1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Universal Flash Storage Host controller driver Core 4 * Copyright (C) 2011-2013 Samsung India Software Operations 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 6 * 7 * Authors: 8 * Santosh Yaraganavi <santosh.sy@samsung.com> 9 * Vinayak Holikatti <h.vinayak@samsung.com> 10 */ 11 12 #include <linux/async.h> 13 #include <linux/devfreq.h> 14 #include <linux/nls.h> 15 #include <linux/of.h> 16 #include <linux/bitfield.h> 17 #include <linux/blk-pm.h> 18 #include <linux/blkdev.h> 19 #include <linux/clk.h> 20 #include <linux/delay.h> 21 #include <linux/hex.h> 22 #include <linux/interrupt.h> 23 #include <linux/module.h> 24 #include <linux/pm_opp.h> 25 #include <linux/regulator/consumer.h> 26 #include <linux/sched/clock.h> 27 #include <linux/sizes.h> 28 #include <linux/iopoll.h> 29 #include <scsi/scsi_cmnd.h> 30 #include <scsi/scsi_dbg.h> 31 #include <scsi/scsi_driver.h> 32 #include <scsi/scsi_eh.h> 33 #include <scsi/scsi_tcq.h> 34 #include "ufshcd-priv.h" 35 #include <ufs/ufs_quirks.h> 36 #include <ufs/unipro.h> 37 #include "ufs-sysfs.h" 38 #include "ufs-debugfs.h" 39 #include "ufs-fault-injection.h" 40 #include "ufs_bsg.h" 41 #include "ufshcd-crypto.h" 42 #include <linux/unaligned.h> 43 44 #define CREATE_TRACE_POINTS 45 #include "ufs_trace.h" 46 47 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 48 UTP_TASK_REQ_COMPL |\ 49 UFSHCD_ERROR_MASK) 50 51 /* UIC command timeout, unit: ms */ 52 enum { 53 UIC_CMD_TIMEOUT_DEFAULT = 500, 54 UIC_CMD_TIMEOUT_MAX = 5000, 55 }; 56 /* NOP OUT retries waiting for NOP IN response */ 57 #define NOP_OUT_RETRIES 10 58 /* Timeout after 50 msecs if NOP OUT hangs without response */ 59 #define NOP_OUT_TIMEOUT 50 /* msecs */ 60 61 /* Query request retries */ 62 #define QUERY_REQ_RETRIES 3 63 /* Query request timeout */ 64 enum { 65 QUERY_REQ_TIMEOUT_MIN = 1, 66 QUERY_REQ_TIMEOUT_DEFAULT = 1500, 67 QUERY_REQ_TIMEOUT_MAX = 30000 68 }; 69 70 /* Advanced RPMB request timeout */ 71 #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ 72 73 /* Task management command timeout */ 74 #define TM_CMD_TIMEOUT 100 /* msecs */ 75 76 /* maximum number of retries for a general UIC command */ 77 #define UFS_UIC_COMMAND_RETRIES 3 78 79 /* maximum number of link-startup retries */ 80 #define DME_LINKSTARTUP_RETRIES 3 81 82 /* maximum number of reset retries before giving up */ 83 #define MAX_HOST_RESET_RETRIES 5 84 85 /* Maximum number of error handler retries before giving up */ 86 #define MAX_ERR_HANDLER_RETRIES 5 87 88 /* Expose the flag value from utp_upiu_query.value */ 89 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF 90 91 /* Interrupt aggregation default timeout, unit: 40us */ 92 #define INT_AGGR_DEF_TO 0x02 93 94 /* default delay of autosuspend: 2000 ms */ 95 #define RPM_AUTOSUSPEND_DELAY_MS 2000 96 97 /* Default delay of RPM device flush delayed work */ 98 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000 99 100 /* Default value of wait time before gating device ref clock */ 101 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */ 102 103 /* Polling time to wait for fDeviceInit */ 104 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ 105 106 /* Default RTC update every 10 seconds */ 107 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) 108 109 /* bMaxNumOfRTT is equal to two after device manufacturing */ 110 #define DEFAULT_MAX_NUM_RTT 2 111 112 /* UFSHC 4.0 compliant HC support this mode. */ 113 static bool use_mcq_mode = true; 114 115 static bool is_mcq_supported(struct ufs_hba *hba) 116 { 117 return hba->mcq_sup && use_mcq_mode; 118 } 119 120 module_param(use_mcq_mode, bool, 0644); 121 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); 122 123 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; 124 125 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) 126 { 127 return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, 128 UIC_CMD_TIMEOUT_MAX); 129 } 130 131 static const struct kernel_param_ops uic_cmd_timeout_ops = { 132 .set = uic_cmd_timeout_set, 133 .get = param_get_uint, 134 }; 135 136 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); 137 MODULE_PARM_DESC(uic_cmd_timeout, 138 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); 139 140 static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; 141 142 static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) 143 { 144 return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, 145 QUERY_REQ_TIMEOUT_MAX); 146 } 147 148 static const struct kernel_param_ops dev_cmd_timeout_ops = { 149 .set = dev_cmd_timeout_set, 150 .get = param_get_uint, 151 }; 152 153 module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); 154 MODULE_PARM_DESC(dev_cmd_timeout, 155 "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); 156 157 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 158 ({ \ 159 int _ret; \ 160 if (_on) \ 161 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 162 else \ 163 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 164 _ret; \ 165 }) 166 167 #define ufshcd_hex_dump(prefix_str, buf, len) do { \ 168 size_t __len = (len); \ 169 print_hex_dump(KERN_ERR, prefix_str, \ 170 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\ 171 16, 4, buf, __len, false); \ 172 } while (0) 173 174 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, 175 const char *prefix) 176 { 177 u32 *regs; 178 size_t pos; 179 180 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */ 181 return -EINVAL; 182 183 regs = kzalloc(len, GFP_ATOMIC); 184 if (!regs) 185 return -ENOMEM; 186 187 for (pos = 0; pos < len; pos += 4) { 188 if (offset == 0 && 189 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && 190 pos <= REG_UIC_ERROR_CODE_DME) 191 continue; 192 regs[pos / 4] = ufshcd_readl(hba, offset + pos); 193 } 194 195 ufshcd_hex_dump(prefix, regs, len); 196 kfree(regs); 197 198 return 0; 199 } 200 EXPORT_SYMBOL_GPL(ufshcd_dump_regs); 201 202 enum { 203 UFSHCD_MAX_CHANNEL = 0, 204 UFSHCD_MAX_ID = 1, 205 }; 206 207 static const char *const ufshcd_state_name[] = { 208 [UFSHCD_STATE_RESET] = "reset", 209 [UFSHCD_STATE_OPERATIONAL] = "operational", 210 [UFSHCD_STATE_ERROR] = "error", 211 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal", 212 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal", 213 }; 214 215 /* UFSHCD error handling flags */ 216 enum { 217 UFSHCD_EH_IN_PROGRESS = (1 << 0), 218 }; 219 220 /* UFSHCD UIC layer error flags */ 221 enum { 222 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 223 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */ 224 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */ 225 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */ 226 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */ 227 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */ 228 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */ 229 }; 230 231 #define ufshcd_set_eh_in_progress(h) \ 232 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS) 233 #define ufshcd_eh_in_progress(h) \ 234 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS) 235 #define ufshcd_clear_eh_in_progress(h) \ 236 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 237 238 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 239 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 240 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 241 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 242 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 243 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 244 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 245 /* 246 * For DeepSleep, the link is first put in hibern8 and then off. 247 * Leaving the link in hibern8 is not supported. 248 */ 249 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, 250 }; 251 252 static inline enum ufs_dev_pwr_mode 253 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 254 { 255 return ufs_pm_lvl_states[lvl].dev_state; 256 } 257 258 static inline enum uic_link_state 259 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 260 { 261 return ufs_pm_lvl_states[lvl].link_state; 262 } 263 264 static inline enum ufs_pm_level 265 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, 266 enum uic_link_state link_state) 267 { 268 enum ufs_pm_level lvl; 269 270 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { 271 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && 272 (ufs_pm_lvl_states[lvl].link_state == link_state)) 273 return lvl; 274 } 275 276 /* if no match found, return the level 0 */ 277 return UFS_PM_LVL_0; 278 } 279 280 static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) 281 { 282 return hba->outstanding_tasks || hba->active_uic_cmd || 283 hba->uic_async_done; 284 } 285 286 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) 287 { 288 return (hba->scsi_host_added && scsi_host_busy(hba->host)) || 289 ufshcd_has_pending_tasks(hba); 290 } 291 292 static const struct ufs_dev_quirk ufs_fixups[] = { 293 /* UFS cards deviations table */ 294 { .wmanufacturerid = UFS_VENDOR_MICRON, 295 .model = UFS_ANY_MODEL, 296 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 297 { .wmanufacturerid = UFS_VENDOR_SAMSUNG, 298 .model = UFS_ANY_MODEL, 299 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 300 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | 301 UFS_DEVICE_QUIRK_PA_HIBER8TIME | 302 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, 303 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 304 .model = UFS_ANY_MODEL, 305 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME }, 306 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 307 .model = "hB8aL1" /*H28U62301AMR*/, 308 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME }, 309 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 310 .model = UFS_ANY_MODEL, 311 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 312 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 313 .model = "THGLF2G9C8KBADG", 314 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 315 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 316 .model = "THGLF2G9D8KBADG", 317 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 318 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 319 .model = "THGJFJT0E25BAIP", 320 .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, 321 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 322 .model = "THGJFJT1E45BATP", 323 .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, 324 {} 325 }; 326 327 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba); 328 static void ufshcd_async_scan(void *data, async_cookie_t cookie); 329 static int ufshcd_reset_and_restore(struct ufs_hba *hba); 330 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); 331 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 332 static void ufshcd_hba_exit(struct ufs_hba *hba); 333 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); 334 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); 335 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 336 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 337 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 338 static void ufshcd_resume_clkscaling(struct ufs_hba *hba); 339 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); 340 static irqreturn_t ufshcd_intr(int irq, void *__hba); 341 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); 342 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); 343 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 344 struct ufs_vreg *vreg); 345 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 346 bool enable); 347 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); 348 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); 349 350 void ufshcd_enable_irq(struct ufs_hba *hba) 351 { 352 if (!hba->is_irq_enabled) { 353 enable_irq(hba->irq); 354 hba->is_irq_enabled = true; 355 } 356 } 357 EXPORT_SYMBOL_GPL(ufshcd_enable_irq); 358 359 void ufshcd_disable_irq(struct ufs_hba *hba) 360 { 361 if (hba->is_irq_enabled) { 362 disable_irq(hba->irq); 363 hba->is_irq_enabled = false; 364 } 365 } 366 EXPORT_SYMBOL_GPL(ufshcd_disable_irq); 367 368 /** 369 * ufshcd_enable_intr - enable interrupts 370 * @hba: per adapter instance 371 * @intrs: interrupt bits 372 */ 373 void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 374 { 375 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 376 u32 new_val = old_val | intrs; 377 378 if (new_val != old_val) 379 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 380 } 381 382 /** 383 * ufshcd_disable_intr - disable interrupts 384 * @hba: per adapter instance 385 * @intrs: interrupt bits 386 */ 387 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 388 { 389 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 390 u32 new_val = old_val & ~intrs; 391 392 if (new_val != old_val) 393 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 394 } 395 396 static void ufshcd_configure_wb(struct ufs_hba *hba) 397 { 398 if (!ufshcd_is_wb_allowed(hba)) 399 return; 400 401 ufshcd_wb_toggle(hba, true); 402 403 ufshcd_wb_toggle_buf_flush_during_h8(hba, true); 404 405 if (ufshcd_is_wb_buf_flush_allowed(hba)) 406 ufshcd_wb_toggle_buf_flush(hba, true); 407 } 408 409 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, 410 struct ufshcd_lrb *lrb, 411 enum ufs_trace_str_t str_t) 412 { 413 struct utp_upiu_req *rq = lrb->ucd_req_ptr; 414 struct utp_upiu_header *header; 415 416 if (!trace_ufshcd_upiu_enabled()) 417 return; 418 419 if (str_t == UFS_CMD_SEND) 420 header = &rq->header; 421 else 422 header = &lrb->ucd_rsp_ptr->header; 423 424 trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, 425 UFS_TSF_CDB); 426 } 427 428 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, 429 enum ufs_trace_str_t str_t, 430 struct utp_upiu_req *rq_rsp) 431 { 432 if (!trace_ufshcd_upiu_enabled()) 433 return; 434 435 trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, 436 &rq_rsp->qr, UFS_TSF_OSF); 437 } 438 439 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, 440 enum ufs_trace_str_t str_t) 441 { 442 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag]; 443 444 if (!trace_ufshcd_upiu_enabled()) 445 return; 446 447 if (str_t == UFS_TM_SEND) 448 trace_ufshcd_upiu(hba, str_t, 449 &descp->upiu_req.req_header, 450 &descp->upiu_req.input_param1, 451 UFS_TSF_TM_INPUT); 452 else 453 trace_ufshcd_upiu(hba, str_t, 454 &descp->upiu_rsp.rsp_header, 455 &descp->upiu_rsp.output_param1, 456 UFS_TSF_TM_OUTPUT); 457 } 458 459 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, 460 const struct uic_command *ucmd, 461 enum ufs_trace_str_t str_t) 462 { 463 u32 cmd; 464 465 if (!trace_ufshcd_uic_command_enabled()) 466 return; 467 468 if (str_t == UFS_CMD_SEND) 469 cmd = ucmd->command; 470 else 471 cmd = ufshcd_readl(hba, REG_UIC_COMMAND); 472 473 trace_ufshcd_uic_command(hba, str_t, cmd, 474 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), 475 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), 476 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 477 } 478 479 static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, 480 enum ufs_trace_str_t str_t) 481 { 482 u64 lba = 0; 483 u8 opcode = 0, group_id = 0; 484 u32 doorbell = 0; 485 u32 intr; 486 u32 hwq_id = 0; 487 struct request *rq = scsi_cmd_to_rq(cmd); 488 unsigned int tag = rq->tag; 489 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 490 int transfer_len = -1; 491 492 /* trace UPIU also */ 493 ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); 494 if (!trace_ufshcd_command_enabled()) 495 return; 496 497 opcode = cmd->cmnd[0]; 498 499 if (opcode == READ_10 || opcode == WRITE_10) { 500 /* 501 * Currently we only fully trace read(10) and write(10) commands 502 */ 503 transfer_len = 504 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 505 lba = scsi_get_lba(cmd); 506 if (opcode == WRITE_10) 507 group_id = cmd->cmnd[6]; 508 } else if (opcode == UNMAP) { 509 /* 510 * The number of Bytes to be unmapped beginning with the lba. 511 */ 512 transfer_len = blk_rq_bytes(rq); 513 lba = scsi_get_lba(cmd); 514 } 515 516 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 517 518 if (hba->mcq_enabled) { 519 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 520 if (hwq) 521 hwq_id = hwq->id; 522 } else { 523 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 524 } 525 trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, 526 transfer_len, intr, lba, opcode, group_id); 527 } 528 529 static void ufshcd_print_clk_freqs(struct ufs_hba *hba) 530 { 531 struct ufs_clk_info *clki; 532 struct list_head *head = &hba->clk_list_head; 533 534 if (list_empty(head)) 535 return; 536 537 list_for_each_entry(clki, head, list) { 538 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && 539 clki->max_freq) 540 dev_err(hba->dev, "clk: %s, rate: %u\n", 541 clki->name, clki->curr_freq); 542 } 543 } 544 545 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, 546 const char *err_name) 547 { 548 int i; 549 bool found = false; 550 const struct ufs_event_hist *e; 551 552 if (id >= UFS_EVT_CNT) 553 return; 554 555 e = &hba->ufs_stats.event[id]; 556 557 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { 558 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; 559 560 if (e->tstamp[p] == 0) 561 continue; 562 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, 563 e->val[p], div_u64(e->tstamp[p], 1000)); 564 found = true; 565 } 566 567 if (!found) 568 dev_err(hba->dev, "No record of %s\n", err_name); 569 else 570 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); 571 } 572 573 static void ufshcd_print_evt_hist(struct ufs_hba *hba) 574 { 575 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 576 577 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); 578 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); 579 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); 580 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); 581 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); 582 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, 583 "auto_hibern8_err"); 584 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); 585 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, 586 "link_startup_fail"); 587 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); 588 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, 589 "suspend_fail"); 590 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail"); 591 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR, 592 "wlun suspend_fail"); 593 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); 594 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); 595 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); 596 597 ufshcd_vops_dbg_register_dump(hba); 598 } 599 600 static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, 601 bool pr_prdt) 602 { 603 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 604 const int tag = scsi_cmd_to_rq(cmd)->tag; 605 int prdt_length; 606 607 if (hba->monitor.enabled) { 608 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, 609 div_u64(lrbp->issue_time_stamp_local_clock, 1000)); 610 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, 611 div_u64(lrbp->compl_time_stamp_local_clock, 1000)); 612 } 613 dev_err(hba->dev, 614 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", 615 tag, (u64)lrbp->utrd_dma_addr); 616 617 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, 618 sizeof(struct utp_transfer_req_desc)); 619 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, 620 (u64)lrbp->ucd_req_dma_addr); 621 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, 622 sizeof(struct utp_upiu_req)); 623 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, 624 (u64)lrbp->ucd_rsp_dma_addr); 625 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, 626 sizeof(struct utp_upiu_rsp)); 627 628 prdt_length = le16_to_cpu( 629 lrbp->utr_descriptor_ptr->prd_table_length); 630 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 631 prdt_length /= ufshcd_sg_entry_size(hba); 632 633 dev_err(hba->dev, 634 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", 635 tag, prdt_length, 636 (u64)lrbp->ucd_prdt_dma_addr); 637 638 if (pr_prdt) 639 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, 640 ufshcd_sg_entry_size(hba) * prdt_length); 641 } 642 643 static bool ufshcd_print_tr_iter(struct request *req, void *priv) 644 { 645 struct scsi_device *sdev = req->q->queuedata; 646 struct Scsi_Host *shost = sdev->host; 647 struct ufs_hba *hba = shost_priv(shost); 648 649 if (!blk_mq_is_reserved_rq(req)) 650 ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); 651 652 return true; 653 } 654 655 /** 656 * ufshcd_print_trs_all - print trs for all started requests. 657 * @hba: per-adapter instance. 658 * @pr_prdt: need to print prdt or not. 659 */ 660 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt) 661 { 662 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt); 663 } 664 665 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) 666 { 667 int tag; 668 669 for_each_set_bit(tag, &bitmap, hba->nutmrs) { 670 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag]; 671 672 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); 673 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp)); 674 } 675 } 676 677 static void ufshcd_print_host_state(struct ufs_hba *hba) 678 { 679 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; 680 681 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); 682 dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", 683 hba->scsi_host_added ? scsi_host_busy(hba->host) : 0, 684 hba->outstanding_tasks); 685 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", 686 hba->saved_err, hba->saved_uic_err); 687 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", 688 hba->curr_dev_pwr_mode, hba->uic_link_state); 689 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", 690 hba->pm_op_in_progress, hba->is_sys_suspended); 691 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", 692 hba->auto_bkops_enabled, hba->host->host_self_blocked); 693 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); 694 dev_err(hba->dev, 695 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", 696 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), 697 hba->ufs_stats.hibern8_exit_cnt); 698 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", 699 hba->eh_flags, hba->req_abort_count); 700 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", 701 hba->ufs_version, hba->capabilities, hba->caps); 702 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, 703 hba->dev_quirks); 704 if (sdev_ufs) 705 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n", 706 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev); 707 708 ufshcd_print_clk_freqs(hba); 709 } 710 711 /** 712 * ufshcd_print_pwr_info - print power params as saved in hba 713 * power info 714 * @hba: per-adapter instance 715 */ 716 static void ufshcd_print_pwr_info(struct ufs_hba *hba) 717 { 718 static const char * const names[] = { 719 "INVALID MODE", 720 "FAST MODE", 721 "SLOW_MODE", 722 "INVALID MODE", 723 "FASTAUTO_MODE", 724 "SLOWAUTO_MODE", 725 "INVALID MODE", 726 }; 727 728 /* 729 * Using dev_dbg to avoid messages during runtime PM to avoid 730 * never-ending cycles of messages written back to storage by user space 731 * causing runtime resume, causing more messages and so on. 732 */ 733 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", 734 __func__, 735 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, 736 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, 737 names[hba->pwr_info.pwr_rx], 738 names[hba->pwr_info.pwr_tx], 739 hba->pwr_info.hs_rate); 740 } 741 742 static void ufshcd_device_reset(struct ufs_hba *hba) 743 { 744 int err; 745 746 err = ufshcd_vops_device_reset(hba); 747 748 if (!err) { 749 ufshcd_set_ufs_dev_active(hba); 750 if (ufshcd_is_wb_allowed(hba)) { 751 hba->dev_info.wb_enabled = false; 752 hba->dev_info.wb_buf_flush_enabled = false; 753 } 754 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 755 hba->dev_info.rtc_time_baseline = 0; 756 } 757 if (err != -EOPNOTSUPP) 758 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); 759 } 760 761 void ufshcd_delay_us(unsigned long us, unsigned long tolerance) 762 { 763 if (!us) 764 return; 765 766 if (us < 10) 767 udelay(us); 768 else 769 usleep_range(us, us + tolerance); 770 } 771 EXPORT_SYMBOL_GPL(ufshcd_delay_us); 772 773 /** 774 * ufshcd_wait_for_register - wait for register value to change 775 * @hba: per-adapter interface 776 * @reg: mmio register offset 777 * @mask: mask to apply to the read register value 778 * @val: value to wait for 779 * @interval_us: polling interval in microseconds 780 * @timeout_ms: timeout in milliseconds 781 * 782 * Return: -ETIMEDOUT on error, zero on success. 783 */ 784 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 785 u32 val, unsigned long interval_us, 786 unsigned long timeout_ms) 787 { 788 u32 v; 789 790 val &= mask; /* ignore bits that we don't intend to wait on */ 791 792 return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, 793 interval_us, timeout_ms * 1000, false, hba, reg); 794 } 795 796 /** 797 * ufshcd_get_intr_mask - Get the interrupt bit mask 798 * @hba: Pointer to adapter instance 799 * 800 * Return: interrupt bit mask per version 801 */ 802 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 803 { 804 if (hba->ufs_version <= ufshci_version(2, 0)) 805 return INTERRUPT_MASK_ALL_VER_11; 806 807 return INTERRUPT_MASK_ALL_VER_21; 808 } 809 810 /** 811 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 812 * @hba: Pointer to adapter instance 813 * 814 * Return: UFSHCI version supported by the controller 815 */ 816 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 817 { 818 u32 ufshci_ver; 819 820 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION) 821 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba); 822 else 823 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION); 824 825 /* 826 * UFSHCI v1.x uses a different version scheme, in order 827 * to allow the use of comparisons with the ufshci_version 828 * function, we convert it to the same scheme as ufs 2.0+. 829 */ 830 if (ufshci_ver & 0x00010000) 831 return ufshci_version(1, ufshci_ver & 0x00000100); 832 833 return ufshci_ver; 834 } 835 836 /** 837 * ufshcd_is_device_present - Check if any device connected to 838 * the host controller 839 * @hba: pointer to adapter instance 840 * 841 * Return: true if device present, false if no device detected 842 */ 843 static inline bool ufshcd_is_device_present(struct ufs_hba *hba) 844 { 845 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT; 846 } 847 848 /** 849 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 850 * @lrbp: pointer to local command reference block 851 * @cqe: pointer to the completion queue entry 852 * 853 * This function is used to get the OCS field from UTRD 854 * 855 * Return: the OCS field in the UTRD. 856 */ 857 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, 858 struct cq_entry *cqe) 859 { 860 if (cqe) 861 return cqe->overall_status & MASK_OCS; 862 863 return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; 864 } 865 866 /** 867 * ufshcd_utrl_clear() - Clear requests from the controller request list. 868 * @hba: per adapter instance 869 * @mask: mask with one bit set for each request to be cleared 870 */ 871 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) 872 { 873 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 874 mask = ~mask; 875 /* 876 * From the UFSHCI specification: "UTP Transfer Request List CLear 877 * Register (UTRLCLR): This field is bit significant. Each bit 878 * corresponds to a slot in the UTP Transfer Request List, where bit 0 879 * corresponds to request slot 0. A bit in this field is set to ‘0’ 880 * by host software to indicate to the host controller that a transfer 881 * request slot is cleared. The host controller 882 * shall free up any resources associated to the request slot 883 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The 884 * host software indicates no change to request slots by setting the 885 * associated bits in this field to ‘1’. Bits in this field shall only 886 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’." 887 */ 888 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR); 889 } 890 891 /** 892 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register 893 * @hba: per adapter instance 894 * @pos: position of the bit to be cleared 895 */ 896 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) 897 { 898 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 899 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 900 else 901 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 902 } 903 904 /** 905 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 906 * @reg: Register value of host controller status 907 * 908 * Return: 0 on success; a positive value if failed. 909 */ 910 static inline int ufshcd_get_lists_status(u32 reg) 911 { 912 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); 913 } 914 915 /** 916 * ufshcd_get_uic_cmd_result - Get the UIC command result 917 * @hba: Pointer to adapter instance 918 * 919 * This function gets the result of UIC command completion 920 * 921 * Return: 0 on success; non-zero value on error. 922 */ 923 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 924 { 925 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 926 MASK_UIC_COMMAND_RESULT; 927 } 928 929 /** 930 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 931 * @hba: Pointer to adapter instance 932 * 933 * This function gets UIC command argument3 934 * 935 * Return: 0 on success; non-zero value on error. 936 */ 937 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 938 { 939 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 940 } 941 942 /** 943 * ufshcd_get_req_rsp - returns the TR response transaction type 944 * @ucd_rsp_ptr: pointer to response UPIU 945 * 946 * Return: UPIU type. 947 */ 948 static inline enum upiu_response_transaction 949 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 950 { 951 return ucd_rsp_ptr->header.transaction_code; 952 } 953 954 /** 955 * ufshcd_is_exception_event - Check if the device raised an exception event 956 * @ucd_rsp_ptr: pointer to response UPIU 957 * 958 * The function checks if the device raised an exception event indicated in 959 * the Device Information field of response UPIU. 960 * 961 * Return: true if exception is raised, false otherwise. 962 */ 963 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 964 { 965 return ucd_rsp_ptr->header.device_information & 1; 966 } 967 968 /** 969 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 970 * @hba: per adapter instance 971 */ 972 static inline void 973 ufshcd_reset_intr_aggr(struct ufs_hba *hba) 974 { 975 ufshcd_writel(hba, INT_AGGR_ENABLE | 976 INT_AGGR_COUNTER_AND_TIMER_RESET, 977 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 978 } 979 980 /** 981 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 982 * @hba: per adapter instance 983 * @cnt: Interrupt aggregation counter threshold 984 * @tmout: Interrupt aggregation timeout value 985 */ 986 static inline void 987 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 988 { 989 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 990 INT_AGGR_COUNTER_THLD_VAL(cnt) | 991 INT_AGGR_TIMEOUT_VAL(tmout), 992 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 993 } 994 995 /** 996 * ufshcd_disable_intr_aggr - Disables interrupt aggregation. 997 * @hba: per adapter instance 998 */ 999 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) 1000 { 1001 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 1002 } 1003 1004 /** 1005 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 1006 * When run-stop registers are set to 1, it indicates the 1007 * host controller that it can process the requests 1008 * @hba: per adapter instance 1009 */ 1010 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 1011 { 1012 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 1013 REG_UTP_TASK_REQ_LIST_RUN_STOP); 1014 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 1015 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 1016 } 1017 1018 /** 1019 * ufshcd_hba_start - Start controller initialization sequence 1020 * @hba: per adapter instance 1021 */ 1022 static inline void ufshcd_hba_start(struct ufs_hba *hba) 1023 { 1024 u32 val = CONTROLLER_ENABLE; 1025 1026 if (ufshcd_crypto_enable(hba)) 1027 val |= CRYPTO_GENERAL_ENABLE; 1028 1029 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); 1030 } 1031 1032 /** 1033 * ufshcd_is_hba_active - Get controller state 1034 * @hba: per adapter instance 1035 * 1036 * Return: true if and only if the controller is active. 1037 */ 1038 bool ufshcd_is_hba_active(struct ufs_hba *hba) 1039 { 1040 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; 1041 } 1042 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); 1043 1044 /** 1045 * ufshcd_pm_qos_init - initialize PM QoS request 1046 * @hba: per adapter instance 1047 */ 1048 void ufshcd_pm_qos_init(struct ufs_hba *hba) 1049 { 1050 guard(mutex)(&hba->pm_qos_mutex); 1051 1052 if (hba->pm_qos_enabled) 1053 return; 1054 1055 cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); 1056 1057 if (cpu_latency_qos_request_active(&hba->pm_qos_req)) 1058 hba->pm_qos_enabled = true; 1059 } 1060 1061 /** 1062 * ufshcd_pm_qos_exit - remove request from PM QoS 1063 * @hba: per adapter instance 1064 */ 1065 void ufshcd_pm_qos_exit(struct ufs_hba *hba) 1066 { 1067 guard(mutex)(&hba->pm_qos_mutex); 1068 1069 if (!hba->pm_qos_enabled) 1070 return; 1071 1072 cpu_latency_qos_remove_request(&hba->pm_qos_req); 1073 hba->pm_qos_enabled = false; 1074 } 1075 1076 /** 1077 * ufshcd_pm_qos_update - update PM QoS request 1078 * @hba: per adapter instance 1079 * @on: If True, vote for perf PM QoS mode otherwise power save mode 1080 */ 1081 void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) 1082 { 1083 guard(mutex)(&hba->pm_qos_mutex); 1084 1085 if (!hba->pm_qos_enabled) 1086 return; 1087 1088 cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); 1089 } 1090 EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update); 1091 1092 /** 1093 * ufshcd_set_clk_freq - set UFS controller clock frequencies 1094 * @hba: per adapter instance 1095 * @scale_up: If True, set max possible frequency othewise set low frequency 1096 * 1097 * Return: 0 if successful; < 0 upon failure. 1098 */ 1099 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) 1100 { 1101 int ret = 0; 1102 struct ufs_clk_info *clki; 1103 struct list_head *head = &hba->clk_list_head; 1104 1105 if (list_empty(head)) 1106 goto out; 1107 1108 list_for_each_entry(clki, head, list) { 1109 if (!IS_ERR_OR_NULL(clki->clk)) { 1110 if (scale_up && clki->max_freq) { 1111 if (clki->curr_freq == clki->max_freq) 1112 continue; 1113 1114 ret = clk_set_rate(clki->clk, clki->max_freq); 1115 if (ret) { 1116 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1117 __func__, clki->name, 1118 clki->max_freq, ret); 1119 break; 1120 } 1121 trace_ufshcd_clk_scaling(hba, 1122 "scaled up", clki->name, 1123 clki->curr_freq, 1124 clki->max_freq); 1125 1126 clki->curr_freq = clki->max_freq; 1127 1128 } else if (!scale_up && clki->min_freq) { 1129 if (clki->curr_freq == clki->min_freq) 1130 continue; 1131 1132 ret = clk_set_rate(clki->clk, clki->min_freq); 1133 if (ret) { 1134 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1135 __func__, clki->name, 1136 clki->min_freq, ret); 1137 break; 1138 } 1139 trace_ufshcd_clk_scaling(hba, 1140 "scaled down", clki->name, 1141 clki->curr_freq, 1142 clki->min_freq); 1143 clki->curr_freq = clki->min_freq; 1144 } 1145 } 1146 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 1147 clki->name, clk_get_rate(clki->clk)); 1148 } 1149 1150 out: 1151 return ret; 1152 } 1153 1154 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, 1155 struct dev_pm_opp *opp, void *data, 1156 bool scaling_down) 1157 { 1158 struct ufs_hba *hba = dev_get_drvdata(dev); 1159 struct list_head *head = &hba->clk_list_head; 1160 struct ufs_clk_info *clki; 1161 unsigned long freq; 1162 u8 idx = 0; 1163 int ret; 1164 1165 list_for_each_entry(clki, head, list) { 1166 if (!IS_ERR_OR_NULL(clki->clk)) { 1167 freq = dev_pm_opp_get_freq_indexed(opp, idx++); 1168 1169 /* Do not set rate for clocks having frequency as 0 */ 1170 if (!freq) 1171 continue; 1172 1173 ret = clk_set_rate(clki->clk, freq); 1174 if (ret) { 1175 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", 1176 __func__, clki->name, freq, ret); 1177 return ret; 1178 } 1179 1180 trace_ufshcd_clk_scaling(hba, 1181 (scaling_down ? "scaled down" : "scaled up"), 1182 clki->name, hba->clk_scaling.target_freq, freq); 1183 } 1184 } 1185 1186 return 0; 1187 } 1188 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); 1189 1190 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) 1191 { 1192 struct dev_pm_opp *opp; 1193 int ret; 1194 1195 opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, 1196 &freq, 0); 1197 if (IS_ERR(opp)) 1198 return PTR_ERR(opp); 1199 1200 ret = dev_pm_opp_set_opp(hba->dev, opp); 1201 dev_pm_opp_put(opp); 1202 1203 return ret; 1204 } 1205 1206 /** 1207 * ufshcd_scale_clks - scale up or scale down UFS controller clocks 1208 * @hba: per adapter instance 1209 * @freq: frequency to scale 1210 * @scale_up: True if scaling up and false if scaling down 1211 * 1212 * Return: 0 if successful; < 0 upon failure. 1213 */ 1214 int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, bool scale_up) 1215 { 1216 int ret = 0; 1217 ktime_t start = ktime_get(); 1218 1219 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); 1220 if (ret) 1221 goto out; 1222 1223 if (hba->use_pm_opp) 1224 ret = ufshcd_opp_set_rate(hba, freq); 1225 else 1226 ret = ufshcd_set_clk_freq(hba, scale_up); 1227 if (ret) 1228 goto out; 1229 1230 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); 1231 if (ret) { 1232 if (hba->use_pm_opp) 1233 ufshcd_opp_set_rate(hba, 1234 hba->devfreq->previous_freq); 1235 else 1236 ufshcd_set_clk_freq(hba, !scale_up); 1237 goto out; 1238 } 1239 1240 ufshcd_pm_qos_update(hba, scale_up); 1241 1242 out: 1243 trace_ufshcd_profile_clk_scaling(hba, 1244 (scale_up ? "up" : "down"), 1245 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1246 return ret; 1247 } 1248 1249 /** 1250 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not 1251 * @hba: per adapter instance 1252 * @freq: frequency to scale 1253 * @scale_up: True if scaling up and false if scaling down 1254 * 1255 * Return: true if scaling is required, false otherwise. 1256 */ 1257 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, 1258 unsigned long freq, bool scale_up) 1259 { 1260 struct ufs_clk_info *clki; 1261 struct list_head *head = &hba->clk_list_head; 1262 1263 if (list_empty(head)) 1264 return false; 1265 1266 if (hba->use_pm_opp) 1267 return freq != hba->clk_scaling.target_freq; 1268 1269 list_for_each_entry(clki, head, list) { 1270 if (!IS_ERR_OR_NULL(clki->clk)) { 1271 if (scale_up && clki->max_freq) { 1272 if (clki->curr_freq == clki->max_freq) 1273 continue; 1274 return true; 1275 } else if (!scale_up && clki->min_freq) { 1276 if (clki->curr_freq == clki->min_freq) 1277 continue; 1278 return true; 1279 } 1280 } 1281 } 1282 1283 return false; 1284 } 1285 1286 /* 1287 * Determine the number of pending commands by counting the bits in the SCSI 1288 * device budget maps. This approach has been selected because a bit is set in 1289 * the budget map before scsi_host_queue_ready() checks the host_self_blocked 1290 * flag. The host_self_blocked flag can be modified by calling 1291 * scsi_block_requests() or scsi_unblock_requests(). 1292 */ 1293 static u32 ufshcd_pending_cmds(struct ufs_hba *hba) 1294 { 1295 struct scsi_device *sdev; 1296 unsigned long flags; 1297 u32 pending = 0; 1298 1299 spin_lock_irqsave(hba->host->host_lock, flags); 1300 __shost_for_each_device(sdev, hba->host) 1301 pending += scsi_device_busy(sdev); 1302 spin_unlock_irqrestore(hba->host->host_lock, flags); 1303 1304 return pending; 1305 } 1306 1307 /* 1308 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1309 * has expired. 1310 * 1311 * Return: 0 upon success; -EBUSY upon timeout. 1312 */ 1313 static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, 1314 u64 wait_timeout_us) 1315 { 1316 int ret = 0; 1317 u32 tm_doorbell; 1318 u32 tr_pending; 1319 bool timeout = false, do_last_check = false; 1320 ktime_t start; 1321 1322 ufshcd_hold(hba); 1323 /* 1324 * Wait for all the outstanding tasks/transfer requests. 1325 * Verify by checking the doorbell registers are clear. 1326 */ 1327 start = ktime_get(); 1328 do { 1329 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { 1330 ret = -EBUSY; 1331 goto out; 1332 } 1333 1334 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 1335 tr_pending = ufshcd_pending_cmds(hba); 1336 if (!tm_doorbell && !tr_pending) { 1337 timeout = false; 1338 break; 1339 } else if (do_last_check) { 1340 break; 1341 } 1342 1343 io_schedule_timeout(msecs_to_jiffies(20)); 1344 if (ktime_to_us(ktime_sub(ktime_get(), start)) > 1345 wait_timeout_us) { 1346 timeout = true; 1347 /* 1348 * We might have scheduled out for long time so make 1349 * sure to check if doorbells are cleared by this time 1350 * or not. 1351 */ 1352 do_last_check = true; 1353 } 1354 } while (tm_doorbell || tr_pending); 1355 1356 if (timeout) { 1357 dev_err(hba->dev, 1358 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", 1359 __func__, tm_doorbell, tr_pending); 1360 ret = -EBUSY; 1361 } 1362 out: 1363 ufshcd_release(hba); 1364 return ret; 1365 } 1366 1367 /** 1368 * ufshcd_pause_command_processing - Pause command processing 1369 * @hba: per-adapter instance 1370 * @timeout_us: timeout in microseconds to wait for pending commands to finish 1371 * 1372 * This function stops new command submissions and waits for existing commands 1373 * to complete. 1374 * 1375 * Return: 0 on success, %-EBUSY if commands did not finish within @timeout_us. 1376 * On failure, all acquired locks are released and the tagset is unquiesced. 1377 */ 1378 int ufshcd_pause_command_processing(struct ufs_hba *hba, u64 timeout_us) 1379 { 1380 int ret = 0; 1381 1382 mutex_lock(&hba->host->scan_mutex); 1383 blk_mq_quiesce_tagset(&hba->host->tag_set); 1384 down_write(&hba->clk_scaling_lock); 1385 1386 if (ufshcd_wait_for_pending_cmds(hba, timeout_us)) { 1387 ret = -EBUSY; 1388 up_write(&hba->clk_scaling_lock); 1389 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1390 mutex_unlock(&hba->host->scan_mutex); 1391 } 1392 1393 return ret; 1394 } 1395 1396 /** 1397 * ufshcd_resume_command_processing - Resume command processing 1398 * @hba: per-adapter instance 1399 * 1400 * This function resumes command submissions. 1401 */ 1402 void ufshcd_resume_command_processing(struct ufs_hba *hba) 1403 { 1404 up_write(&hba->clk_scaling_lock); 1405 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1406 mutex_unlock(&hba->host->scan_mutex); 1407 } 1408 1409 /** 1410 * ufshcd_scale_gear - scale up/down UFS gear 1411 * @hba: per adapter instance 1412 * @target_gear: target gear to scale to 1413 * @scale_up: True for scaling up gear and false for scaling down 1414 * 1415 * Return: 0 for success; -EBUSY if scaling can't happen at this time; 1416 * non-zero for any other errors. 1417 */ 1418 static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) 1419 { 1420 int ret = 0; 1421 struct ufs_pa_layer_attr new_pwr_info; 1422 1423 if (target_gear) { 1424 new_pwr_info = hba->pwr_info; 1425 new_pwr_info.gear_tx = target_gear; 1426 new_pwr_info.gear_rx = target_gear; 1427 1428 goto config_pwr_mode; 1429 } 1430 1431 /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ 1432 if (scale_up) { 1433 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, 1434 sizeof(struct ufs_pa_layer_attr)); 1435 } else { 1436 memcpy(&new_pwr_info, &hba->pwr_info, 1437 sizeof(struct ufs_pa_layer_attr)); 1438 1439 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || 1440 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { 1441 /* save the current power mode */ 1442 memcpy(&hba->clk_scaling.saved_pwr_info, 1443 &hba->pwr_info, 1444 sizeof(struct ufs_pa_layer_attr)); 1445 1446 /* scale down gear */ 1447 new_pwr_info.gear_tx = hba->clk_scaling.min_gear; 1448 new_pwr_info.gear_rx = hba->clk_scaling.min_gear; 1449 } 1450 } 1451 1452 config_pwr_mode: 1453 /* check if the power mode needs to be changed or not? */ 1454 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info, 1455 UFSHCD_PMC_POLICY_DONT_FORCE); 1456 if (ret) 1457 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", 1458 __func__, ret, 1459 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, 1460 new_pwr_info.gear_tx, new_pwr_info.gear_rx); 1461 1462 return ret; 1463 } 1464 1465 /* 1466 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1467 * has expired. 1468 * 1469 * Return: 0 upon success; -EBUSY upon timeout. 1470 */ 1471 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) 1472 { 1473 int ret = 0; 1474 /* 1475 * make sure that there are no outstanding requests when 1476 * clock scaling is in progress 1477 */ 1478 mutex_lock(&hba->host->scan_mutex); 1479 blk_mq_quiesce_tagset(&hba->host->tag_set); 1480 mutex_lock(&hba->wb_mutex); 1481 down_write(&hba->clk_scaling_lock); 1482 1483 if (!hba->clk_scaling.is_allowed || 1484 ufshcd_wait_for_pending_cmds(hba, timeout_us)) { 1485 ret = -EBUSY; 1486 up_write(&hba->clk_scaling_lock); 1487 mutex_unlock(&hba->wb_mutex); 1488 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1489 mutex_unlock(&hba->host->scan_mutex); 1490 goto out; 1491 } 1492 1493 /* let's not get into low power until clock scaling is completed */ 1494 ufshcd_hold(hba); 1495 1496 out: 1497 return ret; 1498 } 1499 1500 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) 1501 { 1502 up_write(&hba->clk_scaling_lock); 1503 mutex_unlock(&hba->wb_mutex); 1504 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1505 mutex_unlock(&hba->host->scan_mutex); 1506 1507 /* Enable Write Booster if current gear requires it else disable it */ 1508 if (ufshcd_enable_wb_if_scaling_up(hba) && !err) 1509 ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); 1510 1511 ufshcd_release(hba); 1512 } 1513 1514 /** 1515 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear 1516 * @hba: per adapter instance 1517 * @freq: frequency to scale 1518 * @scale_up: True for scaling up and false for scalin down 1519 * 1520 * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero 1521 * for any other errors. 1522 */ 1523 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, 1524 bool scale_up) 1525 { 1526 u32 old_gear = hba->pwr_info.gear_rx; 1527 u32 new_gear = 0; 1528 int ret = 0; 1529 1530 new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); 1531 1532 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); 1533 if (ret) 1534 return ret; 1535 1536 /* scale down the gear before scaling down clocks */ 1537 if (!scale_up) { 1538 ret = ufshcd_scale_gear(hba, new_gear, false); 1539 if (ret) 1540 goto out_unprepare; 1541 } 1542 1543 ret = ufshcd_scale_clks(hba, freq, scale_up); 1544 if (ret) { 1545 if (!scale_up) 1546 ufshcd_scale_gear(hba, old_gear, true); 1547 goto out_unprepare; 1548 } 1549 1550 /* scale up the gear after scaling up clocks */ 1551 if (scale_up) { 1552 ret = ufshcd_scale_gear(hba, new_gear, true); 1553 if (ret) { 1554 ufshcd_scale_clks(hba, hba->devfreq->previous_freq, 1555 false); 1556 goto out_unprepare; 1557 } 1558 } 1559 1560 out_unprepare: 1561 ufshcd_clock_scaling_unprepare(hba, ret); 1562 return ret; 1563 } 1564 1565 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) 1566 { 1567 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1568 clk_scaling.suspend_work); 1569 1570 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1571 { 1572 if (hba->clk_scaling.active_reqs || 1573 hba->clk_scaling.is_suspended) 1574 return; 1575 1576 hba->clk_scaling.is_suspended = true; 1577 hba->clk_scaling.window_start_t = 0; 1578 } 1579 1580 devfreq_suspend_device(hba->devfreq); 1581 } 1582 1583 static void ufshcd_clk_scaling_resume_work(struct work_struct *work) 1584 { 1585 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1586 clk_scaling.resume_work); 1587 1588 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1589 { 1590 if (!hba->clk_scaling.is_suspended) 1591 return; 1592 hba->clk_scaling.is_suspended = false; 1593 } 1594 1595 devfreq_resume_device(hba->devfreq); 1596 } 1597 1598 static int ufshcd_devfreq_target(struct device *dev, 1599 unsigned long *freq, u32 flags) 1600 { 1601 int ret = 0; 1602 struct ufs_hba *hba = dev_get_drvdata(dev); 1603 ktime_t start; 1604 bool scale_up = false, sched_clk_scaling_suspend_work = false; 1605 struct list_head *clk_list = &hba->clk_list_head; 1606 struct ufs_clk_info *clki; 1607 1608 if (!ufshcd_is_clkscaling_supported(hba)) 1609 return -EINVAL; 1610 1611 if (hba->use_pm_opp) { 1612 struct dev_pm_opp *opp; 1613 1614 /* Get the recommended frequency from OPP framework */ 1615 opp = devfreq_recommended_opp(dev, freq, flags); 1616 if (IS_ERR(opp)) 1617 return PTR_ERR(opp); 1618 1619 dev_pm_opp_put(opp); 1620 } else { 1621 /* Override with the closest supported frequency */ 1622 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, 1623 list); 1624 *freq = (unsigned long) clk_round_rate(clki->clk, *freq); 1625 } 1626 1627 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1628 { 1629 if (ufshcd_eh_in_progress(hba)) 1630 return 0; 1631 1632 /* Skip scaling clock when clock scaling is suspended */ 1633 if (hba->clk_scaling.is_suspended) { 1634 dev_warn(hba->dev, "clock scaling is suspended, skip"); 1635 return 0; 1636 } 1637 1638 if (!hba->clk_scaling.active_reqs) 1639 sched_clk_scaling_suspend_work = true; 1640 1641 if (list_empty(clk_list)) 1642 goto out; 1643 1644 /* Decide based on the target or rounded-off frequency and update */ 1645 if (hba->use_pm_opp) 1646 scale_up = *freq > hba->clk_scaling.target_freq; 1647 else 1648 scale_up = *freq == clki->max_freq; 1649 1650 if (!hba->use_pm_opp && !scale_up) 1651 *freq = clki->min_freq; 1652 1653 /* Update the frequency */ 1654 if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { 1655 ret = 0; 1656 goto out; /* no state change required */ 1657 } 1658 } 1659 1660 start = ktime_get(); 1661 ret = ufshcd_devfreq_scale(hba, *freq, scale_up); 1662 if (!ret) 1663 hba->clk_scaling.target_freq = *freq; 1664 1665 trace_ufshcd_profile_clk_scaling(hba, 1666 (scale_up ? "up" : "down"), 1667 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1668 1669 out: 1670 if (sched_clk_scaling_suspend_work && 1671 (!scale_up || hba->clk_scaling.suspend_on_no_request)) 1672 queue_work(hba->clk_scaling.workq, 1673 &hba->clk_scaling.suspend_work); 1674 1675 return ret; 1676 } 1677 1678 static int ufshcd_devfreq_get_dev_status(struct device *dev, 1679 struct devfreq_dev_status *stat) 1680 { 1681 struct ufs_hba *hba = dev_get_drvdata(dev); 1682 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 1683 ktime_t curr_t; 1684 1685 if (!ufshcd_is_clkscaling_supported(hba)) 1686 return -EINVAL; 1687 1688 memset(stat, 0, sizeof(*stat)); 1689 1690 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 1691 1692 curr_t = ktime_get(); 1693 if (!scaling->window_start_t) 1694 goto start_window; 1695 1696 /* 1697 * If current frequency is 0, then the ondemand governor considers 1698 * there's no initial frequency set. And it always requests to set 1699 * to max. frequency. 1700 */ 1701 if (hba->use_pm_opp) { 1702 stat->current_frequency = hba->clk_scaling.target_freq; 1703 } else { 1704 struct list_head *clk_list = &hba->clk_list_head; 1705 struct ufs_clk_info *clki; 1706 1707 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1708 stat->current_frequency = clki->curr_freq; 1709 } 1710 1711 if (scaling->is_busy_started) 1712 scaling->tot_busy_t += ktime_us_delta(curr_t, 1713 scaling->busy_start_t); 1714 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); 1715 stat->busy_time = scaling->tot_busy_t; 1716 start_window: 1717 scaling->window_start_t = curr_t; 1718 scaling->tot_busy_t = 0; 1719 1720 if (scaling->active_reqs) { 1721 scaling->busy_start_t = curr_t; 1722 scaling->is_busy_started = true; 1723 } else { 1724 scaling->busy_start_t = 0; 1725 scaling->is_busy_started = false; 1726 } 1727 1728 return 0; 1729 } 1730 1731 static int ufshcd_devfreq_init(struct ufs_hba *hba) 1732 { 1733 struct list_head *clk_list = &hba->clk_list_head; 1734 struct ufs_clk_info *clki; 1735 struct devfreq *devfreq; 1736 int ret; 1737 1738 /* Skip devfreq if we don't have any clocks in the list */ 1739 if (list_empty(clk_list)) 1740 return 0; 1741 1742 if (!hba->use_pm_opp) { 1743 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1744 dev_pm_opp_add(hba->dev, clki->min_freq, 0); 1745 dev_pm_opp_add(hba->dev, clki->max_freq, 0); 1746 } 1747 1748 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, 1749 &hba->vps->ondemand_data); 1750 devfreq = devfreq_add_device(hba->dev, 1751 &hba->vps->devfreq_profile, 1752 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1753 &hba->vps->ondemand_data); 1754 if (IS_ERR(devfreq)) { 1755 ret = PTR_ERR(devfreq); 1756 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); 1757 1758 if (!hba->use_pm_opp) { 1759 dev_pm_opp_remove(hba->dev, clki->min_freq); 1760 dev_pm_opp_remove(hba->dev, clki->max_freq); 1761 } 1762 return ret; 1763 } 1764 1765 hba->devfreq = devfreq; 1766 1767 return 0; 1768 } 1769 1770 static void ufshcd_devfreq_remove(struct ufs_hba *hba) 1771 { 1772 struct list_head *clk_list = &hba->clk_list_head; 1773 1774 if (!hba->devfreq) 1775 return; 1776 1777 devfreq_remove_device(hba->devfreq); 1778 hba->devfreq = NULL; 1779 1780 if (!hba->use_pm_opp) { 1781 struct ufs_clk_info *clki; 1782 1783 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1784 dev_pm_opp_remove(hba->dev, clki->min_freq); 1785 dev_pm_opp_remove(hba->dev, clki->max_freq); 1786 } 1787 } 1788 1789 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1790 { 1791 bool suspend = false; 1792 1793 cancel_work_sync(&hba->clk_scaling.suspend_work); 1794 cancel_work_sync(&hba->clk_scaling.resume_work); 1795 1796 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1797 { 1798 if (!hba->clk_scaling.is_suspended) { 1799 suspend = true; 1800 hba->clk_scaling.is_suspended = true; 1801 hba->clk_scaling.window_start_t = 0; 1802 } 1803 } 1804 1805 if (suspend) 1806 devfreq_suspend_device(hba->devfreq); 1807 } 1808 1809 static void ufshcd_resume_clkscaling(struct ufs_hba *hba) 1810 { 1811 bool resume = false; 1812 1813 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1814 { 1815 if (hba->clk_scaling.is_suspended) { 1816 resume = true; 1817 hba->clk_scaling.is_suspended = false; 1818 } 1819 } 1820 1821 if (resume) 1822 devfreq_resume_device(hba->devfreq); 1823 } 1824 1825 static ssize_t ufshcd_clkscale_enable_show(struct device *dev, 1826 struct device_attribute *attr, char *buf) 1827 { 1828 struct ufs_hba *hba = dev_get_drvdata(dev); 1829 1830 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled); 1831 } 1832 1833 static ssize_t ufshcd_clkscale_enable_store(struct device *dev, 1834 struct device_attribute *attr, const char *buf, size_t count) 1835 { 1836 struct ufs_hba *hba = dev_get_drvdata(dev); 1837 struct ufs_clk_info *clki; 1838 unsigned long freq; 1839 u32 value; 1840 int err = 0; 1841 1842 if (kstrtou32(buf, 0, &value)) 1843 return -EINVAL; 1844 1845 down(&hba->host_sem); 1846 if (!ufshcd_is_user_access_allowed(hba)) { 1847 err = -EBUSY; 1848 goto out; 1849 } 1850 1851 value = !!value; 1852 if (value == hba->clk_scaling.is_enabled) 1853 goto out; 1854 1855 ufshcd_rpm_get_sync(hba); 1856 ufshcd_hold(hba); 1857 1858 hba->clk_scaling.is_enabled = value; 1859 1860 if (value) { 1861 ufshcd_resume_clkscaling(hba); 1862 goto out_rel; 1863 } 1864 1865 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); 1866 freq = clki->max_freq; 1867 1868 ufshcd_suspend_clkscaling(hba); 1869 1870 if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) 1871 goto out_rel; 1872 1873 err = ufshcd_devfreq_scale(hba, freq, true); 1874 if (err) 1875 dev_err(hba->dev, "%s: failed to scale clocks up %d\n", 1876 __func__, err); 1877 else 1878 hba->clk_scaling.target_freq = freq; 1879 1880 out_rel: 1881 ufshcd_release(hba); 1882 ufshcd_rpm_put_sync(hba); 1883 out: 1884 up(&hba->host_sem); 1885 return err ? err : count; 1886 } 1887 1888 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba) 1889 { 1890 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; 1891 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; 1892 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); 1893 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; 1894 hba->clk_scaling.enable_attr.attr.mode = 0644; 1895 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) 1896 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); 1897 } 1898 1899 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) 1900 { 1901 if (hba->clk_scaling.enable_attr.attr.name) 1902 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); 1903 } 1904 1905 static void ufshcd_init_clk_scaling(struct ufs_hba *hba) 1906 { 1907 if (!ufshcd_is_clkscaling_supported(hba)) 1908 return; 1909 1910 if (!hba->clk_scaling.min_gear) 1911 hba->clk_scaling.min_gear = UFS_HS_G1; 1912 1913 if (!hba->clk_scaling.wb_gear) 1914 /* Use intermediate gear speed HS_G3 as the default wb_gear */ 1915 hba->clk_scaling.wb_gear = UFS_HS_G3; 1916 1917 INIT_WORK(&hba->clk_scaling.suspend_work, 1918 ufshcd_clk_scaling_suspend_work); 1919 INIT_WORK(&hba->clk_scaling.resume_work, 1920 ufshcd_clk_scaling_resume_work); 1921 1922 spin_lock_init(&hba->clk_scaling.lock); 1923 1924 hba->clk_scaling.workq = alloc_ordered_workqueue( 1925 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); 1926 1927 hba->clk_scaling.is_initialized = true; 1928 } 1929 1930 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) 1931 { 1932 if (!hba->clk_scaling.is_initialized) 1933 return; 1934 1935 ufshcd_remove_clk_scaling_sysfs(hba); 1936 destroy_workqueue(hba->clk_scaling.workq); 1937 ufshcd_devfreq_remove(hba); 1938 hba->clk_scaling.is_initialized = false; 1939 } 1940 1941 static void ufshcd_ungate_work(struct work_struct *work) 1942 { 1943 int ret; 1944 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1945 clk_gating.ungate_work); 1946 1947 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 1948 1949 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 1950 if (hba->clk_gating.state == CLKS_ON) 1951 return; 1952 } 1953 1954 ufshcd_hba_vreg_set_hpm(hba); 1955 ufshcd_setup_clocks(hba, true); 1956 1957 ufshcd_enable_irq(hba); 1958 1959 /* Exit from hibern8 */ 1960 if (ufshcd_can_hibern8_during_gating(hba)) { 1961 /* Prevent gating in this path */ 1962 hba->clk_gating.is_suspended = true; 1963 if (ufshcd_is_link_hibern8(hba)) { 1964 ret = ufshcd_uic_hibern8_exit(hba); 1965 if (ret) 1966 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 1967 __func__, ret); 1968 else 1969 ufshcd_set_link_active(hba); 1970 } 1971 hba->clk_gating.is_suspended = false; 1972 } 1973 } 1974 1975 /** 1976 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 1977 * Also, exit from hibern8 mode and set the link as active. 1978 * @hba: per adapter instance 1979 */ 1980 void ufshcd_hold(struct ufs_hba *hba) 1981 { 1982 bool flush_result; 1983 unsigned long flags; 1984 1985 if (!ufshcd_is_clkgating_allowed(hba) || 1986 !hba->clk_gating.is_initialized) 1987 return; 1988 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1989 hba->clk_gating.active_reqs++; 1990 1991 start: 1992 switch (hba->clk_gating.state) { 1993 case CLKS_ON: 1994 /* 1995 * Wait for the ungate work to complete if in progress. 1996 * Though the clocks may be in ON state, the link could 1997 * still be in hibner8 state if hibern8 is allowed 1998 * during clock gating. 1999 * Make sure we exit hibern8 state also in addition to 2000 * clocks being ON. 2001 */ 2002 if (ufshcd_can_hibern8_during_gating(hba) && 2003 ufshcd_is_link_hibern8(hba)) { 2004 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 2005 flush_result = flush_work(&hba->clk_gating.ungate_work); 2006 if (hba->clk_gating.is_suspended && !flush_result) 2007 return; 2008 spin_lock_irqsave(&hba->clk_gating.lock, flags); 2009 goto start; 2010 } 2011 break; 2012 case REQ_CLKS_OFF: 2013 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 2014 hba->clk_gating.state = CLKS_ON; 2015 trace_ufshcd_clk_gating(hba, 2016 hba->clk_gating.state); 2017 break; 2018 } 2019 /* 2020 * If we are here, it means gating work is either done or 2021 * currently running. Hence, fall through to cancel gating 2022 * work and to enable clocks. 2023 */ 2024 fallthrough; 2025 case CLKS_OFF: 2026 hba->clk_gating.state = REQ_CLKS_ON; 2027 trace_ufshcd_clk_gating(hba, 2028 hba->clk_gating.state); 2029 queue_work(hba->clk_gating.clk_gating_workq, 2030 &hba->clk_gating.ungate_work); 2031 /* 2032 * fall through to check if we should wait for this 2033 * work to be done or not. 2034 */ 2035 fallthrough; 2036 case REQ_CLKS_ON: 2037 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 2038 flush_work(&hba->clk_gating.ungate_work); 2039 /* Make sure state is CLKS_ON before returning */ 2040 spin_lock_irqsave(&hba->clk_gating.lock, flags); 2041 goto start; 2042 default: 2043 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 2044 __func__, hba->clk_gating.state); 2045 break; 2046 } 2047 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 2048 } 2049 EXPORT_SYMBOL_GPL(ufshcd_hold); 2050 2051 static void ufshcd_gate_work(struct work_struct *work) 2052 { 2053 struct ufs_hba *hba = container_of(work, struct ufs_hba, 2054 clk_gating.gate_work.work); 2055 int ret; 2056 2057 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 2058 /* 2059 * In case you are here to cancel this work the gating state 2060 * would be marked as REQ_CLKS_ON. In this case save time by 2061 * skipping the gating work and exit after changing the clock 2062 * state to CLKS_ON. 2063 */ 2064 if (hba->clk_gating.is_suspended || 2065 hba->clk_gating.state != REQ_CLKS_OFF) { 2066 hba->clk_gating.state = CLKS_ON; 2067 trace_ufshcd_clk_gating(hba, 2068 hba->clk_gating.state); 2069 return; 2070 } 2071 2072 if (hba->clk_gating.active_reqs) 2073 return; 2074 } 2075 2076 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2077 if (ufshcd_is_ufs_dev_busy(hba) || 2078 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2079 return; 2080 } 2081 2082 /* put the link into hibern8 mode before turning off clocks */ 2083 if (ufshcd_can_hibern8_during_gating(hba)) { 2084 ret = ufshcd_uic_hibern8_enter(hba); 2085 if (ret) { 2086 hba->clk_gating.state = CLKS_ON; 2087 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 2088 __func__, ret); 2089 trace_ufshcd_clk_gating(hba, 2090 hba->clk_gating.state); 2091 return; 2092 } 2093 ufshcd_set_link_hibern8(hba); 2094 } 2095 2096 ufshcd_disable_irq(hba); 2097 2098 ufshcd_setup_clocks(hba, false); 2099 2100 /* Put the host controller in low power mode if possible */ 2101 ufshcd_hba_vreg_set_lpm(hba); 2102 /* 2103 * In case you are here to cancel this work the gating state 2104 * would be marked as REQ_CLKS_ON. In this case keep the state 2105 * as REQ_CLKS_ON which would anyway imply that clocks are off 2106 * and a request to turn them on is pending. By doing this way, 2107 * we keep the state machine in tact and this would ultimately 2108 * prevent from doing cancel work multiple times when there are 2109 * new requests arriving before the current cancel work is done. 2110 */ 2111 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2112 if (hba->clk_gating.state == REQ_CLKS_OFF) { 2113 hba->clk_gating.state = CLKS_OFF; 2114 trace_ufshcd_clk_gating(hba, 2115 hba->clk_gating.state); 2116 } 2117 } 2118 2119 static void __ufshcd_release(struct ufs_hba *hba) 2120 { 2121 lockdep_assert_held(&hba->clk_gating.lock); 2122 2123 if (!ufshcd_is_clkgating_allowed(hba)) 2124 return; 2125 2126 hba->clk_gating.active_reqs--; 2127 2128 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || 2129 !hba->clk_gating.is_initialized || 2130 hba->clk_gating.state == CLKS_OFF) 2131 return; 2132 2133 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2134 if (ufshcd_has_pending_tasks(hba) || 2135 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2136 return; 2137 } 2138 2139 hba->clk_gating.state = REQ_CLKS_OFF; 2140 trace_ufshcd_clk_gating(hba, hba->clk_gating.state); 2141 queue_delayed_work(hba->clk_gating.clk_gating_workq, 2142 &hba->clk_gating.gate_work, 2143 msecs_to_jiffies(hba->clk_gating.delay_ms)); 2144 } 2145 2146 void ufshcd_release(struct ufs_hba *hba) 2147 { 2148 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2149 __ufshcd_release(hba); 2150 } 2151 EXPORT_SYMBOL_GPL(ufshcd_release); 2152 2153 static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 2154 struct device_attribute *attr, char *buf) 2155 { 2156 struct ufs_hba *hba = dev_get_drvdata(dev); 2157 2158 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); 2159 } 2160 2161 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) 2162 { 2163 struct ufs_hba *hba = dev_get_drvdata(dev); 2164 2165 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2166 hba->clk_gating.delay_ms = value; 2167 } 2168 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); 2169 2170 static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 2171 struct device_attribute *attr, const char *buf, size_t count) 2172 { 2173 unsigned long value; 2174 2175 if (kstrtoul(buf, 0, &value)) 2176 return -EINVAL; 2177 2178 ufshcd_clkgate_delay_set(dev, value); 2179 return count; 2180 } 2181 2182 static ssize_t ufshcd_clkgate_enable_show(struct device *dev, 2183 struct device_attribute *attr, char *buf) 2184 { 2185 struct ufs_hba *hba = dev_get_drvdata(dev); 2186 2187 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); 2188 } 2189 2190 static ssize_t ufshcd_clkgate_enable_store(struct device *dev, 2191 struct device_attribute *attr, const char *buf, size_t count) 2192 { 2193 struct ufs_hba *hba = dev_get_drvdata(dev); 2194 u32 value; 2195 2196 if (kstrtou32(buf, 0, &value)) 2197 return -EINVAL; 2198 2199 value = !!value; 2200 2201 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2202 2203 if (value == hba->clk_gating.is_enabled) 2204 return count; 2205 2206 if (value) 2207 __ufshcd_release(hba); 2208 else 2209 hba->clk_gating.active_reqs++; 2210 2211 hba->clk_gating.is_enabled = value; 2212 2213 return count; 2214 } 2215 2216 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba) 2217 { 2218 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 2219 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 2220 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 2221 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 2222 hba->clk_gating.delay_attr.attr.mode = 0644; 2223 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 2224 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 2225 2226 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; 2227 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; 2228 sysfs_attr_init(&hba->clk_gating.enable_attr.attr); 2229 hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; 2230 hba->clk_gating.enable_attr.attr.mode = 0644; 2231 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) 2232 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); 2233 } 2234 2235 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) 2236 { 2237 if (hba->clk_gating.delay_attr.attr.name) 2238 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 2239 if (hba->clk_gating.enable_attr.attr.name) 2240 device_remove_file(hba->dev, &hba->clk_gating.enable_attr); 2241 } 2242 2243 static void ufshcd_init_clk_gating(struct ufs_hba *hba) 2244 { 2245 if (!ufshcd_is_clkgating_allowed(hba)) 2246 return; 2247 2248 hba->clk_gating.state = CLKS_ON; 2249 2250 hba->clk_gating.delay_ms = 150; 2251 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 2252 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 2253 2254 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( 2255 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, 2256 hba->host->host_no); 2257 2258 ufshcd_init_clk_gating_sysfs(hba); 2259 2260 hba->clk_gating.is_enabled = true; 2261 hba->clk_gating.is_initialized = true; 2262 } 2263 2264 static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 2265 { 2266 if (!hba->clk_gating.is_initialized) 2267 return; 2268 2269 ufshcd_remove_clk_gating_sysfs(hba); 2270 2271 /* Ungate the clock if necessary. */ 2272 ufshcd_hold(hba); 2273 hba->clk_gating.is_initialized = false; 2274 ufshcd_release(hba); 2275 2276 destroy_workqueue(hba->clk_gating.clk_gating_workq); 2277 } 2278 2279 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 2280 { 2281 bool queue_resume_work = false; 2282 ktime_t curr_t; 2283 2284 if (!ufshcd_is_clkscaling_supported(hba)) 2285 return; 2286 2287 curr_t = ktime_get(); 2288 2289 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2290 2291 if (!hba->clk_scaling.active_reqs++) 2292 queue_resume_work = true; 2293 2294 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) 2295 return; 2296 2297 if (queue_resume_work) 2298 queue_work(hba->clk_scaling.workq, 2299 &hba->clk_scaling.resume_work); 2300 2301 if (!hba->clk_scaling.window_start_t) { 2302 hba->clk_scaling.window_start_t = curr_t; 2303 hba->clk_scaling.tot_busy_t = 0; 2304 hba->clk_scaling.is_busy_started = false; 2305 } 2306 2307 if (!hba->clk_scaling.is_busy_started) { 2308 hba->clk_scaling.busy_start_t = curr_t; 2309 hba->clk_scaling.is_busy_started = true; 2310 } 2311 } 2312 2313 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 2314 { 2315 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 2316 2317 if (!ufshcd_is_clkscaling_supported(hba)) 2318 return; 2319 2320 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2321 2322 hba->clk_scaling.active_reqs--; 2323 if (!scaling->active_reqs && scaling->is_busy_started) { 2324 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 2325 scaling->busy_start_t)); 2326 scaling->busy_start_t = 0; 2327 scaling->is_busy_started = false; 2328 } 2329 } 2330 2331 static inline int ufshcd_monitor_opcode2dir(u8 opcode) 2332 { 2333 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16) 2334 return READ; 2335 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16) 2336 return WRITE; 2337 else 2338 return -EINVAL; 2339 } 2340 2341 /* Must only be called for SCSI commands. */ 2342 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2343 struct scsi_cmnd *cmd) 2344 { 2345 const struct ufs_hba_monitor *m = &hba->monitor; 2346 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2347 2348 return m->enabled && 2349 (!m->chunk_size || m->chunk_size == cmd->sdb.length) && 2350 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); 2351 } 2352 2353 static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2354 { 2355 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2356 unsigned long flags; 2357 2358 spin_lock_irqsave(hba->host->host_lock, flags); 2359 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0) 2360 hba->monitor.busy_start_ts[dir] = ktime_get(); 2361 spin_unlock_irqrestore(hba->host->host_lock, flags); 2362 } 2363 2364 static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2365 { 2366 struct request *req = scsi_cmd_to_rq(cmd); 2367 const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2368 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2369 unsigned long flags; 2370 2371 spin_lock_irqsave(hba->host->host_lock, flags); 2372 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2373 struct ufs_hba_monitor *m = &hba->monitor; 2374 ktime_t now, inc, lat; 2375 2376 now = lrbp->compl_time_stamp; 2377 inc = ktime_sub(now, m->busy_start_ts[dir]); 2378 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc); 2379 m->nr_sec_rw[dir] += blk_rq_sectors(req); 2380 2381 /* Update latencies */ 2382 m->nr_req[dir]++; 2383 lat = ktime_sub(now, lrbp->issue_time_stamp); 2384 m->lat_sum[dir] += lat; 2385 if (m->lat_max[dir] < lat || !m->lat_max[dir]) 2386 m->lat_max[dir] = lat; 2387 if (m->lat_min[dir] > lat || !m->lat_min[dir]) 2388 m->lat_min[dir] = lat; 2389 2390 m->nr_queued[dir]--; 2391 /* Push forward the busy start of monitor */ 2392 m->busy_start_ts[dir] = now; 2393 } 2394 spin_unlock_irqrestore(hba->host->host_lock, flags); 2395 } 2396 2397 /* Returns %true for SCSI commands and %false for device management commands. */ 2398 static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) 2399 { 2400 return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); 2401 } 2402 2403 /** 2404 * ufshcd_send_command - Send SCSI or device management commands 2405 * @hba: per adapter instance 2406 * @cmd: SCSI command or device management command pointer 2407 * @hwq: pointer to hardware queue instance 2408 */ 2409 static inline void ufshcd_send_command(struct ufs_hba *hba, 2410 struct scsi_cmnd *cmd, 2411 struct ufs_hw_queue *hwq) 2412 { 2413 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2414 const int tag = scsi_cmd_to_rq(cmd)->tag; 2415 unsigned long flags; 2416 2417 if (hba->monitor.enabled) { 2418 lrbp->issue_time_stamp = ktime_get(); 2419 lrbp->issue_time_stamp_local_clock = local_clock(); 2420 lrbp->compl_time_stamp = ktime_set(0, 0); 2421 lrbp->compl_time_stamp_local_clock = 0; 2422 } 2423 if (ufshcd_is_scsi_cmd(cmd)) { 2424 ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); 2425 ufshcd_clk_scaling_start_busy(hba); 2426 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 2427 ufshcd_start_monitor(hba, cmd); 2428 } 2429 2430 if (hba->mcq_enabled) { 2431 int utrd_size = sizeof(struct utp_transfer_req_desc); 2432 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; 2433 struct utp_transfer_req_desc *dest; 2434 2435 spin_lock(&hwq->sq_lock); 2436 dest = hwq->sqe_base_addr + hwq->sq_tail_slot; 2437 memcpy(dest, src, utrd_size); 2438 ufshcd_inc_sq_tail(hwq); 2439 spin_unlock(&hwq->sq_lock); 2440 } else { 2441 spin_lock_irqsave(&hba->outstanding_lock, flags); 2442 if (hba->vops && hba->vops->setup_xfer_req) 2443 hba->vops->setup_xfer_req(hba, tag, 2444 ufshcd_is_scsi_cmd(cmd)); 2445 __set_bit(tag, &hba->outstanding_reqs); 2446 ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 2447 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2448 } 2449 } 2450 2451 /** 2452 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2453 * @cmd: SCSI command 2454 */ 2455 static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) 2456 { 2457 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2458 u8 *const sense_buffer = cmd->sense_buffer; 2459 u16 resp_len; 2460 int len; 2461 2462 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); 2463 if (sense_buffer && resp_len) { 2464 int len_to_copy; 2465 2466 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 2467 len_to_copy = min_t(int, UFS_SENSE_SIZE, len); 2468 2469 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, 2470 len_to_copy); 2471 } 2472 } 2473 2474 /** 2475 * ufshcd_copy_query_response() - Copy the Query Response and the data 2476 * descriptor 2477 * @hba: per adapter instance 2478 * @lrbp: pointer to local reference block 2479 * 2480 * Return: 0 upon success; < 0 upon failure. 2481 */ 2482 static 2483 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2484 { 2485 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2486 2487 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 2488 2489 /* Get the descriptor */ 2490 if (hba->dev_cmd.query.descriptor && 2491 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 2492 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 2493 GENERAL_UPIU_REQUEST_SIZE; 2494 u16 resp_len; 2495 u16 buf_len; 2496 2497 /* data segment length */ 2498 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 2499 .data_segment_length); 2500 buf_len = be16_to_cpu( 2501 hba->dev_cmd.query.request.upiu_req.length); 2502 if (likely(buf_len >= resp_len)) { 2503 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 2504 } else { 2505 dev_warn(hba->dev, 2506 "%s: rsp size %d is bigger than buffer size %d", 2507 __func__, resp_len, buf_len); 2508 return -EINVAL; 2509 } 2510 } 2511 2512 return 0; 2513 } 2514 2515 /** 2516 * ufshcd_hba_capabilities - Read controller capabilities 2517 * @hba: per adapter instance 2518 * 2519 * Return: 0 on success, negative on error. 2520 */ 2521 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) 2522 { 2523 int err; 2524 2525 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 2526 2527 /* nutrs and nutmrs are 0 based values */ 2528 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; 2529 hba->nutmrs = 2530 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2531 2532 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; 2533 2534 /* Read crypto capabilities */ 2535 err = ufshcd_hba_init_crypto_capabilities(hba); 2536 if (err) { 2537 dev_err(hba->dev, "crypto setup failed\n"); 2538 return err; 2539 } 2540 2541 /* 2542 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and 2543 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which 2544 * means we can simply read values regardless of version. 2545 */ 2546 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); 2547 /* 2548 * 0h: legacy single doorbell support is available 2549 * 1h: indicate that legacy single doorbell support has been removed 2550 */ 2551 if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) 2552 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); 2553 else 2554 hba->lsdb_sup = true; 2555 2556 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); 2557 2558 return 0; 2559 } 2560 2561 /** 2562 * ufshcd_ready_for_uic_cmd - Check if controller is ready 2563 * to accept UIC commands 2564 * @hba: per adapter instance 2565 * 2566 * Return: true on success, else false. 2567 */ 2568 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 2569 { 2570 u32 val; 2571 int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, 2572 500, uic_cmd_timeout * 1000, false, hba, 2573 REG_CONTROLLER_STATUS); 2574 return ret == 0; 2575 } 2576 2577 /** 2578 * ufshcd_get_upmcrs - Get the power mode change request status 2579 * @hba: Pointer to adapter instance 2580 * 2581 * This function gets the UPMCRS field of HCS register 2582 * 2583 * Return: value of UPMCRS field. 2584 */ 2585 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 2586 { 2587 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 2588 } 2589 2590 /** 2591 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer 2592 * @hba: per adapter instance 2593 * @uic_cmd: UIC command 2594 */ 2595 static inline void 2596 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2597 { 2598 lockdep_assert_held(&hba->uic_cmd_mutex); 2599 2600 WARN_ON(hba->active_uic_cmd); 2601 2602 hba->active_uic_cmd = uic_cmd; 2603 2604 /* Write Args */ 2605 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 2606 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 2607 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 2608 2609 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND); 2610 2611 /* Write UIC Cmd */ 2612 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 2613 REG_UIC_COMMAND); 2614 } 2615 2616 /** 2617 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command 2618 * @hba: per adapter instance 2619 * @uic_cmd: UIC command 2620 * 2621 * Return: 0 only if success. 2622 */ 2623 static int 2624 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2625 { 2626 int ret; 2627 unsigned long flags; 2628 2629 lockdep_assert_held(&hba->uic_cmd_mutex); 2630 2631 if (wait_for_completion_timeout(&uic_cmd->done, 2632 msecs_to_jiffies(uic_cmd_timeout))) { 2633 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2634 } else { 2635 ret = -ETIMEDOUT; 2636 dev_err(hba->dev, 2637 "uic cmd 0x%x with arg3 0x%x completion timeout\n", 2638 uic_cmd->command, uic_cmd->argument3); 2639 2640 if (!uic_cmd->cmd_active) { 2641 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n", 2642 __func__); 2643 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2644 } 2645 } 2646 2647 spin_lock_irqsave(hba->host->host_lock, flags); 2648 hba->active_uic_cmd = NULL; 2649 spin_unlock_irqrestore(hba->host->host_lock, flags); 2650 2651 return ret; 2652 } 2653 2654 /** 2655 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2656 * @hba: per adapter instance 2657 * @uic_cmd: UIC command 2658 * 2659 * Return: 0 if successful; < 0 upon failure. 2660 */ 2661 static int 2662 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2663 { 2664 lockdep_assert_held(&hba->uic_cmd_mutex); 2665 2666 if (!ufshcd_ready_for_uic_cmd(hba)) { 2667 dev_err(hba->dev, 2668 "Controller not ready to accept UIC commands\n"); 2669 return -EIO; 2670 } 2671 2672 init_completion(&uic_cmd->done); 2673 2674 uic_cmd->cmd_active = true; 2675 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 2676 2677 return 0; 2678 } 2679 2680 /** 2681 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2682 * @hba: per adapter instance 2683 * @uic_cmd: UIC command 2684 * 2685 * Return: 0 only if success. 2686 */ 2687 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2688 { 2689 unsigned long flags; 2690 int ret; 2691 2692 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 2693 return 0; 2694 2695 ufshcd_hold(hba); 2696 mutex_lock(&hba->uic_cmd_mutex); 2697 ufshcd_add_delay_before_dme_cmd(hba); 2698 2699 spin_lock_irqsave(hba->host->host_lock, flags); 2700 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); 2701 spin_unlock_irqrestore(hba->host->host_lock, flags); 2702 2703 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 2704 if (!ret) 2705 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 2706 2707 mutex_unlock(&hba->uic_cmd_mutex); 2708 2709 ufshcd_release(hba); 2710 return ret; 2711 } 2712 2713 /** 2714 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) 2715 * @hba: per-adapter instance 2716 * @lrbp: pointer to local reference block 2717 * @sg_entries: The number of sg lists actually used 2718 * @sg_list: Pointer to SG list 2719 */ 2720 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, 2721 struct scatterlist *sg_list) 2722 { 2723 struct ufshcd_sg_entry *prd; 2724 struct scatterlist *sg; 2725 int i; 2726 2727 if (sg_entries) { 2728 2729 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 2730 lrbp->utr_descriptor_ptr->prd_table_length = 2731 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); 2732 else 2733 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); 2734 2735 prd = lrbp->ucd_prdt_ptr; 2736 2737 for_each_sg(sg_list, sg, sg_entries, i) { 2738 const unsigned int len = sg_dma_len(sg); 2739 2740 /* 2741 * From the UFSHCI spec: "Data Byte Count (DBC): A '0' 2742 * based value that indicates the length, in bytes, of 2743 * the data block. A maximum of length of 256KB may 2744 * exist for any entry. Bits 1:0 of this field shall be 2745 * 11b to indicate Dword granularity. A value of '3' 2746 * indicates 4 bytes, '7' indicates 8 bytes, etc." 2747 */ 2748 WARN_ONCE(len > SZ_256K, "len = %#x\n", len); 2749 prd->size = cpu_to_le32(len - 1); 2750 prd->addr = cpu_to_le64(sg->dma_address); 2751 prd->reserved = 0; 2752 prd = (void *)prd + ufshcd_sg_entry_size(hba); 2753 } 2754 } else { 2755 lrbp->utr_descriptor_ptr->prd_table_length = 0; 2756 } 2757 } 2758 2759 /** 2760 * ufshcd_map_sg - Map scatter-gather list to prdt 2761 * @hba: per adapter instance 2762 * @cmd: SCSI command 2763 * 2764 * Return: 0 in case of success, non-zero value in case of failure. 2765 */ 2766 static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2767 { 2768 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2769 int sg_segments = scsi_dma_map(cmd); 2770 2771 if (sg_segments < 0) 2772 return sg_segments; 2773 2774 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2775 2776 return ufshcd_crypto_fill_prdt(hba, cmd); 2777 } 2778 2779 /** 2780 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request 2781 * descriptor according to request 2782 * @hba: per adapter instance 2783 * @lrbp: pointer to local reference block 2784 * @upiu_flags: flags required in the header 2785 * @cmd_dir: requests data direction 2786 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) 2787 */ 2788 static void 2789 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2790 u8 *upiu_flags, enum dma_data_direction cmd_dir, 2791 int ehs_length) 2792 { 2793 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 2794 struct request_desc_header *h = &req_desc->header; 2795 enum utp_data_direction data_direction; 2796 2797 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2798 2799 *h = (typeof(*h)){ }; 2800 2801 if (cmd_dir == DMA_FROM_DEVICE) { 2802 data_direction = UTP_DEVICE_TO_HOST; 2803 *upiu_flags = UPIU_CMD_FLAGS_READ; 2804 } else if (cmd_dir == DMA_TO_DEVICE) { 2805 data_direction = UTP_HOST_TO_DEVICE; 2806 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 2807 } else { 2808 data_direction = UTP_NO_DATA_TRANSFER; 2809 *upiu_flags = UPIU_CMD_FLAGS_NONE; 2810 } 2811 2812 h->command_type = lrbp->command_type; 2813 h->data_direction = data_direction; 2814 h->ehs_length = ehs_length; 2815 2816 if (lrbp->intr_cmd) 2817 h->interrupt = 1; 2818 2819 /* Prepare crypto related dwords */ 2820 ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); 2821 2822 /* 2823 * assigning invalid value for command status. Controller 2824 * updates OCS on command completion, with the command 2825 * status 2826 */ 2827 h->ocs = OCS_INVALID_COMMAND_STATUS; 2828 2829 req_desc->prd_table_length = 0; 2830 } 2831 2832 /** 2833 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2834 * for scsi commands 2835 * @cmd: SCSI command 2836 * @upiu_flags: flags 2837 */ 2838 static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, 2839 u8 upiu_flags) 2840 { 2841 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2842 const int tag = scsi_cmd_to_rq(cmd)->tag; 2843 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2844 unsigned short cdb_len; 2845 2846 ucd_req_ptr->header = (struct utp_upiu_header){ 2847 .transaction_code = UPIU_TRANSACTION_COMMAND, 2848 .flags = upiu_flags, 2849 .lun = lrbp->lun, 2850 .task_tag = tag, 2851 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, 2852 }; 2853 2854 WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); 2855 2856 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2857 2858 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); 2859 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); 2860 2861 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2862 } 2863 2864 /** 2865 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2866 * @hba: UFS hba 2867 * @cmd: SCSI command pointer 2868 * @upiu_flags: flags 2869 */ 2870 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2871 struct scsi_cmnd *cmd, u8 upiu_flags) 2872 { 2873 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2874 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2875 const int tag = scsi_cmd_to_rq(cmd)->tag; 2876 struct ufs_query *query = &hba->dev_cmd.query; 2877 u16 len = be16_to_cpu(query->request.upiu_req.length); 2878 2879 /* Query request header */ 2880 ucd_req_ptr->header = (struct utp_upiu_header){ 2881 .transaction_code = UPIU_TRANSACTION_QUERY_REQ, 2882 .flags = upiu_flags, 2883 .lun = lrbp->lun, 2884 .task_tag = tag, 2885 .query_function = query->request.query_func, 2886 /* Data segment length only need for WRITE_DESC */ 2887 .data_segment_length = 2888 query->request.upiu_req.opcode == 2889 UPIU_QUERY_OPCODE_WRITE_DESC ? 2890 cpu_to_be16(len) : 2891 0, 2892 }; 2893 2894 /* Copy the Query Request buffer as is */ 2895 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 2896 QUERY_OSF_SIZE); 2897 2898 /* Copy the Descriptor */ 2899 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2900 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2901 } 2902 2903 static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) 2904 { 2905 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2906 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2907 const int tag = scsi_cmd_to_rq(cmd)->tag; 2908 2909 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2910 2911 ucd_req_ptr->header = (struct utp_upiu_header){ 2912 .transaction_code = UPIU_TRANSACTION_NOP_OUT, 2913 .task_tag = tag, 2914 }; 2915 } 2916 2917 /** 2918 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2919 * for Device Management Purposes 2920 * @hba: per adapter instance 2921 * @cmd: SCSI command pointer 2922 * 2923 * Return: 0 upon success; < 0 upon failure. 2924 */ 2925 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2926 struct scsi_cmnd *cmd) 2927 { 2928 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2929 u8 upiu_flags; 2930 int ret = 0; 2931 2932 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 2933 2934 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2935 ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); 2936 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2937 ufshcd_prepare_utp_nop_upiu(cmd); 2938 else 2939 ret = -EINVAL; 2940 2941 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2942 2943 return ret; 2944 } 2945 2946 /** 2947 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2948 * for SCSI Purposes 2949 * @hba: per adapter instance 2950 * @cmd: SCSI command 2951 */ 2952 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2953 { 2954 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2955 struct request *rq = scsi_cmd_to_rq(cmd); 2956 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2957 u8 upiu_flags; 2958 2959 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, 2960 cmd->sc_data_direction, 0); 2961 if (ioprio_class == IOPRIO_CLASS_RT) 2962 upiu_flags |= UPIU_CMD_FLAGS_CP; 2963 ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); 2964 } 2965 2966 static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2967 { 2968 const int i = scsi_cmd_to_rq(cmd)->tag; 2969 struct utp_transfer_cmd_desc *cmd_descp = 2970 (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); 2971 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2972 dma_addr_t cmd_desc_element_addr = 2973 hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); 2974 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2975 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2976 struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); 2977 2978 lrb->utr_descriptor_ptr = utrdlp + i; 2979 lrb->utrd_dma_addr = 2980 hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); 2981 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2982 lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2983 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2984 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2985 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2986 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2987 } 2988 2989 static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2990 u8 lun, int tag) 2991 { 2992 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2993 2994 ufshcd_init_lrb(hba, cmd); 2995 2996 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2997 2998 lrbp->lun = lun; 2999 ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? 3000 scsi_cmd_to_rq(cmd) : NULL, lrbp); 3001 } 3002 3003 static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3004 u8 lun, int tag) 3005 { 3006 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3007 3008 __ufshcd_setup_cmd(hba, cmd, lun, tag); 3009 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 3010 lrbp->req_abort_skip = false; 3011 3012 ufshcd_comp_scsi_upiu(hba, cmd); 3013 } 3014 3015 /** 3016 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 3017 * @upiu_wlun_id: UPIU W-LUN id 3018 * 3019 * Return: SCSI W-LUN id. 3020 */ 3021 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 3022 { 3023 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 3024 } 3025 3026 static inline bool is_device_wlun(struct scsi_device *sdev) 3027 { 3028 return sdev->lun == 3029 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 3030 } 3031 3032 /* 3033 * Associate the UFS controller queue with the default and poll HCTX types. 3034 * Initialize the mq_map[] arrays. 3035 */ 3036 static void ufshcd_map_queues(struct Scsi_Host *shost) 3037 { 3038 struct ufs_hba *hba = shost_priv(shost); 3039 int i, queue_offset = 0; 3040 3041 if (!is_mcq_supported(hba)) { 3042 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; 3043 hba->nr_queues[HCTX_TYPE_READ] = 0; 3044 hba->nr_queues[HCTX_TYPE_POLL] = 1; 3045 hba->nr_hw_queues = 1; 3046 } 3047 3048 for (i = 0; i < shost->nr_maps; i++) { 3049 struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 3050 3051 map->nr_queues = hba->nr_queues[i]; 3052 if (!map->nr_queues) 3053 continue; 3054 map->queue_offset = queue_offset; 3055 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) 3056 map->queue_offset = 0; 3057 3058 blk_mq_map_queues(map); 3059 queue_offset += map->nr_queues; 3060 } 3061 } 3062 3063 /* 3064 * The only purpose of this function is to make the SCSI core skip the memset() 3065 * call for the private command data. 3066 */ 3067 static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3068 { 3069 return 0; 3070 } 3071 3072 /** 3073 * ufshcd_queuecommand - main entry point for SCSI requests 3074 * @host: SCSI host pointer 3075 * @cmd: command from SCSI Midlayer 3076 * 3077 * Return: 0 for success, non-zero in case of failure. 3078 */ 3079 static enum scsi_qc_status ufshcd_queuecommand(struct Scsi_Host *host, 3080 struct scsi_cmnd *cmd) 3081 { 3082 struct ufs_hba *hba = shost_priv(host); 3083 int tag = scsi_cmd_to_rq(cmd)->tag; 3084 int err = 0; 3085 struct ufs_hw_queue *hwq = NULL; 3086 3087 switch (hba->ufshcd_state) { 3088 case UFSHCD_STATE_OPERATIONAL: 3089 break; 3090 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: 3091 /* 3092 * SCSI error handler can call ->queuecommand() while UFS error 3093 * handler is in progress. Error interrupts could change the 3094 * state from UFSHCD_STATE_RESET to 3095 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests 3096 * being issued in that case. 3097 */ 3098 if (ufshcd_eh_in_progress(hba)) { 3099 err = SCSI_MLQUEUE_HOST_BUSY; 3100 goto out; 3101 } 3102 break; 3103 case UFSHCD_STATE_EH_SCHEDULED_FATAL: 3104 /* 3105 * pm_runtime_get_sync() is used at error handling preparation 3106 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's 3107 * PM ops, it can never be finished if we let SCSI layer keep 3108 * retrying it, which gets err handler stuck forever. Neither 3109 * can we let the scsi cmd pass through, because UFS is in bad 3110 * state, the scsi cmd may eventually time out, which will get 3111 * err handler blocked for too long. So, just fail the scsi cmd 3112 * sent from PM ops, err handler can recover PM error anyways. 3113 */ 3114 if (hba->pm_op_in_progress) { 3115 hba->force_reset = true; 3116 set_host_byte(cmd, DID_BAD_TARGET); 3117 scsi_done(cmd); 3118 goto out; 3119 } 3120 fallthrough; 3121 case UFSHCD_STATE_RESET: 3122 err = SCSI_MLQUEUE_HOST_BUSY; 3123 goto out; 3124 case UFSHCD_STATE_ERROR: 3125 set_host_byte(cmd, DID_ERROR); 3126 scsi_done(cmd); 3127 goto out; 3128 } 3129 3130 hba->req_abort_count = 0; 3131 3132 ufshcd_hold(hba); 3133 3134 ufshcd_setup_scsi_cmd(hba, cmd, 3135 ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3136 3137 err = ufshcd_map_sg(hba, cmd); 3138 if (err) { 3139 ufshcd_release(hba); 3140 goto out; 3141 } 3142 3143 if (hba->mcq_enabled) 3144 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3145 3146 ufshcd_send_command(hba, cmd, hwq); 3147 3148 out: 3149 if (ufs_trigger_eh(hba)) { 3150 unsigned long flags; 3151 3152 spin_lock_irqsave(hba->host->host_lock, flags); 3153 ufshcd_schedule_eh_work(hba); 3154 spin_unlock_irqrestore(hba->host->host_lock, flags); 3155 } 3156 3157 return err; 3158 } 3159 3160 static enum scsi_qc_status ufshcd_queue_reserved_command(struct Scsi_Host *host, 3161 struct scsi_cmnd *cmd) 3162 { 3163 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3164 struct request *rq = scsi_cmd_to_rq(cmd); 3165 struct ufs_hba *hba = shost_priv(host); 3166 struct ufs_hw_queue *hwq = 3167 hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; 3168 3169 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3170 ufshcd_send_command(hba, cmd, hwq); 3171 return 0; 3172 } 3173 3174 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3175 enum dev_cmd_type cmd_type, u8 lun, int tag) 3176 { 3177 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3178 3179 __ufshcd_setup_cmd(hba, cmd, lun, tag); 3180 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3181 hba->dev_cmd.type = cmd_type; 3182 } 3183 3184 /* 3185 * Return: 0 upon success; < 0 upon failure. 3186 */ 3187 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3188 enum dev_cmd_type cmd_type, int tag) 3189 { 3190 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 3191 3192 return ufshcd_compose_devman_upiu(hba, cmd); 3193 } 3194 3195 /* 3196 * Check with the block layer if the command is inflight 3197 * @cmd: command to check. 3198 * 3199 * Return: true if command is inflight; false if not. 3200 */ 3201 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) 3202 { 3203 return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; 3204 } 3205 3206 /* 3207 * Clear the pending command in the controller and wait until 3208 * the controller confirms that the command has been cleared. 3209 * @hba: per adapter instance 3210 * @task_tag: The tag number of the command to be cleared. 3211 */ 3212 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) 3213 { 3214 u32 mask; 3215 int err; 3216 3217 if (hba->mcq_enabled) { 3218 /* 3219 * MCQ mode. Clean up the MCQ resources similar to 3220 * what the ufshcd_utrl_clear() does for SDB mode. 3221 */ 3222 err = ufshcd_mcq_sq_cleanup(hba, task_tag); 3223 if (err) { 3224 dev_err(hba->dev, "%s: failed tag=%d. err=%d\n", 3225 __func__, task_tag, err); 3226 return err; 3227 } 3228 return 0; 3229 } 3230 3231 mask = 1U << task_tag; 3232 3233 /* clear outstanding transaction before retry */ 3234 ufshcd_utrl_clear(hba, mask); 3235 3236 /* 3237 * wait for h/w to clear corresponding bit in door-bell. 3238 * max. wait is 1 sec. 3239 */ 3240 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, 3241 mask, ~mask, 1000, 1000); 3242 } 3243 3244 /** 3245 * ufshcd_dev_cmd_completion() - handles device management command responses 3246 * @hba: per adapter instance 3247 * @lrbp: pointer to local reference block 3248 * 3249 * Return: 0 upon success; < 0 upon failure. 3250 */ 3251 static int 3252 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 3253 { 3254 enum upiu_response_transaction resp; 3255 int err = 0; 3256 3257 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 3258 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 3259 3260 switch (resp) { 3261 case UPIU_TRANSACTION_NOP_IN: 3262 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 3263 err = -EINVAL; 3264 dev_err(hba->dev, "%s: unexpected response %x\n", 3265 __func__, resp); 3266 } 3267 break; 3268 case UPIU_TRANSACTION_QUERY_RSP: { 3269 u8 response = lrbp->ucd_rsp_ptr->header.response; 3270 3271 if (response == 0) { 3272 err = ufshcd_copy_query_response(hba, lrbp); 3273 } else { 3274 err = -EINVAL; 3275 dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", 3276 __func__, response); 3277 } 3278 break; 3279 } 3280 case UPIU_TRANSACTION_REJECT_UPIU: 3281 /* TODO: handle Reject UPIU Response */ 3282 err = -EPERM; 3283 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 3284 __func__); 3285 break; 3286 case UPIU_TRANSACTION_RESPONSE: 3287 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { 3288 err = -EINVAL; 3289 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); 3290 } 3291 break; 3292 default: 3293 err = -EINVAL; 3294 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 3295 __func__, resp); 3296 break; 3297 } 3298 3299 WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); 3300 return err; 3301 } 3302 3303 static void ufshcd_dev_man_lock(struct ufs_hba *hba) 3304 { 3305 ufshcd_hold(hba); 3306 mutex_lock(&hba->dev_cmd.lock); 3307 down_read(&hba->clk_scaling_lock); 3308 } 3309 3310 static void ufshcd_dev_man_unlock(struct ufs_hba *hba) 3311 { 3312 up_read(&hba->clk_scaling_lock); 3313 mutex_unlock(&hba->dev_cmd.lock); 3314 ufshcd_release(hba); 3315 } 3316 3317 static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) 3318 { 3319 /* 3320 * The caller must hold this lock to guarantee that the NOWAIT 3321 * allocation will succeed. 3322 */ 3323 lockdep_assert_held(&hba->dev_cmd.lock); 3324 3325 return scsi_get_internal_cmd( 3326 hba->host->pseudo_sdev, DMA_TO_DEVICE, 3327 BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); 3328 } 3329 3330 static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) 3331 { 3332 scsi_put_internal_cmd(cmd); 3333 } 3334 3335 /* 3336 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3337 * < 0 if another error occurred. 3338 */ 3339 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3340 const u32 tag, int timeout) 3341 { 3342 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3343 struct request *rq = scsi_cmd_to_rq(cmd); 3344 blk_status_t sts; 3345 3346 rq->timeout = timeout; 3347 sts = blk_execute_rq(rq, true); 3348 if (sts != BLK_STS_OK) 3349 return blk_status_to_errno(sts); 3350 return lrbp->utr_descriptor_ptr->header.ocs; 3351 } 3352 3353 /** 3354 * ufshcd_exec_dev_cmd - API for sending device management requests 3355 * @hba: UFS hba 3356 * @cmd_type: specifies the type (NOP, Query...) 3357 * @timeout: timeout in milliseconds 3358 * 3359 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3360 * < 0 if another error occurred. 3361 * 3362 * NOTE: Since there is only one available tag for device management commands, 3363 * it is expected you hold the hba->dev_cmd.lock mutex. 3364 */ 3365 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3366 enum dev_cmd_type cmd_type, int timeout) 3367 { 3368 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 3369 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3370 u32 tag; 3371 int err; 3372 3373 /* Protects use of hba->dev_cmd. */ 3374 lockdep_assert_held(&hba->dev_cmd.lock); 3375 3376 if (WARN_ON_ONCE(!cmd)) 3377 return -ENOMEM; 3378 3379 tag = scsi_cmd_to_rq(cmd)->tag; 3380 3381 err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); 3382 if (unlikely(err)) 3383 goto out; 3384 3385 err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); 3386 if (err == 0) 3387 err = ufshcd_dev_cmd_completion(hba, lrbp); 3388 3389 out: 3390 ufshcd_put_dev_mgmt_cmd(cmd); 3391 3392 return err; 3393 } 3394 3395 /** 3396 * ufshcd_init_query() - init the query response and request parameters 3397 * @hba: per-adapter instance 3398 * @request: address of the request pointer to be initialized 3399 * @response: address of the response pointer to be initialized 3400 * @opcode: operation to perform 3401 * @idn: flag idn to access 3402 * @index: LU number to access 3403 * @selector: query/flag/descriptor further identification 3404 */ 3405 static inline void ufshcd_init_query(struct ufs_hba *hba, 3406 struct ufs_query_req **request, struct ufs_query_res **response, 3407 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 3408 { 3409 *request = &hba->dev_cmd.query.request; 3410 *response = &hba->dev_cmd.query.response; 3411 memset(*request, 0, sizeof(struct ufs_query_req)); 3412 memset(*response, 0, sizeof(struct ufs_query_res)); 3413 (*request)->upiu_req.opcode = opcode; 3414 (*request)->upiu_req.idn = idn; 3415 (*request)->upiu_req.index = index; 3416 (*request)->upiu_req.selector = selector; 3417 } 3418 3419 /* 3420 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3421 * < 0 if another error occurred. 3422 */ 3423 static int ufshcd_query_flag_retry(struct ufs_hba *hba, 3424 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) 3425 { 3426 int ret; 3427 int retries; 3428 3429 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { 3430 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res); 3431 if (ret) 3432 dev_dbg(hba->dev, 3433 "%s: failed with error %d, retries %d\n", 3434 __func__, ret, retries); 3435 else 3436 break; 3437 } 3438 3439 if (ret) 3440 dev_err(hba->dev, 3441 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", 3442 __func__, opcode, idn, ret, retries); 3443 return ret; 3444 } 3445 3446 /** 3447 * ufshcd_query_flag() - API function for sending flag query requests 3448 * @hba: per-adapter instance 3449 * @opcode: flag query to perform 3450 * @idn: flag idn to access 3451 * @index: flag index to access 3452 * @flag_res: the flag value after the query request completes 3453 * 3454 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3455 * < 0 if another error occurred. 3456 */ 3457 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 3458 enum flag_idn idn, u8 index, bool *flag_res) 3459 { 3460 struct ufs_query_req *request = NULL; 3461 struct ufs_query_res *response = NULL; 3462 int err, selector = 0; 3463 int timeout = dev_cmd_timeout; 3464 3465 BUG_ON(!hba); 3466 3467 ufshcd_dev_man_lock(hba); 3468 3469 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3470 selector); 3471 3472 switch (opcode) { 3473 case UPIU_QUERY_OPCODE_SET_FLAG: 3474 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 3475 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 3476 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3477 break; 3478 case UPIU_QUERY_OPCODE_READ_FLAG: 3479 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3480 if (!flag_res) { 3481 /* No dummy reads */ 3482 dev_err(hba->dev, "%s: Invalid argument for read request\n", 3483 __func__); 3484 err = -EINVAL; 3485 goto out_unlock; 3486 } 3487 break; 3488 default: 3489 dev_err(hba->dev, 3490 "%s: Expected query flag opcode but got = %d\n", 3491 __func__, opcode); 3492 err = -EINVAL; 3493 goto out_unlock; 3494 } 3495 3496 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); 3497 3498 if (err) { 3499 dev_err(hba->dev, 3500 "%s: Sending flag query for idn %d failed, err = %d\n", 3501 __func__, idn, err); 3502 goto out_unlock; 3503 } 3504 3505 if (flag_res) 3506 *flag_res = (be32_to_cpu(response->upiu_res.value) & 3507 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 3508 3509 out_unlock: 3510 ufshcd_dev_man_unlock(hba); 3511 return err; 3512 } 3513 3514 /** 3515 * ufshcd_query_attr - API function for sending attribute requests 3516 * @hba: per-adapter instance 3517 * @opcode: attribute opcode 3518 * @idn: attribute idn to access 3519 * @index: index field 3520 * @selector: selector field 3521 * @attr_val: the attribute value after the query request completes 3522 * 3523 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3524 * < 0 if another error occurred. 3525 */ 3526 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 3527 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 3528 { 3529 struct ufs_query_req *request = NULL; 3530 struct ufs_query_res *response = NULL; 3531 int err; 3532 3533 BUG_ON(!hba); 3534 3535 if (!attr_val) { 3536 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 3537 __func__, opcode); 3538 return -EINVAL; 3539 } 3540 3541 ufshcd_dev_man_lock(hba); 3542 3543 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3544 selector); 3545 3546 switch (opcode) { 3547 case UPIU_QUERY_OPCODE_WRITE_ATTR: 3548 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3549 request->upiu_req.value = cpu_to_be32(*attr_val); 3550 break; 3551 case UPIU_QUERY_OPCODE_READ_ATTR: 3552 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3553 break; 3554 default: 3555 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 3556 __func__, opcode); 3557 err = -EINVAL; 3558 goto out_unlock; 3559 } 3560 3561 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3562 3563 if (err) { 3564 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3565 __func__, opcode, idn, index, err); 3566 goto out_unlock; 3567 } 3568 3569 *attr_val = be32_to_cpu(response->upiu_res.value); 3570 3571 out_unlock: 3572 ufshcd_dev_man_unlock(hba); 3573 return err; 3574 } 3575 3576 /** 3577 * ufshcd_query_attr_retry() - API function for sending query 3578 * attribute with retries 3579 * @hba: per-adapter instance 3580 * @opcode: attribute opcode 3581 * @idn: attribute idn to access 3582 * @index: index field 3583 * @selector: selector field 3584 * @attr_val: the attribute value after the query request 3585 * completes 3586 * 3587 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3588 * < 0 if another error occurred. 3589 */ 3590 int ufshcd_query_attr_retry(struct ufs_hba *hba, 3591 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, 3592 u32 *attr_val) 3593 { 3594 int ret = 0; 3595 u32 retries; 3596 3597 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3598 ret = ufshcd_query_attr(hba, opcode, idn, index, 3599 selector, attr_val); 3600 if (ret) 3601 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n", 3602 __func__, ret, retries); 3603 else 3604 break; 3605 } 3606 3607 if (ret) 3608 dev_err(hba->dev, 3609 "%s: query attribute, idn %d, failed with error %d after %d retries\n", 3610 __func__, idn, ret, QUERY_REQ_RETRIES); 3611 return ret; 3612 } 3613 3614 /* 3615 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3616 * < 0 if another error occurred. 3617 */ 3618 static int __ufshcd_query_descriptor(struct ufs_hba *hba, 3619 enum query_opcode opcode, enum desc_idn idn, u8 index, 3620 u8 selector, u8 *desc_buf, int *buf_len) 3621 { 3622 struct ufs_query_req *request = NULL; 3623 struct ufs_query_res *response = NULL; 3624 int err; 3625 3626 BUG_ON(!hba); 3627 3628 if (!desc_buf) { 3629 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 3630 __func__, opcode); 3631 return -EINVAL; 3632 } 3633 3634 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 3635 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 3636 __func__, *buf_len); 3637 return -EINVAL; 3638 } 3639 3640 ufshcd_dev_man_lock(hba); 3641 3642 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3643 selector); 3644 hba->dev_cmd.query.descriptor = desc_buf; 3645 request->upiu_req.length = cpu_to_be16(*buf_len); 3646 3647 switch (opcode) { 3648 case UPIU_QUERY_OPCODE_WRITE_DESC: 3649 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3650 break; 3651 case UPIU_QUERY_OPCODE_READ_DESC: 3652 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3653 break; 3654 default: 3655 dev_err(hba->dev, 3656 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 3657 __func__, opcode); 3658 err = -EINVAL; 3659 goto out_unlock; 3660 } 3661 3662 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3663 3664 if (err) { 3665 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3666 __func__, opcode, idn, index, err); 3667 goto out_unlock; 3668 } 3669 3670 *buf_len = be16_to_cpu(response->upiu_res.length); 3671 3672 out_unlock: 3673 hba->dev_cmd.query.descriptor = NULL; 3674 ufshcd_dev_man_unlock(hba); 3675 return err; 3676 } 3677 3678 /** 3679 * ufshcd_query_descriptor_retry - API function for sending descriptor requests 3680 * @hba: per-adapter instance 3681 * @opcode: attribute opcode 3682 * @idn: attribute idn to access 3683 * @index: index field 3684 * @selector: selector field 3685 * @desc_buf: the buffer that contains the descriptor 3686 * @buf_len: length parameter passed to the device 3687 * 3688 * The buf_len parameter will contain, on return, the length parameter 3689 * received on the response. 3690 * 3691 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3692 * < 0 if another error occurred. 3693 */ 3694 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 3695 enum query_opcode opcode, 3696 enum desc_idn idn, u8 index, 3697 u8 selector, 3698 u8 *desc_buf, int *buf_len) 3699 { 3700 int err; 3701 int retries; 3702 3703 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3704 err = __ufshcd_query_descriptor(hba, opcode, idn, index, 3705 selector, desc_buf, buf_len); 3706 if (!err || err == -EINVAL) 3707 break; 3708 } 3709 3710 return err; 3711 } 3712 3713 /** 3714 * ufshcd_read_desc_param - read the specified descriptor parameter 3715 * @hba: Pointer to adapter instance 3716 * @desc_id: descriptor idn value 3717 * @desc_index: descriptor index 3718 * @param_offset: offset of the parameter to read 3719 * @param_read_buf: pointer to buffer where parameter would be read 3720 * @param_size: sizeof(param_read_buf) 3721 * 3722 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3723 * < 0 if another error occurred. 3724 */ 3725 int ufshcd_read_desc_param(struct ufs_hba *hba, 3726 enum desc_idn desc_id, 3727 int desc_index, 3728 u8 param_offset, 3729 u8 *param_read_buf, 3730 u8 param_size) 3731 { 3732 int ret; 3733 u8 *desc_buf; 3734 int buff_len = QUERY_DESC_MAX_SIZE; 3735 bool is_kmalloc = true; 3736 3737 /* Safety check */ 3738 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) 3739 return -EINVAL; 3740 3741 /* Check whether we need temp memory */ 3742 if (param_offset != 0 || param_size < buff_len) { 3743 desc_buf = kzalloc(buff_len, GFP_KERNEL); 3744 if (!desc_buf) 3745 return -ENOMEM; 3746 } else { 3747 desc_buf = param_read_buf; 3748 is_kmalloc = false; 3749 } 3750 3751 /* Request for full descriptor */ 3752 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, 3753 desc_id, desc_index, 0, 3754 desc_buf, &buff_len); 3755 if (ret) { 3756 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", 3757 __func__, desc_id, desc_index, param_offset, ret); 3758 goto out; 3759 } 3760 3761 /* Update descriptor length */ 3762 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; 3763 3764 if (param_offset >= buff_len) { 3765 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", 3766 __func__, param_offset, desc_id, buff_len); 3767 ret = -EINVAL; 3768 goto out; 3769 } 3770 3771 /* Sanity check */ 3772 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { 3773 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", 3774 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); 3775 ret = -EINVAL; 3776 goto out; 3777 } 3778 3779 if (is_kmalloc) { 3780 /* Make sure we don't copy more data than available */ 3781 if (param_offset >= buff_len) 3782 ret = -EINVAL; 3783 else 3784 memcpy(param_read_buf, &desc_buf[param_offset], 3785 min_t(u32, param_size, buff_len - param_offset)); 3786 } 3787 out: 3788 if (is_kmalloc) 3789 kfree(desc_buf); 3790 return ret; 3791 } 3792 3793 /** 3794 * struct uc_string_id - unicode string 3795 * 3796 * @len: size of this descriptor inclusive 3797 * @type: descriptor type 3798 * @uc: unicode string character 3799 */ 3800 struct uc_string_id { 3801 u8 len; 3802 u8 type; 3803 wchar_t uc[]; 3804 } __packed; 3805 3806 /* replace non-printable or non-ASCII characters with spaces */ 3807 static inline char ufshcd_remove_non_printable(u8 ch) 3808 { 3809 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' '; 3810 } 3811 3812 /** 3813 * ufshcd_read_string_desc - read string descriptor 3814 * @hba: pointer to adapter instance 3815 * @desc_index: descriptor index 3816 * @buf: pointer to buffer where descriptor would be read, 3817 * the caller should free the memory. 3818 * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII 3819 * 3820 * Return: 3821 * * string size on success. 3822 * * -ENOMEM: on allocation failure 3823 * * -EINVAL: on a wrong parameter 3824 */ 3825 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt) 3826 { 3827 struct uc_string_id *uc_str; 3828 u8 *str; 3829 int ret; 3830 3831 if (!buf) 3832 return -EINVAL; 3833 3834 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 3835 if (!uc_str) 3836 return -ENOMEM; 3837 3838 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, 3839 (u8 *)uc_str, QUERY_DESC_MAX_SIZE); 3840 if (ret < 0) { 3841 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", 3842 QUERY_REQ_RETRIES, ret); 3843 str = NULL; 3844 goto out; 3845 } 3846 3847 if (uc_str->len <= QUERY_DESC_HDR_SIZE) { 3848 dev_dbg(hba->dev, "String Desc is of zero length\n"); 3849 str = NULL; 3850 ret = 0; 3851 goto out; 3852 } 3853 3854 if (fmt == SD_ASCII_STD) { 3855 ssize_t ascii_len; 3856 int i; 3857 /* remove header and divide by 2 to move from UTF16 to UTF8 */ 3858 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; 3859 str = kzalloc(ascii_len, GFP_KERNEL); 3860 if (!str) { 3861 ret = -ENOMEM; 3862 goto out; 3863 } 3864 3865 /* 3866 * the descriptor contains string in UTF16 format 3867 * we need to convert to utf-8 so it can be displayed 3868 */ 3869 ret = utf16s_to_utf8s(uc_str->uc, 3870 uc_str->len - QUERY_DESC_HDR_SIZE, 3871 UTF16_BIG_ENDIAN, str, ascii_len - 1); 3872 3873 /* replace non-printable or non-ASCII characters with spaces */ 3874 for (i = 0; i < ret; i++) 3875 str[i] = ufshcd_remove_non_printable(str[i]); 3876 3877 str[ret++] = '\0'; 3878 3879 } else { 3880 str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); 3881 if (!str) { 3882 ret = -ENOMEM; 3883 goto out; 3884 } 3885 ret = uc_str->len; 3886 } 3887 out: 3888 *buf = str; 3889 kfree(uc_str); 3890 return ret; 3891 } 3892 3893 /** 3894 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 3895 * @hba: Pointer to adapter instance 3896 * @lun: lun id 3897 * @param_offset: offset of the parameter to read 3898 * @param_read_buf: pointer to buffer where parameter would be read 3899 * @param_size: sizeof(param_read_buf) 3900 * 3901 * Return: 0 in case of success; < 0 upon failure. 3902 */ 3903 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 3904 int lun, 3905 enum unit_desc_param param_offset, 3906 u8 *param_read_buf, 3907 u32 param_size) 3908 { 3909 /* 3910 * Unit descriptors are only available for general purpose LUs (LUN id 3911 * from 0 to 7) and RPMB Well known LU. 3912 */ 3913 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun)) 3914 return -EOPNOTSUPP; 3915 3916 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 3917 param_offset, param_read_buf, param_size); 3918 } 3919 3920 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) 3921 { 3922 int err = 0; 3923 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3924 3925 if (hba->dev_info.wspecversion >= 0x300) { 3926 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3927 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0, 3928 &gating_wait); 3929 if (err) 3930 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n", 3931 err, gating_wait); 3932 3933 if (gating_wait == 0) { 3934 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3935 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n", 3936 gating_wait); 3937 } 3938 3939 hba->dev_info.clk_gating_wait_us = gating_wait; 3940 } 3941 3942 return err; 3943 } 3944 3945 /** 3946 * ufshcd_memory_alloc - allocate memory for host memory space data structures 3947 * @hba: per adapter instance 3948 * 3949 * 1. Allocate DMA memory for Command Descriptor array 3950 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 3951 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 3952 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 3953 * (UTMRDL) 3954 * 4. Allocate memory for local reference block(lrb). 3955 * 3956 * Return: 0 for success, non-zero in case of failure. 3957 */ 3958 static int ufshcd_memory_alloc(struct ufs_hba *hba) 3959 { 3960 size_t utmrdl_size, utrdl_size, ucdl_size; 3961 3962 /* Allocate memory for UTP command descriptors */ 3963 ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs; 3964 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 3965 ucdl_size, 3966 &hba->ucdl_dma_addr, 3967 GFP_KERNEL); 3968 3969 /* 3970 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 3971 */ 3972 if (!hba->ucdl_base_addr || 3973 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { 3974 dev_err(hba->dev, 3975 "Command Descriptor Memory allocation failed\n"); 3976 goto out; 3977 } 3978 3979 /* 3980 * Allocate memory for UTP Transfer descriptors 3981 * UFSHCI requires 1KB alignment of UTRD 3982 */ 3983 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 3984 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 3985 utrdl_size, 3986 &hba->utrdl_dma_addr, 3987 GFP_KERNEL); 3988 if (!hba->utrdl_base_addr || 3989 WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) { 3990 dev_err(hba->dev, 3991 "Transfer Descriptor Memory allocation failed\n"); 3992 goto out; 3993 } 3994 3995 /* 3996 * Skip utmrdl allocation; it may have been 3997 * allocated during first pass and not released during 3998 * MCQ memory allocation. 3999 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() 4000 */ 4001 if (hba->utmrdl_base_addr) 4002 goto skip_utmrdl; 4003 /* 4004 * Allocate memory for UTP Task Management descriptors 4005 * UFSHCI requires 1KB alignment of UTMRD 4006 */ 4007 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 4008 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 4009 utmrdl_size, 4010 &hba->utmrdl_dma_addr, 4011 GFP_KERNEL); 4012 if (!hba->utmrdl_base_addr || 4013 WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) { 4014 dev_err(hba->dev, 4015 "Task Management Descriptor Memory allocation failed\n"); 4016 goto out; 4017 } 4018 4019 skip_utmrdl: 4020 return 0; 4021 out: 4022 return -ENOMEM; 4023 } 4024 4025 /** 4026 * ufshcd_host_memory_configure - configure local reference block with 4027 * memory offsets 4028 * @hba: per adapter instance 4029 * 4030 * Configure Host memory space 4031 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 4032 * address. 4033 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 4034 * and PRDT offset. 4035 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 4036 * into local reference block. 4037 */ 4038 static void ufshcd_host_memory_configure(struct ufs_hba *hba) 4039 { 4040 struct utp_transfer_req_desc *utrdlp; 4041 dma_addr_t cmd_desc_dma_addr; 4042 dma_addr_t cmd_desc_element_addr; 4043 u16 response_offset; 4044 u16 prdt_offset; 4045 int cmd_desc_size; 4046 int i; 4047 4048 utrdlp = hba->utrdl_base_addr; 4049 4050 response_offset = 4051 offsetof(struct utp_transfer_cmd_desc, response_upiu); 4052 prdt_offset = 4053 offsetof(struct utp_transfer_cmd_desc, prd_table); 4054 4055 cmd_desc_size = ufshcd_get_ucd_size(hba); 4056 cmd_desc_dma_addr = hba->ucdl_dma_addr; 4057 4058 for (i = 0; i < hba->nutrs; i++) { 4059 /* Configure UTRD with command descriptor base address */ 4060 cmd_desc_element_addr = 4061 (cmd_desc_dma_addr + (cmd_desc_size * i)); 4062 utrdlp[i].command_desc_base_addr = 4063 cpu_to_le64(cmd_desc_element_addr); 4064 4065 /* Response upiu and prdt offset should be in double words */ 4066 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { 4067 utrdlp[i].response_upiu_offset = 4068 cpu_to_le16(response_offset); 4069 utrdlp[i].prd_table_offset = 4070 cpu_to_le16(prdt_offset); 4071 utrdlp[i].response_upiu_length = 4072 cpu_to_le16(ALIGNED_UPIU_SIZE); 4073 } else { 4074 utrdlp[i].response_upiu_offset = 4075 cpu_to_le16(response_offset >> 2); 4076 utrdlp[i].prd_table_offset = 4077 cpu_to_le16(prdt_offset >> 2); 4078 utrdlp[i].response_upiu_length = 4079 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 4080 } 4081 } 4082 } 4083 4084 /** 4085 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 4086 * @hba: per adapter instance 4087 * 4088 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 4089 * in order to initialize the Unipro link startup procedure. 4090 * Once the Unipro links are up, the device connected to the controller 4091 * is detected. 4092 * 4093 * Return: 0 on success, non-zero value on failure. 4094 */ 4095 static int ufshcd_dme_link_startup(struct ufs_hba *hba) 4096 { 4097 struct uic_command uic_cmd = { 4098 .command = UIC_CMD_DME_LINK_STARTUP, 4099 }; 4100 int ret; 4101 4102 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4103 if (ret) 4104 dev_dbg(hba->dev, 4105 "dme-link-startup: error code %d\n", ret); 4106 return ret; 4107 } 4108 /** 4109 * ufshcd_dme_reset - UIC command for DME_RESET 4110 * @hba: per adapter instance 4111 * 4112 * DME_RESET command is issued in order to reset UniPro stack. 4113 * This function now deals with cold reset. 4114 * 4115 * Return: 0 on success, non-zero value on failure. 4116 */ 4117 int ufshcd_dme_reset(struct ufs_hba *hba) 4118 { 4119 struct uic_command uic_cmd = { 4120 .command = UIC_CMD_DME_RESET, 4121 }; 4122 int ret; 4123 4124 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4125 if (ret) 4126 dev_err(hba->dev, 4127 "dme-reset: error code %d\n", ret); 4128 4129 return ret; 4130 } 4131 EXPORT_SYMBOL_GPL(ufshcd_dme_reset); 4132 4133 int ufshcd_dme_configure_adapt(struct ufs_hba *hba, 4134 int agreed_gear, 4135 int adapt_val) 4136 { 4137 int ret; 4138 4139 if (agreed_gear < UFS_HS_G4) 4140 adapt_val = PA_NO_ADAPT; 4141 4142 ret = ufshcd_dme_set(hba, 4143 UIC_ARG_MIB(PA_TXHSADAPTTYPE), 4144 adapt_val); 4145 return ret; 4146 } 4147 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); 4148 4149 /** 4150 * ufshcd_dme_enable - UIC command for DME_ENABLE 4151 * @hba: per adapter instance 4152 * 4153 * DME_ENABLE command is issued in order to enable UniPro stack. 4154 * 4155 * Return: 0 on success, non-zero value on failure. 4156 */ 4157 int ufshcd_dme_enable(struct ufs_hba *hba) 4158 { 4159 struct uic_command uic_cmd = { 4160 .command = UIC_CMD_DME_ENABLE, 4161 }; 4162 int ret; 4163 4164 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4165 if (ret) 4166 dev_err(hba->dev, 4167 "dme-enable: error code %d\n", ret); 4168 4169 return ret; 4170 } 4171 EXPORT_SYMBOL_GPL(ufshcd_dme_enable); 4172 4173 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 4174 { 4175 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 4176 unsigned long min_sleep_time_us; 4177 4178 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 4179 return; 4180 4181 /* 4182 * last_dme_cmd_tstamp will be 0 only for 1st call to 4183 * this function 4184 */ 4185 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 4186 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 4187 } else { 4188 unsigned long delta = 4189 (unsigned long) ktime_to_us( 4190 ktime_sub(ktime_get(), 4191 hba->last_dme_cmd_tstamp)); 4192 4193 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 4194 min_sleep_time_us = 4195 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 4196 else 4197 min_sleep_time_us = 0; /* no more delay required */ 4198 } 4199 4200 if (min_sleep_time_us > 0) { 4201 /* allow sleep for extra 50us if needed */ 4202 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 4203 } 4204 4205 /* update the last_dme_cmd_tstamp */ 4206 hba->last_dme_cmd_tstamp = ktime_get(); 4207 } 4208 4209 /** 4210 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 4211 * @hba: per adapter instance 4212 * @attr_sel: uic command argument1 4213 * @attr_set: attribute set type as uic command argument2 4214 * @mib_val: setting value as uic command argument3 4215 * @peer: indicate whether peer or local 4216 * 4217 * Return: 0 on success, non-zero value on failure. 4218 */ 4219 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 4220 u8 attr_set, u32 mib_val, u8 peer) 4221 { 4222 struct uic_command uic_cmd = { 4223 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, 4224 .argument1 = attr_sel, 4225 .argument2 = UIC_ARG_ATTR_TYPE(attr_set), 4226 .argument3 = mib_val, 4227 }; 4228 static const char *const action[] = { 4229 "dme-set", 4230 "dme-peer-set" 4231 }; 4232 const char *set = action[!!peer]; 4233 int ret; 4234 int retries = UFS_UIC_COMMAND_RETRIES; 4235 4236 do { 4237 /* for peer attributes we retry upon failure */ 4238 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4239 if (ret) 4240 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 4241 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 4242 } while (ret && peer && --retries); 4243 4244 if (ret) 4245 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", 4246 set, UIC_GET_ATTR_ID(attr_sel), mib_val, 4247 UFS_UIC_COMMAND_RETRIES - retries); 4248 4249 return ret; 4250 } 4251 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 4252 4253 /** 4254 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 4255 * @hba: per adapter instance 4256 * @attr_sel: uic command argument1 4257 * @mib_val: the value of the attribute as returned by the UIC command 4258 * @peer: indicate whether peer or local 4259 * 4260 * Return: 0 on success, non-zero value on failure. 4261 */ 4262 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 4263 u32 *mib_val, u8 peer) 4264 { 4265 struct uic_command uic_cmd = { 4266 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, 4267 .argument1 = attr_sel, 4268 }; 4269 static const char *const action[] = { 4270 "dme-get", 4271 "dme-peer-get" 4272 }; 4273 const char *get = action[!!peer]; 4274 int ret; 4275 int retries = UFS_UIC_COMMAND_RETRIES; 4276 struct ufs_pa_layer_attr orig_pwr_info; 4277 struct ufs_pa_layer_attr temp_pwr_info; 4278 bool pwr_mode_change = false; 4279 4280 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) { 4281 orig_pwr_info = hba->pwr_info; 4282 temp_pwr_info = orig_pwr_info; 4283 4284 if (orig_pwr_info.pwr_tx == FAST_MODE || 4285 orig_pwr_info.pwr_rx == FAST_MODE) { 4286 temp_pwr_info.pwr_tx = FASTAUTO_MODE; 4287 temp_pwr_info.pwr_rx = FASTAUTO_MODE; 4288 pwr_mode_change = true; 4289 } else if (orig_pwr_info.pwr_tx == SLOW_MODE || 4290 orig_pwr_info.pwr_rx == SLOW_MODE) { 4291 temp_pwr_info.pwr_tx = SLOWAUTO_MODE; 4292 temp_pwr_info.pwr_rx = SLOWAUTO_MODE; 4293 pwr_mode_change = true; 4294 } 4295 if (pwr_mode_change) { 4296 ret = ufshcd_change_power_mode(hba, &temp_pwr_info, 4297 UFSHCD_PMC_POLICY_DONT_FORCE); 4298 if (ret) 4299 goto out; 4300 } 4301 } 4302 4303 do { 4304 /* for peer attributes we retry upon failure */ 4305 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4306 if (ret) 4307 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", 4308 get, UIC_GET_ATTR_ID(attr_sel), ret); 4309 } while (ret && peer && --retries); 4310 4311 if (ret) 4312 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", 4313 get, UIC_GET_ATTR_ID(attr_sel), 4314 UFS_UIC_COMMAND_RETRIES - retries); 4315 4316 if (mib_val) 4317 *mib_val = ret == 0 ? uic_cmd.argument3 : 0; 4318 4319 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) 4320 && pwr_mode_change) 4321 ufshcd_change_power_mode(hba, &orig_pwr_info, 4322 UFSHCD_PMC_POLICY_DONT_FORCE); 4323 out: 4324 return ret; 4325 } 4326 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 4327 4328 /** 4329 * ufshcd_dme_rmw - get modify set a DME attribute 4330 * @hba: per adapter instance 4331 * @mask: indicates which bits to clear from the value that has been read 4332 * @val: actual value to write 4333 * @attr: dme attribute 4334 */ 4335 int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, 4336 u32 val, u32 attr) 4337 { 4338 u32 cfg = 0; 4339 int err; 4340 4341 err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); 4342 if (err) 4343 return err; 4344 4345 cfg &= ~mask; 4346 cfg |= (val & mask); 4347 4348 return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); 4349 } 4350 EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); 4351 4352 /** 4353 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 4354 * state) and waits for it to take effect. 4355 * 4356 * @hba: per adapter instance 4357 * @cmd: UIC command to execute 4358 * 4359 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 4360 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 4361 * and device UniPro link and hence it's final completion would be indicated by 4362 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 4363 * addition to normal UIC command completion Status (UCCS). This function only 4364 * returns after the relevant status bits indicate the completion. 4365 * 4366 * Return: 0 on success, non-zero value on failure. 4367 */ 4368 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 4369 { 4370 DECLARE_COMPLETION_ONSTACK(uic_async_done); 4371 unsigned long flags; 4372 u8 status; 4373 int ret; 4374 4375 mutex_lock(&hba->uic_cmd_mutex); 4376 ufshcd_add_delay_before_dme_cmd(hba); 4377 4378 spin_lock_irqsave(hba->host->host_lock, flags); 4379 if (ufshcd_is_link_broken(hba)) { 4380 ret = -ENOLINK; 4381 goto out_unlock; 4382 } 4383 hba->uic_async_done = &uic_async_done; 4384 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); 4385 spin_unlock_irqrestore(hba->host->host_lock, flags); 4386 ret = __ufshcd_send_uic_cmd(hba, cmd); 4387 if (ret) { 4388 dev_err(hba->dev, 4389 "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) uic error %d\n", 4390 cmd->command, UIC_GET_ATTR_ID(cmd->argument1), 4391 cmd->argument3, ret); 4392 goto out; 4393 } 4394 4395 if (!wait_for_completion_timeout(hba->uic_async_done, 4396 msecs_to_jiffies(uic_cmd_timeout))) { 4397 dev_err(hba->dev, 4398 "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) completion timeout\n", 4399 cmd->command, UIC_GET_ATTR_ID(cmd->argument1), 4400 cmd->argument3); 4401 4402 if (!cmd->cmd_active) { 4403 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", 4404 __func__); 4405 goto check_upmcrs; 4406 } 4407 4408 ret = -ETIMEDOUT; 4409 goto out; 4410 } 4411 4412 check_upmcrs: 4413 status = ufshcd_get_upmcrs(hba); 4414 if (status != PWR_LOCAL) { 4415 dev_err(hba->dev, 4416 "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) failed, host upmcrs:0x%x\n", 4417 cmd->command, UIC_GET_ATTR_ID(cmd->argument1), 4418 cmd->argument3, status); 4419 ret = (status != PWR_OK) ? status : -1; 4420 } 4421 out: 4422 if (ret) { 4423 ufshcd_print_host_state(hba); 4424 ufshcd_print_pwr_info(hba); 4425 ufshcd_print_tx_eq_params(hba); 4426 ufshcd_print_evt_hist(hba); 4427 } 4428 4429 spin_lock_irqsave(hba->host->host_lock, flags); 4430 hba->active_uic_cmd = NULL; 4431 hba->uic_async_done = NULL; 4432 if (ret && !hba->pm_op_in_progress) { 4433 ufshcd_set_link_broken(hba); 4434 ufshcd_schedule_eh_work(hba); 4435 } 4436 out_unlock: 4437 spin_unlock_irqrestore(hba->host->host_lock, flags); 4438 mutex_unlock(&hba->uic_cmd_mutex); 4439 4440 return ret; 4441 } 4442 4443 /** 4444 * ufshcd_uic_tx_eqtr - Perform UIC TX Equalization Training 4445 * @hba: per adapter instance 4446 * @gear: target gear for EQTR 4447 * 4448 * Returns 0 on success, negative error code otherwise 4449 */ 4450 int ufshcd_uic_tx_eqtr(struct ufs_hba *hba, int gear) 4451 { 4452 struct uic_command uic_cmd = { 4453 .command = UIC_CMD_DME_SET, 4454 .argument1 = UIC_ARG_MIB(PA_EQTR_GEAR), 4455 .argument3 = gear, 4456 }; 4457 int ret; 4458 4459 ufshcd_hold(hba); 4460 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4461 ufshcd_release(hba); 4462 4463 return ret; 4464 } 4465 4466 /** 4467 * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result 4468 * @hba: per adapter instance 4469 * @uic_cmd: UIC command 4470 * 4471 * Return: 0 only if success. 4472 */ 4473 int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 4474 { 4475 int ret; 4476 4477 if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || 4478 uic_cmd->command != UIC_CMD_DME_SET) 4479 return ufshcd_send_uic_cmd(hba, uic_cmd); 4480 4481 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 4482 return 0; 4483 4484 ufshcd_hold(hba); 4485 ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); 4486 ufshcd_release(hba); 4487 4488 return ret; 4489 } 4490 4491 /** 4492 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 4493 * using DME_SET primitives. 4494 * @hba: per adapter instance 4495 * @mode: powr mode value 4496 * 4497 * Return: 0 on success, non-zero value on failure. 4498 */ 4499 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 4500 { 4501 struct uic_command uic_cmd = { 4502 .command = UIC_CMD_DME_SET, 4503 .argument1 = UIC_ARG_MIB(PA_PWRMODE), 4504 .argument3 = mode, 4505 }; 4506 int ret; 4507 4508 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { 4509 ret = ufshcd_dme_set(hba, 4510 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1); 4511 if (ret) { 4512 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n", 4513 __func__, ret); 4514 goto out; 4515 } 4516 } 4517 4518 ufshcd_hold(hba); 4519 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4520 ufshcd_release(hba); 4521 4522 out: 4523 return ret; 4524 } 4525 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode); 4526 4527 int ufshcd_link_recovery(struct ufs_hba *hba) 4528 { 4529 int ret; 4530 unsigned long flags; 4531 4532 spin_lock_irqsave(hba->host->host_lock, flags); 4533 hba->ufshcd_state = UFSHCD_STATE_RESET; 4534 ufshcd_set_eh_in_progress(hba); 4535 spin_unlock_irqrestore(hba->host->host_lock, flags); 4536 4537 /* Reset the attached device */ 4538 ufshcd_device_reset(hba); 4539 4540 ret = ufshcd_host_reset_and_restore(hba); 4541 4542 spin_lock_irqsave(hba->host->host_lock, flags); 4543 if (ret) 4544 hba->ufshcd_state = UFSHCD_STATE_ERROR; 4545 ufshcd_clear_eh_in_progress(hba); 4546 spin_unlock_irqrestore(hba->host->host_lock, flags); 4547 4548 if (ret) 4549 dev_err(hba->dev, "%s: link recovery failed, err %d", 4550 __func__, ret); 4551 4552 return ret; 4553 } 4554 EXPORT_SYMBOL_GPL(ufshcd_link_recovery); 4555 4556 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 4557 { 4558 struct uic_command uic_cmd = { 4559 .command = UIC_CMD_DME_HIBER_ENTER, 4560 }; 4561 ktime_t start = ktime_get(); 4562 int ret; 4563 4564 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); 4565 4566 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4567 trace_ufshcd_profile_hibern8(hba, "enter", 4568 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4569 4570 if (ret) 4571 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", 4572 __func__, ret); 4573 else 4574 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, 4575 POST_CHANGE); 4576 4577 return ret; 4578 } 4579 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); 4580 4581 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 4582 { 4583 struct uic_command uic_cmd = { 4584 .command = UIC_CMD_DME_HIBER_EXIT, 4585 }; 4586 int ret; 4587 ktime_t start = ktime_get(); 4588 4589 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); 4590 4591 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4592 trace_ufshcd_profile_hibern8(hba, "exit", 4593 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4594 4595 if (ret) { 4596 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", 4597 __func__, ret); 4598 } else { 4599 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, 4600 POST_CHANGE); 4601 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); 4602 hba->ufs_stats.hibern8_exit_cnt++; 4603 } 4604 4605 return ret; 4606 } 4607 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); 4608 4609 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) 4610 { 4611 if (!ufshcd_is_auto_hibern8_supported(hba)) 4612 return; 4613 4614 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); 4615 } 4616 4617 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) 4618 { 4619 const u32 cur_ahit = READ_ONCE(hba->ahit); 4620 4621 if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) 4622 return; 4623 4624 WRITE_ONCE(hba->ahit, ahit); 4625 if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { 4626 ufshcd_rpm_get_sync(hba); 4627 ufshcd_hold(hba); 4628 ufshcd_configure_auto_hibern8(hba); 4629 ufshcd_release(hba); 4630 ufshcd_rpm_put_sync(hba); 4631 } 4632 } 4633 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); 4634 4635 /** 4636 * ufshcd_init_pwr_info - setting the POR (power on reset) 4637 * values in hba power info 4638 * @hba: per-adapter instance 4639 */ 4640 static void ufshcd_init_pwr_info(struct ufs_hba *hba) 4641 { 4642 hba->pwr_info.gear_rx = UFS_PWM_G1; 4643 hba->pwr_info.gear_tx = UFS_PWM_G1; 4644 hba->pwr_info.lane_rx = UFS_LANE_1; 4645 hba->pwr_info.lane_tx = UFS_LANE_1; 4646 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 4647 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 4648 hba->pwr_info.hs_rate = 0; 4649 } 4650 4651 /** 4652 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 4653 * @hba: per-adapter instance 4654 * 4655 * Return: 0 upon success; < 0 upon failure. 4656 */ 4657 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 4658 { 4659 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 4660 4661 if (hba->max_pwr_info.is_valid) 4662 return 0; 4663 4664 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) { 4665 pwr_info->pwr_tx = FASTAUTO_MODE; 4666 pwr_info->pwr_rx = FASTAUTO_MODE; 4667 } else { 4668 pwr_info->pwr_tx = FAST_MODE; 4669 pwr_info->pwr_rx = FAST_MODE; 4670 } 4671 pwr_info->hs_rate = PA_HS_MODE_B; 4672 4673 /* Get the connected lane count */ 4674 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 4675 &pwr_info->lane_rx); 4676 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4677 &pwr_info->lane_tx); 4678 4679 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 4680 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 4681 __func__, 4682 pwr_info->lane_rx, 4683 pwr_info->lane_tx); 4684 return -EINVAL; 4685 } 4686 4687 if (pwr_info->lane_rx != pwr_info->lane_tx) { 4688 dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", 4689 __func__, 4690 pwr_info->lane_rx, 4691 pwr_info->lane_tx); 4692 return -EINVAL; 4693 } 4694 4695 /* 4696 * First, get the maximum gears of HS speed. 4697 * If a zero value, it means there is no HSGEAR capability. 4698 * Then, get the maximum gears of PWM speed. 4699 */ 4700 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 4701 if (!pwr_info->gear_rx) { 4702 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4703 &pwr_info->gear_rx); 4704 if (!pwr_info->gear_rx) { 4705 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 4706 __func__, pwr_info->gear_rx); 4707 return -EINVAL; 4708 } 4709 pwr_info->pwr_rx = SLOW_MODE; 4710 } 4711 4712 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 4713 &pwr_info->gear_tx); 4714 if (!pwr_info->gear_tx) { 4715 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4716 &pwr_info->gear_tx); 4717 if (!pwr_info->gear_tx) { 4718 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 4719 __func__, pwr_info->gear_tx); 4720 return -EINVAL; 4721 } 4722 pwr_info->pwr_tx = SLOW_MODE; 4723 } 4724 4725 hba->max_pwr_info.is_valid = true; 4726 return 0; 4727 } 4728 4729 /** 4730 * ufshcd_dme_change_power_mode() - UniPro DME Power Mode change sequence 4731 * @hba: per-adapter instance 4732 * @pwr_mode: pointer to the target power mode (gear/lane) attributes 4733 * @pmc_policy: Power Mode change policy 4734 * 4735 * This function handles the low-level DME (Device Management Entity) 4736 * configuration required to transition the UFS link to a new power mode. It 4737 * performs the following steps: 4738 * 1. Checks if the requested mode matches the current state. 4739 * 2. Sets M-PHY and UniPro attributes including Gear (PA_RXGEAR/TXGEAR), 4740 * Lanes, Termination, and HS Series (PA_HSSERIES). 4741 * 3. Configures default UniPro timeout values (DL_FC0, etc.) unless 4742 * explicitly skipped via quirks. 4743 * 4. Triggers the actual hardware mode change via ufshcd_uic_change_pwr_mode(). 4744 * 5. Updates the HBA's cached power information on success. 4745 * 4746 * Return: 0 on success, non-zero error code on failure. 4747 */ 4748 static int ufshcd_dme_change_power_mode(struct ufs_hba *hba, 4749 struct ufs_pa_layer_attr *pwr_mode, 4750 enum ufshcd_pmc_policy pmc_policy) 4751 { 4752 int ret; 4753 4754 /* if already configured to the requested pwr_mode */ 4755 if (pmc_policy == UFSHCD_PMC_POLICY_DONT_FORCE && 4756 pwr_mode->gear_rx == hba->pwr_info.gear_rx && 4757 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 4758 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 4759 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 4760 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 4761 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 4762 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 4763 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 4764 return 0; 4765 } 4766 4767 /* 4768 * Configure attributes for power mode change with below. 4769 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 4770 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 4771 * - PA_HSSERIES 4772 */ 4773 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 4774 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 4775 pwr_mode->lane_rx); 4776 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4777 pwr_mode->pwr_rx == FAST_MODE) 4778 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); 4779 else 4780 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false); 4781 4782 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 4783 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 4784 pwr_mode->lane_tx); 4785 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 4786 pwr_mode->pwr_tx == FAST_MODE) 4787 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); 4788 else 4789 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false); 4790 4791 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4792 pwr_mode->pwr_tx == FASTAUTO_MODE || 4793 pwr_mode->pwr_rx == FAST_MODE || 4794 pwr_mode->pwr_tx == FAST_MODE) 4795 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 4796 pwr_mode->hs_rate); 4797 4798 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { 4799 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 4800 DL_FC0ProtectionTimeOutVal_Default); 4801 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 4802 DL_TC0ReplayTimeOutVal_Default); 4803 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 4804 DL_AFC0ReqTimeOutVal_Default); 4805 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 4806 DL_FC1ProtectionTimeOutVal_Default); 4807 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 4808 DL_TC1ReplayTimeOutVal_Default); 4809 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 4810 DL_AFC1ReqTimeOutVal_Default); 4811 4812 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), 4813 DL_FC0ProtectionTimeOutVal_Default); 4814 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), 4815 DL_TC0ReplayTimeOutVal_Default); 4816 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), 4817 DL_AFC0ReqTimeOutVal_Default); 4818 } 4819 4820 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 4821 | pwr_mode->pwr_tx); 4822 4823 if (ret) { 4824 dev_err(hba->dev, 4825 "%s: power mode change failed %d\n", __func__, ret); 4826 } else { 4827 memcpy(&hba->pwr_info, pwr_mode, 4828 sizeof(struct ufs_pa_layer_attr)); 4829 } 4830 4831 return ret; 4832 } 4833 4834 /** 4835 * ufshcd_change_power_mode() - Change UFS Link Power Mode 4836 * @hba: per-adapter instance 4837 * @pwr_mode: pointer to the target power mode (gear/lane) attributes 4838 * @pmc_policy: Power Mode change policy 4839 * 4840 * This function handles the high-level sequence for changing the UFS link 4841 * power mode. It triggers vendor-specific pre-change notification, 4842 * executes the DME (Device Management Entity) power mode change sequence, 4843 * and, upon success, triggers vendor-specific post-change notification. 4844 * 4845 * Return: 0 on success, non-zero error code on failure. 4846 */ 4847 int ufshcd_change_power_mode(struct ufs_hba *hba, 4848 struct ufs_pa_layer_attr *pwr_mode, 4849 enum ufshcd_pmc_policy pmc_policy) 4850 { 4851 int ret; 4852 4853 ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE, pwr_mode); 4854 4855 ret = ufshcd_dme_change_power_mode(hba, pwr_mode, pmc_policy); 4856 4857 if (!ret) 4858 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, pwr_mode); 4859 4860 return ret; 4861 } 4862 EXPORT_SYMBOL_GPL(ufshcd_change_power_mode); 4863 4864 /** 4865 * ufshcd_config_pwr_mode - configure a new power mode 4866 * @hba: per-adapter instance 4867 * @desired_pwr_mode: desired power configuration 4868 * @pmc_policy: Power Mode change policy 4869 * 4870 * Return: 0 upon success; < 0 upon failure. 4871 */ 4872 int ufshcd_config_pwr_mode(struct ufs_hba *hba, 4873 struct ufs_pa_layer_attr *desired_pwr_mode, 4874 enum ufshcd_pmc_policy pmc_policy) 4875 { 4876 struct ufs_pa_layer_attr final_params = { 0 }; 4877 int ret; 4878 4879 ret = ufshcd_vops_negotiate_pwr_mode(hba, desired_pwr_mode, 4880 &final_params); 4881 if (ret) { 4882 if (ret != -ENOTSUPP) 4883 dev_err(hba->dev, "Failed to negotiate power mode: %d, use desired as is\n", 4884 ret); 4885 4886 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 4887 } 4888 4889 ret = ufshcd_config_tx_eq_settings(hba, &final_params, false); 4890 if (ret) 4891 dev_warn(hba->dev, "Failed to configure TX Equalization for HS-G%u, Rate-%s: %d\n", 4892 final_params.gear_tx, 4893 ufs_hs_rate_to_str(final_params.hs_rate), ret); 4894 4895 return ufshcd_change_power_mode(hba, &final_params, pmc_policy); 4896 } 4897 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); 4898 4899 /** 4900 * ufshcd_complete_dev_init() - checks device readiness 4901 * @hba: per-adapter instance 4902 * 4903 * Set fDeviceInit flag and poll until device toggles it. 4904 * 4905 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 4906 * < 0 if another error occurred. 4907 */ 4908 static int ufshcd_complete_dev_init(struct ufs_hba *hba) 4909 { 4910 int err; 4911 bool flag_res = true; 4912 ktime_t timeout; 4913 4914 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 4915 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL); 4916 if (err) { 4917 dev_err(hba->dev, 4918 "%s: setting fDeviceInit flag failed with error %d\n", 4919 __func__, err); 4920 goto out; 4921 } 4922 4923 /* Poll fDeviceInit flag to be cleared */ 4924 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT); 4925 do { 4926 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4927 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res); 4928 if (!flag_res) 4929 break; 4930 usleep_range(500, 1000); 4931 } while (ktime_before(ktime_get(), timeout)); 4932 4933 if (err) { 4934 dev_err(hba->dev, 4935 "%s: reading fDeviceInit flag failed with error %d\n", 4936 __func__, err); 4937 } else if (flag_res) { 4938 dev_err(hba->dev, 4939 "%s: fDeviceInit was not cleared by the device\n", 4940 __func__); 4941 err = -EBUSY; 4942 } 4943 out: 4944 return err; 4945 } 4946 4947 /** 4948 * ufshcd_make_hba_operational - Make UFS controller operational 4949 * @hba: per adapter instance 4950 * 4951 * To bring UFS host controller to operational state, 4952 * 1. Enable required interrupts 4953 * 2. Configure interrupt aggregation 4954 * 3. Program UTRL and UTMRL base address 4955 * 4. Configure run-stop-registers 4956 * 4957 * Return: 0 if successful; < 0 upon failure. 4958 */ 4959 int ufshcd_make_hba_operational(struct ufs_hba *hba) 4960 { 4961 int err = 0; 4962 u32 reg; 4963 4964 /* Enable required interrupts */ 4965 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 4966 4967 /* Configure interrupt aggregation */ 4968 if (ufshcd_is_intr_aggr_allowed(hba)) 4969 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 4970 else 4971 ufshcd_disable_intr_aggr(hba); 4972 4973 /* Configure UTRL and UTMRL base address registers */ 4974 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 4975 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 4976 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 4977 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 4978 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 4979 REG_UTP_TASK_REQ_LIST_BASE_L); 4980 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 4981 REG_UTP_TASK_REQ_LIST_BASE_H); 4982 4983 /* 4984 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 4985 */ 4986 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 4987 if (!(ufshcd_get_lists_status(reg))) { 4988 ufshcd_enable_run_stop_reg(hba); 4989 } else { 4990 dev_err(hba->dev, 4991 "Host controller not ready to process requests"); 4992 err = -EIO; 4993 } 4994 4995 return err; 4996 } 4997 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); 4998 4999 /** 5000 * ufshcd_hba_stop - Send controller to reset state 5001 * @hba: per adapter instance 5002 */ 5003 void ufshcd_hba_stop(struct ufs_hba *hba) 5004 { 5005 int err; 5006 5007 ufshcd_disable_irq(hba); 5008 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); 5009 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, 5010 CONTROLLER_ENABLE, CONTROLLER_DISABLE, 5011 10, 1); 5012 ufshcd_enable_irq(hba); 5013 if (err) 5014 dev_err(hba->dev, "%s: Controller disable failed\n", __func__); 5015 } 5016 EXPORT_SYMBOL_GPL(ufshcd_hba_stop); 5017 5018 /** 5019 * ufshcd_hba_execute_hce - initialize the controller 5020 * @hba: per adapter instance 5021 * 5022 * The controller resets itself and controller firmware initialization 5023 * sequence kicks off. When controller is ready it will set 5024 * the Host Controller Enable bit to 1. 5025 * 5026 * Return: 0 on success, non-zero value on failure. 5027 */ 5028 static int ufshcd_hba_execute_hce(struct ufs_hba *hba) 5029 { 5030 int retry; 5031 5032 for (retry = 3; retry > 0; retry--) { 5033 if (ufshcd_is_hba_active(hba)) 5034 /* change controller state to "reset state" */ 5035 ufshcd_hba_stop(hba); 5036 5037 /* UniPro link is disabled at this point */ 5038 ufshcd_set_link_off(hba); 5039 5040 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 5041 5042 /* start controller initialization sequence */ 5043 ufshcd_hba_start(hba); 5044 5045 /* 5046 * To initialize a UFS host controller HCE bit must be set to 1. 5047 * During initialization the HCE bit value changes from 1->0->1. 5048 * When the host controller completes initialization sequence 5049 * it sets the value of HCE bit to 1. The same HCE bit is read back 5050 * to check if the controller has completed initialization sequence. 5051 * So without this delay the value HCE = 1, set in the previous 5052 * instruction might be read back. 5053 * This delay can be changed based on the controller. 5054 */ 5055 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); 5056 5057 /* wait for the host controller to complete initialization */ 5058 if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, 5059 CONTROLLER_ENABLE, 1000, 50)) 5060 break; 5061 5062 dev_err(hba->dev, "Enabling the controller failed\n"); 5063 } 5064 5065 if (!retry) 5066 return -EIO; 5067 5068 /* enable UIC related interrupts */ 5069 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 5070 5071 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 5072 5073 return 0; 5074 } 5075 5076 int ufshcd_hba_enable(struct ufs_hba *hba) 5077 { 5078 int ret; 5079 5080 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { 5081 ufshcd_set_link_off(hba); 5082 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 5083 5084 /* enable UIC related interrupts */ 5085 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 5086 ret = ufshcd_dme_reset(hba); 5087 if (ret) { 5088 dev_err(hba->dev, "DME_RESET failed\n"); 5089 return ret; 5090 } 5091 5092 ret = ufshcd_dme_enable(hba); 5093 if (ret) { 5094 dev_err(hba->dev, "Enabling DME failed\n"); 5095 return ret; 5096 } 5097 5098 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 5099 } else { 5100 ret = ufshcd_hba_execute_hce(hba); 5101 } 5102 5103 return ret; 5104 } 5105 EXPORT_SYMBOL_GPL(ufshcd_hba_enable); 5106 5107 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) 5108 { 5109 int tx_lanes, i, err = 0; 5110 5111 if (!peer) 5112 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 5113 &tx_lanes); 5114 else 5115 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 5116 &tx_lanes); 5117 for (i = 0; i < tx_lanes; i++) { 5118 if (!peer) 5119 err = ufshcd_dme_set(hba, 5120 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 5121 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5122 0); 5123 else 5124 err = ufshcd_dme_peer_set(hba, 5125 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 5126 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5127 0); 5128 if (err) { 5129 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", 5130 __func__, peer, i, err); 5131 break; 5132 } 5133 } 5134 5135 return err; 5136 } 5137 5138 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) 5139 { 5140 return ufshcd_disable_tx_lcc(hba, true); 5141 } 5142 5143 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) 5144 { 5145 struct ufs_event_hist *e; 5146 5147 if (id >= UFS_EVT_CNT) 5148 return; 5149 5150 e = &hba->ufs_stats.event[id]; 5151 e->val[e->pos] = val; 5152 e->tstamp[e->pos] = local_clock(); 5153 e->cnt += 1; 5154 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; 5155 5156 ufshcd_vops_event_notify(hba, id, &val); 5157 } 5158 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); 5159 5160 /** 5161 * ufshcd_link_startup - Initialize unipro link startup 5162 * @hba: per adapter instance 5163 * 5164 * Return: 0 for success, non-zero in case of failure. 5165 */ 5166 static int ufshcd_link_startup(struct ufs_hba *hba) 5167 { 5168 int ret; 5169 int retries = DME_LINKSTARTUP_RETRIES; 5170 bool link_startup_again = false; 5171 5172 /* 5173 * If UFS device isn't active then we will have to issue link startup 5174 * 2 times to make sure the device state move to active. 5175 */ 5176 if (!(hba->quirks & UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE) && 5177 !ufshcd_is_ufs_dev_active(hba)) 5178 link_startup_again = true; 5179 5180 link_startup: 5181 do { 5182 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); 5183 5184 ret = ufshcd_dme_link_startup(hba); 5185 5186 /* check if device is detected by inter-connect layer */ 5187 if (!ret && !ufshcd_is_device_present(hba)) { 5188 ufshcd_update_evt_hist(hba, 5189 UFS_EVT_LINK_STARTUP_FAIL, 5190 0); 5191 dev_err(hba->dev, "%s: Device not present\n", __func__); 5192 ret = -ENXIO; 5193 goto out; 5194 } 5195 5196 /* 5197 * DME link lost indication is only received when link is up, 5198 * but we can't be sure if the link is up until link startup 5199 * succeeds. So reset the local Uni-Pro and try again. 5200 */ 5201 if (ret && retries && ufshcd_hba_enable(hba)) { 5202 ufshcd_update_evt_hist(hba, 5203 UFS_EVT_LINK_STARTUP_FAIL, 5204 (u32)ret); 5205 goto out; 5206 } 5207 } while (ret && retries--); 5208 5209 if (ret) { 5210 /* failed to get the link up... retire */ 5211 ufshcd_update_evt_hist(hba, 5212 UFS_EVT_LINK_STARTUP_FAIL, 5213 (u32)ret); 5214 goto out; 5215 } 5216 5217 if (link_startup_again) { 5218 link_startup_again = false; 5219 retries = DME_LINKSTARTUP_RETRIES; 5220 goto link_startup; 5221 } 5222 5223 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ 5224 ufshcd_init_pwr_info(hba); 5225 ufshcd_print_pwr_info(hba); 5226 5227 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { 5228 ret = ufshcd_disable_device_tx_lcc(hba); 5229 if (ret) 5230 goto out; 5231 } 5232 5233 /* Include any host controller configuration via UIC commands */ 5234 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE); 5235 if (ret) 5236 goto out; 5237 5238 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */ 5239 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 5240 ret = ufshcd_make_hba_operational(hba); 5241 out: 5242 if (ret) 5243 dev_err(hba->dev, "link startup failed %d\n", ret); 5244 return ret; 5245 } 5246 5247 /** 5248 * ufshcd_verify_dev_init() - Verify device initialization 5249 * @hba: per-adapter instance 5250 * 5251 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 5252 * device Transport Protocol (UTP) layer is ready after a reset. 5253 * If the UTP layer at the device side is not initialized, it may 5254 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 5255 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 5256 * 5257 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5258 * < 0 if another error occurred. 5259 */ 5260 static int ufshcd_verify_dev_init(struct ufs_hba *hba) 5261 { 5262 int err = 0; 5263 int retries; 5264 5265 ufshcd_dev_man_lock(hba); 5266 5267 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 5268 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 5269 hba->nop_out_timeout); 5270 5271 if (!err || err == -ETIMEDOUT) 5272 break; 5273 5274 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 5275 } 5276 5277 ufshcd_dev_man_unlock(hba); 5278 5279 if (err) 5280 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 5281 return err; 5282 } 5283 5284 /** 5285 * ufshcd_setup_links - associate link b/w device wlun and other luns 5286 * @sdev: pointer to SCSI device 5287 * @hba: pointer to ufs hba 5288 */ 5289 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) 5290 { 5291 struct device_link *link; 5292 5293 /* 5294 * Device wlun is the supplier & rest of the luns are consumers. 5295 * This ensures that device wlun suspends after all other luns. 5296 */ 5297 if (hba->ufs_device_wlun) { 5298 link = device_link_add(&sdev->sdev_gendev, 5299 &hba->ufs_device_wlun->sdev_gendev, 5300 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); 5301 if (!link) { 5302 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n", 5303 dev_name(&hba->ufs_device_wlun->sdev_gendev)); 5304 return; 5305 } 5306 hba->luns_avail--; 5307 /* Ignore REPORT_LUN wlun probing */ 5308 if (hba->luns_avail == 1) { 5309 ufshcd_rpm_put(hba); 5310 return; 5311 } 5312 } else { 5313 /* 5314 * Device wlun is probed. The assumption is that WLUNs are 5315 * scanned before other LUNs. 5316 */ 5317 hba->luns_avail--; 5318 } 5319 } 5320 5321 /** 5322 * ufshcd_lu_init - Initialize the relevant parameters of the LU 5323 * @hba: per-adapter instance 5324 * @sdev: pointer to SCSI device 5325 */ 5326 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) 5327 { 5328 int len = QUERY_DESC_MAX_SIZE; 5329 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); 5330 u8 lun_qdepth = hba->nutrs; 5331 u8 *desc_buf; 5332 int ret; 5333 5334 desc_buf = kzalloc(len, GFP_KERNEL); 5335 if (!desc_buf) 5336 goto set_qdepth; 5337 5338 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len); 5339 if (ret < 0) { 5340 if (ret == -EOPNOTSUPP) 5341 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */ 5342 lun_qdepth = 1; 5343 kfree(desc_buf); 5344 goto set_qdepth; 5345 } 5346 5347 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) { 5348 /* 5349 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will 5350 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth 5351 */ 5352 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs); 5353 } 5354 /* 5355 * According to UFS device specification, the write protection mode is only supported by 5356 * normal LU, not supported by WLUN. 5357 */ 5358 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported && 5359 !hba->dev_info.is_lu_power_on_wp && 5360 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) 5361 hba->dev_info.is_lu_power_on_wp = true; 5362 5363 /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */ 5364 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) { 5365 if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) 5366 hba->dev_info.b_advanced_rpmb_en = true; 5367 hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE]; 5368 hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; 5369 hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; 5370 hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; 5371 5372 if (hba->dev_info.wspecversion <= 0x0220) { 5373 /* 5374 * These older spec chips have only one RPMB region, 5375 * sized between 128 kB minimum and 16 MB maximum. 5376 * No per region size fields are provided (respective 5377 * REGIONX_SIZE fields always contain zeros), so get 5378 * it from the logical block count and size fields for 5379 * compatibility 5380 * 5381 * (See JESD220C-2_2 Section 14.1.4.6 5382 * RPMB Unit Descriptor,* offset 13h, 4 bytes) 5383 */ 5384 hba->dev_info.rpmb_region_size[0] = 5385 (get_unaligned_be64(desc_buf 5386 + RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_COUNT) 5387 << desc_buf[RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_SIZE]) 5388 / SZ_128K; 5389 } 5390 } 5391 5392 5393 kfree(desc_buf); 5394 set_qdepth: 5395 /* 5396 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose 5397 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue. 5398 */ 5399 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth); 5400 scsi_change_queue_depth(sdev, lun_qdepth); 5401 } 5402 5403 /** 5404 * ufshcd_sdev_init - handle initial SCSI device configurations 5405 * @sdev: pointer to SCSI device 5406 * 5407 * Return: success. 5408 */ 5409 static int ufshcd_sdev_init(struct scsi_device *sdev) 5410 { 5411 struct ufs_hba *hba; 5412 5413 hba = shost_priv(sdev->host); 5414 5415 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 5416 sdev->use_10_for_ms = 1; 5417 5418 /* DBD field should be set to 1 in mode sense(10) */ 5419 sdev->set_dbd_for_ms = 1; 5420 5421 /* allow SCSI layer to restart the device in case of errors */ 5422 sdev->allow_restart = 1; 5423 5424 /* REPORT SUPPORTED OPERATION CODES is not supported */ 5425 sdev->no_report_opcodes = 1; 5426 5427 /* WRITE_SAME command is not supported */ 5428 sdev->no_write_same = 1; 5429 5430 ufshcd_lu_init(hba, sdev); 5431 5432 ufshcd_setup_links(hba, sdev); 5433 5434 return 0; 5435 } 5436 5437 /** 5438 * ufshcd_change_queue_depth - change queue depth 5439 * @sdev: pointer to SCSI device 5440 * @depth: required depth to set 5441 * 5442 * Change queue depth and make sure the max. limits are not crossed. 5443 * 5444 * Return: new queue depth. 5445 */ 5446 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 5447 { 5448 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); 5449 } 5450 5451 /** 5452 * ufshcd_sdev_configure - adjust SCSI device configurations 5453 * @sdev: pointer to SCSI device 5454 * @lim: queue limits 5455 * 5456 * Return: 0 (success). 5457 */ 5458 static int ufshcd_sdev_configure(struct scsi_device *sdev, 5459 struct queue_limits *lim) 5460 { 5461 struct ufs_hba *hba = shost_priv(sdev->host); 5462 struct request_queue *q = sdev->request_queue; 5463 5464 lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; 5465 5466 /* 5467 * Block runtime-pm until all consumers are added. 5468 * Refer ufshcd_setup_links(). 5469 */ 5470 if (is_device_wlun(sdev)) 5471 pm_runtime_get_noresume(&sdev->sdev_gendev); 5472 else if (ufshcd_is_rpm_autosuspend_allowed(hba)) 5473 sdev->rpm_autosuspend = 1; 5474 /* 5475 * Do not print messages during runtime PM to avoid never-ending cycles 5476 * of messages written back to storage by user space causing runtime 5477 * resume, causing more messages and so on. 5478 */ 5479 sdev->silence_suspend = 1; 5480 5481 if (hba->vops && hba->vops->config_scsi_dev) 5482 hba->vops->config_scsi_dev(sdev); 5483 5484 ufshcd_crypto_register(hba, q); 5485 5486 return 0; 5487 } 5488 5489 /** 5490 * ufshcd_sdev_destroy - remove SCSI device configurations 5491 * @sdev: pointer to SCSI device 5492 */ 5493 static void ufshcd_sdev_destroy(struct scsi_device *sdev) 5494 { 5495 struct ufs_hba *hba; 5496 unsigned long flags; 5497 5498 hba = shost_priv(sdev->host); 5499 5500 /* Drop the reference as it won't be needed anymore */ 5501 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 5502 spin_lock_irqsave(hba->host->host_lock, flags); 5503 hba->ufs_device_wlun = NULL; 5504 spin_unlock_irqrestore(hba->host->host_lock, flags); 5505 } else if (hba->ufs_device_wlun) { 5506 struct device *supplier = NULL; 5507 5508 /* Ensure UFS Device WLUN exists and does not disappear */ 5509 spin_lock_irqsave(hba->host->host_lock, flags); 5510 if (hba->ufs_device_wlun) { 5511 supplier = &hba->ufs_device_wlun->sdev_gendev; 5512 get_device(supplier); 5513 } 5514 spin_unlock_irqrestore(hba->host->host_lock, flags); 5515 5516 if (supplier) { 5517 /* 5518 * If a LUN fails to probe (e.g. absent BOOT WLUN), the 5519 * device will not have been registered but can still 5520 * have a device link holding a reference to the device. 5521 */ 5522 device_link_remove(&sdev->sdev_gendev, supplier); 5523 put_device(supplier); 5524 } 5525 } 5526 } 5527 5528 /** 5529 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5530 * @cmd: SCSI command 5531 * @scsi_status: SCSI command status 5532 * 5533 * Return: value base on SCSI command status. 5534 */ 5535 static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) 5536 { 5537 int result = 0; 5538 5539 switch (scsi_status) { 5540 case SAM_STAT_CHECK_CONDITION: 5541 ufshcd_copy_sense_data(cmd); 5542 fallthrough; 5543 case SAM_STAT_GOOD: 5544 result |= DID_OK << 16 | scsi_status; 5545 break; 5546 case SAM_STAT_TASK_SET_FULL: 5547 case SAM_STAT_BUSY: 5548 case SAM_STAT_TASK_ABORTED: 5549 ufshcd_copy_sense_data(cmd); 5550 result |= scsi_status; 5551 break; 5552 default: 5553 result |= DID_ERROR << 16; 5554 break; 5555 } /* end of switch */ 5556 5557 return result; 5558 } 5559 5560 /** 5561 * ufshcd_transfer_rsp_status - Get overall status of the response 5562 * @hba: per adapter instance 5563 * @cmd: SCSI command 5564 * @cqe: pointer to the completion queue entry 5565 * 5566 * Return: result of the command to notify SCSI midlayer. 5567 */ 5568 static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, 5569 struct scsi_cmnd *cmd, 5570 struct cq_entry *cqe) 5571 { 5572 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5573 const int tag = scsi_cmd_to_rq(cmd)->tag; 5574 int result = 0; 5575 int scsi_status; 5576 enum utp_ocs ocs; 5577 u8 upiu_flags; 5578 u32 resid; 5579 5580 upiu_flags = lrbp->ucd_rsp_ptr->header.flags; 5581 resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); 5582 /* 5583 * Test !overflow instead of underflow to support UFS devices that do 5584 * not set either flag. 5585 */ 5586 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5587 scsi_set_resid(cmd, resid); 5588 5589 /* overall command status of utrd */ 5590 ocs = ufshcd_get_tr_ocs(lrbp, cqe); 5591 5592 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { 5593 if (lrbp->ucd_rsp_ptr->header.response || 5594 lrbp->ucd_rsp_ptr->header.status) 5595 ocs = OCS_SUCCESS; 5596 } 5597 5598 switch (ocs) { 5599 case OCS_SUCCESS: 5600 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 5601 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { 5602 case UPIU_TRANSACTION_RESPONSE: 5603 /* 5604 * get the result based on SCSI status response 5605 * to notify the SCSI midlayer of the command status 5606 */ 5607 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5608 result = ufshcd_scsi_cmd_status(cmd, scsi_status); 5609 5610 /* 5611 * Currently we are only supporting BKOPs exception 5612 * events hence we can ignore BKOPs exception event 5613 * during power management callbacks. BKOPs exception 5614 * event is not expected to be raised in runtime suspend 5615 * callback as it allows the urgent bkops. 5616 * During system suspend, we are anyway forcefully 5617 * disabling the bkops and if urgent bkops is needed 5618 * it will be enabled on system resume. Long term 5619 * solution could be to abort the system suspend if 5620 * UFS device needs urgent BKOPs. 5621 */ 5622 if (!hba->pm_op_in_progress && 5623 !ufshcd_eh_in_progress(hba) && 5624 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 5625 /* Flushed in suspend */ 5626 schedule_work(&hba->eeh_work); 5627 break; 5628 case UPIU_TRANSACTION_REJECT_UPIU: 5629 /* TODO: handle Reject UPIU Response */ 5630 result = DID_ERROR << 16; 5631 dev_err(hba->dev, 5632 "Reject UPIU not fully implemented\n"); 5633 break; 5634 default: 5635 dev_err(hba->dev, 5636 "Unexpected request response code = %x\n", 5637 result); 5638 result = DID_ERROR << 16; 5639 break; 5640 } 5641 break; 5642 case OCS_ABORTED: 5643 case OCS_INVALID_COMMAND_STATUS: 5644 result |= DID_REQUEUE << 16; 5645 dev_warn(hba->dev, "OCS %s from controller for tag %d\n", 5646 ocs == OCS_ABORTED ? "aborted" : "invalid", tag); 5647 break; 5648 case OCS_INVALID_CMD_TABLE_ATTR: 5649 case OCS_INVALID_PRDT_ATTR: 5650 case OCS_MISMATCH_DATA_BUF_SIZE: 5651 case OCS_MISMATCH_RESP_UPIU_SIZE: 5652 case OCS_PEER_COMM_FAILURE: 5653 case OCS_FATAL_ERROR: 5654 case OCS_DEVICE_FATAL_ERROR: 5655 case OCS_INVALID_CRYPTO_CONFIG: 5656 case OCS_GENERAL_CRYPTO_ERROR: 5657 default: 5658 result |= DID_ERROR << 16; 5659 dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", 5660 ocs, tag); 5661 ufshcd_print_evt_hist(hba); 5662 ufshcd_print_host_state(hba); 5663 break; 5664 } /* end of switch */ 5665 5666 if ((host_byte(result) != DID_OK) && 5667 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { 5668 if (cqe) 5669 ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); 5670 ufshcd_print_tr(hba, cmd, true); 5671 } 5672 return result; 5673 } 5674 5675 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, 5676 u32 intr_mask) 5677 { 5678 if (!ufshcd_is_auto_hibern8_supported(hba) || 5679 !ufshcd_is_auto_hibern8_enabled(hba)) 5680 return false; 5681 5682 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK)) 5683 return false; 5684 5685 if (hba->active_uic_cmd && 5686 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER || 5687 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT)) 5688 return false; 5689 5690 return true; 5691 } 5692 5693 /** 5694 * ufshcd_uic_cmd_compl - handle completion of uic command 5695 * @hba: per adapter instance 5696 * @intr_status: interrupt status generated by the controller 5697 * 5698 * Return: 5699 * IRQ_HANDLED - If interrupt is valid 5700 * IRQ_NONE - If invalid interrupt 5701 */ 5702 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 5703 { 5704 irqreturn_t retval = IRQ_NONE; 5705 struct uic_command *cmd; 5706 5707 guard(spinlock_irqsave)(hba->host->host_lock); 5708 cmd = hba->active_uic_cmd; 5709 if (!cmd) { 5710 dev_err(hba->dev, 5711 "No active UIC command. Maybe a timeout occurred?\n"); 5712 return retval; 5713 } 5714 5715 if (ufshcd_is_auto_hibern8_error(hba, intr_status)) 5716 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); 5717 5718 if (intr_status & UIC_COMMAND_COMPL) { 5719 cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); 5720 cmd->argument3 = ufshcd_get_dme_attr_val(hba); 5721 if (!hba->uic_async_done) 5722 cmd->cmd_active = false; 5723 complete(&cmd->done); 5724 retval = IRQ_HANDLED; 5725 } 5726 5727 if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { 5728 cmd->cmd_active = false; 5729 complete(hba->uic_async_done); 5730 retval = IRQ_HANDLED; 5731 } 5732 5733 if (retval == IRQ_HANDLED) 5734 ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); 5735 5736 return retval; 5737 } 5738 5739 /* Release the resources allocated for processing a SCSI command. */ 5740 void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) 5741 { 5742 scsi_dma_unmap(cmd); 5743 ufshcd_crypto_clear_prdt(hba, cmd); 5744 ufshcd_release(hba); 5745 ufshcd_clk_scaling_update_busy(hba); 5746 } 5747 5748 /** 5749 * ufshcd_compl_one_cqe - handle a completion queue entry 5750 * @hba: per adapter instance 5751 * @task_tag: the task tag of the request to be completed 5752 * @cqe: pointer to the completion queue entry 5753 */ 5754 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5755 struct cq_entry *cqe) 5756 { 5757 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 5758 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5759 enum utp_ocs ocs; 5760 5761 if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", 5762 le64_to_cpu(cqe->command_desc_base_addr))) 5763 return; 5764 5765 if (hba->monitor.enabled) { 5766 lrbp->compl_time_stamp = ktime_get(); 5767 lrbp->compl_time_stamp_local_clock = local_clock(); 5768 } 5769 if (ufshcd_is_scsi_cmd(cmd)) { 5770 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 5771 ufshcd_update_monitor(hba, cmd); 5772 ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); 5773 cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); 5774 ufshcd_release_scsi_cmd(hba, cmd); 5775 } else { 5776 if (cqe) { 5777 ocs = cqe->overall_status & MASK_OCS; 5778 lrbp->utr_descriptor_ptr->header.ocs = ocs; 5779 } else { 5780 ocs = lrbp->utr_descriptor_ptr->header.ocs; 5781 } 5782 ufshcd_add_query_upiu_trace( 5783 hba, 5784 ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, 5785 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 5786 cmd->result = 0; 5787 } 5788 /* Do not touch lrbp after scsi_done() has been called. */ 5789 scsi_done(cmd); 5790 } 5791 5792 /** 5793 * __ufshcd_transfer_req_compl - handle SCSI and query command completion 5794 * @hba: per adapter instance 5795 * @completed_reqs: bitmask that indicates which requests to complete 5796 */ 5797 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, 5798 unsigned long completed_reqs) 5799 { 5800 int tag; 5801 5802 for_each_set_bit(tag, &completed_reqs, hba->nutrs) 5803 ufshcd_compl_one_cqe(hba, tag, NULL); 5804 } 5805 5806 /* Any value that is not an existing queue number is fine for this constant. */ 5807 enum { 5808 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1 5809 }; 5810 5811 static void ufshcd_clear_polled(struct ufs_hba *hba, 5812 unsigned long *completed_reqs) 5813 { 5814 int tag; 5815 5816 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5817 struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); 5818 5819 if (!cmd) 5820 continue; 5821 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED) 5822 __clear_bit(tag, completed_reqs); 5823 } 5824 } 5825 5826 /* 5827 * Return: > 0 if one or more commands have been completed or 0 if no 5828 * requests have been completed. 5829 */ 5830 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5831 { 5832 struct ufs_hba *hba = shost_priv(shost); 5833 unsigned long completed_reqs, flags; 5834 u32 tr_doorbell; 5835 struct ufs_hw_queue *hwq; 5836 5837 if (hba->mcq_enabled) { 5838 hwq = &hba->uhq[queue_num]; 5839 5840 return ufshcd_mcq_poll_cqe_lock(hba, hwq); 5841 } 5842 5843 spin_lock_irqsave(&hba->outstanding_lock, flags); 5844 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5845 completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5846 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5847 "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5848 hba->outstanding_reqs); 5849 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) { 5850 /* Do not complete polled requests from interrupt context. */ 5851 ufshcd_clear_polled(hba, &completed_reqs); 5852 } 5853 hba->outstanding_reqs &= ~completed_reqs; 5854 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5855 5856 if (completed_reqs) 5857 __ufshcd_transfer_req_compl(hba, completed_reqs); 5858 5859 return completed_reqs != 0; 5860 } 5861 5862 static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) 5863 { 5864 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 5865 struct scsi_device *sdev = rq->q->queuedata; 5866 struct Scsi_Host *shost = sdev->host; 5867 struct ufs_hba *hba = shost_priv(shost); 5868 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5869 5870 if (blk_mq_is_reserved_rq(rq) || !hwq) 5871 return true; 5872 5873 ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5874 5875 /* 5876 * For those cmds of which the cqes are not present in the cq, complete 5877 * them explicitly. 5878 */ 5879 scoped_guard(spinlock_irqsave, &hwq->cq_lock) { 5880 if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5881 set_host_byte(cmd, DID_REQUEUE); 5882 ufshcd_release_scsi_cmd(hba, cmd); 5883 scsi_done(cmd); 5884 } 5885 } 5886 5887 return true; 5888 } 5889 5890 static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) 5891 { 5892 struct scsi_device *sdev = rq->q->queuedata; 5893 struct Scsi_Host *shost = sdev->host; 5894 struct ufs_hba *hba = shost_priv(shost); 5895 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5896 5897 if (!blk_mq_is_reserved_rq(rq) && hwq) 5898 ufshcd_mcq_poll_cqe_lock(hba, hwq); 5899 5900 return true; 5901 } 5902 5903 /** 5904 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is 5905 * invoked from the error handler context or ufshcd_host_reset_and_restore() 5906 * to complete the pending transfers and free the resources associated with 5907 * the scsi command. 5908 * 5909 * @hba: per adapter instance 5910 * @force_compl: This flag is set to true when invoked 5911 * from ufshcd_host_reset_and_restore() in which case it requires special 5912 * handling because the host controller has been reset by ufshcd_hba_stop(). 5913 */ 5914 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, 5915 bool force_compl) 5916 { 5917 blk_mq_tagset_busy_iter(&hba->host->tag_set, 5918 force_compl ? ufshcd_mcq_force_compl_one : 5919 ufshcd_mcq_compl_one, 5920 NULL); 5921 } 5922 5923 /** 5924 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5925 * @hba: per adapter instance 5926 * 5927 * Return: 5928 * IRQ_HANDLED - If interrupt is valid 5929 * IRQ_NONE - If invalid interrupt 5930 */ 5931 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5932 { 5933 /* Resetting interrupt aggregation counters first and reading the 5934 * DOOR_BELL afterward allows us to handle all the completed requests. 5935 * In order to prevent other interrupts starvation the DB is read once 5936 * after reset. The down side of this solution is the possibility of 5937 * false interrupt if device completes another request after resetting 5938 * aggregation and before reading the DB. 5939 */ 5940 if (ufshcd_is_intr_aggr_allowed(hba) && 5941 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) 5942 ufshcd_reset_intr_aggr(hba); 5943 5944 if (ufs_fail_completion(hba)) 5945 return IRQ_HANDLED; 5946 5947 /* 5948 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5949 * do not want polling to trigger spurious interrupt complaints. 5950 */ 5951 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT); 5952 5953 return IRQ_HANDLED; 5954 } 5955 5956 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) 5957 { 5958 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 5959 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, 5960 &ee_ctrl_mask); 5961 } 5962 5963 int ufshcd_write_ee_control(struct ufs_hba *hba) 5964 { 5965 int err; 5966 5967 mutex_lock(&hba->ee_ctrl_mutex); 5968 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask); 5969 mutex_unlock(&hba->ee_ctrl_mutex); 5970 if (err) 5971 dev_err(hba->dev, "%s: failed to write ee control %d\n", 5972 __func__, err); 5973 return err; 5974 } 5975 5976 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 5977 const u16 *other_mask, u16 set, u16 clr) 5978 { 5979 u16 new_mask, ee_ctrl_mask; 5980 int err = 0; 5981 5982 mutex_lock(&hba->ee_ctrl_mutex); 5983 new_mask = (*mask & ~clr) | set; 5984 ee_ctrl_mask = new_mask | *other_mask; 5985 if (ee_ctrl_mask != hba->ee_ctrl_mask) 5986 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask); 5987 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */ 5988 if (!err) { 5989 hba->ee_ctrl_mask = ee_ctrl_mask; 5990 *mask = new_mask; 5991 } 5992 mutex_unlock(&hba->ee_ctrl_mutex); 5993 return err; 5994 } 5995 5996 /** 5997 * ufshcd_disable_ee - disable exception event 5998 * @hba: per-adapter instance 5999 * @mask: exception event to disable 6000 * 6001 * Disables exception event in the device so that the EVENT_ALERT 6002 * bit is not set. 6003 * 6004 * Return: zero on success, non-zero error value on failure. 6005 */ 6006 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 6007 { 6008 return ufshcd_update_ee_drv_mask(hba, 0, mask); 6009 } 6010 6011 /** 6012 * ufshcd_enable_ee - enable exception event 6013 * @hba: per-adapter instance 6014 * @mask: exception event to enable 6015 * 6016 * Enable corresponding exception event in the device to allow 6017 * device to alert host in critical scenarios. 6018 * 6019 * Return: zero on success, non-zero error value on failure. 6020 */ 6021 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 6022 { 6023 return ufshcd_update_ee_drv_mask(hba, mask, 0); 6024 } 6025 6026 /** 6027 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 6028 * @hba: per-adapter instance 6029 * 6030 * Allow device to manage background operations on its own. Enabling 6031 * this might lead to inconsistent latencies during normal data transfers 6032 * as the device is allowed to manage its own way of handling background 6033 * operations. 6034 * 6035 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6036 * < 0 if another error occurred. 6037 */ 6038 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 6039 { 6040 int err = 0; 6041 6042 if (hba->auto_bkops_enabled) 6043 goto out; 6044 6045 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 6046 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 6047 if (err) { 6048 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 6049 __func__, err); 6050 goto out; 6051 } 6052 6053 hba->auto_bkops_enabled = true; 6054 trace_ufshcd_auto_bkops_state(hba, "Enabled"); 6055 6056 /* No need of URGENT_BKOPS exception from the device */ 6057 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 6058 if (err) 6059 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 6060 __func__, err); 6061 out: 6062 return err; 6063 } 6064 6065 /** 6066 * ufshcd_disable_auto_bkops - block device in doing background operations 6067 * @hba: per-adapter instance 6068 * 6069 * Disabling background operations improves command response latency but 6070 * has drawback of device moving into critical state where the device is 6071 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 6072 * host is idle so that BKOPS are managed effectively without any negative 6073 * impacts. 6074 * 6075 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6076 * < 0 if another error occurred. 6077 */ 6078 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 6079 { 6080 int err = 0; 6081 6082 if (!hba->auto_bkops_enabled) 6083 goto out; 6084 6085 /* 6086 * If host assisted BKOPs is to be enabled, make sure 6087 * urgent bkops exception is allowed. 6088 */ 6089 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 6090 if (err) { 6091 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 6092 __func__, err); 6093 goto out; 6094 } 6095 6096 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 6097 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 6098 if (err) { 6099 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 6100 __func__, err); 6101 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 6102 goto out; 6103 } 6104 6105 hba->auto_bkops_enabled = false; 6106 trace_ufshcd_auto_bkops_state(hba, "Disabled"); 6107 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 6108 hba->is_urgent_bkops_lvl_checked = false; 6109 out: 6110 return err; 6111 } 6112 6113 /** 6114 * ufshcd_force_reset_auto_bkops - force reset auto bkops state 6115 * @hba: per adapter instance 6116 * 6117 * After a device reset the device may toggle the BKOPS_EN flag 6118 * to default value. The s/w tracking variables should be updated 6119 * as well. This function would change the auto-bkops state based on 6120 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. 6121 */ 6122 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 6123 { 6124 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { 6125 hba->auto_bkops_enabled = false; 6126 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 6127 ufshcd_enable_auto_bkops(hba); 6128 } else { 6129 hba->auto_bkops_enabled = true; 6130 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; 6131 ufshcd_disable_auto_bkops(hba); 6132 } 6133 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 6134 hba->is_urgent_bkops_lvl_checked = false; 6135 } 6136 6137 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 6138 { 6139 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6140 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 6141 } 6142 6143 /** 6144 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 6145 * @hba: per-adapter instance 6146 * 6147 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 6148 * flag in the device to permit background operations if the device 6149 * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", 6150 * disable otherwise. 6151 * 6152 * Return: 0 for success, non-zero in case of failure. 6153 * 6154 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 6155 * to know whether auto bkops is enabled or disabled after this function 6156 * returns control to it. 6157 */ 6158 static int ufshcd_bkops_ctrl(struct ufs_hba *hba) 6159 { 6160 enum bkops_status status = hba->urgent_bkops_lvl; 6161 u32 curr_status = 0; 6162 int err; 6163 6164 err = ufshcd_get_bkops_status(hba, &curr_status); 6165 if (err) { 6166 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6167 __func__, err); 6168 goto out; 6169 } else if (curr_status > BKOPS_STATUS_MAX) { 6170 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 6171 __func__, curr_status); 6172 err = -EINVAL; 6173 goto out; 6174 } 6175 6176 if (curr_status >= status) 6177 err = ufshcd_enable_auto_bkops(hba); 6178 else 6179 err = ufshcd_disable_auto_bkops(hba); 6180 out: 6181 return err; 6182 } 6183 6184 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 6185 { 6186 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6187 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 6188 } 6189 6190 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) 6191 { 6192 int err; 6193 u32 curr_status = 0; 6194 6195 if (hba->is_urgent_bkops_lvl_checked) 6196 goto enable_auto_bkops; 6197 6198 err = ufshcd_get_bkops_status(hba, &curr_status); 6199 if (err) { 6200 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6201 __func__, err); 6202 goto out; 6203 } 6204 6205 /* 6206 * We are seeing that some devices are raising the urgent bkops 6207 * exception events even when BKOPS status doesn't indicate performace 6208 * impacted or critical. Handle these device by determining their urgent 6209 * bkops status at runtime. 6210 */ 6211 if ((curr_status > BKOPS_STATUS_NO_OP) && (curr_status < BKOPS_STATUS_PERF_IMPACT)) { 6212 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", 6213 __func__, curr_status); 6214 /* update the current status as the urgent bkops level */ 6215 hba->urgent_bkops_lvl = curr_status; 6216 hba->is_urgent_bkops_lvl_checked = true; 6217 } 6218 6219 enable_auto_bkops: 6220 err = ufshcd_enable_auto_bkops(hba); 6221 out: 6222 if (err < 0) 6223 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 6224 __func__, err); 6225 } 6226 6227 /* 6228 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6229 * < 0 if another error occurred. 6230 */ 6231 int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) 6232 { 6233 struct utp_upiu_query_v4_0 *upiu_resp; 6234 struct ufs_query_req *request = NULL; 6235 struct ufs_query_res *response = NULL; 6236 int err; 6237 6238 if (hba->dev_info.wspecversion < 0x410) 6239 return -EOPNOTSUPP; 6240 6241 ufshcd_hold(hba); 6242 mutex_lock(&hba->dev_cmd.lock); 6243 6244 ufshcd_init_query(hba, &request, &response, 6245 UPIU_QUERY_OPCODE_READ_ATTR, 6246 QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); 6247 6248 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 6249 6250 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 6251 6252 if (err) { 6253 dev_err(hba->dev, "%s: failed to read device level exception %d\n", 6254 __func__, err); 6255 goto out; 6256 } 6257 6258 upiu_resp = (struct utp_upiu_query_v4_0 *)response; 6259 *exception_id = get_unaligned_be64(&upiu_resp->osf3); 6260 out: 6261 mutex_unlock(&hba->dev_cmd.lock); 6262 ufshcd_release(hba); 6263 6264 return err; 6265 } 6266 6267 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) 6268 { 6269 u8 index; 6270 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG : 6271 UPIU_QUERY_OPCODE_CLEAR_FLAG; 6272 6273 index = ufshcd_wb_get_query_index(hba); 6274 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL); 6275 } 6276 6277 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) 6278 { 6279 int ret; 6280 6281 if (!ufshcd_is_wb_allowed(hba) || 6282 hba->dev_info.wb_enabled == enable) 6283 return 0; 6284 6285 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); 6286 if (ret) { 6287 dev_err(hba->dev, "%s: Write Booster %s failed %d\n", 6288 __func__, enable ? "enabling" : "disabling", ret); 6289 return ret; 6290 } 6291 6292 hba->dev_info.wb_enabled = enable; 6293 dev_dbg(hba->dev, "%s: Write Booster %s\n", 6294 __func__, enable ? "enabled" : "disabled"); 6295 6296 return ret; 6297 } 6298 6299 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 6300 bool enable) 6301 { 6302 int ret; 6303 6304 ret = __ufshcd_wb_toggle(hba, enable, 6305 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); 6306 if (ret) { 6307 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", 6308 __func__, enable ? "enabling" : "disabling", ret); 6309 return; 6310 } 6311 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", 6312 __func__, enable ? "enabled" : "disabled"); 6313 } 6314 6315 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) 6316 { 6317 int ret; 6318 6319 if (!ufshcd_is_wb_allowed(hba) || 6320 hba->dev_info.wb_buf_flush_enabled == enable) 6321 return 0; 6322 6323 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); 6324 if (ret) { 6325 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", 6326 __func__, enable ? "enabling" : "disabling", ret); 6327 return ret; 6328 } 6329 6330 hba->dev_info.wb_buf_flush_enabled = enable; 6331 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", 6332 __func__, enable ? "enabled" : "disabled"); 6333 6334 return ret; 6335 } 6336 6337 int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) 6338 { 6339 int ret; 6340 u8 index; 6341 6342 index = ufshcd_wb_get_query_index(hba); 6343 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 6344 QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); 6345 if (ret) 6346 dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", 6347 __func__, ret); 6348 6349 return ret; 6350 } 6351 6352 static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, 6353 u32 avail_buf) 6354 { 6355 u32 cur_buf; 6356 int ret; 6357 u8 index; 6358 6359 index = ufshcd_wb_get_query_index(hba); 6360 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6361 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, 6362 index, 0, &cur_buf); 6363 if (ret) { 6364 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", 6365 __func__, ret); 6366 return false; 6367 } 6368 6369 if (!cur_buf) { 6370 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n", 6371 cur_buf); 6372 return false; 6373 } 6374 /* Let it continue to flush when available buffer exceeds threshold */ 6375 return avail_buf < hba->vps->wb_flush_threshold; 6376 } 6377 6378 static void ufshcd_wb_force_disable(struct ufs_hba *hba) 6379 { 6380 if (ufshcd_is_wb_buf_flush_allowed(hba)) 6381 ufshcd_wb_toggle_buf_flush(hba, false); 6382 6383 ufshcd_wb_toggle_buf_flush_during_h8(hba, false); 6384 ufshcd_wb_toggle(hba, false); 6385 hba->caps &= ~UFSHCD_CAP_WB_EN; 6386 6387 dev_info(hba->dev, "%s: WB force disabled\n", __func__); 6388 } 6389 6390 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba) 6391 { 6392 u32 lifetime; 6393 int ret; 6394 u8 index; 6395 6396 index = ufshcd_wb_get_query_index(hba); 6397 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6398 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST, 6399 index, 0, &lifetime); 6400 if (ret) { 6401 dev_err(hba->dev, 6402 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n", 6403 __func__, ret); 6404 return false; 6405 } 6406 6407 if (lifetime == UFS_WB_EXCEED_LIFETIME) { 6408 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n", 6409 __func__, lifetime); 6410 return false; 6411 } 6412 6413 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n", 6414 __func__, lifetime); 6415 6416 return true; 6417 } 6418 6419 static bool ufshcd_wb_need_flush(struct ufs_hba *hba) 6420 { 6421 int ret; 6422 u32 avail_buf; 6423 u8 index; 6424 6425 if (!ufshcd_is_wb_allowed(hba)) 6426 return false; 6427 6428 if (!ufshcd_is_wb_buf_lifetime_available(hba)) { 6429 ufshcd_wb_force_disable(hba); 6430 return false; 6431 } 6432 6433 /* 6434 * With user-space reduction enabled, it's enough to enable flush 6435 * by checking only the available buffer. The threshold 6436 * defined here is > 90% full. 6437 * With user-space preserved enabled, the current-buffer 6438 * should be checked too because the wb buffer size can reduce 6439 * when disk tends to be full. This info is provided by current 6440 * buffer (dCurrentWriteBoosterBufferSize). 6441 */ 6442 index = ufshcd_wb_get_query_index(hba); 6443 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6444 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, 6445 index, 0, &avail_buf); 6446 if (ret) { 6447 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", 6448 __func__, ret); 6449 return false; 6450 } 6451 6452 if (!hba->dev_info.b_presrv_uspc_en) 6453 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); 6454 6455 return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); 6456 } 6457 6458 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) 6459 { 6460 struct ufs_hba *hba = container_of(to_delayed_work(work), 6461 struct ufs_hba, 6462 rpm_dev_flush_recheck_work); 6463 /* 6464 * To prevent unnecessary VCC power drain after device finishes 6465 * WriteBooster buffer flush or Auto BKOPs, force runtime resume 6466 * after a certain delay to recheck the threshold by next runtime 6467 * suspend. 6468 */ 6469 ufshcd_rpm_get_sync(hba); 6470 ufshcd_rpm_put_sync(hba); 6471 } 6472 6473 /** 6474 * ufshcd_exception_event_handler - handle exceptions raised by device 6475 * @work: pointer to work data 6476 * 6477 * Read bExceptionEventStatus attribute from the device and handle the 6478 * exception event accordingly. 6479 */ 6480 static void ufshcd_exception_event_handler(struct work_struct *work) 6481 { 6482 struct ufs_hba *hba; 6483 int err; 6484 u32 status = 0; 6485 hba = container_of(work, struct ufs_hba, eeh_work); 6486 6487 err = ufshcd_get_ee_status(hba, &status); 6488 if (err) { 6489 dev_err(hba->dev, "%s: failed to get exception status %d\n", 6490 __func__, err); 6491 return; 6492 } 6493 6494 trace_ufshcd_exception_event(hba, status); 6495 6496 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) 6497 ufshcd_bkops_exception_event_handler(hba); 6498 6499 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) 6500 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); 6501 6502 if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { 6503 hba->critical_health_count++; 6504 sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); 6505 } 6506 6507 if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { 6508 atomic_inc(&hba->dev_lvl_exception_count); 6509 sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); 6510 } 6511 6512 ufs_debugfs_exception_event(hba, status); 6513 } 6514 6515 /* Complete requests that have door-bell cleared */ 6516 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) 6517 { 6518 if (hba->mcq_enabled) 6519 ufshcd_mcq_compl_pending_transfer(hba, force_compl); 6520 else 6521 ufshcd_transfer_req_compl(hba); 6522 6523 ufshcd_tmc_handler(hba); 6524 } 6525 6526 /** 6527 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is 6528 * to recover from the DL NAC errors or not. 6529 * @hba: per-adapter instance 6530 * 6531 * Return: true if error handling is required, false otherwise. 6532 */ 6533 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) 6534 { 6535 unsigned long flags; 6536 bool err_handling = true; 6537 6538 spin_lock_irqsave(hba->host->host_lock, flags); 6539 /* 6540 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the 6541 * device fatal error and/or DL NAC & REPLAY timeout errors. 6542 */ 6543 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR)) 6544 goto out; 6545 6546 if ((hba->saved_err & DEVICE_FATAL_ERROR) || 6547 ((hba->saved_err & UIC_ERROR) && 6548 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR))) 6549 goto out; 6550 6551 if ((hba->saved_err & UIC_ERROR) && 6552 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) { 6553 int err; 6554 /* 6555 * wait for 50ms to see if we can get any other errors or not. 6556 */ 6557 spin_unlock_irqrestore(hba->host->host_lock, flags); 6558 msleep(50); 6559 spin_lock_irqsave(hba->host->host_lock, flags); 6560 6561 /* 6562 * now check if we have got any other severe errors other than 6563 * DL NAC error? 6564 */ 6565 if ((hba->saved_err & INT_FATAL_ERRORS) || 6566 ((hba->saved_err & UIC_ERROR) && 6567 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR))) 6568 goto out; 6569 6570 /* 6571 * As DL NAC is the only error received so far, send out NOP 6572 * command to confirm if link is still active or not. 6573 * - If we don't get any response then do error recovery. 6574 * - If we get response then clear the DL NAC error bit. 6575 */ 6576 6577 spin_unlock_irqrestore(hba->host->host_lock, flags); 6578 err = ufshcd_verify_dev_init(hba); 6579 spin_lock_irqsave(hba->host->host_lock, flags); 6580 6581 if (err) 6582 goto out; 6583 6584 /* Link seems to be alive hence ignore the DL NAC errors */ 6585 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR) 6586 hba->saved_err &= ~UIC_ERROR; 6587 /* clear NAC error */ 6588 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6589 if (!hba->saved_uic_err) 6590 err_handling = false; 6591 } 6592 out: 6593 spin_unlock_irqrestore(hba->host->host_lock, flags); 6594 return err_handling; 6595 } 6596 6597 /* host lock must be held before calling this func */ 6598 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba) 6599 { 6600 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) || 6601 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)); 6602 } 6603 6604 void ufshcd_schedule_eh_work(struct ufs_hba *hba) 6605 { 6606 lockdep_assert_held(hba->host->host_lock); 6607 6608 /* handle fatal errors only when link is not in error state */ 6609 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6610 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6611 ufshcd_is_saved_err_fatal(hba)) 6612 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL; 6613 else 6614 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL; 6615 queue_work(hba->eh_wq, &hba->eh_work); 6616 } 6617 } 6618 6619 void ufshcd_force_error_recovery(struct ufs_hba *hba) 6620 { 6621 spin_lock_irq(hba->host->host_lock); 6622 hba->force_reset = true; 6623 ufshcd_schedule_eh_work(hba); 6624 spin_unlock_irq(hba->host->host_lock); 6625 } 6626 EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); 6627 6628 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) 6629 { 6630 mutex_lock(&hba->wb_mutex); 6631 down_write(&hba->clk_scaling_lock); 6632 hba->clk_scaling.is_allowed = allow; 6633 up_write(&hba->clk_scaling_lock); 6634 mutex_unlock(&hba->wb_mutex); 6635 } 6636 6637 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) 6638 { 6639 if (suspend) { 6640 if (hba->clk_scaling.is_enabled) 6641 ufshcd_suspend_clkscaling(hba); 6642 ufshcd_clk_scaling_allow(hba, false); 6643 } else { 6644 ufshcd_clk_scaling_allow(hba, true); 6645 if (hba->clk_scaling.is_enabled) 6646 ufshcd_resume_clkscaling(hba); 6647 } 6648 } 6649 6650 static void ufshcd_err_handling_prepare(struct ufs_hba *hba) 6651 { 6652 /* 6653 * A WLUN resume failure could potentially lead to the HBA being 6654 * runtime suspended, so take an extra reference on hba->dev. 6655 */ 6656 pm_runtime_get_sync(hba->dev); 6657 ufshcd_rpm_get_sync(hba); 6658 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || 6659 hba->is_sys_suspended) { 6660 enum ufs_pm_op pm_op; 6661 6662 /* 6663 * Don't assume anything of resume, if 6664 * resume fails, irq and clocks can be OFF, and powers 6665 * can be OFF or in LPM. 6666 */ 6667 ufshcd_setup_hba_vreg(hba, true); 6668 ufshcd_enable_irq(hba); 6669 ufshcd_setup_vreg(hba, true); 6670 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 6671 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 6672 ufshcd_hold(hba); 6673 if (!ufshcd_is_clkgating_allowed(hba)) 6674 ufshcd_setup_clocks(hba, true); 6675 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; 6676 ufshcd_vops_resume(hba, pm_op); 6677 } else { 6678 ufshcd_hold(hba); 6679 if (ufshcd_is_clkscaling_supported(hba) && 6680 hba->clk_scaling.is_enabled) 6681 ufshcd_suspend_clkscaling(hba); 6682 ufshcd_clk_scaling_allow(hba, false); 6683 } 6684 /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ 6685 blk_mq_quiesce_tagset(&hba->host->tag_set); 6686 cancel_work_sync(&hba->eeh_work); 6687 } 6688 6689 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) 6690 { 6691 blk_mq_unquiesce_tagset(&hba->host->tag_set); 6692 ufshcd_release(hba); 6693 if (ufshcd_is_clkscaling_supported(hba)) 6694 ufshcd_clk_scaling_suspend(hba, false); 6695 ufshcd_rpm_put(hba); 6696 pm_runtime_put(hba->dev); 6697 } 6698 6699 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) 6700 { 6701 return (!hba->is_powered || hba->shutting_down || 6702 !hba->ufs_device_wlun || 6703 hba->ufshcd_state == UFSHCD_STATE_ERROR || 6704 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || 6705 ufshcd_is_link_broken(hba)))); 6706 } 6707 6708 #ifdef CONFIG_PM 6709 static void ufshcd_recover_pm_error(struct ufs_hba *hba) 6710 { 6711 struct scsi_target *starget = hba->ufs_device_wlun->sdev_target; 6712 struct Scsi_Host *shost = hba->host; 6713 struct scsi_device *sdev; 6714 struct request_queue *q; 6715 bool resume_sdev_queues = false; 6716 6717 hba->is_sys_suspended = false; 6718 6719 /* 6720 * Ensure the parent's error status is cleared before proceeding 6721 * to the child, as the parent must be active to activate the child. 6722 */ 6723 if (hba->dev->power.runtime_error) { 6724 /* hba->dev has no functional parent thus simplily set RPM_ACTIVE */ 6725 pm_runtime_set_active(hba->dev); 6726 resume_sdev_queues = true; 6727 } 6728 6729 if (hba->ufs_device_wlun->sdev_gendev.power.runtime_error) { 6730 /* 6731 * starget, parent of wlun, might be suspended if wlun resume failed. 6732 * Make sure parent is resumed before set child (wlun) active. 6733 */ 6734 pm_runtime_get_sync(&starget->dev); 6735 pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); 6736 pm_runtime_put_sync(&starget->dev); 6737 resume_sdev_queues = true; 6738 } 6739 6740 /* 6741 * If wlun device had runtime error, we also need to resume those 6742 * consumer scsi devices in case any of them has failed to be 6743 * resumed due to supplier runtime resume failure. This is to unblock 6744 * blk_queue_enter in case there are bios waiting inside it. 6745 */ 6746 if (resume_sdev_queues) { 6747 shost_for_each_device(sdev, shost) { 6748 q = sdev->request_queue; 6749 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 6750 q->rpm_status == RPM_SUSPENDING)) 6751 pm_request_resume(q->dev); 6752 } 6753 } 6754 } 6755 #else 6756 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba) 6757 { 6758 } 6759 #endif 6760 6761 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) 6762 { 6763 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info; 6764 u32 mode; 6765 6766 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode); 6767 6768 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK)) 6769 return true; 6770 6771 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK)) 6772 return true; 6773 6774 return false; 6775 } 6776 6777 static bool ufshcd_abort_one(struct request *rq, void *priv) 6778 { 6779 int *ret = priv; 6780 u32 tag = rq->tag; 6781 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 6782 struct scsi_device *sdev = cmd->device; 6783 struct Scsi_Host *shost = sdev->host; 6784 struct ufs_hba *hba = shost_priv(shost); 6785 6786 if (blk_mq_is_reserved_rq(rq)) 6787 return true; 6788 6789 *ret = ufshcd_try_to_abort_task(hba, tag); 6790 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6791 ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, 6792 *ret ? "failed" : "succeeded"); 6793 6794 return *ret == 0; 6795 } 6796 6797 /** 6798 * ufshcd_abort_all - Abort all pending commands. 6799 * @hba: Host bus adapter pointer. 6800 * 6801 * Return: true if and only if the host controller needs to be reset. 6802 */ 6803 static bool ufshcd_abort_all(struct ufs_hba *hba) 6804 { 6805 int tag, ret = 0; 6806 6807 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); 6808 if (ret) 6809 goto out; 6810 6811 /* Clear pending task management requests */ 6812 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { 6813 ret = ufshcd_clear_tm_cmd(hba, tag); 6814 if (ret) 6815 goto out; 6816 } 6817 6818 out: 6819 /* Complete the requests that are cleared by s/w */ 6820 ufshcd_complete_requests(hba, false); 6821 6822 return ret != 0; 6823 } 6824 6825 /** 6826 * ufshcd_err_handler - handle UFS errors that require s/w attention 6827 * @work: pointer to work structure 6828 */ 6829 static void ufshcd_err_handler(struct work_struct *work) 6830 { 6831 int retries = MAX_ERR_HANDLER_RETRIES; 6832 struct ufs_hba *hba; 6833 unsigned long flags; 6834 bool needs_restore; 6835 bool needs_reset; 6836 int pmc_err; 6837 6838 hba = container_of(work, struct ufs_hba, eh_work); 6839 6840 dev_info(hba->dev, 6841 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", 6842 __func__, ufshcd_state_name[hba->ufshcd_state], 6843 hba->is_powered, hba->shutting_down, hba->saved_err, 6844 hba->saved_uic_err, hba->force_reset, 6845 ufshcd_is_link_broken(hba) ? "; link is broken" : ""); 6846 6847 if (hba->ufs_device_wlun) { 6848 /* 6849 * Use ufshcd_rpm_get_noresume() here to safely perform link 6850 * recovery even if an error occurs during runtime suspend or 6851 * runtime resume. This avoids potential deadlocks that could 6852 * happen if we tried to resume the device while a PM operation 6853 * is already in progress. 6854 */ 6855 ufshcd_rpm_get_noresume(hba); 6856 if (hba->pm_op_in_progress) { 6857 ufshcd_link_recovery(hba); 6858 ufshcd_rpm_put(hba); 6859 return; 6860 } 6861 ufshcd_rpm_put(hba); 6862 } 6863 6864 down(&hba->host_sem); 6865 spin_lock_irqsave(hba->host->host_lock, flags); 6866 if (ufshcd_err_handling_should_stop(hba)) { 6867 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6868 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6869 spin_unlock_irqrestore(hba->host->host_lock, flags); 6870 up(&hba->host_sem); 6871 return; 6872 } 6873 spin_unlock_irqrestore(hba->host->host_lock, flags); 6874 6875 ufshcd_err_handling_prepare(hba); 6876 6877 spin_lock_irqsave(hba->host->host_lock, flags); 6878 ufshcd_set_eh_in_progress(hba); 6879 spin_unlock_irqrestore(hba->host->host_lock, flags); 6880 6881 /* Complete requests that have door-bell cleared by h/w */ 6882 ufshcd_complete_requests(hba, false); 6883 spin_lock_irqsave(hba->host->host_lock, flags); 6884 again: 6885 needs_restore = false; 6886 needs_reset = false; 6887 6888 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6889 hba->ufshcd_state = UFSHCD_STATE_RESET; 6890 /* 6891 * A full reset and restore might have happened after preparation 6892 * is finished, double check whether we should stop. 6893 */ 6894 if (ufshcd_err_handling_should_stop(hba)) 6895 goto skip_err_handling; 6896 6897 if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && 6898 !hba->force_reset) { 6899 bool ret; 6900 6901 spin_unlock_irqrestore(hba->host->host_lock, flags); 6902 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ 6903 ret = ufshcd_quirk_dl_nac_errors(hba); 6904 spin_lock_irqsave(hba->host->host_lock, flags); 6905 if (!ret && ufshcd_err_handling_should_stop(hba)) 6906 goto skip_err_handling; 6907 } 6908 6909 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6910 (hba->saved_uic_err && 6911 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6912 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR); 6913 6914 spin_unlock_irqrestore(hba->host->host_lock, flags); 6915 ufshcd_print_host_state(hba); 6916 ufshcd_print_pwr_info(hba); 6917 ufshcd_print_tx_eq_params(hba); 6918 ufshcd_print_evt_hist(hba); 6919 ufshcd_print_tmrs(hba, hba->outstanding_tasks); 6920 ufshcd_print_trs_all(hba, pr_prdt); 6921 spin_lock_irqsave(hba->host->host_lock, flags); 6922 } 6923 6924 /* 6925 * if host reset is required then skip clearing the pending 6926 * transfers forcefully because they will get cleared during 6927 * host reset and restore 6928 */ 6929 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6930 ufshcd_is_saved_err_fatal(hba) || 6931 ((hba->saved_err & UIC_ERROR) && 6932 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | 6933 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { 6934 needs_reset = true; 6935 goto do_reset; 6936 } 6937 6938 /* 6939 * If LINERESET was caught, UFS might have been put to PWM mode, 6940 * check if power mode restore is needed. 6941 */ 6942 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) { 6943 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6944 if (!hba->saved_uic_err) 6945 hba->saved_err &= ~UIC_ERROR; 6946 spin_unlock_irqrestore(hba->host->host_lock, flags); 6947 if (ufshcd_is_pwr_mode_restore_needed(hba)) 6948 needs_restore = true; 6949 spin_lock_irqsave(hba->host->host_lock, flags); 6950 if (!hba->saved_err && !needs_restore) 6951 goto skip_err_handling; 6952 } 6953 6954 hba->silence_err_logs = true; 6955 /* release lock as clear command might sleep */ 6956 spin_unlock_irqrestore(hba->host->host_lock, flags); 6957 6958 needs_reset = ufshcd_abort_all(hba); 6959 6960 spin_lock_irqsave(hba->host->host_lock, flags); 6961 hba->silence_err_logs = false; 6962 if (needs_reset) 6963 goto do_reset; 6964 6965 /* 6966 * After all reqs and tasks are cleared from doorbell, 6967 * now it is safe to retore power mode. 6968 */ 6969 if (needs_restore) { 6970 spin_unlock_irqrestore(hba->host->host_lock, flags); 6971 /* 6972 * Hold the scaling lock just in case dev cmds 6973 * are sent via bsg and/or sysfs. 6974 */ 6975 down_write(&hba->clk_scaling_lock); 6976 pmc_err = ufshcd_config_pwr_mode(hba, &hba->pwr_info, 6977 UFSHCD_PMC_POLICY_FORCE); 6978 if (pmc_err) { 6979 needs_reset = true; 6980 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n", 6981 __func__, pmc_err); 6982 } 6983 ufshcd_print_pwr_info(hba); 6984 up_write(&hba->clk_scaling_lock); 6985 spin_lock_irqsave(hba->host->host_lock, flags); 6986 } 6987 6988 do_reset: 6989 /* Fatal errors need reset */ 6990 if (needs_reset) { 6991 int err; 6992 6993 hba->force_reset = false; 6994 spin_unlock_irqrestore(hba->host->host_lock, flags); 6995 err = ufshcd_reset_and_restore(hba); 6996 if (err) 6997 dev_err(hba->dev, "%s: reset and restore failed with err %d\n", 6998 __func__, err); 6999 else 7000 ufshcd_recover_pm_error(hba); 7001 spin_lock_irqsave(hba->host->host_lock, flags); 7002 } 7003 7004 skip_err_handling: 7005 if (!needs_reset) { 7006 if (hba->ufshcd_state == UFSHCD_STATE_RESET) 7007 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 7008 if (hba->saved_err || hba->saved_uic_err) 7009 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", 7010 __func__, hba->saved_err, hba->saved_uic_err); 7011 } 7012 /* Exit in an operational state or dead */ 7013 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 7014 hba->ufshcd_state != UFSHCD_STATE_ERROR) { 7015 if (--retries) 7016 goto again; 7017 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7018 } 7019 ufshcd_clear_eh_in_progress(hba); 7020 spin_unlock_irqrestore(hba->host->host_lock, flags); 7021 ufshcd_err_handling_unprepare(hba); 7022 up(&hba->host_sem); 7023 7024 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__, 7025 ufshcd_state_name[hba->ufshcd_state]); 7026 } 7027 7028 /** 7029 * ufshcd_update_uic_error - check and set fatal UIC error flags. 7030 * @hba: per-adapter instance 7031 * 7032 * Return: 7033 * IRQ_HANDLED - If interrupt is valid 7034 * IRQ_NONE - If invalid interrupt 7035 */ 7036 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) 7037 { 7038 u32 reg; 7039 irqreturn_t retval = IRQ_NONE; 7040 7041 /* PHY layer error */ 7042 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 7043 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && 7044 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { 7045 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); 7046 /* 7047 * To know whether this error is fatal or not, DB timeout 7048 * must be checked but this error is handled separately. 7049 */ 7050 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK) 7051 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", 7052 __func__); 7053 7054 /* Got a LINERESET indication. */ 7055 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) { 7056 struct uic_command *cmd = NULL; 7057 7058 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR; 7059 if (hba->uic_async_done && hba->active_uic_cmd) 7060 cmd = hba->active_uic_cmd; 7061 /* 7062 * Ignore the LINERESET during power mode change 7063 * operation via DME_SET command. 7064 */ 7065 if (cmd && (cmd->command == UIC_CMD_DME_SET)) 7066 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 7067 } 7068 retval |= IRQ_HANDLED; 7069 } 7070 7071 /* PA_INIT_ERROR is fatal and needs UIC reset */ 7072 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 7073 if ((reg & UIC_DATA_LINK_LAYER_ERROR) && 7074 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { 7075 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); 7076 7077 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 7078 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 7079 else if (hba->dev_quirks & 7080 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 7081 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED) 7082 hba->uic_error |= 7083 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 7084 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT) 7085 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR; 7086 } 7087 retval |= IRQ_HANDLED; 7088 } 7089 7090 /* UIC NL/TL/DME errors needs software retry */ 7091 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 7092 if ((reg & UIC_NETWORK_LAYER_ERROR) && 7093 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { 7094 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); 7095 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 7096 retval |= IRQ_HANDLED; 7097 } 7098 7099 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 7100 if ((reg & UIC_TRANSPORT_LAYER_ERROR) && 7101 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { 7102 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); 7103 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 7104 retval |= IRQ_HANDLED; 7105 } 7106 7107 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 7108 if (reg & UIC_DME_ERROR) { 7109 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); 7110 7111 if (reg & UIC_DME_ERROR_CODE_MASK) 7112 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 7113 7114 if (reg & UIC_DME_QOS_MASK) { 7115 atomic_set(&hba->dme_qos_notification, 7116 reg & UIC_DME_QOS_MASK); 7117 if (hba->dme_qos_sysfs_handle) 7118 sysfs_notify_dirent(hba->dme_qos_sysfs_handle); 7119 } 7120 7121 retval |= IRQ_HANDLED; 7122 } 7123 7124 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 7125 __func__, hba->uic_error); 7126 return retval; 7127 } 7128 7129 /** 7130 * ufshcd_check_errors - Check for errors that need s/w attention 7131 * @hba: per-adapter instance 7132 * @intr_status: interrupt status generated by the controller 7133 * 7134 * Return: 7135 * IRQ_HANDLED - If interrupt is valid 7136 * IRQ_NONE - If invalid interrupt 7137 */ 7138 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) 7139 { 7140 bool queue_eh_work = false; 7141 irqreturn_t retval = IRQ_NONE; 7142 7143 guard(spinlock_irqsave)(hba->host->host_lock); 7144 hba->errors |= UFSHCD_ERROR_MASK & intr_status; 7145 7146 if (hba->errors & INT_FATAL_ERRORS) { 7147 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, 7148 hba->errors); 7149 queue_eh_work = true; 7150 } 7151 7152 if (hba->errors & UIC_ERROR) { 7153 hba->uic_error = 0; 7154 retval = ufshcd_update_uic_error(hba); 7155 if (hba->uic_error) 7156 queue_eh_work = true; 7157 } 7158 7159 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) { 7160 dev_err(hba->dev, 7161 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n", 7162 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? 7163 "Enter" : "Exit", 7164 hba->errors, ufshcd_get_upmcrs(hba)); 7165 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, 7166 hba->errors); 7167 ufshcd_set_link_broken(hba); 7168 queue_eh_work = true; 7169 } 7170 7171 if (queue_eh_work) { 7172 /* 7173 * update the transfer error masks to sticky bits, let's do this 7174 * irrespective of current ufshcd_state. 7175 */ 7176 hba->saved_err |= hba->errors; 7177 hba->saved_uic_err |= hba->uic_error; 7178 7179 /* dump controller state before resetting */ 7180 if ((hba->saved_err & 7181 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 7182 (hba->saved_uic_err && 7183 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 7184 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", 7185 __func__, hba->saved_err, 7186 hba->saved_uic_err); 7187 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, 7188 "host_regs: "); 7189 ufshcd_print_pwr_info(hba); 7190 ufshcd_print_tx_eq_params(hba); 7191 } 7192 ufshcd_schedule_eh_work(hba); 7193 retval |= IRQ_HANDLED; 7194 } 7195 /* 7196 * if (!queue_eh_work) - 7197 * Other errors are either non-fatal where host recovers 7198 * itself without s/w intervention or errors that will be 7199 * handled by the SCSI core layer. 7200 */ 7201 hba->errors = 0; 7202 hba->uic_error = 0; 7203 7204 return retval; 7205 } 7206 7207 /** 7208 * ufshcd_tmc_handler - handle task management function completion 7209 * @hba: per adapter instance 7210 * 7211 * Return: 7212 * IRQ_HANDLED - If interrupt is valid 7213 * IRQ_NONE - If invalid interrupt 7214 */ 7215 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) 7216 { 7217 unsigned long flags, pending, issued; 7218 irqreturn_t ret = IRQ_NONE; 7219 int tag; 7220 7221 spin_lock_irqsave(hba->host->host_lock, flags); 7222 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 7223 issued = hba->outstanding_tasks & ~pending; 7224 for_each_set_bit(tag, &issued, hba->nutmrs) { 7225 struct request *req = hba->tmf_rqs[tag]; 7226 struct completion *c = req->end_io_data; 7227 7228 complete(c); 7229 ret = IRQ_HANDLED; 7230 } 7231 spin_unlock_irqrestore(hba->host->host_lock, flags); 7232 7233 return ret; 7234 } 7235 7236 /** 7237 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events 7238 * @hba: per adapter instance 7239 * @reset_iag: true, to reset MCQ IAG counter and timer of the CQ 7240 * 7241 * Return: IRQ_HANDLED if interrupt is handled. 7242 */ 7243 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba, bool reset_iag) 7244 { 7245 struct ufs_hw_queue *hwq; 7246 unsigned long outstanding_cqs; 7247 unsigned int nr_queues; 7248 int i, ret; 7249 u32 events, reg; 7250 7251 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); 7252 if (ret) 7253 outstanding_cqs = (1ULL << hba->nr_hw_queues) - 1; 7254 7255 /* Exclude the poll queues */ 7256 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; 7257 for_each_set_bit(i, &outstanding_cqs, nr_queues) { 7258 hwq = &hba->uhq[i]; 7259 7260 events = ufshcd_mcq_read_cqis(hba, i); 7261 if (events) 7262 ufshcd_mcq_write_cqis(hba, events, i); 7263 7264 if (reset_iag) { 7265 reg = ufshcd_mcq_read_mcqiacr(hba, i); 7266 reg |= INT_AGGR_COUNTER_AND_TIMER_RESET; 7267 ufshcd_mcq_write_mcqiacr(hba, reg, i); 7268 } 7269 7270 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) 7271 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7272 } 7273 7274 return IRQ_HANDLED; 7275 } 7276 7277 /** 7278 * ufshcd_sl_intr - Interrupt service routine 7279 * @hba: per adapter instance 7280 * @intr_status: contains interrupts generated by the controller 7281 * 7282 * Return: 7283 * IRQ_HANDLED - If interrupt is valid 7284 * IRQ_NONE - If invalid interrupt 7285 */ 7286 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 7287 { 7288 irqreturn_t retval = IRQ_NONE; 7289 7290 if (intr_status & UFSHCD_UIC_MASK) 7291 retval |= ufshcd_uic_cmd_compl(hba, intr_status); 7292 7293 if (intr_status & UFSHCD_ERROR_MASK || hba->errors) 7294 retval |= ufshcd_check_errors(hba, intr_status); 7295 7296 if (intr_status & UTP_TASK_REQ_COMPL) 7297 retval |= ufshcd_tmc_handler(hba); 7298 7299 if (intr_status & UTP_TRANSFER_REQ_COMPL) 7300 retval |= ufshcd_transfer_req_compl(hba); 7301 7302 if (intr_status & MCQ_CQ_EVENT_STATUS) 7303 retval |= ufshcd_handle_mcq_cq_events(hba, false); 7304 7305 if (intr_status & MCQ_IAG_EVENT_STATUS) 7306 retval |= ufshcd_handle_mcq_cq_events(hba, true); 7307 7308 return retval; 7309 } 7310 7311 /** 7312 * ufshcd_threaded_intr - Threaded interrupt service routine 7313 * @irq: irq number 7314 * @__hba: pointer to adapter instance 7315 * 7316 * Return: 7317 * IRQ_HANDLED - If interrupt is valid 7318 * IRQ_NONE - If invalid interrupt 7319 */ 7320 static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) 7321 { 7322 u32 last_intr_status, intr_status, enabled_intr_status = 0; 7323 irqreturn_t retval = IRQ_NONE; 7324 struct ufs_hba *hba = __hba; 7325 int retries = hba->nutrs; 7326 7327 last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7328 7329 /* 7330 * There could be max of hba->nutrs reqs in flight and in worst case 7331 * if the reqs get finished 1 by 1 after the interrupt status is 7332 * read, make sure we handle them by checking the interrupt status 7333 * again in a loop until we process all of the reqs before returning. 7334 */ 7335 while (intr_status && retries--) { 7336 enabled_intr_status = 7337 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7338 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7339 if (enabled_intr_status) 7340 retval |= ufshcd_sl_intr(hba, enabled_intr_status); 7341 7342 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7343 } 7344 7345 if (enabled_intr_status && retval == IRQ_NONE && 7346 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) || 7347 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) { 7348 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", 7349 __func__, 7350 intr_status, 7351 last_intr_status, 7352 enabled_intr_status); 7353 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 7354 } 7355 7356 return retval; 7357 } 7358 7359 /** 7360 * ufshcd_intr - Main interrupt service routine 7361 * @irq: irq number 7362 * @__hba: pointer to adapter instance 7363 * 7364 * Return: 7365 * IRQ_HANDLED - If interrupt is valid 7366 * IRQ_WAKE_THREAD - If handling is moved to threaded handled 7367 * IRQ_NONE - If invalid interrupt 7368 */ 7369 static irqreturn_t ufshcd_intr(int irq, void *__hba) 7370 { 7371 struct ufs_hba *hba = __hba; 7372 u32 intr_status, enabled_intr_status; 7373 7374 /* 7375 * Handle interrupt in thread if MCQ or ESI is disabled, 7376 * and no active UIC command. 7377 */ 7378 if ((!hba->mcq_enabled || !hba->mcq_esi_enabled) && 7379 !hba->active_uic_cmd) 7380 return IRQ_WAKE_THREAD; 7381 7382 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7383 enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7384 7385 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7386 7387 /* Directly handle interrupts since MCQ ESI handlers does the hard job */ 7388 return ufshcd_sl_intr(hba, enabled_intr_status); 7389 } 7390 7391 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 7392 { 7393 int err = 0; 7394 u32 mask = 1 << tag; 7395 7396 if (!test_bit(tag, &hba->outstanding_tasks)) 7397 goto out; 7398 7399 ufshcd_utmrl_clear(hba, tag); 7400 7401 /* poll for max. 1 sec to clear door bell register by h/w */ 7402 err = ufshcd_wait_for_register(hba, 7403 REG_UTP_TASK_REQ_DOOR_BELL, 7404 mask, 0, 1000, 1000); 7405 7406 dev_err(hba->dev, "Clearing task management function with tag %d %s\n", 7407 tag, err < 0 ? "failed" : "succeeded"); 7408 7409 out: 7410 return err; 7411 } 7412 7413 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, 7414 struct utp_task_req_desc *treq, u8 tm_function) 7415 { 7416 struct request_queue *q = hba->tmf_queue; 7417 struct Scsi_Host *host = hba->host; 7418 DECLARE_COMPLETION_ONSTACK(wait); 7419 struct request *req; 7420 unsigned long flags; 7421 int task_tag, err; 7422 7423 /* 7424 * blk_mq_alloc_request() is used here only to get a free tag. 7425 */ 7426 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0); 7427 if (IS_ERR(req)) 7428 return PTR_ERR(req); 7429 7430 req->end_io_data = &wait; 7431 ufshcd_hold(hba); 7432 7433 spin_lock_irqsave(host->host_lock, flags); 7434 7435 task_tag = req->tag; 7436 hba->tmf_rqs[req->tag] = req; 7437 treq->upiu_req.req_header.task_tag = task_tag; 7438 7439 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); 7440 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); 7441 7442 __set_bit(task_tag, &hba->outstanding_tasks); 7443 7444 spin_unlock_irqrestore(host->host_lock, flags); 7445 7446 /* send command to the controller */ 7447 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); 7448 7449 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); 7450 7451 /* wait until the task management command is completed */ 7452 err = wait_for_completion_io_timeout(&wait, 7453 msecs_to_jiffies(TM_CMD_TIMEOUT)); 7454 if (!err) { 7455 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR); 7456 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 7457 __func__, tm_function); 7458 if (ufshcd_clear_tm_cmd(hba, task_tag)) 7459 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n", 7460 __func__, task_tag); 7461 err = -ETIMEDOUT; 7462 } else { 7463 err = 0; 7464 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq)); 7465 7466 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP); 7467 } 7468 7469 spin_lock_irqsave(hba->host->host_lock, flags); 7470 hba->tmf_rqs[req->tag] = NULL; 7471 __clear_bit(task_tag, &hba->outstanding_tasks); 7472 spin_unlock_irqrestore(hba->host->host_lock, flags); 7473 7474 ufshcd_release(hba); 7475 blk_mq_free_request(req); 7476 7477 return err; 7478 } 7479 7480 /** 7481 * ufshcd_issue_tm_cmd - issues task management commands to controller 7482 * @hba: per adapter instance 7483 * @lun_id: LUN ID to which TM command is sent 7484 * @task_id: task ID to which the TM command is applicable 7485 * @tm_function: task management function opcode 7486 * @tm_response: task management service response return value 7487 * 7488 * Return: non-zero value on error, zero on success. 7489 */ 7490 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 7491 u8 tm_function, u8 *tm_response) 7492 { 7493 struct utp_task_req_desc treq = { }; 7494 enum utp_ocs ocs_value; 7495 int err; 7496 7497 /* Configure task request descriptor */ 7498 treq.header.interrupt = 1; 7499 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7500 7501 /* Configure task request UPIU */ 7502 treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; 7503 treq.upiu_req.req_header.lun = lun_id; 7504 treq.upiu_req.req_header.tm_function = tm_function; 7505 7506 /* 7507 * The host shall provide the same value for LUN field in the basic 7508 * header and for Input Parameter. 7509 */ 7510 treq.upiu_req.input_param1 = cpu_to_be32(lun_id); 7511 treq.upiu_req.input_param2 = cpu_to_be32(task_id); 7512 7513 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function); 7514 if (err == -ETIMEDOUT) 7515 return err; 7516 7517 ocs_value = treq.header.ocs & MASK_OCS; 7518 if (ocs_value != OCS_SUCCESS) 7519 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 7520 __func__, ocs_value); 7521 else if (tm_response) 7522 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) & 7523 MASK_TM_SERVICE_RESP; 7524 return err; 7525 } 7526 7527 /** 7528 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests 7529 * @hba: per-adapter instance 7530 * @req_upiu: upiu request 7531 * @rsp_upiu: upiu reply 7532 * @desc_buff: pointer to descriptor buffer, NULL if NA 7533 * @buff_len: descriptor size, 0 if NA 7534 * @cmd_type: specifies the type (NOP, Query...) 7535 * @desc_op: descriptor operation 7536 * 7537 * Those type of requests uses UTP Transfer Request Descriptor - utrd. 7538 * Therefore, it "rides" the device management infrastructure: uses its tag and 7539 * tasks work queues. 7540 * 7541 * Since there is only one available tag for device management commands, 7542 * the caller is expected to hold the hba->dev_cmd.lock mutex. 7543 * 7544 * Return: 0 upon success; < 0 upon failure. 7545 */ 7546 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, 7547 struct utp_upiu_req *req_upiu, 7548 struct utp_upiu_req *rsp_upiu, 7549 u8 *desc_buff, int *buff_len, 7550 enum dev_cmd_type cmd_type, 7551 enum query_opcode desc_op) 7552 { 7553 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 7554 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7555 u32 tag; 7556 int err = 0; 7557 u8 upiu_flags; 7558 7559 /* Protects use of hba->dev_cmd. */ 7560 lockdep_assert_held(&hba->dev_cmd.lock); 7561 7562 if (WARN_ON_ONCE(!cmd)) 7563 return -ENOMEM; 7564 7565 tag = scsi_cmd_to_rq(cmd)->tag; 7566 7567 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 7568 7569 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7570 7571 /* update the task tag in the request upiu */ 7572 req_upiu->header.task_tag = tag; 7573 7574 /* just copy the upiu request as it is */ 7575 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7576 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) { 7577 /* The Data Segment Area is optional depending upon the query 7578 * function value. for WRITE DESCRIPTOR, the data segment 7579 * follows right after the tsf. 7580 */ 7581 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len); 7582 *buff_len = 0; 7583 } 7584 7585 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7586 7587 err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); 7588 if (err) 7589 goto put_dev_mgmt_cmd; 7590 7591 /* just copy the upiu response as it is */ 7592 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7593 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { 7594 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); 7595 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 7596 .data_segment_length); 7597 7598 if (*buff_len >= resp_len) { 7599 memcpy(desc_buff, descp, resp_len); 7600 *buff_len = resp_len; 7601 } else { 7602 dev_warn(hba->dev, 7603 "%s: rsp size %d is bigger than buffer size %d", 7604 __func__, resp_len, *buff_len); 7605 *buff_len = 0; 7606 err = -EINVAL; 7607 } 7608 } 7609 7610 put_dev_mgmt_cmd: 7611 ufshcd_put_dev_mgmt_cmd(cmd); 7612 7613 return err; 7614 } 7615 7616 /** 7617 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands 7618 * @hba: per-adapter instance 7619 * @req_upiu: upiu request 7620 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands 7621 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target 7622 * @desc_buff: pointer to descriptor buffer, NULL if NA 7623 * @buff_len: descriptor size, 0 if NA 7624 * @desc_op: descriptor operation 7625 * 7626 * Supports UTP Transfer requests (nop and query), and UTP Task 7627 * Management requests. 7628 * It is up to the caller to fill the upiu conent properly, as it will 7629 * be copied without any further input validations. 7630 * 7631 * Return: 0 upon success; < 0 upon failure. 7632 */ 7633 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 7634 struct utp_upiu_req *req_upiu, 7635 struct utp_upiu_req *rsp_upiu, 7636 enum upiu_request_transaction msgcode, 7637 u8 *desc_buff, int *buff_len, 7638 enum query_opcode desc_op) 7639 { 7640 int err; 7641 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; 7642 struct utp_task_req_desc treq = { }; 7643 enum utp_ocs ocs_value; 7644 u8 tm_f = req_upiu->header.tm_function; 7645 7646 switch (msgcode) { 7647 case UPIU_TRANSACTION_NOP_OUT: 7648 cmd_type = DEV_CMD_TYPE_NOP; 7649 fallthrough; 7650 case UPIU_TRANSACTION_QUERY_REQ: 7651 ufshcd_dev_man_lock(hba); 7652 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, 7653 desc_buff, buff_len, 7654 cmd_type, desc_op); 7655 ufshcd_dev_man_unlock(hba); 7656 7657 break; 7658 case UPIU_TRANSACTION_TASK_REQ: 7659 treq.header.interrupt = 1; 7660 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7661 7662 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); 7663 7664 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f); 7665 if (err == -ETIMEDOUT) 7666 break; 7667 7668 ocs_value = treq.header.ocs & MASK_OCS; 7669 if (ocs_value != OCS_SUCCESS) { 7670 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, 7671 ocs_value); 7672 break; 7673 } 7674 7675 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu)); 7676 7677 break; 7678 default: 7679 err = -EINVAL; 7680 7681 break; 7682 } 7683 7684 return err; 7685 } 7686 7687 /** 7688 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request 7689 * @hba: per adapter instance 7690 * @req_upiu: upiu request 7691 * @rsp_upiu: upiu reply 7692 * @req_ehs: EHS field which contains Advanced RPMB Request Message 7693 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message 7694 * @sg_cnt: The number of sg lists actually used 7695 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation 7696 * @dir: DMA direction 7697 * 7698 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 7699 * < 0 if another error occurred. 7700 */ 7701 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, 7702 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, 7703 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7704 enum dma_data_direction dir) 7705 { 7706 struct scsi_cmnd *cmd; 7707 struct ufshcd_lrb *lrbp; 7708 u32 tag; 7709 int err = 0; 7710 int result; 7711 u8 upiu_flags; 7712 u8 *ehs_data; 7713 u16 ehs_len; 7714 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; 7715 7716 ufshcd_dev_man_lock(hba); 7717 7718 cmd = ufshcd_get_dev_mgmt_cmd(hba); 7719 7720 if (WARN_ON_ONCE(!cmd)) { 7721 err = -ENOMEM; 7722 goto unlock; 7723 } 7724 7725 lrbp = scsi_cmd_priv(cmd); 7726 tag = scsi_cmd_to_rq(cmd)->tag; 7727 7728 ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, 7729 tag); 7730 7731 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7732 7733 /* update the task tag */ 7734 req_upiu->header.task_tag = tag; 7735 7736 /* copy the UPIU(contains CDB) request as it is */ 7737 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7738 /* Copy EHS, starting with byte32, immediately after the CDB package */ 7739 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); 7740 7741 if (dir != DMA_NONE && sg_list) 7742 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); 7743 7744 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7745 7746 err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7747 if (err) 7748 goto put_dev_mgmt_cmd; 7749 7750 err = ufshcd_dev_cmd_completion(hba, lrbp); 7751 if (!err) { 7752 /* Just copy the upiu response as it is */ 7753 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7754 /* Get the response UPIU result */ 7755 result = (lrbp->ucd_rsp_ptr->header.response << 8) | 7756 lrbp->ucd_rsp_ptr->header.status; 7757 7758 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; 7759 /* 7760 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data 7761 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB 7762 * Message is 02h 7763 */ 7764 if (ehs_len == 2 && rsp_ehs) { 7765 /* 7766 * ucd_rsp_ptr points to a buffer with a length of 512 bytes 7767 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 7768 */ 7769 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; 7770 memcpy(rsp_ehs, ehs_data, ehs_len * 32); 7771 } 7772 } 7773 7774 put_dev_mgmt_cmd: 7775 ufshcd_put_dev_mgmt_cmd(cmd); 7776 7777 unlock: 7778 ufshcd_dev_man_unlock(hba); 7779 7780 return err ? : result; 7781 } 7782 7783 static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) 7784 { 7785 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 7786 struct scsi_device *sdev = cmd->device; 7787 struct Scsi_Host *shost = sdev->host; 7788 struct ufs_hba *hba = shost_priv(shost); 7789 const u64 lun = *(u64 *)priv; 7790 const u32 tag = req->tag; 7791 7792 if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) 7793 return true; 7794 7795 if (ufshcd_clear_cmd(hba, tag) < 0) { 7796 dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, 7797 tag); 7798 return true; 7799 } 7800 7801 if (hba->mcq_enabled) { 7802 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); 7803 7804 if (hwq) 7805 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7806 return true; 7807 } 7808 7809 ufshcd_compl_one_cqe(hba, tag, NULL); 7810 return true; 7811 } 7812 7813 /** 7814 * ufshcd_eh_device_reset_handler() - Reset a single logical unit. 7815 * @cmd: SCSI command pointer 7816 * 7817 * Return: SUCCESS or FAILED. 7818 */ 7819 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7820 { 7821 struct Scsi_Host *host; 7822 struct ufs_hba *hba; 7823 int err; 7824 u8 resp = 0xF, lun; 7825 7826 host = cmd->device->host; 7827 hba = shost_priv(host); 7828 7829 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7830 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7831 if (err) { 7832 } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7833 err = resp; 7834 } else { 7835 /* clear the commands that were pending for corresponding LUN */ 7836 blk_mq_tagset_busy_iter(&hba->host->tag_set, 7837 ufshcd_clear_lu_cmds, 7838 &cmd->device->lun); 7839 } 7840 7841 hba->req_abort_count = 0; 7842 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7843 if (!err) { 7844 err = SUCCESS; 7845 } else { 7846 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7847 err = FAILED; 7848 } 7849 return err; 7850 } 7851 7852 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7853 { 7854 int tag; 7855 7856 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7857 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7858 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7859 7860 lrbp->req_abort_skip = true; 7861 } 7862 } 7863 7864 /** 7865 * ufshcd_try_to_abort_task - abort a specific task 7866 * @hba: Pointer to adapter instance 7867 * @tag: Tag of the task to be aborted 7868 * 7869 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7870 * command, and in host controller by clearing the door-bell register. There can 7871 * be race between controller sending the command to the device while abort is 7872 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 7873 * really issued and then try to abort it. 7874 * 7875 * Return: zero on success, non-zero on failure. 7876 */ 7877 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7878 { 7879 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7880 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7881 int err; 7882 int poll_cnt; 7883 u8 resp = 0xF; 7884 7885 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7886 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, 7887 &resp); 7888 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7889 /* cmd pending in the device */ 7890 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", 7891 __func__, tag); 7892 break; 7893 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7894 /* 7895 * cmd not pending in the device, check if it is 7896 * in transition. 7897 */ 7898 dev_info( 7899 hba->dev, 7900 "%s: cmd with tag %d not pending in the device.\n", 7901 __func__, tag); 7902 if (!ufshcd_cmd_inflight(cmd)) { 7903 dev_info(hba->dev, 7904 "%s: cmd with tag=%d completed.\n", 7905 __func__, tag); 7906 return 0; 7907 } 7908 usleep_range(100, 200); 7909 } else { 7910 dev_err(hba->dev, 7911 "%s: no response from device. tag = %d, err %d\n", 7912 __func__, tag, err); 7913 return err ? : resp; 7914 } 7915 } 7916 7917 if (!poll_cnt) 7918 return -EBUSY; 7919 7920 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); 7921 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7922 if (!err) { 7923 err = resp; /* service response error */ 7924 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", 7925 __func__, tag, err); 7926 } 7927 return err; 7928 } 7929 7930 err = ufshcd_clear_cmd(hba, tag); 7931 if (err) 7932 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", 7933 __func__, tag, err); 7934 7935 return err; 7936 } 7937 7938 /** 7939 * ufshcd_abort - scsi host template eh_abort_handler callback 7940 * @cmd: SCSI command pointer 7941 * 7942 * Return: SUCCESS or FAILED. 7943 */ 7944 static int ufshcd_abort(struct scsi_cmnd *cmd) 7945 { 7946 struct Scsi_Host *host = cmd->device->host; 7947 struct ufs_hba *hba = shost_priv(host); 7948 struct request *rq = scsi_cmd_to_rq(cmd); 7949 int tag = rq->tag; 7950 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7951 unsigned long flags; 7952 int err = FAILED; 7953 bool outstanding; 7954 u32 reg; 7955 7956 ufshcd_hold(hba); 7957 7958 if (!hba->mcq_enabled) { 7959 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7960 if (!test_bit(tag, &hba->outstanding_reqs)) { 7961 /* If command is already aborted/completed, return FAILED. */ 7962 dev_err(hba->dev, 7963 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", 7964 __func__, tag, hba->outstanding_reqs, reg); 7965 goto release; 7966 } 7967 } 7968 7969 /* Print Transfer Request of aborted task */ 7970 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); 7971 7972 /* 7973 * Print detailed info about aborted request. 7974 * As more than one request might get aborted at the same time, 7975 * print full information only for the first aborted request in order 7976 * to reduce repeated printouts. For other aborted requests only print 7977 * basic details. 7978 */ 7979 if (ufshcd_is_scsi_cmd(cmd)) 7980 scsi_print_command(cmd); 7981 if (!hba->req_abort_count) { 7982 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7983 ufshcd_print_evt_hist(hba); 7984 ufshcd_print_host_state(hba); 7985 ufshcd_print_pwr_info(hba); 7986 ufshcd_print_tx_eq_params(hba); 7987 ufshcd_print_tr(hba, cmd, true); 7988 } else { 7989 ufshcd_print_tr(hba, cmd, false); 7990 } 7991 hba->req_abort_count++; 7992 7993 if (!hba->mcq_enabled && !(reg & (1 << tag))) { 7994 /* only execute this code in single doorbell mode */ 7995 dev_err(hba->dev, 7996 "%s: cmd was completed, but without a notifying intr, tag = %d", 7997 __func__, tag); 7998 __ufshcd_transfer_req_compl(hba, 1UL << tag); 7999 goto release; 8000 } 8001 8002 /* 8003 * Task abort to the device W-LUN is illegal. When this command 8004 * will fail, due to spec violation, scsi err handling next step 8005 * will be to send LU reset which, again, is a spec violation. 8006 * To avoid these unnecessary/illegal steps, first we clean up 8007 * the lrb taken by this cmd and re-set it in outstanding_reqs, 8008 * then queue the eh_work and bail. 8009 */ 8010 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { 8011 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); 8012 8013 spin_lock_irqsave(host->host_lock, flags); 8014 hba->force_reset = true; 8015 ufshcd_schedule_eh_work(hba); 8016 spin_unlock_irqrestore(host->host_lock, flags); 8017 goto release; 8018 } 8019 8020 if (hba->mcq_enabled) { 8021 /* MCQ mode. Branch off to handle abort for mcq mode */ 8022 err = ufshcd_mcq_abort(cmd); 8023 goto release; 8024 } 8025 8026 /* Skip task abort in case previous aborts failed and report failure */ 8027 if (lrbp->req_abort_skip) { 8028 dev_err(hba->dev, "%s: skipping abort\n", __func__); 8029 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 8030 goto release; 8031 } 8032 8033 if (blk_mq_is_reserved_rq(rq)) 8034 err = ufshcd_clear_cmd(hba, tag); 8035 else 8036 err = ufshcd_try_to_abort_task(hba, tag); 8037 if (err) { 8038 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 8039 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 8040 err = FAILED; 8041 goto release; 8042 } 8043 8044 /* 8045 * Clear the corresponding bit from outstanding_reqs since the command 8046 * has been aborted successfully. 8047 */ 8048 spin_lock_irqsave(&hba->outstanding_lock, flags); 8049 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs); 8050 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 8051 8052 if (outstanding) 8053 ufshcd_release_scsi_cmd(hba, cmd); 8054 8055 err = SUCCESS; 8056 8057 release: 8058 /* Matches the ufshcd_hold() call at the start of this function. */ 8059 ufshcd_release(hba); 8060 return err; 8061 } 8062 8063 /** 8064 * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. 8065 * @hba: UFS host controller instance. 8066 * @probe_start: time when the ufshcd_probe_hba() call started. 8067 * @ret: ufshcd_probe_hba() return value. 8068 */ 8069 static void ufshcd_process_probe_result(struct ufs_hba *hba, 8070 ktime_t probe_start, int ret) 8071 { 8072 unsigned long flags; 8073 8074 spin_lock_irqsave(hba->host->host_lock, flags); 8075 if (ret) 8076 hba->ufshcd_state = UFSHCD_STATE_ERROR; 8077 else if (hba->ufshcd_state == UFSHCD_STATE_RESET) 8078 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 8079 spin_unlock_irqrestore(hba->host->host_lock, flags); 8080 8081 trace_ufshcd_init(hba, ret, 8082 ktime_to_us(ktime_sub(ktime_get(), probe_start)), 8083 hba->curr_dev_pwr_mode, hba->uic_link_state); 8084 } 8085 8086 /** 8087 * ufshcd_host_reset_and_restore - reset and restore host controller 8088 * @hba: per-adapter instance 8089 * 8090 * Note that host controller reset may issue DME_RESET to 8091 * local and remote (device) Uni-Pro stack and the attributes 8092 * are reset to default state. 8093 * 8094 * Return: zero on success, non-zero on failure. 8095 */ 8096 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 8097 { 8098 int err; 8099 8100 /* 8101 * Stop the host controller and complete the requests 8102 * cleared by h/w 8103 */ 8104 ufshcd_hba_stop(hba); 8105 hba->silence_err_logs = true; 8106 ufshcd_complete_requests(hba, true); 8107 hba->silence_err_logs = false; 8108 8109 /* scale up clocks to max frequency before full reinitialization */ 8110 if (ufshcd_is_clkscaling_supported(hba)) 8111 ufshcd_scale_clks(hba, ULONG_MAX, true); 8112 8113 err = ufshcd_hba_enable(hba); 8114 8115 /* Establish the link again and restore the device */ 8116 if (!err) { 8117 ktime_t probe_start = ktime_get(); 8118 8119 err = ufshcd_device_init(hba, /*init_dev_params=*/false); 8120 if (!err) 8121 err = ufshcd_probe_hba(hba, false); 8122 ufshcd_process_probe_result(hba, probe_start, err); 8123 } 8124 8125 if (err) 8126 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 8127 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); 8128 return err; 8129 } 8130 8131 /** 8132 * ufshcd_reset_and_restore - reset and re-initialize host/device 8133 * @hba: per-adapter instance 8134 * 8135 * Reset and recover device, host and re-establish link. This 8136 * is helpful to recover the communication in fatal error conditions. 8137 * 8138 * Return: zero on success, non-zero on failure. 8139 */ 8140 static int ufshcd_reset_and_restore(struct ufs_hba *hba) 8141 { 8142 u32 saved_err = 0; 8143 u32 saved_uic_err = 0; 8144 int err = 0; 8145 unsigned long flags; 8146 int retries = MAX_HOST_RESET_RETRIES; 8147 8148 spin_lock_irqsave(hba->host->host_lock, flags); 8149 do { 8150 /* 8151 * This is a fresh start, cache and clear saved error first, 8152 * in case new error generated during reset and restore. 8153 */ 8154 saved_err |= hba->saved_err; 8155 saved_uic_err |= hba->saved_uic_err; 8156 hba->saved_err = 0; 8157 hba->saved_uic_err = 0; 8158 hba->force_reset = false; 8159 hba->ufshcd_state = UFSHCD_STATE_RESET; 8160 spin_unlock_irqrestore(hba->host->host_lock, flags); 8161 8162 /* Reset the attached device */ 8163 ufshcd_device_reset(hba); 8164 8165 err = ufshcd_host_reset_and_restore(hba); 8166 8167 spin_lock_irqsave(hba->host->host_lock, flags); 8168 if (err) 8169 continue; 8170 /* Do not exit unless operational or dead */ 8171 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 8172 hba->ufshcd_state != UFSHCD_STATE_ERROR && 8173 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL) 8174 err = -EAGAIN; 8175 } while (err && --retries); 8176 8177 /* 8178 * Inform scsi mid-layer that we did reset and allow to handle 8179 * Unit Attention properly. 8180 */ 8181 scsi_report_bus_reset(hba->host, 0); 8182 if (err) { 8183 hba->ufshcd_state = UFSHCD_STATE_ERROR; 8184 hba->saved_err |= saved_err; 8185 hba->saved_uic_err |= saved_uic_err; 8186 } 8187 spin_unlock_irqrestore(hba->host->host_lock, flags); 8188 8189 return err; 8190 } 8191 8192 /** 8193 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 8194 * @cmd: SCSI command pointer 8195 * 8196 * Return: SUCCESS or FAILED. 8197 */ 8198 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 8199 { 8200 int err = SUCCESS; 8201 unsigned long flags; 8202 struct ufs_hba *hba; 8203 8204 hba = shost_priv(cmd->device->host); 8205 8206 /* 8207 * If runtime PM sent SSU and got a timeout, scsi_error_handler is 8208 * stuck in this function waiting for flush_work(&hba->eh_work). And 8209 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do 8210 * ufshcd_link_recovery instead of eh_work to prevent deadlock. 8211 */ 8212 if (hba->pm_op_in_progress) { 8213 if (ufshcd_link_recovery(hba)) 8214 err = FAILED; 8215 8216 return err; 8217 } 8218 8219 spin_lock_irqsave(hba->host->host_lock, flags); 8220 hba->force_reset = true; 8221 ufshcd_schedule_eh_work(hba); 8222 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__); 8223 spin_unlock_irqrestore(hba->host->host_lock, flags); 8224 8225 flush_work(&hba->eh_work); 8226 8227 spin_lock_irqsave(hba->host->host_lock, flags); 8228 if (hba->ufshcd_state == UFSHCD_STATE_ERROR) 8229 err = FAILED; 8230 spin_unlock_irqrestore(hba->host->host_lock, flags); 8231 8232 return err; 8233 } 8234 8235 /** 8236 * ufshcd_get_max_icc_level - calculate the ICC level 8237 * @sup_curr_uA: max. current supported by the regulator 8238 * @start_scan: row at the desc table to start scan from 8239 * @buff: power descriptor buffer 8240 * 8241 * Return: calculated max ICC level for specific regulator. 8242 */ 8243 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, 8244 const char *buff) 8245 { 8246 int i; 8247 int curr_uA; 8248 u16 data; 8249 u16 unit; 8250 8251 for (i = start_scan; i >= 0; i--) { 8252 data = get_unaligned_be16(&buff[2 * i]); 8253 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 8254 ATTR_ICC_LVL_UNIT_OFFSET; 8255 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 8256 switch (unit) { 8257 case UFSHCD_NANO_AMP: 8258 curr_uA = curr_uA / 1000; 8259 break; 8260 case UFSHCD_MILI_AMP: 8261 curr_uA = curr_uA * 1000; 8262 break; 8263 case UFSHCD_AMP: 8264 curr_uA = curr_uA * 1000 * 1000; 8265 break; 8266 case UFSHCD_MICRO_AMP: 8267 default: 8268 break; 8269 } 8270 if (sup_curr_uA >= curr_uA) 8271 break; 8272 } 8273 if (i < 0) { 8274 i = 0; 8275 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 8276 } 8277 8278 return (u32)i; 8279 } 8280 8281 /** 8282 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level 8283 * In case regulators are not initialized we'll return 0 8284 * @hba: per-adapter instance 8285 * @desc_buf: power descriptor buffer to extract ICC levels from. 8286 * 8287 * Return: calculated ICC level. 8288 */ 8289 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 8290 const u8 *desc_buf) 8291 { 8292 u32 icc_level = 0; 8293 8294 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 8295 !hba->vreg_info.vccq2) { 8296 /* 8297 * Using dev_dbg to avoid messages during runtime PM to avoid 8298 * never-ending cycles of messages written back to storage by 8299 * user space causing runtime resume, causing more messages and 8300 * so on. 8301 */ 8302 dev_dbg(hba->dev, 8303 "%s: Regulator capability was not set, actvIccLevel=%d", 8304 __func__, icc_level); 8305 goto out; 8306 } 8307 8308 if (hba->vreg_info.vcc->max_uA) 8309 icc_level = ufshcd_get_max_icc_level( 8310 hba->vreg_info.vcc->max_uA, 8311 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 8312 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 8313 8314 if (hba->vreg_info.vccq->max_uA) 8315 icc_level = ufshcd_get_max_icc_level( 8316 hba->vreg_info.vccq->max_uA, 8317 icc_level, 8318 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 8319 8320 if (hba->vreg_info.vccq2->max_uA) 8321 icc_level = ufshcd_get_max_icc_level( 8322 hba->vreg_info.vccq2->max_uA, 8323 icc_level, 8324 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 8325 out: 8326 return icc_level; 8327 } 8328 8329 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) 8330 { 8331 int ret; 8332 u8 *desc_buf; 8333 u32 icc_level; 8334 8335 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8336 if (!desc_buf) 8337 return; 8338 8339 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, 8340 desc_buf, QUERY_DESC_MAX_SIZE); 8341 if (ret) { 8342 dev_err(hba->dev, 8343 "%s: Failed reading power descriptor ret = %d", 8344 __func__, ret); 8345 goto out; 8346 } 8347 8348 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); 8349 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); 8350 8351 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8352 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); 8353 8354 if (ret) 8355 dev_err(hba->dev, 8356 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 8357 __func__, icc_level, ret); 8358 8359 out: 8360 kfree(desc_buf); 8361 } 8362 8363 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) 8364 { 8365 struct Scsi_Host *shost = sdev->host; 8366 8367 scsi_autopm_get_device(sdev); 8368 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); 8369 if (sdev->rpm_autosuspend) 8370 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 8371 shost->rpm_autosuspend_delay); 8372 scsi_autopm_put_device(sdev); 8373 } 8374 8375 /** 8376 * ufshcd_scsi_add_wlus - Adds required W-LUs 8377 * @hba: per-adapter instance 8378 * 8379 * UFS device specification requires the UFS devices to support 4 well known 8380 * logical units: 8381 * "REPORT_LUNS" (address: 01h) 8382 * "UFS Device" (address: 50h) 8383 * "RPMB" (address: 44h) 8384 * "BOOT" (address: 30h) 8385 * UFS device's power management needs to be controlled by "POWER CONDITION" 8386 * field of SSU (START STOP UNIT) command. But this "power condition" field 8387 * will take effect only when its sent to "UFS device" well known logical unit 8388 * hence we require the scsi_device instance to represent this logical unit in 8389 * order for the UFS host driver to send the SSU command for power management. 8390 * 8391 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 8392 * Block) LU so user space process can control this LU. User space may also 8393 * want to have access to BOOT LU. 8394 * 8395 * This function adds scsi device instances for each of all well known LUs 8396 * (except "REPORT LUNS" LU). 8397 * 8398 * Return: zero on success (all required W-LUs are added successfully), 8399 * non-zero error value on failure (if failed to add any of the required W-LU). 8400 */ 8401 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 8402 { 8403 int ret = 0; 8404 struct scsi_device *sdev_boot, *sdev_rpmb; 8405 8406 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0, 8407 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 8408 if (IS_ERR(hba->ufs_device_wlun)) { 8409 ret = PTR_ERR(hba->ufs_device_wlun); 8410 hba->ufs_device_wlun = NULL; 8411 goto out; 8412 } 8413 scsi_device_put(hba->ufs_device_wlun); 8414 8415 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 8416 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 8417 if (IS_ERR(sdev_rpmb)) { 8418 ret = PTR_ERR(sdev_rpmb); 8419 hba->ufs_rpmb_wlun = NULL; 8420 dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__); 8421 goto remove_ufs_device_wlun; 8422 } 8423 hba->ufs_rpmb_wlun = sdev_rpmb; 8424 ufshcd_blk_pm_runtime_init(sdev_rpmb); 8425 scsi_device_put(sdev_rpmb); 8426 8427 sdev_boot = __scsi_add_device(hba->host, 0, 0, 8428 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 8429 if (IS_ERR(sdev_boot)) { 8430 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__); 8431 } else { 8432 ufshcd_blk_pm_runtime_init(sdev_boot); 8433 scsi_device_put(sdev_boot); 8434 } 8435 goto out; 8436 8437 remove_ufs_device_wlun: 8438 scsi_remove_device(hba->ufs_device_wlun); 8439 out: 8440 return ret; 8441 } 8442 8443 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) 8444 { 8445 struct ufs_dev_info *dev_info = &hba->dev_info; 8446 u8 lun; 8447 u32 d_lu_wb_buf_alloc; 8448 u32 ext_ufs_feature; 8449 8450 if (!ufshcd_is_wb_allowed(hba)) 8451 return; 8452 8453 /* 8454 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or 8455 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES 8456 * enabled 8457 */ 8458 if (!(dev_info->wspecversion >= 0x310 || 8459 dev_info->wspecversion == 0x220 || 8460 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) 8461 goto wb_disabled; 8462 8463 ext_ufs_feature = get_unaligned_be32(desc_buf + 8464 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8465 8466 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP)) 8467 goto wb_disabled; 8468 8469 /* 8470 * WB may be supported but not configured while provisioning. The spec 8471 * says, in dedicated wb buffer mode, a max of 1 lun would have wb 8472 * buffer configured. 8473 */ 8474 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; 8475 8476 dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + 8477 DEVICE_DESC_PARAM_EXT_WB_SUP); 8478 8479 dev_info->b_presrv_uspc_en = 8480 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; 8481 8482 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) { 8483 if (!get_unaligned_be32(desc_buf + 8484 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS)) 8485 goto wb_disabled; 8486 } else { 8487 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { 8488 d_lu_wb_buf_alloc = 0; 8489 ufshcd_read_unit_desc_param(hba, 8490 lun, 8491 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS, 8492 (u8 *)&d_lu_wb_buf_alloc, 8493 sizeof(d_lu_wb_buf_alloc)); 8494 if (d_lu_wb_buf_alloc) { 8495 dev_info->wb_dedicated_lu = lun; 8496 break; 8497 } 8498 } 8499 8500 if (!d_lu_wb_buf_alloc) 8501 goto wb_disabled; 8502 } 8503 8504 if (!ufshcd_is_wb_buf_lifetime_available(hba)) 8505 goto wb_disabled; 8506 8507 return; 8508 8509 wb_disabled: 8510 hba->caps &= ~UFSHCD_CAP_WB_EN; 8511 } 8512 8513 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) 8514 { 8515 struct ufs_dev_info *dev_info = &hba->dev_info; 8516 u32 ext_ufs_feature; 8517 u8 mask = 0; 8518 8519 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300) 8520 return; 8521 8522 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8523 8524 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF) 8525 mask |= MASK_EE_TOO_LOW_TEMP; 8526 8527 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF) 8528 mask |= MASK_EE_TOO_HIGH_TEMP; 8529 8530 if (mask) { 8531 ufshcd_enable_ee(hba, mask); 8532 ufs_hwmon_probe(hba, mask); 8533 } 8534 } 8535 8536 static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) 8537 { 8538 u32 ext_ufs_feature; 8539 8540 if (hba->dev_info.wspecversion < 0x410) 8541 return; 8542 8543 ext_ufs_feature = get_unaligned_be32(desc_buf + 8544 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8545 if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) 8546 return; 8547 8548 atomic_set(&hba->dev_lvl_exception_count, 0); 8549 ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); 8550 } 8551 8552 static void ufshcd_set_rtt(struct ufs_hba *hba) 8553 { 8554 struct ufs_dev_info *dev_info = &hba->dev_info; 8555 u32 rtt = 0; 8556 u32 dev_rtt = 0; 8557 int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? 8558 hba->vops->max_num_rtt : hba->nortt; 8559 8560 /* RTT override makes sense only for UFS-4.0 and above */ 8561 if (dev_info->wspecversion < 0x400) 8562 return; 8563 8564 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8565 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { 8566 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); 8567 return; 8568 } 8569 8570 /* do not override if it was already written */ 8571 if (dev_rtt != DEFAULT_MAX_NUM_RTT) 8572 return; 8573 8574 rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); 8575 8576 if (rtt == dev_rtt) 8577 return; 8578 8579 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8580 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) 8581 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); 8582 } 8583 8584 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, 8585 const struct ufs_dev_quirk *fixups) 8586 { 8587 const struct ufs_dev_quirk *f; 8588 struct ufs_dev_info *dev_info = &hba->dev_info; 8589 8590 if (!fixups) 8591 return; 8592 8593 for (f = fixups; f->quirk; f++) { 8594 if ((f->wmanufacturerid == dev_info->wmanufacturerid || 8595 f->wmanufacturerid == UFS_ANY_VENDOR) && 8596 ((dev_info->model && 8597 STR_PRFX_EQUAL(f->model, dev_info->model)) || 8598 !strcmp(f->model, UFS_ANY_MODEL))) 8599 hba->dev_quirks |= f->quirk; 8600 } 8601 } 8602 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks); 8603 8604 static void ufs_fixup_device_setup(struct ufs_hba *hba) 8605 { 8606 /* fix by general quirk table */ 8607 ufshcd_fixup_dev_quirks(hba, ufs_fixups); 8608 8609 /* allow vendors to fix quirks */ 8610 ufshcd_vops_fixup_dev_quirks(hba); 8611 } 8612 8613 static void ufshcd_update_rtc(struct ufs_hba *hba) 8614 { 8615 struct timespec64 ts64; 8616 int err; 8617 u32 val; 8618 8619 ktime_get_real_ts64(&ts64); 8620 8621 if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { 8622 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); 8623 return; 8624 } 8625 8626 /* 8627 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond 8628 * 2146 is required, it is recommended to choose the relative RTC mode. 8629 */ 8630 val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; 8631 8632 /* Skip update RTC if RPM state is not RPM_ACTIVE */ 8633 if (ufshcd_rpm_get_if_active(hba) <= 0) 8634 return; 8635 8636 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, 8637 0, 0, &val); 8638 ufshcd_rpm_put(hba); 8639 8640 if (err) 8641 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); 8642 else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 8643 hba->dev_info.rtc_time_baseline = ts64.tv_sec; 8644 } 8645 8646 static void ufshcd_rtc_work(struct work_struct *work) 8647 { 8648 struct ufs_hba *hba; 8649 8650 hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); 8651 8652 /* Update RTC only when there are no requests in progress and UFSHCI is operational */ 8653 if (!ufshcd_is_ufs_dev_busy(hba) && 8654 hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && 8655 !hba->clk_gating.active_reqs) 8656 ufshcd_update_rtc(hba); 8657 8658 if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) 8659 schedule_delayed_work(&hba->ufs_rtc_update_work, 8660 msecs_to_jiffies(hba->dev_info.rtc_update_period)); 8661 } 8662 8663 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) 8664 { 8665 u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); 8666 struct ufs_dev_info *dev_info = &hba->dev_info; 8667 8668 if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { 8669 dev_info->rtc_type = UFS_RTC_ABSOLUTE; 8670 8671 /* 8672 * The concept of measuring time in Linux as the number of seconds elapsed since 8673 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st 8674 * 2010 00:00, here we need to adjust ABS baseline. 8675 */ 8676 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - 8677 mktime64(1970, 1, 1, 0, 0, 0); 8678 } else { 8679 dev_info->rtc_type = UFS_RTC_RELATIVE; 8680 dev_info->rtc_time_baseline = 0; 8681 } 8682 8683 /* 8684 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state 8685 * how to calculate the specific update period for each time unit. And we disable periodic 8686 * RTC update work, let user configure by sysfs node according to specific circumstance. 8687 */ 8688 dev_info->rtc_update_period = 0; 8689 } 8690 8691 /** 8692 * ufshcd_create_device_id - Generate unique device identifier string 8693 * @hba: per-adapter instance 8694 * @desc_buf: device descriptor buffer 8695 * 8696 * Creates a unique device ID string combining manufacturer ID, spec version, 8697 * model name, serial number (as hex), device version, and manufacture date. 8698 * 8699 * Returns: Allocated device ID string on success, NULL on failure 8700 */ 8701 static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf) 8702 { 8703 struct ufs_dev_info *dev_info = &hba->dev_info; 8704 u16 manufacture_date; 8705 u16 device_version; 8706 u8 *serial_number; 8707 char *serial_hex; 8708 char *device_id; 8709 u8 serial_index; 8710 int serial_len; 8711 int ret; 8712 8713 serial_index = desc_buf[DEVICE_DESC_PARAM_SN]; 8714 8715 ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW); 8716 if (ret < 0) { 8717 dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret); 8718 return NULL; 8719 } 8720 8721 device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]); 8722 manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]); 8723 8724 serial_len = ret; 8725 /* Allocate buffer for hex string: 2 chars per byte + null terminator */ 8726 serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL); 8727 if (!serial_hex) { 8728 kfree(serial_number); 8729 return NULL; 8730 } 8731 8732 bin2hex(serial_hex, serial_number, serial_len); 8733 8734 /* 8735 * Device ID format is ABI with secure world - do not change without firmware 8736 * coordination. 8737 */ 8738 device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X", 8739 dev_info->wmanufacturerid, dev_info->wspecversion, 8740 dev_info->model, serial_hex, device_version, 8741 manufacture_date); 8742 8743 kfree(serial_hex); 8744 kfree(serial_number); 8745 8746 if (!device_id) 8747 dev_warn(hba->dev, "Failed to allocate unique device ID\n"); 8748 8749 return device_id; 8750 } 8751 8752 static int ufs_get_device_desc(struct ufs_hba *hba) 8753 { 8754 struct ufs_dev_info *dev_info = &hba->dev_info; 8755 struct Scsi_Host *shost = hba->host; 8756 int err; 8757 u8 model_index; 8758 u8 *desc_buf; 8759 8760 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8761 if (!desc_buf) { 8762 err = -ENOMEM; 8763 goto out; 8764 } 8765 8766 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, 8767 QUERY_DESC_MAX_SIZE); 8768 if (err) { 8769 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", 8770 __func__, err); 8771 goto out; 8772 } 8773 8774 /* 8775 * getting vendor (manufacturerID) and Bank Index in big endian 8776 * format 8777 */ 8778 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | 8779 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; 8780 8781 /* getting Specification Version in big endian format */ 8782 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8783 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8784 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8785 8786 /* 8787 * According to the UFS standard, the UFS device queue depth 8788 * (bQueueDepth) must be in the range 1..255 if the shared queueing 8789 * architecture is supported. bQueueDepth is zero if the shared queueing 8790 * architecture is not supported. 8791 */ 8792 if (dev_info->bqueuedepth) 8793 shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - 8794 UFSHCD_NUM_RESERVED; 8795 else 8796 shost->cmd_per_lun = shost->can_queue; 8797 8798 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; 8799 8800 dev_info->hid_sup = get_unaligned_be32(desc_buf + 8801 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & 8802 UFS_DEV_HID_SUPPORT; 8803 8804 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; 8805 8806 err = ufshcd_read_string_desc(hba, model_index, 8807 &dev_info->model, SD_ASCII_STD); 8808 if (err < 0) { 8809 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", 8810 __func__, err); 8811 goto out; 8812 } 8813 8814 /* Generate unique device ID */ 8815 dev_info->device_id = ufshcd_create_device_id(hba, desc_buf); 8816 8817 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + 8818 desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; 8819 8820 ufs_fixup_device_setup(hba); 8821 8822 ufshcd_wb_probe(hba, desc_buf); 8823 8824 ufshcd_temp_notif_probe(hba, desc_buf); 8825 8826 if (dev_info->wspecversion >= 0x410) { 8827 hba->critical_health_count = 0; 8828 ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); 8829 } 8830 8831 ufs_init_rtc(hba, desc_buf); 8832 8833 ufshcd_device_lvl_exception_probe(hba, desc_buf); 8834 8835 /* 8836 * ufshcd_read_string_desc returns size of the string 8837 * reset the error value 8838 */ 8839 err = 0; 8840 8841 out: 8842 kfree(desc_buf); 8843 return err; 8844 } 8845 8846 static void ufs_put_device_desc(struct ufs_hba *hba) 8847 { 8848 struct ufs_dev_info *dev_info = &hba->dev_info; 8849 8850 kfree(dev_info->model); 8851 dev_info->model = NULL; 8852 kfree(dev_info->device_id); 8853 dev_info->device_id = NULL; 8854 } 8855 8856 /** 8857 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is 8858 * less than device PA_TACTIVATE time. 8859 * @hba: per-adapter instance 8860 * 8861 * Some UFS devices require host PA_TACTIVATE to be lower than device 8862 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk 8863 * for such devices. 8864 * 8865 * Return: zero on success, non-zero error value on failure. 8866 */ 8867 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) 8868 { 8869 int ret = 0; 8870 u32 granularity, peer_granularity; 8871 u32 pa_tactivate, peer_pa_tactivate; 8872 u32 pa_tactivate_us, peer_pa_tactivate_us; 8873 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; 8874 8875 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8876 &granularity); 8877 if (ret) 8878 goto out; 8879 8880 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8881 &peer_granularity); 8882 if (ret) 8883 goto out; 8884 8885 if ((granularity < PA_GRANULARITY_MIN_VAL) || 8886 (granularity > PA_GRANULARITY_MAX_VAL)) { 8887 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", 8888 __func__, granularity); 8889 return -EINVAL; 8890 } 8891 8892 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || 8893 (peer_granularity > PA_GRANULARITY_MAX_VAL)) { 8894 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", 8895 __func__, peer_granularity); 8896 return -EINVAL; 8897 } 8898 8899 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); 8900 if (ret) 8901 goto out; 8902 8903 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), 8904 &peer_pa_tactivate); 8905 if (ret) 8906 goto out; 8907 8908 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; 8909 peer_pa_tactivate_us = peer_pa_tactivate * 8910 gran_to_us_table[peer_granularity - 1]; 8911 8912 if (pa_tactivate_us >= peer_pa_tactivate_us) { 8913 u32 new_peer_pa_tactivate; 8914 8915 new_peer_pa_tactivate = pa_tactivate_us / 8916 gran_to_us_table[peer_granularity - 1]; 8917 new_peer_pa_tactivate++; 8918 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8919 new_peer_pa_tactivate); 8920 } 8921 8922 out: 8923 return ret; 8924 } 8925 8926 /** 8927 * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. 8928 * @hba: per-adapter instance 8929 * 8930 * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter 8931 * to ensure proper hibernation timing. This function retrieves the current 8932 * PA_HIBERN8TIME value and increments it by 100us. 8933 */ 8934 static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) 8935 { 8936 u32 pa_h8time; 8937 int ret; 8938 8939 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); 8940 if (ret) { 8941 dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); 8942 return; 8943 } 8944 8945 /* Increment by 1 to increase hibernation time by 100 µs */ 8946 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); 8947 if (ret) 8948 dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); 8949 } 8950 8951 static void ufshcd_tune_unipro_params(struct ufs_hba *hba) 8952 { 8953 ufshcd_vops_apply_dev_quirks(hba); 8954 8955 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) 8956 /* set 1ms timeout for PA_TACTIVATE */ 8957 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); 8958 8959 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) 8960 ufshcd_quirk_tune_host_pa_tactivate(hba); 8961 8962 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) 8963 ufshcd_quirk_override_pa_h8time(hba); 8964 8965 ufshcd_apply_valid_tx_eq_settings(hba); 8966 } 8967 8968 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) 8969 { 8970 hba->ufs_stats.hibern8_exit_cnt = 0; 8971 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 8972 hba->req_abort_count = 0; 8973 } 8974 8975 static int ufshcd_device_geo_params_init(struct ufs_hba *hba) 8976 { 8977 int err; 8978 u8 *desc_buf; 8979 8980 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8981 if (!desc_buf) { 8982 err = -ENOMEM; 8983 goto out; 8984 } 8985 8986 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, 8987 desc_buf, QUERY_DESC_MAX_SIZE); 8988 if (err) { 8989 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", 8990 __func__, err); 8991 goto out; 8992 } 8993 8994 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1) 8995 hba->dev_info.max_lu_supported = 32; 8996 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) 8997 hba->dev_info.max_lu_supported = 8; 8998 8999 hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE]; 9000 9001 out: 9002 kfree(desc_buf); 9003 return err; 9004 } 9005 9006 struct ufs_ref_clk { 9007 unsigned long freq_hz; 9008 enum ufs_ref_clk_freq val; 9009 }; 9010 9011 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = { 9012 {19200000, REF_CLK_FREQ_19_2_MHZ}, 9013 {26000000, REF_CLK_FREQ_26_MHZ}, 9014 {38400000, REF_CLK_FREQ_38_4_MHZ}, 9015 {52000000, REF_CLK_FREQ_52_MHZ}, 9016 {0, REF_CLK_FREQ_INVAL}, 9017 }; 9018 9019 static enum ufs_ref_clk_freq 9020 ufs_get_bref_clk_from_hz(unsigned long freq) 9021 { 9022 int i; 9023 9024 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++) 9025 if (ufs_ref_clk_freqs[i].freq_hz == freq) 9026 return ufs_ref_clk_freqs[i].val; 9027 9028 return REF_CLK_FREQ_INVAL; 9029 } 9030 9031 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk) 9032 { 9033 unsigned long freq; 9034 9035 freq = clk_get_rate(refclk); 9036 9037 hba->dev_ref_clk_freq = 9038 ufs_get_bref_clk_from_hz(freq); 9039 9040 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9041 dev_err(hba->dev, 9042 "invalid ref_clk setting = %ld\n", freq); 9043 } 9044 9045 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba) 9046 { 9047 int err; 9048 u32 ref_clk; 9049 u32 freq = hba->dev_ref_clk_freq; 9050 9051 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 9052 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk); 9053 9054 if (err) { 9055 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n", 9056 err); 9057 goto out; 9058 } 9059 9060 if (ref_clk == freq) 9061 goto out; /* nothing to update */ 9062 9063 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 9064 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq); 9065 9066 if (err) { 9067 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n", 9068 ufs_ref_clk_freqs[freq].freq_hz); 9069 goto out; 9070 } 9071 9072 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n", 9073 ufs_ref_clk_freqs[freq].freq_hz); 9074 9075 out: 9076 return err; 9077 } 9078 9079 static int ufshcd_device_params_init(struct ufs_hba *hba) 9080 { 9081 bool flag; 9082 int ret; 9083 9084 /* Init UFS geometry descriptor related parameters */ 9085 ret = ufshcd_device_geo_params_init(hba); 9086 if (ret) 9087 goto out; 9088 9089 /* Check and apply UFS device quirks */ 9090 ret = ufs_get_device_desc(hba); 9091 if (ret) { 9092 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", 9093 __func__, ret); 9094 goto out; 9095 } 9096 9097 ufshcd_set_rtt(hba); 9098 9099 ufshcd_get_ref_clk_gating_wait(hba); 9100 9101 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, 9102 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag)) 9103 hba->dev_info.f_power_on_wp_en = flag; 9104 9105 /* Probe maximum power mode co-supported by both UFS host and device */ 9106 if (ufshcd_get_max_pwr_mode(hba)) 9107 dev_err(hba->dev, 9108 "%s: Failed getting max supported power mode\n", 9109 __func__); 9110 out: 9111 return ret; 9112 } 9113 9114 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) 9115 { 9116 int err; 9117 struct ufs_query_req *request = NULL; 9118 struct ufs_query_res *response = NULL; 9119 struct ufs_dev_info *dev_info = &hba->dev_info; 9120 struct utp_upiu_query_v4_0 *upiu_data; 9121 9122 if (dev_info->wspecversion < 0x400 || 9123 hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) 9124 return; 9125 9126 ufshcd_dev_man_lock(hba); 9127 9128 ufshcd_init_query(hba, &request, &response, 9129 UPIU_QUERY_OPCODE_WRITE_ATTR, 9130 QUERY_ATTR_IDN_TIMESTAMP, 0, 0); 9131 9132 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 9133 9134 upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req; 9135 9136 put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); 9137 9138 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 9139 9140 if (err) 9141 dev_err(hba->dev, "%s: failed to set timestamp %d\n", 9142 __func__, err); 9143 9144 ufshcd_dev_man_unlock(hba); 9145 } 9146 9147 /** 9148 * ufshcd_add_lus - probe and add UFS logical units 9149 * @hba: per-adapter instance 9150 * 9151 * Return: 0 upon success; < 0 upon failure. 9152 */ 9153 static int ufshcd_add_lus(struct ufs_hba *hba) 9154 { 9155 int ret; 9156 9157 /* Add required well known logical units to scsi mid layer */ 9158 ret = ufshcd_scsi_add_wlus(hba); 9159 if (ret) 9160 goto out; 9161 9162 /* Initialize devfreq after UFS device is detected */ 9163 if (ufshcd_is_clkscaling_supported(hba)) { 9164 memcpy(&hba->clk_scaling.saved_pwr_info, 9165 &hba->pwr_info, 9166 sizeof(struct ufs_pa_layer_attr)); 9167 hba->clk_scaling.is_allowed = true; 9168 9169 ret = ufshcd_devfreq_init(hba); 9170 if (ret) 9171 goto out; 9172 9173 hba->clk_scaling.is_enabled = true; 9174 ufshcd_init_clk_scaling_sysfs(hba); 9175 } 9176 9177 /* 9178 * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev 9179 * pointer and hence must only be started after the WLUN pointer has 9180 * been initialized by ufshcd_scsi_add_wlus(). 9181 */ 9182 schedule_delayed_work(&hba->ufs_rtc_update_work, 9183 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 9184 9185 ufs_bsg_probe(hba); 9186 scsi_scan_host(hba->host); 9187 ufs_rpmb_probe(hba); 9188 9189 out: 9190 return ret; 9191 } 9192 9193 /* SDB - Single Doorbell */ 9194 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) 9195 { 9196 size_t ucdl_size, utrdl_size; 9197 9198 ucdl_size = ufshcd_get_ucd_size(hba) * nutrs; 9199 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, 9200 hba->ucdl_dma_addr); 9201 9202 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 9203 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 9204 hba->utrdl_dma_addr); 9205 } 9206 9207 static int ufshcd_alloc_mcq(struct ufs_hba *hba) 9208 { 9209 int ret; 9210 int old_nutrs = hba->nutrs; 9211 9212 ret = ufshcd_get_hba_mac(hba); 9213 if (ret < 0) 9214 return ret; 9215 9216 hba->nutrs = ret; 9217 ret = ufshcd_mcq_init(hba); 9218 if (ret) 9219 goto err; 9220 9221 /* 9222 * Previously allocated memory for nutrs may not be enough in MCQ mode. 9223 * Number of supported tags in MCQ mode may be larger than SDB mode. 9224 */ 9225 if (hba->nutrs != old_nutrs) { 9226 ufshcd_release_sdb_queue(hba, old_nutrs); 9227 ret = ufshcd_memory_alloc(hba); 9228 if (ret) 9229 goto err; 9230 ufshcd_host_memory_configure(hba); 9231 } 9232 9233 ret = ufshcd_mcq_memory_alloc(hba); 9234 if (ret) 9235 goto err; 9236 9237 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 9238 9239 return 0; 9240 err: 9241 hba->nutrs = old_nutrs; 9242 return ret; 9243 } 9244 9245 static void ufshcd_config_mcq(struct ufs_hba *hba) 9246 { 9247 int ret; 9248 9249 ret = ufshcd_mcq_vops_config_esi(hba); 9250 hba->mcq_esi_enabled = !ret; 9251 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); 9252 9253 ufshcd_mcq_make_queues_operational(hba); 9254 ufshcd_mcq_config_mac(hba, hba->nutrs); 9255 9256 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", 9257 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], 9258 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], 9259 hba->nutrs); 9260 } 9261 9262 static int ufshcd_post_device_init(struct ufs_hba *hba) 9263 { 9264 int ret; 9265 9266 ufshcd_tune_unipro_params(hba); 9267 9268 /* UFS device is also active now */ 9269 ufshcd_set_ufs_dev_active(hba); 9270 9271 /* Indicate that DME QoS Monitor has been reset */ 9272 atomic_set(&hba->dme_qos_notification, 0x1); 9273 if (hba->dme_qos_sysfs_handle) 9274 sysfs_notify_dirent(hba->dme_qos_sysfs_handle); 9275 9276 ufshcd_force_reset_auto_bkops(hba); 9277 9278 ufshcd_set_timestamp_attr(hba); 9279 9280 if (!hba->max_pwr_info.is_valid) 9281 return 0; 9282 9283 /* 9284 * Set the right value to bRefClkFreq before attempting to 9285 * switch to HS gears. 9286 */ 9287 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) 9288 ufshcd_set_dev_ref_clk(hba); 9289 /* Gear up to HS gear. */ 9290 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info, 9291 UFSHCD_PMC_POLICY_DONT_FORCE); 9292 if (ret) { 9293 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 9294 __func__, ret); 9295 return ret; 9296 } 9297 9298 return 0; 9299 } 9300 9301 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) 9302 { 9303 int ret; 9304 9305 WARN_ON_ONCE(!hba->scsi_host_added); 9306 9307 hba->ufshcd_state = UFSHCD_STATE_RESET; 9308 9309 ret = ufshcd_link_startup(hba); 9310 if (ret) 9311 return ret; 9312 9313 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 9314 return ret; 9315 9316 /* Debug counters initialization */ 9317 ufshcd_clear_dbg_ufs_stats(hba); 9318 9319 /* UniPro link is active now */ 9320 ufshcd_set_link_active(hba); 9321 9322 /* Reconfigure MCQ upon reset */ 9323 if (hba->mcq_enabled && !init_dev_params) { 9324 ufshcd_config_mcq(hba); 9325 ufshcd_mcq_enable(hba); 9326 } 9327 9328 /* Verify device initialization by sending NOP OUT UPIU */ 9329 ret = ufshcd_verify_dev_init(hba); 9330 if (ret) 9331 return ret; 9332 9333 /* Initiate UFS initialization, and waiting until completion */ 9334 ret = ufshcd_complete_dev_init(hba); 9335 if (ret) 9336 return ret; 9337 9338 /* 9339 * Initialize UFS device parameters used by driver, these 9340 * parameters are associated with UFS descriptors. 9341 */ 9342 if (init_dev_params) { 9343 ret = ufshcd_device_params_init(hba); 9344 if (ret) 9345 return ret; 9346 if (is_mcq_supported(hba) && 9347 hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 9348 ufshcd_config_mcq(hba); 9349 ufshcd_mcq_enable(hba); 9350 } 9351 } 9352 9353 return ufshcd_post_device_init(hba); 9354 } 9355 9356 /** 9357 * ufshcd_probe_hba - probe hba to detect device and initialize it 9358 * @hba: per-adapter instance 9359 * @init_dev_params: whether or not to call ufshcd_device_params_init(). 9360 * 9361 * Execute link-startup and verify device initialization 9362 * 9363 * Return: 0 upon success; < 0 upon failure. 9364 */ 9365 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) 9366 { 9367 int ret; 9368 9369 if (!hba->pm_op_in_progress && 9370 (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { 9371 /* Reset the device and controller before doing reinit */ 9372 ufshcd_device_reset(hba); 9373 ufs_put_device_desc(hba); 9374 ufshcd_hba_stop(hba); 9375 ret = ufshcd_hba_enable(hba); 9376 if (ret) { 9377 dev_err(hba->dev, "Host controller enable failed\n"); 9378 ufshcd_print_evt_hist(hba); 9379 ufshcd_print_host_state(hba); 9380 return ret; 9381 } 9382 9383 /* Reinit the device */ 9384 ret = ufshcd_device_init(hba, init_dev_params); 9385 if (ret) 9386 return ret; 9387 } 9388 9389 ufshcd_print_pwr_info(hba); 9390 9391 /* 9392 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) 9393 * and for removable UFS card as well, hence always set the parameter. 9394 * Note: Error handler may issue the device reset hence resetting 9395 * bActiveICCLevel as well so it is always safe to set this here. 9396 */ 9397 ufshcd_set_active_icc_lvl(hba); 9398 9399 /* Enable UFS Write Booster if supported */ 9400 ufshcd_configure_wb(hba); 9401 9402 if (hba->ee_usr_mask) 9403 ufshcd_write_ee_control(hba); 9404 ufshcd_configure_auto_hibern8(hba); 9405 9406 return 0; 9407 } 9408 9409 /** 9410 * ufshcd_async_scan - asynchronous execution for probing hba 9411 * @data: data pointer to pass to this function 9412 * @cookie: cookie data 9413 */ 9414 static void ufshcd_async_scan(void *data, async_cookie_t cookie) 9415 { 9416 struct ufs_hba *hba = (struct ufs_hba *)data; 9417 ktime_t probe_start; 9418 int ret; 9419 9420 down(&hba->host_sem); 9421 /* Initialize hba, detect and initialize UFS device */ 9422 probe_start = ktime_get(); 9423 ret = ufshcd_probe_hba(hba, true); 9424 ufshcd_process_probe_result(hba, probe_start, ret); 9425 up(&hba->host_sem); 9426 if (ret) 9427 goto out; 9428 9429 /* Probe and add UFS logical units */ 9430 ret = ufshcd_add_lus(hba); 9431 9432 out: 9433 pm_runtime_put_sync(hba->dev); 9434 9435 if (ret) 9436 dev_err(hba->dev, "%s failed: %d\n", __func__, ret); 9437 } 9438 9439 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) 9440 { 9441 struct ufs_hba *hba = shost_priv(scmd->device->host); 9442 9443 if (!hba->system_suspending) { 9444 /* Activate the error handler in the SCSI core. */ 9445 return SCSI_EH_NOT_HANDLED; 9446 } 9447 9448 /* 9449 * If we get here we know that no TMFs are outstanding and also that 9450 * the only pending command is a START STOP UNIT command. Handle the 9451 * timeout of that command directly to prevent a deadlock between 9452 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). 9453 */ 9454 ufshcd_link_recovery(hba); 9455 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", 9456 __func__, hba->outstanding_tasks); 9457 9458 return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; 9459 } 9460 9461 static const struct attribute_group *ufshcd_driver_groups[] = { 9462 &ufs_sysfs_unit_descriptor_group, 9463 &ufs_sysfs_lun_attributes_group, 9464 NULL, 9465 }; 9466 9467 static struct ufs_hba_variant_params ufs_hba_vps = { 9468 .hba_enable_delay_us = 1000, 9469 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40), 9470 .devfreq_profile.polling_ms = 100, 9471 .devfreq_profile.target = ufshcd_devfreq_target, 9472 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status, 9473 .ondemand_data.upthreshold = 70, 9474 .ondemand_data.downdifferential = 5, 9475 }; 9476 9477 static const struct scsi_host_template ufshcd_driver_template = { 9478 .module = THIS_MODULE, 9479 .name = UFSHCD, 9480 .proc_name = UFSHCD, 9481 .map_queues = ufshcd_map_queues, 9482 .cmd_size = sizeof(struct ufshcd_lrb), 9483 .init_cmd_priv = ufshcd_init_cmd_priv, 9484 .queuecommand = ufshcd_queuecommand, 9485 .queue_reserved_command = ufshcd_queue_reserved_command, 9486 .nr_reserved_cmds = UFSHCD_NUM_RESERVED, 9487 .mq_poll = ufshcd_poll, 9488 .sdev_init = ufshcd_sdev_init, 9489 .sdev_configure = ufshcd_sdev_configure, 9490 .sdev_destroy = ufshcd_sdev_destroy, 9491 .change_queue_depth = ufshcd_change_queue_depth, 9492 .eh_abort_handler = ufshcd_abort, 9493 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 9494 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 9495 .eh_timed_out = ufshcd_eh_timed_out, 9496 .this_id = -1, 9497 .sg_tablesize = SG_ALL, 9498 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, 9499 .max_sectors = SZ_1M / SECTOR_SIZE, 9500 .max_host_blocked = 1, 9501 .host_tagset = true, 9502 .track_queue_depth = 1, 9503 .skip_settle_delay = 1, 9504 .sdev_groups = ufshcd_driver_groups, 9505 }; 9506 9507 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 9508 int ua) 9509 { 9510 int ret; 9511 9512 if (!vreg) 9513 return 0; 9514 9515 /* 9516 * "set_load" operation shall be required on those regulators 9517 * which specifically configured current limitation. Otherwise 9518 * zero max_uA may cause unexpected behavior when regulator is 9519 * enabled or set as high power mode. 9520 */ 9521 if (!vreg->max_uA) 9522 return 0; 9523 9524 ret = regulator_set_load(vreg->reg, ua); 9525 if (ret < 0) { 9526 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 9527 __func__, vreg->name, ua, ret); 9528 } 9529 9530 return ret; 9531 } 9532 9533 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 9534 struct ufs_vreg *vreg) 9535 { 9536 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 9537 } 9538 9539 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 9540 struct ufs_vreg *vreg) 9541 { 9542 if (!vreg) 9543 return 0; 9544 9545 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 9546 } 9547 9548 static int ufshcd_config_vreg(struct device *dev, 9549 struct ufs_vreg *vreg, bool on) 9550 { 9551 if (regulator_count_voltages(vreg->reg) <= 0) 9552 return 0; 9553 9554 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0); 9555 } 9556 9557 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 9558 { 9559 int ret = 0; 9560 9561 if (!vreg || vreg->enabled) 9562 goto out; 9563 9564 ret = ufshcd_config_vreg(dev, vreg, true); 9565 if (!ret) 9566 ret = regulator_enable(vreg->reg); 9567 9568 if (!ret) 9569 vreg->enabled = true; 9570 else 9571 dev_err(dev, "%s: %s enable failed, err=%d\n", 9572 __func__, vreg->name, ret); 9573 out: 9574 return ret; 9575 } 9576 9577 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 9578 { 9579 int ret = 0; 9580 9581 if (!vreg || !vreg->enabled || vreg->always_on) 9582 goto out; 9583 9584 ret = regulator_disable(vreg->reg); 9585 9586 if (!ret) { 9587 /* ignore errors on applying disable config */ 9588 ufshcd_config_vreg(dev, vreg, false); 9589 vreg->enabled = false; 9590 } else { 9591 dev_err(dev, "%s: %s disable failed, err=%d\n", 9592 __func__, vreg->name, ret); 9593 } 9594 out: 9595 return ret; 9596 } 9597 9598 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 9599 { 9600 int ret = 0; 9601 struct device *dev = hba->dev; 9602 struct ufs_vreg_info *info = &hba->vreg_info; 9603 9604 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 9605 if (ret) 9606 goto out; 9607 9608 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 9609 if (ret) 9610 goto out; 9611 9612 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 9613 9614 out: 9615 if (ret) { 9616 ufshcd_toggle_vreg(dev, info->vccq2, false); 9617 ufshcd_toggle_vreg(dev, info->vccq, false); 9618 ufshcd_toggle_vreg(dev, info->vcc, false); 9619 } 9620 return ret; 9621 } 9622 9623 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 9624 { 9625 struct ufs_vreg_info *info = &hba->vreg_info; 9626 9627 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 9628 } 9629 9630 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 9631 { 9632 int ret = 0; 9633 9634 if (!vreg) 9635 goto out; 9636 9637 vreg->reg = devm_regulator_get(dev, vreg->name); 9638 if (IS_ERR(vreg->reg)) { 9639 ret = PTR_ERR(vreg->reg); 9640 dev_err(dev, "%s: %s get failed, err=%d\n", 9641 __func__, vreg->name, ret); 9642 } 9643 out: 9644 return ret; 9645 } 9646 EXPORT_SYMBOL_GPL(ufshcd_get_vreg); 9647 9648 static int ufshcd_init_vreg(struct ufs_hba *hba) 9649 { 9650 int ret = 0; 9651 struct device *dev = hba->dev; 9652 struct ufs_vreg_info *info = &hba->vreg_info; 9653 9654 ret = ufshcd_get_vreg(dev, info->vcc); 9655 if (ret) 9656 goto out; 9657 9658 ret = ufshcd_get_vreg(dev, info->vccq); 9659 if (!ret) 9660 ret = ufshcd_get_vreg(dev, info->vccq2); 9661 out: 9662 return ret; 9663 } 9664 9665 static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 9666 { 9667 struct ufs_vreg_info *info = &hba->vreg_info; 9668 9669 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 9670 } 9671 9672 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 9673 { 9674 int ret = 0; 9675 struct ufs_clk_info *clki; 9676 struct list_head *head = &hba->clk_list_head; 9677 ktime_t start = ktime_get(); 9678 bool clk_state_changed = false; 9679 9680 if (list_empty(head)) 9681 goto out; 9682 9683 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); 9684 if (ret) 9685 return ret; 9686 9687 list_for_each_entry(clki, head, list) { 9688 if (!IS_ERR_OR_NULL(clki->clk)) { 9689 /* 9690 * Don't disable clocks which are needed 9691 * to keep the link active. 9692 */ 9693 if (ufshcd_is_link_active(hba) && 9694 clki->keep_link_active) 9695 continue; 9696 9697 clk_state_changed = on ^ clki->enabled; 9698 if (on && !clki->enabled) { 9699 ret = clk_prepare_enable(clki->clk); 9700 if (ret) { 9701 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 9702 __func__, clki->name, ret); 9703 goto out; 9704 } 9705 } else if (!on && clki->enabled) { 9706 clk_disable_unprepare(clki->clk); 9707 } 9708 clki->enabled = on; 9709 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 9710 clki->name, on ? "en" : "dis"); 9711 } 9712 } 9713 9714 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); 9715 if (ret) 9716 return ret; 9717 9718 if (!ufshcd_is_clkscaling_supported(hba)) 9719 ufshcd_pm_qos_update(hba, on); 9720 out: 9721 if (ret) { 9722 list_for_each_entry(clki, head, list) { 9723 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 9724 clk_disable_unprepare(clki->clk); 9725 } 9726 } else if (!ret && on && hba->clk_gating.is_initialized) { 9727 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) 9728 hba->clk_gating.state = CLKS_ON; 9729 trace_ufshcd_clk_gating(hba, 9730 hba->clk_gating.state); 9731 } 9732 9733 if (clk_state_changed) 9734 trace_ufshcd_profile_clk_gating(hba, 9735 (on ? "on" : "off"), 9736 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 9737 return ret; 9738 } 9739 9740 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba) 9741 { 9742 u32 freq; 9743 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq); 9744 9745 if (ret) { 9746 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret); 9747 return REF_CLK_FREQ_INVAL; 9748 } 9749 9750 return ufs_get_bref_clk_from_hz(freq); 9751 } 9752 9753 static int ufshcd_init_clocks(struct ufs_hba *hba) 9754 { 9755 int ret = 0; 9756 struct ufs_clk_info *clki; 9757 struct device *dev = hba->dev; 9758 struct list_head *head = &hba->clk_list_head; 9759 9760 if (list_empty(head)) 9761 goto out; 9762 9763 list_for_each_entry(clki, head, list) { 9764 if (!clki->name) 9765 continue; 9766 9767 clki->clk = devm_clk_get(dev, clki->name); 9768 if (IS_ERR(clki->clk)) { 9769 ret = PTR_ERR(clki->clk); 9770 dev_err(dev, "%s: %s clk get failed, %d\n", 9771 __func__, clki->name, ret); 9772 goto out; 9773 } 9774 9775 /* 9776 * Parse device ref clk freq as per device tree "ref_clk". 9777 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL 9778 * in ufshcd_alloc_host(). 9779 */ 9780 if (!strcmp(clki->name, "ref_clk")) 9781 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk); 9782 9783 if (clki->max_freq) { 9784 ret = clk_set_rate(clki->clk, clki->max_freq); 9785 if (ret) { 9786 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 9787 __func__, clki->name, 9788 clki->max_freq, ret); 9789 goto out; 9790 } 9791 clki->curr_freq = clki->max_freq; 9792 } 9793 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 9794 clki->name, clk_get_rate(clki->clk)); 9795 } 9796 9797 /* Set Max. frequency for all clocks */ 9798 if (hba->use_pm_opp) { 9799 ret = ufshcd_opp_set_rate(hba, ULONG_MAX); 9800 if (ret) { 9801 dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, 9802 ret); 9803 goto out; 9804 } 9805 } 9806 9807 out: 9808 return ret; 9809 } 9810 9811 static int ufshcd_variant_hba_init(struct ufs_hba *hba) 9812 { 9813 int err = 0; 9814 9815 if (!hba->vops) 9816 goto out; 9817 9818 err = ufshcd_vops_init(hba); 9819 if (err) 9820 dev_err_probe(hba->dev, err, 9821 "%s: variant %s init failed with err %d\n", 9822 __func__, ufshcd_get_var_name(hba), err); 9823 out: 9824 return err; 9825 } 9826 9827 static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 9828 { 9829 if (!hba->vops) 9830 return; 9831 9832 ufshcd_vops_exit(hba); 9833 } 9834 9835 static int ufshcd_hba_init(struct ufs_hba *hba) 9836 { 9837 int err; 9838 9839 /* 9840 * Handle host controller power separately from the UFS device power 9841 * rails as it will help controlling the UFS host controller power 9842 * collapse easily which is different than UFS device power collapse. 9843 * Also, enable the host controller power before we go ahead with rest 9844 * of the initialization here. 9845 */ 9846 err = ufshcd_init_hba_vreg(hba); 9847 if (err) 9848 goto out; 9849 9850 err = ufshcd_setup_hba_vreg(hba, true); 9851 if (err) 9852 goto out; 9853 9854 err = ufshcd_init_clocks(hba); 9855 if (err) 9856 goto out_disable_hba_vreg; 9857 9858 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9859 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba); 9860 9861 err = ufshcd_setup_clocks(hba, true); 9862 if (err) 9863 goto out_disable_hba_vreg; 9864 9865 err = ufshcd_init_vreg(hba); 9866 if (err) 9867 goto out_disable_clks; 9868 9869 err = ufshcd_setup_vreg(hba, true); 9870 if (err) 9871 goto out_disable_clks; 9872 9873 err = ufshcd_variant_hba_init(hba); 9874 if (err) 9875 goto out_disable_vreg; 9876 9877 ufs_debugfs_hba_init(hba); 9878 ufs_fault_inject_hba_init(hba); 9879 9880 hba->is_powered = true; 9881 goto out; 9882 9883 out_disable_vreg: 9884 ufshcd_setup_vreg(hba, false); 9885 out_disable_clks: 9886 ufshcd_setup_clocks(hba, false); 9887 out_disable_hba_vreg: 9888 ufshcd_setup_hba_vreg(hba, false); 9889 out: 9890 return err; 9891 } 9892 9893 static void ufshcd_hba_exit(struct ufs_hba *hba) 9894 { 9895 if (hba->is_powered) { 9896 ufshcd_pm_qos_exit(hba); 9897 ufshcd_exit_clk_scaling(hba); 9898 ufshcd_exit_clk_gating(hba); 9899 if (hba->eh_wq) 9900 destroy_workqueue(hba->eh_wq); 9901 ufs_debugfs_hba_exit(hba); 9902 ufshcd_variant_hba_exit(hba); 9903 ufshcd_setup_vreg(hba, false); 9904 ufshcd_setup_clocks(hba, false); 9905 ufshcd_setup_hba_vreg(hba, false); 9906 hba->is_powered = false; 9907 ufs_put_device_desc(hba); 9908 } 9909 sysfs_put(hba->dme_qos_sysfs_handle); 9910 } 9911 9912 static int ufshcd_execute_start_stop(struct scsi_device *sdev, 9913 enum ufs_dev_pwr_mode pwr_mode, 9914 struct scsi_sense_hdr *sshdr) 9915 { 9916 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; 9917 struct scsi_failure failure_defs[] = { 9918 { 9919 .allowed = 2, 9920 .result = SCMD_FAILURE_RESULT_ANY, 9921 }, 9922 }; 9923 struct scsi_failures failures = { 9924 .failure_definitions = failure_defs, 9925 }; 9926 const struct scsi_exec_args args = { 9927 .failures = &failures, 9928 .sshdr = sshdr, 9929 .req_flags = BLK_MQ_REQ_PM, 9930 .scmd_flags = SCMD_FAIL_IF_RECOVERING, 9931 }; 9932 9933 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, 9934 /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0, 9935 &args); 9936 } 9937 9938 /** 9939 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 9940 * power mode 9941 * @hba: per adapter instance 9942 * @pwr_mode: device power mode to set 9943 * 9944 * Return: 0 if requested power mode is set successfully; 9945 * < 0 if failed to set the requested power mode. 9946 */ 9947 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 9948 enum ufs_dev_pwr_mode pwr_mode) 9949 { 9950 struct scsi_sense_hdr sshdr; 9951 struct scsi_device *sdp; 9952 unsigned long flags; 9953 int ret; 9954 9955 spin_lock_irqsave(hba->host->host_lock, flags); 9956 sdp = hba->ufs_device_wlun; 9957 if (sdp && scsi_device_online(sdp)) 9958 ret = scsi_device_get(sdp); 9959 else 9960 ret = -ENODEV; 9961 spin_unlock_irqrestore(hba->host->host_lock, flags); 9962 9963 if (ret) 9964 return ret; 9965 9966 /* 9967 * If scsi commands fail, the scsi mid-layer schedules scsi error- 9968 * handling, which would wait for host to be resumed. Since we know 9969 * we are functional while we are here, skip host resume in error 9970 * handling context. 9971 */ 9972 hba->host->eh_noresume = 1; 9973 9974 /* 9975 * Current function would be generally called from the power management 9976 * callbacks hence set the RQF_PM flag so that it doesn't resume the 9977 * already suspended childs. 9978 */ 9979 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); 9980 if (ret) { 9981 sdev_printk(KERN_WARNING, sdp, 9982 "START_STOP failed for power mode: %d, result %x\n", 9983 pwr_mode, ret); 9984 if (ret > 0) { 9985 if (scsi_sense_valid(&sshdr)) 9986 scsi_print_sense_hdr(sdp, NULL, &sshdr); 9987 ret = -EIO; 9988 } 9989 } else { 9990 hba->curr_dev_pwr_mode = pwr_mode; 9991 } 9992 9993 scsi_device_put(sdp); 9994 hba->host->eh_noresume = 0; 9995 return ret; 9996 } 9997 9998 static int ufshcd_link_state_transition(struct ufs_hba *hba, 9999 enum uic_link_state req_link_state, 10000 bool check_for_bkops) 10001 { 10002 int ret = 0; 10003 10004 if (req_link_state == hba->uic_link_state) 10005 return 0; 10006 10007 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 10008 ret = ufshcd_uic_hibern8_enter(hba); 10009 if (!ret) { 10010 ufshcd_set_link_hibern8(hba); 10011 } else { 10012 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 10013 __func__, ret); 10014 goto out; 10015 } 10016 } 10017 /* 10018 * If autobkops is enabled, link can't be turned off because 10019 * turning off the link would also turn off the device, except in the 10020 * case of DeepSleep where the device is expected to remain powered. 10021 */ 10022 else if ((req_link_state == UIC_LINK_OFF_STATE) && 10023 (!check_for_bkops || !hba->auto_bkops_enabled)) { 10024 /* 10025 * Let's make sure that link is in low power mode, we are doing 10026 * this currently by putting the link in Hibern8. Otherway to 10027 * put the link in low power mode is to send the DME end point 10028 * to device and then send the DME reset command to local 10029 * unipro. But putting the link in hibern8 is much faster. 10030 * 10031 * Note also that putting the link in Hibern8 is a requirement 10032 * for entering DeepSleep. 10033 */ 10034 ret = ufshcd_uic_hibern8_enter(hba); 10035 if (ret) { 10036 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 10037 __func__, ret); 10038 goto out; 10039 } 10040 /* 10041 * Change controller state to "reset state" which 10042 * should also put the link in off/reset state 10043 */ 10044 ufshcd_hba_stop(hba); 10045 /* 10046 * TODO: Check if we need any delay to make sure that 10047 * controller is reset 10048 */ 10049 ufshcd_set_link_off(hba); 10050 } 10051 10052 out: 10053 return ret; 10054 } 10055 10056 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 10057 { 10058 bool vcc_off = false; 10059 10060 /* 10061 * It seems some UFS devices may keep drawing more than sleep current 10062 * (atleast for 500us) from UFS rails (especially from VCCQ rail). 10063 * To avoid this situation, add 2ms delay before putting these UFS 10064 * rails in LPM mode. 10065 */ 10066 if (!ufshcd_is_link_active(hba) && 10067 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM) 10068 usleep_range(2000, 2100); 10069 10070 /* 10071 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 10072 * power. 10073 * 10074 * If UFS device and link is in OFF state, all power supplies (VCC, 10075 * VCCQ, VCCQ2) can be turned off if power on write protect is not 10076 * required. If UFS link is inactive (Hibern8 or OFF state) and device 10077 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 10078 * 10079 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 10080 * in low power state which would save some power. 10081 * 10082 * If Write Booster is enabled and the device needs to flush the WB 10083 * buffer OR if bkops status is urgent for WB, keep Vcc on. 10084 */ 10085 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 10086 !hba->dev_info.is_lu_power_on_wp) { 10087 ufshcd_setup_vreg(hba, false); 10088 vcc_off = true; 10089 } else if (!ufshcd_is_ufs_dev_active(hba)) { 10090 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 10091 vcc_off = true; 10092 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { 10093 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 10094 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 10095 } 10096 } 10097 10098 /* 10099 * All UFS devices require delay after VCC power rail is turned-off. 10100 */ 10101 if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) 10102 usleep_range(hba->vcc_off_delay_us, 10103 hba->vcc_off_delay_us + 100); 10104 } 10105 10106 #ifdef CONFIG_PM 10107 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 10108 { 10109 bool vcc_on = false; 10110 int ret = 0; 10111 10112 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 10113 !hba->dev_info.is_lu_power_on_wp) { 10114 ret = ufshcd_setup_vreg(hba, true); 10115 vcc_on = true; 10116 } else if (!ufshcd_is_ufs_dev_active(hba)) { 10117 if (!ufshcd_is_link_active(hba)) { 10118 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 10119 if (ret) 10120 goto vcc_disable; 10121 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 10122 if (ret) 10123 goto vccq_lpm; 10124 } 10125 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 10126 vcc_on = true; 10127 } 10128 goto out; 10129 10130 vccq_lpm: 10131 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 10132 vcc_disable: 10133 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 10134 out: 10135 /* 10136 * On platforms with a slow VCC ramp-up, a delay is needed after 10137 * turning on VCC to ensure the voltage is stable before the 10138 * reference clock is enabled. 10139 */ 10140 if (hba->quirks & UFSHCD_QUIRK_VCC_ON_DELAY && !ret && vcc_on && 10141 hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) 10142 usleep_range(1000, 1100); 10143 10144 return ret; 10145 } 10146 #endif /* CONFIG_PM */ 10147 10148 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 10149 { 10150 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 10151 ufshcd_setup_hba_vreg(hba, false); 10152 } 10153 10154 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 10155 { 10156 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 10157 ufshcd_setup_hba_vreg(hba, true); 10158 } 10159 10160 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 10161 { 10162 int ret = 0; 10163 bool check_for_bkops; 10164 enum ufs_pm_level pm_lvl; 10165 enum ufs_dev_pwr_mode req_dev_pwr_mode; 10166 enum uic_link_state req_link_state; 10167 10168 hba->pm_op_in_progress = true; 10169 if (pm_op != UFS_SHUTDOWN_PM) { 10170 pm_lvl = pm_op == UFS_RUNTIME_PM ? 10171 hba->rpm_lvl : hba->spm_lvl; 10172 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 10173 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 10174 } else { 10175 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 10176 req_link_state = UIC_LINK_OFF_STATE; 10177 } 10178 10179 /* 10180 * If we can't transition into any of the low power modes 10181 * just gate the clocks. 10182 */ 10183 ufshcd_hold(hba); 10184 hba->clk_gating.is_suspended = true; 10185 10186 if (ufshcd_is_clkscaling_supported(hba)) 10187 ufshcd_clk_scaling_suspend(hba, true); 10188 10189 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 10190 req_link_state == UIC_LINK_ACTIVE_STATE) { 10191 ufshcd_disable_auto_bkops(hba); 10192 flush_work(&hba->eeh_work); 10193 goto vops_suspend; 10194 } 10195 10196 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 10197 (req_link_state == hba->uic_link_state)) 10198 goto enable_scaling; 10199 10200 /* UFS device & link must be active before we enter in this function */ 10201 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 10202 /* Wait err handler finish or trigger err recovery */ 10203 if (!ufshcd_eh_in_progress(hba)) 10204 ufshcd_force_error_recovery(hba); 10205 ret = -EBUSY; 10206 goto enable_scaling; 10207 } 10208 10209 if (pm_op == UFS_RUNTIME_PM) { 10210 if (ufshcd_can_autobkops_during_suspend(hba)) { 10211 /* 10212 * The device is idle with no requests in the queue, 10213 * allow background operations if bkops status shows 10214 * that performance might be impacted. 10215 */ 10216 ret = ufshcd_bkops_ctrl(hba); 10217 if (ret) { 10218 /* 10219 * If return err in suspend flow, IO will hang. 10220 * Trigger error handler and break suspend for 10221 * error recovery. 10222 */ 10223 ufshcd_force_error_recovery(hba); 10224 ret = -EBUSY; 10225 goto enable_scaling; 10226 } 10227 } else { 10228 /* make sure that auto bkops is disabled */ 10229 ufshcd_disable_auto_bkops(hba); 10230 } 10231 /* 10232 * If device needs to do BKOP or WB buffer flush during 10233 * Hibern8, keep device power mode as "active power mode" 10234 * and VCC supply. 10235 */ 10236 hba->dev_info.b_rpm_dev_flush_capable = 10237 hba->auto_bkops_enabled || 10238 (((req_link_state == UIC_LINK_HIBERN8_STATE) || 10239 ((req_link_state == UIC_LINK_ACTIVE_STATE) && 10240 ufshcd_is_auto_hibern8_enabled(hba))) && 10241 ufshcd_wb_need_flush(hba)); 10242 } 10243 10244 flush_work(&hba->eeh_work); 10245 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10246 10247 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10248 if (ret) 10249 goto enable_scaling; 10250 10251 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { 10252 if (pm_op != UFS_RUNTIME_PM) 10253 /* ensure that bkops is disabled */ 10254 ufshcd_disable_auto_bkops(hba); 10255 10256 if (!hba->dev_info.b_rpm_dev_flush_capable) { 10257 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 10258 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10259 /* 10260 * If return err in suspend flow, IO will hang. 10261 * Trigger error handler and break suspend for 10262 * error recovery. 10263 */ 10264 ufshcd_force_error_recovery(hba); 10265 ret = -EBUSY; 10266 } 10267 if (ret) 10268 goto enable_scaling; 10269 } 10270 } 10271 10272 /* 10273 * In the case of DeepSleep, the device is expected to remain powered 10274 * with the link off, so do not check for bkops. 10275 */ 10276 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); 10277 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); 10278 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10279 /* 10280 * If return err in suspend flow, IO will hang. 10281 * Trigger error handler and break suspend for 10282 * error recovery. 10283 */ 10284 ufshcd_force_error_recovery(hba); 10285 ret = -EBUSY; 10286 } 10287 if (ret) 10288 goto set_dev_active; 10289 10290 vops_suspend: 10291 /* 10292 * Call vendor specific suspend callback. As these callbacks may access 10293 * vendor specific host controller register space call them before the 10294 * host clocks are ON. 10295 */ 10296 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10297 if (ret) 10298 goto set_link_active; 10299 10300 goto out; 10301 10302 set_link_active: 10303 /* 10304 * Device hardware reset is required to exit DeepSleep. Also, for 10305 * DeepSleep, the link is off so host reset and restore will be done 10306 * further below. 10307 */ 10308 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10309 ufshcd_device_reset(hba); 10310 WARN_ON(!ufshcd_is_link_off(hba)); 10311 } 10312 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 10313 ufshcd_set_link_active(hba); 10314 else if (ufshcd_is_link_off(hba)) 10315 ufshcd_host_reset_and_restore(hba); 10316 set_dev_active: 10317 /* Can also get here needing to exit DeepSleep */ 10318 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10319 ufshcd_device_reset(hba); 10320 ufshcd_host_reset_and_restore(hba); 10321 } 10322 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 10323 ufshcd_disable_auto_bkops(hba); 10324 enable_scaling: 10325 if (ufshcd_is_clkscaling_supported(hba)) 10326 ufshcd_clk_scaling_suspend(hba, false); 10327 10328 hba->dev_info.b_rpm_dev_flush_capable = false; 10329 out: 10330 if (hba->dev_info.b_rpm_dev_flush_capable) { 10331 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, 10332 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS)); 10333 } 10334 10335 if (ret) { 10336 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); 10337 hba->clk_gating.is_suspended = false; 10338 ufshcd_release(hba); 10339 } 10340 hba->pm_op_in_progress = false; 10341 return ret; 10342 } 10343 10344 #ifdef CONFIG_PM 10345 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 10346 { 10347 int ret; 10348 enum uic_link_state old_link_state = hba->uic_link_state; 10349 10350 hba->pm_op_in_progress = true; 10351 10352 /* 10353 * Call vendor specific resume callback. As these callbacks may access 10354 * vendor specific host controller register space call them when the 10355 * host clocks are ON. 10356 */ 10357 ret = ufshcd_vops_resume(hba, pm_op); 10358 if (ret) 10359 goto out; 10360 10361 /* For DeepSleep, the only supported option is to have the link off */ 10362 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); 10363 10364 if (ufshcd_is_link_hibern8(hba)) { 10365 ret = ufshcd_uic_hibern8_exit(hba); 10366 if (!ret) { 10367 ufshcd_set_link_active(hba); 10368 } else { 10369 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 10370 __func__, ret); 10371 /* 10372 * If the h8 exit fails during the runtime resume 10373 * process, it becomes stuck and cannot be recovered 10374 * through the error handler. To fix this, use link 10375 * recovery instead of the error handler. 10376 */ 10377 ret = ufshcd_link_recovery(hba); 10378 if (ret) 10379 goto vendor_suspend; 10380 } 10381 } else if (ufshcd_is_link_off(hba)) { 10382 /* 10383 * A full initialization of the host and the device is 10384 * required since the link was put to off during suspend. 10385 * Note, in the case of DeepSleep, the device will exit 10386 * DeepSleep due to device reset. 10387 */ 10388 ret = ufshcd_reset_and_restore(hba); 10389 /* 10390 * ufshcd_reset_and_restore() should have already 10391 * set the link state as active 10392 */ 10393 if (ret || !ufshcd_is_link_active(hba)) 10394 goto vendor_suspend; 10395 } 10396 10397 if (!ufshcd_is_ufs_dev_active(hba)) { 10398 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 10399 if (ret) 10400 goto set_old_link_state; 10401 ufshcd_set_timestamp_attr(hba); 10402 schedule_delayed_work(&hba->ufs_rtc_update_work, 10403 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 10404 } 10405 10406 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) 10407 ufshcd_enable_auto_bkops(hba); 10408 else 10409 /* 10410 * If BKOPs operations are urgently needed at this moment then 10411 * keep auto-bkops enabled or else disable it. 10412 */ 10413 ufshcd_bkops_ctrl(hba); 10414 10415 if (hba->ee_usr_mask) 10416 ufshcd_write_ee_control(hba); 10417 10418 if (ufshcd_is_clkscaling_supported(hba)) 10419 ufshcd_clk_scaling_suspend(hba, false); 10420 10421 if (hba->dev_info.b_rpm_dev_flush_capable) { 10422 hba->dev_info.b_rpm_dev_flush_capable = false; 10423 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); 10424 } 10425 10426 ufshcd_configure_auto_hibern8(hba); 10427 10428 goto out; 10429 10430 set_old_link_state: 10431 ufshcd_link_state_transition(hba, old_link_state, 0); 10432 vendor_suspend: 10433 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10434 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10435 out: 10436 if (ret) 10437 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret); 10438 hba->clk_gating.is_suspended = false; 10439 ufshcd_release(hba); 10440 hba->pm_op_in_progress = false; 10441 return ret; 10442 } 10443 10444 static int ufshcd_wl_runtime_suspend(struct device *dev) 10445 { 10446 struct scsi_device *sdev = to_scsi_device(dev); 10447 struct ufs_hba *hba; 10448 int ret; 10449 ktime_t start = ktime_get(); 10450 10451 hba = shost_priv(sdev->host); 10452 10453 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM); 10454 if (ret) 10455 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10456 10457 trace_ufshcd_wl_runtime_suspend(hba, ret, 10458 ktime_to_us(ktime_sub(ktime_get(), start)), 10459 hba->curr_dev_pwr_mode, hba->uic_link_state); 10460 10461 return ret; 10462 } 10463 10464 static int ufshcd_wl_runtime_resume(struct device *dev) 10465 { 10466 struct scsi_device *sdev = to_scsi_device(dev); 10467 struct ufs_hba *hba; 10468 int ret = 0; 10469 ktime_t start = ktime_get(); 10470 10471 hba = shost_priv(sdev->host); 10472 10473 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM); 10474 if (ret) 10475 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10476 10477 trace_ufshcd_wl_runtime_resume(hba, ret, 10478 ktime_to_us(ktime_sub(ktime_get(), start)), 10479 hba->curr_dev_pwr_mode, hba->uic_link_state); 10480 10481 return ret; 10482 } 10483 #endif 10484 10485 #ifdef CONFIG_PM_SLEEP 10486 static int ufshcd_wl_suspend(struct device *dev) 10487 { 10488 struct scsi_device *sdev = to_scsi_device(dev); 10489 struct ufs_hba *hba; 10490 int ret = 0; 10491 ktime_t start = ktime_get(); 10492 10493 hba = shost_priv(sdev->host); 10494 down(&hba->host_sem); 10495 hba->system_suspending = true; 10496 10497 if (pm_runtime_suspended(dev)) 10498 goto out; 10499 10500 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM); 10501 if (ret) { 10502 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10503 up(&hba->host_sem); 10504 } 10505 10506 out: 10507 if (!ret) 10508 hba->is_sys_suspended = true; 10509 trace_ufshcd_wl_suspend(hba, ret, 10510 ktime_to_us(ktime_sub(ktime_get(), start)), 10511 hba->curr_dev_pwr_mode, hba->uic_link_state); 10512 10513 return ret; 10514 } 10515 10516 static int ufshcd_wl_resume(struct device *dev) 10517 { 10518 struct scsi_device *sdev = to_scsi_device(dev); 10519 struct ufs_hba *hba; 10520 int ret = 0; 10521 ktime_t start = ktime_get(); 10522 10523 hba = shost_priv(sdev->host); 10524 10525 if (pm_runtime_suspended(dev)) 10526 goto out; 10527 10528 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM); 10529 if (ret) 10530 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10531 out: 10532 trace_ufshcd_wl_resume(hba, ret, 10533 ktime_to_us(ktime_sub(ktime_get(), start)), 10534 hba->curr_dev_pwr_mode, hba->uic_link_state); 10535 if (!ret) 10536 hba->is_sys_suspended = false; 10537 hba->system_suspending = false; 10538 up(&hba->host_sem); 10539 return ret; 10540 } 10541 #endif 10542 10543 /** 10544 * ufshcd_suspend - helper function for suspend operations 10545 * @hba: per adapter instance 10546 * 10547 * This function will put disable irqs, turn off clocks 10548 * and set vreg and hba-vreg in lpm mode. 10549 * 10550 * Return: 0 upon success; < 0 upon failure. 10551 */ 10552 static int ufshcd_suspend(struct ufs_hba *hba) 10553 { 10554 int ret; 10555 10556 if (!hba->is_powered) 10557 return 0; 10558 /* 10559 * Disable the host irq as host controller as there won't be any 10560 * host controller transaction expected till resume. 10561 */ 10562 ufshcd_disable_irq(hba); 10563 ret = ufshcd_setup_clocks(hba, false); 10564 if (ret) { 10565 ufshcd_enable_irq(hba); 10566 goto out; 10567 } 10568 if (ufshcd_is_clkgating_allowed(hba)) { 10569 hba->clk_gating.state = CLKS_OFF; 10570 trace_ufshcd_clk_gating(hba, 10571 hba->clk_gating.state); 10572 } 10573 10574 ufshcd_vreg_set_lpm(hba); 10575 /* Put the host controller in low power mode if possible */ 10576 ufshcd_hba_vreg_set_lpm(hba); 10577 ufshcd_pm_qos_update(hba, false); 10578 out: 10579 if (ret) 10580 ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret); 10581 return ret; 10582 } 10583 10584 #ifdef CONFIG_PM 10585 /** 10586 * ufshcd_resume - helper function for resume operations 10587 * @hba: per adapter instance 10588 * 10589 * This function basically turns on the regulators, clocks and 10590 * irqs of the hba. 10591 * 10592 * Return: 0 for success and non-zero for failure. 10593 */ 10594 static int ufshcd_resume(struct ufs_hba *hba) 10595 { 10596 int ret; 10597 10598 if (!hba->is_powered) 10599 return 0; 10600 10601 ufshcd_hba_vreg_set_hpm(hba); 10602 ret = ufshcd_vreg_set_hpm(hba); 10603 if (ret) 10604 goto out; 10605 10606 /* Make sure clocks are enabled before accessing controller */ 10607 ret = ufshcd_setup_clocks(hba, true); 10608 if (ret) 10609 goto disable_vreg; 10610 10611 /* enable the host irq as host controller would be active soon */ 10612 ufshcd_enable_irq(hba); 10613 10614 goto out; 10615 10616 disable_vreg: 10617 ufshcd_vreg_set_lpm(hba); 10618 out: 10619 if (ret) 10620 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); 10621 return ret; 10622 } 10623 #endif /* CONFIG_PM */ 10624 10625 #ifdef CONFIG_PM_SLEEP 10626 /** 10627 * ufshcd_system_suspend - system suspend callback 10628 * @dev: Device associated with the UFS controller. 10629 * 10630 * Executed before putting the system into a sleep state in which the contents 10631 * of main memory are preserved. 10632 * 10633 * Return: 0 for success and non-zero for failure. 10634 */ 10635 int ufshcd_system_suspend(struct device *dev) 10636 { 10637 struct ufs_hba *hba = dev_get_drvdata(dev); 10638 int ret = 0; 10639 ktime_t start = ktime_get(); 10640 10641 if (pm_runtime_suspended(hba->dev)) 10642 goto out; 10643 10644 ret = ufshcd_suspend(hba); 10645 out: 10646 trace_ufshcd_system_suspend(hba, ret, 10647 ktime_to_us(ktime_sub(ktime_get(), start)), 10648 hba->curr_dev_pwr_mode, hba->uic_link_state); 10649 return ret; 10650 } 10651 EXPORT_SYMBOL(ufshcd_system_suspend); 10652 10653 /** 10654 * ufshcd_system_resume - system resume callback 10655 * @dev: Device associated with the UFS controller. 10656 * 10657 * Executed after waking the system up from a sleep state in which the contents 10658 * of main memory were preserved. 10659 * 10660 * Return: 0 for success and non-zero for failure. 10661 */ 10662 int ufshcd_system_resume(struct device *dev) 10663 { 10664 struct ufs_hba *hba = dev_get_drvdata(dev); 10665 ktime_t start = ktime_get(); 10666 int ret = 0; 10667 10668 if (pm_runtime_suspended(hba->dev)) 10669 goto out; 10670 10671 ret = ufshcd_resume(hba); 10672 10673 out: 10674 trace_ufshcd_system_resume(hba, ret, 10675 ktime_to_us(ktime_sub(ktime_get(), start)), 10676 hba->curr_dev_pwr_mode, hba->uic_link_state); 10677 10678 return ret; 10679 } 10680 EXPORT_SYMBOL(ufshcd_system_resume); 10681 #endif /* CONFIG_PM_SLEEP */ 10682 10683 #ifdef CONFIG_PM 10684 /** 10685 * ufshcd_runtime_suspend - runtime suspend callback 10686 * @dev: Device associated with the UFS controller. 10687 * 10688 * Check the description of ufshcd_suspend() function for more details. 10689 * 10690 * Return: 0 for success and non-zero for failure. 10691 */ 10692 int ufshcd_runtime_suspend(struct device *dev) 10693 { 10694 struct ufs_hba *hba = dev_get_drvdata(dev); 10695 int ret; 10696 ktime_t start = ktime_get(); 10697 10698 ret = ufshcd_suspend(hba); 10699 10700 trace_ufshcd_runtime_suspend(hba, ret, 10701 ktime_to_us(ktime_sub(ktime_get(), start)), 10702 hba->curr_dev_pwr_mode, hba->uic_link_state); 10703 return ret; 10704 } 10705 EXPORT_SYMBOL(ufshcd_runtime_suspend); 10706 10707 /** 10708 * ufshcd_runtime_resume - runtime resume routine 10709 * @dev: Device associated with the UFS controller. 10710 * 10711 * This function basically brings controller 10712 * to active state. Following operations are done in this function: 10713 * 10714 * 1. Turn on all the controller related clocks 10715 * 2. Turn ON VCC rail 10716 * 10717 * Return: 0 upon success; < 0 upon failure. 10718 */ 10719 int ufshcd_runtime_resume(struct device *dev) 10720 { 10721 struct ufs_hba *hba = dev_get_drvdata(dev); 10722 int ret; 10723 ktime_t start = ktime_get(); 10724 10725 ret = ufshcd_resume(hba); 10726 10727 trace_ufshcd_runtime_resume(hba, ret, 10728 ktime_to_us(ktime_sub(ktime_get(), start)), 10729 hba->curr_dev_pwr_mode, hba->uic_link_state); 10730 return ret; 10731 } 10732 EXPORT_SYMBOL(ufshcd_runtime_resume); 10733 #endif /* CONFIG_PM */ 10734 10735 static void ufshcd_wl_shutdown(struct scsi_device *sdev) 10736 { 10737 struct ufs_hba *hba = shost_priv(sdev->host); 10738 10739 down(&hba->host_sem); 10740 hba->shutting_down = true; 10741 up(&hba->host_sem); 10742 10743 /* Turn on everything while shutting down */ 10744 ufshcd_rpm_get_sync(hba); 10745 scsi_device_quiesce(sdev); 10746 shost_for_each_device(sdev, hba->host) { 10747 if (sdev == hba->ufs_device_wlun) 10748 continue; 10749 mutex_lock(&sdev->state_mutex); 10750 scsi_device_set_state(sdev, SDEV_OFFLINE); 10751 mutex_unlock(&sdev->state_mutex); 10752 } 10753 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10754 10755 /* 10756 * Next, turn off the UFS controller and the UFS regulators. Disable 10757 * clocks. 10758 */ 10759 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 10760 ufshcd_suspend(hba); 10761 10762 hba->is_powered = false; 10763 } 10764 10765 /** 10766 * ufshcd_remove - de-allocate SCSI host and host memory space 10767 * data structure memory 10768 * @hba: per adapter instance 10769 */ 10770 void ufshcd_remove(struct ufs_hba *hba) 10771 { 10772 if (hba->ufs_device_wlun) 10773 ufshcd_rpm_get_sync(hba); 10774 ufs_hwmon_remove(hba); 10775 ufs_bsg_remove(hba); 10776 ufs_rpmb_remove(hba); 10777 ufs_sysfs_remove_nodes(hba->dev); 10778 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10779 blk_mq_destroy_queue(hba->tmf_queue); 10780 blk_put_queue(hba->tmf_queue); 10781 blk_mq_free_tag_set(&hba->tmf_tag_set); 10782 if (hba->scsi_host_added) 10783 scsi_remove_host(hba->host); 10784 /* disable interrupts */ 10785 ufshcd_disable_intr(hba, hba->intr_mask); 10786 ufshcd_hba_stop(hba); 10787 ufshcd_hba_exit(hba); 10788 } 10789 EXPORT_SYMBOL_GPL(ufshcd_remove); 10790 10791 #ifdef CONFIG_PM_SLEEP 10792 int ufshcd_system_freeze(struct device *dev) 10793 { 10794 10795 return ufshcd_system_suspend(dev); 10796 10797 } 10798 EXPORT_SYMBOL_GPL(ufshcd_system_freeze); 10799 10800 int ufshcd_system_restore(struct device *dev) 10801 { 10802 10803 struct ufs_hba *hba = dev_get_drvdata(dev); 10804 int ret; 10805 10806 ret = ufshcd_system_resume(dev); 10807 if (ret) 10808 return ret; 10809 10810 /* Configure UTRL and UTMRL base address registers */ 10811 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 10812 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 10813 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 10814 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 10815 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 10816 REG_UTP_TASK_REQ_LIST_BASE_L); 10817 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 10818 REG_UTP_TASK_REQ_LIST_BASE_H); 10819 /* 10820 * Make sure that UTRL and UTMRL base address registers 10821 * are updated with the latest queue addresses. Only after 10822 * updating these addresses, we can queue the new commands. 10823 */ 10824 ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); 10825 10826 return 0; 10827 10828 } 10829 EXPORT_SYMBOL_GPL(ufshcd_system_restore); 10830 10831 int ufshcd_system_thaw(struct device *dev) 10832 { 10833 return ufshcd_system_resume(dev); 10834 } 10835 EXPORT_SYMBOL_GPL(ufshcd_system_thaw); 10836 #endif /* CONFIG_PM_SLEEP */ 10837 10838 /** 10839 * ufshcd_set_dma_mask - Set dma mask based on the controller 10840 * addressing capability 10841 * @hba: per adapter instance 10842 * 10843 * Return: 0 for success, non-zero for failure. 10844 */ 10845 static int ufshcd_set_dma_mask(struct ufs_hba *hba) 10846 { 10847 if (hba->vops && hba->vops->set_dma_mask) 10848 return hba->vops->set_dma_mask(hba); 10849 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 10850 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 10851 return 0; 10852 } 10853 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 10854 } 10855 10856 /** 10857 * ufshcd_devres_release - devres cleanup handler, invoked during release of 10858 * hba->dev 10859 * @host: pointer to SCSI host 10860 */ 10861 static void ufshcd_devres_release(void *host) 10862 { 10863 scsi_host_put(host); 10864 } 10865 10866 /** 10867 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 10868 * @dev: pointer to device handle 10869 * @hba_handle: driver private handle 10870 * 10871 * Return: 0 on success, non-zero value on failure. 10872 * 10873 * NOTE: There is no corresponding ufshcd_dealloc_host() because this function 10874 * keeps track of its allocations using devres and deallocates everything on 10875 * device removal automatically. 10876 */ 10877 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 10878 { 10879 struct Scsi_Host *host; 10880 struct ufs_hba *hba; 10881 int err = 0; 10882 10883 if (!dev) { 10884 dev_err(dev, 10885 "Invalid memory reference for dev is NULL\n"); 10886 err = -ENODEV; 10887 goto out_error; 10888 } 10889 10890 host = scsi_host_alloc(&ufshcd_driver_template, 10891 sizeof(struct ufs_hba)); 10892 if (!host) { 10893 dev_err(dev, "scsi_host_alloc failed\n"); 10894 err = -ENOMEM; 10895 goto out_error; 10896 } 10897 10898 err = devm_add_action_or_reset(dev, ufshcd_devres_release, 10899 host); 10900 if (err) 10901 return err; 10902 10903 host->nr_maps = HCTX_TYPE_POLL + 1; 10904 hba = shost_priv(host); 10905 hba->host = host; 10906 hba->dev = dev; 10907 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; 10908 hba->nop_out_timeout = NOP_OUT_TIMEOUT; 10909 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); 10910 INIT_LIST_HEAD(&hba->clk_list_head); 10911 spin_lock_init(&hba->outstanding_lock); 10912 10913 *hba_handle = hba; 10914 10915 out_error: 10916 return err; 10917 } 10918 EXPORT_SYMBOL(ufshcd_alloc_host); 10919 10920 /* This function exists because blk_mq_alloc_tag_set() requires this. */ 10921 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx, 10922 const struct blk_mq_queue_data *qd) 10923 { 10924 WARN_ON_ONCE(true); 10925 return BLK_STS_NOTSUPP; 10926 } 10927 10928 static const struct blk_mq_ops ufshcd_tmf_ops = { 10929 .queue_rq = ufshcd_queue_tmf, 10930 }; 10931 10932 static int ufshcd_add_scsi_host(struct ufs_hba *hba) 10933 { 10934 int err; 10935 10936 WARN_ON_ONCE(!hba->host->can_queue); 10937 WARN_ON_ONCE(!hba->host->cmd_per_lun); 10938 10939 if (is_mcq_supported(hba)) { 10940 ufshcd_mcq_enable(hba); 10941 err = ufshcd_alloc_mcq(hba); 10942 if (err) { 10943 /* Continue with SDB mode */ 10944 ufshcd_mcq_disable(hba); 10945 use_mcq_mode = false; 10946 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", 10947 err); 10948 } 10949 } 10950 if (!is_mcq_supported(hba) && !hba->lsdb_sup) { 10951 dev_err(hba->dev, 10952 "%s: failed to initialize (legacy doorbell mode not supported)\n", 10953 __func__); 10954 return -EINVAL; 10955 } 10956 10957 err = scsi_add_host(hba->host, hba->dev); 10958 if (err) { 10959 dev_err(hba->dev, "scsi_add_host failed\n"); 10960 return err; 10961 } 10962 hba->scsi_host_added = true; 10963 10964 hba->tmf_tag_set = (struct blk_mq_tag_set) { 10965 .nr_hw_queues = 1, 10966 .queue_depth = hba->nutmrs, 10967 .ops = &ufshcd_tmf_ops, 10968 }; 10969 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); 10970 if (err < 0) 10971 goto remove_scsi_host; 10972 hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); 10973 if (IS_ERR(hba->tmf_queue)) { 10974 err = PTR_ERR(hba->tmf_queue); 10975 goto free_tmf_tag_set; 10976 } 10977 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, 10978 sizeof(*hba->tmf_rqs), GFP_KERNEL); 10979 if (!hba->tmf_rqs) { 10980 err = -ENOMEM; 10981 goto free_tmf_queue; 10982 } 10983 10984 return 0; 10985 10986 free_tmf_queue: 10987 blk_mq_destroy_queue(hba->tmf_queue); 10988 blk_put_queue(hba->tmf_queue); 10989 10990 free_tmf_tag_set: 10991 blk_mq_free_tag_set(&hba->tmf_tag_set); 10992 10993 remove_scsi_host: 10994 if (hba->scsi_host_added) 10995 scsi_remove_host(hba->host); 10996 10997 return err; 10998 } 10999 11000 /** 11001 * ufshcd_init - Driver initialization routine 11002 * @hba: per-adapter instance 11003 * @mmio_base: base register address 11004 * @irq: Interrupt line of device 11005 * 11006 * Return: 0 on success; < 0 on failure. 11007 */ 11008 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 11009 { 11010 int err; 11011 struct Scsi_Host *host = hba->host; 11012 struct device *dev = hba->dev; 11013 11014 /* 11015 * dev_set_drvdata() must be called before any callbacks are registered 11016 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon, 11017 * sysfs). 11018 */ 11019 dev_set_drvdata(dev, hba); 11020 11021 if (!mmio_base) { 11022 dev_err(hba->dev, 11023 "Invalid memory reference for mmio_base is NULL\n"); 11024 err = -ENODEV; 11025 goto out_error; 11026 } 11027 11028 hba->mmio_base = mmio_base; 11029 hba->irq = irq; 11030 hba->vps = &ufs_hba_vps; 11031 11032 /* 11033 * Initialize clk_gating.lock early since it is being used in 11034 * ufshcd_setup_clocks() 11035 */ 11036 spin_lock_init(&hba->clk_gating.lock); 11037 11038 /* Initialize mutex for PM QoS request synchronization */ 11039 mutex_init(&hba->pm_qos_mutex); 11040 11041 /* 11042 * Set the default power management level for runtime and system PM. 11043 * Host controller drivers can override them in their 11044 * 'ufs_hba_variant_ops::init' callback. 11045 * 11046 * Default power saving mode is to keep UFS link in Hibern8 state 11047 * and UFS device in sleep state. 11048 */ 11049 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 11050 UFS_SLEEP_PWR_MODE, 11051 UIC_LINK_HIBERN8_STATE); 11052 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 11053 UFS_SLEEP_PWR_MODE, 11054 UIC_LINK_HIBERN8_STATE); 11055 11056 /* 11057 * Most ufs devices require 1ms delay after vcc is powered off before 11058 * it can be powered on again. Set the default to 2ms. The platform 11059 * drivers can override this setting as needed. 11060 */ 11061 hba->vcc_off_delay_us = 2000; 11062 11063 err = ufshcd_hba_init(hba); 11064 if (err) 11065 goto out_error; 11066 11067 /* Read capabilities registers */ 11068 err = ufshcd_hba_capabilities(hba); 11069 if (err) 11070 goto out_disable; 11071 11072 /* Get UFS version supported by the controller */ 11073 hba->ufs_version = ufshcd_get_ufs_version(hba); 11074 11075 /* Get Interrupt bit mask per version */ 11076 hba->intr_mask = ufshcd_get_intr_mask(hba); 11077 11078 err = ufshcd_set_dma_mask(hba); 11079 if (err) { 11080 dev_err(hba->dev, "set dma mask failed\n"); 11081 goto out_disable; 11082 } 11083 11084 /* Allocate memory for host memory space */ 11085 err = ufshcd_memory_alloc(hba); 11086 if (err) { 11087 dev_err(hba->dev, "Memory allocation failed\n"); 11088 goto out_disable; 11089 } 11090 11091 /* Configure LRB */ 11092 ufshcd_host_memory_configure(hba); 11093 11094 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 11095 /* 11096 * Set the queue depth for WLUNs. ufs_get_device_desc() will increase 11097 * host->cmd_per_lun to a larger value. 11098 */ 11099 host->cmd_per_lun = 1; 11100 host->max_id = UFSHCD_MAX_ID; 11101 host->max_lun = UFS_MAX_LUNS; 11102 host->max_channel = UFSHCD_MAX_CHANNEL; 11103 host->unique_id = host->host_no; 11104 host->max_cmd_len = UFS_CDB_SIZE; 11105 host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); 11106 11107 /* Use default RPM delay if host not set */ 11108 if (host->rpm_autosuspend_delay == 0) 11109 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; 11110 11111 hba->max_pwr_info.is_valid = false; 11112 11113 /* Initialize work queues */ 11114 hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, 11115 hba->host->host_no); 11116 if (!hba->eh_wq) { 11117 dev_err(hba->dev, "%s: failed to create eh workqueue\n", 11118 __func__); 11119 err = -ENOMEM; 11120 goto out_disable; 11121 } 11122 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 11123 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 11124 11125 sema_init(&hba->host_sem, 1); 11126 11127 /* Initialize UIC command mutex */ 11128 mutex_init(&hba->uic_cmd_mutex); 11129 11130 /* Initialize mutex for device management commands */ 11131 mutex_init(&hba->dev_cmd.lock); 11132 11133 /* Initialize mutex for exception event control */ 11134 mutex_init(&hba->ee_ctrl_mutex); 11135 11136 mutex_init(&hba->wb_mutex); 11137 11138 init_rwsem(&hba->clk_scaling_lock); 11139 11140 ufshcd_init_clk_gating(hba); 11141 11142 ufshcd_init_clk_scaling(hba); 11143 11144 /* 11145 * In order to avoid any spurious interrupt immediately after 11146 * registering UFS controller interrupt handler, clear any pending UFS 11147 * interrupt status and disable all the UFS interrupts. 11148 */ 11149 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), 11150 REG_INTERRUPT_STATUS); 11151 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); 11152 /* 11153 * Make sure that UFS interrupts are disabled and any pending interrupt 11154 * status is cleared before registering UFS interrupt handler. 11155 */ 11156 ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 11157 11158 /* IRQ registration */ 11159 err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, 11160 IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); 11161 if (err) { 11162 dev_err(hba->dev, "request irq failed\n"); 11163 goto out_disable; 11164 } else { 11165 hba->is_irq_enabled = true; 11166 } 11167 11168 /* Reset the attached device */ 11169 ufshcd_device_reset(hba); 11170 11171 ufshcd_init_crypto(hba); 11172 11173 /* Host controller enable */ 11174 err = ufshcd_hba_enable(hba); 11175 if (err) { 11176 dev_err(hba->dev, "Host controller enable failed\n"); 11177 ufshcd_print_evt_hist(hba); 11178 ufshcd_print_host_state(hba); 11179 goto out_disable; 11180 } 11181 11182 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); 11183 INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); 11184 11185 /* Set the default auto-hiberate idle timer value to 150 ms */ 11186 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { 11187 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) | 11188 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 11189 } 11190 11191 err = ufshcd_add_scsi_host(hba); 11192 if (err) 11193 goto out_disable; 11194 11195 /* Hold auto suspend until async scan completes */ 11196 pm_runtime_get_sync(dev); 11197 11198 /* 11199 * We are assuming that device wasn't put in sleep/power-down 11200 * state exclusively during the boot stage before kernel. 11201 * This assumption helps avoid doing link startup twice during 11202 * ufshcd_probe_hba(). 11203 */ 11204 ufshcd_set_ufs_dev_active(hba); 11205 11206 /* Initialize hba, detect and initialize UFS device */ 11207 ktime_t probe_start = ktime_get(); 11208 11209 hba->ufshcd_state = UFSHCD_STATE_RESET; 11210 11211 err = ufshcd_link_startup(hba); 11212 if (err) 11213 goto out_disable; 11214 11215 if (hba->mcq_enabled) 11216 ufshcd_config_mcq(hba); 11217 11218 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 11219 goto initialized; 11220 11221 /* Debug counters initialization */ 11222 ufshcd_clear_dbg_ufs_stats(hba); 11223 11224 /* UniPro link is active now */ 11225 ufshcd_set_link_active(hba); 11226 11227 /* Verify device initialization by sending NOP OUT UPIU */ 11228 err = ufshcd_verify_dev_init(hba); 11229 if (err) 11230 goto out_disable; 11231 11232 /* Initiate UFS initialization, and waiting until completion */ 11233 err = ufshcd_complete_dev_init(hba); 11234 if (err) 11235 goto out_disable; 11236 11237 err = ufshcd_device_params_init(hba); 11238 if (err) 11239 goto out_disable; 11240 11241 err = ufshcd_post_device_init(hba); 11242 11243 initialized: 11244 ufshcd_process_probe_result(hba, probe_start, err); 11245 if (err) 11246 goto out_disable; 11247 11248 ufs_sysfs_add_nodes(hba->dev); 11249 hba->dme_qos_sysfs_handle = sysfs_get_dirent(hba->dev->kobj.sd, 11250 "dme_qos_notification"); 11251 async_schedule(ufshcd_async_scan, hba); 11252 11253 device_enable_async_suspend(dev); 11254 ufshcd_pm_qos_init(hba); 11255 return 0; 11256 11257 out_disable: 11258 hba->is_irq_enabled = false; 11259 ufshcd_hba_exit(hba); 11260 out_error: 11261 return err > 0 ? -EIO : err; 11262 } 11263 EXPORT_SYMBOL_GPL(ufshcd_init); 11264 11265 void ufshcd_resume_complete(struct device *dev) 11266 { 11267 struct ufs_hba *hba = dev_get_drvdata(dev); 11268 11269 if (hba->complete_put) { 11270 ufshcd_rpm_put(hba); 11271 hba->complete_put = false; 11272 } 11273 } 11274 EXPORT_SYMBOL_GPL(ufshcd_resume_complete); 11275 11276 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba) 11277 { 11278 struct device *dev = &hba->ufs_device_wlun->sdev_gendev; 11279 enum ufs_dev_pwr_mode dev_pwr_mode; 11280 enum uic_link_state link_state; 11281 unsigned long flags; 11282 bool res; 11283 11284 spin_lock_irqsave(&dev->power.lock, flags); 11285 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl); 11286 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl); 11287 res = pm_runtime_suspended(dev) && 11288 hba->curr_dev_pwr_mode == dev_pwr_mode && 11289 hba->uic_link_state == link_state && 11290 !hba->dev_info.b_rpm_dev_flush_capable; 11291 spin_unlock_irqrestore(&dev->power.lock, flags); 11292 11293 return res; 11294 } 11295 11296 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm) 11297 { 11298 struct ufs_hba *hba = dev_get_drvdata(dev); 11299 int ret; 11300 11301 /* 11302 * SCSI assumes that runtime-pm and system-pm for scsi drivers 11303 * are same. And it doesn't wake up the device for system-suspend 11304 * if it's runtime suspended. But ufs doesn't follow that. 11305 * Refer ufshcd_resume_complete() 11306 */ 11307 if (hba->ufs_device_wlun) { 11308 /* Prevent runtime suspend */ 11309 ufshcd_rpm_get_noresume(hba); 11310 /* 11311 * Check if already runtime suspended in same state as system 11312 * suspend would be. 11313 */ 11314 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) { 11315 /* RPM state is not ok for SPM, so runtime resume */ 11316 ret = ufshcd_rpm_resume(hba); 11317 if (ret < 0 && ret != -EACCES) { 11318 ufshcd_rpm_put(hba); 11319 return ret; 11320 } 11321 } 11322 hba->complete_put = true; 11323 } 11324 return 0; 11325 } 11326 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare); 11327 11328 int ufshcd_suspend_prepare(struct device *dev) 11329 { 11330 return __ufshcd_suspend_prepare(dev, true); 11331 } 11332 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare); 11333 11334 #ifdef CONFIG_PM_SLEEP 11335 static int ufshcd_wl_poweroff(struct device *dev) 11336 { 11337 struct scsi_device *sdev = to_scsi_device(dev); 11338 struct ufs_hba *hba = shost_priv(sdev->host); 11339 11340 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 11341 return 0; 11342 } 11343 #endif 11344 11345 static int ufshcd_wl_probe(struct scsi_device *sdev) 11346 { 11347 struct device *dev = &sdev->sdev_gendev; 11348 11349 if (!is_device_wlun(sdev)) 11350 return -ENODEV; 11351 11352 blk_pm_runtime_init(sdev->request_queue, dev); 11353 pm_runtime_set_autosuspend_delay(dev, 0); 11354 pm_runtime_allow(dev); 11355 11356 return 0; 11357 } 11358 11359 static void ufshcd_wl_remove(struct scsi_device *sdev) 11360 { 11361 struct device *dev = &sdev->sdev_gendev; 11362 11363 pm_runtime_forbid(dev); 11364 } 11365 11366 static const struct dev_pm_ops ufshcd_wl_pm_ops = { 11367 #ifdef CONFIG_PM_SLEEP 11368 .suspend = ufshcd_wl_suspend, 11369 .resume = ufshcd_wl_resume, 11370 .freeze = ufshcd_wl_suspend, 11371 .thaw = ufshcd_wl_resume, 11372 .poweroff = ufshcd_wl_poweroff, 11373 .restore = ufshcd_wl_resume, 11374 #endif 11375 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) 11376 }; 11377 11378 static void ufshcd_check_header_layout(void) 11379 { 11380 /* 11381 * gcc compilers before version 10 cannot do constant-folding for 11382 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and 11383 * before. 11384 */ 11385 if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) 11386 return; 11387 11388 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11389 .cci = 3})[0] != 3); 11390 11391 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11392 .ehs_length = 2})[1] != 2); 11393 11394 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11395 .enable_crypto = 1})[2] 11396 != 0x80); 11397 11398 BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ 11399 .command_type = 5, 11400 .data_direction = 3, 11401 .interrupt = 1, 11402 })[3]) != ((5 << 4) | (3 << 1) | 1)); 11403 11404 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11405 .dunl = cpu_to_le32(0xdeadbeef)})[1] != 11406 cpu_to_le32(0xdeadbeef)); 11407 11408 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11409 .ocs = 4})[8] != 4); 11410 11411 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11412 .cds = 5})[9] != 5); 11413 11414 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11415 .dunu = cpu_to_le32(0xbadcafe)})[3] != 11416 cpu_to_le32(0xbadcafe)); 11417 11418 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11419 .iid = 0xf })[4] != 0xf0); 11420 11421 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11422 .command_set_type = 0xf })[4] != 0xf); 11423 } 11424 11425 /* 11426 * ufs_dev_wlun_template - describes ufs device wlun 11427 * ufs-device wlun - used to send pm commands 11428 * All luns are consumers of ufs-device wlun. 11429 * 11430 * Currently, no sd driver is present for wluns. 11431 * Hence the no specific pm operations are performed. 11432 * With ufs design, SSU should be sent to ufs-device wlun. 11433 * Hence register a scsi driver for ufs wluns only. 11434 */ 11435 static struct scsi_driver ufs_dev_wlun_template = { 11436 .probe = ufshcd_wl_probe, 11437 .remove = ufshcd_wl_remove, 11438 .shutdown = ufshcd_wl_shutdown, 11439 .gendrv = { 11440 .name = "ufs_device_wlun", 11441 .pm = &ufshcd_wl_pm_ops, 11442 }, 11443 }; 11444 11445 static int __init ufshcd_core_init(void) 11446 { 11447 int ret; 11448 11449 ufshcd_check_header_layout(); 11450 11451 ufs_debugfs_init(); 11452 11453 ret = scsi_register_driver(&ufs_dev_wlun_template); 11454 if (ret) 11455 ufs_debugfs_exit(); 11456 return ret; 11457 } 11458 11459 static void __exit ufshcd_core_exit(void) 11460 { 11461 ufs_debugfs_exit(); 11462 scsi_unregister_driver(&ufs_dev_wlun_template); 11463 } 11464 11465 module_init(ufshcd_core_init); 11466 module_exit(ufshcd_core_exit); 11467 11468 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 11469 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 11470 MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 11471 MODULE_SOFTDEP("pre: governor_simpleondemand"); 11472 MODULE_LICENSE("GPL"); 11473