1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Universal Flash Storage Host controller driver Core 4 * Copyright (C) 2011-2013 Samsung India Software Operations 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 6 * 7 * Authors: 8 * Santosh Yaraganavi <santosh.sy@samsung.com> 9 * Vinayak Holikatti <h.vinayak@samsung.com> 10 */ 11 12 #include <linux/async.h> 13 #include <linux/devfreq.h> 14 #include <linux/nls.h> 15 #include <linux/of.h> 16 #include <linux/bitfield.h> 17 #include <linux/blk-pm.h> 18 #include <linux/blkdev.h> 19 #include <linux/clk.h> 20 #include <linux/delay.h> 21 #include <linux/interrupt.h> 22 #include <linux/module.h> 23 #include <linux/pm_opp.h> 24 #include <linux/regulator/consumer.h> 25 #include <linux/sched/clock.h> 26 #include <linux/iopoll.h> 27 #include <scsi/scsi_cmnd.h> 28 #include <scsi/scsi_dbg.h> 29 #include <scsi/scsi_driver.h> 30 #include <scsi/scsi_eh.h> 31 #include "ufshcd-priv.h" 32 #include <ufs/ufs_quirks.h> 33 #include <ufs/unipro.h> 34 #include "ufs-sysfs.h" 35 #include "ufs-debugfs.h" 36 #include "ufs-fault-injection.h" 37 #include "ufs_bsg.h" 38 #include "ufshcd-crypto.h" 39 #include <linux/unaligned.h> 40 41 #define CREATE_TRACE_POINTS 42 #include "ufs_trace.h" 43 44 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 45 UTP_TASK_REQ_COMPL |\ 46 UFSHCD_ERROR_MASK) 47 48 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\ 49 UFSHCD_ERROR_MASK |\ 50 MCQ_CQ_EVENT_STATUS) 51 52 53 /* UIC command timeout, unit: ms */ 54 enum { 55 UIC_CMD_TIMEOUT_DEFAULT = 500, 56 UIC_CMD_TIMEOUT_MAX = 5000, 57 }; 58 /* NOP OUT retries waiting for NOP IN response */ 59 #define NOP_OUT_RETRIES 10 60 /* Timeout after 50 msecs if NOP OUT hangs without response */ 61 #define NOP_OUT_TIMEOUT 50 /* msecs */ 62 63 /* Query request retries */ 64 #define QUERY_REQ_RETRIES 3 65 /* Query request timeout */ 66 enum { 67 QUERY_REQ_TIMEOUT_MIN = 1, 68 QUERY_REQ_TIMEOUT_DEFAULT = 1500, 69 QUERY_REQ_TIMEOUT_MAX = 30000 70 }; 71 72 /* Advanced RPMB request timeout */ 73 #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ 74 75 /* Task management command timeout */ 76 #define TM_CMD_TIMEOUT 100 /* msecs */ 77 78 /* maximum number of retries for a general UIC command */ 79 #define UFS_UIC_COMMAND_RETRIES 3 80 81 /* maximum number of link-startup retries */ 82 #define DME_LINKSTARTUP_RETRIES 3 83 84 /* maximum number of reset retries before giving up */ 85 #define MAX_HOST_RESET_RETRIES 5 86 87 /* Maximum number of error handler retries before giving up */ 88 #define MAX_ERR_HANDLER_RETRIES 5 89 90 /* Expose the flag value from utp_upiu_query.value */ 91 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF 92 93 /* Interrupt aggregation default timeout, unit: 40us */ 94 #define INT_AGGR_DEF_TO 0x02 95 96 /* default delay of autosuspend: 2000 ms */ 97 #define RPM_AUTOSUSPEND_DELAY_MS 2000 98 99 /* Default delay of RPM device flush delayed work */ 100 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000 101 102 /* Default value of wait time before gating device ref clock */ 103 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */ 104 105 /* Polling time to wait for fDeviceInit */ 106 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ 107 108 /* Default RTC update every 10 seconds */ 109 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) 110 111 /* bMaxNumOfRTT is equal to two after device manufacturing */ 112 #define DEFAULT_MAX_NUM_RTT 2 113 114 /* UFSHC 4.0 compliant HC support this mode. */ 115 static bool use_mcq_mode = true; 116 117 static bool is_mcq_supported(struct ufs_hba *hba) 118 { 119 return hba->mcq_sup && use_mcq_mode; 120 } 121 122 module_param(use_mcq_mode, bool, 0644); 123 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); 124 125 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; 126 127 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) 128 { 129 return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, 130 UIC_CMD_TIMEOUT_MAX); 131 } 132 133 static const struct kernel_param_ops uic_cmd_timeout_ops = { 134 .set = uic_cmd_timeout_set, 135 .get = param_get_uint, 136 }; 137 138 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); 139 MODULE_PARM_DESC(uic_cmd_timeout, 140 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); 141 142 static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; 143 144 static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) 145 { 146 return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, 147 QUERY_REQ_TIMEOUT_MAX); 148 } 149 150 static const struct kernel_param_ops dev_cmd_timeout_ops = { 151 .set = dev_cmd_timeout_set, 152 .get = param_get_uint, 153 }; 154 155 module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); 156 MODULE_PARM_DESC(dev_cmd_timeout, 157 "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); 158 159 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 160 ({ \ 161 int _ret; \ 162 if (_on) \ 163 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 164 else \ 165 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 166 _ret; \ 167 }) 168 169 #define ufshcd_hex_dump(prefix_str, buf, len) do { \ 170 size_t __len = (len); \ 171 print_hex_dump(KERN_ERR, prefix_str, \ 172 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\ 173 16, 4, buf, __len, false); \ 174 } while (0) 175 176 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, 177 const char *prefix) 178 { 179 u32 *regs; 180 size_t pos; 181 182 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */ 183 return -EINVAL; 184 185 regs = kzalloc(len, GFP_ATOMIC); 186 if (!regs) 187 return -ENOMEM; 188 189 for (pos = 0; pos < len; pos += 4) { 190 if (offset == 0 && 191 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && 192 pos <= REG_UIC_ERROR_CODE_DME) 193 continue; 194 regs[pos / 4] = ufshcd_readl(hba, offset + pos); 195 } 196 197 ufshcd_hex_dump(prefix, regs, len); 198 kfree(regs); 199 200 return 0; 201 } 202 EXPORT_SYMBOL_GPL(ufshcd_dump_regs); 203 204 enum { 205 UFSHCD_MAX_CHANNEL = 0, 206 UFSHCD_MAX_ID = 1, 207 }; 208 209 static const char *const ufshcd_state_name[] = { 210 [UFSHCD_STATE_RESET] = "reset", 211 [UFSHCD_STATE_OPERATIONAL] = "operational", 212 [UFSHCD_STATE_ERROR] = "error", 213 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal", 214 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal", 215 }; 216 217 /* UFSHCD error handling flags */ 218 enum { 219 UFSHCD_EH_IN_PROGRESS = (1 << 0), 220 }; 221 222 /* UFSHCD UIC layer error flags */ 223 enum { 224 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 225 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */ 226 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */ 227 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */ 228 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */ 229 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */ 230 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */ 231 }; 232 233 #define ufshcd_set_eh_in_progress(h) \ 234 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS) 235 #define ufshcd_eh_in_progress(h) \ 236 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS) 237 #define ufshcd_clear_eh_in_progress(h) \ 238 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 239 240 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 241 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 242 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 243 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 244 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 245 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 246 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 247 /* 248 * For DeepSleep, the link is first put in hibern8 and then off. 249 * Leaving the link in hibern8 is not supported. 250 */ 251 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, 252 }; 253 254 static inline enum ufs_dev_pwr_mode 255 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 256 { 257 return ufs_pm_lvl_states[lvl].dev_state; 258 } 259 260 static inline enum uic_link_state 261 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 262 { 263 return ufs_pm_lvl_states[lvl].link_state; 264 } 265 266 static inline enum ufs_pm_level 267 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, 268 enum uic_link_state link_state) 269 { 270 enum ufs_pm_level lvl; 271 272 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { 273 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && 274 (ufs_pm_lvl_states[lvl].link_state == link_state)) 275 return lvl; 276 } 277 278 /* if no match found, return the level 0 */ 279 return UFS_PM_LVL_0; 280 } 281 282 static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) 283 { 284 return hba->outstanding_tasks || hba->active_uic_cmd || 285 hba->uic_async_done; 286 } 287 288 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) 289 { 290 return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); 291 } 292 293 static const struct ufs_dev_quirk ufs_fixups[] = { 294 /* UFS cards deviations table */ 295 { .wmanufacturerid = UFS_VENDOR_MICRON, 296 .model = UFS_ANY_MODEL, 297 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 298 { .wmanufacturerid = UFS_VENDOR_SAMSUNG, 299 .model = UFS_ANY_MODEL, 300 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 301 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | 302 UFS_DEVICE_QUIRK_PA_HIBER8TIME | 303 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, 304 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 305 .model = UFS_ANY_MODEL, 306 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME }, 307 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 308 .model = "hB8aL1" /*H28U62301AMR*/, 309 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME }, 310 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 311 .model = UFS_ANY_MODEL, 312 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 313 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 314 .model = "THGLF2G9C8KBADG", 315 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 316 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 317 .model = "THGLF2G9D8KBADG", 318 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 319 {} 320 }; 321 322 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba); 323 static void ufshcd_async_scan(void *data, async_cookie_t cookie); 324 static int ufshcd_reset_and_restore(struct ufs_hba *hba); 325 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); 326 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 327 static void ufshcd_hba_exit(struct ufs_hba *hba); 328 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); 329 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); 330 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 331 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 332 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 333 static void ufshcd_resume_clkscaling(struct ufs_hba *hba); 334 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); 335 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 336 bool scale_up); 337 static irqreturn_t ufshcd_intr(int irq, void *__hba); 338 static int ufshcd_change_power_mode(struct ufs_hba *hba, 339 struct ufs_pa_layer_attr *pwr_mode); 340 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); 341 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); 342 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 343 struct ufs_vreg *vreg); 344 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 345 bool enable); 346 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); 347 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); 348 349 void ufshcd_enable_irq(struct ufs_hba *hba) 350 { 351 if (!hba->is_irq_enabled) { 352 enable_irq(hba->irq); 353 hba->is_irq_enabled = true; 354 } 355 } 356 EXPORT_SYMBOL_GPL(ufshcd_enable_irq); 357 358 void ufshcd_disable_irq(struct ufs_hba *hba) 359 { 360 if (hba->is_irq_enabled) { 361 disable_irq(hba->irq); 362 hba->is_irq_enabled = false; 363 } 364 } 365 EXPORT_SYMBOL_GPL(ufshcd_disable_irq); 366 367 /** 368 * ufshcd_enable_intr - enable interrupts 369 * @hba: per adapter instance 370 * @intrs: interrupt bits 371 */ 372 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 373 { 374 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 375 u32 new_val = old_val | intrs; 376 377 if (new_val != old_val) 378 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 379 } 380 381 /** 382 * ufshcd_disable_intr - disable interrupts 383 * @hba: per adapter instance 384 * @intrs: interrupt bits 385 */ 386 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 387 { 388 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 389 u32 new_val = old_val & ~intrs; 390 391 if (new_val != old_val) 392 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 393 } 394 395 static void ufshcd_configure_wb(struct ufs_hba *hba) 396 { 397 if (!ufshcd_is_wb_allowed(hba)) 398 return; 399 400 ufshcd_wb_toggle(hba, true); 401 402 ufshcd_wb_toggle_buf_flush_during_h8(hba, true); 403 404 if (ufshcd_is_wb_buf_flush_allowed(hba)) 405 ufshcd_wb_toggle_buf_flush(hba, true); 406 } 407 408 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, 409 enum ufs_trace_str_t str_t) 410 { 411 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; 412 struct utp_upiu_header *header; 413 414 if (!trace_ufshcd_upiu_enabled()) 415 return; 416 417 if (str_t == UFS_CMD_SEND) 418 header = &rq->header; 419 else 420 header = &hba->lrb[tag].ucd_rsp_ptr->header; 421 422 trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, 423 UFS_TSF_CDB); 424 } 425 426 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, 427 enum ufs_trace_str_t str_t, 428 struct utp_upiu_req *rq_rsp) 429 { 430 if (!trace_ufshcd_upiu_enabled()) 431 return; 432 433 trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, 434 &rq_rsp->qr, UFS_TSF_OSF); 435 } 436 437 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, 438 enum ufs_trace_str_t str_t) 439 { 440 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag]; 441 442 if (!trace_ufshcd_upiu_enabled()) 443 return; 444 445 if (str_t == UFS_TM_SEND) 446 trace_ufshcd_upiu(hba, str_t, 447 &descp->upiu_req.req_header, 448 &descp->upiu_req.input_param1, 449 UFS_TSF_TM_INPUT); 450 else 451 trace_ufshcd_upiu(hba, str_t, 452 &descp->upiu_rsp.rsp_header, 453 &descp->upiu_rsp.output_param1, 454 UFS_TSF_TM_OUTPUT); 455 } 456 457 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, 458 const struct uic_command *ucmd, 459 enum ufs_trace_str_t str_t) 460 { 461 u32 cmd; 462 463 if (!trace_ufshcd_uic_command_enabled()) 464 return; 465 466 if (str_t == UFS_CMD_SEND) 467 cmd = ucmd->command; 468 else 469 cmd = ufshcd_readl(hba, REG_UIC_COMMAND); 470 471 trace_ufshcd_uic_command(hba, str_t, cmd, 472 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), 473 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), 474 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 475 } 476 477 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, 478 enum ufs_trace_str_t str_t) 479 { 480 u64 lba = 0; 481 u8 opcode = 0, group_id = 0; 482 u32 doorbell = 0; 483 u32 intr; 484 u32 hwq_id = 0; 485 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 486 struct scsi_cmnd *cmd = lrbp->cmd; 487 struct request *rq = scsi_cmd_to_rq(cmd); 488 int transfer_len = -1; 489 490 if (!cmd) 491 return; 492 493 /* trace UPIU also */ 494 ufshcd_add_cmd_upiu_trace(hba, tag, str_t); 495 if (!trace_ufshcd_command_enabled()) 496 return; 497 498 opcode = cmd->cmnd[0]; 499 500 if (opcode == READ_10 || opcode == WRITE_10) { 501 /* 502 * Currently we only fully trace read(10) and write(10) commands 503 */ 504 transfer_len = 505 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 506 lba = scsi_get_lba(cmd); 507 if (opcode == WRITE_10) 508 group_id = lrbp->cmd->cmnd[6]; 509 } else if (opcode == UNMAP) { 510 /* 511 * The number of Bytes to be unmapped beginning with the lba. 512 */ 513 transfer_len = blk_rq_bytes(rq); 514 lba = scsi_get_lba(cmd); 515 } 516 517 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 518 519 if (hba->mcq_enabled) { 520 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 521 522 hwq_id = hwq->id; 523 } else { 524 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 525 } 526 trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, 527 transfer_len, intr, lba, opcode, group_id); 528 } 529 530 static void ufshcd_print_clk_freqs(struct ufs_hba *hba) 531 { 532 struct ufs_clk_info *clki; 533 struct list_head *head = &hba->clk_list_head; 534 535 if (list_empty(head)) 536 return; 537 538 list_for_each_entry(clki, head, list) { 539 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && 540 clki->max_freq) 541 dev_err(hba->dev, "clk: %s, rate: %u\n", 542 clki->name, clki->curr_freq); 543 } 544 } 545 546 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, 547 const char *err_name) 548 { 549 int i; 550 bool found = false; 551 const struct ufs_event_hist *e; 552 553 if (id >= UFS_EVT_CNT) 554 return; 555 556 e = &hba->ufs_stats.event[id]; 557 558 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { 559 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; 560 561 if (e->tstamp[p] == 0) 562 continue; 563 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, 564 e->val[p], div_u64(e->tstamp[p], 1000)); 565 found = true; 566 } 567 568 if (!found) 569 dev_err(hba->dev, "No record of %s\n", err_name); 570 else 571 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); 572 } 573 574 static void ufshcd_print_evt_hist(struct ufs_hba *hba) 575 { 576 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 577 578 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); 579 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); 580 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); 581 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); 582 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); 583 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, 584 "auto_hibern8_err"); 585 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); 586 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, 587 "link_startup_fail"); 588 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); 589 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, 590 "suspend_fail"); 591 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail"); 592 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR, 593 "wlun suspend_fail"); 594 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); 595 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); 596 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); 597 598 ufshcd_vops_dbg_register_dump(hba); 599 } 600 601 static 602 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) 603 { 604 const struct ufshcd_lrb *lrbp; 605 int prdt_length; 606 607 lrbp = &hba->lrb[tag]; 608 609 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", 610 tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); 611 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", 612 tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); 613 dev_err(hba->dev, 614 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", 615 tag, (u64)lrbp->utrd_dma_addr); 616 617 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, 618 sizeof(struct utp_transfer_req_desc)); 619 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, 620 (u64)lrbp->ucd_req_dma_addr); 621 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, 622 sizeof(struct utp_upiu_req)); 623 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, 624 (u64)lrbp->ucd_rsp_dma_addr); 625 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, 626 sizeof(struct utp_upiu_rsp)); 627 628 prdt_length = le16_to_cpu( 629 lrbp->utr_descriptor_ptr->prd_table_length); 630 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 631 prdt_length /= ufshcd_sg_entry_size(hba); 632 633 dev_err(hba->dev, 634 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", 635 tag, prdt_length, 636 (u64)lrbp->ucd_prdt_dma_addr); 637 638 if (pr_prdt) 639 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, 640 ufshcd_sg_entry_size(hba) * prdt_length); 641 } 642 643 static bool ufshcd_print_tr_iter(struct request *req, void *priv) 644 { 645 struct scsi_device *sdev = req->q->queuedata; 646 struct Scsi_Host *shost = sdev->host; 647 struct ufs_hba *hba = shost_priv(shost); 648 649 ufshcd_print_tr(hba, req->tag, *(bool *)priv); 650 651 return true; 652 } 653 654 /** 655 * ufshcd_print_trs_all - print trs for all started requests. 656 * @hba: per-adapter instance. 657 * @pr_prdt: need to print prdt or not. 658 */ 659 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt) 660 { 661 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt); 662 } 663 664 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) 665 { 666 int tag; 667 668 for_each_set_bit(tag, &bitmap, hba->nutmrs) { 669 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag]; 670 671 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); 672 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp)); 673 } 674 } 675 676 static void ufshcd_print_host_state(struct ufs_hba *hba) 677 { 678 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; 679 680 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); 681 dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", 682 scsi_host_busy(hba->host), hba->outstanding_tasks); 683 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", 684 hba->saved_err, hba->saved_uic_err); 685 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", 686 hba->curr_dev_pwr_mode, hba->uic_link_state); 687 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", 688 hba->pm_op_in_progress, hba->is_sys_suspended); 689 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", 690 hba->auto_bkops_enabled, hba->host->host_self_blocked); 691 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); 692 dev_err(hba->dev, 693 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", 694 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), 695 hba->ufs_stats.hibern8_exit_cnt); 696 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", 697 hba->eh_flags, hba->req_abort_count); 698 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", 699 hba->ufs_version, hba->capabilities, hba->caps); 700 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, 701 hba->dev_quirks); 702 if (sdev_ufs) 703 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n", 704 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev); 705 706 ufshcd_print_clk_freqs(hba); 707 } 708 709 /** 710 * ufshcd_print_pwr_info - print power params as saved in hba 711 * power info 712 * @hba: per-adapter instance 713 */ 714 static void ufshcd_print_pwr_info(struct ufs_hba *hba) 715 { 716 static const char * const names[] = { 717 "INVALID MODE", 718 "FAST MODE", 719 "SLOW_MODE", 720 "INVALID MODE", 721 "FASTAUTO_MODE", 722 "SLOWAUTO_MODE", 723 "INVALID MODE", 724 }; 725 726 /* 727 * Using dev_dbg to avoid messages during runtime PM to avoid 728 * never-ending cycles of messages written back to storage by user space 729 * causing runtime resume, causing more messages and so on. 730 */ 731 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", 732 __func__, 733 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, 734 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, 735 names[hba->pwr_info.pwr_rx], 736 names[hba->pwr_info.pwr_tx], 737 hba->pwr_info.hs_rate); 738 } 739 740 static void ufshcd_device_reset(struct ufs_hba *hba) 741 { 742 int err; 743 744 err = ufshcd_vops_device_reset(hba); 745 746 if (!err) { 747 ufshcd_set_ufs_dev_active(hba); 748 if (ufshcd_is_wb_allowed(hba)) { 749 hba->dev_info.wb_enabled = false; 750 hba->dev_info.wb_buf_flush_enabled = false; 751 } 752 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 753 hba->dev_info.rtc_time_baseline = 0; 754 } 755 if (err != -EOPNOTSUPP) 756 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); 757 } 758 759 void ufshcd_delay_us(unsigned long us, unsigned long tolerance) 760 { 761 if (!us) 762 return; 763 764 if (us < 10) 765 udelay(us); 766 else 767 usleep_range(us, us + tolerance); 768 } 769 EXPORT_SYMBOL_GPL(ufshcd_delay_us); 770 771 /** 772 * ufshcd_wait_for_register - wait for register value to change 773 * @hba: per-adapter interface 774 * @reg: mmio register offset 775 * @mask: mask to apply to the read register value 776 * @val: value to wait for 777 * @interval_us: polling interval in microseconds 778 * @timeout_ms: timeout in milliseconds 779 * 780 * Return: -ETIMEDOUT on error, zero on success. 781 */ 782 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 783 u32 val, unsigned long interval_us, 784 unsigned long timeout_ms) 785 { 786 u32 v; 787 788 val &= mask; /* ignore bits that we don't intend to wait on */ 789 790 return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, 791 interval_us, timeout_ms * 1000, false, hba, reg); 792 } 793 794 /** 795 * ufshcd_get_intr_mask - Get the interrupt bit mask 796 * @hba: Pointer to adapter instance 797 * 798 * Return: interrupt bit mask per version 799 */ 800 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 801 { 802 if (hba->ufs_version <= ufshci_version(2, 0)) 803 return INTERRUPT_MASK_ALL_VER_11; 804 805 return INTERRUPT_MASK_ALL_VER_21; 806 } 807 808 /** 809 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 810 * @hba: Pointer to adapter instance 811 * 812 * Return: UFSHCI version supported by the controller 813 */ 814 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 815 { 816 u32 ufshci_ver; 817 818 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION) 819 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba); 820 else 821 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION); 822 823 /* 824 * UFSHCI v1.x uses a different version scheme, in order 825 * to allow the use of comparisons with the ufshci_version 826 * function, we convert it to the same scheme as ufs 2.0+. 827 */ 828 if (ufshci_ver & 0x00010000) 829 return ufshci_version(1, ufshci_ver & 0x00000100); 830 831 return ufshci_ver; 832 } 833 834 /** 835 * ufshcd_is_device_present - Check if any device connected to 836 * the host controller 837 * @hba: pointer to adapter instance 838 * 839 * Return: true if device present, false if no device detected 840 */ 841 static inline bool ufshcd_is_device_present(struct ufs_hba *hba) 842 { 843 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT; 844 } 845 846 /** 847 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 848 * @lrbp: pointer to local command reference block 849 * @cqe: pointer to the completion queue entry 850 * 851 * This function is used to get the OCS field from UTRD 852 * 853 * Return: the OCS field in the UTRD. 854 */ 855 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, 856 struct cq_entry *cqe) 857 { 858 if (cqe) 859 return le32_to_cpu(cqe->status) & MASK_OCS; 860 861 return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; 862 } 863 864 /** 865 * ufshcd_utrl_clear() - Clear requests from the controller request list. 866 * @hba: per adapter instance 867 * @mask: mask with one bit set for each request to be cleared 868 */ 869 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) 870 { 871 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 872 mask = ~mask; 873 /* 874 * From the UFSHCI specification: "UTP Transfer Request List CLear 875 * Register (UTRLCLR): This field is bit significant. Each bit 876 * corresponds to a slot in the UTP Transfer Request List, where bit 0 877 * corresponds to request slot 0. A bit in this field is set to ‘0’ 878 * by host software to indicate to the host controller that a transfer 879 * request slot is cleared. The host controller 880 * shall free up any resources associated to the request slot 881 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The 882 * host software indicates no change to request slots by setting the 883 * associated bits in this field to ‘1’. Bits in this field shall only 884 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’." 885 */ 886 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR); 887 } 888 889 /** 890 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register 891 * @hba: per adapter instance 892 * @pos: position of the bit to be cleared 893 */ 894 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) 895 { 896 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 897 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 898 else 899 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 900 } 901 902 /** 903 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 904 * @reg: Register value of host controller status 905 * 906 * Return: 0 on success; a positive value if failed. 907 */ 908 static inline int ufshcd_get_lists_status(u32 reg) 909 { 910 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); 911 } 912 913 /** 914 * ufshcd_get_uic_cmd_result - Get the UIC command result 915 * @hba: Pointer to adapter instance 916 * 917 * This function gets the result of UIC command completion 918 * 919 * Return: 0 on success; non-zero value on error. 920 */ 921 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 922 { 923 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 924 MASK_UIC_COMMAND_RESULT; 925 } 926 927 /** 928 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 929 * @hba: Pointer to adapter instance 930 * 931 * This function gets UIC command argument3 932 * 933 * Return: 0 on success; non-zero value on error. 934 */ 935 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 936 { 937 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 938 } 939 940 /** 941 * ufshcd_get_req_rsp - returns the TR response transaction type 942 * @ucd_rsp_ptr: pointer to response UPIU 943 * 944 * Return: UPIU type. 945 */ 946 static inline enum upiu_response_transaction 947 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 948 { 949 return ucd_rsp_ptr->header.transaction_code; 950 } 951 952 /** 953 * ufshcd_is_exception_event - Check if the device raised an exception event 954 * @ucd_rsp_ptr: pointer to response UPIU 955 * 956 * The function checks if the device raised an exception event indicated in 957 * the Device Information field of response UPIU. 958 * 959 * Return: true if exception is raised, false otherwise. 960 */ 961 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 962 { 963 return ucd_rsp_ptr->header.device_information & 1; 964 } 965 966 /** 967 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 968 * @hba: per adapter instance 969 */ 970 static inline void 971 ufshcd_reset_intr_aggr(struct ufs_hba *hba) 972 { 973 ufshcd_writel(hba, INT_AGGR_ENABLE | 974 INT_AGGR_COUNTER_AND_TIMER_RESET, 975 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 976 } 977 978 /** 979 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 980 * @hba: per adapter instance 981 * @cnt: Interrupt aggregation counter threshold 982 * @tmout: Interrupt aggregation timeout value 983 */ 984 static inline void 985 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 986 { 987 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 988 INT_AGGR_COUNTER_THLD_VAL(cnt) | 989 INT_AGGR_TIMEOUT_VAL(tmout), 990 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 991 } 992 993 /** 994 * ufshcd_disable_intr_aggr - Disables interrupt aggregation. 995 * @hba: per adapter instance 996 */ 997 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) 998 { 999 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 1000 } 1001 1002 /** 1003 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 1004 * When run-stop registers are set to 1, it indicates the 1005 * host controller that it can process the requests 1006 * @hba: per adapter instance 1007 */ 1008 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 1009 { 1010 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 1011 REG_UTP_TASK_REQ_LIST_RUN_STOP); 1012 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 1013 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 1014 } 1015 1016 /** 1017 * ufshcd_hba_start - Start controller initialization sequence 1018 * @hba: per adapter instance 1019 */ 1020 static inline void ufshcd_hba_start(struct ufs_hba *hba) 1021 { 1022 u32 val = CONTROLLER_ENABLE; 1023 1024 if (ufshcd_crypto_enable(hba)) 1025 val |= CRYPTO_GENERAL_ENABLE; 1026 1027 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); 1028 } 1029 1030 /** 1031 * ufshcd_is_hba_active - Get controller state 1032 * @hba: per adapter instance 1033 * 1034 * Return: true if and only if the controller is active. 1035 */ 1036 bool ufshcd_is_hba_active(struct ufs_hba *hba) 1037 { 1038 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; 1039 } 1040 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); 1041 1042 /** 1043 * ufshcd_pm_qos_init - initialize PM QoS request 1044 * @hba: per adapter instance 1045 */ 1046 void ufshcd_pm_qos_init(struct ufs_hba *hba) 1047 { 1048 1049 if (hba->pm_qos_enabled) 1050 return; 1051 1052 cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); 1053 1054 if (cpu_latency_qos_request_active(&hba->pm_qos_req)) 1055 hba->pm_qos_enabled = true; 1056 } 1057 1058 /** 1059 * ufshcd_pm_qos_exit - remove request from PM QoS 1060 * @hba: per adapter instance 1061 */ 1062 void ufshcd_pm_qos_exit(struct ufs_hba *hba) 1063 { 1064 if (!hba->pm_qos_enabled) 1065 return; 1066 1067 cpu_latency_qos_remove_request(&hba->pm_qos_req); 1068 hba->pm_qos_enabled = false; 1069 } 1070 1071 /** 1072 * ufshcd_pm_qos_update - update PM QoS request 1073 * @hba: per adapter instance 1074 * @on: If True, vote for perf PM QoS mode otherwise power save mode 1075 */ 1076 static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) 1077 { 1078 if (!hba->pm_qos_enabled) 1079 return; 1080 1081 cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); 1082 } 1083 1084 /** 1085 * ufshcd_set_clk_freq - set UFS controller clock frequencies 1086 * @hba: per adapter instance 1087 * @scale_up: If True, set max possible frequency othewise set low frequency 1088 * 1089 * Return: 0 if successful; < 0 upon failure. 1090 */ 1091 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) 1092 { 1093 int ret = 0; 1094 struct ufs_clk_info *clki; 1095 struct list_head *head = &hba->clk_list_head; 1096 1097 if (list_empty(head)) 1098 goto out; 1099 1100 list_for_each_entry(clki, head, list) { 1101 if (!IS_ERR_OR_NULL(clki->clk)) { 1102 if (scale_up && clki->max_freq) { 1103 if (clki->curr_freq == clki->max_freq) 1104 continue; 1105 1106 ret = clk_set_rate(clki->clk, clki->max_freq); 1107 if (ret) { 1108 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1109 __func__, clki->name, 1110 clki->max_freq, ret); 1111 break; 1112 } 1113 trace_ufshcd_clk_scaling(hba, 1114 "scaled up", clki->name, 1115 clki->curr_freq, 1116 clki->max_freq); 1117 1118 clki->curr_freq = clki->max_freq; 1119 1120 } else if (!scale_up && clki->min_freq) { 1121 if (clki->curr_freq == clki->min_freq) 1122 continue; 1123 1124 ret = clk_set_rate(clki->clk, clki->min_freq); 1125 if (ret) { 1126 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1127 __func__, clki->name, 1128 clki->min_freq, ret); 1129 break; 1130 } 1131 trace_ufshcd_clk_scaling(hba, 1132 "scaled down", clki->name, 1133 clki->curr_freq, 1134 clki->min_freq); 1135 clki->curr_freq = clki->min_freq; 1136 } 1137 } 1138 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 1139 clki->name, clk_get_rate(clki->clk)); 1140 } 1141 1142 out: 1143 return ret; 1144 } 1145 1146 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, 1147 struct dev_pm_opp *opp, void *data, 1148 bool scaling_down) 1149 { 1150 struct ufs_hba *hba = dev_get_drvdata(dev); 1151 struct list_head *head = &hba->clk_list_head; 1152 struct ufs_clk_info *clki; 1153 unsigned long freq; 1154 u8 idx = 0; 1155 int ret; 1156 1157 list_for_each_entry(clki, head, list) { 1158 if (!IS_ERR_OR_NULL(clki->clk)) { 1159 freq = dev_pm_opp_get_freq_indexed(opp, idx++); 1160 1161 /* Do not set rate for clocks having frequency as 0 */ 1162 if (!freq) 1163 continue; 1164 1165 ret = clk_set_rate(clki->clk, freq); 1166 if (ret) { 1167 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", 1168 __func__, clki->name, freq, ret); 1169 return ret; 1170 } 1171 1172 trace_ufshcd_clk_scaling(hba, 1173 (scaling_down ? "scaled down" : "scaled up"), 1174 clki->name, hba->clk_scaling.target_freq, freq); 1175 } 1176 } 1177 1178 return 0; 1179 } 1180 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); 1181 1182 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) 1183 { 1184 struct dev_pm_opp *opp; 1185 int ret; 1186 1187 opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, 1188 &freq, 0); 1189 if (IS_ERR(opp)) 1190 return PTR_ERR(opp); 1191 1192 ret = dev_pm_opp_set_opp(hba->dev, opp); 1193 dev_pm_opp_put(opp); 1194 1195 return ret; 1196 } 1197 1198 /** 1199 * ufshcd_scale_clks - scale up or scale down UFS controller clocks 1200 * @hba: per adapter instance 1201 * @freq: frequency to scale 1202 * @scale_up: True if scaling up and false if scaling down 1203 * 1204 * Return: 0 if successful; < 0 upon failure. 1205 */ 1206 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 1207 bool scale_up) 1208 { 1209 int ret = 0; 1210 ktime_t start = ktime_get(); 1211 1212 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); 1213 if (ret) 1214 goto out; 1215 1216 if (hba->use_pm_opp) 1217 ret = ufshcd_opp_set_rate(hba, freq); 1218 else 1219 ret = ufshcd_set_clk_freq(hba, scale_up); 1220 if (ret) 1221 goto out; 1222 1223 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); 1224 if (ret) { 1225 if (hba->use_pm_opp) 1226 ufshcd_opp_set_rate(hba, 1227 hba->devfreq->previous_freq); 1228 else 1229 ufshcd_set_clk_freq(hba, !scale_up); 1230 goto out; 1231 } 1232 1233 ufshcd_pm_qos_update(hba, scale_up); 1234 1235 out: 1236 trace_ufshcd_profile_clk_scaling(hba, 1237 (scale_up ? "up" : "down"), 1238 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1239 return ret; 1240 } 1241 1242 /** 1243 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not 1244 * @hba: per adapter instance 1245 * @freq: frequency to scale 1246 * @scale_up: True if scaling up and false if scaling down 1247 * 1248 * Return: true if scaling is required, false otherwise. 1249 */ 1250 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, 1251 unsigned long freq, bool scale_up) 1252 { 1253 struct ufs_clk_info *clki; 1254 struct list_head *head = &hba->clk_list_head; 1255 1256 if (list_empty(head)) 1257 return false; 1258 1259 if (hba->use_pm_opp) 1260 return freq != hba->clk_scaling.target_freq; 1261 1262 list_for_each_entry(clki, head, list) { 1263 if (!IS_ERR_OR_NULL(clki->clk)) { 1264 if (scale_up && clki->max_freq) { 1265 if (clki->curr_freq == clki->max_freq) 1266 continue; 1267 return true; 1268 } else if (!scale_up && clki->min_freq) { 1269 if (clki->curr_freq == clki->min_freq) 1270 continue; 1271 return true; 1272 } 1273 } 1274 } 1275 1276 return false; 1277 } 1278 1279 /* 1280 * Determine the number of pending commands by counting the bits in the SCSI 1281 * device budget maps. This approach has been selected because a bit is set in 1282 * the budget map before scsi_host_queue_ready() checks the host_self_blocked 1283 * flag. The host_self_blocked flag can be modified by calling 1284 * scsi_block_requests() or scsi_unblock_requests(). 1285 */ 1286 static u32 ufshcd_pending_cmds(struct ufs_hba *hba) 1287 { 1288 const struct scsi_device *sdev; 1289 unsigned long flags; 1290 u32 pending = 0; 1291 1292 spin_lock_irqsave(hba->host->host_lock, flags); 1293 __shost_for_each_device(sdev, hba->host) 1294 pending += sbitmap_weight(&sdev->budget_map); 1295 spin_unlock_irqrestore(hba->host->host_lock, flags); 1296 1297 return pending; 1298 } 1299 1300 /* 1301 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1302 * has expired. 1303 * 1304 * Return: 0 upon success; -EBUSY upon timeout. 1305 */ 1306 static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, 1307 u64 wait_timeout_us) 1308 { 1309 int ret = 0; 1310 u32 tm_doorbell; 1311 u32 tr_pending; 1312 bool timeout = false, do_last_check = false; 1313 ktime_t start; 1314 1315 ufshcd_hold(hba); 1316 /* 1317 * Wait for all the outstanding tasks/transfer requests. 1318 * Verify by checking the doorbell registers are clear. 1319 */ 1320 start = ktime_get(); 1321 do { 1322 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { 1323 ret = -EBUSY; 1324 goto out; 1325 } 1326 1327 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 1328 tr_pending = ufshcd_pending_cmds(hba); 1329 if (!tm_doorbell && !tr_pending) { 1330 timeout = false; 1331 break; 1332 } else if (do_last_check) { 1333 break; 1334 } 1335 1336 io_schedule_timeout(msecs_to_jiffies(20)); 1337 if (ktime_to_us(ktime_sub(ktime_get(), start)) > 1338 wait_timeout_us) { 1339 timeout = true; 1340 /* 1341 * We might have scheduled out for long time so make 1342 * sure to check if doorbells are cleared by this time 1343 * or not. 1344 */ 1345 do_last_check = true; 1346 } 1347 } while (tm_doorbell || tr_pending); 1348 1349 if (timeout) { 1350 dev_err(hba->dev, 1351 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", 1352 __func__, tm_doorbell, tr_pending); 1353 ret = -EBUSY; 1354 } 1355 out: 1356 ufshcd_release(hba); 1357 return ret; 1358 } 1359 1360 /** 1361 * ufshcd_scale_gear - scale up/down UFS gear 1362 * @hba: per adapter instance 1363 * @target_gear: target gear to scale to 1364 * @scale_up: True for scaling up gear and false for scaling down 1365 * 1366 * Return: 0 for success; -EBUSY if scaling can't happen at this time; 1367 * non-zero for any other errors. 1368 */ 1369 static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) 1370 { 1371 int ret = 0; 1372 struct ufs_pa_layer_attr new_pwr_info; 1373 1374 if (target_gear) { 1375 new_pwr_info = hba->pwr_info; 1376 new_pwr_info.gear_tx = target_gear; 1377 new_pwr_info.gear_rx = target_gear; 1378 1379 goto config_pwr_mode; 1380 } 1381 1382 /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ 1383 if (scale_up) { 1384 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, 1385 sizeof(struct ufs_pa_layer_attr)); 1386 } else { 1387 memcpy(&new_pwr_info, &hba->pwr_info, 1388 sizeof(struct ufs_pa_layer_attr)); 1389 1390 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || 1391 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { 1392 /* save the current power mode */ 1393 memcpy(&hba->clk_scaling.saved_pwr_info, 1394 &hba->pwr_info, 1395 sizeof(struct ufs_pa_layer_attr)); 1396 1397 /* scale down gear */ 1398 new_pwr_info.gear_tx = hba->clk_scaling.min_gear; 1399 new_pwr_info.gear_rx = hba->clk_scaling.min_gear; 1400 } 1401 } 1402 1403 config_pwr_mode: 1404 /* check if the power mode needs to be changed or not? */ 1405 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); 1406 if (ret) 1407 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", 1408 __func__, ret, 1409 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, 1410 new_pwr_info.gear_tx, new_pwr_info.gear_rx); 1411 1412 return ret; 1413 } 1414 1415 /* 1416 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1417 * has expired. 1418 * 1419 * Return: 0 upon success; -EBUSY upon timeout. 1420 */ 1421 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) 1422 { 1423 int ret = 0; 1424 /* 1425 * make sure that there are no outstanding requests when 1426 * clock scaling is in progress 1427 */ 1428 mutex_lock(&hba->host->scan_mutex); 1429 blk_mq_quiesce_tagset(&hba->host->tag_set); 1430 mutex_lock(&hba->wb_mutex); 1431 down_write(&hba->clk_scaling_lock); 1432 1433 if (!hba->clk_scaling.is_allowed || 1434 ufshcd_wait_for_pending_cmds(hba, timeout_us)) { 1435 ret = -EBUSY; 1436 up_write(&hba->clk_scaling_lock); 1437 mutex_unlock(&hba->wb_mutex); 1438 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1439 mutex_unlock(&hba->host->scan_mutex); 1440 goto out; 1441 } 1442 1443 /* let's not get into low power until clock scaling is completed */ 1444 ufshcd_hold(hba); 1445 1446 out: 1447 return ret; 1448 } 1449 1450 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) 1451 { 1452 up_write(&hba->clk_scaling_lock); 1453 1454 /* Enable Write Booster if current gear requires it else disable it */ 1455 if (ufshcd_enable_wb_if_scaling_up(hba) && !err) 1456 ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); 1457 1458 mutex_unlock(&hba->wb_mutex); 1459 1460 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1461 mutex_unlock(&hba->host->scan_mutex); 1462 ufshcd_release(hba); 1463 } 1464 1465 /** 1466 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear 1467 * @hba: per adapter instance 1468 * @freq: frequency to scale 1469 * @scale_up: True for scaling up and false for scalin down 1470 * 1471 * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero 1472 * for any other errors. 1473 */ 1474 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, 1475 bool scale_up) 1476 { 1477 u32 old_gear = hba->pwr_info.gear_rx; 1478 u32 new_gear = 0; 1479 int ret = 0; 1480 1481 new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); 1482 1483 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); 1484 if (ret) 1485 return ret; 1486 1487 /* scale down the gear before scaling down clocks */ 1488 if (!scale_up) { 1489 ret = ufshcd_scale_gear(hba, new_gear, false); 1490 if (ret) 1491 goto out_unprepare; 1492 } 1493 1494 ret = ufshcd_scale_clks(hba, freq, scale_up); 1495 if (ret) { 1496 if (!scale_up) 1497 ufshcd_scale_gear(hba, old_gear, true); 1498 goto out_unprepare; 1499 } 1500 1501 /* scale up the gear after scaling up clocks */ 1502 if (scale_up) { 1503 ret = ufshcd_scale_gear(hba, new_gear, true); 1504 if (ret) { 1505 ufshcd_scale_clks(hba, hba->devfreq->previous_freq, 1506 false); 1507 goto out_unprepare; 1508 } 1509 } 1510 1511 out_unprepare: 1512 ufshcd_clock_scaling_unprepare(hba, ret); 1513 return ret; 1514 } 1515 1516 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) 1517 { 1518 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1519 clk_scaling.suspend_work); 1520 1521 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1522 { 1523 if (hba->clk_scaling.active_reqs || 1524 hba->clk_scaling.is_suspended) 1525 return; 1526 1527 hba->clk_scaling.is_suspended = true; 1528 hba->clk_scaling.window_start_t = 0; 1529 } 1530 1531 devfreq_suspend_device(hba->devfreq); 1532 } 1533 1534 static void ufshcd_clk_scaling_resume_work(struct work_struct *work) 1535 { 1536 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1537 clk_scaling.resume_work); 1538 1539 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1540 { 1541 if (!hba->clk_scaling.is_suspended) 1542 return; 1543 hba->clk_scaling.is_suspended = false; 1544 } 1545 1546 devfreq_resume_device(hba->devfreq); 1547 } 1548 1549 static int ufshcd_devfreq_target(struct device *dev, 1550 unsigned long *freq, u32 flags) 1551 { 1552 int ret = 0; 1553 struct ufs_hba *hba = dev_get_drvdata(dev); 1554 ktime_t start; 1555 bool scale_up = false, sched_clk_scaling_suspend_work = false; 1556 struct list_head *clk_list = &hba->clk_list_head; 1557 struct ufs_clk_info *clki; 1558 1559 if (!ufshcd_is_clkscaling_supported(hba)) 1560 return -EINVAL; 1561 1562 if (hba->use_pm_opp) { 1563 struct dev_pm_opp *opp; 1564 1565 /* Get the recommended frequency from OPP framework */ 1566 opp = devfreq_recommended_opp(dev, freq, flags); 1567 if (IS_ERR(opp)) 1568 return PTR_ERR(opp); 1569 1570 dev_pm_opp_put(opp); 1571 } else { 1572 /* Override with the closest supported frequency */ 1573 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, 1574 list); 1575 *freq = (unsigned long) clk_round_rate(clki->clk, *freq); 1576 } 1577 1578 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1579 { 1580 if (ufshcd_eh_in_progress(hba)) 1581 return 0; 1582 1583 /* Skip scaling clock when clock scaling is suspended */ 1584 if (hba->clk_scaling.is_suspended) { 1585 dev_warn(hba->dev, "clock scaling is suspended, skip"); 1586 return 0; 1587 } 1588 1589 if (!hba->clk_scaling.active_reqs) 1590 sched_clk_scaling_suspend_work = true; 1591 1592 if (list_empty(clk_list)) 1593 goto out; 1594 1595 /* Decide based on the target or rounded-off frequency and update */ 1596 if (hba->use_pm_opp) 1597 scale_up = *freq > hba->clk_scaling.target_freq; 1598 else 1599 scale_up = *freq == clki->max_freq; 1600 1601 if (!hba->use_pm_opp && !scale_up) 1602 *freq = clki->min_freq; 1603 1604 /* Update the frequency */ 1605 if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { 1606 ret = 0; 1607 goto out; /* no state change required */ 1608 } 1609 } 1610 1611 start = ktime_get(); 1612 ret = ufshcd_devfreq_scale(hba, *freq, scale_up); 1613 if (!ret) 1614 hba->clk_scaling.target_freq = *freq; 1615 1616 trace_ufshcd_profile_clk_scaling(hba, 1617 (scale_up ? "up" : "down"), 1618 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1619 1620 out: 1621 if (sched_clk_scaling_suspend_work && 1622 (!scale_up || hba->clk_scaling.suspend_on_no_request)) 1623 queue_work(hba->clk_scaling.workq, 1624 &hba->clk_scaling.suspend_work); 1625 1626 return ret; 1627 } 1628 1629 static int ufshcd_devfreq_get_dev_status(struct device *dev, 1630 struct devfreq_dev_status *stat) 1631 { 1632 struct ufs_hba *hba = dev_get_drvdata(dev); 1633 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 1634 ktime_t curr_t; 1635 1636 if (!ufshcd_is_clkscaling_supported(hba)) 1637 return -EINVAL; 1638 1639 memset(stat, 0, sizeof(*stat)); 1640 1641 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 1642 1643 curr_t = ktime_get(); 1644 if (!scaling->window_start_t) 1645 goto start_window; 1646 1647 /* 1648 * If current frequency is 0, then the ondemand governor considers 1649 * there's no initial frequency set. And it always requests to set 1650 * to max. frequency. 1651 */ 1652 if (hba->use_pm_opp) { 1653 stat->current_frequency = hba->clk_scaling.target_freq; 1654 } else { 1655 struct list_head *clk_list = &hba->clk_list_head; 1656 struct ufs_clk_info *clki; 1657 1658 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1659 stat->current_frequency = clki->curr_freq; 1660 } 1661 1662 if (scaling->is_busy_started) 1663 scaling->tot_busy_t += ktime_us_delta(curr_t, 1664 scaling->busy_start_t); 1665 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); 1666 stat->busy_time = scaling->tot_busy_t; 1667 start_window: 1668 scaling->window_start_t = curr_t; 1669 scaling->tot_busy_t = 0; 1670 1671 if (scaling->active_reqs) { 1672 scaling->busy_start_t = curr_t; 1673 scaling->is_busy_started = true; 1674 } else { 1675 scaling->busy_start_t = 0; 1676 scaling->is_busy_started = false; 1677 } 1678 1679 return 0; 1680 } 1681 1682 static int ufshcd_devfreq_init(struct ufs_hba *hba) 1683 { 1684 struct list_head *clk_list = &hba->clk_list_head; 1685 struct ufs_clk_info *clki; 1686 struct devfreq *devfreq; 1687 int ret; 1688 1689 /* Skip devfreq if we don't have any clocks in the list */ 1690 if (list_empty(clk_list)) 1691 return 0; 1692 1693 if (!hba->use_pm_opp) { 1694 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1695 dev_pm_opp_add(hba->dev, clki->min_freq, 0); 1696 dev_pm_opp_add(hba->dev, clki->max_freq, 0); 1697 } 1698 1699 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, 1700 &hba->vps->ondemand_data); 1701 devfreq = devfreq_add_device(hba->dev, 1702 &hba->vps->devfreq_profile, 1703 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1704 &hba->vps->ondemand_data); 1705 if (IS_ERR(devfreq)) { 1706 ret = PTR_ERR(devfreq); 1707 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); 1708 1709 if (!hba->use_pm_opp) { 1710 dev_pm_opp_remove(hba->dev, clki->min_freq); 1711 dev_pm_opp_remove(hba->dev, clki->max_freq); 1712 } 1713 return ret; 1714 } 1715 1716 hba->devfreq = devfreq; 1717 1718 return 0; 1719 } 1720 1721 static void ufshcd_devfreq_remove(struct ufs_hba *hba) 1722 { 1723 struct list_head *clk_list = &hba->clk_list_head; 1724 1725 if (!hba->devfreq) 1726 return; 1727 1728 devfreq_remove_device(hba->devfreq); 1729 hba->devfreq = NULL; 1730 1731 if (!hba->use_pm_opp) { 1732 struct ufs_clk_info *clki; 1733 1734 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1735 dev_pm_opp_remove(hba->dev, clki->min_freq); 1736 dev_pm_opp_remove(hba->dev, clki->max_freq); 1737 } 1738 } 1739 1740 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1741 { 1742 bool suspend = false; 1743 1744 cancel_work_sync(&hba->clk_scaling.suspend_work); 1745 cancel_work_sync(&hba->clk_scaling.resume_work); 1746 1747 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1748 { 1749 if (!hba->clk_scaling.is_suspended) { 1750 suspend = true; 1751 hba->clk_scaling.is_suspended = true; 1752 hba->clk_scaling.window_start_t = 0; 1753 } 1754 } 1755 1756 if (suspend) 1757 devfreq_suspend_device(hba->devfreq); 1758 } 1759 1760 static void ufshcd_resume_clkscaling(struct ufs_hba *hba) 1761 { 1762 bool resume = false; 1763 1764 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1765 { 1766 if (hba->clk_scaling.is_suspended) { 1767 resume = true; 1768 hba->clk_scaling.is_suspended = false; 1769 } 1770 } 1771 1772 if (resume) 1773 devfreq_resume_device(hba->devfreq); 1774 } 1775 1776 static ssize_t ufshcd_clkscale_enable_show(struct device *dev, 1777 struct device_attribute *attr, char *buf) 1778 { 1779 struct ufs_hba *hba = dev_get_drvdata(dev); 1780 1781 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled); 1782 } 1783 1784 static ssize_t ufshcd_clkscale_enable_store(struct device *dev, 1785 struct device_attribute *attr, const char *buf, size_t count) 1786 { 1787 struct ufs_hba *hba = dev_get_drvdata(dev); 1788 struct ufs_clk_info *clki; 1789 unsigned long freq; 1790 u32 value; 1791 int err = 0; 1792 1793 if (kstrtou32(buf, 0, &value)) 1794 return -EINVAL; 1795 1796 down(&hba->host_sem); 1797 if (!ufshcd_is_user_access_allowed(hba)) { 1798 err = -EBUSY; 1799 goto out; 1800 } 1801 1802 value = !!value; 1803 if (value == hba->clk_scaling.is_enabled) 1804 goto out; 1805 1806 ufshcd_rpm_get_sync(hba); 1807 ufshcd_hold(hba); 1808 1809 hba->clk_scaling.is_enabled = value; 1810 1811 if (value) { 1812 ufshcd_resume_clkscaling(hba); 1813 goto out_rel; 1814 } 1815 1816 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); 1817 freq = clki->max_freq; 1818 1819 ufshcd_suspend_clkscaling(hba); 1820 1821 if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) 1822 goto out_rel; 1823 1824 err = ufshcd_devfreq_scale(hba, freq, true); 1825 if (err) 1826 dev_err(hba->dev, "%s: failed to scale clocks up %d\n", 1827 __func__, err); 1828 else 1829 hba->clk_scaling.target_freq = freq; 1830 1831 out_rel: 1832 ufshcd_release(hba); 1833 ufshcd_rpm_put_sync(hba); 1834 out: 1835 up(&hba->host_sem); 1836 return err ? err : count; 1837 } 1838 1839 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba) 1840 { 1841 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; 1842 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; 1843 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); 1844 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; 1845 hba->clk_scaling.enable_attr.attr.mode = 0644; 1846 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) 1847 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); 1848 } 1849 1850 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) 1851 { 1852 if (hba->clk_scaling.enable_attr.attr.name) 1853 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); 1854 } 1855 1856 static void ufshcd_init_clk_scaling(struct ufs_hba *hba) 1857 { 1858 if (!ufshcd_is_clkscaling_supported(hba)) 1859 return; 1860 1861 if (!hba->clk_scaling.min_gear) 1862 hba->clk_scaling.min_gear = UFS_HS_G1; 1863 1864 if (!hba->clk_scaling.wb_gear) 1865 /* Use intermediate gear speed HS_G3 as the default wb_gear */ 1866 hba->clk_scaling.wb_gear = UFS_HS_G3; 1867 1868 INIT_WORK(&hba->clk_scaling.suspend_work, 1869 ufshcd_clk_scaling_suspend_work); 1870 INIT_WORK(&hba->clk_scaling.resume_work, 1871 ufshcd_clk_scaling_resume_work); 1872 1873 spin_lock_init(&hba->clk_scaling.lock); 1874 1875 hba->clk_scaling.workq = alloc_ordered_workqueue( 1876 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); 1877 1878 hba->clk_scaling.is_initialized = true; 1879 } 1880 1881 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) 1882 { 1883 if (!hba->clk_scaling.is_initialized) 1884 return; 1885 1886 ufshcd_remove_clk_scaling_sysfs(hba); 1887 destroy_workqueue(hba->clk_scaling.workq); 1888 ufshcd_devfreq_remove(hba); 1889 hba->clk_scaling.is_initialized = false; 1890 } 1891 1892 static void ufshcd_ungate_work(struct work_struct *work) 1893 { 1894 int ret; 1895 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1896 clk_gating.ungate_work); 1897 1898 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 1899 1900 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 1901 if (hba->clk_gating.state == CLKS_ON) 1902 return; 1903 } 1904 1905 ufshcd_hba_vreg_set_hpm(hba); 1906 ufshcd_setup_clocks(hba, true); 1907 1908 ufshcd_enable_irq(hba); 1909 1910 /* Exit from hibern8 */ 1911 if (ufshcd_can_hibern8_during_gating(hba)) { 1912 /* Prevent gating in this path */ 1913 hba->clk_gating.is_suspended = true; 1914 if (ufshcd_is_link_hibern8(hba)) { 1915 ret = ufshcd_uic_hibern8_exit(hba); 1916 if (ret) 1917 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 1918 __func__, ret); 1919 else 1920 ufshcd_set_link_active(hba); 1921 } 1922 hba->clk_gating.is_suspended = false; 1923 } 1924 } 1925 1926 /** 1927 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 1928 * Also, exit from hibern8 mode and set the link as active. 1929 * @hba: per adapter instance 1930 */ 1931 void ufshcd_hold(struct ufs_hba *hba) 1932 { 1933 bool flush_result; 1934 unsigned long flags; 1935 1936 if (!ufshcd_is_clkgating_allowed(hba) || 1937 !hba->clk_gating.is_initialized) 1938 return; 1939 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1940 hba->clk_gating.active_reqs++; 1941 1942 start: 1943 switch (hba->clk_gating.state) { 1944 case CLKS_ON: 1945 /* 1946 * Wait for the ungate work to complete if in progress. 1947 * Though the clocks may be in ON state, the link could 1948 * still be in hibner8 state if hibern8 is allowed 1949 * during clock gating. 1950 * Make sure we exit hibern8 state also in addition to 1951 * clocks being ON. 1952 */ 1953 if (ufshcd_can_hibern8_during_gating(hba) && 1954 ufshcd_is_link_hibern8(hba)) { 1955 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1956 flush_result = flush_work(&hba->clk_gating.ungate_work); 1957 if (hba->clk_gating.is_suspended && !flush_result) 1958 return; 1959 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1960 goto start; 1961 } 1962 break; 1963 case REQ_CLKS_OFF: 1964 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 1965 hba->clk_gating.state = CLKS_ON; 1966 trace_ufshcd_clk_gating(hba, 1967 hba->clk_gating.state); 1968 break; 1969 } 1970 /* 1971 * If we are here, it means gating work is either done or 1972 * currently running. Hence, fall through to cancel gating 1973 * work and to enable clocks. 1974 */ 1975 fallthrough; 1976 case CLKS_OFF: 1977 hba->clk_gating.state = REQ_CLKS_ON; 1978 trace_ufshcd_clk_gating(hba, 1979 hba->clk_gating.state); 1980 queue_work(hba->clk_gating.clk_gating_workq, 1981 &hba->clk_gating.ungate_work); 1982 /* 1983 * fall through to check if we should wait for this 1984 * work to be done or not. 1985 */ 1986 fallthrough; 1987 case REQ_CLKS_ON: 1988 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1989 flush_work(&hba->clk_gating.ungate_work); 1990 /* Make sure state is CLKS_ON before returning */ 1991 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1992 goto start; 1993 default: 1994 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 1995 __func__, hba->clk_gating.state); 1996 break; 1997 } 1998 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1999 } 2000 EXPORT_SYMBOL_GPL(ufshcd_hold); 2001 2002 static void ufshcd_gate_work(struct work_struct *work) 2003 { 2004 struct ufs_hba *hba = container_of(work, struct ufs_hba, 2005 clk_gating.gate_work.work); 2006 int ret; 2007 2008 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 2009 /* 2010 * In case you are here to cancel this work the gating state 2011 * would be marked as REQ_CLKS_ON. In this case save time by 2012 * skipping the gating work and exit after changing the clock 2013 * state to CLKS_ON. 2014 */ 2015 if (hba->clk_gating.is_suspended || 2016 hba->clk_gating.state != REQ_CLKS_OFF) { 2017 hba->clk_gating.state = CLKS_ON; 2018 trace_ufshcd_clk_gating(hba, 2019 hba->clk_gating.state); 2020 return; 2021 } 2022 2023 if (hba->clk_gating.active_reqs) 2024 return; 2025 } 2026 2027 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2028 if (ufshcd_is_ufs_dev_busy(hba) || 2029 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2030 return; 2031 } 2032 2033 /* put the link into hibern8 mode before turning off clocks */ 2034 if (ufshcd_can_hibern8_during_gating(hba)) { 2035 ret = ufshcd_uic_hibern8_enter(hba); 2036 if (ret) { 2037 hba->clk_gating.state = CLKS_ON; 2038 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 2039 __func__, ret); 2040 trace_ufshcd_clk_gating(hba, 2041 hba->clk_gating.state); 2042 return; 2043 } 2044 ufshcd_set_link_hibern8(hba); 2045 } 2046 2047 ufshcd_disable_irq(hba); 2048 2049 ufshcd_setup_clocks(hba, false); 2050 2051 /* Put the host controller in low power mode if possible */ 2052 ufshcd_hba_vreg_set_lpm(hba); 2053 /* 2054 * In case you are here to cancel this work the gating state 2055 * would be marked as REQ_CLKS_ON. In this case keep the state 2056 * as REQ_CLKS_ON which would anyway imply that clocks are off 2057 * and a request to turn them on is pending. By doing this way, 2058 * we keep the state machine in tact and this would ultimately 2059 * prevent from doing cancel work multiple times when there are 2060 * new requests arriving before the current cancel work is done. 2061 */ 2062 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2063 if (hba->clk_gating.state == REQ_CLKS_OFF) { 2064 hba->clk_gating.state = CLKS_OFF; 2065 trace_ufshcd_clk_gating(hba, 2066 hba->clk_gating.state); 2067 } 2068 } 2069 2070 static void __ufshcd_release(struct ufs_hba *hba) 2071 { 2072 lockdep_assert_held(&hba->clk_gating.lock); 2073 2074 if (!ufshcd_is_clkgating_allowed(hba)) 2075 return; 2076 2077 hba->clk_gating.active_reqs--; 2078 2079 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || 2080 !hba->clk_gating.is_initialized || 2081 hba->clk_gating.state == CLKS_OFF) 2082 return; 2083 2084 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2085 if (ufshcd_has_pending_tasks(hba) || 2086 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2087 return; 2088 } 2089 2090 hba->clk_gating.state = REQ_CLKS_OFF; 2091 trace_ufshcd_clk_gating(hba, hba->clk_gating.state); 2092 queue_delayed_work(hba->clk_gating.clk_gating_workq, 2093 &hba->clk_gating.gate_work, 2094 msecs_to_jiffies(hba->clk_gating.delay_ms)); 2095 } 2096 2097 void ufshcd_release(struct ufs_hba *hba) 2098 { 2099 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2100 __ufshcd_release(hba); 2101 } 2102 EXPORT_SYMBOL_GPL(ufshcd_release); 2103 2104 static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 2105 struct device_attribute *attr, char *buf) 2106 { 2107 struct ufs_hba *hba = dev_get_drvdata(dev); 2108 2109 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); 2110 } 2111 2112 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) 2113 { 2114 struct ufs_hba *hba = dev_get_drvdata(dev); 2115 2116 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2117 hba->clk_gating.delay_ms = value; 2118 } 2119 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); 2120 2121 static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 2122 struct device_attribute *attr, const char *buf, size_t count) 2123 { 2124 unsigned long value; 2125 2126 if (kstrtoul(buf, 0, &value)) 2127 return -EINVAL; 2128 2129 ufshcd_clkgate_delay_set(dev, value); 2130 return count; 2131 } 2132 2133 static ssize_t ufshcd_clkgate_enable_show(struct device *dev, 2134 struct device_attribute *attr, char *buf) 2135 { 2136 struct ufs_hba *hba = dev_get_drvdata(dev); 2137 2138 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); 2139 } 2140 2141 static ssize_t ufshcd_clkgate_enable_store(struct device *dev, 2142 struct device_attribute *attr, const char *buf, size_t count) 2143 { 2144 struct ufs_hba *hba = dev_get_drvdata(dev); 2145 u32 value; 2146 2147 if (kstrtou32(buf, 0, &value)) 2148 return -EINVAL; 2149 2150 value = !!value; 2151 2152 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2153 2154 if (value == hba->clk_gating.is_enabled) 2155 return count; 2156 2157 if (value) 2158 __ufshcd_release(hba); 2159 else 2160 hba->clk_gating.active_reqs++; 2161 2162 hba->clk_gating.is_enabled = value; 2163 2164 return count; 2165 } 2166 2167 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba) 2168 { 2169 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 2170 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 2171 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 2172 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 2173 hba->clk_gating.delay_attr.attr.mode = 0644; 2174 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 2175 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 2176 2177 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; 2178 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; 2179 sysfs_attr_init(&hba->clk_gating.enable_attr.attr); 2180 hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; 2181 hba->clk_gating.enable_attr.attr.mode = 0644; 2182 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) 2183 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); 2184 } 2185 2186 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) 2187 { 2188 if (hba->clk_gating.delay_attr.attr.name) 2189 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 2190 if (hba->clk_gating.enable_attr.attr.name) 2191 device_remove_file(hba->dev, &hba->clk_gating.enable_attr); 2192 } 2193 2194 static void ufshcd_init_clk_gating(struct ufs_hba *hba) 2195 { 2196 if (!ufshcd_is_clkgating_allowed(hba)) 2197 return; 2198 2199 hba->clk_gating.state = CLKS_ON; 2200 2201 hba->clk_gating.delay_ms = 150; 2202 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 2203 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 2204 2205 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( 2206 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, 2207 hba->host->host_no); 2208 2209 ufshcd_init_clk_gating_sysfs(hba); 2210 2211 hba->clk_gating.is_enabled = true; 2212 hba->clk_gating.is_initialized = true; 2213 } 2214 2215 static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 2216 { 2217 if (!hba->clk_gating.is_initialized) 2218 return; 2219 2220 ufshcd_remove_clk_gating_sysfs(hba); 2221 2222 /* Ungate the clock if necessary. */ 2223 ufshcd_hold(hba); 2224 hba->clk_gating.is_initialized = false; 2225 ufshcd_release(hba); 2226 2227 destroy_workqueue(hba->clk_gating.clk_gating_workq); 2228 } 2229 2230 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 2231 { 2232 bool queue_resume_work = false; 2233 ktime_t curr_t = ktime_get(); 2234 2235 if (!ufshcd_is_clkscaling_supported(hba)) 2236 return; 2237 2238 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2239 2240 if (!hba->clk_scaling.active_reqs++) 2241 queue_resume_work = true; 2242 2243 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) 2244 return; 2245 2246 if (queue_resume_work) 2247 queue_work(hba->clk_scaling.workq, 2248 &hba->clk_scaling.resume_work); 2249 2250 if (!hba->clk_scaling.window_start_t) { 2251 hba->clk_scaling.window_start_t = curr_t; 2252 hba->clk_scaling.tot_busy_t = 0; 2253 hba->clk_scaling.is_busy_started = false; 2254 } 2255 2256 if (!hba->clk_scaling.is_busy_started) { 2257 hba->clk_scaling.busy_start_t = curr_t; 2258 hba->clk_scaling.is_busy_started = true; 2259 } 2260 } 2261 2262 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 2263 { 2264 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 2265 2266 if (!ufshcd_is_clkscaling_supported(hba)) 2267 return; 2268 2269 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2270 2271 hba->clk_scaling.active_reqs--; 2272 if (!scaling->active_reqs && scaling->is_busy_started) { 2273 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 2274 scaling->busy_start_t)); 2275 scaling->busy_start_t = 0; 2276 scaling->is_busy_started = false; 2277 } 2278 } 2279 2280 static inline int ufshcd_monitor_opcode2dir(u8 opcode) 2281 { 2282 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16) 2283 return READ; 2284 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16) 2285 return WRITE; 2286 else 2287 return -EINVAL; 2288 } 2289 2290 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2291 struct ufshcd_lrb *lrbp) 2292 { 2293 const struct ufs_hba_monitor *m = &hba->monitor; 2294 2295 return (m->enabled && lrbp && lrbp->cmd && 2296 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && 2297 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); 2298 } 2299 2300 static void ufshcd_start_monitor(struct ufs_hba *hba, 2301 const struct ufshcd_lrb *lrbp) 2302 { 2303 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2304 unsigned long flags; 2305 2306 spin_lock_irqsave(hba->host->host_lock, flags); 2307 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0) 2308 hba->monitor.busy_start_ts[dir] = ktime_get(); 2309 spin_unlock_irqrestore(hba->host->host_lock, flags); 2310 } 2311 2312 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) 2313 { 2314 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2315 unsigned long flags; 2316 2317 spin_lock_irqsave(hba->host->host_lock, flags); 2318 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2319 const struct request *req = scsi_cmd_to_rq(lrbp->cmd); 2320 struct ufs_hba_monitor *m = &hba->monitor; 2321 ktime_t now, inc, lat; 2322 2323 now = lrbp->compl_time_stamp; 2324 inc = ktime_sub(now, m->busy_start_ts[dir]); 2325 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc); 2326 m->nr_sec_rw[dir] += blk_rq_sectors(req); 2327 2328 /* Update latencies */ 2329 m->nr_req[dir]++; 2330 lat = ktime_sub(now, lrbp->issue_time_stamp); 2331 m->lat_sum[dir] += lat; 2332 if (m->lat_max[dir] < lat || !m->lat_max[dir]) 2333 m->lat_max[dir] = lat; 2334 if (m->lat_min[dir] > lat || !m->lat_min[dir]) 2335 m->lat_min[dir] = lat; 2336 2337 m->nr_queued[dir]--; 2338 /* Push forward the busy start of monitor */ 2339 m->busy_start_ts[dir] = now; 2340 } 2341 spin_unlock_irqrestore(hba->host->host_lock, flags); 2342 } 2343 2344 /** 2345 * ufshcd_send_command - Send SCSI or device management commands 2346 * @hba: per adapter instance 2347 * @task_tag: Task tag of the command 2348 * @hwq: pointer to hardware queue instance 2349 */ 2350 static inline 2351 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, 2352 struct ufs_hw_queue *hwq) 2353 { 2354 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 2355 unsigned long flags; 2356 2357 lrbp->issue_time_stamp = ktime_get(); 2358 lrbp->issue_time_stamp_local_clock = local_clock(); 2359 lrbp->compl_time_stamp = ktime_set(0, 0); 2360 lrbp->compl_time_stamp_local_clock = 0; 2361 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); 2362 if (lrbp->cmd) 2363 ufshcd_clk_scaling_start_busy(hba); 2364 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 2365 ufshcd_start_monitor(hba, lrbp); 2366 2367 if (hba->mcq_enabled) { 2368 int utrd_size = sizeof(struct utp_transfer_req_desc); 2369 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; 2370 struct utp_transfer_req_desc *dest; 2371 2372 spin_lock(&hwq->sq_lock); 2373 dest = hwq->sqe_base_addr + hwq->sq_tail_slot; 2374 memcpy(dest, src, utrd_size); 2375 ufshcd_inc_sq_tail(hwq); 2376 spin_unlock(&hwq->sq_lock); 2377 } else { 2378 spin_lock_irqsave(&hba->outstanding_lock, flags); 2379 if (hba->vops && hba->vops->setup_xfer_req) 2380 hba->vops->setup_xfer_req(hba, lrbp->task_tag, 2381 !!lrbp->cmd); 2382 __set_bit(lrbp->task_tag, &hba->outstanding_reqs); 2383 ufshcd_writel(hba, 1 << lrbp->task_tag, 2384 REG_UTP_TRANSFER_REQ_DOOR_BELL); 2385 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2386 } 2387 } 2388 2389 /** 2390 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2391 * @lrbp: pointer to local reference block 2392 */ 2393 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 2394 { 2395 u8 *const sense_buffer = lrbp->cmd->sense_buffer; 2396 u16 resp_len; 2397 int len; 2398 2399 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); 2400 if (sense_buffer && resp_len) { 2401 int len_to_copy; 2402 2403 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 2404 len_to_copy = min_t(int, UFS_SENSE_SIZE, len); 2405 2406 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, 2407 len_to_copy); 2408 } 2409 } 2410 2411 /** 2412 * ufshcd_copy_query_response() - Copy the Query Response and the data 2413 * descriptor 2414 * @hba: per adapter instance 2415 * @lrbp: pointer to local reference block 2416 * 2417 * Return: 0 upon success; < 0 upon failure. 2418 */ 2419 static 2420 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2421 { 2422 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2423 2424 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 2425 2426 /* Get the descriptor */ 2427 if (hba->dev_cmd.query.descriptor && 2428 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 2429 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 2430 GENERAL_UPIU_REQUEST_SIZE; 2431 u16 resp_len; 2432 u16 buf_len; 2433 2434 /* data segment length */ 2435 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 2436 .data_segment_length); 2437 buf_len = be16_to_cpu( 2438 hba->dev_cmd.query.request.upiu_req.length); 2439 if (likely(buf_len >= resp_len)) { 2440 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 2441 } else { 2442 dev_warn(hba->dev, 2443 "%s: rsp size %d is bigger than buffer size %d", 2444 __func__, resp_len, buf_len); 2445 return -EINVAL; 2446 } 2447 } 2448 2449 return 0; 2450 } 2451 2452 /** 2453 * ufshcd_hba_capabilities - Read controller capabilities 2454 * @hba: per adapter instance 2455 * 2456 * Return: 0 on success, negative on error. 2457 */ 2458 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) 2459 { 2460 int err; 2461 2462 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 2463 2464 /* nutrs and nutmrs are 0 based values */ 2465 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; 2466 hba->nutmrs = 2467 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2468 hba->reserved_slot = hba->nutrs - 1; 2469 2470 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; 2471 2472 /* Read crypto capabilities */ 2473 err = ufshcd_hba_init_crypto_capabilities(hba); 2474 if (err) { 2475 dev_err(hba->dev, "crypto setup failed\n"); 2476 return err; 2477 } 2478 2479 /* 2480 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and 2481 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which 2482 * means we can simply read values regardless of version. 2483 */ 2484 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); 2485 /* 2486 * 0h: legacy single doorbell support is available 2487 * 1h: indicate that legacy single doorbell support has been removed 2488 */ 2489 if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) 2490 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); 2491 else 2492 hba->lsdb_sup = true; 2493 2494 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); 2495 2496 return 0; 2497 } 2498 2499 /** 2500 * ufshcd_ready_for_uic_cmd - Check if controller is ready 2501 * to accept UIC commands 2502 * @hba: per adapter instance 2503 * 2504 * Return: true on success, else false. 2505 */ 2506 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 2507 { 2508 u32 val; 2509 int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, 2510 500, uic_cmd_timeout * 1000, false, hba, 2511 REG_CONTROLLER_STATUS); 2512 return ret == 0; 2513 } 2514 2515 /** 2516 * ufshcd_get_upmcrs - Get the power mode change request status 2517 * @hba: Pointer to adapter instance 2518 * 2519 * This function gets the UPMCRS field of HCS register 2520 * 2521 * Return: value of UPMCRS field. 2522 */ 2523 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 2524 { 2525 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 2526 } 2527 2528 /** 2529 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer 2530 * @hba: per adapter instance 2531 * @uic_cmd: UIC command 2532 */ 2533 static inline void 2534 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2535 { 2536 lockdep_assert_held(&hba->uic_cmd_mutex); 2537 2538 WARN_ON(hba->active_uic_cmd); 2539 2540 hba->active_uic_cmd = uic_cmd; 2541 2542 /* Write Args */ 2543 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 2544 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 2545 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 2546 2547 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND); 2548 2549 /* Write UIC Cmd */ 2550 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 2551 REG_UIC_COMMAND); 2552 } 2553 2554 /** 2555 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command 2556 * @hba: per adapter instance 2557 * @uic_cmd: UIC command 2558 * 2559 * Return: 0 only if success. 2560 */ 2561 static int 2562 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2563 { 2564 int ret; 2565 unsigned long flags; 2566 2567 lockdep_assert_held(&hba->uic_cmd_mutex); 2568 2569 if (wait_for_completion_timeout(&uic_cmd->done, 2570 msecs_to_jiffies(uic_cmd_timeout))) { 2571 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2572 } else { 2573 ret = -ETIMEDOUT; 2574 dev_err(hba->dev, 2575 "uic cmd 0x%x with arg3 0x%x completion timeout\n", 2576 uic_cmd->command, uic_cmd->argument3); 2577 2578 if (!uic_cmd->cmd_active) { 2579 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n", 2580 __func__); 2581 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2582 } 2583 } 2584 2585 spin_lock_irqsave(hba->host->host_lock, flags); 2586 hba->active_uic_cmd = NULL; 2587 spin_unlock_irqrestore(hba->host->host_lock, flags); 2588 2589 return ret; 2590 } 2591 2592 /** 2593 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2594 * @hba: per adapter instance 2595 * @uic_cmd: UIC command 2596 * 2597 * Return: 0 if successful; < 0 upon failure. 2598 */ 2599 static int 2600 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2601 { 2602 lockdep_assert_held(&hba->uic_cmd_mutex); 2603 2604 if (!ufshcd_ready_for_uic_cmd(hba)) { 2605 dev_err(hba->dev, 2606 "Controller not ready to accept UIC commands\n"); 2607 return -EIO; 2608 } 2609 2610 init_completion(&uic_cmd->done); 2611 2612 uic_cmd->cmd_active = 1; 2613 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 2614 2615 return 0; 2616 } 2617 2618 /** 2619 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2620 * @hba: per adapter instance 2621 * @uic_cmd: UIC command 2622 * 2623 * Return: 0 only if success. 2624 */ 2625 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2626 { 2627 unsigned long flags; 2628 int ret; 2629 2630 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 2631 return 0; 2632 2633 ufshcd_hold(hba); 2634 mutex_lock(&hba->uic_cmd_mutex); 2635 ufshcd_add_delay_before_dme_cmd(hba); 2636 2637 spin_lock_irqsave(hba->host->host_lock, flags); 2638 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); 2639 spin_unlock_irqrestore(hba->host->host_lock, flags); 2640 2641 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 2642 if (!ret) 2643 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 2644 2645 mutex_unlock(&hba->uic_cmd_mutex); 2646 2647 ufshcd_release(hba); 2648 return ret; 2649 } 2650 2651 /** 2652 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) 2653 * @hba: per-adapter instance 2654 * @lrbp: pointer to local reference block 2655 * @sg_entries: The number of sg lists actually used 2656 * @sg_list: Pointer to SG list 2657 */ 2658 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, 2659 struct scatterlist *sg_list) 2660 { 2661 struct ufshcd_sg_entry *prd; 2662 struct scatterlist *sg; 2663 int i; 2664 2665 if (sg_entries) { 2666 2667 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 2668 lrbp->utr_descriptor_ptr->prd_table_length = 2669 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); 2670 else 2671 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); 2672 2673 prd = lrbp->ucd_prdt_ptr; 2674 2675 for_each_sg(sg_list, sg, sg_entries, i) { 2676 const unsigned int len = sg_dma_len(sg); 2677 2678 /* 2679 * From the UFSHCI spec: "Data Byte Count (DBC): A '0' 2680 * based value that indicates the length, in bytes, of 2681 * the data block. A maximum of length of 256KB may 2682 * exist for any entry. Bits 1:0 of this field shall be 2683 * 11b to indicate Dword granularity. A value of '3' 2684 * indicates 4 bytes, '7' indicates 8 bytes, etc." 2685 */ 2686 WARN_ONCE(len > SZ_256K, "len = %#x\n", len); 2687 prd->size = cpu_to_le32(len - 1); 2688 prd->addr = cpu_to_le64(sg->dma_address); 2689 prd->reserved = 0; 2690 prd = (void *)prd + ufshcd_sg_entry_size(hba); 2691 } 2692 } else { 2693 lrbp->utr_descriptor_ptr->prd_table_length = 0; 2694 } 2695 } 2696 2697 /** 2698 * ufshcd_map_sg - Map scatter-gather list to prdt 2699 * @hba: per adapter instance 2700 * @lrbp: pointer to local reference block 2701 * 2702 * Return: 0 in case of success, non-zero value in case of failure. 2703 */ 2704 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2705 { 2706 struct scsi_cmnd *cmd = lrbp->cmd; 2707 int sg_segments = scsi_dma_map(cmd); 2708 2709 if (sg_segments < 0) 2710 return sg_segments; 2711 2712 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2713 2714 return ufshcd_crypto_fill_prdt(hba, lrbp); 2715 } 2716 2717 /** 2718 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request 2719 * descriptor according to request 2720 * @hba: per adapter instance 2721 * @lrbp: pointer to local reference block 2722 * @upiu_flags: flags required in the header 2723 * @cmd_dir: requests data direction 2724 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) 2725 */ 2726 static void 2727 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2728 u8 *upiu_flags, enum dma_data_direction cmd_dir, 2729 int ehs_length) 2730 { 2731 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 2732 struct request_desc_header *h = &req_desc->header; 2733 enum utp_data_direction data_direction; 2734 2735 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2736 2737 *h = (typeof(*h)){ }; 2738 2739 if (cmd_dir == DMA_FROM_DEVICE) { 2740 data_direction = UTP_DEVICE_TO_HOST; 2741 *upiu_flags = UPIU_CMD_FLAGS_READ; 2742 } else if (cmd_dir == DMA_TO_DEVICE) { 2743 data_direction = UTP_HOST_TO_DEVICE; 2744 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 2745 } else { 2746 data_direction = UTP_NO_DATA_TRANSFER; 2747 *upiu_flags = UPIU_CMD_FLAGS_NONE; 2748 } 2749 2750 h->command_type = lrbp->command_type; 2751 h->data_direction = data_direction; 2752 h->ehs_length = ehs_length; 2753 2754 if (lrbp->intr_cmd) 2755 h->interrupt = 1; 2756 2757 /* Prepare crypto related dwords */ 2758 ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); 2759 2760 /* 2761 * assigning invalid value for command status. Controller 2762 * updates OCS on command completion, with the command 2763 * status 2764 */ 2765 h->ocs = OCS_INVALID_COMMAND_STATUS; 2766 2767 req_desc->prd_table_length = 0; 2768 } 2769 2770 /** 2771 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2772 * for scsi commands 2773 * @lrbp: local reference block pointer 2774 * @upiu_flags: flags 2775 */ 2776 static 2777 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) 2778 { 2779 struct scsi_cmnd *cmd = lrbp->cmd; 2780 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2781 unsigned short cdb_len; 2782 2783 ucd_req_ptr->header = (struct utp_upiu_header){ 2784 .transaction_code = UPIU_TRANSACTION_COMMAND, 2785 .flags = upiu_flags, 2786 .lun = lrbp->lun, 2787 .task_tag = lrbp->task_tag, 2788 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, 2789 }; 2790 2791 WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag); 2792 2793 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2794 2795 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); 2796 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); 2797 2798 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2799 } 2800 2801 /** 2802 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2803 * @hba: UFS hba 2804 * @lrbp: local reference block pointer 2805 * @upiu_flags: flags 2806 */ 2807 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2808 struct ufshcd_lrb *lrbp, u8 upiu_flags) 2809 { 2810 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2811 struct ufs_query *query = &hba->dev_cmd.query; 2812 u16 len = be16_to_cpu(query->request.upiu_req.length); 2813 2814 /* Query request header */ 2815 ucd_req_ptr->header = (struct utp_upiu_header){ 2816 .transaction_code = UPIU_TRANSACTION_QUERY_REQ, 2817 .flags = upiu_flags, 2818 .lun = lrbp->lun, 2819 .task_tag = lrbp->task_tag, 2820 .query_function = query->request.query_func, 2821 /* Data segment length only need for WRITE_DESC */ 2822 .data_segment_length = 2823 query->request.upiu_req.opcode == 2824 UPIU_QUERY_OPCODE_WRITE_DESC ? 2825 cpu_to_be16(len) : 2826 0, 2827 }; 2828 2829 /* Copy the Query Request buffer as is */ 2830 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 2831 QUERY_OSF_SIZE); 2832 2833 /* Copy the Descriptor */ 2834 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2835 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2836 } 2837 2838 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) 2839 { 2840 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2841 2842 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2843 2844 ucd_req_ptr->header = (struct utp_upiu_header){ 2845 .transaction_code = UPIU_TRANSACTION_NOP_OUT, 2846 .task_tag = lrbp->task_tag, 2847 }; 2848 } 2849 2850 /** 2851 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2852 * for Device Management Purposes 2853 * @hba: per adapter instance 2854 * @lrbp: pointer to local reference block 2855 * 2856 * Return: 0 upon success; < 0 upon failure. 2857 */ 2858 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2859 struct ufshcd_lrb *lrbp) 2860 { 2861 u8 upiu_flags; 2862 int ret = 0; 2863 2864 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 2865 2866 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2867 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); 2868 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2869 ufshcd_prepare_utp_nop_upiu(lrbp); 2870 else 2871 ret = -EINVAL; 2872 2873 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2874 2875 return ret; 2876 } 2877 2878 /** 2879 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2880 * for SCSI Purposes 2881 * @hba: per adapter instance 2882 * @lrbp: pointer to local reference block 2883 */ 2884 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2885 { 2886 struct request *rq = scsi_cmd_to_rq(lrbp->cmd); 2887 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2888 u8 upiu_flags; 2889 2890 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); 2891 if (ioprio_class == IOPRIO_CLASS_RT) 2892 upiu_flags |= UPIU_CMD_FLAGS_CP; 2893 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 2894 } 2895 2896 static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag) 2897 { 2898 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2899 2900 lrbp->cmd = cmd; 2901 lrbp->task_tag = tag; 2902 lrbp->lun = lun; 2903 ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp); 2904 } 2905 2906 static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2907 struct scsi_cmnd *cmd, u8 lun, int tag) 2908 { 2909 __ufshcd_setup_cmd(lrbp, cmd, lun, tag); 2910 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2911 lrbp->req_abort_skip = false; 2912 2913 ufshcd_comp_scsi_upiu(hba, lrbp); 2914 } 2915 2916 /** 2917 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 2918 * @upiu_wlun_id: UPIU W-LUN id 2919 * 2920 * Return: SCSI W-LUN id. 2921 */ 2922 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 2923 { 2924 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 2925 } 2926 2927 static inline bool is_device_wlun(struct scsi_device *sdev) 2928 { 2929 return sdev->lun == 2930 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 2931 } 2932 2933 /* 2934 * Associate the UFS controller queue with the default and poll HCTX types. 2935 * Initialize the mq_map[] arrays. 2936 */ 2937 static void ufshcd_map_queues(struct Scsi_Host *shost) 2938 { 2939 struct ufs_hba *hba = shost_priv(shost); 2940 int i, queue_offset = 0; 2941 2942 if (!is_mcq_supported(hba)) { 2943 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; 2944 hba->nr_queues[HCTX_TYPE_READ] = 0; 2945 hba->nr_queues[HCTX_TYPE_POLL] = 1; 2946 hba->nr_hw_queues = 1; 2947 } 2948 2949 for (i = 0; i < shost->nr_maps; i++) { 2950 struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 2951 2952 map->nr_queues = hba->nr_queues[i]; 2953 if (!map->nr_queues) 2954 continue; 2955 map->queue_offset = queue_offset; 2956 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) 2957 map->queue_offset = 0; 2958 2959 blk_mq_map_queues(map); 2960 queue_offset += map->nr_queues; 2961 } 2962 } 2963 2964 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) 2965 { 2966 struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + 2967 i * ufshcd_get_ucd_size(hba); 2968 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2969 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + 2970 i * ufshcd_get_ucd_size(hba); 2971 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2972 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2973 2974 lrb->utr_descriptor_ptr = utrdlp + i; 2975 lrb->utrd_dma_addr = hba->utrdl_dma_addr + 2976 i * sizeof(struct utp_transfer_req_desc); 2977 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2978 lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2979 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2980 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2981 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2982 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2983 } 2984 2985 /** 2986 * ufshcd_queuecommand - main entry point for SCSI requests 2987 * @host: SCSI host pointer 2988 * @cmd: command from SCSI Midlayer 2989 * 2990 * Return: 0 for success, non-zero in case of failure. 2991 */ 2992 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 2993 { 2994 struct ufs_hba *hba = shost_priv(host); 2995 int tag = scsi_cmd_to_rq(cmd)->tag; 2996 struct ufshcd_lrb *lrbp; 2997 int err = 0; 2998 struct ufs_hw_queue *hwq = NULL; 2999 3000 switch (hba->ufshcd_state) { 3001 case UFSHCD_STATE_OPERATIONAL: 3002 break; 3003 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: 3004 /* 3005 * SCSI error handler can call ->queuecommand() while UFS error 3006 * handler is in progress. Error interrupts could change the 3007 * state from UFSHCD_STATE_RESET to 3008 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests 3009 * being issued in that case. 3010 */ 3011 if (ufshcd_eh_in_progress(hba)) { 3012 err = SCSI_MLQUEUE_HOST_BUSY; 3013 goto out; 3014 } 3015 break; 3016 case UFSHCD_STATE_EH_SCHEDULED_FATAL: 3017 /* 3018 * pm_runtime_get_sync() is used at error handling preparation 3019 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's 3020 * PM ops, it can never be finished if we let SCSI layer keep 3021 * retrying it, which gets err handler stuck forever. Neither 3022 * can we let the scsi cmd pass through, because UFS is in bad 3023 * state, the scsi cmd may eventually time out, which will get 3024 * err handler blocked for too long. So, just fail the scsi cmd 3025 * sent from PM ops, err handler can recover PM error anyways. 3026 */ 3027 if (hba->pm_op_in_progress) { 3028 hba->force_reset = true; 3029 set_host_byte(cmd, DID_BAD_TARGET); 3030 scsi_done(cmd); 3031 goto out; 3032 } 3033 fallthrough; 3034 case UFSHCD_STATE_RESET: 3035 err = SCSI_MLQUEUE_HOST_BUSY; 3036 goto out; 3037 case UFSHCD_STATE_ERROR: 3038 set_host_byte(cmd, DID_ERROR); 3039 scsi_done(cmd); 3040 goto out; 3041 } 3042 3043 hba->req_abort_count = 0; 3044 3045 ufshcd_hold(hba); 3046 3047 lrbp = &hba->lrb[tag]; 3048 3049 ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3050 3051 err = ufshcd_map_sg(hba, lrbp); 3052 if (err) { 3053 ufshcd_release(hba); 3054 goto out; 3055 } 3056 3057 if (hba->mcq_enabled) 3058 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3059 3060 ufshcd_send_command(hba, tag, hwq); 3061 3062 out: 3063 if (ufs_trigger_eh(hba)) { 3064 unsigned long flags; 3065 3066 spin_lock_irqsave(hba->host->host_lock, flags); 3067 ufshcd_schedule_eh_work(hba); 3068 spin_unlock_irqrestore(hba->host->host_lock, flags); 3069 } 3070 3071 return err; 3072 } 3073 3074 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3075 enum dev_cmd_type cmd_type, u8 lun, int tag) 3076 { 3077 __ufshcd_setup_cmd(lrbp, NULL, lun, tag); 3078 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3079 hba->dev_cmd.type = cmd_type; 3080 } 3081 3082 /* 3083 * Return: 0 upon success; < 0 upon failure. 3084 */ 3085 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 3086 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 3087 { 3088 ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 3089 3090 return ufshcd_compose_devman_upiu(hba, lrbp); 3091 } 3092 3093 /* 3094 * Check with the block layer if the command is inflight 3095 * @cmd: command to check. 3096 * 3097 * Return: true if command is inflight; false if not. 3098 */ 3099 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) 3100 { 3101 return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; 3102 } 3103 3104 /* 3105 * Clear the pending command in the controller and wait until 3106 * the controller confirms that the command has been cleared. 3107 * @hba: per adapter instance 3108 * @task_tag: The tag number of the command to be cleared. 3109 */ 3110 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) 3111 { 3112 u32 mask; 3113 int err; 3114 3115 if (hba->mcq_enabled) { 3116 /* 3117 * MCQ mode. Clean up the MCQ resources similar to 3118 * what the ufshcd_utrl_clear() does for SDB mode. 3119 */ 3120 err = ufshcd_mcq_sq_cleanup(hba, task_tag); 3121 if (err) { 3122 dev_err(hba->dev, "%s: failed tag=%d. err=%d\n", 3123 __func__, task_tag, err); 3124 return err; 3125 } 3126 return 0; 3127 } 3128 3129 mask = 1U << task_tag; 3130 3131 /* clear outstanding transaction before retry */ 3132 ufshcd_utrl_clear(hba, mask); 3133 3134 /* 3135 * wait for h/w to clear corresponding bit in door-bell. 3136 * max. wait is 1 sec. 3137 */ 3138 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, 3139 mask, ~mask, 1000, 1000); 3140 } 3141 3142 /** 3143 * ufshcd_dev_cmd_completion() - handles device management command responses 3144 * @hba: per adapter instance 3145 * @lrbp: pointer to local reference block 3146 * 3147 * Return: 0 upon success; < 0 upon failure. 3148 */ 3149 static int 3150 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 3151 { 3152 enum upiu_response_transaction resp; 3153 int err = 0; 3154 3155 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 3156 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 3157 3158 switch (resp) { 3159 case UPIU_TRANSACTION_NOP_IN: 3160 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 3161 err = -EINVAL; 3162 dev_err(hba->dev, "%s: unexpected response %x\n", 3163 __func__, resp); 3164 } 3165 break; 3166 case UPIU_TRANSACTION_QUERY_RSP: { 3167 u8 response = lrbp->ucd_rsp_ptr->header.response; 3168 3169 if (response == 0) { 3170 err = ufshcd_copy_query_response(hba, lrbp); 3171 } else { 3172 err = -EINVAL; 3173 dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", 3174 __func__, response); 3175 } 3176 break; 3177 } 3178 case UPIU_TRANSACTION_REJECT_UPIU: 3179 /* TODO: handle Reject UPIU Response */ 3180 err = -EPERM; 3181 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 3182 __func__); 3183 break; 3184 case UPIU_TRANSACTION_RESPONSE: 3185 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { 3186 err = -EINVAL; 3187 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); 3188 } 3189 break; 3190 default: 3191 err = -EINVAL; 3192 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 3193 __func__, resp); 3194 break; 3195 } 3196 3197 WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); 3198 return err; 3199 } 3200 3201 /* 3202 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3203 * < 0 if another error occurred. 3204 */ 3205 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, 3206 struct ufshcd_lrb *lrbp, int max_timeout) 3207 { 3208 unsigned long time_left = msecs_to_jiffies(max_timeout); 3209 unsigned long flags; 3210 bool pending; 3211 int err; 3212 3213 retry: 3214 time_left = wait_for_completion_timeout(&hba->dev_cmd.complete, 3215 time_left); 3216 3217 if (likely(time_left)) { 3218 err = ufshcd_get_tr_ocs(lrbp, NULL); 3219 if (!err) 3220 err = ufshcd_dev_cmd_completion(hba, lrbp); 3221 } else { 3222 err = -ETIMEDOUT; 3223 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", 3224 __func__, lrbp->task_tag); 3225 3226 /* MCQ mode */ 3227 if (hba->mcq_enabled) { 3228 /* successfully cleared the command, retry if needed */ 3229 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) 3230 err = -EAGAIN; 3231 return err; 3232 } 3233 3234 /* SDB mode */ 3235 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) { 3236 /* successfully cleared the command, retry if needed */ 3237 err = -EAGAIN; 3238 /* 3239 * Since clearing the command succeeded we also need to 3240 * clear the task tag bit from the outstanding_reqs 3241 * variable. 3242 */ 3243 spin_lock_irqsave(&hba->outstanding_lock, flags); 3244 pending = test_bit(lrbp->task_tag, 3245 &hba->outstanding_reqs); 3246 if (pending) 3247 __clear_bit(lrbp->task_tag, 3248 &hba->outstanding_reqs); 3249 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3250 3251 if (!pending) { 3252 /* 3253 * The completion handler ran while we tried to 3254 * clear the command. 3255 */ 3256 time_left = 1; 3257 goto retry; 3258 } 3259 } else { 3260 dev_err(hba->dev, "%s: failed to clear tag %d\n", 3261 __func__, lrbp->task_tag); 3262 3263 spin_lock_irqsave(&hba->outstanding_lock, flags); 3264 pending = test_bit(lrbp->task_tag, 3265 &hba->outstanding_reqs); 3266 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3267 3268 if (!pending) { 3269 /* 3270 * The completion handler ran while we tried to 3271 * clear the command. 3272 */ 3273 time_left = 1; 3274 goto retry; 3275 } 3276 } 3277 } 3278 3279 return err; 3280 } 3281 3282 static void ufshcd_dev_man_lock(struct ufs_hba *hba) 3283 { 3284 ufshcd_hold(hba); 3285 mutex_lock(&hba->dev_cmd.lock); 3286 down_read(&hba->clk_scaling_lock); 3287 } 3288 3289 static void ufshcd_dev_man_unlock(struct ufs_hba *hba) 3290 { 3291 up_read(&hba->clk_scaling_lock); 3292 mutex_unlock(&hba->dev_cmd.lock); 3293 ufshcd_release(hba); 3294 } 3295 3296 /* 3297 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3298 * < 0 if another error occurred. 3299 */ 3300 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 3301 const u32 tag, int timeout) 3302 { 3303 int err; 3304 3305 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3306 ufshcd_send_command(hba, tag, hba->dev_cmd_queue); 3307 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 3308 3309 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, 3310 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 3311 3312 return err; 3313 } 3314 3315 /** 3316 * ufshcd_exec_dev_cmd - API for sending device management requests 3317 * @hba: UFS hba 3318 * @cmd_type: specifies the type (NOP, Query...) 3319 * @timeout: timeout in milliseconds 3320 * 3321 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3322 * < 0 if another error occurred. 3323 * 3324 * NOTE: Since there is only one available tag for device management commands, 3325 * it is expected you hold the hba->dev_cmd.lock mutex. 3326 */ 3327 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3328 enum dev_cmd_type cmd_type, int timeout) 3329 { 3330 const u32 tag = hba->reserved_slot; 3331 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 3332 int err; 3333 3334 /* Protects use of hba->reserved_slot. */ 3335 lockdep_assert_held(&hba->dev_cmd.lock); 3336 3337 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 3338 if (unlikely(err)) 3339 return err; 3340 3341 return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout); 3342 } 3343 3344 /** 3345 * ufshcd_init_query() - init the query response and request parameters 3346 * @hba: per-adapter instance 3347 * @request: address of the request pointer to be initialized 3348 * @response: address of the response pointer to be initialized 3349 * @opcode: operation to perform 3350 * @idn: flag idn to access 3351 * @index: LU number to access 3352 * @selector: query/flag/descriptor further identification 3353 */ 3354 static inline void ufshcd_init_query(struct ufs_hba *hba, 3355 struct ufs_query_req **request, struct ufs_query_res **response, 3356 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 3357 { 3358 *request = &hba->dev_cmd.query.request; 3359 *response = &hba->dev_cmd.query.response; 3360 memset(*request, 0, sizeof(struct ufs_query_req)); 3361 memset(*response, 0, sizeof(struct ufs_query_res)); 3362 (*request)->upiu_req.opcode = opcode; 3363 (*request)->upiu_req.idn = idn; 3364 (*request)->upiu_req.index = index; 3365 (*request)->upiu_req.selector = selector; 3366 } 3367 3368 /* 3369 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3370 * < 0 if another error occurred. 3371 */ 3372 static int ufshcd_query_flag_retry(struct ufs_hba *hba, 3373 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) 3374 { 3375 int ret; 3376 int retries; 3377 3378 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { 3379 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res); 3380 if (ret) 3381 dev_dbg(hba->dev, 3382 "%s: failed with error %d, retries %d\n", 3383 __func__, ret, retries); 3384 else 3385 break; 3386 } 3387 3388 if (ret) 3389 dev_err(hba->dev, 3390 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", 3391 __func__, opcode, idn, ret, retries); 3392 return ret; 3393 } 3394 3395 /** 3396 * ufshcd_query_flag() - API function for sending flag query requests 3397 * @hba: per-adapter instance 3398 * @opcode: flag query to perform 3399 * @idn: flag idn to access 3400 * @index: flag index to access 3401 * @flag_res: the flag value after the query request completes 3402 * 3403 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3404 * < 0 if another error occurred. 3405 */ 3406 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 3407 enum flag_idn idn, u8 index, bool *flag_res) 3408 { 3409 struct ufs_query_req *request = NULL; 3410 struct ufs_query_res *response = NULL; 3411 int err, selector = 0; 3412 int timeout = dev_cmd_timeout; 3413 3414 BUG_ON(!hba); 3415 3416 ufshcd_dev_man_lock(hba); 3417 3418 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3419 selector); 3420 3421 switch (opcode) { 3422 case UPIU_QUERY_OPCODE_SET_FLAG: 3423 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 3424 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 3425 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3426 break; 3427 case UPIU_QUERY_OPCODE_READ_FLAG: 3428 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3429 if (!flag_res) { 3430 /* No dummy reads */ 3431 dev_err(hba->dev, "%s: Invalid argument for read request\n", 3432 __func__); 3433 err = -EINVAL; 3434 goto out_unlock; 3435 } 3436 break; 3437 default: 3438 dev_err(hba->dev, 3439 "%s: Expected query flag opcode but got = %d\n", 3440 __func__, opcode); 3441 err = -EINVAL; 3442 goto out_unlock; 3443 } 3444 3445 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); 3446 3447 if (err) { 3448 dev_err(hba->dev, 3449 "%s: Sending flag query for idn %d failed, err = %d\n", 3450 __func__, idn, err); 3451 goto out_unlock; 3452 } 3453 3454 if (flag_res) 3455 *flag_res = (be32_to_cpu(response->upiu_res.value) & 3456 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 3457 3458 out_unlock: 3459 ufshcd_dev_man_unlock(hba); 3460 return err; 3461 } 3462 3463 /** 3464 * ufshcd_query_attr - API function for sending attribute requests 3465 * @hba: per-adapter instance 3466 * @opcode: attribute opcode 3467 * @idn: attribute idn to access 3468 * @index: index field 3469 * @selector: selector field 3470 * @attr_val: the attribute value after the query request completes 3471 * 3472 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3473 * < 0 if another error occurred. 3474 */ 3475 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 3476 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 3477 { 3478 struct ufs_query_req *request = NULL; 3479 struct ufs_query_res *response = NULL; 3480 int err; 3481 3482 BUG_ON(!hba); 3483 3484 if (!attr_val) { 3485 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 3486 __func__, opcode); 3487 return -EINVAL; 3488 } 3489 3490 ufshcd_dev_man_lock(hba); 3491 3492 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3493 selector); 3494 3495 switch (opcode) { 3496 case UPIU_QUERY_OPCODE_WRITE_ATTR: 3497 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3498 request->upiu_req.value = cpu_to_be32(*attr_val); 3499 break; 3500 case UPIU_QUERY_OPCODE_READ_ATTR: 3501 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3502 break; 3503 default: 3504 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 3505 __func__, opcode); 3506 err = -EINVAL; 3507 goto out_unlock; 3508 } 3509 3510 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3511 3512 if (err) { 3513 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3514 __func__, opcode, idn, index, err); 3515 goto out_unlock; 3516 } 3517 3518 *attr_val = be32_to_cpu(response->upiu_res.value); 3519 3520 out_unlock: 3521 ufshcd_dev_man_unlock(hba); 3522 return err; 3523 } 3524 3525 /** 3526 * ufshcd_query_attr_retry() - API function for sending query 3527 * attribute with retries 3528 * @hba: per-adapter instance 3529 * @opcode: attribute opcode 3530 * @idn: attribute idn to access 3531 * @index: index field 3532 * @selector: selector field 3533 * @attr_val: the attribute value after the query request 3534 * completes 3535 * 3536 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3537 * < 0 if another error occurred. 3538 */ 3539 int ufshcd_query_attr_retry(struct ufs_hba *hba, 3540 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, 3541 u32 *attr_val) 3542 { 3543 int ret = 0; 3544 u32 retries; 3545 3546 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3547 ret = ufshcd_query_attr(hba, opcode, idn, index, 3548 selector, attr_val); 3549 if (ret) 3550 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n", 3551 __func__, ret, retries); 3552 else 3553 break; 3554 } 3555 3556 if (ret) 3557 dev_err(hba->dev, 3558 "%s: query attribute, idn %d, failed with error %d after %d retries\n", 3559 __func__, idn, ret, QUERY_REQ_RETRIES); 3560 return ret; 3561 } 3562 3563 /* 3564 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3565 * < 0 if another error occurred. 3566 */ 3567 static int __ufshcd_query_descriptor(struct ufs_hba *hba, 3568 enum query_opcode opcode, enum desc_idn idn, u8 index, 3569 u8 selector, u8 *desc_buf, int *buf_len) 3570 { 3571 struct ufs_query_req *request = NULL; 3572 struct ufs_query_res *response = NULL; 3573 int err; 3574 3575 BUG_ON(!hba); 3576 3577 if (!desc_buf) { 3578 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 3579 __func__, opcode); 3580 return -EINVAL; 3581 } 3582 3583 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 3584 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 3585 __func__, *buf_len); 3586 return -EINVAL; 3587 } 3588 3589 ufshcd_dev_man_lock(hba); 3590 3591 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3592 selector); 3593 hba->dev_cmd.query.descriptor = desc_buf; 3594 request->upiu_req.length = cpu_to_be16(*buf_len); 3595 3596 switch (opcode) { 3597 case UPIU_QUERY_OPCODE_WRITE_DESC: 3598 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3599 break; 3600 case UPIU_QUERY_OPCODE_READ_DESC: 3601 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3602 break; 3603 default: 3604 dev_err(hba->dev, 3605 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 3606 __func__, opcode); 3607 err = -EINVAL; 3608 goto out_unlock; 3609 } 3610 3611 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3612 3613 if (err) { 3614 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3615 __func__, opcode, idn, index, err); 3616 goto out_unlock; 3617 } 3618 3619 *buf_len = be16_to_cpu(response->upiu_res.length); 3620 3621 out_unlock: 3622 hba->dev_cmd.query.descriptor = NULL; 3623 ufshcd_dev_man_unlock(hba); 3624 return err; 3625 } 3626 3627 /** 3628 * ufshcd_query_descriptor_retry - API function for sending descriptor requests 3629 * @hba: per-adapter instance 3630 * @opcode: attribute opcode 3631 * @idn: attribute idn to access 3632 * @index: index field 3633 * @selector: selector field 3634 * @desc_buf: the buffer that contains the descriptor 3635 * @buf_len: length parameter passed to the device 3636 * 3637 * The buf_len parameter will contain, on return, the length parameter 3638 * received on the response. 3639 * 3640 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3641 * < 0 if another error occurred. 3642 */ 3643 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 3644 enum query_opcode opcode, 3645 enum desc_idn idn, u8 index, 3646 u8 selector, 3647 u8 *desc_buf, int *buf_len) 3648 { 3649 int err; 3650 int retries; 3651 3652 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3653 err = __ufshcd_query_descriptor(hba, opcode, idn, index, 3654 selector, desc_buf, buf_len); 3655 if (!err || err == -EINVAL) 3656 break; 3657 } 3658 3659 return err; 3660 } 3661 3662 /** 3663 * ufshcd_read_desc_param - read the specified descriptor parameter 3664 * @hba: Pointer to adapter instance 3665 * @desc_id: descriptor idn value 3666 * @desc_index: descriptor index 3667 * @param_offset: offset of the parameter to read 3668 * @param_read_buf: pointer to buffer where parameter would be read 3669 * @param_size: sizeof(param_read_buf) 3670 * 3671 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3672 * < 0 if another error occurred. 3673 */ 3674 int ufshcd_read_desc_param(struct ufs_hba *hba, 3675 enum desc_idn desc_id, 3676 int desc_index, 3677 u8 param_offset, 3678 u8 *param_read_buf, 3679 u8 param_size) 3680 { 3681 int ret; 3682 u8 *desc_buf; 3683 int buff_len = QUERY_DESC_MAX_SIZE; 3684 bool is_kmalloc = true; 3685 3686 /* Safety check */ 3687 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) 3688 return -EINVAL; 3689 3690 /* Check whether we need temp memory */ 3691 if (param_offset != 0 || param_size < buff_len) { 3692 desc_buf = kzalloc(buff_len, GFP_KERNEL); 3693 if (!desc_buf) 3694 return -ENOMEM; 3695 } else { 3696 desc_buf = param_read_buf; 3697 is_kmalloc = false; 3698 } 3699 3700 /* Request for full descriptor */ 3701 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, 3702 desc_id, desc_index, 0, 3703 desc_buf, &buff_len); 3704 if (ret) { 3705 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", 3706 __func__, desc_id, desc_index, param_offset, ret); 3707 goto out; 3708 } 3709 3710 /* Update descriptor length */ 3711 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; 3712 3713 if (param_offset >= buff_len) { 3714 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", 3715 __func__, param_offset, desc_id, buff_len); 3716 ret = -EINVAL; 3717 goto out; 3718 } 3719 3720 /* Sanity check */ 3721 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { 3722 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", 3723 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); 3724 ret = -EINVAL; 3725 goto out; 3726 } 3727 3728 if (is_kmalloc) { 3729 /* Make sure we don't copy more data than available */ 3730 if (param_offset >= buff_len) 3731 ret = -EINVAL; 3732 else 3733 memcpy(param_read_buf, &desc_buf[param_offset], 3734 min_t(u32, param_size, buff_len - param_offset)); 3735 } 3736 out: 3737 if (is_kmalloc) 3738 kfree(desc_buf); 3739 return ret; 3740 } 3741 3742 /** 3743 * struct uc_string_id - unicode string 3744 * 3745 * @len: size of this descriptor inclusive 3746 * @type: descriptor type 3747 * @uc: unicode string character 3748 */ 3749 struct uc_string_id { 3750 u8 len; 3751 u8 type; 3752 wchar_t uc[]; 3753 } __packed; 3754 3755 /* replace non-printable or non-ASCII characters with spaces */ 3756 static inline char ufshcd_remove_non_printable(u8 ch) 3757 { 3758 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' '; 3759 } 3760 3761 /** 3762 * ufshcd_read_string_desc - read string descriptor 3763 * @hba: pointer to adapter instance 3764 * @desc_index: descriptor index 3765 * @buf: pointer to buffer where descriptor would be read, 3766 * the caller should free the memory. 3767 * @ascii: if true convert from unicode to ascii characters 3768 * null terminated string. 3769 * 3770 * Return: 3771 * * string size on success. 3772 * * -ENOMEM: on allocation failure 3773 * * -EINVAL: on a wrong parameter 3774 */ 3775 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, 3776 u8 **buf, bool ascii) 3777 { 3778 struct uc_string_id *uc_str; 3779 u8 *str; 3780 int ret; 3781 3782 if (!buf) 3783 return -EINVAL; 3784 3785 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 3786 if (!uc_str) 3787 return -ENOMEM; 3788 3789 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, 3790 (u8 *)uc_str, QUERY_DESC_MAX_SIZE); 3791 if (ret < 0) { 3792 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", 3793 QUERY_REQ_RETRIES, ret); 3794 str = NULL; 3795 goto out; 3796 } 3797 3798 if (uc_str->len <= QUERY_DESC_HDR_SIZE) { 3799 dev_dbg(hba->dev, "String Desc is of zero length\n"); 3800 str = NULL; 3801 ret = 0; 3802 goto out; 3803 } 3804 3805 if (ascii) { 3806 ssize_t ascii_len; 3807 int i; 3808 /* remove header and divide by 2 to move from UTF16 to UTF8 */ 3809 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; 3810 str = kzalloc(ascii_len, GFP_KERNEL); 3811 if (!str) { 3812 ret = -ENOMEM; 3813 goto out; 3814 } 3815 3816 /* 3817 * the descriptor contains string in UTF16 format 3818 * we need to convert to utf-8 so it can be displayed 3819 */ 3820 ret = utf16s_to_utf8s(uc_str->uc, 3821 uc_str->len - QUERY_DESC_HDR_SIZE, 3822 UTF16_BIG_ENDIAN, str, ascii_len - 1); 3823 3824 /* replace non-printable or non-ASCII characters with spaces */ 3825 for (i = 0; i < ret; i++) 3826 str[i] = ufshcd_remove_non_printable(str[i]); 3827 3828 str[ret++] = '\0'; 3829 3830 } else { 3831 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL); 3832 if (!str) { 3833 ret = -ENOMEM; 3834 goto out; 3835 } 3836 ret = uc_str->len; 3837 } 3838 out: 3839 *buf = str; 3840 kfree(uc_str); 3841 return ret; 3842 } 3843 3844 /** 3845 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 3846 * @hba: Pointer to adapter instance 3847 * @lun: lun id 3848 * @param_offset: offset of the parameter to read 3849 * @param_read_buf: pointer to buffer where parameter would be read 3850 * @param_size: sizeof(param_read_buf) 3851 * 3852 * Return: 0 in case of success; < 0 upon failure. 3853 */ 3854 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 3855 int lun, 3856 enum unit_desc_param param_offset, 3857 u8 *param_read_buf, 3858 u32 param_size) 3859 { 3860 /* 3861 * Unit descriptors are only available for general purpose LUs (LUN id 3862 * from 0 to 7) and RPMB Well known LU. 3863 */ 3864 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun)) 3865 return -EOPNOTSUPP; 3866 3867 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 3868 param_offset, param_read_buf, param_size); 3869 } 3870 3871 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) 3872 { 3873 int err = 0; 3874 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3875 3876 if (hba->dev_info.wspecversion >= 0x300) { 3877 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3878 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0, 3879 &gating_wait); 3880 if (err) 3881 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n", 3882 err, gating_wait); 3883 3884 if (gating_wait == 0) { 3885 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3886 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n", 3887 gating_wait); 3888 } 3889 3890 hba->dev_info.clk_gating_wait_us = gating_wait; 3891 } 3892 3893 return err; 3894 } 3895 3896 /** 3897 * ufshcd_memory_alloc - allocate memory for host memory space data structures 3898 * @hba: per adapter instance 3899 * 3900 * 1. Allocate DMA memory for Command Descriptor array 3901 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 3902 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 3903 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 3904 * (UTMRDL) 3905 * 4. Allocate memory for local reference block(lrb). 3906 * 3907 * Return: 0 for success, non-zero in case of failure. 3908 */ 3909 static int ufshcd_memory_alloc(struct ufs_hba *hba) 3910 { 3911 size_t utmrdl_size, utrdl_size, ucdl_size; 3912 3913 /* Allocate memory for UTP command descriptors */ 3914 ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs; 3915 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 3916 ucdl_size, 3917 &hba->ucdl_dma_addr, 3918 GFP_KERNEL); 3919 3920 /* 3921 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 3922 */ 3923 if (!hba->ucdl_base_addr || 3924 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { 3925 dev_err(hba->dev, 3926 "Command Descriptor Memory allocation failed\n"); 3927 goto out; 3928 } 3929 3930 /* 3931 * Allocate memory for UTP Transfer descriptors 3932 * UFSHCI requires 1KB alignment of UTRD 3933 */ 3934 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 3935 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 3936 utrdl_size, 3937 &hba->utrdl_dma_addr, 3938 GFP_KERNEL); 3939 if (!hba->utrdl_base_addr || 3940 WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) { 3941 dev_err(hba->dev, 3942 "Transfer Descriptor Memory allocation failed\n"); 3943 goto out; 3944 } 3945 3946 /* 3947 * Skip utmrdl allocation; it may have been 3948 * allocated during first pass and not released during 3949 * MCQ memory allocation. 3950 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() 3951 */ 3952 if (hba->utmrdl_base_addr) 3953 goto skip_utmrdl; 3954 /* 3955 * Allocate memory for UTP Task Management descriptors 3956 * UFSHCI requires 1KB alignment of UTMRD 3957 */ 3958 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 3959 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 3960 utmrdl_size, 3961 &hba->utmrdl_dma_addr, 3962 GFP_KERNEL); 3963 if (!hba->utmrdl_base_addr || 3964 WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) { 3965 dev_err(hba->dev, 3966 "Task Management Descriptor Memory allocation failed\n"); 3967 goto out; 3968 } 3969 3970 skip_utmrdl: 3971 /* Allocate memory for local reference block */ 3972 hba->lrb = devm_kcalloc(hba->dev, 3973 hba->nutrs, sizeof(struct ufshcd_lrb), 3974 GFP_KERNEL); 3975 if (!hba->lrb) { 3976 dev_err(hba->dev, "LRB Memory allocation failed\n"); 3977 goto out; 3978 } 3979 return 0; 3980 out: 3981 return -ENOMEM; 3982 } 3983 3984 /** 3985 * ufshcd_host_memory_configure - configure local reference block with 3986 * memory offsets 3987 * @hba: per adapter instance 3988 * 3989 * Configure Host memory space 3990 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 3991 * address. 3992 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 3993 * and PRDT offset. 3994 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 3995 * into local reference block. 3996 */ 3997 static void ufshcd_host_memory_configure(struct ufs_hba *hba) 3998 { 3999 struct utp_transfer_req_desc *utrdlp; 4000 dma_addr_t cmd_desc_dma_addr; 4001 dma_addr_t cmd_desc_element_addr; 4002 u16 response_offset; 4003 u16 prdt_offset; 4004 int cmd_desc_size; 4005 int i; 4006 4007 utrdlp = hba->utrdl_base_addr; 4008 4009 response_offset = 4010 offsetof(struct utp_transfer_cmd_desc, response_upiu); 4011 prdt_offset = 4012 offsetof(struct utp_transfer_cmd_desc, prd_table); 4013 4014 cmd_desc_size = ufshcd_get_ucd_size(hba); 4015 cmd_desc_dma_addr = hba->ucdl_dma_addr; 4016 4017 for (i = 0; i < hba->nutrs; i++) { 4018 /* Configure UTRD with command descriptor base address */ 4019 cmd_desc_element_addr = 4020 (cmd_desc_dma_addr + (cmd_desc_size * i)); 4021 utrdlp[i].command_desc_base_addr = 4022 cpu_to_le64(cmd_desc_element_addr); 4023 4024 /* Response upiu and prdt offset should be in double words */ 4025 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { 4026 utrdlp[i].response_upiu_offset = 4027 cpu_to_le16(response_offset); 4028 utrdlp[i].prd_table_offset = 4029 cpu_to_le16(prdt_offset); 4030 utrdlp[i].response_upiu_length = 4031 cpu_to_le16(ALIGNED_UPIU_SIZE); 4032 } else { 4033 utrdlp[i].response_upiu_offset = 4034 cpu_to_le16(response_offset >> 2); 4035 utrdlp[i].prd_table_offset = 4036 cpu_to_le16(prdt_offset >> 2); 4037 utrdlp[i].response_upiu_length = 4038 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 4039 } 4040 4041 ufshcd_init_lrb(hba, &hba->lrb[i], i); 4042 } 4043 } 4044 4045 /** 4046 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 4047 * @hba: per adapter instance 4048 * 4049 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 4050 * in order to initialize the Unipro link startup procedure. 4051 * Once the Unipro links are up, the device connected to the controller 4052 * is detected. 4053 * 4054 * Return: 0 on success, non-zero value on failure. 4055 */ 4056 static int ufshcd_dme_link_startup(struct ufs_hba *hba) 4057 { 4058 struct uic_command uic_cmd = { 4059 .command = UIC_CMD_DME_LINK_STARTUP, 4060 }; 4061 int ret; 4062 4063 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4064 if (ret) 4065 dev_dbg(hba->dev, 4066 "dme-link-startup: error code %d\n", ret); 4067 return ret; 4068 } 4069 /** 4070 * ufshcd_dme_reset - UIC command for DME_RESET 4071 * @hba: per adapter instance 4072 * 4073 * DME_RESET command is issued in order to reset UniPro stack. 4074 * This function now deals with cold reset. 4075 * 4076 * Return: 0 on success, non-zero value on failure. 4077 */ 4078 int ufshcd_dme_reset(struct ufs_hba *hba) 4079 { 4080 struct uic_command uic_cmd = { 4081 .command = UIC_CMD_DME_RESET, 4082 }; 4083 int ret; 4084 4085 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4086 if (ret) 4087 dev_err(hba->dev, 4088 "dme-reset: error code %d\n", ret); 4089 4090 return ret; 4091 } 4092 EXPORT_SYMBOL_GPL(ufshcd_dme_reset); 4093 4094 int ufshcd_dme_configure_adapt(struct ufs_hba *hba, 4095 int agreed_gear, 4096 int adapt_val) 4097 { 4098 int ret; 4099 4100 if (agreed_gear < UFS_HS_G4) 4101 adapt_val = PA_NO_ADAPT; 4102 4103 ret = ufshcd_dme_set(hba, 4104 UIC_ARG_MIB(PA_TXHSADAPTTYPE), 4105 adapt_val); 4106 return ret; 4107 } 4108 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); 4109 4110 /** 4111 * ufshcd_dme_enable - UIC command for DME_ENABLE 4112 * @hba: per adapter instance 4113 * 4114 * DME_ENABLE command is issued in order to enable UniPro stack. 4115 * 4116 * Return: 0 on success, non-zero value on failure. 4117 */ 4118 int ufshcd_dme_enable(struct ufs_hba *hba) 4119 { 4120 struct uic_command uic_cmd = { 4121 .command = UIC_CMD_DME_ENABLE, 4122 }; 4123 int ret; 4124 4125 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4126 if (ret) 4127 dev_err(hba->dev, 4128 "dme-enable: error code %d\n", ret); 4129 4130 return ret; 4131 } 4132 EXPORT_SYMBOL_GPL(ufshcd_dme_enable); 4133 4134 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 4135 { 4136 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 4137 unsigned long min_sleep_time_us; 4138 4139 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 4140 return; 4141 4142 /* 4143 * last_dme_cmd_tstamp will be 0 only for 1st call to 4144 * this function 4145 */ 4146 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 4147 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 4148 } else { 4149 unsigned long delta = 4150 (unsigned long) ktime_to_us( 4151 ktime_sub(ktime_get(), 4152 hba->last_dme_cmd_tstamp)); 4153 4154 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 4155 min_sleep_time_us = 4156 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 4157 else 4158 min_sleep_time_us = 0; /* no more delay required */ 4159 } 4160 4161 if (min_sleep_time_us > 0) { 4162 /* allow sleep for extra 50us if needed */ 4163 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 4164 } 4165 4166 /* update the last_dme_cmd_tstamp */ 4167 hba->last_dme_cmd_tstamp = ktime_get(); 4168 } 4169 4170 /** 4171 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 4172 * @hba: per adapter instance 4173 * @attr_sel: uic command argument1 4174 * @attr_set: attribute set type as uic command argument2 4175 * @mib_val: setting value as uic command argument3 4176 * @peer: indicate whether peer or local 4177 * 4178 * Return: 0 on success, non-zero value on failure. 4179 */ 4180 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 4181 u8 attr_set, u32 mib_val, u8 peer) 4182 { 4183 struct uic_command uic_cmd = { 4184 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, 4185 .argument1 = attr_sel, 4186 .argument2 = UIC_ARG_ATTR_TYPE(attr_set), 4187 .argument3 = mib_val, 4188 }; 4189 static const char *const action[] = { 4190 "dme-set", 4191 "dme-peer-set" 4192 }; 4193 const char *set = action[!!peer]; 4194 int ret; 4195 int retries = UFS_UIC_COMMAND_RETRIES; 4196 4197 do { 4198 /* for peer attributes we retry upon failure */ 4199 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4200 if (ret) 4201 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 4202 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 4203 } while (ret && peer && --retries); 4204 4205 if (ret) 4206 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", 4207 set, UIC_GET_ATTR_ID(attr_sel), mib_val, 4208 UFS_UIC_COMMAND_RETRIES - retries); 4209 4210 return ret; 4211 } 4212 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 4213 4214 /** 4215 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 4216 * @hba: per adapter instance 4217 * @attr_sel: uic command argument1 4218 * @mib_val: the value of the attribute as returned by the UIC command 4219 * @peer: indicate whether peer or local 4220 * 4221 * Return: 0 on success, non-zero value on failure. 4222 */ 4223 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 4224 u32 *mib_val, u8 peer) 4225 { 4226 struct uic_command uic_cmd = { 4227 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, 4228 .argument1 = attr_sel, 4229 }; 4230 static const char *const action[] = { 4231 "dme-get", 4232 "dme-peer-get" 4233 }; 4234 const char *get = action[!!peer]; 4235 int ret; 4236 int retries = UFS_UIC_COMMAND_RETRIES; 4237 struct ufs_pa_layer_attr orig_pwr_info; 4238 struct ufs_pa_layer_attr temp_pwr_info; 4239 bool pwr_mode_change = false; 4240 4241 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) { 4242 orig_pwr_info = hba->pwr_info; 4243 temp_pwr_info = orig_pwr_info; 4244 4245 if (orig_pwr_info.pwr_tx == FAST_MODE || 4246 orig_pwr_info.pwr_rx == FAST_MODE) { 4247 temp_pwr_info.pwr_tx = FASTAUTO_MODE; 4248 temp_pwr_info.pwr_rx = FASTAUTO_MODE; 4249 pwr_mode_change = true; 4250 } else if (orig_pwr_info.pwr_tx == SLOW_MODE || 4251 orig_pwr_info.pwr_rx == SLOW_MODE) { 4252 temp_pwr_info.pwr_tx = SLOWAUTO_MODE; 4253 temp_pwr_info.pwr_rx = SLOWAUTO_MODE; 4254 pwr_mode_change = true; 4255 } 4256 if (pwr_mode_change) { 4257 ret = ufshcd_change_power_mode(hba, &temp_pwr_info); 4258 if (ret) 4259 goto out; 4260 } 4261 } 4262 4263 do { 4264 /* for peer attributes we retry upon failure */ 4265 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4266 if (ret) 4267 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", 4268 get, UIC_GET_ATTR_ID(attr_sel), ret); 4269 } while (ret && peer && --retries); 4270 4271 if (ret) 4272 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", 4273 get, UIC_GET_ATTR_ID(attr_sel), 4274 UFS_UIC_COMMAND_RETRIES - retries); 4275 4276 if (mib_val && !ret) 4277 *mib_val = uic_cmd.argument3; 4278 4279 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) 4280 && pwr_mode_change) 4281 ufshcd_change_power_mode(hba, &orig_pwr_info); 4282 out: 4283 return ret; 4284 } 4285 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 4286 4287 /** 4288 * ufshcd_dme_rmw - get modify set a DME attribute 4289 * @hba: per adapter instance 4290 * @mask: indicates which bits to clear from the value that has been read 4291 * @val: actual value to write 4292 * @attr: dme attribute 4293 */ 4294 int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, 4295 u32 val, u32 attr) 4296 { 4297 u32 cfg = 0; 4298 int err; 4299 4300 err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); 4301 if (err) 4302 return err; 4303 4304 cfg &= ~mask; 4305 cfg |= (val & mask); 4306 4307 return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); 4308 } 4309 EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); 4310 4311 /** 4312 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 4313 * state) and waits for it to take effect. 4314 * 4315 * @hba: per adapter instance 4316 * @cmd: UIC command to execute 4317 * 4318 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 4319 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 4320 * and device UniPro link and hence it's final completion would be indicated by 4321 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 4322 * addition to normal UIC command completion Status (UCCS). This function only 4323 * returns after the relevant status bits indicate the completion. 4324 * 4325 * Return: 0 on success, non-zero value on failure. 4326 */ 4327 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 4328 { 4329 DECLARE_COMPLETION_ONSTACK(uic_async_done); 4330 unsigned long flags; 4331 u8 status; 4332 int ret; 4333 4334 mutex_lock(&hba->uic_cmd_mutex); 4335 ufshcd_add_delay_before_dme_cmd(hba); 4336 4337 spin_lock_irqsave(hba->host->host_lock, flags); 4338 if (ufshcd_is_link_broken(hba)) { 4339 ret = -ENOLINK; 4340 goto out_unlock; 4341 } 4342 hba->uic_async_done = &uic_async_done; 4343 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); 4344 spin_unlock_irqrestore(hba->host->host_lock, flags); 4345 ret = __ufshcd_send_uic_cmd(hba, cmd); 4346 if (ret) { 4347 dev_err(hba->dev, 4348 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 4349 cmd->command, cmd->argument3, ret); 4350 goto out; 4351 } 4352 4353 if (!wait_for_completion_timeout(hba->uic_async_done, 4354 msecs_to_jiffies(uic_cmd_timeout))) { 4355 dev_err(hba->dev, 4356 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 4357 cmd->command, cmd->argument3); 4358 4359 if (!cmd->cmd_active) { 4360 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", 4361 __func__); 4362 goto check_upmcrs; 4363 } 4364 4365 ret = -ETIMEDOUT; 4366 goto out; 4367 } 4368 4369 check_upmcrs: 4370 status = ufshcd_get_upmcrs(hba); 4371 if (status != PWR_LOCAL) { 4372 dev_err(hba->dev, 4373 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", 4374 cmd->command, status); 4375 ret = (status != PWR_OK) ? status : -1; 4376 } 4377 out: 4378 if (ret) { 4379 ufshcd_print_host_state(hba); 4380 ufshcd_print_pwr_info(hba); 4381 ufshcd_print_evt_hist(hba); 4382 } 4383 4384 spin_lock_irqsave(hba->host->host_lock, flags); 4385 hba->active_uic_cmd = NULL; 4386 hba->uic_async_done = NULL; 4387 if (ret && !hba->pm_op_in_progress) { 4388 ufshcd_set_link_broken(hba); 4389 ufshcd_schedule_eh_work(hba); 4390 } 4391 out_unlock: 4392 spin_unlock_irqrestore(hba->host->host_lock, flags); 4393 mutex_unlock(&hba->uic_cmd_mutex); 4394 4395 /* 4396 * If the h8 exit fails during the runtime resume process, it becomes 4397 * stuck and cannot be recovered through the error handler. To fix 4398 * this, use link recovery instead of the error handler. 4399 */ 4400 if (ret && hba->pm_op_in_progress) 4401 ret = ufshcd_link_recovery(hba); 4402 4403 return ret; 4404 } 4405 4406 /** 4407 * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result 4408 * @hba: per adapter instance 4409 * @uic_cmd: UIC command 4410 * 4411 * Return: 0 only if success. 4412 */ 4413 int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 4414 { 4415 int ret; 4416 4417 if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || 4418 uic_cmd->command != UIC_CMD_DME_SET) 4419 return ufshcd_send_uic_cmd(hba, uic_cmd); 4420 4421 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 4422 return 0; 4423 4424 ufshcd_hold(hba); 4425 ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); 4426 ufshcd_release(hba); 4427 4428 return ret; 4429 } 4430 4431 /** 4432 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 4433 * using DME_SET primitives. 4434 * @hba: per adapter instance 4435 * @mode: powr mode value 4436 * 4437 * Return: 0 on success, non-zero value on failure. 4438 */ 4439 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 4440 { 4441 struct uic_command uic_cmd = { 4442 .command = UIC_CMD_DME_SET, 4443 .argument1 = UIC_ARG_MIB(PA_PWRMODE), 4444 .argument3 = mode, 4445 }; 4446 int ret; 4447 4448 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { 4449 ret = ufshcd_dme_set(hba, 4450 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1); 4451 if (ret) { 4452 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n", 4453 __func__, ret); 4454 goto out; 4455 } 4456 } 4457 4458 ufshcd_hold(hba); 4459 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4460 ufshcd_release(hba); 4461 4462 out: 4463 return ret; 4464 } 4465 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode); 4466 4467 int ufshcd_link_recovery(struct ufs_hba *hba) 4468 { 4469 int ret; 4470 unsigned long flags; 4471 4472 spin_lock_irqsave(hba->host->host_lock, flags); 4473 hba->ufshcd_state = UFSHCD_STATE_RESET; 4474 ufshcd_set_eh_in_progress(hba); 4475 spin_unlock_irqrestore(hba->host->host_lock, flags); 4476 4477 /* Reset the attached device */ 4478 ufshcd_device_reset(hba); 4479 4480 ret = ufshcd_host_reset_and_restore(hba); 4481 4482 spin_lock_irqsave(hba->host->host_lock, flags); 4483 if (ret) 4484 hba->ufshcd_state = UFSHCD_STATE_ERROR; 4485 ufshcd_clear_eh_in_progress(hba); 4486 spin_unlock_irqrestore(hba->host->host_lock, flags); 4487 4488 if (ret) 4489 dev_err(hba->dev, "%s: link recovery failed, err %d", 4490 __func__, ret); 4491 4492 return ret; 4493 } 4494 EXPORT_SYMBOL_GPL(ufshcd_link_recovery); 4495 4496 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 4497 { 4498 struct uic_command uic_cmd = { 4499 .command = UIC_CMD_DME_HIBER_ENTER, 4500 }; 4501 ktime_t start = ktime_get(); 4502 int ret; 4503 4504 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); 4505 4506 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4507 trace_ufshcd_profile_hibern8(hba, "enter", 4508 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4509 4510 if (ret) 4511 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", 4512 __func__, ret); 4513 else 4514 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, 4515 POST_CHANGE); 4516 4517 return ret; 4518 } 4519 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); 4520 4521 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 4522 { 4523 struct uic_command uic_cmd = { 4524 .command = UIC_CMD_DME_HIBER_EXIT, 4525 }; 4526 int ret; 4527 ktime_t start = ktime_get(); 4528 4529 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); 4530 4531 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4532 trace_ufshcd_profile_hibern8(hba, "exit", 4533 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4534 4535 if (ret) { 4536 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", 4537 __func__, ret); 4538 } else { 4539 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, 4540 POST_CHANGE); 4541 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); 4542 hba->ufs_stats.hibern8_exit_cnt++; 4543 } 4544 4545 return ret; 4546 } 4547 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); 4548 4549 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) 4550 { 4551 if (!ufshcd_is_auto_hibern8_supported(hba)) 4552 return; 4553 4554 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); 4555 } 4556 4557 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) 4558 { 4559 const u32 cur_ahit = READ_ONCE(hba->ahit); 4560 4561 if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) 4562 return; 4563 4564 WRITE_ONCE(hba->ahit, ahit); 4565 if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { 4566 ufshcd_rpm_get_sync(hba); 4567 ufshcd_hold(hba); 4568 ufshcd_configure_auto_hibern8(hba); 4569 ufshcd_release(hba); 4570 ufshcd_rpm_put_sync(hba); 4571 } 4572 } 4573 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); 4574 4575 /** 4576 * ufshcd_init_pwr_info - setting the POR (power on reset) 4577 * values in hba power info 4578 * @hba: per-adapter instance 4579 */ 4580 static void ufshcd_init_pwr_info(struct ufs_hba *hba) 4581 { 4582 hba->pwr_info.gear_rx = UFS_PWM_G1; 4583 hba->pwr_info.gear_tx = UFS_PWM_G1; 4584 hba->pwr_info.lane_rx = UFS_LANE_1; 4585 hba->pwr_info.lane_tx = UFS_LANE_1; 4586 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 4587 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 4588 hba->pwr_info.hs_rate = 0; 4589 } 4590 4591 /** 4592 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 4593 * @hba: per-adapter instance 4594 * 4595 * Return: 0 upon success; < 0 upon failure. 4596 */ 4597 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 4598 { 4599 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 4600 4601 if (hba->max_pwr_info.is_valid) 4602 return 0; 4603 4604 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) { 4605 pwr_info->pwr_tx = FASTAUTO_MODE; 4606 pwr_info->pwr_rx = FASTAUTO_MODE; 4607 } else { 4608 pwr_info->pwr_tx = FAST_MODE; 4609 pwr_info->pwr_rx = FAST_MODE; 4610 } 4611 pwr_info->hs_rate = PA_HS_MODE_B; 4612 4613 /* Get the connected lane count */ 4614 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 4615 &pwr_info->lane_rx); 4616 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4617 &pwr_info->lane_tx); 4618 4619 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 4620 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 4621 __func__, 4622 pwr_info->lane_rx, 4623 pwr_info->lane_tx); 4624 return -EINVAL; 4625 } 4626 4627 if (pwr_info->lane_rx != pwr_info->lane_tx) { 4628 dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", 4629 __func__, 4630 pwr_info->lane_rx, 4631 pwr_info->lane_tx); 4632 return -EINVAL; 4633 } 4634 4635 /* 4636 * First, get the maximum gears of HS speed. 4637 * If a zero value, it means there is no HSGEAR capability. 4638 * Then, get the maximum gears of PWM speed. 4639 */ 4640 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 4641 if (!pwr_info->gear_rx) { 4642 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4643 &pwr_info->gear_rx); 4644 if (!pwr_info->gear_rx) { 4645 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 4646 __func__, pwr_info->gear_rx); 4647 return -EINVAL; 4648 } 4649 pwr_info->pwr_rx = SLOW_MODE; 4650 } 4651 4652 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 4653 &pwr_info->gear_tx); 4654 if (!pwr_info->gear_tx) { 4655 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4656 &pwr_info->gear_tx); 4657 if (!pwr_info->gear_tx) { 4658 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 4659 __func__, pwr_info->gear_tx); 4660 return -EINVAL; 4661 } 4662 pwr_info->pwr_tx = SLOW_MODE; 4663 } 4664 4665 hba->max_pwr_info.is_valid = true; 4666 return 0; 4667 } 4668 4669 static int ufshcd_change_power_mode(struct ufs_hba *hba, 4670 struct ufs_pa_layer_attr *pwr_mode) 4671 { 4672 int ret; 4673 4674 /* if already configured to the requested pwr_mode */ 4675 if (!hba->force_pmc && 4676 pwr_mode->gear_rx == hba->pwr_info.gear_rx && 4677 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 4678 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 4679 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 4680 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 4681 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 4682 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 4683 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 4684 return 0; 4685 } 4686 4687 /* 4688 * Configure attributes for power mode change with below. 4689 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 4690 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 4691 * - PA_HSSERIES 4692 */ 4693 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 4694 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 4695 pwr_mode->lane_rx); 4696 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4697 pwr_mode->pwr_rx == FAST_MODE) 4698 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); 4699 else 4700 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false); 4701 4702 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 4703 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 4704 pwr_mode->lane_tx); 4705 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 4706 pwr_mode->pwr_tx == FAST_MODE) 4707 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); 4708 else 4709 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false); 4710 4711 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4712 pwr_mode->pwr_tx == FASTAUTO_MODE || 4713 pwr_mode->pwr_rx == FAST_MODE || 4714 pwr_mode->pwr_tx == FAST_MODE) 4715 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 4716 pwr_mode->hs_rate); 4717 4718 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { 4719 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 4720 DL_FC0ProtectionTimeOutVal_Default); 4721 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 4722 DL_TC0ReplayTimeOutVal_Default); 4723 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 4724 DL_AFC0ReqTimeOutVal_Default); 4725 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 4726 DL_FC1ProtectionTimeOutVal_Default); 4727 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 4728 DL_TC1ReplayTimeOutVal_Default); 4729 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 4730 DL_AFC1ReqTimeOutVal_Default); 4731 4732 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), 4733 DL_FC0ProtectionTimeOutVal_Default); 4734 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), 4735 DL_TC0ReplayTimeOutVal_Default); 4736 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), 4737 DL_AFC0ReqTimeOutVal_Default); 4738 } 4739 4740 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 4741 | pwr_mode->pwr_tx); 4742 4743 if (ret) { 4744 dev_err(hba->dev, 4745 "%s: power mode change failed %d\n", __func__, ret); 4746 } else { 4747 memcpy(&hba->pwr_info, pwr_mode, 4748 sizeof(struct ufs_pa_layer_attr)); 4749 } 4750 4751 return ret; 4752 } 4753 4754 /** 4755 * ufshcd_config_pwr_mode - configure a new power mode 4756 * @hba: per-adapter instance 4757 * @desired_pwr_mode: desired power configuration 4758 * 4759 * Return: 0 upon success; < 0 upon failure. 4760 */ 4761 int ufshcd_config_pwr_mode(struct ufs_hba *hba, 4762 struct ufs_pa_layer_attr *desired_pwr_mode) 4763 { 4764 struct ufs_pa_layer_attr final_params = { 0 }; 4765 int ret; 4766 4767 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE, 4768 desired_pwr_mode, &final_params); 4769 4770 if (ret) 4771 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 4772 4773 ret = ufshcd_change_power_mode(hba, &final_params); 4774 4775 if (!ret) 4776 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, 4777 &final_params); 4778 4779 return ret; 4780 } 4781 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); 4782 4783 /** 4784 * ufshcd_complete_dev_init() - checks device readiness 4785 * @hba: per-adapter instance 4786 * 4787 * Set fDeviceInit flag and poll until device toggles it. 4788 * 4789 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 4790 * < 0 if another error occurred. 4791 */ 4792 static int ufshcd_complete_dev_init(struct ufs_hba *hba) 4793 { 4794 int err; 4795 bool flag_res = true; 4796 ktime_t timeout; 4797 4798 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 4799 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL); 4800 if (err) { 4801 dev_err(hba->dev, 4802 "%s: setting fDeviceInit flag failed with error %d\n", 4803 __func__, err); 4804 goto out; 4805 } 4806 4807 /* Poll fDeviceInit flag to be cleared */ 4808 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT); 4809 do { 4810 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4811 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res); 4812 if (!flag_res) 4813 break; 4814 usleep_range(500, 1000); 4815 } while (ktime_before(ktime_get(), timeout)); 4816 4817 if (err) { 4818 dev_err(hba->dev, 4819 "%s: reading fDeviceInit flag failed with error %d\n", 4820 __func__, err); 4821 } else if (flag_res) { 4822 dev_err(hba->dev, 4823 "%s: fDeviceInit was not cleared by the device\n", 4824 __func__); 4825 err = -EBUSY; 4826 } 4827 out: 4828 return err; 4829 } 4830 4831 /** 4832 * ufshcd_make_hba_operational - Make UFS controller operational 4833 * @hba: per adapter instance 4834 * 4835 * To bring UFS host controller to operational state, 4836 * 1. Enable required interrupts 4837 * 2. Configure interrupt aggregation 4838 * 3. Program UTRL and UTMRL base address 4839 * 4. Configure run-stop-registers 4840 * 4841 * Return: 0 if successful; < 0 upon failure. 4842 */ 4843 int ufshcd_make_hba_operational(struct ufs_hba *hba) 4844 { 4845 int err = 0; 4846 u32 reg; 4847 4848 /* Enable required interrupts */ 4849 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 4850 4851 /* Configure interrupt aggregation */ 4852 if (ufshcd_is_intr_aggr_allowed(hba)) 4853 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 4854 else 4855 ufshcd_disable_intr_aggr(hba); 4856 4857 /* Configure UTRL and UTMRL base address registers */ 4858 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 4859 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 4860 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 4861 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 4862 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 4863 REG_UTP_TASK_REQ_LIST_BASE_L); 4864 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 4865 REG_UTP_TASK_REQ_LIST_BASE_H); 4866 4867 /* 4868 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 4869 */ 4870 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 4871 if (!(ufshcd_get_lists_status(reg))) { 4872 ufshcd_enable_run_stop_reg(hba); 4873 } else { 4874 dev_err(hba->dev, 4875 "Host controller not ready to process requests"); 4876 err = -EIO; 4877 } 4878 4879 return err; 4880 } 4881 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); 4882 4883 /** 4884 * ufshcd_hba_stop - Send controller to reset state 4885 * @hba: per adapter instance 4886 */ 4887 void ufshcd_hba_stop(struct ufs_hba *hba) 4888 { 4889 int err; 4890 4891 ufshcd_disable_irq(hba); 4892 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); 4893 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, 4894 CONTROLLER_ENABLE, CONTROLLER_DISABLE, 4895 10, 1); 4896 ufshcd_enable_irq(hba); 4897 if (err) 4898 dev_err(hba->dev, "%s: Controller disable failed\n", __func__); 4899 } 4900 EXPORT_SYMBOL_GPL(ufshcd_hba_stop); 4901 4902 /** 4903 * ufshcd_hba_execute_hce - initialize the controller 4904 * @hba: per adapter instance 4905 * 4906 * The controller resets itself and controller firmware initialization 4907 * sequence kicks off. When controller is ready it will set 4908 * the Host Controller Enable bit to 1. 4909 * 4910 * Return: 0 on success, non-zero value on failure. 4911 */ 4912 static int ufshcd_hba_execute_hce(struct ufs_hba *hba) 4913 { 4914 int retry; 4915 4916 for (retry = 3; retry > 0; retry--) { 4917 if (ufshcd_is_hba_active(hba)) 4918 /* change controller state to "reset state" */ 4919 ufshcd_hba_stop(hba); 4920 4921 /* UniPro link is disabled at this point */ 4922 ufshcd_set_link_off(hba); 4923 4924 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4925 4926 /* start controller initialization sequence */ 4927 ufshcd_hba_start(hba); 4928 4929 /* 4930 * To initialize a UFS host controller HCE bit must be set to 1. 4931 * During initialization the HCE bit value changes from 1->0->1. 4932 * When the host controller completes initialization sequence 4933 * it sets the value of HCE bit to 1. The same HCE bit is read back 4934 * to check if the controller has completed initialization sequence. 4935 * So without this delay the value HCE = 1, set in the previous 4936 * instruction might be read back. 4937 * This delay can be changed based on the controller. 4938 */ 4939 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); 4940 4941 /* wait for the host controller to complete initialization */ 4942 if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, 4943 CONTROLLER_ENABLE, 1000, 50)) 4944 break; 4945 4946 dev_err(hba->dev, "Enabling the controller failed\n"); 4947 } 4948 4949 if (!retry) 4950 return -EIO; 4951 4952 /* enable UIC related interrupts */ 4953 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4954 4955 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4956 4957 return 0; 4958 } 4959 4960 int ufshcd_hba_enable(struct ufs_hba *hba) 4961 { 4962 int ret; 4963 4964 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { 4965 ufshcd_set_link_off(hba); 4966 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4967 4968 /* enable UIC related interrupts */ 4969 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4970 ret = ufshcd_dme_reset(hba); 4971 if (ret) { 4972 dev_err(hba->dev, "DME_RESET failed\n"); 4973 return ret; 4974 } 4975 4976 ret = ufshcd_dme_enable(hba); 4977 if (ret) { 4978 dev_err(hba->dev, "Enabling DME failed\n"); 4979 return ret; 4980 } 4981 4982 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4983 } else { 4984 ret = ufshcd_hba_execute_hce(hba); 4985 } 4986 4987 return ret; 4988 } 4989 EXPORT_SYMBOL_GPL(ufshcd_hba_enable); 4990 4991 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) 4992 { 4993 int tx_lanes = 0, i, err = 0; 4994 4995 if (!peer) 4996 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4997 &tx_lanes); 4998 else 4999 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 5000 &tx_lanes); 5001 for (i = 0; i < tx_lanes; i++) { 5002 if (!peer) 5003 err = ufshcd_dme_set(hba, 5004 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 5005 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5006 0); 5007 else 5008 err = ufshcd_dme_peer_set(hba, 5009 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 5010 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 5011 0); 5012 if (err) { 5013 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", 5014 __func__, peer, i, err); 5015 break; 5016 } 5017 } 5018 5019 return err; 5020 } 5021 5022 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) 5023 { 5024 return ufshcd_disable_tx_lcc(hba, true); 5025 } 5026 5027 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) 5028 { 5029 struct ufs_event_hist *e; 5030 5031 if (id >= UFS_EVT_CNT) 5032 return; 5033 5034 e = &hba->ufs_stats.event[id]; 5035 e->val[e->pos] = val; 5036 e->tstamp[e->pos] = local_clock(); 5037 e->cnt += 1; 5038 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; 5039 5040 ufshcd_vops_event_notify(hba, id, &val); 5041 } 5042 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); 5043 5044 /** 5045 * ufshcd_link_startup - Initialize unipro link startup 5046 * @hba: per adapter instance 5047 * 5048 * Return: 0 for success, non-zero in case of failure. 5049 */ 5050 static int ufshcd_link_startup(struct ufs_hba *hba) 5051 { 5052 int ret; 5053 int retries = DME_LINKSTARTUP_RETRIES; 5054 bool link_startup_again = false; 5055 5056 /* 5057 * If UFS device isn't active then we will have to issue link startup 5058 * 2 times to make sure the device state move to active. 5059 */ 5060 if (!ufshcd_is_ufs_dev_active(hba)) 5061 link_startup_again = true; 5062 5063 link_startup: 5064 do { 5065 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); 5066 5067 ret = ufshcd_dme_link_startup(hba); 5068 5069 /* check if device is detected by inter-connect layer */ 5070 if (!ret && !ufshcd_is_device_present(hba)) { 5071 ufshcd_update_evt_hist(hba, 5072 UFS_EVT_LINK_STARTUP_FAIL, 5073 0); 5074 dev_err(hba->dev, "%s: Device not present\n", __func__); 5075 ret = -ENXIO; 5076 goto out; 5077 } 5078 5079 /* 5080 * DME link lost indication is only received when link is up, 5081 * but we can't be sure if the link is up until link startup 5082 * succeeds. So reset the local Uni-Pro and try again. 5083 */ 5084 if (ret && retries && ufshcd_hba_enable(hba)) { 5085 ufshcd_update_evt_hist(hba, 5086 UFS_EVT_LINK_STARTUP_FAIL, 5087 (u32)ret); 5088 goto out; 5089 } 5090 } while (ret && retries--); 5091 5092 if (ret) { 5093 /* failed to get the link up... retire */ 5094 ufshcd_update_evt_hist(hba, 5095 UFS_EVT_LINK_STARTUP_FAIL, 5096 (u32)ret); 5097 goto out; 5098 } 5099 5100 if (link_startup_again) { 5101 link_startup_again = false; 5102 retries = DME_LINKSTARTUP_RETRIES; 5103 goto link_startup; 5104 } 5105 5106 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ 5107 ufshcd_init_pwr_info(hba); 5108 ufshcd_print_pwr_info(hba); 5109 5110 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { 5111 ret = ufshcd_disable_device_tx_lcc(hba); 5112 if (ret) 5113 goto out; 5114 } 5115 5116 /* Include any host controller configuration via UIC commands */ 5117 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE); 5118 if (ret) 5119 goto out; 5120 5121 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */ 5122 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 5123 ret = ufshcd_make_hba_operational(hba); 5124 out: 5125 if (ret) { 5126 dev_err(hba->dev, "link startup failed %d\n", ret); 5127 ufshcd_print_host_state(hba); 5128 ufshcd_print_pwr_info(hba); 5129 ufshcd_print_evt_hist(hba); 5130 } 5131 return ret; 5132 } 5133 5134 /** 5135 * ufshcd_verify_dev_init() - Verify device initialization 5136 * @hba: per-adapter instance 5137 * 5138 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 5139 * device Transport Protocol (UTP) layer is ready after a reset. 5140 * If the UTP layer at the device side is not initialized, it may 5141 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 5142 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 5143 * 5144 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5145 * < 0 if another error occurred. 5146 */ 5147 static int ufshcd_verify_dev_init(struct ufs_hba *hba) 5148 { 5149 int err = 0; 5150 int retries; 5151 5152 ufshcd_dev_man_lock(hba); 5153 5154 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 5155 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 5156 hba->nop_out_timeout); 5157 5158 if (!err || err == -ETIMEDOUT) 5159 break; 5160 5161 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 5162 } 5163 5164 ufshcd_dev_man_unlock(hba); 5165 5166 if (err) 5167 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 5168 return err; 5169 } 5170 5171 /** 5172 * ufshcd_setup_links - associate link b/w device wlun and other luns 5173 * @sdev: pointer to SCSI device 5174 * @hba: pointer to ufs hba 5175 */ 5176 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) 5177 { 5178 struct device_link *link; 5179 5180 /* 5181 * Device wlun is the supplier & rest of the luns are consumers. 5182 * This ensures that device wlun suspends after all other luns. 5183 */ 5184 if (hba->ufs_device_wlun) { 5185 link = device_link_add(&sdev->sdev_gendev, 5186 &hba->ufs_device_wlun->sdev_gendev, 5187 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); 5188 if (!link) { 5189 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n", 5190 dev_name(&hba->ufs_device_wlun->sdev_gendev)); 5191 return; 5192 } 5193 hba->luns_avail--; 5194 /* Ignore REPORT_LUN wlun probing */ 5195 if (hba->luns_avail == 1) { 5196 ufshcd_rpm_put(hba); 5197 return; 5198 } 5199 } else { 5200 /* 5201 * Device wlun is probed. The assumption is that WLUNs are 5202 * scanned before other LUNs. 5203 */ 5204 hba->luns_avail--; 5205 } 5206 } 5207 5208 /** 5209 * ufshcd_lu_init - Initialize the relevant parameters of the LU 5210 * @hba: per-adapter instance 5211 * @sdev: pointer to SCSI device 5212 */ 5213 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) 5214 { 5215 int len = QUERY_DESC_MAX_SIZE; 5216 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); 5217 u8 lun_qdepth = hba->nutrs; 5218 u8 *desc_buf; 5219 int ret; 5220 5221 desc_buf = kzalloc(len, GFP_KERNEL); 5222 if (!desc_buf) 5223 goto set_qdepth; 5224 5225 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len); 5226 if (ret < 0) { 5227 if (ret == -EOPNOTSUPP) 5228 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */ 5229 lun_qdepth = 1; 5230 kfree(desc_buf); 5231 goto set_qdepth; 5232 } 5233 5234 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) { 5235 /* 5236 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will 5237 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth 5238 */ 5239 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs); 5240 } 5241 /* 5242 * According to UFS device specification, the write protection mode is only supported by 5243 * normal LU, not supported by WLUN. 5244 */ 5245 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported && 5246 !hba->dev_info.is_lu_power_on_wp && 5247 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) 5248 hba->dev_info.is_lu_power_on_wp = true; 5249 5250 /* In case of RPMB LU, check if advanced RPMB mode is enabled */ 5251 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN && 5252 desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) 5253 hba->dev_info.b_advanced_rpmb_en = true; 5254 5255 5256 kfree(desc_buf); 5257 set_qdepth: 5258 /* 5259 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose 5260 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue. 5261 */ 5262 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth); 5263 scsi_change_queue_depth(sdev, lun_qdepth); 5264 } 5265 5266 /** 5267 * ufshcd_sdev_init - handle initial SCSI device configurations 5268 * @sdev: pointer to SCSI device 5269 * 5270 * Return: success. 5271 */ 5272 static int ufshcd_sdev_init(struct scsi_device *sdev) 5273 { 5274 struct ufs_hba *hba; 5275 5276 hba = shost_priv(sdev->host); 5277 5278 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 5279 sdev->use_10_for_ms = 1; 5280 5281 /* DBD field should be set to 1 in mode sense(10) */ 5282 sdev->set_dbd_for_ms = 1; 5283 5284 /* allow SCSI layer to restart the device in case of errors */ 5285 sdev->allow_restart = 1; 5286 5287 /* REPORT SUPPORTED OPERATION CODES is not supported */ 5288 sdev->no_report_opcodes = 1; 5289 5290 /* WRITE_SAME command is not supported */ 5291 sdev->no_write_same = 1; 5292 5293 ufshcd_lu_init(hba, sdev); 5294 5295 ufshcd_setup_links(hba, sdev); 5296 5297 return 0; 5298 } 5299 5300 /** 5301 * ufshcd_change_queue_depth - change queue depth 5302 * @sdev: pointer to SCSI device 5303 * @depth: required depth to set 5304 * 5305 * Change queue depth and make sure the max. limits are not crossed. 5306 * 5307 * Return: new queue depth. 5308 */ 5309 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 5310 { 5311 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); 5312 } 5313 5314 /** 5315 * ufshcd_sdev_configure - adjust SCSI device configurations 5316 * @sdev: pointer to SCSI device 5317 * @lim: queue limits 5318 * 5319 * Return: 0 (success). 5320 */ 5321 static int ufshcd_sdev_configure(struct scsi_device *sdev, 5322 struct queue_limits *lim) 5323 { 5324 struct ufs_hba *hba = shost_priv(sdev->host); 5325 struct request_queue *q = sdev->request_queue; 5326 5327 lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; 5328 5329 /* 5330 * Block runtime-pm until all consumers are added. 5331 * Refer ufshcd_setup_links(). 5332 */ 5333 if (is_device_wlun(sdev)) 5334 pm_runtime_get_noresume(&sdev->sdev_gendev); 5335 else if (ufshcd_is_rpm_autosuspend_allowed(hba)) 5336 sdev->rpm_autosuspend = 1; 5337 /* 5338 * Do not print messages during runtime PM to avoid never-ending cycles 5339 * of messages written back to storage by user space causing runtime 5340 * resume, causing more messages and so on. 5341 */ 5342 sdev->silence_suspend = 1; 5343 5344 if (hba->vops && hba->vops->config_scsi_dev) 5345 hba->vops->config_scsi_dev(sdev); 5346 5347 ufshcd_crypto_register(hba, q); 5348 5349 return 0; 5350 } 5351 5352 /** 5353 * ufshcd_sdev_destroy - remove SCSI device configurations 5354 * @sdev: pointer to SCSI device 5355 */ 5356 static void ufshcd_sdev_destroy(struct scsi_device *sdev) 5357 { 5358 struct ufs_hba *hba; 5359 unsigned long flags; 5360 5361 hba = shost_priv(sdev->host); 5362 5363 /* Drop the reference as it won't be needed anymore */ 5364 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 5365 spin_lock_irqsave(hba->host->host_lock, flags); 5366 hba->ufs_device_wlun = NULL; 5367 spin_unlock_irqrestore(hba->host->host_lock, flags); 5368 } else if (hba->ufs_device_wlun) { 5369 struct device *supplier = NULL; 5370 5371 /* Ensure UFS Device WLUN exists and does not disappear */ 5372 spin_lock_irqsave(hba->host->host_lock, flags); 5373 if (hba->ufs_device_wlun) { 5374 supplier = &hba->ufs_device_wlun->sdev_gendev; 5375 get_device(supplier); 5376 } 5377 spin_unlock_irqrestore(hba->host->host_lock, flags); 5378 5379 if (supplier) { 5380 /* 5381 * If a LUN fails to probe (e.g. absent BOOT WLUN), the 5382 * device will not have been registered but can still 5383 * have a device link holding a reference to the device. 5384 */ 5385 device_link_remove(&sdev->sdev_gendev, supplier); 5386 put_device(supplier); 5387 } 5388 } 5389 } 5390 5391 /** 5392 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5393 * @lrbp: pointer to local reference block of completed command 5394 * @scsi_status: SCSI command status 5395 * 5396 * Return: value base on SCSI command status. 5397 */ 5398 static inline int 5399 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 5400 { 5401 int result = 0; 5402 5403 switch (scsi_status) { 5404 case SAM_STAT_CHECK_CONDITION: 5405 ufshcd_copy_sense_data(lrbp); 5406 fallthrough; 5407 case SAM_STAT_GOOD: 5408 result |= DID_OK << 16 | scsi_status; 5409 break; 5410 case SAM_STAT_TASK_SET_FULL: 5411 case SAM_STAT_BUSY: 5412 case SAM_STAT_TASK_ABORTED: 5413 ufshcd_copy_sense_data(lrbp); 5414 result |= scsi_status; 5415 break; 5416 default: 5417 result |= DID_ERROR << 16; 5418 break; 5419 } /* end of switch */ 5420 5421 return result; 5422 } 5423 5424 /** 5425 * ufshcd_transfer_rsp_status - Get overall status of the response 5426 * @hba: per adapter instance 5427 * @lrbp: pointer to local reference block of completed command 5428 * @cqe: pointer to the completion queue entry 5429 * 5430 * Return: result of the command to notify SCSI midlayer. 5431 */ 5432 static inline int 5433 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 5434 struct cq_entry *cqe) 5435 { 5436 int result = 0; 5437 int scsi_status; 5438 enum utp_ocs ocs; 5439 u8 upiu_flags; 5440 u32 resid; 5441 5442 upiu_flags = lrbp->ucd_rsp_ptr->header.flags; 5443 resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); 5444 /* 5445 * Test !overflow instead of underflow to support UFS devices that do 5446 * not set either flag. 5447 */ 5448 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5449 scsi_set_resid(lrbp->cmd, resid); 5450 5451 /* overall command status of utrd */ 5452 ocs = ufshcd_get_tr_ocs(lrbp, cqe); 5453 5454 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { 5455 if (lrbp->ucd_rsp_ptr->header.response || 5456 lrbp->ucd_rsp_ptr->header.status) 5457 ocs = OCS_SUCCESS; 5458 } 5459 5460 switch (ocs) { 5461 case OCS_SUCCESS: 5462 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 5463 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { 5464 case UPIU_TRANSACTION_RESPONSE: 5465 /* 5466 * get the result based on SCSI status response 5467 * to notify the SCSI midlayer of the command status 5468 */ 5469 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5470 result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 5471 5472 /* 5473 * Currently we are only supporting BKOPs exception 5474 * events hence we can ignore BKOPs exception event 5475 * during power management callbacks. BKOPs exception 5476 * event is not expected to be raised in runtime suspend 5477 * callback as it allows the urgent bkops. 5478 * During system suspend, we are anyway forcefully 5479 * disabling the bkops and if urgent bkops is needed 5480 * it will be enabled on system resume. Long term 5481 * solution could be to abort the system suspend if 5482 * UFS device needs urgent BKOPs. 5483 */ 5484 if (!hba->pm_op_in_progress && 5485 !ufshcd_eh_in_progress(hba) && 5486 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 5487 /* Flushed in suspend */ 5488 schedule_work(&hba->eeh_work); 5489 break; 5490 case UPIU_TRANSACTION_REJECT_UPIU: 5491 /* TODO: handle Reject UPIU Response */ 5492 result = DID_ERROR << 16; 5493 dev_err(hba->dev, 5494 "Reject UPIU not fully implemented\n"); 5495 break; 5496 default: 5497 dev_err(hba->dev, 5498 "Unexpected request response code = %x\n", 5499 result); 5500 result = DID_ERROR << 16; 5501 break; 5502 } 5503 break; 5504 case OCS_ABORTED: 5505 case OCS_INVALID_COMMAND_STATUS: 5506 result |= DID_REQUEUE << 16; 5507 dev_warn(hba->dev, 5508 "OCS %s from controller for tag %d\n", 5509 (ocs == OCS_ABORTED ? "aborted" : "invalid"), 5510 lrbp->task_tag); 5511 break; 5512 case OCS_INVALID_CMD_TABLE_ATTR: 5513 case OCS_INVALID_PRDT_ATTR: 5514 case OCS_MISMATCH_DATA_BUF_SIZE: 5515 case OCS_MISMATCH_RESP_UPIU_SIZE: 5516 case OCS_PEER_COMM_FAILURE: 5517 case OCS_FATAL_ERROR: 5518 case OCS_DEVICE_FATAL_ERROR: 5519 case OCS_INVALID_CRYPTO_CONFIG: 5520 case OCS_GENERAL_CRYPTO_ERROR: 5521 default: 5522 result |= DID_ERROR << 16; 5523 dev_err(hba->dev, 5524 "OCS error from controller = %x for tag %d\n", 5525 ocs, lrbp->task_tag); 5526 ufshcd_print_evt_hist(hba); 5527 ufshcd_print_host_state(hba); 5528 break; 5529 } /* end of switch */ 5530 5531 if ((host_byte(result) != DID_OK) && 5532 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) 5533 ufshcd_print_tr(hba, lrbp->task_tag, true); 5534 return result; 5535 } 5536 5537 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, 5538 u32 intr_mask) 5539 { 5540 if (!ufshcd_is_auto_hibern8_supported(hba) || 5541 !ufshcd_is_auto_hibern8_enabled(hba)) 5542 return false; 5543 5544 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK)) 5545 return false; 5546 5547 if (hba->active_uic_cmd && 5548 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER || 5549 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT)) 5550 return false; 5551 5552 return true; 5553 } 5554 5555 /** 5556 * ufshcd_uic_cmd_compl - handle completion of uic command 5557 * @hba: per adapter instance 5558 * @intr_status: interrupt status generated by the controller 5559 * 5560 * Return: 5561 * IRQ_HANDLED - If interrupt is valid 5562 * IRQ_NONE - If invalid interrupt 5563 */ 5564 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 5565 { 5566 irqreturn_t retval = IRQ_NONE; 5567 struct uic_command *cmd; 5568 5569 guard(spinlock_irqsave)(hba->host->host_lock); 5570 cmd = hba->active_uic_cmd; 5571 if (!cmd) 5572 goto unlock; 5573 5574 if (ufshcd_is_auto_hibern8_error(hba, intr_status)) 5575 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); 5576 5577 if (intr_status & UIC_COMMAND_COMPL) { 5578 cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); 5579 cmd->argument3 = ufshcd_get_dme_attr_val(hba); 5580 if (!hba->uic_async_done) 5581 cmd->cmd_active = 0; 5582 complete(&cmd->done); 5583 retval = IRQ_HANDLED; 5584 } 5585 5586 if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { 5587 cmd->cmd_active = 0; 5588 complete(hba->uic_async_done); 5589 retval = IRQ_HANDLED; 5590 } 5591 5592 if (retval == IRQ_HANDLED) 5593 ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); 5594 5595 unlock: 5596 return retval; 5597 } 5598 5599 /* Release the resources allocated for processing a SCSI command. */ 5600 void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 5601 struct ufshcd_lrb *lrbp) 5602 { 5603 struct scsi_cmnd *cmd = lrbp->cmd; 5604 5605 scsi_dma_unmap(cmd); 5606 ufshcd_crypto_clear_prdt(hba, lrbp); 5607 ufshcd_release(hba); 5608 ufshcd_clk_scaling_update_busy(hba); 5609 } 5610 5611 /** 5612 * ufshcd_compl_one_cqe - handle a completion queue entry 5613 * @hba: per adapter instance 5614 * @task_tag: the task tag of the request to be completed 5615 * @cqe: pointer to the completion queue entry 5616 */ 5617 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5618 struct cq_entry *cqe) 5619 { 5620 struct ufshcd_lrb *lrbp; 5621 struct scsi_cmnd *cmd; 5622 enum utp_ocs ocs; 5623 5624 lrbp = &hba->lrb[task_tag]; 5625 lrbp->compl_time_stamp = ktime_get(); 5626 lrbp->compl_time_stamp_local_clock = local_clock(); 5627 cmd = lrbp->cmd; 5628 if (cmd) { 5629 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 5630 ufshcd_update_monitor(hba, lrbp); 5631 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP); 5632 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); 5633 ufshcd_release_scsi_cmd(hba, lrbp); 5634 /* Do not touch lrbp after scsi done */ 5635 scsi_done(cmd); 5636 } else { 5637 if (cqe) { 5638 ocs = le32_to_cpu(cqe->status) & MASK_OCS; 5639 lrbp->utr_descriptor_ptr->header.ocs = ocs; 5640 } 5641 complete(&hba->dev_cmd.complete); 5642 } 5643 } 5644 5645 /** 5646 * __ufshcd_transfer_req_compl - handle SCSI and query command completion 5647 * @hba: per adapter instance 5648 * @completed_reqs: bitmask that indicates which requests to complete 5649 */ 5650 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, 5651 unsigned long completed_reqs) 5652 { 5653 int tag; 5654 5655 for_each_set_bit(tag, &completed_reqs, hba->nutrs) 5656 ufshcd_compl_one_cqe(hba, tag, NULL); 5657 } 5658 5659 /* Any value that is not an existing queue number is fine for this constant. */ 5660 enum { 5661 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1 5662 }; 5663 5664 static void ufshcd_clear_polled(struct ufs_hba *hba, 5665 unsigned long *completed_reqs) 5666 { 5667 int tag; 5668 5669 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5670 struct scsi_cmnd *cmd = hba->lrb[tag].cmd; 5671 5672 if (!cmd) 5673 continue; 5674 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED) 5675 __clear_bit(tag, completed_reqs); 5676 } 5677 } 5678 5679 /* 5680 * Return: > 0 if one or more commands have been completed or 0 if no 5681 * requests have been completed. 5682 */ 5683 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5684 { 5685 struct ufs_hba *hba = shost_priv(shost); 5686 unsigned long completed_reqs, flags; 5687 u32 tr_doorbell; 5688 struct ufs_hw_queue *hwq; 5689 5690 if (hba->mcq_enabled) { 5691 hwq = &hba->uhq[queue_num]; 5692 5693 return ufshcd_mcq_poll_cqe_lock(hba, hwq); 5694 } 5695 5696 spin_lock_irqsave(&hba->outstanding_lock, flags); 5697 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5698 completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5699 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5700 "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5701 hba->outstanding_reqs); 5702 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) { 5703 /* Do not complete polled requests from interrupt context. */ 5704 ufshcd_clear_polled(hba, &completed_reqs); 5705 } 5706 hba->outstanding_reqs &= ~completed_reqs; 5707 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5708 5709 if (completed_reqs) 5710 __ufshcd_transfer_req_compl(hba, completed_reqs); 5711 5712 return completed_reqs != 0; 5713 } 5714 5715 /** 5716 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is 5717 * invoked from the error handler context or ufshcd_host_reset_and_restore() 5718 * to complete the pending transfers and free the resources associated with 5719 * the scsi command. 5720 * 5721 * @hba: per adapter instance 5722 * @force_compl: This flag is set to true when invoked 5723 * from ufshcd_host_reset_and_restore() in which case it requires special 5724 * handling because the host controller has been reset by ufshcd_hba_stop(). 5725 */ 5726 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, 5727 bool force_compl) 5728 { 5729 struct ufs_hw_queue *hwq; 5730 struct ufshcd_lrb *lrbp; 5731 struct scsi_cmnd *cmd; 5732 unsigned long flags; 5733 int tag; 5734 5735 for (tag = 0; tag < hba->nutrs; tag++) { 5736 lrbp = &hba->lrb[tag]; 5737 cmd = lrbp->cmd; 5738 if (!ufshcd_cmd_inflight(cmd) || 5739 test_bit(SCMD_STATE_COMPLETE, &cmd->state)) 5740 continue; 5741 5742 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 5743 if (!hwq) 5744 continue; 5745 5746 if (force_compl) { 5747 ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5748 /* 5749 * For those cmds of which the cqes are not present 5750 * in the cq, complete them explicitly. 5751 */ 5752 spin_lock_irqsave(&hwq->cq_lock, flags); 5753 if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5754 set_host_byte(cmd, DID_REQUEUE); 5755 ufshcd_release_scsi_cmd(hba, lrbp); 5756 scsi_done(cmd); 5757 } 5758 spin_unlock_irqrestore(&hwq->cq_lock, flags); 5759 } else { 5760 ufshcd_mcq_poll_cqe_lock(hba, hwq); 5761 } 5762 } 5763 } 5764 5765 /** 5766 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5767 * @hba: per adapter instance 5768 * 5769 * Return: 5770 * IRQ_HANDLED - If interrupt is valid 5771 * IRQ_NONE - If invalid interrupt 5772 */ 5773 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5774 { 5775 /* Resetting interrupt aggregation counters first and reading the 5776 * DOOR_BELL afterward allows us to handle all the completed requests. 5777 * In order to prevent other interrupts starvation the DB is read once 5778 * after reset. The down side of this solution is the possibility of 5779 * false interrupt if device completes another request after resetting 5780 * aggregation and before reading the DB. 5781 */ 5782 if (ufshcd_is_intr_aggr_allowed(hba) && 5783 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) 5784 ufshcd_reset_intr_aggr(hba); 5785 5786 if (ufs_fail_completion(hba)) 5787 return IRQ_HANDLED; 5788 5789 /* 5790 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5791 * do not want polling to trigger spurious interrupt complaints. 5792 */ 5793 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT); 5794 5795 return IRQ_HANDLED; 5796 } 5797 5798 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) 5799 { 5800 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 5801 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, 5802 &ee_ctrl_mask); 5803 } 5804 5805 int ufshcd_write_ee_control(struct ufs_hba *hba) 5806 { 5807 int err; 5808 5809 mutex_lock(&hba->ee_ctrl_mutex); 5810 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask); 5811 mutex_unlock(&hba->ee_ctrl_mutex); 5812 if (err) 5813 dev_err(hba->dev, "%s: failed to write ee control %d\n", 5814 __func__, err); 5815 return err; 5816 } 5817 5818 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 5819 const u16 *other_mask, u16 set, u16 clr) 5820 { 5821 u16 new_mask, ee_ctrl_mask; 5822 int err = 0; 5823 5824 mutex_lock(&hba->ee_ctrl_mutex); 5825 new_mask = (*mask & ~clr) | set; 5826 ee_ctrl_mask = new_mask | *other_mask; 5827 if (ee_ctrl_mask != hba->ee_ctrl_mask) 5828 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask); 5829 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */ 5830 if (!err) { 5831 hba->ee_ctrl_mask = ee_ctrl_mask; 5832 *mask = new_mask; 5833 } 5834 mutex_unlock(&hba->ee_ctrl_mutex); 5835 return err; 5836 } 5837 5838 /** 5839 * ufshcd_disable_ee - disable exception event 5840 * @hba: per-adapter instance 5841 * @mask: exception event to disable 5842 * 5843 * Disables exception event in the device so that the EVENT_ALERT 5844 * bit is not set. 5845 * 5846 * Return: zero on success, non-zero error value on failure. 5847 */ 5848 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 5849 { 5850 return ufshcd_update_ee_drv_mask(hba, 0, mask); 5851 } 5852 5853 /** 5854 * ufshcd_enable_ee - enable exception event 5855 * @hba: per-adapter instance 5856 * @mask: exception event to enable 5857 * 5858 * Enable corresponding exception event in the device to allow 5859 * device to alert host in critical scenarios. 5860 * 5861 * Return: zero on success, non-zero error value on failure. 5862 */ 5863 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 5864 { 5865 return ufshcd_update_ee_drv_mask(hba, mask, 0); 5866 } 5867 5868 /** 5869 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 5870 * @hba: per-adapter instance 5871 * 5872 * Allow device to manage background operations on its own. Enabling 5873 * this might lead to inconsistent latencies during normal data transfers 5874 * as the device is allowed to manage its own way of handling background 5875 * operations. 5876 * 5877 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5878 * < 0 if another error occurred. 5879 */ 5880 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 5881 { 5882 int err = 0; 5883 5884 if (hba->auto_bkops_enabled) 5885 goto out; 5886 5887 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 5888 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5889 if (err) { 5890 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 5891 __func__, err); 5892 goto out; 5893 } 5894 5895 hba->auto_bkops_enabled = true; 5896 trace_ufshcd_auto_bkops_state(hba, "Enabled"); 5897 5898 /* No need of URGENT_BKOPS exception from the device */ 5899 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5900 if (err) 5901 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 5902 __func__, err); 5903 out: 5904 return err; 5905 } 5906 5907 /** 5908 * ufshcd_disable_auto_bkops - block device in doing background operations 5909 * @hba: per-adapter instance 5910 * 5911 * Disabling background operations improves command response latency but 5912 * has drawback of device moving into critical state where the device is 5913 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 5914 * host is idle so that BKOPS are managed effectively without any negative 5915 * impacts. 5916 * 5917 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5918 * < 0 if another error occurred. 5919 */ 5920 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 5921 { 5922 int err = 0; 5923 5924 if (!hba->auto_bkops_enabled) 5925 goto out; 5926 5927 /* 5928 * If host assisted BKOPs is to be enabled, make sure 5929 * urgent bkops exception is allowed. 5930 */ 5931 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 5932 if (err) { 5933 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 5934 __func__, err); 5935 goto out; 5936 } 5937 5938 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 5939 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5940 if (err) { 5941 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 5942 __func__, err); 5943 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5944 goto out; 5945 } 5946 5947 hba->auto_bkops_enabled = false; 5948 trace_ufshcd_auto_bkops_state(hba, "Disabled"); 5949 hba->is_urgent_bkops_lvl_checked = false; 5950 out: 5951 return err; 5952 } 5953 5954 /** 5955 * ufshcd_force_reset_auto_bkops - force reset auto bkops state 5956 * @hba: per adapter instance 5957 * 5958 * After a device reset the device may toggle the BKOPS_EN flag 5959 * to default value. The s/w tracking variables should be updated 5960 * as well. This function would change the auto-bkops state based on 5961 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. 5962 */ 5963 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 5964 { 5965 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { 5966 hba->auto_bkops_enabled = false; 5967 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 5968 ufshcd_enable_auto_bkops(hba); 5969 } else { 5970 hba->auto_bkops_enabled = true; 5971 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; 5972 ufshcd_disable_auto_bkops(hba); 5973 } 5974 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 5975 hba->is_urgent_bkops_lvl_checked = false; 5976 } 5977 5978 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 5979 { 5980 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5981 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 5982 } 5983 5984 /** 5985 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 5986 * @hba: per-adapter instance 5987 * 5988 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 5989 * flag in the device to permit background operations if the device 5990 * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", 5991 * disable otherwise. 5992 * 5993 * Return: 0 for success, non-zero in case of failure. 5994 * 5995 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 5996 * to know whether auto bkops is enabled or disabled after this function 5997 * returns control to it. 5998 */ 5999 static int ufshcd_bkops_ctrl(struct ufs_hba *hba) 6000 { 6001 enum bkops_status status = hba->urgent_bkops_lvl; 6002 u32 curr_status = 0; 6003 int err; 6004 6005 err = ufshcd_get_bkops_status(hba, &curr_status); 6006 if (err) { 6007 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6008 __func__, err); 6009 goto out; 6010 } else if (curr_status > BKOPS_STATUS_MAX) { 6011 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 6012 __func__, curr_status); 6013 err = -EINVAL; 6014 goto out; 6015 } 6016 6017 if (curr_status >= status) 6018 err = ufshcd_enable_auto_bkops(hba); 6019 else 6020 err = ufshcd_disable_auto_bkops(hba); 6021 out: 6022 return err; 6023 } 6024 6025 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 6026 { 6027 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6028 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 6029 } 6030 6031 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) 6032 { 6033 int err; 6034 u32 curr_status = 0; 6035 6036 if (hba->is_urgent_bkops_lvl_checked) 6037 goto enable_auto_bkops; 6038 6039 err = ufshcd_get_bkops_status(hba, &curr_status); 6040 if (err) { 6041 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6042 __func__, err); 6043 goto out; 6044 } 6045 6046 /* 6047 * We are seeing that some devices are raising the urgent bkops 6048 * exception events even when BKOPS status doesn't indicate performace 6049 * impacted or critical. Handle these device by determining their urgent 6050 * bkops status at runtime. 6051 */ 6052 if (curr_status < BKOPS_STATUS_PERF_IMPACT) { 6053 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", 6054 __func__, curr_status); 6055 /* update the current status as the urgent bkops level */ 6056 hba->urgent_bkops_lvl = curr_status; 6057 hba->is_urgent_bkops_lvl_checked = true; 6058 } 6059 6060 enable_auto_bkops: 6061 err = ufshcd_enable_auto_bkops(hba); 6062 out: 6063 if (err < 0) 6064 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 6065 __func__, err); 6066 } 6067 6068 /* 6069 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6070 * < 0 if another error occurred. 6071 */ 6072 int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) 6073 { 6074 struct utp_upiu_query_v4_0 *upiu_resp; 6075 struct ufs_query_req *request = NULL; 6076 struct ufs_query_res *response = NULL; 6077 int err; 6078 6079 if (hba->dev_info.wspecversion < 0x410) 6080 return -EOPNOTSUPP; 6081 6082 ufshcd_hold(hba); 6083 mutex_lock(&hba->dev_cmd.lock); 6084 6085 ufshcd_init_query(hba, &request, &response, 6086 UPIU_QUERY_OPCODE_READ_ATTR, 6087 QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); 6088 6089 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 6090 6091 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 6092 6093 if (err) { 6094 dev_err(hba->dev, "%s: failed to read device level exception %d\n", 6095 __func__, err); 6096 goto out; 6097 } 6098 6099 upiu_resp = (struct utp_upiu_query_v4_0 *)response; 6100 *exception_id = get_unaligned_be64(&upiu_resp->osf3); 6101 out: 6102 mutex_unlock(&hba->dev_cmd.lock); 6103 ufshcd_release(hba); 6104 6105 return err; 6106 } 6107 6108 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) 6109 { 6110 u8 index; 6111 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG : 6112 UPIU_QUERY_OPCODE_CLEAR_FLAG; 6113 6114 index = ufshcd_wb_get_query_index(hba); 6115 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL); 6116 } 6117 6118 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) 6119 { 6120 int ret; 6121 6122 if (!ufshcd_is_wb_allowed(hba) || 6123 hba->dev_info.wb_enabled == enable) 6124 return 0; 6125 6126 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); 6127 if (ret) { 6128 dev_err(hba->dev, "%s: Write Booster %s failed %d\n", 6129 __func__, enable ? "enabling" : "disabling", ret); 6130 return ret; 6131 } 6132 6133 hba->dev_info.wb_enabled = enable; 6134 dev_dbg(hba->dev, "%s: Write Booster %s\n", 6135 __func__, enable ? "enabled" : "disabled"); 6136 6137 return ret; 6138 } 6139 6140 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 6141 bool enable) 6142 { 6143 int ret; 6144 6145 ret = __ufshcd_wb_toggle(hba, enable, 6146 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); 6147 if (ret) { 6148 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", 6149 __func__, enable ? "enabling" : "disabling", ret); 6150 return; 6151 } 6152 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", 6153 __func__, enable ? "enabled" : "disabled"); 6154 } 6155 6156 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) 6157 { 6158 int ret; 6159 6160 if (!ufshcd_is_wb_allowed(hba) || 6161 hba->dev_info.wb_buf_flush_enabled == enable) 6162 return 0; 6163 6164 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); 6165 if (ret) { 6166 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", 6167 __func__, enable ? "enabling" : "disabling", ret); 6168 return ret; 6169 } 6170 6171 hba->dev_info.wb_buf_flush_enabled = enable; 6172 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", 6173 __func__, enable ? "enabled" : "disabled"); 6174 6175 return ret; 6176 } 6177 6178 int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) 6179 { 6180 int ret; 6181 u8 index; 6182 6183 index = ufshcd_wb_get_query_index(hba); 6184 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 6185 QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); 6186 if (ret) 6187 dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", 6188 __func__, ret); 6189 6190 return ret; 6191 } 6192 6193 static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, 6194 u32 avail_buf) 6195 { 6196 u32 cur_buf; 6197 int ret; 6198 u8 index; 6199 6200 index = ufshcd_wb_get_query_index(hba); 6201 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6202 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, 6203 index, 0, &cur_buf); 6204 if (ret) { 6205 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", 6206 __func__, ret); 6207 return false; 6208 } 6209 6210 if (!cur_buf) { 6211 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n", 6212 cur_buf); 6213 return false; 6214 } 6215 /* Let it continue to flush when available buffer exceeds threshold */ 6216 return avail_buf < hba->vps->wb_flush_threshold; 6217 } 6218 6219 static void ufshcd_wb_force_disable(struct ufs_hba *hba) 6220 { 6221 if (ufshcd_is_wb_buf_flush_allowed(hba)) 6222 ufshcd_wb_toggle_buf_flush(hba, false); 6223 6224 ufshcd_wb_toggle_buf_flush_during_h8(hba, false); 6225 ufshcd_wb_toggle(hba, false); 6226 hba->caps &= ~UFSHCD_CAP_WB_EN; 6227 6228 dev_info(hba->dev, "%s: WB force disabled\n", __func__); 6229 } 6230 6231 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba) 6232 { 6233 u32 lifetime; 6234 int ret; 6235 u8 index; 6236 6237 index = ufshcd_wb_get_query_index(hba); 6238 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6239 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST, 6240 index, 0, &lifetime); 6241 if (ret) { 6242 dev_err(hba->dev, 6243 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n", 6244 __func__, ret); 6245 return false; 6246 } 6247 6248 if (lifetime == UFS_WB_EXCEED_LIFETIME) { 6249 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n", 6250 __func__, lifetime); 6251 return false; 6252 } 6253 6254 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n", 6255 __func__, lifetime); 6256 6257 return true; 6258 } 6259 6260 static bool ufshcd_wb_need_flush(struct ufs_hba *hba) 6261 { 6262 int ret; 6263 u32 avail_buf; 6264 u8 index; 6265 6266 if (!ufshcd_is_wb_allowed(hba)) 6267 return false; 6268 6269 if (!ufshcd_is_wb_buf_lifetime_available(hba)) { 6270 ufshcd_wb_force_disable(hba); 6271 return false; 6272 } 6273 6274 /* 6275 * With user-space reduction enabled, it's enough to enable flush 6276 * by checking only the available buffer. The threshold 6277 * defined here is > 90% full. 6278 * With user-space preserved enabled, the current-buffer 6279 * should be checked too because the wb buffer size can reduce 6280 * when disk tends to be full. This info is provided by current 6281 * buffer (dCurrentWriteBoosterBufferSize). 6282 */ 6283 index = ufshcd_wb_get_query_index(hba); 6284 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6285 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, 6286 index, 0, &avail_buf); 6287 if (ret) { 6288 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", 6289 __func__, ret); 6290 return false; 6291 } 6292 6293 if (!hba->dev_info.b_presrv_uspc_en) 6294 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); 6295 6296 return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); 6297 } 6298 6299 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) 6300 { 6301 struct ufs_hba *hba = container_of(to_delayed_work(work), 6302 struct ufs_hba, 6303 rpm_dev_flush_recheck_work); 6304 /* 6305 * To prevent unnecessary VCC power drain after device finishes 6306 * WriteBooster buffer flush or Auto BKOPs, force runtime resume 6307 * after a certain delay to recheck the threshold by next runtime 6308 * suspend. 6309 */ 6310 ufshcd_rpm_get_sync(hba); 6311 ufshcd_rpm_put_sync(hba); 6312 } 6313 6314 /** 6315 * ufshcd_exception_event_handler - handle exceptions raised by device 6316 * @work: pointer to work data 6317 * 6318 * Read bExceptionEventStatus attribute from the device and handle the 6319 * exception event accordingly. 6320 */ 6321 static void ufshcd_exception_event_handler(struct work_struct *work) 6322 { 6323 struct ufs_hba *hba; 6324 int err; 6325 u32 status = 0; 6326 hba = container_of(work, struct ufs_hba, eeh_work); 6327 6328 err = ufshcd_get_ee_status(hba, &status); 6329 if (err) { 6330 dev_err(hba->dev, "%s: failed to get exception status %d\n", 6331 __func__, err); 6332 return; 6333 } 6334 6335 trace_ufshcd_exception_event(hba, status); 6336 6337 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) 6338 ufshcd_bkops_exception_event_handler(hba); 6339 6340 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) 6341 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); 6342 6343 if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { 6344 hba->critical_health_count++; 6345 sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); 6346 } 6347 6348 if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { 6349 atomic_inc(&hba->dev_lvl_exception_count); 6350 sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); 6351 } 6352 6353 ufs_debugfs_exception_event(hba, status); 6354 } 6355 6356 /* Complete requests that have door-bell cleared */ 6357 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) 6358 { 6359 if (hba->mcq_enabled) 6360 ufshcd_mcq_compl_pending_transfer(hba, force_compl); 6361 else 6362 ufshcd_transfer_req_compl(hba); 6363 6364 ufshcd_tmc_handler(hba); 6365 } 6366 6367 /** 6368 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is 6369 * to recover from the DL NAC errors or not. 6370 * @hba: per-adapter instance 6371 * 6372 * Return: true if error handling is required, false otherwise. 6373 */ 6374 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) 6375 { 6376 unsigned long flags; 6377 bool err_handling = true; 6378 6379 spin_lock_irqsave(hba->host->host_lock, flags); 6380 /* 6381 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the 6382 * device fatal error and/or DL NAC & REPLAY timeout errors. 6383 */ 6384 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR)) 6385 goto out; 6386 6387 if ((hba->saved_err & DEVICE_FATAL_ERROR) || 6388 ((hba->saved_err & UIC_ERROR) && 6389 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR))) 6390 goto out; 6391 6392 if ((hba->saved_err & UIC_ERROR) && 6393 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) { 6394 int err; 6395 /* 6396 * wait for 50ms to see if we can get any other errors or not. 6397 */ 6398 spin_unlock_irqrestore(hba->host->host_lock, flags); 6399 msleep(50); 6400 spin_lock_irqsave(hba->host->host_lock, flags); 6401 6402 /* 6403 * now check if we have got any other severe errors other than 6404 * DL NAC error? 6405 */ 6406 if ((hba->saved_err & INT_FATAL_ERRORS) || 6407 ((hba->saved_err & UIC_ERROR) && 6408 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR))) 6409 goto out; 6410 6411 /* 6412 * As DL NAC is the only error received so far, send out NOP 6413 * command to confirm if link is still active or not. 6414 * - If we don't get any response then do error recovery. 6415 * - If we get response then clear the DL NAC error bit. 6416 */ 6417 6418 spin_unlock_irqrestore(hba->host->host_lock, flags); 6419 err = ufshcd_verify_dev_init(hba); 6420 spin_lock_irqsave(hba->host->host_lock, flags); 6421 6422 if (err) 6423 goto out; 6424 6425 /* Link seems to be alive hence ignore the DL NAC errors */ 6426 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR) 6427 hba->saved_err &= ~UIC_ERROR; 6428 /* clear NAC error */ 6429 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6430 if (!hba->saved_uic_err) 6431 err_handling = false; 6432 } 6433 out: 6434 spin_unlock_irqrestore(hba->host->host_lock, flags); 6435 return err_handling; 6436 } 6437 6438 /* host lock must be held before calling this func */ 6439 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba) 6440 { 6441 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) || 6442 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)); 6443 } 6444 6445 void ufshcd_schedule_eh_work(struct ufs_hba *hba) 6446 { 6447 lockdep_assert_held(hba->host->host_lock); 6448 6449 /* handle fatal errors only when link is not in error state */ 6450 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6451 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6452 ufshcd_is_saved_err_fatal(hba)) 6453 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL; 6454 else 6455 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL; 6456 queue_work(hba->eh_wq, &hba->eh_work); 6457 } 6458 } 6459 6460 static void ufshcd_force_error_recovery(struct ufs_hba *hba) 6461 { 6462 spin_lock_irq(hba->host->host_lock); 6463 hba->force_reset = true; 6464 ufshcd_schedule_eh_work(hba); 6465 spin_unlock_irq(hba->host->host_lock); 6466 } 6467 6468 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) 6469 { 6470 mutex_lock(&hba->wb_mutex); 6471 down_write(&hba->clk_scaling_lock); 6472 hba->clk_scaling.is_allowed = allow; 6473 up_write(&hba->clk_scaling_lock); 6474 mutex_unlock(&hba->wb_mutex); 6475 } 6476 6477 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) 6478 { 6479 if (suspend) { 6480 if (hba->clk_scaling.is_enabled) 6481 ufshcd_suspend_clkscaling(hba); 6482 ufshcd_clk_scaling_allow(hba, false); 6483 } else { 6484 ufshcd_clk_scaling_allow(hba, true); 6485 if (hba->clk_scaling.is_enabled) 6486 ufshcd_resume_clkscaling(hba); 6487 } 6488 } 6489 6490 static void ufshcd_err_handling_prepare(struct ufs_hba *hba) 6491 { 6492 ufshcd_rpm_get_sync(hba); 6493 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || 6494 hba->is_sys_suspended) { 6495 enum ufs_pm_op pm_op; 6496 6497 /* 6498 * Don't assume anything of resume, if 6499 * resume fails, irq and clocks can be OFF, and powers 6500 * can be OFF or in LPM. 6501 */ 6502 ufshcd_setup_hba_vreg(hba, true); 6503 ufshcd_enable_irq(hba); 6504 ufshcd_setup_vreg(hba, true); 6505 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 6506 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 6507 ufshcd_hold(hba); 6508 if (!ufshcd_is_clkgating_allowed(hba)) 6509 ufshcd_setup_clocks(hba, true); 6510 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; 6511 ufshcd_vops_resume(hba, pm_op); 6512 } else { 6513 ufshcd_hold(hba); 6514 if (ufshcd_is_clkscaling_supported(hba) && 6515 hba->clk_scaling.is_enabled) 6516 ufshcd_suspend_clkscaling(hba); 6517 ufshcd_clk_scaling_allow(hba, false); 6518 } 6519 /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ 6520 blk_mq_quiesce_tagset(&hba->host->tag_set); 6521 cancel_work_sync(&hba->eeh_work); 6522 } 6523 6524 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) 6525 { 6526 blk_mq_unquiesce_tagset(&hba->host->tag_set); 6527 ufshcd_release(hba); 6528 if (ufshcd_is_clkscaling_supported(hba)) 6529 ufshcd_clk_scaling_suspend(hba, false); 6530 ufshcd_rpm_put(hba); 6531 } 6532 6533 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) 6534 { 6535 return (!hba->is_powered || hba->shutting_down || 6536 !hba->ufs_device_wlun || 6537 hba->ufshcd_state == UFSHCD_STATE_ERROR || 6538 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || 6539 ufshcd_is_link_broken(hba)))); 6540 } 6541 6542 #ifdef CONFIG_PM 6543 static void ufshcd_recover_pm_error(struct ufs_hba *hba) 6544 { 6545 struct Scsi_Host *shost = hba->host; 6546 struct scsi_device *sdev; 6547 struct request_queue *q; 6548 int ret; 6549 6550 hba->is_sys_suspended = false; 6551 /* 6552 * Set RPM status of wlun device to RPM_ACTIVE, 6553 * this also clears its runtime error. 6554 */ 6555 ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); 6556 6557 /* hba device might have a runtime error otherwise */ 6558 if (ret) 6559 ret = pm_runtime_set_active(hba->dev); 6560 /* 6561 * If wlun device had runtime error, we also need to resume those 6562 * consumer scsi devices in case any of them has failed to be 6563 * resumed due to supplier runtime resume failure. This is to unblock 6564 * blk_queue_enter in case there are bios waiting inside it. 6565 */ 6566 if (!ret) { 6567 shost_for_each_device(sdev, shost) { 6568 q = sdev->request_queue; 6569 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 6570 q->rpm_status == RPM_SUSPENDING)) 6571 pm_request_resume(q->dev); 6572 } 6573 } 6574 } 6575 #else 6576 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba) 6577 { 6578 } 6579 #endif 6580 6581 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) 6582 { 6583 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info; 6584 u32 mode; 6585 6586 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode); 6587 6588 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK)) 6589 return true; 6590 6591 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK)) 6592 return true; 6593 6594 return false; 6595 } 6596 6597 static bool ufshcd_abort_one(struct request *rq, void *priv) 6598 { 6599 int *ret = priv; 6600 u32 tag = rq->tag; 6601 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 6602 struct scsi_device *sdev = cmd->device; 6603 struct Scsi_Host *shost = sdev->host; 6604 struct ufs_hba *hba = shost_priv(shost); 6605 6606 *ret = ufshcd_try_to_abort_task(hba, tag); 6607 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6608 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, 6609 *ret ? "failed" : "succeeded"); 6610 6611 return *ret == 0; 6612 } 6613 6614 /** 6615 * ufshcd_abort_all - Abort all pending commands. 6616 * @hba: Host bus adapter pointer. 6617 * 6618 * Return: true if and only if the host controller needs to be reset. 6619 */ 6620 static bool ufshcd_abort_all(struct ufs_hba *hba) 6621 { 6622 int tag, ret = 0; 6623 6624 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); 6625 if (ret) 6626 goto out; 6627 6628 /* Clear pending task management requests */ 6629 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { 6630 ret = ufshcd_clear_tm_cmd(hba, tag); 6631 if (ret) 6632 goto out; 6633 } 6634 6635 out: 6636 /* Complete the requests that are cleared by s/w */ 6637 ufshcd_complete_requests(hba, false); 6638 6639 return ret != 0; 6640 } 6641 6642 /** 6643 * ufshcd_err_handler - handle UFS errors that require s/w attention 6644 * @work: pointer to work structure 6645 */ 6646 static void ufshcd_err_handler(struct work_struct *work) 6647 { 6648 int retries = MAX_ERR_HANDLER_RETRIES; 6649 struct ufs_hba *hba; 6650 unsigned long flags; 6651 bool needs_restore; 6652 bool needs_reset; 6653 int pmc_err; 6654 6655 hba = container_of(work, struct ufs_hba, eh_work); 6656 6657 dev_info(hba->dev, 6658 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", 6659 __func__, ufshcd_state_name[hba->ufshcd_state], 6660 hba->is_powered, hba->shutting_down, hba->saved_err, 6661 hba->saved_uic_err, hba->force_reset, 6662 ufshcd_is_link_broken(hba) ? "; link is broken" : ""); 6663 6664 down(&hba->host_sem); 6665 spin_lock_irqsave(hba->host->host_lock, flags); 6666 if (ufshcd_err_handling_should_stop(hba)) { 6667 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6668 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6669 spin_unlock_irqrestore(hba->host->host_lock, flags); 6670 up(&hba->host_sem); 6671 return; 6672 } 6673 spin_unlock_irqrestore(hba->host->host_lock, flags); 6674 6675 ufshcd_err_handling_prepare(hba); 6676 6677 spin_lock_irqsave(hba->host->host_lock, flags); 6678 ufshcd_set_eh_in_progress(hba); 6679 spin_unlock_irqrestore(hba->host->host_lock, flags); 6680 6681 /* Complete requests that have door-bell cleared by h/w */ 6682 ufshcd_complete_requests(hba, false); 6683 spin_lock_irqsave(hba->host->host_lock, flags); 6684 again: 6685 needs_restore = false; 6686 needs_reset = false; 6687 6688 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6689 hba->ufshcd_state = UFSHCD_STATE_RESET; 6690 /* 6691 * A full reset and restore might have happened after preparation 6692 * is finished, double check whether we should stop. 6693 */ 6694 if (ufshcd_err_handling_should_stop(hba)) 6695 goto skip_err_handling; 6696 6697 if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && 6698 !hba->force_reset) { 6699 bool ret; 6700 6701 spin_unlock_irqrestore(hba->host->host_lock, flags); 6702 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ 6703 ret = ufshcd_quirk_dl_nac_errors(hba); 6704 spin_lock_irqsave(hba->host->host_lock, flags); 6705 if (!ret && ufshcd_err_handling_should_stop(hba)) 6706 goto skip_err_handling; 6707 } 6708 6709 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6710 (hba->saved_uic_err && 6711 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6712 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR); 6713 6714 spin_unlock_irqrestore(hba->host->host_lock, flags); 6715 ufshcd_print_host_state(hba); 6716 ufshcd_print_pwr_info(hba); 6717 ufshcd_print_evt_hist(hba); 6718 ufshcd_print_tmrs(hba, hba->outstanding_tasks); 6719 ufshcd_print_trs_all(hba, pr_prdt); 6720 spin_lock_irqsave(hba->host->host_lock, flags); 6721 } 6722 6723 /* 6724 * if host reset is required then skip clearing the pending 6725 * transfers forcefully because they will get cleared during 6726 * host reset and restore 6727 */ 6728 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6729 ufshcd_is_saved_err_fatal(hba) || 6730 ((hba->saved_err & UIC_ERROR) && 6731 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | 6732 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { 6733 needs_reset = true; 6734 goto do_reset; 6735 } 6736 6737 /* 6738 * If LINERESET was caught, UFS might have been put to PWM mode, 6739 * check if power mode restore is needed. 6740 */ 6741 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) { 6742 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6743 if (!hba->saved_uic_err) 6744 hba->saved_err &= ~UIC_ERROR; 6745 spin_unlock_irqrestore(hba->host->host_lock, flags); 6746 if (ufshcd_is_pwr_mode_restore_needed(hba)) 6747 needs_restore = true; 6748 spin_lock_irqsave(hba->host->host_lock, flags); 6749 if (!hba->saved_err && !needs_restore) 6750 goto skip_err_handling; 6751 } 6752 6753 hba->silence_err_logs = true; 6754 /* release lock as clear command might sleep */ 6755 spin_unlock_irqrestore(hba->host->host_lock, flags); 6756 6757 needs_reset = ufshcd_abort_all(hba); 6758 6759 spin_lock_irqsave(hba->host->host_lock, flags); 6760 hba->silence_err_logs = false; 6761 if (needs_reset) 6762 goto do_reset; 6763 6764 /* 6765 * After all reqs and tasks are cleared from doorbell, 6766 * now it is safe to retore power mode. 6767 */ 6768 if (needs_restore) { 6769 spin_unlock_irqrestore(hba->host->host_lock, flags); 6770 /* 6771 * Hold the scaling lock just in case dev cmds 6772 * are sent via bsg and/or sysfs. 6773 */ 6774 down_write(&hba->clk_scaling_lock); 6775 hba->force_pmc = true; 6776 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info)); 6777 if (pmc_err) { 6778 needs_reset = true; 6779 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n", 6780 __func__, pmc_err); 6781 } 6782 hba->force_pmc = false; 6783 ufshcd_print_pwr_info(hba); 6784 up_write(&hba->clk_scaling_lock); 6785 spin_lock_irqsave(hba->host->host_lock, flags); 6786 } 6787 6788 do_reset: 6789 /* Fatal errors need reset */ 6790 if (needs_reset) { 6791 int err; 6792 6793 hba->force_reset = false; 6794 spin_unlock_irqrestore(hba->host->host_lock, flags); 6795 err = ufshcd_reset_and_restore(hba); 6796 if (err) 6797 dev_err(hba->dev, "%s: reset and restore failed with err %d\n", 6798 __func__, err); 6799 else 6800 ufshcd_recover_pm_error(hba); 6801 spin_lock_irqsave(hba->host->host_lock, flags); 6802 } 6803 6804 skip_err_handling: 6805 if (!needs_reset) { 6806 if (hba->ufshcd_state == UFSHCD_STATE_RESET) 6807 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6808 if (hba->saved_err || hba->saved_uic_err) 6809 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", 6810 __func__, hba->saved_err, hba->saved_uic_err); 6811 } 6812 /* Exit in an operational state or dead */ 6813 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 6814 hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6815 if (--retries) 6816 goto again; 6817 hba->ufshcd_state = UFSHCD_STATE_ERROR; 6818 } 6819 ufshcd_clear_eh_in_progress(hba); 6820 spin_unlock_irqrestore(hba->host->host_lock, flags); 6821 ufshcd_err_handling_unprepare(hba); 6822 up(&hba->host_sem); 6823 6824 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__, 6825 ufshcd_state_name[hba->ufshcd_state]); 6826 } 6827 6828 /** 6829 * ufshcd_update_uic_error - check and set fatal UIC error flags. 6830 * @hba: per-adapter instance 6831 * 6832 * Return: 6833 * IRQ_HANDLED - If interrupt is valid 6834 * IRQ_NONE - If invalid interrupt 6835 */ 6836 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) 6837 { 6838 u32 reg; 6839 irqreturn_t retval = IRQ_NONE; 6840 6841 /* PHY layer error */ 6842 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 6843 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && 6844 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { 6845 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); 6846 /* 6847 * To know whether this error is fatal or not, DB timeout 6848 * must be checked but this error is handled separately. 6849 */ 6850 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK) 6851 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", 6852 __func__); 6853 6854 /* Got a LINERESET indication. */ 6855 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) { 6856 struct uic_command *cmd = NULL; 6857 6858 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR; 6859 if (hba->uic_async_done && hba->active_uic_cmd) 6860 cmd = hba->active_uic_cmd; 6861 /* 6862 * Ignore the LINERESET during power mode change 6863 * operation via DME_SET command. 6864 */ 6865 if (cmd && (cmd->command == UIC_CMD_DME_SET)) 6866 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6867 } 6868 retval |= IRQ_HANDLED; 6869 } 6870 6871 /* PA_INIT_ERROR is fatal and needs UIC reset */ 6872 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 6873 if ((reg & UIC_DATA_LINK_LAYER_ERROR) && 6874 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { 6875 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); 6876 6877 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 6878 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 6879 else if (hba->dev_quirks & 6880 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 6881 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED) 6882 hba->uic_error |= 6883 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6884 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT) 6885 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR; 6886 } 6887 retval |= IRQ_HANDLED; 6888 } 6889 6890 /* UIC NL/TL/DME errors needs software retry */ 6891 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 6892 if ((reg & UIC_NETWORK_LAYER_ERROR) && 6893 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { 6894 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); 6895 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 6896 retval |= IRQ_HANDLED; 6897 } 6898 6899 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 6900 if ((reg & UIC_TRANSPORT_LAYER_ERROR) && 6901 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { 6902 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); 6903 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 6904 retval |= IRQ_HANDLED; 6905 } 6906 6907 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 6908 if ((reg & UIC_DME_ERROR) && 6909 (reg & UIC_DME_ERROR_CODE_MASK)) { 6910 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); 6911 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 6912 retval |= IRQ_HANDLED; 6913 } 6914 6915 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 6916 __func__, hba->uic_error); 6917 return retval; 6918 } 6919 6920 /** 6921 * ufshcd_check_errors - Check for errors that need s/w attention 6922 * @hba: per-adapter instance 6923 * @intr_status: interrupt status generated by the controller 6924 * 6925 * Return: 6926 * IRQ_HANDLED - If interrupt is valid 6927 * IRQ_NONE - If invalid interrupt 6928 */ 6929 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) 6930 { 6931 bool queue_eh_work = false; 6932 irqreturn_t retval = IRQ_NONE; 6933 6934 guard(spinlock_irqsave)(hba->host->host_lock); 6935 hba->errors |= UFSHCD_ERROR_MASK & intr_status; 6936 6937 if (hba->errors & INT_FATAL_ERRORS) { 6938 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, 6939 hba->errors); 6940 queue_eh_work = true; 6941 } 6942 6943 if (hba->errors & UIC_ERROR) { 6944 hba->uic_error = 0; 6945 retval = ufshcd_update_uic_error(hba); 6946 if (hba->uic_error) 6947 queue_eh_work = true; 6948 } 6949 6950 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) { 6951 dev_err(hba->dev, 6952 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n", 6953 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? 6954 "Enter" : "Exit", 6955 hba->errors, ufshcd_get_upmcrs(hba)); 6956 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, 6957 hba->errors); 6958 ufshcd_set_link_broken(hba); 6959 queue_eh_work = true; 6960 } 6961 6962 if (queue_eh_work) { 6963 /* 6964 * update the transfer error masks to sticky bits, let's do this 6965 * irrespective of current ufshcd_state. 6966 */ 6967 hba->saved_err |= hba->errors; 6968 hba->saved_uic_err |= hba->uic_error; 6969 6970 /* dump controller state before resetting */ 6971 if ((hba->saved_err & 6972 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6973 (hba->saved_uic_err && 6974 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6975 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", 6976 __func__, hba->saved_err, 6977 hba->saved_uic_err); 6978 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, 6979 "host_regs: "); 6980 ufshcd_print_pwr_info(hba); 6981 } 6982 ufshcd_schedule_eh_work(hba); 6983 retval |= IRQ_HANDLED; 6984 } 6985 /* 6986 * if (!queue_eh_work) - 6987 * Other errors are either non-fatal where host recovers 6988 * itself without s/w intervention or errors that will be 6989 * handled by the SCSI core layer. 6990 */ 6991 hba->errors = 0; 6992 hba->uic_error = 0; 6993 6994 return retval; 6995 } 6996 6997 /** 6998 * ufshcd_tmc_handler - handle task management function completion 6999 * @hba: per adapter instance 7000 * 7001 * Return: 7002 * IRQ_HANDLED - If interrupt is valid 7003 * IRQ_NONE - If invalid interrupt 7004 */ 7005 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) 7006 { 7007 unsigned long flags, pending, issued; 7008 irqreturn_t ret = IRQ_NONE; 7009 int tag; 7010 7011 spin_lock_irqsave(hba->host->host_lock, flags); 7012 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 7013 issued = hba->outstanding_tasks & ~pending; 7014 for_each_set_bit(tag, &issued, hba->nutmrs) { 7015 struct request *req = hba->tmf_rqs[tag]; 7016 struct completion *c = req->end_io_data; 7017 7018 complete(c); 7019 ret = IRQ_HANDLED; 7020 } 7021 spin_unlock_irqrestore(hba->host->host_lock, flags); 7022 7023 return ret; 7024 } 7025 7026 /** 7027 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events 7028 * @hba: per adapter instance 7029 * 7030 * Return: IRQ_HANDLED if interrupt is handled. 7031 */ 7032 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) 7033 { 7034 struct ufs_hw_queue *hwq; 7035 unsigned long outstanding_cqs; 7036 unsigned int nr_queues; 7037 int i, ret; 7038 u32 events; 7039 7040 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); 7041 if (ret) 7042 outstanding_cqs = (1U << hba->nr_hw_queues) - 1; 7043 7044 /* Exclude the poll queues */ 7045 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; 7046 for_each_set_bit(i, &outstanding_cqs, nr_queues) { 7047 hwq = &hba->uhq[i]; 7048 7049 events = ufshcd_mcq_read_cqis(hba, i); 7050 if (events) 7051 ufshcd_mcq_write_cqis(hba, events, i); 7052 7053 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) 7054 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7055 } 7056 7057 return IRQ_HANDLED; 7058 } 7059 7060 /** 7061 * ufshcd_sl_intr - Interrupt service routine 7062 * @hba: per adapter instance 7063 * @intr_status: contains interrupts generated by the controller 7064 * 7065 * Return: 7066 * IRQ_HANDLED - If interrupt is valid 7067 * IRQ_NONE - If invalid interrupt 7068 */ 7069 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 7070 { 7071 irqreturn_t retval = IRQ_NONE; 7072 7073 if (intr_status & UFSHCD_UIC_MASK) 7074 retval |= ufshcd_uic_cmd_compl(hba, intr_status); 7075 7076 if (intr_status & UFSHCD_ERROR_MASK || hba->errors) 7077 retval |= ufshcd_check_errors(hba, intr_status); 7078 7079 if (intr_status & UTP_TASK_REQ_COMPL) 7080 retval |= ufshcd_tmc_handler(hba); 7081 7082 if (intr_status & UTP_TRANSFER_REQ_COMPL) 7083 retval |= ufshcd_transfer_req_compl(hba); 7084 7085 if (intr_status & MCQ_CQ_EVENT_STATUS) 7086 retval |= ufshcd_handle_mcq_cq_events(hba); 7087 7088 return retval; 7089 } 7090 7091 /** 7092 * ufshcd_threaded_intr - Threaded interrupt service routine 7093 * @irq: irq number 7094 * @__hba: pointer to adapter instance 7095 * 7096 * Return: 7097 * IRQ_HANDLED - If interrupt is valid 7098 * IRQ_NONE - If invalid interrupt 7099 */ 7100 static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) 7101 { 7102 u32 last_intr_status, intr_status, enabled_intr_status = 0; 7103 irqreturn_t retval = IRQ_NONE; 7104 struct ufs_hba *hba = __hba; 7105 int retries = hba->nutrs; 7106 7107 last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7108 7109 /* 7110 * There could be max of hba->nutrs reqs in flight and in worst case 7111 * if the reqs get finished 1 by 1 after the interrupt status is 7112 * read, make sure we handle them by checking the interrupt status 7113 * again in a loop until we process all of the reqs before returning. 7114 */ 7115 while (intr_status && retries--) { 7116 enabled_intr_status = 7117 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7118 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7119 if (enabled_intr_status) 7120 retval |= ufshcd_sl_intr(hba, enabled_intr_status); 7121 7122 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7123 } 7124 7125 if (enabled_intr_status && retval == IRQ_NONE && 7126 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) || 7127 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) { 7128 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", 7129 __func__, 7130 intr_status, 7131 last_intr_status, 7132 enabled_intr_status); 7133 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 7134 } 7135 7136 return retval; 7137 } 7138 7139 /** 7140 * ufshcd_intr - Main interrupt service routine 7141 * @irq: irq number 7142 * @__hba: pointer to adapter instance 7143 * 7144 * Return: 7145 * IRQ_HANDLED - If interrupt is valid 7146 * IRQ_WAKE_THREAD - If handling is moved to threaded handled 7147 * IRQ_NONE - If invalid interrupt 7148 */ 7149 static irqreturn_t ufshcd_intr(int irq, void *__hba) 7150 { 7151 struct ufs_hba *hba = __hba; 7152 u32 intr_status, enabled_intr_status; 7153 7154 /* Move interrupt handling to thread when MCQ & ESI are not enabled */ 7155 if (!hba->mcq_enabled || !hba->mcq_esi_enabled) 7156 return IRQ_WAKE_THREAD; 7157 7158 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7159 enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7160 7161 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7162 7163 /* Directly handle interrupts since MCQ ESI handlers does the hard job */ 7164 return ufshcd_sl_intr(hba, enabled_intr_status); 7165 } 7166 7167 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 7168 { 7169 int err = 0; 7170 u32 mask = 1 << tag; 7171 7172 if (!test_bit(tag, &hba->outstanding_tasks)) 7173 goto out; 7174 7175 ufshcd_utmrl_clear(hba, tag); 7176 7177 /* poll for max. 1 sec to clear door bell register by h/w */ 7178 err = ufshcd_wait_for_register(hba, 7179 REG_UTP_TASK_REQ_DOOR_BELL, 7180 mask, 0, 1000, 1000); 7181 7182 dev_err(hba->dev, "Clearing task management function with tag %d %s\n", 7183 tag, err < 0 ? "failed" : "succeeded"); 7184 7185 out: 7186 return err; 7187 } 7188 7189 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, 7190 struct utp_task_req_desc *treq, u8 tm_function) 7191 { 7192 struct request_queue *q = hba->tmf_queue; 7193 struct Scsi_Host *host = hba->host; 7194 DECLARE_COMPLETION_ONSTACK(wait); 7195 struct request *req; 7196 unsigned long flags; 7197 int task_tag, err; 7198 7199 /* 7200 * blk_mq_alloc_request() is used here only to get a free tag. 7201 */ 7202 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0); 7203 if (IS_ERR(req)) 7204 return PTR_ERR(req); 7205 7206 req->end_io_data = &wait; 7207 ufshcd_hold(hba); 7208 7209 spin_lock_irqsave(host->host_lock, flags); 7210 7211 task_tag = req->tag; 7212 hba->tmf_rqs[req->tag] = req; 7213 treq->upiu_req.req_header.task_tag = task_tag; 7214 7215 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); 7216 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); 7217 7218 __set_bit(task_tag, &hba->outstanding_tasks); 7219 7220 spin_unlock_irqrestore(host->host_lock, flags); 7221 7222 /* send command to the controller */ 7223 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); 7224 7225 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); 7226 7227 /* wait until the task management command is completed */ 7228 err = wait_for_completion_io_timeout(&wait, 7229 msecs_to_jiffies(TM_CMD_TIMEOUT)); 7230 if (!err) { 7231 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR); 7232 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 7233 __func__, tm_function); 7234 if (ufshcd_clear_tm_cmd(hba, task_tag)) 7235 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n", 7236 __func__, task_tag); 7237 err = -ETIMEDOUT; 7238 } else { 7239 err = 0; 7240 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq)); 7241 7242 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP); 7243 } 7244 7245 spin_lock_irqsave(hba->host->host_lock, flags); 7246 hba->tmf_rqs[req->tag] = NULL; 7247 __clear_bit(task_tag, &hba->outstanding_tasks); 7248 spin_unlock_irqrestore(hba->host->host_lock, flags); 7249 7250 ufshcd_release(hba); 7251 blk_mq_free_request(req); 7252 7253 return err; 7254 } 7255 7256 /** 7257 * ufshcd_issue_tm_cmd - issues task management commands to controller 7258 * @hba: per adapter instance 7259 * @lun_id: LUN ID to which TM command is sent 7260 * @task_id: task ID to which the TM command is applicable 7261 * @tm_function: task management function opcode 7262 * @tm_response: task management service response return value 7263 * 7264 * Return: non-zero value on error, zero on success. 7265 */ 7266 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 7267 u8 tm_function, u8 *tm_response) 7268 { 7269 struct utp_task_req_desc treq = { }; 7270 enum utp_ocs ocs_value; 7271 int err; 7272 7273 /* Configure task request descriptor */ 7274 treq.header.interrupt = 1; 7275 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7276 7277 /* Configure task request UPIU */ 7278 treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; 7279 treq.upiu_req.req_header.lun = lun_id; 7280 treq.upiu_req.req_header.tm_function = tm_function; 7281 7282 /* 7283 * The host shall provide the same value for LUN field in the basic 7284 * header and for Input Parameter. 7285 */ 7286 treq.upiu_req.input_param1 = cpu_to_be32(lun_id); 7287 treq.upiu_req.input_param2 = cpu_to_be32(task_id); 7288 7289 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function); 7290 if (err == -ETIMEDOUT) 7291 return err; 7292 7293 ocs_value = treq.header.ocs & MASK_OCS; 7294 if (ocs_value != OCS_SUCCESS) 7295 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 7296 __func__, ocs_value); 7297 else if (tm_response) 7298 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) & 7299 MASK_TM_SERVICE_RESP; 7300 return err; 7301 } 7302 7303 /** 7304 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests 7305 * @hba: per-adapter instance 7306 * @req_upiu: upiu request 7307 * @rsp_upiu: upiu reply 7308 * @desc_buff: pointer to descriptor buffer, NULL if NA 7309 * @buff_len: descriptor size, 0 if NA 7310 * @cmd_type: specifies the type (NOP, Query...) 7311 * @desc_op: descriptor operation 7312 * 7313 * Those type of requests uses UTP Transfer Request Descriptor - utrd. 7314 * Therefore, it "rides" the device management infrastructure: uses its tag and 7315 * tasks work queues. 7316 * 7317 * Since there is only one available tag for device management commands, 7318 * the caller is expected to hold the hba->dev_cmd.lock mutex. 7319 * 7320 * Return: 0 upon success; < 0 upon failure. 7321 */ 7322 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, 7323 struct utp_upiu_req *req_upiu, 7324 struct utp_upiu_req *rsp_upiu, 7325 u8 *desc_buff, int *buff_len, 7326 enum dev_cmd_type cmd_type, 7327 enum query_opcode desc_op) 7328 { 7329 const u32 tag = hba->reserved_slot; 7330 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7331 int err = 0; 7332 u8 upiu_flags; 7333 7334 /* Protects use of hba->reserved_slot. */ 7335 lockdep_assert_held(&hba->dev_cmd.lock); 7336 7337 ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag); 7338 7339 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7340 7341 /* update the task tag in the request upiu */ 7342 req_upiu->header.task_tag = tag; 7343 7344 /* just copy the upiu request as it is */ 7345 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7346 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) { 7347 /* The Data Segment Area is optional depending upon the query 7348 * function value. for WRITE DESCRIPTOR, the data segment 7349 * follows right after the tsf. 7350 */ 7351 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len); 7352 *buff_len = 0; 7353 } 7354 7355 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7356 7357 /* 7358 * ignore the returning value here - ufshcd_check_query_response is 7359 * bound to fail since dev_cmd.query and dev_cmd.type were left empty. 7360 * read the response directly ignoring all errors. 7361 */ 7362 ufshcd_issue_dev_cmd(hba, lrbp, tag, dev_cmd_timeout); 7363 7364 /* just copy the upiu response as it is */ 7365 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7366 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { 7367 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); 7368 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 7369 .data_segment_length); 7370 7371 if (*buff_len >= resp_len) { 7372 memcpy(desc_buff, descp, resp_len); 7373 *buff_len = resp_len; 7374 } else { 7375 dev_warn(hba->dev, 7376 "%s: rsp size %d is bigger than buffer size %d", 7377 __func__, resp_len, *buff_len); 7378 *buff_len = 0; 7379 err = -EINVAL; 7380 } 7381 } 7382 7383 return err; 7384 } 7385 7386 /** 7387 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands 7388 * @hba: per-adapter instance 7389 * @req_upiu: upiu request 7390 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands 7391 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target 7392 * @desc_buff: pointer to descriptor buffer, NULL if NA 7393 * @buff_len: descriptor size, 0 if NA 7394 * @desc_op: descriptor operation 7395 * 7396 * Supports UTP Transfer requests (nop and query), and UTP Task 7397 * Management requests. 7398 * It is up to the caller to fill the upiu conent properly, as it will 7399 * be copied without any further input validations. 7400 * 7401 * Return: 0 upon success; < 0 upon failure. 7402 */ 7403 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 7404 struct utp_upiu_req *req_upiu, 7405 struct utp_upiu_req *rsp_upiu, 7406 enum upiu_request_transaction msgcode, 7407 u8 *desc_buff, int *buff_len, 7408 enum query_opcode desc_op) 7409 { 7410 int err; 7411 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; 7412 struct utp_task_req_desc treq = { }; 7413 enum utp_ocs ocs_value; 7414 u8 tm_f = req_upiu->header.tm_function; 7415 7416 switch (msgcode) { 7417 case UPIU_TRANSACTION_NOP_OUT: 7418 cmd_type = DEV_CMD_TYPE_NOP; 7419 fallthrough; 7420 case UPIU_TRANSACTION_QUERY_REQ: 7421 ufshcd_dev_man_lock(hba); 7422 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, 7423 desc_buff, buff_len, 7424 cmd_type, desc_op); 7425 ufshcd_dev_man_unlock(hba); 7426 7427 break; 7428 case UPIU_TRANSACTION_TASK_REQ: 7429 treq.header.interrupt = 1; 7430 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7431 7432 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); 7433 7434 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f); 7435 if (err == -ETIMEDOUT) 7436 break; 7437 7438 ocs_value = treq.header.ocs & MASK_OCS; 7439 if (ocs_value != OCS_SUCCESS) { 7440 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, 7441 ocs_value); 7442 break; 7443 } 7444 7445 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu)); 7446 7447 break; 7448 default: 7449 err = -EINVAL; 7450 7451 break; 7452 } 7453 7454 return err; 7455 } 7456 7457 /** 7458 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request 7459 * @hba: per adapter instance 7460 * @req_upiu: upiu request 7461 * @rsp_upiu: upiu reply 7462 * @req_ehs: EHS field which contains Advanced RPMB Request Message 7463 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message 7464 * @sg_cnt: The number of sg lists actually used 7465 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation 7466 * @dir: DMA direction 7467 * 7468 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 7469 * < 0 if another error occurred. 7470 */ 7471 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, 7472 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, 7473 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7474 enum dma_data_direction dir) 7475 { 7476 const u32 tag = hba->reserved_slot; 7477 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7478 int err = 0; 7479 int result; 7480 u8 upiu_flags; 7481 u8 *ehs_data; 7482 u16 ehs_len; 7483 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; 7484 7485 /* Protects use of hba->reserved_slot. */ 7486 ufshcd_dev_man_lock(hba); 7487 7488 ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag); 7489 7490 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7491 7492 /* update the task tag */ 7493 req_upiu->header.task_tag = tag; 7494 7495 /* copy the UPIU(contains CDB) request as it is */ 7496 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7497 /* Copy EHS, starting with byte32, immediately after the CDB package */ 7498 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); 7499 7500 if (dir != DMA_NONE && sg_list) 7501 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); 7502 7503 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7504 7505 err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7506 7507 if (!err) { 7508 /* Just copy the upiu response as it is */ 7509 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7510 /* Get the response UPIU result */ 7511 result = (lrbp->ucd_rsp_ptr->header.response << 8) | 7512 lrbp->ucd_rsp_ptr->header.status; 7513 7514 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; 7515 /* 7516 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data 7517 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB 7518 * Message is 02h 7519 */ 7520 if (ehs_len == 2 && rsp_ehs) { 7521 /* 7522 * ucd_rsp_ptr points to a buffer with a length of 512 bytes 7523 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 7524 */ 7525 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; 7526 memcpy(rsp_ehs, ehs_data, ehs_len * 32); 7527 } 7528 } 7529 7530 ufshcd_dev_man_unlock(hba); 7531 7532 return err ? : result; 7533 } 7534 7535 /** 7536 * ufshcd_eh_device_reset_handler() - Reset a single logical unit. 7537 * @cmd: SCSI command pointer 7538 * 7539 * Return: SUCCESS or FAILED. 7540 */ 7541 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7542 { 7543 unsigned long flags, pending_reqs = 0, not_cleared = 0; 7544 struct Scsi_Host *host; 7545 struct ufs_hba *hba; 7546 struct ufs_hw_queue *hwq; 7547 struct ufshcd_lrb *lrbp; 7548 u32 pos, not_cleared_mask = 0; 7549 int err; 7550 u8 resp = 0xF, lun; 7551 7552 host = cmd->device->host; 7553 hba = shost_priv(host); 7554 7555 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7556 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7557 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7558 if (!err) 7559 err = resp; 7560 goto out; 7561 } 7562 7563 if (hba->mcq_enabled) { 7564 for (pos = 0; pos < hba->nutrs; pos++) { 7565 lrbp = &hba->lrb[pos]; 7566 if (ufshcd_cmd_inflight(lrbp->cmd) && 7567 lrbp->lun == lun) { 7568 ufshcd_clear_cmd(hba, pos); 7569 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd)); 7570 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7571 } 7572 } 7573 err = 0; 7574 goto out; 7575 } 7576 7577 /* clear the commands that were pending for corresponding LUN */ 7578 spin_lock_irqsave(&hba->outstanding_lock, flags); 7579 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) 7580 if (hba->lrb[pos].lun == lun) 7581 __set_bit(pos, &pending_reqs); 7582 hba->outstanding_reqs &= ~pending_reqs; 7583 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7584 7585 for_each_set_bit(pos, &pending_reqs, hba->nutrs) { 7586 if (ufshcd_clear_cmd(hba, pos) < 0) { 7587 spin_lock_irqsave(&hba->outstanding_lock, flags); 7588 not_cleared = 1U << pos & 7589 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7590 hba->outstanding_reqs |= not_cleared; 7591 not_cleared_mask |= not_cleared; 7592 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7593 7594 dev_err(hba->dev, "%s: failed to clear request %d\n", 7595 __func__, pos); 7596 } 7597 } 7598 __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask); 7599 7600 out: 7601 hba->req_abort_count = 0; 7602 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7603 if (!err) { 7604 err = SUCCESS; 7605 } else { 7606 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7607 err = FAILED; 7608 } 7609 return err; 7610 } 7611 7612 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7613 { 7614 struct ufshcd_lrb *lrbp; 7615 int tag; 7616 7617 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7618 lrbp = &hba->lrb[tag]; 7619 lrbp->req_abort_skip = true; 7620 } 7621 } 7622 7623 /** 7624 * ufshcd_try_to_abort_task - abort a specific task 7625 * @hba: Pointer to adapter instance 7626 * @tag: Task tag/index to be aborted 7627 * 7628 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7629 * command, and in host controller by clearing the door-bell register. There can 7630 * be race between controller sending the command to the device while abort is 7631 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 7632 * really issued and then try to abort it. 7633 * 7634 * Return: zero on success, non-zero on failure. 7635 */ 7636 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7637 { 7638 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7639 int err; 7640 int poll_cnt; 7641 u8 resp = 0xF; 7642 7643 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7644 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7645 UFS_QUERY_TASK, &resp); 7646 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7647 /* cmd pending in the device */ 7648 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", 7649 __func__, tag); 7650 break; 7651 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7652 /* 7653 * cmd not pending in the device, check if it is 7654 * in transition. 7655 */ 7656 dev_info( 7657 hba->dev, 7658 "%s: cmd with tag %d not pending in the device.\n", 7659 __func__, tag); 7660 if (!ufshcd_cmd_inflight(lrbp->cmd)) { 7661 dev_info(hba->dev, 7662 "%s: cmd with tag=%d completed.\n", 7663 __func__, tag); 7664 return 0; 7665 } 7666 usleep_range(100, 200); 7667 } else { 7668 dev_err(hba->dev, 7669 "%s: no response from device. tag = %d, err %d\n", 7670 __func__, tag, err); 7671 return err ? : resp; 7672 } 7673 } 7674 7675 if (!poll_cnt) 7676 return -EBUSY; 7677 7678 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7679 UFS_ABORT_TASK, &resp); 7680 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7681 if (!err) { 7682 err = resp; /* service response error */ 7683 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", 7684 __func__, tag, err); 7685 } 7686 return err; 7687 } 7688 7689 err = ufshcd_clear_cmd(hba, tag); 7690 if (err) 7691 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", 7692 __func__, tag, err); 7693 7694 return err; 7695 } 7696 7697 /** 7698 * ufshcd_abort - scsi host template eh_abort_handler callback 7699 * @cmd: SCSI command pointer 7700 * 7701 * Return: SUCCESS or FAILED. 7702 */ 7703 static int ufshcd_abort(struct scsi_cmnd *cmd) 7704 { 7705 struct Scsi_Host *host = cmd->device->host; 7706 struct ufs_hba *hba = shost_priv(host); 7707 int tag = scsi_cmd_to_rq(cmd)->tag; 7708 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7709 unsigned long flags; 7710 int err = FAILED; 7711 bool outstanding; 7712 u32 reg; 7713 7714 ufshcd_hold(hba); 7715 7716 if (!hba->mcq_enabled) { 7717 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7718 if (!test_bit(tag, &hba->outstanding_reqs)) { 7719 /* If command is already aborted/completed, return FAILED. */ 7720 dev_err(hba->dev, 7721 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", 7722 __func__, tag, hba->outstanding_reqs, reg); 7723 goto release; 7724 } 7725 } 7726 7727 /* Print Transfer Request of aborted task */ 7728 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); 7729 7730 /* 7731 * Print detailed info about aborted request. 7732 * As more than one request might get aborted at the same time, 7733 * print full information only for the first aborted request in order 7734 * to reduce repeated printouts. For other aborted requests only print 7735 * basic details. 7736 */ 7737 scsi_print_command(cmd); 7738 if (!hba->req_abort_count) { 7739 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7740 ufshcd_print_evt_hist(hba); 7741 ufshcd_print_host_state(hba); 7742 ufshcd_print_pwr_info(hba); 7743 ufshcd_print_tr(hba, tag, true); 7744 } else { 7745 ufshcd_print_tr(hba, tag, false); 7746 } 7747 hba->req_abort_count++; 7748 7749 if (!hba->mcq_enabled && !(reg & (1 << tag))) { 7750 /* only execute this code in single doorbell mode */ 7751 dev_err(hba->dev, 7752 "%s: cmd was completed, but without a notifying intr, tag = %d", 7753 __func__, tag); 7754 __ufshcd_transfer_req_compl(hba, 1UL << tag); 7755 goto release; 7756 } 7757 7758 /* 7759 * Task abort to the device W-LUN is illegal. When this command 7760 * will fail, due to spec violation, scsi err handling next step 7761 * will be to send LU reset which, again, is a spec violation. 7762 * To avoid these unnecessary/illegal steps, first we clean up 7763 * the lrb taken by this cmd and re-set it in outstanding_reqs, 7764 * then queue the eh_work and bail. 7765 */ 7766 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { 7767 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); 7768 7769 spin_lock_irqsave(host->host_lock, flags); 7770 hba->force_reset = true; 7771 ufshcd_schedule_eh_work(hba); 7772 spin_unlock_irqrestore(host->host_lock, flags); 7773 goto release; 7774 } 7775 7776 if (hba->mcq_enabled) { 7777 /* MCQ mode. Branch off to handle abort for mcq mode */ 7778 err = ufshcd_mcq_abort(cmd); 7779 goto release; 7780 } 7781 7782 /* Skip task abort in case previous aborts failed and report failure */ 7783 if (lrbp->req_abort_skip) { 7784 dev_err(hba->dev, "%s: skipping abort\n", __func__); 7785 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7786 goto release; 7787 } 7788 7789 err = ufshcd_try_to_abort_task(hba, tag); 7790 if (err) { 7791 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7792 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7793 err = FAILED; 7794 goto release; 7795 } 7796 7797 /* 7798 * Clear the corresponding bit from outstanding_reqs since the command 7799 * has been aborted successfully. 7800 */ 7801 spin_lock_irqsave(&hba->outstanding_lock, flags); 7802 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs); 7803 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7804 7805 if (outstanding) 7806 ufshcd_release_scsi_cmd(hba, lrbp); 7807 7808 err = SUCCESS; 7809 7810 release: 7811 /* Matches the ufshcd_hold() call at the start of this function. */ 7812 ufshcd_release(hba); 7813 return err; 7814 } 7815 7816 /** 7817 * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. 7818 * @hba: UFS host controller instance. 7819 * @probe_start: time when the ufshcd_probe_hba() call started. 7820 * @ret: ufshcd_probe_hba() return value. 7821 */ 7822 static void ufshcd_process_probe_result(struct ufs_hba *hba, 7823 ktime_t probe_start, int ret) 7824 { 7825 unsigned long flags; 7826 7827 spin_lock_irqsave(hba->host->host_lock, flags); 7828 if (ret) 7829 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7830 else if (hba->ufshcd_state == UFSHCD_STATE_RESET) 7831 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 7832 spin_unlock_irqrestore(hba->host->host_lock, flags); 7833 7834 trace_ufshcd_init(hba, ret, 7835 ktime_to_us(ktime_sub(ktime_get(), probe_start)), 7836 hba->curr_dev_pwr_mode, hba->uic_link_state); 7837 } 7838 7839 /** 7840 * ufshcd_host_reset_and_restore - reset and restore host controller 7841 * @hba: per-adapter instance 7842 * 7843 * Note that host controller reset may issue DME_RESET to 7844 * local and remote (device) Uni-Pro stack and the attributes 7845 * are reset to default state. 7846 * 7847 * Return: zero on success, non-zero on failure. 7848 */ 7849 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 7850 { 7851 int err; 7852 7853 /* 7854 * Stop the host controller and complete the requests 7855 * cleared by h/w 7856 */ 7857 ufshcd_hba_stop(hba); 7858 hba->silence_err_logs = true; 7859 ufshcd_complete_requests(hba, true); 7860 hba->silence_err_logs = false; 7861 7862 /* scale up clocks to max frequency before full reinitialization */ 7863 if (ufshcd_is_clkscaling_supported(hba)) 7864 ufshcd_scale_clks(hba, ULONG_MAX, true); 7865 7866 err = ufshcd_hba_enable(hba); 7867 7868 /* Establish the link again and restore the device */ 7869 if (!err) { 7870 ktime_t probe_start = ktime_get(); 7871 7872 err = ufshcd_device_init(hba, /*init_dev_params=*/false); 7873 if (!err) 7874 err = ufshcd_probe_hba(hba, false); 7875 ufshcd_process_probe_result(hba, probe_start, err); 7876 } 7877 7878 if (err) 7879 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 7880 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); 7881 return err; 7882 } 7883 7884 /** 7885 * ufshcd_reset_and_restore - reset and re-initialize host/device 7886 * @hba: per-adapter instance 7887 * 7888 * Reset and recover device, host and re-establish link. This 7889 * is helpful to recover the communication in fatal error conditions. 7890 * 7891 * Return: zero on success, non-zero on failure. 7892 */ 7893 static int ufshcd_reset_and_restore(struct ufs_hba *hba) 7894 { 7895 u32 saved_err = 0; 7896 u32 saved_uic_err = 0; 7897 int err = 0; 7898 unsigned long flags; 7899 int retries = MAX_HOST_RESET_RETRIES; 7900 7901 spin_lock_irqsave(hba->host->host_lock, flags); 7902 do { 7903 /* 7904 * This is a fresh start, cache and clear saved error first, 7905 * in case new error generated during reset and restore. 7906 */ 7907 saved_err |= hba->saved_err; 7908 saved_uic_err |= hba->saved_uic_err; 7909 hba->saved_err = 0; 7910 hba->saved_uic_err = 0; 7911 hba->force_reset = false; 7912 hba->ufshcd_state = UFSHCD_STATE_RESET; 7913 spin_unlock_irqrestore(hba->host->host_lock, flags); 7914 7915 /* Reset the attached device */ 7916 ufshcd_device_reset(hba); 7917 7918 err = ufshcd_host_reset_and_restore(hba); 7919 7920 spin_lock_irqsave(hba->host->host_lock, flags); 7921 if (err) 7922 continue; 7923 /* Do not exit unless operational or dead */ 7924 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 7925 hba->ufshcd_state != UFSHCD_STATE_ERROR && 7926 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL) 7927 err = -EAGAIN; 7928 } while (err && --retries); 7929 7930 /* 7931 * Inform scsi mid-layer that we did reset and allow to handle 7932 * Unit Attention properly. 7933 */ 7934 scsi_report_bus_reset(hba->host, 0); 7935 if (err) { 7936 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7937 hba->saved_err |= saved_err; 7938 hba->saved_uic_err |= saved_uic_err; 7939 } 7940 spin_unlock_irqrestore(hba->host->host_lock, flags); 7941 7942 return err; 7943 } 7944 7945 /** 7946 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 7947 * @cmd: SCSI command pointer 7948 * 7949 * Return: SUCCESS or FAILED. 7950 */ 7951 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 7952 { 7953 int err = SUCCESS; 7954 unsigned long flags; 7955 struct ufs_hba *hba; 7956 7957 hba = shost_priv(cmd->device->host); 7958 7959 /* 7960 * If runtime PM sent SSU and got a timeout, scsi_error_handler is 7961 * stuck in this function waiting for flush_work(&hba->eh_work). And 7962 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do 7963 * ufshcd_link_recovery instead of eh_work to prevent deadlock. 7964 */ 7965 if (hba->pm_op_in_progress) { 7966 if (ufshcd_link_recovery(hba)) 7967 err = FAILED; 7968 7969 return err; 7970 } 7971 7972 spin_lock_irqsave(hba->host->host_lock, flags); 7973 hba->force_reset = true; 7974 ufshcd_schedule_eh_work(hba); 7975 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__); 7976 spin_unlock_irqrestore(hba->host->host_lock, flags); 7977 7978 flush_work(&hba->eh_work); 7979 7980 spin_lock_irqsave(hba->host->host_lock, flags); 7981 if (hba->ufshcd_state == UFSHCD_STATE_ERROR) 7982 err = FAILED; 7983 spin_unlock_irqrestore(hba->host->host_lock, flags); 7984 7985 return err; 7986 } 7987 7988 /** 7989 * ufshcd_get_max_icc_level - calculate the ICC level 7990 * @sup_curr_uA: max. current supported by the regulator 7991 * @start_scan: row at the desc table to start scan from 7992 * @buff: power descriptor buffer 7993 * 7994 * Return: calculated max ICC level for specific regulator. 7995 */ 7996 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, 7997 const char *buff) 7998 { 7999 int i; 8000 int curr_uA; 8001 u16 data; 8002 u16 unit; 8003 8004 for (i = start_scan; i >= 0; i--) { 8005 data = get_unaligned_be16(&buff[2 * i]); 8006 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 8007 ATTR_ICC_LVL_UNIT_OFFSET; 8008 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 8009 switch (unit) { 8010 case UFSHCD_NANO_AMP: 8011 curr_uA = curr_uA / 1000; 8012 break; 8013 case UFSHCD_MILI_AMP: 8014 curr_uA = curr_uA * 1000; 8015 break; 8016 case UFSHCD_AMP: 8017 curr_uA = curr_uA * 1000 * 1000; 8018 break; 8019 case UFSHCD_MICRO_AMP: 8020 default: 8021 break; 8022 } 8023 if (sup_curr_uA >= curr_uA) 8024 break; 8025 } 8026 if (i < 0) { 8027 i = 0; 8028 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 8029 } 8030 8031 return (u32)i; 8032 } 8033 8034 /** 8035 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level 8036 * In case regulators are not initialized we'll return 0 8037 * @hba: per-adapter instance 8038 * @desc_buf: power descriptor buffer to extract ICC levels from. 8039 * 8040 * Return: calculated ICC level. 8041 */ 8042 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 8043 const u8 *desc_buf) 8044 { 8045 u32 icc_level = 0; 8046 8047 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 8048 !hba->vreg_info.vccq2) { 8049 /* 8050 * Using dev_dbg to avoid messages during runtime PM to avoid 8051 * never-ending cycles of messages written back to storage by 8052 * user space causing runtime resume, causing more messages and 8053 * so on. 8054 */ 8055 dev_dbg(hba->dev, 8056 "%s: Regulator capability was not set, actvIccLevel=%d", 8057 __func__, icc_level); 8058 goto out; 8059 } 8060 8061 if (hba->vreg_info.vcc->max_uA) 8062 icc_level = ufshcd_get_max_icc_level( 8063 hba->vreg_info.vcc->max_uA, 8064 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 8065 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 8066 8067 if (hba->vreg_info.vccq->max_uA) 8068 icc_level = ufshcd_get_max_icc_level( 8069 hba->vreg_info.vccq->max_uA, 8070 icc_level, 8071 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 8072 8073 if (hba->vreg_info.vccq2->max_uA) 8074 icc_level = ufshcd_get_max_icc_level( 8075 hba->vreg_info.vccq2->max_uA, 8076 icc_level, 8077 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 8078 out: 8079 return icc_level; 8080 } 8081 8082 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) 8083 { 8084 int ret; 8085 u8 *desc_buf; 8086 u32 icc_level; 8087 8088 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8089 if (!desc_buf) 8090 return; 8091 8092 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, 8093 desc_buf, QUERY_DESC_MAX_SIZE); 8094 if (ret) { 8095 dev_err(hba->dev, 8096 "%s: Failed reading power descriptor ret = %d", 8097 __func__, ret); 8098 goto out; 8099 } 8100 8101 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); 8102 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); 8103 8104 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8105 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); 8106 8107 if (ret) 8108 dev_err(hba->dev, 8109 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 8110 __func__, icc_level, ret); 8111 8112 out: 8113 kfree(desc_buf); 8114 } 8115 8116 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) 8117 { 8118 struct Scsi_Host *shost = sdev->host; 8119 8120 scsi_autopm_get_device(sdev); 8121 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); 8122 if (sdev->rpm_autosuspend) 8123 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 8124 shost->rpm_autosuspend_delay); 8125 scsi_autopm_put_device(sdev); 8126 } 8127 8128 /** 8129 * ufshcd_scsi_add_wlus - Adds required W-LUs 8130 * @hba: per-adapter instance 8131 * 8132 * UFS device specification requires the UFS devices to support 4 well known 8133 * logical units: 8134 * "REPORT_LUNS" (address: 01h) 8135 * "UFS Device" (address: 50h) 8136 * "RPMB" (address: 44h) 8137 * "BOOT" (address: 30h) 8138 * UFS device's power management needs to be controlled by "POWER CONDITION" 8139 * field of SSU (START STOP UNIT) command. But this "power condition" field 8140 * will take effect only when its sent to "UFS device" well known logical unit 8141 * hence we require the scsi_device instance to represent this logical unit in 8142 * order for the UFS host driver to send the SSU command for power management. 8143 * 8144 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 8145 * Block) LU so user space process can control this LU. User space may also 8146 * want to have access to BOOT LU. 8147 * 8148 * This function adds scsi device instances for each of all well known LUs 8149 * (except "REPORT LUNS" LU). 8150 * 8151 * Return: zero on success (all required W-LUs are added successfully), 8152 * non-zero error value on failure (if failed to add any of the required W-LU). 8153 */ 8154 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 8155 { 8156 int ret = 0; 8157 struct scsi_device *sdev_boot, *sdev_rpmb; 8158 8159 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0, 8160 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 8161 if (IS_ERR(hba->ufs_device_wlun)) { 8162 ret = PTR_ERR(hba->ufs_device_wlun); 8163 hba->ufs_device_wlun = NULL; 8164 goto out; 8165 } 8166 scsi_device_put(hba->ufs_device_wlun); 8167 8168 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 8169 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 8170 if (IS_ERR(sdev_rpmb)) { 8171 ret = PTR_ERR(sdev_rpmb); 8172 goto remove_ufs_device_wlun; 8173 } 8174 ufshcd_blk_pm_runtime_init(sdev_rpmb); 8175 scsi_device_put(sdev_rpmb); 8176 8177 sdev_boot = __scsi_add_device(hba->host, 0, 0, 8178 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 8179 if (IS_ERR(sdev_boot)) { 8180 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__); 8181 } else { 8182 ufshcd_blk_pm_runtime_init(sdev_boot); 8183 scsi_device_put(sdev_boot); 8184 } 8185 goto out; 8186 8187 remove_ufs_device_wlun: 8188 scsi_remove_device(hba->ufs_device_wlun); 8189 out: 8190 return ret; 8191 } 8192 8193 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) 8194 { 8195 struct ufs_dev_info *dev_info = &hba->dev_info; 8196 u8 lun; 8197 u32 d_lu_wb_buf_alloc; 8198 u32 ext_ufs_feature; 8199 8200 if (!ufshcd_is_wb_allowed(hba)) 8201 return; 8202 8203 /* 8204 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or 8205 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES 8206 * enabled 8207 */ 8208 if (!(dev_info->wspecversion >= 0x310 || 8209 dev_info->wspecversion == 0x220 || 8210 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) 8211 goto wb_disabled; 8212 8213 ext_ufs_feature = get_unaligned_be32(desc_buf + 8214 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8215 8216 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP)) 8217 goto wb_disabled; 8218 8219 /* 8220 * WB may be supported but not configured while provisioning. The spec 8221 * says, in dedicated wb buffer mode, a max of 1 lun would have wb 8222 * buffer configured. 8223 */ 8224 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; 8225 8226 dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + 8227 DEVICE_DESC_PARAM_EXT_WB_SUP); 8228 8229 dev_info->b_presrv_uspc_en = 8230 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; 8231 8232 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) { 8233 if (!get_unaligned_be32(desc_buf + 8234 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS)) 8235 goto wb_disabled; 8236 } else { 8237 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { 8238 d_lu_wb_buf_alloc = 0; 8239 ufshcd_read_unit_desc_param(hba, 8240 lun, 8241 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS, 8242 (u8 *)&d_lu_wb_buf_alloc, 8243 sizeof(d_lu_wb_buf_alloc)); 8244 if (d_lu_wb_buf_alloc) { 8245 dev_info->wb_dedicated_lu = lun; 8246 break; 8247 } 8248 } 8249 8250 if (!d_lu_wb_buf_alloc) 8251 goto wb_disabled; 8252 } 8253 8254 if (!ufshcd_is_wb_buf_lifetime_available(hba)) 8255 goto wb_disabled; 8256 8257 return; 8258 8259 wb_disabled: 8260 hba->caps &= ~UFSHCD_CAP_WB_EN; 8261 } 8262 8263 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) 8264 { 8265 struct ufs_dev_info *dev_info = &hba->dev_info; 8266 u32 ext_ufs_feature; 8267 u8 mask = 0; 8268 8269 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300) 8270 return; 8271 8272 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8273 8274 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF) 8275 mask |= MASK_EE_TOO_LOW_TEMP; 8276 8277 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF) 8278 mask |= MASK_EE_TOO_HIGH_TEMP; 8279 8280 if (mask) { 8281 ufshcd_enable_ee(hba, mask); 8282 ufs_hwmon_probe(hba, mask); 8283 } 8284 } 8285 8286 static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) 8287 { 8288 u32 ext_ufs_feature; 8289 8290 if (hba->dev_info.wspecversion < 0x410) 8291 return; 8292 8293 ext_ufs_feature = get_unaligned_be32(desc_buf + 8294 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8295 if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) 8296 return; 8297 8298 atomic_set(&hba->dev_lvl_exception_count, 0); 8299 ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); 8300 } 8301 8302 static void ufshcd_set_rtt(struct ufs_hba *hba) 8303 { 8304 struct ufs_dev_info *dev_info = &hba->dev_info; 8305 u32 rtt = 0; 8306 u32 dev_rtt = 0; 8307 int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? 8308 hba->vops->max_num_rtt : hba->nortt; 8309 8310 /* RTT override makes sense only for UFS-4.0 and above */ 8311 if (dev_info->wspecversion < 0x400) 8312 return; 8313 8314 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8315 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { 8316 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); 8317 return; 8318 } 8319 8320 /* do not override if it was already written */ 8321 if (dev_rtt != DEFAULT_MAX_NUM_RTT) 8322 return; 8323 8324 rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); 8325 8326 if (rtt == dev_rtt) 8327 return; 8328 8329 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8330 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) 8331 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); 8332 } 8333 8334 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, 8335 const struct ufs_dev_quirk *fixups) 8336 { 8337 const struct ufs_dev_quirk *f; 8338 struct ufs_dev_info *dev_info = &hba->dev_info; 8339 8340 if (!fixups) 8341 return; 8342 8343 for (f = fixups; f->quirk; f++) { 8344 if ((f->wmanufacturerid == dev_info->wmanufacturerid || 8345 f->wmanufacturerid == UFS_ANY_VENDOR) && 8346 ((dev_info->model && 8347 STR_PRFX_EQUAL(f->model, dev_info->model)) || 8348 !strcmp(f->model, UFS_ANY_MODEL))) 8349 hba->dev_quirks |= f->quirk; 8350 } 8351 } 8352 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks); 8353 8354 static void ufs_fixup_device_setup(struct ufs_hba *hba) 8355 { 8356 /* fix by general quirk table */ 8357 ufshcd_fixup_dev_quirks(hba, ufs_fixups); 8358 8359 /* allow vendors to fix quirks */ 8360 ufshcd_vops_fixup_dev_quirks(hba); 8361 } 8362 8363 static void ufshcd_update_rtc(struct ufs_hba *hba) 8364 { 8365 struct timespec64 ts64; 8366 int err; 8367 u32 val; 8368 8369 ktime_get_real_ts64(&ts64); 8370 8371 if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { 8372 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); 8373 return; 8374 } 8375 8376 /* 8377 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond 8378 * 2146 is required, it is recommended to choose the relative RTC mode. 8379 */ 8380 val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; 8381 8382 /* Skip update RTC if RPM state is not RPM_ACTIVE */ 8383 if (ufshcd_rpm_get_if_active(hba) <= 0) 8384 return; 8385 8386 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, 8387 0, 0, &val); 8388 ufshcd_rpm_put(hba); 8389 8390 if (err) 8391 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); 8392 else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 8393 hba->dev_info.rtc_time_baseline = ts64.tv_sec; 8394 } 8395 8396 static void ufshcd_rtc_work(struct work_struct *work) 8397 { 8398 struct ufs_hba *hba; 8399 8400 hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); 8401 8402 /* Update RTC only when there are no requests in progress and UFSHCI is operational */ 8403 if (!ufshcd_is_ufs_dev_busy(hba) && 8404 hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && 8405 !hba->clk_gating.active_reqs) 8406 ufshcd_update_rtc(hba); 8407 8408 if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) 8409 schedule_delayed_work(&hba->ufs_rtc_update_work, 8410 msecs_to_jiffies(hba->dev_info.rtc_update_period)); 8411 } 8412 8413 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) 8414 { 8415 u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); 8416 struct ufs_dev_info *dev_info = &hba->dev_info; 8417 8418 if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { 8419 dev_info->rtc_type = UFS_RTC_ABSOLUTE; 8420 8421 /* 8422 * The concept of measuring time in Linux as the number of seconds elapsed since 8423 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st 8424 * 2010 00:00, here we need to adjust ABS baseline. 8425 */ 8426 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - 8427 mktime64(1970, 1, 1, 0, 0, 0); 8428 } else { 8429 dev_info->rtc_type = UFS_RTC_RELATIVE; 8430 dev_info->rtc_time_baseline = 0; 8431 } 8432 8433 /* 8434 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state 8435 * how to calculate the specific update period for each time unit. And we disable periodic 8436 * RTC update work, let user configure by sysfs node according to specific circumstance. 8437 */ 8438 dev_info->rtc_update_period = 0; 8439 } 8440 8441 static int ufs_get_device_desc(struct ufs_hba *hba) 8442 { 8443 int err; 8444 u8 model_index; 8445 u8 *desc_buf; 8446 struct ufs_dev_info *dev_info = &hba->dev_info; 8447 8448 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8449 if (!desc_buf) { 8450 err = -ENOMEM; 8451 goto out; 8452 } 8453 8454 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, 8455 QUERY_DESC_MAX_SIZE); 8456 if (err) { 8457 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", 8458 __func__, err); 8459 goto out; 8460 } 8461 8462 /* 8463 * getting vendor (manufacturerID) and Bank Index in big endian 8464 * format 8465 */ 8466 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | 8467 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; 8468 8469 /* getting Specification Version in big endian format */ 8470 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8471 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8472 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8473 8474 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; 8475 8476 dev_info->hid_sup = get_unaligned_be32(desc_buf + 8477 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & 8478 UFS_DEV_HID_SUPPORT; 8479 8480 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; 8481 8482 err = ufshcd_read_string_desc(hba, model_index, 8483 &dev_info->model, SD_ASCII_STD); 8484 if (err < 0) { 8485 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", 8486 __func__, err); 8487 goto out; 8488 } 8489 8490 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + 8491 desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; 8492 8493 ufs_fixup_device_setup(hba); 8494 8495 ufshcd_wb_probe(hba, desc_buf); 8496 8497 ufshcd_temp_notif_probe(hba, desc_buf); 8498 8499 if (dev_info->wspecversion >= 0x410) { 8500 hba->critical_health_count = 0; 8501 ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); 8502 } 8503 8504 ufs_init_rtc(hba, desc_buf); 8505 8506 ufshcd_device_lvl_exception_probe(hba, desc_buf); 8507 8508 /* 8509 * ufshcd_read_string_desc returns size of the string 8510 * reset the error value 8511 */ 8512 err = 0; 8513 8514 out: 8515 kfree(desc_buf); 8516 return err; 8517 } 8518 8519 static void ufs_put_device_desc(struct ufs_hba *hba) 8520 { 8521 struct ufs_dev_info *dev_info = &hba->dev_info; 8522 8523 kfree(dev_info->model); 8524 dev_info->model = NULL; 8525 } 8526 8527 /** 8528 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is 8529 * less than device PA_TACTIVATE time. 8530 * @hba: per-adapter instance 8531 * 8532 * Some UFS devices require host PA_TACTIVATE to be lower than device 8533 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk 8534 * for such devices. 8535 * 8536 * Return: zero on success, non-zero error value on failure. 8537 */ 8538 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) 8539 { 8540 int ret = 0; 8541 u32 granularity, peer_granularity; 8542 u32 pa_tactivate, peer_pa_tactivate; 8543 u32 pa_tactivate_us, peer_pa_tactivate_us; 8544 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; 8545 8546 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8547 &granularity); 8548 if (ret) 8549 goto out; 8550 8551 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8552 &peer_granularity); 8553 if (ret) 8554 goto out; 8555 8556 if ((granularity < PA_GRANULARITY_MIN_VAL) || 8557 (granularity > PA_GRANULARITY_MAX_VAL)) { 8558 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", 8559 __func__, granularity); 8560 return -EINVAL; 8561 } 8562 8563 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || 8564 (peer_granularity > PA_GRANULARITY_MAX_VAL)) { 8565 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", 8566 __func__, peer_granularity); 8567 return -EINVAL; 8568 } 8569 8570 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); 8571 if (ret) 8572 goto out; 8573 8574 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), 8575 &peer_pa_tactivate); 8576 if (ret) 8577 goto out; 8578 8579 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; 8580 peer_pa_tactivate_us = peer_pa_tactivate * 8581 gran_to_us_table[peer_granularity - 1]; 8582 8583 if (pa_tactivate_us >= peer_pa_tactivate_us) { 8584 u32 new_peer_pa_tactivate; 8585 8586 new_peer_pa_tactivate = pa_tactivate_us / 8587 gran_to_us_table[peer_granularity - 1]; 8588 new_peer_pa_tactivate++; 8589 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8590 new_peer_pa_tactivate); 8591 } 8592 8593 out: 8594 return ret; 8595 } 8596 8597 /** 8598 * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. 8599 * @hba: per-adapter instance 8600 * 8601 * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter 8602 * to ensure proper hibernation timing. This function retrieves the current 8603 * PA_HIBERN8TIME value and increments it by 100us. 8604 */ 8605 static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) 8606 { 8607 u32 pa_h8time; 8608 int ret; 8609 8610 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); 8611 if (ret) { 8612 dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); 8613 return; 8614 } 8615 8616 /* Increment by 1 to increase hibernation time by 100 µs */ 8617 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); 8618 if (ret) 8619 dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); 8620 } 8621 8622 static void ufshcd_tune_unipro_params(struct ufs_hba *hba) 8623 { 8624 ufshcd_vops_apply_dev_quirks(hba); 8625 8626 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) 8627 /* set 1ms timeout for PA_TACTIVATE */ 8628 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); 8629 8630 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) 8631 ufshcd_quirk_tune_host_pa_tactivate(hba); 8632 8633 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) 8634 ufshcd_quirk_override_pa_h8time(hba); 8635 } 8636 8637 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) 8638 { 8639 hba->ufs_stats.hibern8_exit_cnt = 0; 8640 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 8641 hba->req_abort_count = 0; 8642 } 8643 8644 static int ufshcd_device_geo_params_init(struct ufs_hba *hba) 8645 { 8646 int err; 8647 u8 *desc_buf; 8648 8649 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8650 if (!desc_buf) { 8651 err = -ENOMEM; 8652 goto out; 8653 } 8654 8655 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, 8656 desc_buf, QUERY_DESC_MAX_SIZE); 8657 if (err) { 8658 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", 8659 __func__, err); 8660 goto out; 8661 } 8662 8663 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1) 8664 hba->dev_info.max_lu_supported = 32; 8665 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) 8666 hba->dev_info.max_lu_supported = 8; 8667 8668 out: 8669 kfree(desc_buf); 8670 return err; 8671 } 8672 8673 struct ufs_ref_clk { 8674 unsigned long freq_hz; 8675 enum ufs_ref_clk_freq val; 8676 }; 8677 8678 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = { 8679 {19200000, REF_CLK_FREQ_19_2_MHZ}, 8680 {26000000, REF_CLK_FREQ_26_MHZ}, 8681 {38400000, REF_CLK_FREQ_38_4_MHZ}, 8682 {52000000, REF_CLK_FREQ_52_MHZ}, 8683 {0, REF_CLK_FREQ_INVAL}, 8684 }; 8685 8686 static enum ufs_ref_clk_freq 8687 ufs_get_bref_clk_from_hz(unsigned long freq) 8688 { 8689 int i; 8690 8691 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++) 8692 if (ufs_ref_clk_freqs[i].freq_hz == freq) 8693 return ufs_ref_clk_freqs[i].val; 8694 8695 return REF_CLK_FREQ_INVAL; 8696 } 8697 8698 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk) 8699 { 8700 unsigned long freq; 8701 8702 freq = clk_get_rate(refclk); 8703 8704 hba->dev_ref_clk_freq = 8705 ufs_get_bref_clk_from_hz(freq); 8706 8707 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 8708 dev_err(hba->dev, 8709 "invalid ref_clk setting = %ld\n", freq); 8710 } 8711 8712 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba) 8713 { 8714 int err; 8715 u32 ref_clk; 8716 u32 freq = hba->dev_ref_clk_freq; 8717 8718 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8719 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk); 8720 8721 if (err) { 8722 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n", 8723 err); 8724 goto out; 8725 } 8726 8727 if (ref_clk == freq) 8728 goto out; /* nothing to update */ 8729 8730 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8731 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq); 8732 8733 if (err) { 8734 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n", 8735 ufs_ref_clk_freqs[freq].freq_hz); 8736 goto out; 8737 } 8738 8739 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n", 8740 ufs_ref_clk_freqs[freq].freq_hz); 8741 8742 out: 8743 return err; 8744 } 8745 8746 static int ufshcd_device_params_init(struct ufs_hba *hba) 8747 { 8748 bool flag; 8749 int ret; 8750 8751 /* Init UFS geometry descriptor related parameters */ 8752 ret = ufshcd_device_geo_params_init(hba); 8753 if (ret) 8754 goto out; 8755 8756 /* Check and apply UFS device quirks */ 8757 ret = ufs_get_device_desc(hba); 8758 if (ret) { 8759 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", 8760 __func__, ret); 8761 goto out; 8762 } 8763 8764 ufshcd_set_rtt(hba); 8765 8766 ufshcd_get_ref_clk_gating_wait(hba); 8767 8768 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, 8769 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag)) 8770 hba->dev_info.f_power_on_wp_en = flag; 8771 8772 /* Probe maximum power mode co-supported by both UFS host and device */ 8773 if (ufshcd_get_max_pwr_mode(hba)) 8774 dev_err(hba->dev, 8775 "%s: Failed getting max supported power mode\n", 8776 __func__); 8777 out: 8778 return ret; 8779 } 8780 8781 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) 8782 { 8783 int err; 8784 struct ufs_query_req *request = NULL; 8785 struct ufs_query_res *response = NULL; 8786 struct ufs_dev_info *dev_info = &hba->dev_info; 8787 struct utp_upiu_query_v4_0 *upiu_data; 8788 8789 if (dev_info->wspecversion < 0x400) 8790 return; 8791 8792 ufshcd_dev_man_lock(hba); 8793 8794 ufshcd_init_query(hba, &request, &response, 8795 UPIU_QUERY_OPCODE_WRITE_ATTR, 8796 QUERY_ATTR_IDN_TIMESTAMP, 0, 0); 8797 8798 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 8799 8800 upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req; 8801 8802 put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); 8803 8804 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 8805 8806 if (err) 8807 dev_err(hba->dev, "%s: failed to set timestamp %d\n", 8808 __func__, err); 8809 8810 ufshcd_dev_man_unlock(hba); 8811 } 8812 8813 /** 8814 * ufshcd_add_lus - probe and add UFS logical units 8815 * @hba: per-adapter instance 8816 * 8817 * Return: 0 upon success; < 0 upon failure. 8818 */ 8819 static int ufshcd_add_lus(struct ufs_hba *hba) 8820 { 8821 int ret; 8822 8823 /* Add required well known logical units to scsi mid layer */ 8824 ret = ufshcd_scsi_add_wlus(hba); 8825 if (ret) 8826 goto out; 8827 8828 /* Initialize devfreq after UFS device is detected */ 8829 if (ufshcd_is_clkscaling_supported(hba)) { 8830 memcpy(&hba->clk_scaling.saved_pwr_info, 8831 &hba->pwr_info, 8832 sizeof(struct ufs_pa_layer_attr)); 8833 hba->clk_scaling.is_allowed = true; 8834 8835 ret = ufshcd_devfreq_init(hba); 8836 if (ret) 8837 goto out; 8838 8839 hba->clk_scaling.is_enabled = true; 8840 ufshcd_init_clk_scaling_sysfs(hba); 8841 } 8842 8843 /* 8844 * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev 8845 * pointer and hence must only be started after the WLUN pointer has 8846 * been initialized by ufshcd_scsi_add_wlus(). 8847 */ 8848 schedule_delayed_work(&hba->ufs_rtc_update_work, 8849 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 8850 8851 ufs_bsg_probe(hba); 8852 scsi_scan_host(hba->host); 8853 8854 out: 8855 return ret; 8856 } 8857 8858 /* SDB - Single Doorbell */ 8859 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) 8860 { 8861 size_t ucdl_size, utrdl_size; 8862 8863 ucdl_size = ufshcd_get_ucd_size(hba) * nutrs; 8864 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, 8865 hba->ucdl_dma_addr); 8866 8867 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 8868 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 8869 hba->utrdl_dma_addr); 8870 8871 devm_kfree(hba->dev, hba->lrb); 8872 } 8873 8874 static int ufshcd_alloc_mcq(struct ufs_hba *hba) 8875 { 8876 int ret; 8877 int old_nutrs = hba->nutrs; 8878 8879 ret = ufshcd_mcq_decide_queue_depth(hba); 8880 if (ret < 0) 8881 return ret; 8882 8883 hba->nutrs = ret; 8884 ret = ufshcd_mcq_init(hba); 8885 if (ret) 8886 goto err; 8887 8888 /* 8889 * Previously allocated memory for nutrs may not be enough in MCQ mode. 8890 * Number of supported tags in MCQ mode may be larger than SDB mode. 8891 */ 8892 if (hba->nutrs != old_nutrs) { 8893 ufshcd_release_sdb_queue(hba, old_nutrs); 8894 ret = ufshcd_memory_alloc(hba); 8895 if (ret) 8896 goto err; 8897 ufshcd_host_memory_configure(hba); 8898 } 8899 8900 ret = ufshcd_mcq_memory_alloc(hba); 8901 if (ret) 8902 goto err; 8903 8904 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 8905 hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; 8906 8907 return 0; 8908 err: 8909 hba->nutrs = old_nutrs; 8910 return ret; 8911 } 8912 8913 static void ufshcd_config_mcq(struct ufs_hba *hba) 8914 { 8915 int ret; 8916 u32 intrs; 8917 8918 ret = ufshcd_mcq_vops_config_esi(hba); 8919 hba->mcq_esi_enabled = !ret; 8920 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); 8921 8922 intrs = UFSHCD_ENABLE_MCQ_INTRS; 8923 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR) 8924 intrs &= ~MCQ_CQ_EVENT_STATUS; 8925 ufshcd_enable_intr(hba, intrs); 8926 ufshcd_mcq_make_queues_operational(hba); 8927 ufshcd_mcq_config_mac(hba, hba->nutrs); 8928 8929 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", 8930 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], 8931 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], 8932 hba->nutrs); 8933 } 8934 8935 static int ufshcd_post_device_init(struct ufs_hba *hba) 8936 { 8937 int ret; 8938 8939 ufshcd_tune_unipro_params(hba); 8940 8941 /* UFS device is also active now */ 8942 ufshcd_set_ufs_dev_active(hba); 8943 ufshcd_force_reset_auto_bkops(hba); 8944 8945 ufshcd_set_timestamp_attr(hba); 8946 8947 if (!hba->max_pwr_info.is_valid) 8948 return 0; 8949 8950 /* 8951 * Set the right value to bRefClkFreq before attempting to 8952 * switch to HS gears. 8953 */ 8954 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) 8955 ufshcd_set_dev_ref_clk(hba); 8956 /* Gear up to HS gear. */ 8957 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 8958 if (ret) { 8959 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 8960 __func__, ret); 8961 return ret; 8962 } 8963 8964 return 0; 8965 } 8966 8967 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) 8968 { 8969 int ret; 8970 8971 WARN_ON_ONCE(!hba->scsi_host_added); 8972 8973 hba->ufshcd_state = UFSHCD_STATE_RESET; 8974 8975 ret = ufshcd_link_startup(hba); 8976 if (ret) 8977 return ret; 8978 8979 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 8980 return ret; 8981 8982 /* Debug counters initialization */ 8983 ufshcd_clear_dbg_ufs_stats(hba); 8984 8985 /* UniPro link is active now */ 8986 ufshcd_set_link_active(hba); 8987 8988 /* Reconfigure MCQ upon reset */ 8989 if (hba->mcq_enabled && !init_dev_params) { 8990 ufshcd_config_mcq(hba); 8991 ufshcd_mcq_enable(hba); 8992 } 8993 8994 /* Verify device initialization by sending NOP OUT UPIU */ 8995 ret = ufshcd_verify_dev_init(hba); 8996 if (ret) 8997 return ret; 8998 8999 /* Initiate UFS initialization, and waiting until completion */ 9000 ret = ufshcd_complete_dev_init(hba); 9001 if (ret) 9002 return ret; 9003 9004 /* 9005 * Initialize UFS device parameters used by driver, these 9006 * parameters are associated with UFS descriptors. 9007 */ 9008 if (init_dev_params) { 9009 ret = ufshcd_device_params_init(hba); 9010 if (ret) 9011 return ret; 9012 if (is_mcq_supported(hba) && 9013 hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 9014 ufshcd_config_mcq(hba); 9015 ufshcd_mcq_enable(hba); 9016 } 9017 } 9018 9019 return ufshcd_post_device_init(hba); 9020 } 9021 9022 /** 9023 * ufshcd_probe_hba - probe hba to detect device and initialize it 9024 * @hba: per-adapter instance 9025 * @init_dev_params: whether or not to call ufshcd_device_params_init(). 9026 * 9027 * Execute link-startup and verify device initialization 9028 * 9029 * Return: 0 upon success; < 0 upon failure. 9030 */ 9031 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) 9032 { 9033 int ret; 9034 9035 if (!hba->pm_op_in_progress && 9036 (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { 9037 /* Reset the device and controller before doing reinit */ 9038 ufshcd_device_reset(hba); 9039 ufs_put_device_desc(hba); 9040 ufshcd_hba_stop(hba); 9041 ret = ufshcd_hba_enable(hba); 9042 if (ret) { 9043 dev_err(hba->dev, "Host controller enable failed\n"); 9044 ufshcd_print_evt_hist(hba); 9045 ufshcd_print_host_state(hba); 9046 return ret; 9047 } 9048 9049 /* Reinit the device */ 9050 ret = ufshcd_device_init(hba, init_dev_params); 9051 if (ret) 9052 return ret; 9053 } 9054 9055 ufshcd_print_pwr_info(hba); 9056 9057 /* 9058 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) 9059 * and for removable UFS card as well, hence always set the parameter. 9060 * Note: Error handler may issue the device reset hence resetting 9061 * bActiveICCLevel as well so it is always safe to set this here. 9062 */ 9063 ufshcd_set_active_icc_lvl(hba); 9064 9065 /* Enable UFS Write Booster if supported */ 9066 ufshcd_configure_wb(hba); 9067 9068 if (hba->ee_usr_mask) 9069 ufshcd_write_ee_control(hba); 9070 ufshcd_configure_auto_hibern8(hba); 9071 9072 return 0; 9073 } 9074 9075 /** 9076 * ufshcd_async_scan - asynchronous execution for probing hba 9077 * @data: data pointer to pass to this function 9078 * @cookie: cookie data 9079 */ 9080 static void ufshcd_async_scan(void *data, async_cookie_t cookie) 9081 { 9082 struct ufs_hba *hba = (struct ufs_hba *)data; 9083 ktime_t probe_start; 9084 int ret; 9085 9086 down(&hba->host_sem); 9087 /* Initialize hba, detect and initialize UFS device */ 9088 probe_start = ktime_get(); 9089 ret = ufshcd_probe_hba(hba, true); 9090 ufshcd_process_probe_result(hba, probe_start, ret); 9091 up(&hba->host_sem); 9092 if (ret) 9093 goto out; 9094 9095 /* Probe and add UFS logical units */ 9096 ret = ufshcd_add_lus(hba); 9097 9098 out: 9099 pm_runtime_put_sync(hba->dev); 9100 9101 if (ret) 9102 dev_err(hba->dev, "%s failed: %d\n", __func__, ret); 9103 } 9104 9105 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) 9106 { 9107 struct ufs_hba *hba = shost_priv(scmd->device->host); 9108 9109 if (!hba->system_suspending) { 9110 /* Activate the error handler in the SCSI core. */ 9111 return SCSI_EH_NOT_HANDLED; 9112 } 9113 9114 /* 9115 * If we get here we know that no TMFs are outstanding and also that 9116 * the only pending command is a START STOP UNIT command. Handle the 9117 * timeout of that command directly to prevent a deadlock between 9118 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). 9119 */ 9120 ufshcd_link_recovery(hba); 9121 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", 9122 __func__, hba->outstanding_tasks); 9123 9124 return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; 9125 } 9126 9127 static const struct attribute_group *ufshcd_driver_groups[] = { 9128 &ufs_sysfs_unit_descriptor_group, 9129 &ufs_sysfs_lun_attributes_group, 9130 NULL, 9131 }; 9132 9133 static struct ufs_hba_variant_params ufs_hba_vps = { 9134 .hba_enable_delay_us = 1000, 9135 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40), 9136 .devfreq_profile.polling_ms = 100, 9137 .devfreq_profile.target = ufshcd_devfreq_target, 9138 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status, 9139 .ondemand_data.upthreshold = 70, 9140 .ondemand_data.downdifferential = 5, 9141 }; 9142 9143 static const struct scsi_host_template ufshcd_driver_template = { 9144 .module = THIS_MODULE, 9145 .name = UFSHCD, 9146 .proc_name = UFSHCD, 9147 .map_queues = ufshcd_map_queues, 9148 .queuecommand = ufshcd_queuecommand, 9149 .mq_poll = ufshcd_poll, 9150 .sdev_init = ufshcd_sdev_init, 9151 .sdev_configure = ufshcd_sdev_configure, 9152 .sdev_destroy = ufshcd_sdev_destroy, 9153 .change_queue_depth = ufshcd_change_queue_depth, 9154 .eh_abort_handler = ufshcd_abort, 9155 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 9156 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 9157 .eh_timed_out = ufshcd_eh_timed_out, 9158 .this_id = -1, 9159 .sg_tablesize = SG_ALL, 9160 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, 9161 .max_sectors = SZ_1M / SECTOR_SIZE, 9162 .max_host_blocked = 1, 9163 .track_queue_depth = 1, 9164 .skip_settle_delay = 1, 9165 .sdev_groups = ufshcd_driver_groups, 9166 }; 9167 9168 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 9169 int ua) 9170 { 9171 int ret; 9172 9173 if (!vreg) 9174 return 0; 9175 9176 /* 9177 * "set_load" operation shall be required on those regulators 9178 * which specifically configured current limitation. Otherwise 9179 * zero max_uA may cause unexpected behavior when regulator is 9180 * enabled or set as high power mode. 9181 */ 9182 if (!vreg->max_uA) 9183 return 0; 9184 9185 ret = regulator_set_load(vreg->reg, ua); 9186 if (ret < 0) { 9187 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 9188 __func__, vreg->name, ua, ret); 9189 } 9190 9191 return ret; 9192 } 9193 9194 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 9195 struct ufs_vreg *vreg) 9196 { 9197 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 9198 } 9199 9200 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 9201 struct ufs_vreg *vreg) 9202 { 9203 if (!vreg) 9204 return 0; 9205 9206 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 9207 } 9208 9209 static int ufshcd_config_vreg(struct device *dev, 9210 struct ufs_vreg *vreg, bool on) 9211 { 9212 if (regulator_count_voltages(vreg->reg) <= 0) 9213 return 0; 9214 9215 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0); 9216 } 9217 9218 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 9219 { 9220 int ret = 0; 9221 9222 if (!vreg || vreg->enabled) 9223 goto out; 9224 9225 ret = ufshcd_config_vreg(dev, vreg, true); 9226 if (!ret) 9227 ret = regulator_enable(vreg->reg); 9228 9229 if (!ret) 9230 vreg->enabled = true; 9231 else 9232 dev_err(dev, "%s: %s enable failed, err=%d\n", 9233 __func__, vreg->name, ret); 9234 out: 9235 return ret; 9236 } 9237 9238 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 9239 { 9240 int ret = 0; 9241 9242 if (!vreg || !vreg->enabled || vreg->always_on) 9243 goto out; 9244 9245 ret = regulator_disable(vreg->reg); 9246 9247 if (!ret) { 9248 /* ignore errors on applying disable config */ 9249 ufshcd_config_vreg(dev, vreg, false); 9250 vreg->enabled = false; 9251 } else { 9252 dev_err(dev, "%s: %s disable failed, err=%d\n", 9253 __func__, vreg->name, ret); 9254 } 9255 out: 9256 return ret; 9257 } 9258 9259 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 9260 { 9261 int ret = 0; 9262 struct device *dev = hba->dev; 9263 struct ufs_vreg_info *info = &hba->vreg_info; 9264 9265 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 9266 if (ret) 9267 goto out; 9268 9269 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 9270 if (ret) 9271 goto out; 9272 9273 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 9274 9275 out: 9276 if (ret) { 9277 ufshcd_toggle_vreg(dev, info->vccq2, false); 9278 ufshcd_toggle_vreg(dev, info->vccq, false); 9279 ufshcd_toggle_vreg(dev, info->vcc, false); 9280 } 9281 return ret; 9282 } 9283 9284 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 9285 { 9286 struct ufs_vreg_info *info = &hba->vreg_info; 9287 9288 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 9289 } 9290 9291 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 9292 { 9293 int ret = 0; 9294 9295 if (!vreg) 9296 goto out; 9297 9298 vreg->reg = devm_regulator_get(dev, vreg->name); 9299 if (IS_ERR(vreg->reg)) { 9300 ret = PTR_ERR(vreg->reg); 9301 dev_err(dev, "%s: %s get failed, err=%d\n", 9302 __func__, vreg->name, ret); 9303 } 9304 out: 9305 return ret; 9306 } 9307 EXPORT_SYMBOL_GPL(ufshcd_get_vreg); 9308 9309 static int ufshcd_init_vreg(struct ufs_hba *hba) 9310 { 9311 int ret = 0; 9312 struct device *dev = hba->dev; 9313 struct ufs_vreg_info *info = &hba->vreg_info; 9314 9315 ret = ufshcd_get_vreg(dev, info->vcc); 9316 if (ret) 9317 goto out; 9318 9319 ret = ufshcd_get_vreg(dev, info->vccq); 9320 if (!ret) 9321 ret = ufshcd_get_vreg(dev, info->vccq2); 9322 out: 9323 return ret; 9324 } 9325 9326 static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 9327 { 9328 struct ufs_vreg_info *info = &hba->vreg_info; 9329 9330 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 9331 } 9332 9333 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 9334 { 9335 int ret = 0; 9336 struct ufs_clk_info *clki; 9337 struct list_head *head = &hba->clk_list_head; 9338 ktime_t start = ktime_get(); 9339 bool clk_state_changed = false; 9340 9341 if (list_empty(head)) 9342 goto out; 9343 9344 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); 9345 if (ret) 9346 return ret; 9347 9348 list_for_each_entry(clki, head, list) { 9349 if (!IS_ERR_OR_NULL(clki->clk)) { 9350 /* 9351 * Don't disable clocks which are needed 9352 * to keep the link active. 9353 */ 9354 if (ufshcd_is_link_active(hba) && 9355 clki->keep_link_active) 9356 continue; 9357 9358 clk_state_changed = on ^ clki->enabled; 9359 if (on && !clki->enabled) { 9360 ret = clk_prepare_enable(clki->clk); 9361 if (ret) { 9362 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 9363 __func__, clki->name, ret); 9364 goto out; 9365 } 9366 } else if (!on && clki->enabled) { 9367 clk_disable_unprepare(clki->clk); 9368 } 9369 clki->enabled = on; 9370 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 9371 clki->name, on ? "en" : "dis"); 9372 } 9373 } 9374 9375 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); 9376 if (ret) 9377 return ret; 9378 9379 if (!ufshcd_is_clkscaling_supported(hba)) 9380 ufshcd_pm_qos_update(hba, on); 9381 out: 9382 if (ret) { 9383 list_for_each_entry(clki, head, list) { 9384 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 9385 clk_disable_unprepare(clki->clk); 9386 } 9387 } else if (!ret && on && hba->clk_gating.is_initialized) { 9388 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) 9389 hba->clk_gating.state = CLKS_ON; 9390 trace_ufshcd_clk_gating(hba, 9391 hba->clk_gating.state); 9392 } 9393 9394 if (clk_state_changed) 9395 trace_ufshcd_profile_clk_gating(hba, 9396 (on ? "on" : "off"), 9397 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 9398 return ret; 9399 } 9400 9401 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba) 9402 { 9403 u32 freq; 9404 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq); 9405 9406 if (ret) { 9407 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret); 9408 return REF_CLK_FREQ_INVAL; 9409 } 9410 9411 return ufs_get_bref_clk_from_hz(freq); 9412 } 9413 9414 static int ufshcd_init_clocks(struct ufs_hba *hba) 9415 { 9416 int ret = 0; 9417 struct ufs_clk_info *clki; 9418 struct device *dev = hba->dev; 9419 struct list_head *head = &hba->clk_list_head; 9420 9421 if (list_empty(head)) 9422 goto out; 9423 9424 list_for_each_entry(clki, head, list) { 9425 if (!clki->name) 9426 continue; 9427 9428 clki->clk = devm_clk_get(dev, clki->name); 9429 if (IS_ERR(clki->clk)) { 9430 ret = PTR_ERR(clki->clk); 9431 dev_err(dev, "%s: %s clk get failed, %d\n", 9432 __func__, clki->name, ret); 9433 goto out; 9434 } 9435 9436 /* 9437 * Parse device ref clk freq as per device tree "ref_clk". 9438 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL 9439 * in ufshcd_alloc_host(). 9440 */ 9441 if (!strcmp(clki->name, "ref_clk")) 9442 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk); 9443 9444 if (clki->max_freq) { 9445 ret = clk_set_rate(clki->clk, clki->max_freq); 9446 if (ret) { 9447 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 9448 __func__, clki->name, 9449 clki->max_freq, ret); 9450 goto out; 9451 } 9452 clki->curr_freq = clki->max_freq; 9453 } 9454 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 9455 clki->name, clk_get_rate(clki->clk)); 9456 } 9457 9458 /* Set Max. frequency for all clocks */ 9459 if (hba->use_pm_opp) { 9460 ret = ufshcd_opp_set_rate(hba, ULONG_MAX); 9461 if (ret) { 9462 dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, 9463 ret); 9464 goto out; 9465 } 9466 } 9467 9468 out: 9469 return ret; 9470 } 9471 9472 static int ufshcd_variant_hba_init(struct ufs_hba *hba) 9473 { 9474 int err = 0; 9475 9476 if (!hba->vops) 9477 goto out; 9478 9479 err = ufshcd_vops_init(hba); 9480 if (err) 9481 dev_err_probe(hba->dev, err, 9482 "%s: variant %s init failed with err %d\n", 9483 __func__, ufshcd_get_var_name(hba), err); 9484 out: 9485 return err; 9486 } 9487 9488 static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 9489 { 9490 if (!hba->vops) 9491 return; 9492 9493 ufshcd_vops_exit(hba); 9494 } 9495 9496 static int ufshcd_hba_init(struct ufs_hba *hba) 9497 { 9498 int err; 9499 9500 /* 9501 * Handle host controller power separately from the UFS device power 9502 * rails as it will help controlling the UFS host controller power 9503 * collapse easily which is different than UFS device power collapse. 9504 * Also, enable the host controller power before we go ahead with rest 9505 * of the initialization here. 9506 */ 9507 err = ufshcd_init_hba_vreg(hba); 9508 if (err) 9509 goto out; 9510 9511 err = ufshcd_setup_hba_vreg(hba, true); 9512 if (err) 9513 goto out; 9514 9515 err = ufshcd_init_clocks(hba); 9516 if (err) 9517 goto out_disable_hba_vreg; 9518 9519 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9520 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba); 9521 9522 err = ufshcd_setup_clocks(hba, true); 9523 if (err) 9524 goto out_disable_hba_vreg; 9525 9526 err = ufshcd_init_vreg(hba); 9527 if (err) 9528 goto out_disable_clks; 9529 9530 err = ufshcd_setup_vreg(hba, true); 9531 if (err) 9532 goto out_disable_clks; 9533 9534 err = ufshcd_variant_hba_init(hba); 9535 if (err) 9536 goto out_disable_vreg; 9537 9538 ufs_debugfs_hba_init(hba); 9539 ufs_fault_inject_hba_init(hba); 9540 9541 hba->is_powered = true; 9542 goto out; 9543 9544 out_disable_vreg: 9545 ufshcd_setup_vreg(hba, false); 9546 out_disable_clks: 9547 ufshcd_setup_clocks(hba, false); 9548 out_disable_hba_vreg: 9549 ufshcd_setup_hba_vreg(hba, false); 9550 out: 9551 return err; 9552 } 9553 9554 static void ufshcd_hba_exit(struct ufs_hba *hba) 9555 { 9556 if (hba->is_powered) { 9557 ufshcd_pm_qos_exit(hba); 9558 ufshcd_exit_clk_scaling(hba); 9559 ufshcd_exit_clk_gating(hba); 9560 if (hba->eh_wq) 9561 destroy_workqueue(hba->eh_wq); 9562 ufs_debugfs_hba_exit(hba); 9563 ufshcd_variant_hba_exit(hba); 9564 ufshcd_setup_vreg(hba, false); 9565 ufshcd_setup_clocks(hba, false); 9566 ufshcd_setup_hba_vreg(hba, false); 9567 hba->is_powered = false; 9568 ufs_put_device_desc(hba); 9569 } 9570 } 9571 9572 static int ufshcd_execute_start_stop(struct scsi_device *sdev, 9573 enum ufs_dev_pwr_mode pwr_mode, 9574 struct scsi_sense_hdr *sshdr) 9575 { 9576 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; 9577 struct scsi_failure failure_defs[] = { 9578 { 9579 .allowed = 2, 9580 .result = SCMD_FAILURE_RESULT_ANY, 9581 }, 9582 }; 9583 struct scsi_failures failures = { 9584 .failure_definitions = failure_defs, 9585 }; 9586 const struct scsi_exec_args args = { 9587 .failures = &failures, 9588 .sshdr = sshdr, 9589 .req_flags = BLK_MQ_REQ_PM, 9590 .scmd_flags = SCMD_FAIL_IF_RECOVERING, 9591 }; 9592 9593 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, 9594 /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0, 9595 &args); 9596 } 9597 9598 /** 9599 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 9600 * power mode 9601 * @hba: per adapter instance 9602 * @pwr_mode: device power mode to set 9603 * 9604 * Return: 0 if requested power mode is set successfully; 9605 * < 0 if failed to set the requested power mode. 9606 */ 9607 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 9608 enum ufs_dev_pwr_mode pwr_mode) 9609 { 9610 struct scsi_sense_hdr sshdr; 9611 struct scsi_device *sdp; 9612 unsigned long flags; 9613 int ret; 9614 9615 spin_lock_irqsave(hba->host->host_lock, flags); 9616 sdp = hba->ufs_device_wlun; 9617 if (sdp && scsi_device_online(sdp)) 9618 ret = scsi_device_get(sdp); 9619 else 9620 ret = -ENODEV; 9621 spin_unlock_irqrestore(hba->host->host_lock, flags); 9622 9623 if (ret) 9624 return ret; 9625 9626 /* 9627 * If scsi commands fail, the scsi mid-layer schedules scsi error- 9628 * handling, which would wait for host to be resumed. Since we know 9629 * we are functional while we are here, skip host resume in error 9630 * handling context. 9631 */ 9632 hba->host->eh_noresume = 1; 9633 9634 /* 9635 * Current function would be generally called from the power management 9636 * callbacks hence set the RQF_PM flag so that it doesn't resume the 9637 * already suspended childs. 9638 */ 9639 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); 9640 if (ret) { 9641 sdev_printk(KERN_WARNING, sdp, 9642 "START_STOP failed for power mode: %d, result %x\n", 9643 pwr_mode, ret); 9644 if (ret > 0) { 9645 if (scsi_sense_valid(&sshdr)) 9646 scsi_print_sense_hdr(sdp, NULL, &sshdr); 9647 ret = -EIO; 9648 } 9649 } else { 9650 hba->curr_dev_pwr_mode = pwr_mode; 9651 } 9652 9653 scsi_device_put(sdp); 9654 hba->host->eh_noresume = 0; 9655 return ret; 9656 } 9657 9658 static int ufshcd_link_state_transition(struct ufs_hba *hba, 9659 enum uic_link_state req_link_state, 9660 bool check_for_bkops) 9661 { 9662 int ret = 0; 9663 9664 if (req_link_state == hba->uic_link_state) 9665 return 0; 9666 9667 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 9668 ret = ufshcd_uic_hibern8_enter(hba); 9669 if (!ret) { 9670 ufshcd_set_link_hibern8(hba); 9671 } else { 9672 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9673 __func__, ret); 9674 goto out; 9675 } 9676 } 9677 /* 9678 * If autobkops is enabled, link can't be turned off because 9679 * turning off the link would also turn off the device, except in the 9680 * case of DeepSleep where the device is expected to remain powered. 9681 */ 9682 else if ((req_link_state == UIC_LINK_OFF_STATE) && 9683 (!check_for_bkops || !hba->auto_bkops_enabled)) { 9684 /* 9685 * Let's make sure that link is in low power mode, we are doing 9686 * this currently by putting the link in Hibern8. Otherway to 9687 * put the link in low power mode is to send the DME end point 9688 * to device and then send the DME reset command to local 9689 * unipro. But putting the link in hibern8 is much faster. 9690 * 9691 * Note also that putting the link in Hibern8 is a requirement 9692 * for entering DeepSleep. 9693 */ 9694 ret = ufshcd_uic_hibern8_enter(hba); 9695 if (ret) { 9696 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9697 __func__, ret); 9698 goto out; 9699 } 9700 /* 9701 * Change controller state to "reset state" which 9702 * should also put the link in off/reset state 9703 */ 9704 ufshcd_hba_stop(hba); 9705 /* 9706 * TODO: Check if we need any delay to make sure that 9707 * controller is reset 9708 */ 9709 ufshcd_set_link_off(hba); 9710 } 9711 9712 out: 9713 return ret; 9714 } 9715 9716 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 9717 { 9718 bool vcc_off = false; 9719 9720 /* 9721 * It seems some UFS devices may keep drawing more than sleep current 9722 * (atleast for 500us) from UFS rails (especially from VCCQ rail). 9723 * To avoid this situation, add 2ms delay before putting these UFS 9724 * rails in LPM mode. 9725 */ 9726 if (!ufshcd_is_link_active(hba) && 9727 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM) 9728 usleep_range(2000, 2100); 9729 9730 /* 9731 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 9732 * power. 9733 * 9734 * If UFS device and link is in OFF state, all power supplies (VCC, 9735 * VCCQ, VCCQ2) can be turned off if power on write protect is not 9736 * required. If UFS link is inactive (Hibern8 or OFF state) and device 9737 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 9738 * 9739 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 9740 * in low power state which would save some power. 9741 * 9742 * If Write Booster is enabled and the device needs to flush the WB 9743 * buffer OR if bkops status is urgent for WB, keep Vcc on. 9744 */ 9745 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9746 !hba->dev_info.is_lu_power_on_wp) { 9747 ufshcd_setup_vreg(hba, false); 9748 vcc_off = true; 9749 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9750 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9751 vcc_off = true; 9752 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { 9753 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9754 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 9755 } 9756 } 9757 9758 /* 9759 * Some UFS devices require delay after VCC power rail is turned-off. 9760 */ 9761 if (vcc_off && hba->vreg_info.vcc && 9762 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM) 9763 usleep_range(5000, 5100); 9764 } 9765 9766 #ifdef CONFIG_PM 9767 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 9768 { 9769 int ret = 0; 9770 9771 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9772 !hba->dev_info.is_lu_power_on_wp) { 9773 ret = ufshcd_setup_vreg(hba, true); 9774 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9775 if (!ufshcd_is_link_active(hba)) { 9776 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 9777 if (ret) 9778 goto vcc_disable; 9779 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 9780 if (ret) 9781 goto vccq_lpm; 9782 } 9783 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 9784 } 9785 goto out; 9786 9787 vccq_lpm: 9788 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9789 vcc_disable: 9790 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9791 out: 9792 return ret; 9793 } 9794 #endif /* CONFIG_PM */ 9795 9796 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 9797 { 9798 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9799 ufshcd_setup_hba_vreg(hba, false); 9800 } 9801 9802 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 9803 { 9804 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9805 ufshcd_setup_hba_vreg(hba, true); 9806 } 9807 9808 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9809 { 9810 int ret = 0; 9811 bool check_for_bkops; 9812 enum ufs_pm_level pm_lvl; 9813 enum ufs_dev_pwr_mode req_dev_pwr_mode; 9814 enum uic_link_state req_link_state; 9815 9816 hba->pm_op_in_progress = true; 9817 if (pm_op != UFS_SHUTDOWN_PM) { 9818 pm_lvl = pm_op == UFS_RUNTIME_PM ? 9819 hba->rpm_lvl : hba->spm_lvl; 9820 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 9821 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 9822 } else { 9823 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 9824 req_link_state = UIC_LINK_OFF_STATE; 9825 } 9826 9827 /* 9828 * If we can't transition into any of the low power modes 9829 * just gate the clocks. 9830 */ 9831 ufshcd_hold(hba); 9832 hba->clk_gating.is_suspended = true; 9833 9834 if (ufshcd_is_clkscaling_supported(hba)) 9835 ufshcd_clk_scaling_suspend(hba, true); 9836 9837 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 9838 req_link_state == UIC_LINK_ACTIVE_STATE) { 9839 goto vops_suspend; 9840 } 9841 9842 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 9843 (req_link_state == hba->uic_link_state)) 9844 goto enable_scaling; 9845 9846 /* UFS device & link must be active before we enter in this function */ 9847 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 9848 /* Wait err handler finish or trigger err recovery */ 9849 if (!ufshcd_eh_in_progress(hba)) 9850 ufshcd_force_error_recovery(hba); 9851 ret = -EBUSY; 9852 goto enable_scaling; 9853 } 9854 9855 if (pm_op == UFS_RUNTIME_PM) { 9856 if (ufshcd_can_autobkops_during_suspend(hba)) { 9857 /* 9858 * The device is idle with no requests in the queue, 9859 * allow background operations if bkops status shows 9860 * that performance might be impacted. 9861 */ 9862 ret = ufshcd_bkops_ctrl(hba); 9863 if (ret) { 9864 /* 9865 * If return err in suspend flow, IO will hang. 9866 * Trigger error handler and break suspend for 9867 * error recovery. 9868 */ 9869 ufshcd_force_error_recovery(hba); 9870 ret = -EBUSY; 9871 goto enable_scaling; 9872 } 9873 } else { 9874 /* make sure that auto bkops is disabled */ 9875 ufshcd_disable_auto_bkops(hba); 9876 } 9877 /* 9878 * If device needs to do BKOP or WB buffer flush during 9879 * Hibern8, keep device power mode as "active power mode" 9880 * and VCC supply. 9881 */ 9882 hba->dev_info.b_rpm_dev_flush_capable = 9883 hba->auto_bkops_enabled || 9884 (((req_link_state == UIC_LINK_HIBERN8_STATE) || 9885 ((req_link_state == UIC_LINK_ACTIVE_STATE) && 9886 ufshcd_is_auto_hibern8_enabled(hba))) && 9887 ufshcd_wb_need_flush(hba)); 9888 } 9889 9890 flush_work(&hba->eeh_work); 9891 9892 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 9893 if (ret) 9894 goto enable_scaling; 9895 9896 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { 9897 if (pm_op != UFS_RUNTIME_PM) 9898 /* ensure that bkops is disabled */ 9899 ufshcd_disable_auto_bkops(hba); 9900 9901 if (!hba->dev_info.b_rpm_dev_flush_capable) { 9902 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 9903 if (ret && pm_op != UFS_SHUTDOWN_PM) { 9904 /* 9905 * If return err in suspend flow, IO will hang. 9906 * Trigger error handler and break suspend for 9907 * error recovery. 9908 */ 9909 ufshcd_force_error_recovery(hba); 9910 ret = -EBUSY; 9911 } 9912 if (ret) 9913 goto enable_scaling; 9914 } 9915 } 9916 9917 /* 9918 * In the case of DeepSleep, the device is expected to remain powered 9919 * with the link off, so do not check for bkops. 9920 */ 9921 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); 9922 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); 9923 if (ret && pm_op != UFS_SHUTDOWN_PM) { 9924 /* 9925 * If return err in suspend flow, IO will hang. 9926 * Trigger error handler and break suspend for 9927 * error recovery. 9928 */ 9929 ufshcd_force_error_recovery(hba); 9930 ret = -EBUSY; 9931 } 9932 if (ret) 9933 goto set_dev_active; 9934 9935 vops_suspend: 9936 /* 9937 * Call vendor specific suspend callback. As these callbacks may access 9938 * vendor specific host controller register space call them before the 9939 * host clocks are ON. 9940 */ 9941 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 9942 if (ret) 9943 goto set_link_active; 9944 9945 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 9946 goto out; 9947 9948 set_link_active: 9949 /* 9950 * Device hardware reset is required to exit DeepSleep. Also, for 9951 * DeepSleep, the link is off so host reset and restore will be done 9952 * further below. 9953 */ 9954 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 9955 ufshcd_device_reset(hba); 9956 WARN_ON(!ufshcd_is_link_off(hba)); 9957 } 9958 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 9959 ufshcd_set_link_active(hba); 9960 else if (ufshcd_is_link_off(hba)) 9961 ufshcd_host_reset_and_restore(hba); 9962 set_dev_active: 9963 /* Can also get here needing to exit DeepSleep */ 9964 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 9965 ufshcd_device_reset(hba); 9966 ufshcd_host_reset_and_restore(hba); 9967 } 9968 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 9969 ufshcd_disable_auto_bkops(hba); 9970 enable_scaling: 9971 if (ufshcd_is_clkscaling_supported(hba)) 9972 ufshcd_clk_scaling_suspend(hba, false); 9973 9974 hba->dev_info.b_rpm_dev_flush_capable = false; 9975 out: 9976 if (hba->dev_info.b_rpm_dev_flush_capable) { 9977 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, 9978 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS)); 9979 } 9980 9981 if (ret) { 9982 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); 9983 hba->clk_gating.is_suspended = false; 9984 ufshcd_release(hba); 9985 } 9986 hba->pm_op_in_progress = false; 9987 return ret; 9988 } 9989 9990 #ifdef CONFIG_PM 9991 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9992 { 9993 int ret; 9994 enum uic_link_state old_link_state = hba->uic_link_state; 9995 9996 hba->pm_op_in_progress = true; 9997 9998 /* 9999 * Call vendor specific resume callback. As these callbacks may access 10000 * vendor specific host controller register space call them when the 10001 * host clocks are ON. 10002 */ 10003 ret = ufshcd_vops_resume(hba, pm_op); 10004 if (ret) 10005 goto out; 10006 10007 /* For DeepSleep, the only supported option is to have the link off */ 10008 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); 10009 10010 if (ufshcd_is_link_hibern8(hba)) { 10011 ret = ufshcd_uic_hibern8_exit(hba); 10012 if (!ret) { 10013 ufshcd_set_link_active(hba); 10014 } else { 10015 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 10016 __func__, ret); 10017 goto vendor_suspend; 10018 } 10019 } else if (ufshcd_is_link_off(hba)) { 10020 /* 10021 * A full initialization of the host and the device is 10022 * required since the link was put to off during suspend. 10023 * Note, in the case of DeepSleep, the device will exit 10024 * DeepSleep due to device reset. 10025 */ 10026 ret = ufshcd_reset_and_restore(hba); 10027 /* 10028 * ufshcd_reset_and_restore() should have already 10029 * set the link state as active 10030 */ 10031 if (ret || !ufshcd_is_link_active(hba)) 10032 goto vendor_suspend; 10033 } 10034 10035 if (!ufshcd_is_ufs_dev_active(hba)) { 10036 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 10037 if (ret) 10038 goto set_old_link_state; 10039 ufshcd_set_timestamp_attr(hba); 10040 schedule_delayed_work(&hba->ufs_rtc_update_work, 10041 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 10042 } 10043 10044 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) 10045 ufshcd_enable_auto_bkops(hba); 10046 else 10047 /* 10048 * If BKOPs operations are urgently needed at this moment then 10049 * keep auto-bkops enabled or else disable it. 10050 */ 10051 ufshcd_bkops_ctrl(hba); 10052 10053 if (hba->ee_usr_mask) 10054 ufshcd_write_ee_control(hba); 10055 10056 if (ufshcd_is_clkscaling_supported(hba)) 10057 ufshcd_clk_scaling_suspend(hba, false); 10058 10059 if (hba->dev_info.b_rpm_dev_flush_capable) { 10060 hba->dev_info.b_rpm_dev_flush_capable = false; 10061 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); 10062 } 10063 10064 ufshcd_configure_auto_hibern8(hba); 10065 10066 goto out; 10067 10068 set_old_link_state: 10069 ufshcd_link_state_transition(hba, old_link_state, 0); 10070 vendor_suspend: 10071 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10072 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10073 out: 10074 if (ret) 10075 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret); 10076 hba->clk_gating.is_suspended = false; 10077 ufshcd_release(hba); 10078 hba->pm_op_in_progress = false; 10079 return ret; 10080 } 10081 10082 static int ufshcd_wl_runtime_suspend(struct device *dev) 10083 { 10084 struct scsi_device *sdev = to_scsi_device(dev); 10085 struct ufs_hba *hba; 10086 int ret; 10087 ktime_t start = ktime_get(); 10088 10089 hba = shost_priv(sdev->host); 10090 10091 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM); 10092 if (ret) 10093 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10094 10095 trace_ufshcd_wl_runtime_suspend(hba, ret, 10096 ktime_to_us(ktime_sub(ktime_get(), start)), 10097 hba->curr_dev_pwr_mode, hba->uic_link_state); 10098 10099 return ret; 10100 } 10101 10102 static int ufshcd_wl_runtime_resume(struct device *dev) 10103 { 10104 struct scsi_device *sdev = to_scsi_device(dev); 10105 struct ufs_hba *hba; 10106 int ret = 0; 10107 ktime_t start = ktime_get(); 10108 10109 hba = shost_priv(sdev->host); 10110 10111 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM); 10112 if (ret) 10113 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10114 10115 trace_ufshcd_wl_runtime_resume(hba, ret, 10116 ktime_to_us(ktime_sub(ktime_get(), start)), 10117 hba->curr_dev_pwr_mode, hba->uic_link_state); 10118 10119 return ret; 10120 } 10121 #endif 10122 10123 #ifdef CONFIG_PM_SLEEP 10124 static int ufshcd_wl_suspend(struct device *dev) 10125 { 10126 struct scsi_device *sdev = to_scsi_device(dev); 10127 struct ufs_hba *hba; 10128 int ret = 0; 10129 ktime_t start = ktime_get(); 10130 10131 hba = shost_priv(sdev->host); 10132 down(&hba->host_sem); 10133 hba->system_suspending = true; 10134 10135 if (pm_runtime_suspended(dev)) 10136 goto out; 10137 10138 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM); 10139 if (ret) { 10140 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10141 up(&hba->host_sem); 10142 } 10143 10144 out: 10145 if (!ret) 10146 hba->is_sys_suspended = true; 10147 trace_ufshcd_wl_suspend(hba, ret, 10148 ktime_to_us(ktime_sub(ktime_get(), start)), 10149 hba->curr_dev_pwr_mode, hba->uic_link_state); 10150 10151 return ret; 10152 } 10153 10154 static int ufshcd_wl_resume(struct device *dev) 10155 { 10156 struct scsi_device *sdev = to_scsi_device(dev); 10157 struct ufs_hba *hba; 10158 int ret = 0; 10159 ktime_t start = ktime_get(); 10160 10161 hba = shost_priv(sdev->host); 10162 10163 if (pm_runtime_suspended(dev)) 10164 goto out; 10165 10166 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM); 10167 if (ret) 10168 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10169 out: 10170 trace_ufshcd_wl_resume(hba, ret, 10171 ktime_to_us(ktime_sub(ktime_get(), start)), 10172 hba->curr_dev_pwr_mode, hba->uic_link_state); 10173 if (!ret) 10174 hba->is_sys_suspended = false; 10175 hba->system_suspending = false; 10176 up(&hba->host_sem); 10177 return ret; 10178 } 10179 #endif 10180 10181 /** 10182 * ufshcd_suspend - helper function for suspend operations 10183 * @hba: per adapter instance 10184 * 10185 * This function will put disable irqs, turn off clocks 10186 * and set vreg and hba-vreg in lpm mode. 10187 * 10188 * Return: 0 upon success; < 0 upon failure. 10189 */ 10190 static int ufshcd_suspend(struct ufs_hba *hba) 10191 { 10192 int ret; 10193 10194 if (!hba->is_powered) 10195 return 0; 10196 /* 10197 * Disable the host irq as host controller as there won't be any 10198 * host controller transaction expected till resume. 10199 */ 10200 ufshcd_disable_irq(hba); 10201 ret = ufshcd_setup_clocks(hba, false); 10202 if (ret) { 10203 ufshcd_enable_irq(hba); 10204 return ret; 10205 } 10206 if (ufshcd_is_clkgating_allowed(hba)) { 10207 hba->clk_gating.state = CLKS_OFF; 10208 trace_ufshcd_clk_gating(hba, 10209 hba->clk_gating.state); 10210 } 10211 10212 ufshcd_vreg_set_lpm(hba); 10213 /* Put the host controller in low power mode if possible */ 10214 ufshcd_hba_vreg_set_lpm(hba); 10215 ufshcd_pm_qos_update(hba, false); 10216 return ret; 10217 } 10218 10219 #ifdef CONFIG_PM 10220 /** 10221 * ufshcd_resume - helper function for resume operations 10222 * @hba: per adapter instance 10223 * 10224 * This function basically turns on the regulators, clocks and 10225 * irqs of the hba. 10226 * 10227 * Return: 0 for success and non-zero for failure. 10228 */ 10229 static int ufshcd_resume(struct ufs_hba *hba) 10230 { 10231 int ret; 10232 10233 if (!hba->is_powered) 10234 return 0; 10235 10236 ufshcd_hba_vreg_set_hpm(hba); 10237 ret = ufshcd_vreg_set_hpm(hba); 10238 if (ret) 10239 goto out; 10240 10241 /* Make sure clocks are enabled before accessing controller */ 10242 ret = ufshcd_setup_clocks(hba, true); 10243 if (ret) 10244 goto disable_vreg; 10245 10246 /* enable the host irq as host controller would be active soon */ 10247 ufshcd_enable_irq(hba); 10248 10249 goto out; 10250 10251 disable_vreg: 10252 ufshcd_vreg_set_lpm(hba); 10253 out: 10254 if (ret) 10255 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); 10256 return ret; 10257 } 10258 #endif /* CONFIG_PM */ 10259 10260 #ifdef CONFIG_PM_SLEEP 10261 /** 10262 * ufshcd_system_suspend - system suspend callback 10263 * @dev: Device associated with the UFS controller. 10264 * 10265 * Executed before putting the system into a sleep state in which the contents 10266 * of main memory are preserved. 10267 * 10268 * Return: 0 for success and non-zero for failure. 10269 */ 10270 int ufshcd_system_suspend(struct device *dev) 10271 { 10272 struct ufs_hba *hba = dev_get_drvdata(dev); 10273 int ret = 0; 10274 ktime_t start = ktime_get(); 10275 10276 if (pm_runtime_suspended(hba->dev)) 10277 goto out; 10278 10279 ret = ufshcd_suspend(hba); 10280 out: 10281 trace_ufshcd_system_suspend(hba, ret, 10282 ktime_to_us(ktime_sub(ktime_get(), start)), 10283 hba->curr_dev_pwr_mode, hba->uic_link_state); 10284 return ret; 10285 } 10286 EXPORT_SYMBOL(ufshcd_system_suspend); 10287 10288 /** 10289 * ufshcd_system_resume - system resume callback 10290 * @dev: Device associated with the UFS controller. 10291 * 10292 * Executed after waking the system up from a sleep state in which the contents 10293 * of main memory were preserved. 10294 * 10295 * Return: 0 for success and non-zero for failure. 10296 */ 10297 int ufshcd_system_resume(struct device *dev) 10298 { 10299 struct ufs_hba *hba = dev_get_drvdata(dev); 10300 ktime_t start = ktime_get(); 10301 int ret = 0; 10302 10303 if (pm_runtime_suspended(hba->dev)) 10304 goto out; 10305 10306 ret = ufshcd_resume(hba); 10307 10308 out: 10309 trace_ufshcd_system_resume(hba, ret, 10310 ktime_to_us(ktime_sub(ktime_get(), start)), 10311 hba->curr_dev_pwr_mode, hba->uic_link_state); 10312 10313 return ret; 10314 } 10315 EXPORT_SYMBOL(ufshcd_system_resume); 10316 #endif /* CONFIG_PM_SLEEP */ 10317 10318 #ifdef CONFIG_PM 10319 /** 10320 * ufshcd_runtime_suspend - runtime suspend callback 10321 * @dev: Device associated with the UFS controller. 10322 * 10323 * Check the description of ufshcd_suspend() function for more details. 10324 * 10325 * Return: 0 for success and non-zero for failure. 10326 */ 10327 int ufshcd_runtime_suspend(struct device *dev) 10328 { 10329 struct ufs_hba *hba = dev_get_drvdata(dev); 10330 int ret; 10331 ktime_t start = ktime_get(); 10332 10333 ret = ufshcd_suspend(hba); 10334 10335 trace_ufshcd_runtime_suspend(hba, ret, 10336 ktime_to_us(ktime_sub(ktime_get(), start)), 10337 hba->curr_dev_pwr_mode, hba->uic_link_state); 10338 return ret; 10339 } 10340 EXPORT_SYMBOL(ufshcd_runtime_suspend); 10341 10342 /** 10343 * ufshcd_runtime_resume - runtime resume routine 10344 * @dev: Device associated with the UFS controller. 10345 * 10346 * This function basically brings controller 10347 * to active state. Following operations are done in this function: 10348 * 10349 * 1. Turn on all the controller related clocks 10350 * 2. Turn ON VCC rail 10351 * 10352 * Return: 0 upon success; < 0 upon failure. 10353 */ 10354 int ufshcd_runtime_resume(struct device *dev) 10355 { 10356 struct ufs_hba *hba = dev_get_drvdata(dev); 10357 int ret; 10358 ktime_t start = ktime_get(); 10359 10360 ret = ufshcd_resume(hba); 10361 10362 trace_ufshcd_runtime_resume(hba, ret, 10363 ktime_to_us(ktime_sub(ktime_get(), start)), 10364 hba->curr_dev_pwr_mode, hba->uic_link_state); 10365 return ret; 10366 } 10367 EXPORT_SYMBOL(ufshcd_runtime_resume); 10368 #endif /* CONFIG_PM */ 10369 10370 static void ufshcd_wl_shutdown(struct device *dev) 10371 { 10372 struct scsi_device *sdev = to_scsi_device(dev); 10373 struct ufs_hba *hba = shost_priv(sdev->host); 10374 10375 down(&hba->host_sem); 10376 hba->shutting_down = true; 10377 up(&hba->host_sem); 10378 10379 /* Turn on everything while shutting down */ 10380 ufshcd_rpm_get_sync(hba); 10381 scsi_device_quiesce(sdev); 10382 shost_for_each_device(sdev, hba->host) { 10383 if (sdev == hba->ufs_device_wlun) 10384 continue; 10385 mutex_lock(&sdev->state_mutex); 10386 scsi_device_set_state(sdev, SDEV_OFFLINE); 10387 mutex_unlock(&sdev->state_mutex); 10388 } 10389 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10390 10391 /* 10392 * Next, turn off the UFS controller and the UFS regulators. Disable 10393 * clocks. 10394 */ 10395 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 10396 ufshcd_suspend(hba); 10397 10398 hba->is_powered = false; 10399 } 10400 10401 /** 10402 * ufshcd_remove - de-allocate SCSI host and host memory space 10403 * data structure memory 10404 * @hba: per adapter instance 10405 */ 10406 void ufshcd_remove(struct ufs_hba *hba) 10407 { 10408 if (hba->ufs_device_wlun) 10409 ufshcd_rpm_get_sync(hba); 10410 ufs_hwmon_remove(hba); 10411 ufs_bsg_remove(hba); 10412 ufs_sysfs_remove_nodes(hba->dev); 10413 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10414 blk_mq_destroy_queue(hba->tmf_queue); 10415 blk_put_queue(hba->tmf_queue); 10416 blk_mq_free_tag_set(&hba->tmf_tag_set); 10417 if (hba->scsi_host_added) 10418 scsi_remove_host(hba->host); 10419 /* disable interrupts */ 10420 ufshcd_disable_intr(hba, hba->intr_mask); 10421 ufshcd_hba_stop(hba); 10422 ufshcd_hba_exit(hba); 10423 } 10424 EXPORT_SYMBOL_GPL(ufshcd_remove); 10425 10426 #ifdef CONFIG_PM_SLEEP 10427 int ufshcd_system_freeze(struct device *dev) 10428 { 10429 10430 return ufshcd_system_suspend(dev); 10431 10432 } 10433 EXPORT_SYMBOL_GPL(ufshcd_system_freeze); 10434 10435 int ufshcd_system_restore(struct device *dev) 10436 { 10437 10438 struct ufs_hba *hba = dev_get_drvdata(dev); 10439 int ret; 10440 10441 ret = ufshcd_system_resume(dev); 10442 if (ret) 10443 return ret; 10444 10445 /* Configure UTRL and UTMRL base address registers */ 10446 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 10447 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 10448 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 10449 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 10450 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 10451 REG_UTP_TASK_REQ_LIST_BASE_L); 10452 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 10453 REG_UTP_TASK_REQ_LIST_BASE_H); 10454 /* 10455 * Make sure that UTRL and UTMRL base address registers 10456 * are updated with the latest queue addresses. Only after 10457 * updating these addresses, we can queue the new commands. 10458 */ 10459 ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); 10460 10461 return 0; 10462 10463 } 10464 EXPORT_SYMBOL_GPL(ufshcd_system_restore); 10465 10466 int ufshcd_system_thaw(struct device *dev) 10467 { 10468 return ufshcd_system_resume(dev); 10469 } 10470 EXPORT_SYMBOL_GPL(ufshcd_system_thaw); 10471 #endif /* CONFIG_PM_SLEEP */ 10472 10473 /** 10474 * ufshcd_set_dma_mask - Set dma mask based on the controller 10475 * addressing capability 10476 * @hba: per adapter instance 10477 * 10478 * Return: 0 for success, non-zero for failure. 10479 */ 10480 static int ufshcd_set_dma_mask(struct ufs_hba *hba) 10481 { 10482 if (hba->vops && hba->vops->set_dma_mask) 10483 return hba->vops->set_dma_mask(hba); 10484 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 10485 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 10486 return 0; 10487 } 10488 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 10489 } 10490 10491 /** 10492 * ufshcd_devres_release - devres cleanup handler, invoked during release of 10493 * hba->dev 10494 * @host: pointer to SCSI host 10495 */ 10496 static void ufshcd_devres_release(void *host) 10497 { 10498 scsi_host_put(host); 10499 } 10500 10501 /** 10502 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 10503 * @dev: pointer to device handle 10504 * @hba_handle: driver private handle 10505 * 10506 * Return: 0 on success, non-zero value on failure. 10507 * 10508 * NOTE: There is no corresponding ufshcd_dealloc_host() because this function 10509 * keeps track of its allocations using devres and deallocates everything on 10510 * device removal automatically. 10511 */ 10512 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 10513 { 10514 struct Scsi_Host *host; 10515 struct ufs_hba *hba; 10516 int err = 0; 10517 10518 if (!dev) { 10519 dev_err(dev, 10520 "Invalid memory reference for dev is NULL\n"); 10521 err = -ENODEV; 10522 goto out_error; 10523 } 10524 10525 host = scsi_host_alloc(&ufshcd_driver_template, 10526 sizeof(struct ufs_hba)); 10527 if (!host) { 10528 dev_err(dev, "scsi_host_alloc failed\n"); 10529 err = -ENOMEM; 10530 goto out_error; 10531 } 10532 10533 err = devm_add_action_or_reset(dev, ufshcd_devres_release, 10534 host); 10535 if (err) 10536 return err; 10537 10538 host->nr_maps = HCTX_TYPE_POLL + 1; 10539 hba = shost_priv(host); 10540 hba->host = host; 10541 hba->dev = dev; 10542 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; 10543 hba->nop_out_timeout = NOP_OUT_TIMEOUT; 10544 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); 10545 INIT_LIST_HEAD(&hba->clk_list_head); 10546 spin_lock_init(&hba->outstanding_lock); 10547 10548 *hba_handle = hba; 10549 10550 out_error: 10551 return err; 10552 } 10553 EXPORT_SYMBOL(ufshcd_alloc_host); 10554 10555 /* This function exists because blk_mq_alloc_tag_set() requires this. */ 10556 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx, 10557 const struct blk_mq_queue_data *qd) 10558 { 10559 WARN_ON_ONCE(true); 10560 return BLK_STS_NOTSUPP; 10561 } 10562 10563 static const struct blk_mq_ops ufshcd_tmf_ops = { 10564 .queue_rq = ufshcd_queue_tmf, 10565 }; 10566 10567 static int ufshcd_add_scsi_host(struct ufs_hba *hba) 10568 { 10569 int err; 10570 10571 if (is_mcq_supported(hba)) { 10572 ufshcd_mcq_enable(hba); 10573 err = ufshcd_alloc_mcq(hba); 10574 if (!err) { 10575 ufshcd_config_mcq(hba); 10576 } else { 10577 /* Continue with SDB mode */ 10578 ufshcd_mcq_disable(hba); 10579 use_mcq_mode = false; 10580 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", 10581 err); 10582 } 10583 } 10584 if (!is_mcq_supported(hba) && !hba->lsdb_sup) { 10585 dev_err(hba->dev, 10586 "%s: failed to initialize (legacy doorbell mode not supported)\n", 10587 __func__); 10588 return -EINVAL; 10589 } 10590 10591 err = scsi_add_host(hba->host, hba->dev); 10592 if (err) { 10593 dev_err(hba->dev, "scsi_add_host failed\n"); 10594 return err; 10595 } 10596 hba->scsi_host_added = true; 10597 10598 hba->tmf_tag_set = (struct blk_mq_tag_set) { 10599 .nr_hw_queues = 1, 10600 .queue_depth = hba->nutmrs, 10601 .ops = &ufshcd_tmf_ops, 10602 }; 10603 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); 10604 if (err < 0) 10605 goto remove_scsi_host; 10606 hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); 10607 if (IS_ERR(hba->tmf_queue)) { 10608 err = PTR_ERR(hba->tmf_queue); 10609 goto free_tmf_tag_set; 10610 } 10611 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, 10612 sizeof(*hba->tmf_rqs), GFP_KERNEL); 10613 if (!hba->tmf_rqs) { 10614 err = -ENOMEM; 10615 goto free_tmf_queue; 10616 } 10617 10618 return 0; 10619 10620 free_tmf_queue: 10621 blk_mq_destroy_queue(hba->tmf_queue); 10622 blk_put_queue(hba->tmf_queue); 10623 10624 free_tmf_tag_set: 10625 blk_mq_free_tag_set(&hba->tmf_tag_set); 10626 10627 remove_scsi_host: 10628 if (hba->scsi_host_added) 10629 scsi_remove_host(hba->host); 10630 10631 return err; 10632 } 10633 10634 /** 10635 * ufshcd_init - Driver initialization routine 10636 * @hba: per-adapter instance 10637 * @mmio_base: base register address 10638 * @irq: Interrupt line of device 10639 * 10640 * Return: 0 on success, non-zero value on failure. 10641 */ 10642 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 10643 { 10644 int err; 10645 struct Scsi_Host *host = hba->host; 10646 struct device *dev = hba->dev; 10647 10648 /* 10649 * dev_set_drvdata() must be called before any callbacks are registered 10650 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon, 10651 * sysfs). 10652 */ 10653 dev_set_drvdata(dev, hba); 10654 10655 if (!mmio_base) { 10656 dev_err(hba->dev, 10657 "Invalid memory reference for mmio_base is NULL\n"); 10658 err = -ENODEV; 10659 goto out_error; 10660 } 10661 10662 hba->mmio_base = mmio_base; 10663 hba->irq = irq; 10664 hba->vps = &ufs_hba_vps; 10665 10666 /* 10667 * Initialize clk_gating.lock early since it is being used in 10668 * ufshcd_setup_clocks() 10669 */ 10670 spin_lock_init(&hba->clk_gating.lock); 10671 10672 /* 10673 * Set the default power management level for runtime and system PM. 10674 * Host controller drivers can override them in their 10675 * 'ufs_hba_variant_ops::init' callback. 10676 * 10677 * Default power saving mode is to keep UFS link in Hibern8 state 10678 * and UFS device in sleep state. 10679 */ 10680 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10681 UFS_SLEEP_PWR_MODE, 10682 UIC_LINK_HIBERN8_STATE); 10683 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10684 UFS_SLEEP_PWR_MODE, 10685 UIC_LINK_HIBERN8_STATE); 10686 10687 init_completion(&hba->dev_cmd.complete); 10688 10689 err = ufshcd_hba_init(hba); 10690 if (err) 10691 goto out_error; 10692 10693 /* Read capabilities registers */ 10694 err = ufshcd_hba_capabilities(hba); 10695 if (err) 10696 goto out_disable; 10697 10698 /* Get UFS version supported by the controller */ 10699 hba->ufs_version = ufshcd_get_ufs_version(hba); 10700 10701 /* Get Interrupt bit mask per version */ 10702 hba->intr_mask = ufshcd_get_intr_mask(hba); 10703 10704 err = ufshcd_set_dma_mask(hba); 10705 if (err) { 10706 dev_err(hba->dev, "set dma mask failed\n"); 10707 goto out_disable; 10708 } 10709 10710 /* Allocate memory for host memory space */ 10711 err = ufshcd_memory_alloc(hba); 10712 if (err) { 10713 dev_err(hba->dev, "Memory allocation failed\n"); 10714 goto out_disable; 10715 } 10716 10717 /* Configure LRB */ 10718 ufshcd_host_memory_configure(hba); 10719 10720 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 10721 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; 10722 host->max_id = UFSHCD_MAX_ID; 10723 host->max_lun = UFS_MAX_LUNS; 10724 host->max_channel = UFSHCD_MAX_CHANNEL; 10725 host->unique_id = host->host_no; 10726 host->max_cmd_len = UFS_CDB_SIZE; 10727 host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); 10728 10729 /* Use default RPM delay if host not set */ 10730 if (host->rpm_autosuspend_delay == 0) 10731 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; 10732 10733 hba->max_pwr_info.is_valid = false; 10734 10735 /* Initialize work queues */ 10736 hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, 10737 hba->host->host_no); 10738 if (!hba->eh_wq) { 10739 dev_err(hba->dev, "%s: failed to create eh workqueue\n", 10740 __func__); 10741 err = -ENOMEM; 10742 goto out_disable; 10743 } 10744 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 10745 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 10746 10747 sema_init(&hba->host_sem, 1); 10748 10749 /* Initialize UIC command mutex */ 10750 mutex_init(&hba->uic_cmd_mutex); 10751 10752 /* Initialize mutex for device management commands */ 10753 mutex_init(&hba->dev_cmd.lock); 10754 10755 /* Initialize mutex for exception event control */ 10756 mutex_init(&hba->ee_ctrl_mutex); 10757 10758 mutex_init(&hba->wb_mutex); 10759 init_rwsem(&hba->clk_scaling_lock); 10760 10761 ufshcd_init_clk_gating(hba); 10762 10763 ufshcd_init_clk_scaling(hba); 10764 10765 /* 10766 * In order to avoid any spurious interrupt immediately after 10767 * registering UFS controller interrupt handler, clear any pending UFS 10768 * interrupt status and disable all the UFS interrupts. 10769 */ 10770 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), 10771 REG_INTERRUPT_STATUS); 10772 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); 10773 /* 10774 * Make sure that UFS interrupts are disabled and any pending interrupt 10775 * status is cleared before registering UFS interrupt handler. 10776 */ 10777 ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 10778 10779 /* IRQ registration */ 10780 err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, 10781 IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); 10782 if (err) { 10783 dev_err(hba->dev, "request irq failed\n"); 10784 goto out_disable; 10785 } else { 10786 hba->is_irq_enabled = true; 10787 } 10788 10789 /* Reset the attached device */ 10790 ufshcd_device_reset(hba); 10791 10792 ufshcd_init_crypto(hba); 10793 10794 /* Host controller enable */ 10795 err = ufshcd_hba_enable(hba); 10796 if (err) { 10797 dev_err(hba->dev, "Host controller enable failed\n"); 10798 ufshcd_print_evt_hist(hba); 10799 ufshcd_print_host_state(hba); 10800 goto out_disable; 10801 } 10802 10803 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); 10804 INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); 10805 10806 /* Set the default auto-hiberate idle timer value to 150 ms */ 10807 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { 10808 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) | 10809 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 10810 } 10811 10812 /* Hold auto suspend until async scan completes */ 10813 pm_runtime_get_sync(dev); 10814 10815 /* 10816 * We are assuming that device wasn't put in sleep/power-down 10817 * state exclusively during the boot stage before kernel. 10818 * This assumption helps avoid doing link startup twice during 10819 * ufshcd_probe_hba(). 10820 */ 10821 ufshcd_set_ufs_dev_active(hba); 10822 10823 /* Initialize hba, detect and initialize UFS device */ 10824 ktime_t probe_start = ktime_get(); 10825 10826 hba->ufshcd_state = UFSHCD_STATE_RESET; 10827 10828 err = ufshcd_link_startup(hba); 10829 if (err) 10830 goto out_disable; 10831 10832 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 10833 goto initialized; 10834 10835 /* Debug counters initialization */ 10836 ufshcd_clear_dbg_ufs_stats(hba); 10837 10838 /* UniPro link is active now */ 10839 ufshcd_set_link_active(hba); 10840 10841 /* Verify device initialization by sending NOP OUT UPIU */ 10842 err = ufshcd_verify_dev_init(hba); 10843 if (err) 10844 goto out_disable; 10845 10846 /* Initiate UFS initialization, and waiting until completion */ 10847 err = ufshcd_complete_dev_init(hba); 10848 if (err) 10849 goto out_disable; 10850 10851 err = ufshcd_device_params_init(hba); 10852 if (err) 10853 goto out_disable; 10854 10855 err = ufshcd_post_device_init(hba); 10856 10857 initialized: 10858 ufshcd_process_probe_result(hba, probe_start, err); 10859 if (err) 10860 goto out_disable; 10861 10862 err = ufshcd_add_scsi_host(hba); 10863 if (err) 10864 goto out_disable; 10865 10866 async_schedule(ufshcd_async_scan, hba); 10867 ufs_sysfs_add_nodes(hba->dev); 10868 10869 device_enable_async_suspend(dev); 10870 ufshcd_pm_qos_init(hba); 10871 return 0; 10872 10873 out_disable: 10874 hba->is_irq_enabled = false; 10875 ufshcd_hba_exit(hba); 10876 out_error: 10877 return err; 10878 } 10879 EXPORT_SYMBOL_GPL(ufshcd_init); 10880 10881 void ufshcd_resume_complete(struct device *dev) 10882 { 10883 struct ufs_hba *hba = dev_get_drvdata(dev); 10884 10885 if (hba->complete_put) { 10886 ufshcd_rpm_put(hba); 10887 hba->complete_put = false; 10888 } 10889 } 10890 EXPORT_SYMBOL_GPL(ufshcd_resume_complete); 10891 10892 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba) 10893 { 10894 struct device *dev = &hba->ufs_device_wlun->sdev_gendev; 10895 enum ufs_dev_pwr_mode dev_pwr_mode; 10896 enum uic_link_state link_state; 10897 unsigned long flags; 10898 bool res; 10899 10900 spin_lock_irqsave(&dev->power.lock, flags); 10901 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl); 10902 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl); 10903 res = pm_runtime_suspended(dev) && 10904 hba->curr_dev_pwr_mode == dev_pwr_mode && 10905 hba->uic_link_state == link_state && 10906 !hba->dev_info.b_rpm_dev_flush_capable; 10907 spin_unlock_irqrestore(&dev->power.lock, flags); 10908 10909 return res; 10910 } 10911 10912 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm) 10913 { 10914 struct ufs_hba *hba = dev_get_drvdata(dev); 10915 int ret; 10916 10917 /* 10918 * SCSI assumes that runtime-pm and system-pm for scsi drivers 10919 * are same. And it doesn't wake up the device for system-suspend 10920 * if it's runtime suspended. But ufs doesn't follow that. 10921 * Refer ufshcd_resume_complete() 10922 */ 10923 if (hba->ufs_device_wlun) { 10924 /* Prevent runtime suspend */ 10925 ufshcd_rpm_get_noresume(hba); 10926 /* 10927 * Check if already runtime suspended in same state as system 10928 * suspend would be. 10929 */ 10930 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) { 10931 /* RPM state is not ok for SPM, so runtime resume */ 10932 ret = ufshcd_rpm_resume(hba); 10933 if (ret < 0 && ret != -EACCES) { 10934 ufshcd_rpm_put(hba); 10935 return ret; 10936 } 10937 } 10938 hba->complete_put = true; 10939 } 10940 return 0; 10941 } 10942 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare); 10943 10944 int ufshcd_suspend_prepare(struct device *dev) 10945 { 10946 return __ufshcd_suspend_prepare(dev, true); 10947 } 10948 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare); 10949 10950 #ifdef CONFIG_PM_SLEEP 10951 static int ufshcd_wl_poweroff(struct device *dev) 10952 { 10953 struct scsi_device *sdev = to_scsi_device(dev); 10954 struct ufs_hba *hba = shost_priv(sdev->host); 10955 10956 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10957 return 0; 10958 } 10959 #endif 10960 10961 static int ufshcd_wl_probe(struct device *dev) 10962 { 10963 struct scsi_device *sdev = to_scsi_device(dev); 10964 10965 if (!is_device_wlun(sdev)) 10966 return -ENODEV; 10967 10968 blk_pm_runtime_init(sdev->request_queue, dev); 10969 pm_runtime_set_autosuspend_delay(dev, 0); 10970 pm_runtime_allow(dev); 10971 10972 return 0; 10973 } 10974 10975 static int ufshcd_wl_remove(struct device *dev) 10976 { 10977 pm_runtime_forbid(dev); 10978 return 0; 10979 } 10980 10981 static const struct dev_pm_ops ufshcd_wl_pm_ops = { 10982 #ifdef CONFIG_PM_SLEEP 10983 .suspend = ufshcd_wl_suspend, 10984 .resume = ufshcd_wl_resume, 10985 .freeze = ufshcd_wl_suspend, 10986 .thaw = ufshcd_wl_resume, 10987 .poweroff = ufshcd_wl_poweroff, 10988 .restore = ufshcd_wl_resume, 10989 #endif 10990 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) 10991 }; 10992 10993 static void ufshcd_check_header_layout(void) 10994 { 10995 /* 10996 * gcc compilers before version 10 cannot do constant-folding for 10997 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and 10998 * before. 10999 */ 11000 if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) 11001 return; 11002 11003 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11004 .cci = 3})[0] != 3); 11005 11006 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11007 .ehs_length = 2})[1] != 2); 11008 11009 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11010 .enable_crypto = 1})[2] 11011 != 0x80); 11012 11013 BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ 11014 .command_type = 5, 11015 .data_direction = 3, 11016 .interrupt = 1, 11017 })[3]) != ((5 << 4) | (3 << 1) | 1)); 11018 11019 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11020 .dunl = cpu_to_le32(0xdeadbeef)})[1] != 11021 cpu_to_le32(0xdeadbeef)); 11022 11023 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11024 .ocs = 4})[8] != 4); 11025 11026 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11027 .cds = 5})[9] != 5); 11028 11029 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11030 .dunu = cpu_to_le32(0xbadcafe)})[3] != 11031 cpu_to_le32(0xbadcafe)); 11032 11033 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11034 .iid = 0xf })[4] != 0xf0); 11035 11036 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11037 .command_set_type = 0xf })[4] != 0xf); 11038 } 11039 11040 /* 11041 * ufs_dev_wlun_template - describes ufs device wlun 11042 * ufs-device wlun - used to send pm commands 11043 * All luns are consumers of ufs-device wlun. 11044 * 11045 * Currently, no sd driver is present for wluns. 11046 * Hence the no specific pm operations are performed. 11047 * With ufs design, SSU should be sent to ufs-device wlun. 11048 * Hence register a scsi driver for ufs wluns only. 11049 */ 11050 static struct scsi_driver ufs_dev_wlun_template = { 11051 .gendrv = { 11052 .name = "ufs_device_wlun", 11053 .probe = ufshcd_wl_probe, 11054 .remove = ufshcd_wl_remove, 11055 .pm = &ufshcd_wl_pm_ops, 11056 .shutdown = ufshcd_wl_shutdown, 11057 }, 11058 }; 11059 11060 static int __init ufshcd_core_init(void) 11061 { 11062 int ret; 11063 11064 ufshcd_check_header_layout(); 11065 11066 ufs_debugfs_init(); 11067 11068 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); 11069 if (ret) 11070 ufs_debugfs_exit(); 11071 return ret; 11072 } 11073 11074 static void __exit ufshcd_core_exit(void) 11075 { 11076 ufs_debugfs_exit(); 11077 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv); 11078 } 11079 11080 module_init(ufshcd_core_init); 11081 module_exit(ufshcd_core_exit); 11082 11083 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 11084 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 11085 MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 11086 MODULE_SOFTDEP("pre: governor_simpleondemand"); 11087 MODULE_LICENSE("GPL"); 11088