1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 3 #ifndef _UFSHCD_PRIV_H_ 4 #define _UFSHCD_PRIV_H_ 5 6 #include <linux/pm_runtime.h> 7 #include <ufs/ufshcd.h> 8 9 void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs); 10 11 static inline bool ufshcd_is_user_access_allowed(struct ufs_hba *hba) 12 { 13 return !hba->shutting_down; 14 } 15 16 void ufshcd_schedule_eh_work(struct ufs_hba *hba); 17 18 static inline bool ufshcd_keep_autobkops_enabled_except_suspend( 19 struct ufs_hba *hba) 20 { 21 return hba->caps & UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND; 22 } 23 24 static inline u8 ufshcd_wb_get_query_index(struct ufs_hba *hba) 25 { 26 if (hba->dev_info.wb_buffer_type == WB_BUF_MODE_LU_DEDICATED) 27 return hba->dev_info.wb_dedicated_lu; 28 return 0; 29 } 30 31 static inline bool ufshcd_is_wb_buf_flush_allowed(struct ufs_hba *hba) 32 { 33 return ufshcd_is_wb_allowed(hba) && 34 !(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL); 35 } 36 37 #ifdef CONFIG_SCSI_UFS_HWMON 38 void ufs_hwmon_probe(struct ufs_hba *hba, u8 mask); 39 void ufs_hwmon_remove(struct ufs_hba *hba); 40 void ufs_hwmon_notify_event(struct ufs_hba *hba, u8 ee_mask); 41 #else 42 static inline void ufs_hwmon_probe(struct ufs_hba *hba, u8 mask) {} 43 static inline void ufs_hwmon_remove(struct ufs_hba *hba) {} 44 static inline void ufs_hwmon_notify_event(struct ufs_hba *hba, u8 ee_mask) {} 45 #endif 46 47 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 48 enum query_opcode opcode, 49 enum desc_idn idn, u8 index, 50 u8 selector, 51 u8 *desc_buf, int *buf_len); 52 int ufshcd_read_desc_param(struct ufs_hba *hba, 53 enum desc_idn desc_id, 54 int desc_index, 55 u8 param_offset, 56 u8 *param_read_buf, 57 u8 param_size); 58 int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, 59 enum attr_idn idn, u8 index, u8 selector, 60 u32 *attr_val); 61 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 62 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val); 63 int ufshcd_query_attr_qword(struct ufs_hba *hba, enum query_opcode opcode, 64 enum attr_idn idn, u8 index, u8 sel, u64 *attr_val); 65 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 66 enum flag_idn idn, u8 index, bool *flag_res); 67 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit); 68 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 69 struct cq_entry *cqe); 70 int ufshcd_mcq_init(struct ufs_hba *hba); 71 void ufshcd_mcq_disable(struct ufs_hba *hba); 72 int ufshcd_get_hba_mac(struct ufs_hba *hba); 73 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba); 74 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, 75 struct request *req); 76 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, 77 struct ufs_hw_queue *hwq); 78 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd); 79 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag); 80 int ufshcd_mcq_abort(struct scsi_cmnd *cmd); 81 u32 ufshcd_mcq_read_mcqiacr(struct ufs_hba *hba, int i); 82 void ufshcd_mcq_write_mcqiacr(struct ufs_hba *hba, u32 val, int i); 83 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); 84 void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd); 85 int ufshcd_pause_command_processing(struct ufs_hba *hba, u64 timeout_us); 86 void ufshcd_resume_command_processing(struct ufs_hba *hba); 87 int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, bool scale_up); 88 89 /** 90 * enum ufs_descr_fmt - UFS string descriptor format 91 * @SD_RAW: Raw UTF-16 format 92 * @SD_ASCII_STD: Convert to null-terminated ASCII string 93 */ 94 enum ufs_descr_fmt { 95 SD_RAW = 0, 96 SD_ASCII_STD = 1, 97 }; 98 99 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt); 100 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd); 101 int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd); 102 103 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 104 struct utp_upiu_req *req_upiu, 105 struct utp_upiu_req *rsp_upiu, 106 enum upiu_request_transaction msgcode, 107 u8 *desc_buff, int *buff_len, 108 enum query_opcode desc_op); 109 110 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable); 111 112 int ufshcd_uic_tx_eqtr(struct ufs_hba *hba, int gear); 113 void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba); 114 int ufshcd_config_tx_eq_settings(struct ufs_hba *hba, 115 struct ufs_pa_layer_attr *pwr_mode, 116 bool force_tx_eqtr); 117 void ufshcd_print_tx_eq_params(struct ufs_hba *hba); 118 bool ufshcd_is_txeq_presets_used(struct ufs_hba *hba); 119 bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis); 120 int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear); 121 void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba); 122 void ufshcd_store_tx_eq_settings(struct ufs_hba *hba); 123 124 /* Wrapper functions for safely calling variant operations */ 125 static inline const char *ufshcd_get_var_name(struct ufs_hba *hba) 126 { 127 if (hba->vops) 128 return hba->vops->name; 129 return ""; 130 } 131 132 static inline void ufshcd_vops_exit(struct ufs_hba *hba) 133 { 134 if (hba->vops && hba->vops->exit) 135 hba->vops->exit(hba); 136 } 137 138 static inline u32 ufshcd_vops_get_ufs_hci_version(struct ufs_hba *hba) 139 { 140 if (hba->vops && hba->vops->get_ufs_hci_version) 141 return hba->vops->get_ufs_hci_version(hba); 142 143 return ufshcd_readl(hba, REG_UFS_VERSION); 144 } 145 146 static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba, bool up, 147 unsigned long target_freq, 148 enum ufs_notify_change_status status) 149 { 150 if (hba->vops && hba->vops->clk_scale_notify) 151 return hba->vops->clk_scale_notify(hba, up, target_freq, status); 152 return 0; 153 } 154 155 static inline void ufshcd_vops_event_notify(struct ufs_hba *hba, 156 enum ufs_event_type evt, 157 void *data) 158 { 159 if (hba->vops && hba->vops->event_notify) 160 hba->vops->event_notify(hba, evt, data); 161 } 162 163 static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on, 164 enum ufs_notify_change_status status) 165 { 166 if (hba->vops && hba->vops->setup_clocks) 167 return hba->vops->setup_clocks(hba, on, status); 168 return 0; 169 } 170 171 static inline int ufshcd_vops_hce_enable_notify(struct ufs_hba *hba, 172 bool status) 173 { 174 if (hba->vops && hba->vops->hce_enable_notify) 175 return hba->vops->hce_enable_notify(hba, status); 176 177 return 0; 178 } 179 static inline int ufshcd_vops_link_startup_notify(struct ufs_hba *hba, 180 bool status) 181 { 182 if (hba->vops && hba->vops->link_startup_notify) 183 return hba->vops->link_startup_notify(hba, status); 184 185 return 0; 186 } 187 188 static inline int ufshcd_vops_negotiate_pwr_mode(struct ufs_hba *hba, 189 const struct ufs_pa_layer_attr *dev_max_params, 190 struct ufs_pa_layer_attr *dev_req_params) 191 { 192 if (hba->vops && hba->vops->negotiate_pwr_mode) 193 return hba->vops->negotiate_pwr_mode(hba, dev_max_params, 194 dev_req_params); 195 196 return -ENOTSUPP; 197 } 198 199 static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba, 200 enum ufs_notify_change_status status, 201 struct ufs_pa_layer_attr *dev_req_params) 202 { 203 if (hba->vops && hba->vops->pwr_change_notify) 204 return hba->vops->pwr_change_notify(hba, status, 205 dev_req_params); 206 207 return -ENOTSUPP; 208 } 209 210 static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba, 211 int tag, u8 tm_function) 212 { 213 if (hba->vops && hba->vops->setup_task_mgmt) 214 hba->vops->setup_task_mgmt(hba, tag, tm_function); 215 } 216 217 static inline void ufshcd_vops_hibern8_notify(struct ufs_hba *hba, 218 enum uic_cmd_dme cmd, 219 enum ufs_notify_change_status status) 220 { 221 if (hba->vops && hba->vops->hibern8_notify) 222 hba->vops->hibern8_notify(hba, cmd, status); 223 } 224 225 static inline int ufshcd_vops_apply_dev_quirks(struct ufs_hba *hba) 226 { 227 if (hba->vops && hba->vops->apply_dev_quirks) 228 return hba->vops->apply_dev_quirks(hba); 229 return 0; 230 } 231 232 static inline void ufshcd_vops_fixup_dev_quirks(struct ufs_hba *hba) 233 { 234 if (hba->vops && hba->vops->fixup_dev_quirks) 235 hba->vops->fixup_dev_quirks(hba); 236 } 237 238 static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op, 239 enum ufs_notify_change_status status) 240 { 241 if (hba->vops && hba->vops->suspend) 242 return hba->vops->suspend(hba, op, status); 243 244 return 0; 245 } 246 247 static inline int ufshcd_vops_resume(struct ufs_hba *hba, enum ufs_pm_op op) 248 { 249 if (hba->vops && hba->vops->resume) 250 return hba->vops->resume(hba, op); 251 252 return 0; 253 } 254 255 static inline void ufshcd_vops_dbg_register_dump(struct ufs_hba *hba) 256 { 257 if (hba->vops && hba->vops->dbg_register_dump) 258 hba->vops->dbg_register_dump(hba); 259 } 260 261 static inline int ufshcd_vops_device_reset(struct ufs_hba *hba) 262 { 263 if (hba->vops && hba->vops->device_reset) 264 return hba->vops->device_reset(hba); 265 266 return -EOPNOTSUPP; 267 } 268 269 static inline void ufshcd_vops_config_scaling_param(struct ufs_hba *hba, 270 struct devfreq_dev_profile *p, 271 struct devfreq_simple_ondemand_data *data) 272 { 273 if (hba->vops && hba->vops->config_scaling_param) 274 hba->vops->config_scaling_param(hba, p, data); 275 } 276 277 static inline int ufshcd_vops_mcq_config_resource(struct ufs_hba *hba) 278 { 279 if (hba->vops && hba->vops->mcq_config_resource) 280 return hba->vops->mcq_config_resource(hba); 281 282 return -EOPNOTSUPP; 283 } 284 285 static inline int ufshcd_mcq_vops_op_runtime_config(struct ufs_hba *hba) 286 { 287 if (hba->vops && hba->vops->op_runtime_config) 288 return hba->vops->op_runtime_config(hba); 289 290 return -EOPNOTSUPP; 291 } 292 293 static inline int ufshcd_vops_get_outstanding_cqs(struct ufs_hba *hba, 294 unsigned long *ocqs) 295 { 296 if (hba->vops && hba->vops->get_outstanding_cqs) 297 return hba->vops->get_outstanding_cqs(hba, ocqs); 298 299 return -EOPNOTSUPP; 300 } 301 302 static inline int ufshcd_mcq_vops_config_esi(struct ufs_hba *hba) 303 { 304 if (hba->vops && hba->vops->config_esi) 305 return hba->vops->config_esi(hba); 306 307 return -EOPNOTSUPP; 308 } 309 310 static inline u32 ufshcd_vops_freq_to_gear_speed(struct ufs_hba *hba, unsigned long freq) 311 { 312 if (hba->vops && hba->vops->freq_to_gear_speed) 313 return hba->vops->freq_to_gear_speed(hba, freq); 314 315 return 0; 316 } 317 318 static inline int ufshcd_vops_get_rx_fom(struct ufs_hba *hba, 319 struct ufs_pa_layer_attr *pwr_mode, 320 struct tx_eqtr_iter *h_iter, 321 struct tx_eqtr_iter *d_iter) 322 { 323 if (hba->vops && hba->vops->get_rx_fom) 324 return hba->vops->get_rx_fom(hba, pwr_mode, h_iter, d_iter); 325 326 return 0; 327 } 328 329 static inline int ufshcd_vops_apply_tx_eqtr_settings(struct ufs_hba *hba, 330 struct ufs_pa_layer_attr *pwr_mode, 331 struct tx_eqtr_iter *h_iter, 332 struct tx_eqtr_iter *d_iter) 333 { 334 if (hba->vops && hba->vops->apply_tx_eqtr_settings) 335 return hba->vops->apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter); 336 337 return 0; 338 } 339 340 static inline int ufshcd_vops_tx_eqtr_notify(struct ufs_hba *hba, 341 enum ufs_notify_change_status status, 342 struct ufs_pa_layer_attr *pwr_mode) 343 { 344 if (hba->vops && hba->vops->tx_eqtr_notify) 345 return hba->vops->tx_eqtr_notify(hba, status, pwr_mode); 346 347 return 0; 348 } 349 350 extern const struct ufs_pm_lvl_states ufs_pm_lvl_states[]; 351 352 /** 353 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN 354 * @scsi_lun: scsi LUN id 355 * 356 * Return: UPIU LUN id 357 */ 358 static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun) 359 { 360 if (scsi_is_wlun(scsi_lun)) 361 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID) 362 | UFS_UPIU_WLUN_ID; 363 else 364 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID; 365 } 366 367 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask); 368 int ufshcd_write_ee_control(struct ufs_hba *hba); 369 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 370 const u16 *other_mask, u16 set, u16 clr); 371 372 static inline int ufshcd_update_ee_drv_mask(struct ufs_hba *hba, 373 u16 set, u16 clr) 374 { 375 return ufshcd_update_ee_control(hba, &hba->ee_drv_mask, 376 &hba->ee_usr_mask, set, clr); 377 } 378 379 static inline int ufshcd_update_ee_usr_mask(struct ufs_hba *hba, 380 u16 set, u16 clr) 381 { 382 return ufshcd_update_ee_control(hba, &hba->ee_usr_mask, 383 &hba->ee_drv_mask, set, clr); 384 } 385 386 static inline int ufshcd_rpm_get_sync(struct ufs_hba *hba) 387 { 388 return pm_runtime_get_sync(&hba->ufs_device_wlun->sdev_gendev); 389 } 390 391 static inline int ufshcd_rpm_get_if_active(struct ufs_hba *hba) 392 { 393 return pm_runtime_get_if_active(&hba->ufs_device_wlun->sdev_gendev); 394 } 395 396 static inline int ufshcd_rpm_put_sync(struct ufs_hba *hba) 397 { 398 return pm_runtime_put_sync(&hba->ufs_device_wlun->sdev_gendev); 399 } 400 401 static inline void ufshcd_rpm_get_noresume(struct ufs_hba *hba) 402 { 403 pm_runtime_get_noresume(&hba->ufs_device_wlun->sdev_gendev); 404 } 405 406 static inline int ufshcd_rpm_resume(struct ufs_hba *hba) 407 { 408 return pm_runtime_resume(&hba->ufs_device_wlun->sdev_gendev); 409 } 410 411 static inline void ufshcd_rpm_put(struct ufs_hba *hba) 412 { 413 pm_runtime_put(&hba->ufs_device_wlun->sdev_gendev); 414 } 415 416 /** 417 * ufs_is_valid_unit_desc_lun - checks if the given LUN has a unit descriptor 418 * @dev_info: pointer of instance of struct ufs_dev_info 419 * @lun: LU number to check 420 * @return: true if the lun has a matching unit descriptor, false otherwise 421 */ 422 static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8 lun) 423 { 424 if (!dev_info || !dev_info->max_lu_supported) { 425 pr_err("Max General LU supported by UFS isn't initialized\n"); 426 return false; 427 } 428 return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported); 429 } 430 431 /* 432 * Convert a block layer tag into a SCSI command pointer. This function is 433 * called once per I/O completion path and is also called from error paths. 434 */ 435 static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag) 436 { 437 struct blk_mq_tags *tags = hba->host->tag_set.shared_tags; 438 struct request *rq = blk_mq_tag_to_rq(tags, tag); 439 440 if (WARN_ON_ONCE(!rq)) 441 return NULL; 442 443 return blk_mq_rq_to_pdu(rq); 444 } 445 446 static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q) 447 __must_hold(&q->sq_lock) 448 { 449 u32 val; 450 451 q->sq_tail_slot++; 452 if (q->sq_tail_slot == q->max_entries) 453 q->sq_tail_slot = 0; 454 val = q->sq_tail_slot * sizeof(struct utp_transfer_req_desc); 455 writel(val, q->mcq_sq_tail); 456 } 457 458 static inline void ufshcd_mcq_update_cq_tail_slot(struct ufs_hw_queue *q) 459 { 460 u32 val = readl(q->mcq_cq_tail); 461 462 q->cq_tail_slot = val / sizeof(struct cq_entry); 463 } 464 465 static inline bool ufshcd_mcq_is_cq_empty(struct ufs_hw_queue *q) 466 { 467 return q->cq_head_slot == q->cq_tail_slot; 468 } 469 470 static inline void ufshcd_mcq_inc_cq_head_slot(struct ufs_hw_queue *q) 471 { 472 q->cq_head_slot++; 473 if (q->cq_head_slot == q->max_entries) 474 q->cq_head_slot = 0; 475 } 476 477 static inline void ufshcd_mcq_update_cq_head(struct ufs_hw_queue *q) 478 { 479 writel(q->cq_head_slot * sizeof(struct cq_entry), q->mcq_cq_head); 480 } 481 482 static inline struct cq_entry *ufshcd_mcq_cur_cqe(struct ufs_hw_queue *q) 483 { 484 struct cq_entry *cqe = q->cqe_base_addr; 485 486 return cqe + q->cq_head_slot; 487 } 488 489 static inline u32 ufshcd_mcq_get_sq_head_slot(struct ufs_hw_queue *q) 490 { 491 u32 val = readl(q->mcq_sq_head); 492 493 return val / sizeof(struct utp_transfer_req_desc); 494 } 495 496 #if IS_ENABLED(CONFIG_RPMB) 497 int ufs_rpmb_probe(struct ufs_hba *hba); 498 void ufs_rpmb_remove(struct ufs_hba *hba); 499 #else 500 static inline int ufs_rpmb_probe(struct ufs_hba *hba) 501 { 502 return 0; 503 } 504 static inline void ufs_rpmb_remove(struct ufs_hba *hba) 505 { 506 } 507 #endif 508 509 #endif /* _UFSHCD_PRIV_H_ */ 510