1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *); 98 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *); 99 100 static int ahci_initialize_controller(ahci_ctl_t *); 101 static void ahci_uninitialize_controller(ahci_ctl_t *); 102 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 103 static int ahci_config_space_init(ahci_ctl_t *); 104 105 static void ahci_drain_ports_taskq(ahci_ctl_t *); 106 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *, 107 uint8_t); 108 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *); 109 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t); 110 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *, 111 sata_device_t *); 112 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *, 113 sata_pmult_gscr_t *); 114 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 115 sata_device_t *); 116 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 117 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 118 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 119 sata_device_t *); 120 121 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 122 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, 124 ahci_addr_t *); 125 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 126 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 127 ahci_addr_t *, sata_pkt_t *); 128 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 129 ahci_addr_t *, sata_pkt_t *); 130 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, 131 ahci_addr_t *, int); 132 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 133 static void ahci_copy_ncq_err_page(sata_cmd_t *, 134 struct sata_ncq_error_recovery_page *); 135 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 136 137 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 138 static int ahci_hba_reset(ahci_ctl_t *); 139 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 140 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 141 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 142 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, 143 ahci_addr_t *); 144 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *, 145 ahci_addr_t *); 146 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, 147 ahci_addr_t *); 148 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 149 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 150 uint8_t); 151 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 152 uint8_t, int, int *); 153 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 154 uint32_t, uint32_t, uint32_t, uint32_t); 155 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, sata_pkt_t *); 158 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 159 ahci_addr_t *, uint32_t); 160 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 161 uint8_t, uint32_t); 162 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 163 uint8_t, uint32_t); 164 static void ahci_events_handler(void *); 165 static void ahci_watchdog_handler(ahci_ctl_t *); 166 167 static uint_t ahci_intr(caddr_t, caddr_t); 168 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 169 static int ahci_add_intrs(ahci_ctl_t *, int); 170 static void ahci_rem_intrs(ahci_ctl_t *); 171 static void ahci_enable_all_intrs(ahci_ctl_t *); 172 static void ahci_disable_all_intrs(ahci_ctl_t *); 173 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 174 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 175 176 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 177 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 178 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 179 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t); 180 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 181 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 182 ahci_port_t *, uint8_t); 183 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 184 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 185 uint8_t, uint32_t); 186 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 187 uint8_t, uint32_t); 188 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 189 190 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *); 191 static int ahci_get_num_implemented_ports(uint32_t); 192 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 193 uint32_t); 194 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 195 #if AHCI_DEBUG 196 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 197 #endif 198 199 200 /* 201 * DMA attributes for the data buffer 202 * 203 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 204 * does not support 64-bit addressing 205 */ 206 static ddi_dma_attr_t buffer_dma_attr = { 207 DMA_ATTR_V0, /* dma_attr_version */ 208 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 209 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 210 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 211 0x2ull, /* dma_attr_align: word aligned */ 212 1, /* dma_attr_burstsizes */ 213 1, /* dma_attr_minxfer */ 214 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 215 0xffffffffull, /* dma_attr_seg */ 216 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 217 512, /* dma_attr_granular */ 218 0, /* dma_attr_flags */ 219 }; 220 221 /* 222 * DMA attributes for the rcvd FIS 223 * 224 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 225 * does not support 64-bit addressing 226 */ 227 static ddi_dma_attr_t rcvd_fis_dma_attr = { 228 DMA_ATTR_V0, /* dma_attr_version */ 229 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 230 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 231 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 232 0x100ull, /* dma_attr_align: 256-byte aligned */ 233 1, /* dma_attr_burstsizes */ 234 1, /* dma_attr_minxfer */ 235 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 236 0xffffffffull, /* dma_attr_seg */ 237 1, /* dma_attr_sgllen */ 238 1, /* dma_attr_granular */ 239 0, /* dma_attr_flags */ 240 }; 241 242 /* 243 * DMA attributes for the command list 244 * 245 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 246 * does not support 64-bit addressing 247 */ 248 static ddi_dma_attr_t cmd_list_dma_attr = { 249 DMA_ATTR_V0, /* dma_attr_version */ 250 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 251 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 252 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 253 0x400ull, /* dma_attr_align: 1K-byte aligned */ 254 1, /* dma_attr_burstsizes */ 255 1, /* dma_attr_minxfer */ 256 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 257 0xffffffffull, /* dma_attr_seg */ 258 1, /* dma_attr_sgllen */ 259 1, /* dma_attr_granular */ 260 0, /* dma_attr_flags */ 261 }; 262 263 /* 264 * DMA attributes for cmd tables 265 * 266 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 267 * does not support 64-bit addressing 268 */ 269 static ddi_dma_attr_t cmd_table_dma_attr = { 270 DMA_ATTR_V0, /* dma_attr_version */ 271 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 272 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 273 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 274 0x80ull, /* dma_attr_align: 128-byte aligned */ 275 1, /* dma_attr_burstsizes */ 276 1, /* dma_attr_minxfer */ 277 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 278 0xffffffffull, /* dma_attr_seg */ 279 1, /* dma_attr_sgllen */ 280 1, /* dma_attr_granular */ 281 0, /* dma_attr_flags */ 282 }; 283 284 285 /* Device access attributes */ 286 static ddi_device_acc_attr_t accattr = { 287 DDI_DEVICE_ATTR_V0, 288 DDI_STRUCTURE_LE_ACC, 289 DDI_STRICTORDER_ACC 290 }; 291 292 293 static struct dev_ops ahcictl_dev_ops = { 294 DEVO_REV, /* devo_rev */ 295 0, /* refcnt */ 296 ahci_getinfo, /* info */ 297 nulldev, /* identify */ 298 nulldev, /* probe */ 299 ahci_attach, /* attach */ 300 ahci_detach, /* detach */ 301 nodev, /* no reset */ 302 (struct cb_ops *)0, /* driver operations */ 303 NULL, /* bus operations */ 304 NULL, /* power */ 305 ahci_quiesce, /* quiesce */ 306 }; 307 308 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 309 SATA_TRAN_HOTPLUG_OPS_REV_1, 310 ahci_tran_hotplug_port_activate, 311 ahci_tran_hotplug_port_deactivate 312 }; 313 314 extern struct mod_ops mod_driverops; 315 316 static struct modldrv modldrv = { 317 &mod_driverops, /* driverops */ 318 ahci_ident, /* short description */ 319 &ahcictl_dev_ops, /* driver ops */ 320 }; 321 322 static struct modlinkage modlinkage = { 323 MODREV_1, 324 &modldrv, 325 NULL 326 }; 327 328 /* The following variables are watchdog handler related */ 329 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 330 static int ahci_watchdog_tick; 331 332 /* 333 * This static variable indicates the size of command table, 334 * and it's changeable with prdt number, which ahci_dma_prdt_number 335 * indicates. 336 */ 337 static size_t ahci_cmd_table_size; 338 339 /* 340 * The below global variables are tunable via /etc/system 341 * 342 * ahci_dma_prdt_number 343 * ahci_msi_enabled 344 * ahci_buf_64bit_dma 345 * ahci_commu_64bit_dma 346 */ 347 348 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 349 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 350 351 /* AHCI MSI is tunable */ 352 boolean_t ahci_msi_enabled = B_TRUE; 353 354 /* 355 * 64-bit dma addressing for data buffer is tunable 356 * 357 * The variable controls only the below value: 358 * DBAU (upper 32-bits physical address of data block) 359 */ 360 boolean_t ahci_buf_64bit_dma = B_TRUE; 361 362 /* 363 * 64-bit dma addressing for communication system descriptors is tunable 364 * 365 * The variable controls the below three values: 366 * 367 * PxCLBU (upper 32-bits for the command list base physical address) 368 * PxFBU (upper 32-bits for the received FIS base physical address) 369 * CTBAU (upper 32-bits of command table base) 370 */ 371 boolean_t ahci_commu_64bit_dma = B_TRUE; 372 373 /* 374 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600 375 * chipset. If the users want to have a try with 64-bit dma, please change 376 * the below variable value to enable it. 377 */ 378 boolean_t sb600_buf_64bit_dma_disable = B_TRUE; 379 380 /* 381 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI 382 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma, 383 * please change the below value to enable it. 384 */ 385 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE; 386 387 388 /* 389 * End of global tunable variable definition 390 */ 391 392 #if AHCI_DEBUG 393 uint32_t ahci_debug_flags = 0; 394 #else 395 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 396 #endif 397 398 399 #if AHCI_DEBUG 400 /* The following is needed for ahci_log() */ 401 static kmutex_t ahci_log_mutex; 402 static char ahci_log_buf[512]; 403 #endif 404 405 /* Opaque state pointer initialized by ddi_soft_state_init() */ 406 static void *ahci_statep = NULL; 407 408 /* 409 * ahci module initialization. 410 */ 411 int 412 _init(void) 413 { 414 int ret; 415 416 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 417 if (ret != 0) { 418 goto err_out; 419 } 420 421 #if AHCI_DEBUG 422 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 423 #endif 424 425 if ((ret = sata_hba_init(&modlinkage)) != 0) { 426 #if AHCI_DEBUG 427 mutex_destroy(&ahci_log_mutex); 428 #endif 429 ddi_soft_state_fini(&ahci_statep); 430 goto err_out; 431 } 432 433 ret = mod_install(&modlinkage); 434 if (ret != 0) { 435 sata_hba_fini(&modlinkage); 436 #if AHCI_DEBUG 437 mutex_destroy(&ahci_log_mutex); 438 #endif 439 ddi_soft_state_fini(&ahci_statep); 440 goto err_out; 441 } 442 443 /* watchdog tick */ 444 ahci_watchdog_tick = drv_usectohz( 445 (clock_t)ahci_watchdog_timeout * 1000000); 446 return (ret); 447 448 err_out: 449 cmn_err(CE_WARN, "!ahci: Module init failed"); 450 return (ret); 451 } 452 453 /* 454 * ahci module uninitialize. 455 */ 456 int 457 _fini(void) 458 { 459 int ret; 460 461 ret = mod_remove(&modlinkage); 462 if (ret != 0) { 463 return (ret); 464 } 465 466 /* Remove the resources allocated in _init(). */ 467 sata_hba_fini(&modlinkage); 468 #if AHCI_DEBUG 469 mutex_destroy(&ahci_log_mutex); 470 #endif 471 ddi_soft_state_fini(&ahci_statep); 472 473 return (ret); 474 } 475 476 /* 477 * _info entry point 478 */ 479 int 480 _info(struct modinfo *modinfop) 481 { 482 return (mod_info(&modlinkage, modinfop)); 483 } 484 485 /* 486 * The attach entry point for dev_ops. 487 */ 488 static int 489 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 490 { 491 ahci_ctl_t *ahci_ctlp = NULL; 492 int instance = ddi_get_instance(dip); 493 int status; 494 int attach_state; 495 uint32_t cap_status, ahci_version; 496 int intr_types; 497 int i; 498 pci_regspec_t *regs; 499 int regs_length; 500 int rnumber; 501 #if AHCI_DEBUG 502 int speed; 503 #endif 504 505 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 506 NULL); 507 508 switch (cmd) { 509 case DDI_ATTACH: 510 break; 511 512 case DDI_RESUME: 513 514 /* 515 * During DDI_RESUME, the hardware state of the device 516 * (power may have been removed from the device) must be 517 * restored, allow pending requests to continue, and 518 * service new requests. 519 */ 520 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 521 mutex_enter(&ahci_ctlp->ahcictl_mutex); 522 523 /* Restart watch thread */ 524 if (ahci_ctlp->ahcictl_timeout_id == 0) 525 ahci_ctlp->ahcictl_timeout_id = timeout( 526 (void (*)(void *))ahci_watchdog_handler, 527 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 528 529 mutex_exit(&ahci_ctlp->ahcictl_mutex); 530 531 /* 532 * Re-initialize the controller and enable the interrupts and 533 * restart all the ports. 534 * 535 * Note that so far we don't support hot-plug during 536 * suspend/resume. 537 */ 538 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 539 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 540 "Failed to initialize the controller " 541 "during DDI_RESUME", NULL); 542 return (DDI_FAILURE); 543 } 544 545 mutex_enter(&ahci_ctlp->ahcictl_mutex); 546 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 547 mutex_exit(&ahci_ctlp->ahcictl_mutex); 548 549 return (DDI_SUCCESS); 550 551 default: 552 return (DDI_FAILURE); 553 } 554 555 attach_state = AHCI_ATTACH_STATE_NONE; 556 557 /* Allocate soft state */ 558 status = ddi_soft_state_zalloc(ahci_statep, instance); 559 if (status != DDI_SUCCESS) { 560 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 561 instance); 562 goto err_out; 563 } 564 565 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 566 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 567 ahci_ctlp->ahcictl_dip = dip; 568 569 /* Initialize the cport/port mapping */ 570 for (i = 0; i < AHCI_MAX_PORTS; i++) { 571 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 572 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 573 } 574 575 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 576 577 /* 578 * Now map the AHCI base address; which includes global 579 * registers and port control registers 580 * 581 * According to the spec, the AHCI Base Address is BAR5, 582 * but BAR0-BAR4 are optional, so we need to check which 583 * rnumber is used for BAR5. 584 */ 585 586 /* 587 * search through DDI "reg" property for the AHCI register set 588 */ 589 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 590 DDI_PROP_DONTPASS, "reg", (int **)®s, 591 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 592 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 593 instance); 594 goto err_out; 595 } 596 597 /* AHCI Base Address is located at 0x24 offset */ 598 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 599 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 600 == AHCI_PCI_RNUM) 601 break; 602 } 603 604 ddi_prop_free(regs); 605 606 if (rnumber == regs_length) { 607 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 608 instance); 609 goto err_out; 610 } 611 612 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 613 614 status = ddi_regs_map_setup(dip, 615 rnumber, 616 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 617 0, 618 0, 619 &accattr, 620 &ahci_ctlp->ahcictl_ahci_acc_handle); 621 if (status != DDI_SUCCESS) { 622 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 623 instance); 624 goto err_out; 625 } 626 627 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 628 629 /* Get the AHCI version information */ 630 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 631 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 632 633 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 634 (ahci_version & 0xffff0000) >> 16, 635 ((ahci_version & 0x0000ff00) >> 4 | 636 (ahci_version & 0x000000ff))); 637 638 /* We don't support controllers whose versions are lower than 1.0 */ 639 if (!(ahci_version & 0xffff0000)) { 640 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 641 "than version 1.0", instance); 642 goto err_out; 643 } 644 645 /* Get the HBA capabilities information */ 646 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 647 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 648 649 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 650 cap_status); 651 652 #if AHCI_DEBUG 653 /* Get the interface speed supported by the HBA */ 654 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 655 if (speed == 0x01) { 656 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 657 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 658 } else if (speed == 0x10) { 659 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 660 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 661 } else if (speed == 0x11) { 662 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 663 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 664 } 665 #endif 666 667 /* Get the number of command slots supported by the HBA */ 668 ahci_ctlp->ahcictl_num_cmd_slots = 669 ((cap_status & AHCI_HBA_CAP_NCS) >> 670 AHCI_HBA_CAP_NCS_SHIFT) + 1; 671 672 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 673 ahci_ctlp->ahcictl_num_cmd_slots); 674 675 /* Get the bit map which indicates ports implemented by the HBA */ 676 ahci_ctlp->ahcictl_ports_implemented = 677 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 678 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 679 680 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 681 ahci_ctlp->ahcictl_ports_implemented); 682 683 /* 684 * According to the AHCI spec, CAP.NP should indicate the maximum 685 * number of ports supported by the HBA silicon, but we found 686 * this value of ICH8 chipset only indicates the number of ports 687 * implemented (exposed) by it. Therefore, the driver should calculate 688 * the potential maximum value by checking PI register, and use 689 * the maximum of this value and CAP.NP. 690 */ 691 ahci_ctlp->ahcictl_num_ports = max( 692 (cap_status & AHCI_HBA_CAP_NP) + 1, 693 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 694 695 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 696 ahci_ctlp->ahcictl_num_ports); 697 698 /* Get the number of implemented ports by the HBA */ 699 ahci_ctlp->ahcictl_num_implemented_ports = 700 ahci_get_num_implemented_ports( 701 ahci_ctlp->ahcictl_ports_implemented); 702 703 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 704 "hba number of implemented ports: %d", 705 ahci_ctlp->ahcictl_num_implemented_ports); 706 707 /* Check whether HBA supports 64bit DMA addressing */ 708 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 709 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 710 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 711 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 712 "hba does not support 64-bit addressing", NULL); 713 } 714 715 /* Checking for the support of Port Multiplier */ 716 if (cap_status & AHCI_HBA_CAP_SPM) { 717 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS; 718 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 719 "hba supports port multiplier (CBSS)", NULL); 720 721 /* Support FIS-based switching ? */ 722 if (cap_status & AHCI_HBA_CAP_FBSS) { 723 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS; 724 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 725 "hba supports FIS-based switching (FBSS)", NULL); 726 } 727 } 728 729 /* Checking for Support Command List Override */ 730 if (cap_status & AHCI_HBA_CAP_SCLO) { 731 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 732 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 733 "hba supports command list override.", NULL); 734 } 735 736 /* Checking for Asynchronous Notification */ 737 if (cap_status & AHCI_HBA_CAP_SSNTF) { 738 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF; 739 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 740 "hba supports asynchronous notification.", NULL); 741 } 742 743 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 744 != DDI_SUCCESS) { 745 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 746 instance); 747 goto err_out; 748 } 749 750 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 751 752 /* 753 * Check the pci configuration space, and set caps. We also 754 * handle the hardware defect in this function. 755 * 756 * For example, force ATI SB600 to use 32-bit dma addressing 757 * since it doesn't support 64-bit dma though its CAP register 758 * declares it support. 759 */ 760 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 761 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 762 instance); 763 goto err_out; 764 } 765 766 /* 767 * Disable the whole controller interrupts before adding 768 * interrupt handlers(s). 769 */ 770 ahci_disable_all_intrs(ahci_ctlp); 771 772 /* Get supported interrupt types */ 773 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 774 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 775 instance); 776 goto err_out; 777 } 778 779 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 780 "ddi_intr_get_supported_types() returned: 0x%x", 781 intr_types); 782 783 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 784 /* 785 * Try MSI first, but fall back to FIXED if failed 786 */ 787 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 788 DDI_SUCCESS) { 789 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 790 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 791 "Using MSI interrupt type", NULL); 792 goto intr_done; 793 } 794 795 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 796 "MSI registration failed, " 797 "trying FIXED interrupts", NULL); 798 } 799 800 if (intr_types & DDI_INTR_TYPE_FIXED) { 801 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 802 DDI_SUCCESS) { 803 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 804 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 805 "Using FIXED interrupt type", NULL); 806 goto intr_done; 807 } 808 809 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 810 "FIXED interrupt registration failed", NULL); 811 } 812 813 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 814 815 goto err_out; 816 817 intr_done: 818 819 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 820 821 /* Initialize the controller mutex */ 822 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 823 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 824 825 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 826 827 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 828 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 829 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 830 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 831 } 832 833 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 834 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 835 sizeof (ahci_prdt_item_t)); 836 837 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 838 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 839 " ahci_cmd_table_size is 0x%x", 840 ahci_dma_prdt_number, ahci_cmd_table_size); 841 842 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 843 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 844 ahci_dma_prdt_number; 845 846 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 847 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 848 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 849 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 850 851 /* 852 * enable 64bit dma for data buffer for SB600 if 853 * sb600_buf_64bit_dma_disable is B_FALSE 854 */ 855 if ((ahci_buf_64bit_dma == B_FALSE) || 856 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) && 857 !(sb600_buf_64bit_dma_disable == B_FALSE && 858 ahci_ctlp->ahcictl_venid == 0x1002 && 859 ahci_ctlp->ahcictl_devid == 0x4380))) { 860 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 861 0xffffffffull; 862 } 863 864 /* 865 * enable 64bit dma for command buffer for SB600/700/710/800 866 * if sbxxx_commu_64bit_dma_disable is B_FALSE 867 */ 868 if ((ahci_commu_64bit_dma == B_FALSE) || 869 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) && 870 !(sbxxx_commu_64bit_dma_disable == B_FALSE && 871 ahci_ctlp->ahcictl_venid == 0x1002 && 872 (ahci_ctlp->ahcictl_devid == 0x4380 || 873 ahci_ctlp->ahcictl_devid == 0x4391)))) { 874 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 875 0xffffffffull; 876 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 877 0xffffffffull; 878 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 879 0xffffffffull; 880 } 881 882 /* Allocate the ports structure */ 883 status = ahci_alloc_ports_state(ahci_ctlp); 884 if (status != AHCI_SUCCESS) { 885 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 886 instance); 887 goto err_out; 888 } 889 890 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 891 892 /* 893 * Initialize the controller and ports. 894 */ 895 status = ahci_initialize_controller(ahci_ctlp); 896 if (status != AHCI_SUCCESS) { 897 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 898 instance); 899 goto err_out; 900 } 901 902 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 903 904 /* Start one thread to check packet timeouts */ 905 ahci_ctlp->ahcictl_timeout_id = timeout( 906 (void (*)(void *))ahci_watchdog_handler, 907 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 908 909 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 910 911 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 912 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 913 instance); 914 goto err_out; 915 } 916 917 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 918 919 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 920 921 return (DDI_SUCCESS); 922 923 err_out: 924 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 925 mutex_enter(&ahci_ctlp->ahcictl_mutex); 926 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 927 ahci_ctlp->ahcictl_timeout_id = 0; 928 mutex_exit(&ahci_ctlp->ahcictl_mutex); 929 } 930 931 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 932 ahci_uninitialize_controller(ahci_ctlp); 933 } 934 935 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 936 ahci_dealloc_ports_state(ahci_ctlp); 937 } 938 939 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 940 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 941 } 942 943 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 944 ahci_rem_intrs(ahci_ctlp); 945 } 946 947 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 948 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 949 } 950 951 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 952 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 953 } 954 955 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 956 ddi_soft_state_free(ahci_statep, instance); 957 } 958 959 return (DDI_FAILURE); 960 } 961 962 /* 963 * The detach entry point for dev_ops. 964 */ 965 static int 966 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 967 { 968 ahci_ctl_t *ahci_ctlp; 969 int instance; 970 int ret; 971 972 instance = ddi_get_instance(dip); 973 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 974 975 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 976 977 switch (cmd) { 978 case DDI_DETACH: 979 980 /* disable the interrupts for an uninterrupted detach */ 981 mutex_enter(&ahci_ctlp->ahcictl_mutex); 982 ahci_disable_all_intrs(ahci_ctlp); 983 mutex_exit(&ahci_ctlp->ahcictl_mutex); 984 985 /* unregister from the sata framework. */ 986 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 987 if (ret != AHCI_SUCCESS) { 988 mutex_enter(&ahci_ctlp->ahcictl_mutex); 989 ahci_enable_all_intrs(ahci_ctlp); 990 mutex_exit(&ahci_ctlp->ahcictl_mutex); 991 return (DDI_FAILURE); 992 } 993 994 mutex_enter(&ahci_ctlp->ahcictl_mutex); 995 996 /* stop the watchdog handler */ 997 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 998 ahci_ctlp->ahcictl_timeout_id = 0; 999 1000 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1001 1002 /* uninitialize the controller */ 1003 ahci_uninitialize_controller(ahci_ctlp); 1004 1005 /* remove the interrupts */ 1006 ahci_rem_intrs(ahci_ctlp); 1007 1008 /* deallocate the ports structures */ 1009 ahci_dealloc_ports_state(ahci_ctlp); 1010 1011 /* destroy mutex */ 1012 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1013 1014 /* teardown the pci config */ 1015 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1016 1017 /* remove the reg maps. */ 1018 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1019 1020 /* free the soft state. */ 1021 ddi_soft_state_free(ahci_statep, instance); 1022 1023 return (DDI_SUCCESS); 1024 1025 case DDI_SUSPEND: 1026 1027 /* 1028 * The steps associated with suspension must include putting 1029 * the underlying device into a quiescent state so that it 1030 * will not generate interrupts or modify or access memory. 1031 */ 1032 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1033 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 1034 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1035 return (DDI_SUCCESS); 1036 } 1037 1038 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 1039 1040 /* stop the watchdog handler */ 1041 if (ahci_ctlp->ahcictl_timeout_id) { 1042 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1043 ahci_ctlp->ahcictl_timeout_id = 0; 1044 } 1045 1046 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1047 1048 /* 1049 * drain the taskq 1050 */ 1051 ahci_drain_ports_taskq(ahci_ctlp); 1052 1053 /* 1054 * Disable the interrupts and stop all the ports. 1055 */ 1056 ahci_uninitialize_controller(ahci_ctlp); 1057 1058 return (DDI_SUCCESS); 1059 1060 default: 1061 return (DDI_FAILURE); 1062 } 1063 } 1064 1065 /* 1066 * The info entry point for dev_ops. 1067 * 1068 */ 1069 static int 1070 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 1071 void *arg, void **result) 1072 { 1073 #ifndef __lock_lint 1074 _NOTE(ARGUNUSED(dip)) 1075 #endif /* __lock_lint */ 1076 1077 ahci_ctl_t *ahci_ctlp; 1078 int instance; 1079 dev_t dev; 1080 1081 dev = (dev_t)arg; 1082 instance = getminor(dev); 1083 1084 switch (infocmd) { 1085 case DDI_INFO_DEVT2DEVINFO: 1086 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1087 if (ahci_ctlp != NULL) { 1088 *result = ahci_ctlp->ahcictl_dip; 1089 return (DDI_SUCCESS); 1090 } else { 1091 *result = NULL; 1092 return (DDI_FAILURE); 1093 } 1094 case DDI_INFO_DEVT2INSTANCE: 1095 *(int *)result = instance; 1096 break; 1097 default: 1098 break; 1099 } 1100 1101 return (DDI_SUCCESS); 1102 } 1103 1104 /* 1105 * Registers the ahci with sata framework. 1106 */ 1107 static int 1108 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1109 { 1110 struct sata_hba_tran *sata_hba_tran; 1111 1112 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1113 "ahci_register_sata_hba_tran enter", NULL); 1114 1115 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1116 1117 /* Allocate memory for the sata_hba_tran */ 1118 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1119 1120 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV; 1121 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1122 sata_hba_tran->sata_tran_hba_dma_attr = 1123 &ahci_ctlp->ahcictl_buffer_dma_attr; 1124 1125 /* Report the number of implemented ports */ 1126 sata_hba_tran->sata_tran_hba_num_cports = 1127 ahci_ctlp->ahcictl_num_implemented_ports; 1128 1129 /* Support ATAPI device */ 1130 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1131 1132 /* Get the data transfer capability for PIO command by the HBA */ 1133 if (cap_status & AHCI_HBA_CAP_PMD) { 1134 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1135 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1136 "DRQ block data transfer for PIO command protocol", NULL); 1137 } 1138 1139 /* 1140 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1141 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1142 * QUEUED (EXT), and SERVICE commands). Queued operations are 1143 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1144 * QUEUED commands when the HBA and device support native command 1145 * queuing(NCQ). 1146 * 1147 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1148 * CAP register of the HBA indicates NCQ is supported. 1149 * 1150 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1151 * set because the previous register content of PxCI can be re-written 1152 * in the register write. 1153 */ 1154 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1155 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1156 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1157 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1158 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1159 "Command Queuing", NULL); 1160 } 1161 1162 /* Support port multiplier? */ 1163 if (cap_status & AHCI_HBA_CAP_SPM) { 1164 sata_hba_tran->sata_tran_hba_features_support |= 1165 SATA_CTLF_PORT_MULTIPLIER; 1166 1167 /* Support FIS-based switching for port multiplier? */ 1168 if (cap_status & AHCI_HBA_CAP_FBSS) { 1169 sata_hba_tran->sata_tran_hba_features_support |= 1170 SATA_CTLF_PMULT_FBS; 1171 } 1172 } 1173 1174 /* Report the number of command slots */ 1175 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1176 1177 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1178 sata_hba_tran->sata_tran_start = ahci_tran_start; 1179 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1180 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1181 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1182 #ifdef __lock_lint 1183 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1184 #endif 1185 /* 1186 * When SATA framework adds support for pwrmgt the 1187 * pwrmgt_ops needs to be updated 1188 */ 1189 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1190 sata_hba_tran->sata_tran_ioctl = NULL; 1191 1192 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1193 1194 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1195 1196 /* Attach it to SATA framework */ 1197 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1198 != DDI_SUCCESS) { 1199 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1200 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1201 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1202 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1203 return (AHCI_FAILURE); 1204 } 1205 1206 return (AHCI_SUCCESS); 1207 } 1208 1209 /* 1210 * Unregisters the ahci with sata framework. 1211 */ 1212 static int 1213 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1214 { 1215 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1216 "ahci_unregister_sata_hba_tran enter", NULL); 1217 1218 /* Detach from the SATA framework. */ 1219 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1220 DDI_SUCCESS) { 1221 return (AHCI_FAILURE); 1222 } 1223 1224 /* Deallocate sata_hba_tran. */ 1225 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1226 sizeof (sata_hba_tran_t)); 1227 1228 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1229 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1230 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1231 1232 return (AHCI_SUCCESS); 1233 } 1234 1235 #define SET_PORTSTR(str, addrp) \ 1236 if (AHCI_ADDR_IS_PORT(addrp)) \ 1237 (void) sprintf((str), "%d", (addrp)->aa_port); \ 1238 else if (AHCI_ADDR_IS_PMULT(addrp)) \ 1239 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \ 1240 else \ 1241 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \ 1242 (addrp)->aa_pmport); 1243 1244 /* 1245 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1246 * port status registers and an attached device type via sata_device 1247 * structure. 1248 * 1249 * We return the cached information from a previous hardware probe. The 1250 * actual hardware probing itself was done either from within 1251 * ahci_initialize_controller() during the driver attach or from a phy 1252 * ready change interrupt handler. 1253 */ 1254 static int 1255 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1256 { 1257 ahci_ctl_t *ahci_ctlp; 1258 ahci_port_t *ahci_portp; 1259 ahci_addr_t addr, pmult_addr; 1260 uint8_t cport = sd->satadev_addr.cport; 1261 char portstr[10]; 1262 uint8_t device_type; 1263 uint32_t port_state; 1264 uint8_t port; 1265 int rval = SATA_SUCCESS, rval_init; 1266 1267 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1268 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1269 1270 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1271 1272 mutex_enter(&ahci_portp->ahciport_mutex); 1273 1274 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 1275 ASSERT(AHCI_ADDR_IS_VALID(&addr)); 1276 SET_PORTSTR(portstr, &addr); 1277 1278 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1279 "ahci_tran_probe_port enter: port %s", portstr); 1280 1281 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) && 1282 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1283 ahci_portp->ahciport_pmult_info == NULL)) { 1284 /* port mutliplier is removed. */ 1285 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1286 "ahci_tran_probe_port: " 1287 "pmult is removed from port %s", portstr); 1288 mutex_exit(&ahci_portp->ahciport_mutex); 1289 return (SATA_FAILURE); 1290 } 1291 1292 /* 1293 * The sata_device may refer to 1294 * 1. A controller port. 1295 * A controller port should be ready here. 1296 * 2. A port multiplier. 1297 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize 1298 * it and register the port multiplier to the framework. 1299 * SATA_ADDR_PMULT - check the status of all its device ports. 1300 * 3. A port multiplier port. 1301 * If it has not been initialized, initialized it. 1302 * 1303 * A port multiplier or a port multiplier port may require some 1304 * initialization because we cannot do these time-consuming jobs in an 1305 * interrupt context. 1306 */ 1307 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) { 1308 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 1309 /* Initialize registers on a port multiplier */ 1310 rval_init = ahci_initialize_pmult(ahci_ctlp, 1311 ahci_portp, &pmult_addr, sd); 1312 if (rval_init != AHCI_SUCCESS) { 1313 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1314 "ahci_tran_probe_port: " 1315 "pmult initialization failed.", NULL); 1316 mutex_exit(&ahci_portp->ahciport_mutex); 1317 return (SATA_FAILURE); 1318 } 1319 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) { 1320 /* Check pmports hotplug events */ 1321 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr); 1322 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT | 1323 SATA_ADDR_DPMPORT)) { 1324 if (ahci_probe_pmport(ahci_ctlp, ahci_portp, 1325 &addr, sd) != AHCI_SUCCESS) { 1326 rval = SATA_FAILURE; 1327 goto out; 1328 } 1329 } 1330 1331 /* Update port state and device type */ 1332 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr); 1333 1334 switch (port_state) { 1335 1336 case SATA_PSTATE_FAILED: 1337 sd->satadev_state = SATA_PSTATE_FAILED; 1338 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1339 "ahci_tran_probe_port: port %s PORT FAILED", portstr); 1340 goto out; 1341 1342 case SATA_PSTATE_SHUTDOWN: 1343 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1344 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1345 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr); 1346 goto out; 1347 1348 case SATA_PSTATE_PWROFF: 1349 sd->satadev_state = SATA_PSTATE_PWROFF; 1350 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1351 "ahci_tran_probe_port: port %s PORT PWROFF", portstr); 1352 goto out; 1353 1354 case SATA_PSTATE_PWRON: 1355 sd->satadev_state = SATA_PSTATE_PWRON; 1356 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1357 "ahci_tran_probe_port: port %s PORT PWRON", portstr); 1358 break; 1359 1360 default: 1361 sd->satadev_state = port_state; 1362 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1363 "ahci_tran_probe_port: port %s PORT NORMAL %x", 1364 portstr, port_state); 1365 break; 1366 } 1367 1368 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1369 1370 switch (device_type) { 1371 1372 case SATA_DTYPE_ATADISK: 1373 sd->satadev_type = SATA_DTYPE_ATADISK; 1374 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1375 "ahci_tran_probe_port: port %s DISK found", portstr); 1376 break; 1377 1378 case SATA_DTYPE_ATAPI: 1379 /* 1380 * HBA driver only knows it's an ATAPI device, and don't know 1381 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1382 * type need to be determined by checking IDENTIFY PACKET 1383 * DEVICE data 1384 */ 1385 sd->satadev_type = SATA_DTYPE_ATAPI; 1386 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1387 "ahci_tran_probe_port: port %s ATAPI found", portstr); 1388 break; 1389 1390 case SATA_DTYPE_PMULT: 1391 ASSERT(AHCI_ADDR_IS_PORT(&addr)||AHCI_ADDR_IS_PMULT(&addr)); 1392 sd->satadev_type = SATA_DTYPE_PMULT; 1393 1394 /* Update the number of pmports. */ 1395 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1396 sd->satadev_add_info = ahci_portp-> 1397 ahciport_pmult_info->ahcipmi_num_dev_ports; 1398 1399 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1400 "ahci_tran_probe_port: port %s Port Multiplier found", 1401 portstr); 1402 break; 1403 1404 case SATA_DTYPE_UNKNOWN: 1405 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1406 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1407 "ahci_tran_probe_port: port %s Unknown device found", 1408 portstr); 1409 break; 1410 1411 default: 1412 /* we don't support any other device types */ 1413 sd->satadev_type = SATA_DTYPE_NONE; 1414 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1415 "ahci_tran_probe_port: port %s No device found", portstr); 1416 break; 1417 } 1418 1419 out: 1420 /* Register update only fails while probing a pmult/pmport */ 1421 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) { 1422 ahci_update_sata_registers(ahci_ctlp, port, sd); 1423 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 1424 if (port_state & SATA_STATE_READY) 1425 if (ahci_update_pmult_pscr(ahci_ctlp, 1426 &addr, sd) != AHCI_SUCCESS) 1427 rval = SATA_FAILURE; 1428 } 1429 1430 mutex_exit(&ahci_portp->ahciport_mutex); 1431 return (rval); 1432 } 1433 1434 /* 1435 * There are four operation modes in sata framework: 1436 * SATA_OPMODE_INTERRUPTS 1437 * SATA_OPMODE_POLLING 1438 * SATA_OPMODE_ASYNCH 1439 * SATA_OPMODE_SYNCH 1440 * 1441 * Their combined meanings as following: 1442 * 1443 * SATA_OPMODE_SYNCH 1444 * The command has to be completed before sata_tran_start functions returns. 1445 * Either interrupts or polling could be used - it's up to the driver. 1446 * Mode used currently for internal, sata-module initiated operations. 1447 * 1448 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1449 * It is the same as the one above. 1450 * 1451 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1452 * The command has to be completed before sata_tran_start function returns. 1453 * No interrupt used, polling only. This should be the mode used for scsi 1454 * packets with FLAG_NOINTR. 1455 * 1456 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1457 * The command may be queued (callback function specified). Interrupts could 1458 * be used. It's normal operation mode. 1459 */ 1460 /* 1461 * Called by sata framework to transport a sata packet down stream. 1462 */ 1463 static int 1464 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1465 { 1466 ahci_ctl_t *ahci_ctlp; 1467 ahci_port_t *ahci_portp; 1468 ahci_addr_t addr; 1469 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1470 uint8_t port; 1471 char portstr[10]; 1472 1473 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1474 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1475 1476 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1477 "ahci_tran_start enter: cport %d satapkt 0x%p", 1478 cport, (void *)spkt); 1479 1480 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1481 1482 mutex_enter(&ahci_portp->ahciport_mutex); 1483 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr); 1484 SET_PORTSTR(portstr, &addr); 1485 1486 /* Sanity check */ 1487 if (AHCI_ADDR_IS_PMPORT(&addr)) { 1488 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1489 ahci_portp->ahciport_pmult_info == NULL) { 1490 1491 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1492 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1493 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN; 1494 ahci_update_sata_registers(ahci_ctlp, port, 1495 &spkt->satapkt_device); 1496 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1497 "ahci_tran_start returning PORT_ERROR while " 1498 "pmult removed: port: %s", portstr); 1499 mutex_exit(&ahci_portp->ahciport_mutex); 1500 return (SATA_TRAN_PORT_ERROR); 1501 } 1502 1503 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) & 1504 SATA_STATE_READY)) { 1505 if (!ddi_in_panic() || 1506 ahci_initialize_pmport(ahci_ctlp, 1507 ahci_portp, &addr) != AHCI_SUCCESS) { 1508 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1509 spkt->satapkt_device.satadev_type = 1510 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1511 spkt->satapkt_device.satadev_state = 1512 AHCIPORT_GET_STATE(ahci_portp, &addr); 1513 ahci_update_sata_registers(ahci_ctlp, port, 1514 &spkt->satapkt_device); 1515 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1516 "ahci_tran_start returning PORT_ERROR " 1517 "while sub-link is not initialized " 1518 "at port: %s", portstr); 1519 mutex_exit(&ahci_portp->ahciport_mutex); 1520 return (SATA_TRAN_PORT_ERROR); 1521 } 1522 } 1523 } 1524 1525 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED || 1526 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN|| 1527 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) { 1528 /* 1529 * In case the target driver would send the packet before 1530 * sata framework can have the opportunity to process those 1531 * event reports. 1532 */ 1533 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1534 spkt->satapkt_device.satadev_state = 1535 ahci_portp->ahciport_port_state; 1536 ahci_update_sata_registers(ahci_ctlp, port, 1537 &spkt->satapkt_device); 1538 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1539 "ahci_tran_start returning PORT_ERROR while " 1540 "port in FAILED/SHUTDOWN/PWROFF state: " 1541 "port: %s", portstr); 1542 mutex_exit(&ahci_portp->ahciport_mutex); 1543 return (SATA_TRAN_PORT_ERROR); 1544 } 1545 1546 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) { 1547 /* 1548 * ahci_intr_phyrdy_change() may have rendered it to 1549 * SATA_DTYPE_NONE. 1550 */ 1551 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1552 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1553 spkt->satapkt_device.satadev_state = 1554 ahci_portp->ahciport_port_state; 1555 ahci_update_sata_registers(ahci_ctlp, port, 1556 &spkt->satapkt_device); 1557 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1558 "ahci_tran_start returning PORT_ERROR while " 1559 "no device attached: port: %s", portstr); 1560 mutex_exit(&ahci_portp->ahciport_mutex); 1561 return (SATA_TRAN_PORT_ERROR); 1562 } 1563 1564 /* R/W PMULT command will occupy the whole HBA port */ 1565 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1566 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1567 "ahci_tran_start returning BUSY while " 1568 "executing READ/WRITE PORT-MULT command: " 1569 "port: %s", portstr); 1570 spkt->satapkt_reason = SATA_PKT_BUSY; 1571 mutex_exit(&ahci_portp->ahciport_mutex); 1572 return (SATA_TRAN_BUSY); 1573 } 1574 1575 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 1576 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1577 "ahci_tran_start returning BUSY while " 1578 "hot-plug in progress: port: %s", portstr); 1579 spkt->satapkt_reason = SATA_PKT_BUSY; 1580 mutex_exit(&ahci_portp->ahciport_mutex); 1581 return (SATA_TRAN_BUSY); 1582 } 1583 1584 /* 1585 * SATA HBA driver should remember that a device was reset and it 1586 * is supposed to reject any packets which do not specify either 1587 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1588 * 1589 * This is to prevent a race condition when a device was arbitrarily 1590 * reset by the HBA driver (and lost it's setting) and a target 1591 * driver sending some commands to a device before the sata framework 1592 * has a chance to restore the device setting (such as cache enable/ 1593 * disable or other resettable stuff). 1594 */ 1595 /* 1596 * It is unnecessary to use specific flags to indicate 1597 * reset_in_progress for a pmport. While mopping, all command will be 1598 * mopped so that the entire HBA port is being dealt as a single 1599 * object. 1600 */ 1601 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1602 ahci_portp->ahciport_reset_in_progress = 0; 1603 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1604 "ahci_tran_start [CLEAR] the " 1605 "reset_in_progress for port: %d", port); 1606 } 1607 1608 if (ahci_portp->ahciport_reset_in_progress && 1609 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1610 ! ddi_in_panic()) { 1611 spkt->satapkt_reason = SATA_PKT_BUSY; 1612 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1613 "ahci_tran_start returning BUSY while " 1614 "reset in progress: port: %d", port); 1615 mutex_exit(&ahci_portp->ahciport_mutex); 1616 return (SATA_TRAN_BUSY); 1617 } 1618 1619 #ifdef AHCI_DEBUG 1620 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) { 1621 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1622 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d", 1623 spkt, port); 1624 } 1625 #endif 1626 1627 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1628 spkt->satapkt_reason = SATA_PKT_BUSY; 1629 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1630 "ahci_tran_start returning BUSY while " 1631 "mopping in progress: port: %d", port); 1632 mutex_exit(&ahci_portp->ahciport_mutex); 1633 return (SATA_TRAN_BUSY); 1634 } 1635 1636 if (spkt->satapkt_op_mode & 1637 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1638 /* 1639 * If a SYNC command to be executed in interrupt context, 1640 * bounce it back to sata module. 1641 */ 1642 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1643 servicing_interrupt()) { 1644 spkt->satapkt_reason = SATA_PKT_BUSY; 1645 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1646 "ahci_tran_start returning BUSY while " 1647 "sending SYNC mode under interrupt context: " 1648 "port : %d", port); 1649 mutex_exit(&ahci_portp->ahciport_mutex); 1650 return (SATA_TRAN_BUSY); 1651 } 1652 1653 /* We need to do the sync start now */ 1654 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, 1655 spkt) == AHCI_FAILURE) { 1656 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1657 "return QUEUE_FULL: port %d", port); 1658 mutex_exit(&ahci_portp->ahciport_mutex); 1659 return (SATA_TRAN_QUEUE_FULL); 1660 } 1661 } else { 1662 /* Async start, using interrupt */ 1663 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt) 1664 == AHCI_FAILURE) { 1665 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1666 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1667 "returning QUEUE_FULL: port %d", port); 1668 mutex_exit(&ahci_portp->ahciport_mutex); 1669 return (SATA_TRAN_QUEUE_FULL); 1670 } 1671 } 1672 1673 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1674 "sata tran accepted: port %s", portstr); 1675 1676 mutex_exit(&ahci_portp->ahciport_mutex); 1677 return (SATA_TRAN_ACCEPTED); 1678 } 1679 1680 /* 1681 * SATA_OPMODE_SYNCH flag is set 1682 * 1683 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1684 * without interrupt, otherwise we can still use the interrupt. 1685 * 1686 * WARNING!!! ahciport_mutex should be acquired before the function 1687 * is called. 1688 */ 1689 static int 1690 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1691 ahci_addr_t *addrp, sata_pkt_t *spkt) 1692 { 1693 int pkt_timeout_ticks; 1694 uint32_t timeout_tags; 1695 int rval; 1696 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1697 uint8_t port = addrp->aa_port; 1698 1699 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1700 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt); 1701 1702 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1703 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1704 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1705 addrp, spkt)) == AHCI_FAILURE) { 1706 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1707 return (rval); 1708 } 1709 1710 pkt_timeout_ticks = 1711 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1712 1713 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1714 mutex_exit(&ahci_portp->ahciport_mutex); 1715 1716 /* Simulate the interrupt */ 1717 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1718 1719 drv_usecwait(AHCI_10MS_USECS); 1720 1721 mutex_enter(&ahci_portp->ahciport_mutex); 1722 pkt_timeout_ticks -= AHCI_10MS_TICKS; 1723 if (pkt_timeout_ticks < 0) { 1724 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1725 "port %d satapkt 0x%p timed out\n", 1726 instance, port, (void *)spkt); 1727 timeout_tags = (0x1 << rval); 1728 mutex_exit(&ahci_portp->ahciport_mutex); 1729 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1730 port, timeout_tags); 1731 mutex_enter(&ahci_portp->ahciport_mutex); 1732 } 1733 } 1734 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1735 return (AHCI_SUCCESS); 1736 1737 } else { 1738 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1739 addrp, spkt)) == AHCI_FAILURE) 1740 return (rval); 1741 1742 #if AHCI_DEBUG 1743 /* 1744 * Note that the driver always uses the slot 0 to deliver 1745 * REQUEST SENSE or READ LOG EXT command 1746 */ 1747 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1748 ASSERT(rval == 0); 1749 #endif 1750 1751 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1752 cv_wait(&ahci_portp->ahciport_cv, 1753 &ahci_portp->ahciport_mutex); 1754 1755 return (AHCI_SUCCESS); 1756 } 1757 } 1758 1759 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1760 if (satapkt) { \ 1761 satapkt->satapkt_reason = reason; \ 1762 /* \ 1763 * We set the satapkt_reason in both sync and \ 1764 * non-sync cases. \ 1765 */ \ 1766 } \ 1767 if (satapkt && \ 1768 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1769 satapkt->satapkt_comp) { \ 1770 mutex_exit(&ahci_portp->ahciport_mutex); \ 1771 (*satapkt->satapkt_comp)(satapkt); \ 1772 mutex_enter(&ahci_portp->ahciport_mutex); \ 1773 } else { \ 1774 if (satapkt && \ 1775 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1776 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1777 cv_broadcast(&ahci_portp->ahciport_cv); \ 1778 } 1779 1780 /* 1781 * Searches for and claims a free command slot. 1782 * 1783 * Returns value: 1784 * 1785 * AHCI_FAILURE returned only if 1786 * 1. No empty slot left 1787 * 2. Non-queued command requested while queued command(s) is outstanding 1788 * 3. Queued command requested while non-queued command(s) is outstanding 1789 * 4. HBA doesn't support multiple-use of command list while already a 1790 * non-queued command is oustanding 1791 * 5. Queued command requested while some queued command(s) has been 1792 * outstanding on a different port multiplier port. (AHCI spec 1.2, 1793 * 9.1.2) 1794 * 1795 * claimed slot number returned if succeeded 1796 * 1797 * NOTE: it will always return slot 0 for following commands to simplify the 1798 * algorithm. 1799 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process 1800 * 2. READ/WRITE PORTMULT command 1801 * 1802 * WARNING!!! ahciport_mutex should be acquired before the function 1803 * is called. 1804 */ 1805 static int 1806 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1807 ahci_addr_t *addrp, int command_type) 1808 { 1809 uint32_t port_cmd_issue; 1810 uint32_t free_slots; 1811 int slot; 1812 1813 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1814 "ahciport_pending_tags = 0x%x " 1815 "ahciport_pending_ncq_tags = 0x%x", 1816 ahci_portp->ahciport_pending_tags, 1817 ahci_portp->ahciport_pending_ncq_tags); 1818 1819 /* 1820 * According to the AHCI spec, system software is responsible to 1821 * ensure that queued and non-queued commands are not mixed in 1822 * the command list. 1823 */ 1824 if (command_type == AHCI_NON_NCQ_CMD) { 1825 /* Non-NCQ command request */ 1826 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1827 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1828 "ahci_claim_free_slot: there is still pending " 1829 "queued command(s) in the command list, " 1830 "so no available slot for the non-queued " 1831 "command", NULL); 1832 return (AHCI_FAILURE); 1833 } 1834 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1835 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 1836 "ahci_claim_free_slot: there is still pending " 1837 "read/write port-mult command(s) in command list, " 1838 "so no available slot for the non-queued command", 1839 NULL); 1840 return (AHCI_FAILURE); 1841 } 1842 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1843 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1844 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1845 "ahci_claim_free_slot: HBA cannot support multiple-" 1846 "use of the command list for non-queued commands", 1847 NULL); 1848 return (AHCI_FAILURE); 1849 } 1850 free_slots = (~ahci_portp->ahciport_pending_tags) & 1851 AHCI_SLOT_MASK(ahci_ctlp); 1852 } else if (command_type == AHCI_NCQ_CMD) { 1853 /* NCQ command request */ 1854 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1855 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1856 "ahci_claim_free_slot: there is still pending " 1857 "non-queued command(s) in the command list, " 1858 "so no available slot for the queued command", 1859 NULL); 1860 return (AHCI_FAILURE); 1861 } 1862 1863 /* 1864 * NCQ commands cannot be sent to different port multiplier 1865 * ports in Command-Based Switching mode 1866 */ 1867 /* 1868 * NOTE: In Command-Based Switching mode, AHCI controller 1869 * usually reports a 'Handshake Error' when multiple NCQ 1870 * commands are outstanding simultaneously. 1871 */ 1872 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 1873 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1874 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) && 1875 NCQ_CMD_IN_PROGRESS(ahci_portp) && 1876 AHCIPORT_NCQ_PMPORT(ahci_portp) != 1877 addrp->aa_pmport) { 1878 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1879 "ahci_claim_free_slot: there is still " 1880 "pending queued command(s) in the " 1881 "command list for another Port Multiplier " 1882 "port, so no available slot.", NULL); 1883 return (AHCI_FAILURE); 1884 } 1885 } 1886 1887 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1888 AHCI_NCQ_SLOT_MASK(ahci_portp); 1889 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1890 /* Error retrieval command request */ 1891 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1892 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1893 "SENSE or READ LOG EXT command", NULL); 1894 slot = 0; 1895 goto out; 1896 } else if (command_type == AHCI_RDWR_PMULT_CMD) { 1897 /* 1898 * An extra check on PxCI. Sometimes PxCI bits may not be 1899 * cleared during hot-plug or error recovery process. 1900 */ 1901 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 1902 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port)); 1903 1904 if (port_cmd_issue != 0) { 1905 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 1906 "ahci_claim_free_slot: there is still pending " 1907 "command(s) in command list (0x%x/0x%x, PxCI %x)," 1908 "so no available slot for R/W PMULT command.", 1909 NON_NCQ_CMD_IN_PROGRESS(ahci_portp), 1910 NCQ_CMD_IN_PROGRESS(ahci_portp), 1911 port_cmd_issue); 1912 return (AHCI_FAILURE); 1913 } 1914 1915 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1916 "ahci_claim_free_slot: slot 0 is allocated for " 1917 "READ/WRITE PORTMULT command", NULL); 1918 slot = 0; 1919 goto out; 1920 } 1921 1922 slot = ddi_ffs(free_slots) - 1; 1923 if (slot == -1) { 1924 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1925 "ahci_claim_free_slot: no empty slots", NULL); 1926 return (AHCI_FAILURE); 1927 } 1928 1929 /* 1930 * According to the AHCI spec, to allow a simple mechanism for the 1931 * HBA to map command list slots to queue entries, software must 1932 * match the tag number it uses to the slot it is placing the command 1933 * in. For example, if a queued command is placed in slot 5, the tag 1934 * for that command must be 5. 1935 */ 1936 if (command_type == AHCI_NCQ_CMD) { 1937 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1938 if (AHCI_ADDR_IS_PMPORT(addrp)) { 1939 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1940 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport; 1941 } 1942 } 1943 1944 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1945 1946 out: 1947 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1948 "ahci_claim_free_slot: found slot: 0x%x", slot); 1949 1950 return (slot); 1951 } 1952 1953 /* 1954 * Builds the Command Table for the sata packet and delivers it to controller. 1955 * 1956 * Returns: 1957 * slot number if we can obtain a slot successfully 1958 * otherwise, return AHCI_FAILURE 1959 * 1960 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1961 */ 1962 static int 1963 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1964 ahci_addr_t *addrp, sata_pkt_t *spkt) 1965 { 1966 int cmd_slot; 1967 sata_cmd_t *scmd; 1968 ahci_fis_h2d_register_t *h2d_register_fisp; 1969 ahci_cmd_table_t *cmd_table; 1970 ahci_cmd_header_t *cmd_header; 1971 int ncookies; 1972 int i; 1973 int command_type = AHCI_NON_NCQ_CMD; 1974 int ncq_qdepth; 1975 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1976 uint8_t port, pmport; 1977 #if AHCI_DEBUG 1978 uint32_t *ptr; 1979 uint8_t *ptr2; 1980 #endif 1981 1982 port = addrp->aa_port; 1983 pmport = addrp->aa_pmport; 1984 1985 spkt->satapkt_reason = SATA_PKT_BUSY; 1986 1987 scmd = &spkt->satapkt_cmd; 1988 1989 /* Check if the command is a NCQ command */ 1990 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1991 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1992 command_type = AHCI_NCQ_CMD; 1993 1994 /* 1995 * When NCQ is support, system software must determine the 1996 * maximum tag allowed by the device and the HBA, and it 1997 * must use a value not beyond of the lower bound of the two. 1998 * 1999 * Sata module is going to calculate the qdepth and send 2000 * down to HBA driver via sata_cmd. 2001 */ 2002 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 2003 2004 /* 2005 * At the moment, the driver doesn't support the dynamic 2006 * setting of the maximum ncq depth, and the value can be 2007 * set either during the attach or after hot-plug insertion. 2008 */ 2009 if (ahci_portp->ahciport_max_ncq_tags == 0) { 2010 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 2011 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 2012 "ahci_deliver_satapkt: port %d the max tags for " 2013 "NCQ command is %d", port, ncq_qdepth); 2014 } else { 2015 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 2016 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 2017 " port %d the max tag for NCQ command is " 2018 "requested to change from %d to %d, at the" 2019 " moment the driver doesn't support the " 2020 "dynamic change so it's going to " 2021 "still use the previous tag value", 2022 instance, port, 2023 ahci_portp->ahciport_max_ncq_tags, 2024 ncq_qdepth); 2025 } 2026 } 2027 } 2028 2029 /* Check if the command is an error retrieval command */ 2030 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 2031 command_type = AHCI_ERR_RETRI_CMD; 2032 2033 /* Check if the command is an read/write pmult command */ 2034 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2035 command_type = AHCI_RDWR_PMULT_CMD; 2036 2037 /* Check if there is an empty command slot */ 2038 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, 2039 addrp, command_type); 2040 if (cmd_slot == AHCI_FAILURE) { 2041 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 2042 return (AHCI_FAILURE); 2043 } 2044 2045 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 2046 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 2047 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 2048 cmd_slot, port, (void *)spkt); 2049 2050 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 2051 bzero((void *)cmd_table, ahci_cmd_table_size); 2052 2053 /* For data transfer operations, it is the H2D Register FIS */ 2054 h2d_register_fisp = 2055 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 2056 2057 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 2058 2059 /* 2060 * PMP field only make sense when target is a port multiplier or a 2061 * device behind a port multiplier. Otherwise should set it to 0. 2062 */ 2063 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2064 SET_FIS_PMP(h2d_register_fisp, pmport); 2065 2066 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 2067 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 2068 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 2069 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 2070 2071 switch (scmd->satacmd_addr_type) { 2072 2073 case 0: 2074 /* 2075 * satacmd_addr_type will be 0 for the commands below: 2076 * ATAPI command 2077 * SATAC_IDLE_IM 2078 * SATAC_STANDBY_IM 2079 * SATAC_DOWNLOAD_MICROCODE 2080 * SATAC_FLUSH_CACHE 2081 * SATAC_SET_FEATURES 2082 * SATAC_SMART 2083 * SATAC_ID_PACKET_DEVICE 2084 * SATAC_ID_DEVICE 2085 * SATAC_READ_PORTMULT 2086 * SATAC_WRITE_PORTMULT 2087 */ 2088 /* FALLTHRU */ 2089 2090 case ATA_ADDR_LBA: 2091 /* FALLTHRU */ 2092 2093 case ATA_ADDR_LBA28: 2094 /* LBA[7:0] */ 2095 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2096 2097 /* LBA[15:8] */ 2098 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2099 2100 /* LBA[23:16] */ 2101 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2102 2103 /* LBA [27:24] (also called dev_head) */ 2104 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 2105 2106 break; 2107 2108 case ATA_ADDR_LBA48: 2109 /* LBA[7:0] */ 2110 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2111 2112 /* LBA[15:8] */ 2113 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2114 2115 /* LBA[23:16] */ 2116 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2117 2118 /* LBA [31:24] */ 2119 SET_FIS_SECTOR_EXP(h2d_register_fisp, 2120 scmd->satacmd_lba_low_msb); 2121 2122 /* LBA [39:32] */ 2123 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 2124 scmd->satacmd_lba_mid_msb); 2125 2126 /* LBA [47:40] */ 2127 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 2128 scmd->satacmd_lba_high_msb); 2129 2130 /* Set dev_head */ 2131 SET_FIS_DEV_HEAD(h2d_register_fisp, 2132 scmd->satacmd_device_reg); 2133 2134 /* Set the extended sector count and features */ 2135 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 2136 scmd->satacmd_sec_count_msb); 2137 SET_FIS_FEATURES_EXP(h2d_register_fisp, 2138 scmd->satacmd_features_reg_ext); 2139 break; 2140 } 2141 2142 /* 2143 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 2144 * filled into features field, and sector count 8:15 is filled into 2145 * features (exp) field. The hba driver doesn't need to anything 2146 * special with regard to this, since sata framework has already 2147 * done so. 2148 * 2149 * However the driver needs to make sure TAG is filled into sector 2150 * field. 2151 */ 2152 if (command_type == AHCI_NCQ_CMD) { 2153 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 2154 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 2155 } 2156 2157 ncookies = scmd->satacmd_num_dma_cookies; 2158 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2159 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 2160 ncookies, ahci_dma_prdt_number); 2161 2162 ASSERT(ncookies <= ahci_dma_prdt_number); 2163 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 2164 2165 /* *** now fill the scatter gather list ******* */ 2166 for (i = 0; i < ncookies; i++) { 2167 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 2168 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 2169 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 2170 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 2171 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 2172 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 2173 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 2174 scmd->satacmd_dma_cookie_list[i].dmac_size; 2175 } 2176 2177 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2178 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 2179 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 2180 2181 /* The ACMD field is filled in for ATAPI command */ 2182 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 2183 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 2184 SATA_ATAPI_MAX_CDB_LEN); 2185 } 2186 2187 /* Set Command Header in Command List */ 2188 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 2189 BZERO_DESCR_INFO(cmd_header); 2190 BZERO_PRD_BYTE_COUNT(cmd_header); 2191 2192 /* Set the number of entries in the PRD table */ 2193 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 2194 2195 /* Set the length of the command in the CFIS area */ 2196 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 2197 2198 /* 2199 * PMP field only make sense when target is a port multiplier or a 2200 * device behind a port multiplier. Otherwise should set it to 0. 2201 */ 2202 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2203 SET_PORT_MULTI_PORT(cmd_header, pmport); 2204 2205 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 2206 "sata_data_direction = 0x%x", 2207 scmd->satacmd_flags.sata_data_direction); 2208 2209 /* Set A bit if it is an ATAPI command */ 2210 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 2211 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 2212 2213 /* Set W bit if data is going to the device */ 2214 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 2215 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 2216 2217 /* 2218 * Set the prefetchable bit - this bit is only valid if the PRDTL 2219 * field is non-zero or the ATAPI 'A' bit is set in the command 2220 * header. This bit cannot be set when using native command 2221 * queuing commands or when using FIS-based switching with a Port 2222 * multiplier. 2223 */ 2224 if (command_type != AHCI_NCQ_CMD) 2225 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 2226 2227 /* 2228 * Now remember the sata packet in ahciport_slot_pkts[]. 2229 * Error retrieval command and r/w port multiplier command will 2230 * be stored specifically for each port. 2231 */ 2232 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 2233 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2234 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 2235 2236 /* 2237 * We are overloading satapkt_hba_driver_private with 2238 * watched_cycle count. 2239 */ 2240 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 2241 2242 #if AHCI_DEBUG 2243 if (ahci_debug_flags & AHCIDBG_ATACMD && 2244 scmd->satacmd_cmd_reg != SATAC_PACKET || 2245 ahci_debug_flags & AHCIDBG_ATAPICMD && 2246 scmd->satacmd_cmd_reg == SATAC_PACKET) { 2247 2248 /* Dump the command header and table */ 2249 ahci_log(ahci_ctlp, CE_WARN, "\n"); 2250 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 2251 "0x%p cmd_reg 0x%x port %d", spkt, 2252 scmd->satacmd_cmd_reg, port); 2253 ptr = (uint32_t *)cmd_header; 2254 ahci_log(ahci_ctlp, CE_WARN, 2255 " Command Header:%8x %8x %8x %8x", 2256 ptr[0], ptr[1], ptr[2], ptr[3]); 2257 2258 /* Dump the H2D register FIS */ 2259 ptr = (uint32_t *)h2d_register_fisp; 2260 ahci_log(ahci_ctlp, CE_WARN, 2261 " Command FIS: %8x %8x %8x %8x", 2262 ptr[0], ptr[1], ptr[2], ptr[3]); 2263 2264 /* Dump the ACMD register FIS */ 2265 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 2266 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 2267 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 2268 ahci_log(ahci_ctlp, CE_WARN, 2269 " ATAPI command: %2x %2x %2x %2x " 2270 "%2x %2x %2x %2x", 2271 ptr2[8 * i], ptr2[8 * i + 1], 2272 ptr2[8 * i + 2], ptr2[8 * i + 3], 2273 ptr2[8 * i + 4], ptr2[8 * i + 5], 2274 ptr2[8 * i + 6], ptr2[8 * i + 7]); 2275 2276 /* Dump the PRDT */ 2277 for (i = 0; i < ncookies; i++) { 2278 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 2279 ahci_log(ahci_ctlp, CE_WARN, 2280 " Cookie %d: %8x %8x %8x %8x", 2281 i, ptr[0], ptr[1], ptr[2], ptr[3]); 2282 } 2283 } 2284 #endif 2285 2286 (void) ddi_dma_sync( 2287 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 2288 0, 2289 ahci_cmd_table_size, 2290 DDI_DMA_SYNC_FORDEV); 2291 2292 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 2293 cmd_slot * sizeof (ahci_cmd_header_t), 2294 sizeof (ahci_cmd_header_t), 2295 DDI_DMA_SYNC_FORDEV); 2296 2297 /* Set the corresponding bit in the PxSACT.DS for queued command */ 2298 if (command_type == AHCI_NCQ_CMD) { 2299 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2300 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 2301 (0x1 << cmd_slot)); 2302 } 2303 2304 /* Indicate to the HBA that a command is active. */ 2305 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2306 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 2307 (0x1 << cmd_slot)); 2308 2309 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 2310 "exit: port %d", port); 2311 2312 return (cmd_slot); 2313 } 2314 2315 /* 2316 * Called by the sata framework to abort the previously sent packet(s). 2317 * 2318 * Reset device to abort commands. 2319 */ 2320 static int 2321 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 2322 { 2323 ahci_ctl_t *ahci_ctlp; 2324 ahci_port_t *ahci_portp; 2325 uint32_t slot_status = 0; 2326 uint32_t aborted_tags = 0; 2327 uint32_t finished_tags = 0; 2328 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 2329 uint8_t port; 2330 int tmp_slot; 2331 int instance = ddi_get_instance(dip); 2332 2333 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2334 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2335 2336 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2337 "ahci_tran_abort enter: port %d", port); 2338 2339 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2340 mutex_enter(&ahci_portp->ahciport_mutex); 2341 2342 /* 2343 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2344 * commands are being mopped, therefore there is nothing else to do 2345 */ 2346 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2347 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2348 "ahci_tran_abort: port %d is in " 2349 "mopping process, so just return directly ", port); 2350 mutex_exit(&ahci_portp->ahciport_mutex); 2351 return (SATA_SUCCESS); 2352 } 2353 2354 /* 2355 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT 2356 * command is being executed so no other commands is outstanding, 2357 * nothing to do. 2358 */ 2359 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) { 2360 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2361 "ahci_tran_abort: port %d is reading/writing " 2362 "port multiplier, so just return directly ", port); 2363 mutex_exit(&ahci_portp->ahciport_mutex); 2364 return (SATA_SUCCESS); 2365 } 2366 2367 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2368 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2369 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2370 /* 2371 * In case the targer driver would send the request before 2372 * sata framework can have the opportunity to process those 2373 * event reports. 2374 */ 2375 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2376 spkt->satapkt_device.satadev_state = 2377 ahci_portp->ahciport_port_state; 2378 ahci_update_sata_registers(ahci_ctlp, port, 2379 &spkt->satapkt_device); 2380 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2381 "ahci_tran_abort returning SATA_FAILURE while " 2382 "port in FAILED/SHUTDOWN/PWROFF state: " 2383 "port: %d", port); 2384 mutex_exit(&ahci_portp->ahciport_mutex); 2385 return (SATA_FAILURE); 2386 } 2387 2388 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2389 /* 2390 * ahci_intr_phyrdy_change() may have rendered it to 2391 * AHCI_PORT_TYPE_NODEV. 2392 */ 2393 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2394 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2395 spkt->satapkt_device.satadev_state = 2396 ahci_portp->ahciport_port_state; 2397 ahci_update_sata_registers(ahci_ctlp, port, 2398 &spkt->satapkt_device); 2399 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2400 "ahci_tran_abort returning SATA_FAILURE while " 2401 "no device attached: port: %d", port); 2402 mutex_exit(&ahci_portp->ahciport_mutex); 2403 return (SATA_FAILURE); 2404 } 2405 2406 if (flag == SATA_ABORT_ALL_PACKETS) { 2407 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2408 aborted_tags = ahci_portp->ahciport_pending_tags; 2409 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2410 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2411 2412 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2413 instance, port); 2414 } else { 2415 aborted_tags = 0xffffffff; 2416 /* 2417 * Aborting one specific packet, first search the 2418 * ahciport_slot_pkts[] list for matching spkt. 2419 */ 2420 for (tmp_slot = 0; 2421 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2422 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2423 aborted_tags = (0x1 << tmp_slot); 2424 break; 2425 } 2426 } 2427 2428 if (aborted_tags == 0xffffffff) { 2429 /* request packet is not on the pending list */ 2430 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2431 "Cannot find the aborting pkt 0x%p on the " 2432 "pending list", (void *)spkt); 2433 ahci_update_sata_registers(ahci_ctlp, port, 2434 &spkt->satapkt_device); 2435 mutex_exit(&ahci_portp->ahciport_mutex); 2436 return (SATA_FAILURE); 2437 } 2438 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2439 instance, port, (void *)spkt); 2440 } 2441 2442 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2443 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2444 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2445 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2446 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2447 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2448 2449 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2450 ahci_portp->ahciport_mop_in_progress++; 2451 2452 /* 2453 * To abort the packet(s), first we are trying to clear PxCMD.ST 2454 * to stop the port, and if the port can be stopped 2455 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2456 * then we just send back the aborted packet(s) with ABORTED flag 2457 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2458 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2459 * perform a COMRESET. 2460 */ 2461 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2462 ahci_portp, port, NULL, NULL); 2463 2464 /* 2465 * Compute which have finished and which need to be retried. 2466 * 2467 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2468 * minus the slot_status. The aborted_tags has to be deducted by 2469 * finished_tags since we can't possibly abort a tag which had finished 2470 * already. 2471 */ 2472 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2473 finished_tags = ahci_portp->ahciport_pending_tags & 2474 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2475 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2476 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2477 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2478 2479 aborted_tags &= ~finished_tags; 2480 2481 ahci_mop_commands(ahci_ctlp, 2482 ahci_portp, 2483 slot_status, 2484 0, /* failed tags */ 2485 0, /* timeout tags */ 2486 aborted_tags, 2487 0); /* reset tags */ 2488 2489 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2490 mutex_exit(&ahci_portp->ahciport_mutex); 2491 2492 return (SATA_SUCCESS); 2493 } 2494 2495 /* 2496 * Used to do device reset and reject all the pending packets on a device 2497 * during the reset operation. 2498 * 2499 * NOTE: ONLY called by ahci_tran_reset_dport 2500 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2501 */ 2502 static int 2503 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2504 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2505 { 2506 uint32_t slot_status = 0; 2507 uint32_t reset_tags = 0; 2508 uint32_t finished_tags = 0; 2509 uint8_t port = addrp->aa_port; 2510 sata_device_t sdevice; 2511 int ret; 2512 2513 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2514 "ahci_reset_device_reject_pkts on port: %d", port); 2515 2516 /* 2517 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2518 * commands are being mopped, therefore there is nothing else to do 2519 */ 2520 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2521 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2522 "ahci_reset_device_reject_pkts: port %d is in " 2523 "mopping process, so return directly ", port); 2524 return (SATA_SUCCESS); 2525 } 2526 2527 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2528 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2529 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2530 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2531 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2532 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2533 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2534 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2535 } 2536 2537 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2538 != AHCI_SUCCESS) { 2539 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2540 "Try to do a port reset after software " 2541 "reset failed", port); 2542 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 2543 if (ret != AHCI_SUCCESS) { 2544 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2545 "ahci_reset_device_reject_pkts: port %d " 2546 "failed", port); 2547 return (SATA_FAILURE); 2548 } 2549 } 2550 /* Set the reset in progress flag */ 2551 ahci_portp->ahciport_reset_in_progress = 1; 2552 2553 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2554 ahci_portp->ahciport_mop_in_progress++; 2555 2556 /* Indicate to the framework that a reset has happened */ 2557 bzero((void *)&sdevice, sizeof (sata_device_t)); 2558 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2559 sdevice.satadev_addr.pmport = 0; 2560 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2561 sdevice.satadev_state = SATA_DSTATE_RESET | 2562 SATA_DSTATE_PWR_ACTIVE; 2563 mutex_exit(&ahci_portp->ahciport_mutex); 2564 sata_hba_event_notify( 2565 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2566 &sdevice, 2567 SATA_EVNT_DEVICE_RESET); 2568 mutex_enter(&ahci_portp->ahciport_mutex); 2569 2570 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2571 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 2572 2573 /* Next try to mop the pending commands */ 2574 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2575 finished_tags = ahci_portp->ahciport_pending_tags & 2576 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2577 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2578 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2579 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2580 2581 reset_tags &= ~finished_tags; 2582 2583 ahci_mop_commands(ahci_ctlp, 2584 ahci_portp, 2585 slot_status, 2586 0, /* failed tags */ 2587 0, /* timeout tags */ 2588 0, /* aborted tags */ 2589 reset_tags); /* reset tags */ 2590 2591 return (SATA_SUCCESS); 2592 } 2593 2594 /* 2595 * Used to do device reset and reject all the pending packets on a device 2596 * during the reset operation. 2597 * 2598 * NOTE: ONLY called by ahci_tran_reset_dport 2599 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2600 */ 2601 static int 2602 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp, 2603 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2604 { 2605 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 2606 uint8_t port = addrp->aa_port; 2607 uint8_t pmport = addrp->aa_pmport; 2608 sata_device_t sdevice; 2609 2610 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp, 2611 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport); 2612 2613 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2614 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2615 "ahci_reset_pmdevice_reject_pkts: port %d is in " 2616 "mopping process, so return directly ", port); 2617 return (SATA_SUCCESS); 2618 } 2619 2620 /* Checking for outstanding commands */ 2621 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2622 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2623 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2624 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2625 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2626 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2627 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2628 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2629 } 2630 2631 /* Issue SOFTWARE reset command. */ 2632 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2633 != AHCI_SUCCESS) { 2634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2635 "Try to do a port reset after software " 2636 "reset failed", port); 2637 return (SATA_FAILURE); 2638 } 2639 2640 /* Set the reset in progress flag */ 2641 ahci_portp->ahciport_reset_in_progress = 1; 2642 2643 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2644 ahci_portp->ahciport_mop_in_progress++; 2645 2646 /* Indicate to the framework that a reset has happened */ 2647 bzero((void *)&sdevice, sizeof (sata_device_t)); 2648 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2649 sdevice.satadev_addr.pmport = pmport; 2650 if (AHCI_ADDR_IS_PMULT(addrp)) 2651 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 2652 else 2653 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT; 2654 sdevice.satadev_state = SATA_DSTATE_RESET | 2655 SATA_DSTATE_PWR_ACTIVE; 2656 mutex_exit(&ahci_portp->ahciport_mutex); 2657 sata_hba_event_notify( 2658 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2659 &sdevice, 2660 SATA_EVNT_DEVICE_RESET); 2661 mutex_enter(&ahci_portp->ahciport_mutex); 2662 2663 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2664 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET", 2665 port, pmport); 2666 2667 /* Next try to mop the pending commands */ 2668 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2669 finished_tags = ahci_portp->ahciport_pending_tags & 2670 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2671 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2672 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2673 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2674 reset_tags &= ~finished_tags; 2675 2676 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 2677 "reset_tags = %x, finished_tags = %x, slot_status = %x", 2678 reset_tags, finished_tags, slot_status); 2679 2680 /* 2681 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we 2682 * try to reset a single device behind a port multiplier will 2683 * terminate all the commands on that HBA port. We need mop these 2684 * commands as well. 2685 */ 2686 ahci_mop_commands(ahci_ctlp, 2687 ahci_portp, 2688 slot_status, 2689 0, /* failed tags */ 2690 0, /* timeout tags */ 2691 0, /* aborted tags */ 2692 reset_tags); /* reset tags */ 2693 2694 return (SATA_SUCCESS); 2695 } 2696 2697 /* 2698 * Used to do port reset and reject all the pending packets on a port during 2699 * the reset operation. 2700 * 2701 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2702 */ 2703 static int 2704 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2705 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2706 { 2707 uint32_t slot_status = 0; 2708 uint32_t reset_tags = 0; 2709 uint32_t finished_tags = 0; 2710 uint8_t port = addrp->aa_port; 2711 2712 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2713 "ahci_reset_port_reject_pkts at port: %d", port); 2714 2715 /* 2716 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2717 * commands are being mopped, therefore there is nothing else to do 2718 */ 2719 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2720 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2721 "ahci_reset_port_reject_pkts: port %d is in " 2722 "mopping process, so return directly ", port); 2723 return (SATA_SUCCESS); 2724 } 2725 2726 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2727 ahci_portp->ahciport_mop_in_progress++; 2728 2729 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2730 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2731 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2732 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2733 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2735 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2736 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2737 } 2738 2739 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2740 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2741 NULL) != AHCI_SUCCESS) { 2742 2743 /* Clear mop flag */ 2744 ahci_portp->ahciport_mop_in_progress--; 2745 if (ahci_portp->ahciport_mop_in_progress == 0) 2746 ahci_portp->ahciport_flags &= 2747 ~AHCI_PORT_FLAG_MOPPING; 2748 return (SATA_FAILURE); 2749 } 2750 2751 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2752 finished_tags = ahci_portp->ahciport_pending_tags & 2753 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2754 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2755 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2756 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2757 2758 reset_tags &= ~finished_tags; 2759 2760 ahci_mop_commands(ahci_ctlp, 2761 ahci_portp, 2762 slot_status, 2763 0, /* failed tags */ 2764 0, /* timeout tags */ 2765 0, /* aborted tags */ 2766 reset_tags); /* reset tags */ 2767 2768 return (SATA_SUCCESS); 2769 } 2770 2771 /* 2772 * Used to do hba reset and reject all the pending packets on all ports 2773 * during the reset operation. 2774 */ 2775 static int 2776 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2777 { 2778 ahci_port_t *ahci_portp; 2779 uint32_t slot_status[AHCI_MAX_PORTS]; 2780 uint32_t reset_tags[AHCI_MAX_PORTS]; 2781 uint32_t finished_tags[AHCI_MAX_PORTS]; 2782 int port; 2783 int ret = SATA_SUCCESS; 2784 2785 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2786 "ahci_reset_hba_reject_pkts enter", NULL); 2787 2788 bzero(slot_status, sizeof (slot_status)); 2789 bzero(reset_tags, sizeof (reset_tags)); 2790 bzero(finished_tags, sizeof (finished_tags)); 2791 2792 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2793 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2794 continue; 2795 } 2796 2797 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2798 2799 mutex_enter(&ahci_portp->ahciport_mutex); 2800 ahci_portp->ahciport_reset_in_progress = 1; 2801 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2802 slot_status[port] = ddi_get32( 2803 ahci_ctlp->ahcictl_ahci_acc_handle, 2804 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2805 reset_tags[port] = slot_status[port] & 2806 AHCI_SLOT_MASK(ahci_ctlp); 2807 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2808 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 2809 port, reset_tags[port], 2810 ahci_portp->ahciport_pending_tags); 2811 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2812 slot_status[port] = ddi_get32( 2813 ahci_ctlp->ahcictl_ahci_acc_handle, 2814 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2815 reset_tags[port] = slot_status[port] & 2816 AHCI_NCQ_SLOT_MASK(ahci_portp); 2817 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2818 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 2819 port, reset_tags[port], 2820 ahci_portp->ahciport_pending_tags); 2821 } 2822 mutex_exit(&ahci_portp->ahciport_mutex); 2823 } 2824 2825 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2826 ret = SATA_FAILURE; 2827 } 2828 2829 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2830 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2831 continue; 2832 } 2833 2834 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2835 2836 mutex_enter(&ahci_portp->ahciport_mutex); 2837 /* 2838 * To prevent recursive enter to ahci_mop_commands, we need 2839 * check AHCI_PORT_FLAG_MOPPING flag. 2840 */ 2841 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2842 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2843 "ahci_reset_hba_reject_pkts: port %d is in " 2844 "mopping process, so return directly ", port); 2845 mutex_exit(&ahci_portp->ahciport_mutex); 2846 continue; 2847 } 2848 2849 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2850 ahci_portp->ahciport_mop_in_progress++; 2851 2852 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2853 finished_tags[port] = 2854 ahci_portp->ahciport_pending_tags & 2855 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2856 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2857 finished_tags[port] = 2858 ahci_portp->ahciport_pending_ncq_tags & 2859 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2860 2861 reset_tags[port] &= ~finished_tags[port]; 2862 2863 ahci_mop_commands(ahci_ctlp, 2864 ahci_portp, 2865 slot_status[port], 2866 0, /* failed tags */ 2867 0, /* timeout tags */ 2868 0, /* aborted tags */ 2869 reset_tags[port]); /* reset tags */ 2870 mutex_exit(&ahci_portp->ahciport_mutex); 2871 } 2872 out: 2873 return (ret); 2874 } 2875 2876 /* 2877 * Called by sata framework to reset a port(s) or device. 2878 */ 2879 static int 2880 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2881 { 2882 ahci_ctl_t *ahci_ctlp; 2883 ahci_port_t *ahci_portp; 2884 ahci_addr_t addr; 2885 uint8_t cport = sd->satadev_addr.cport; 2886 uint8_t pmport = sd->satadev_addr.pmport; 2887 uint8_t port; 2888 int ret = SATA_SUCCESS; 2889 int instance = ddi_get_instance(dip); 2890 2891 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2892 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2893 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2894 2895 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 2896 2897 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2898 "ahci_tran_reset_dport enter: cport %d", cport); 2899 2900 switch (sd->satadev_addr.qual) { 2901 case SATA_ADDR_PMPORT: 2902 /* 2903 * If we want to issue a COMRESET on a pmport, we need to 2904 * reject the outstanding commands on that pmport. According 2905 * to AHCI spec, PxCI register could only be cleared by 2906 * clearing PxCMD.ST, which will halt the controller port - as 2907 * well as other pmports. 2908 * 2909 * Therefore we directly reset the controller port for 2910 * simplicity. ahci_tran_probe_port() will handle reset stuff 2911 * like initializing the given pmport. 2912 */ 2913 /* FALLTHRU */ 2914 case SATA_ADDR_CPORT: 2915 /* Port reset */ 2916 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2917 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2918 "port %d reset port", instance, port); 2919 2920 mutex_enter(&ahci_portp->ahciport_mutex); 2921 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr); 2922 mutex_exit(&ahci_portp->ahciport_mutex); 2923 2924 break; 2925 2926 case SATA_ADDR_DPMPORT: 2927 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2928 "port %d:%d reset device", instance, port, pmport); 2929 /* FALLTHRU */ 2930 case SATA_ADDR_DCPORT: 2931 /* Device reset */ 2932 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT) 2933 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2934 "port %d reset device", instance, port); 2935 2936 mutex_enter(&ahci_portp->ahciport_mutex); 2937 /* 2938 * software reset request must be sent to SATA_PMULT_HOSTPORT 2939 * if target is a port multiplier: 2940 */ 2941 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT && 2942 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) 2943 AHCI_ADDR_SET_PMULT(&addr, port); 2944 2945 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2946 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2947 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2948 /* 2949 * In case the targer driver would send the request 2950 * before sata framework can have the opportunity to 2951 * process those event reports. 2952 */ 2953 sd->satadev_state = ahci_portp->ahciport_port_state; 2954 ahci_update_sata_registers(ahci_ctlp, port, sd); 2955 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2956 "ahci_tran_reset_dport returning SATA_FAILURE " 2957 "while port in FAILED/SHUTDOWN/PWROFF state: " 2958 "port: %d", port); 2959 mutex_exit(&ahci_portp->ahciport_mutex); 2960 ret = SATA_FAILURE; 2961 break; 2962 } 2963 2964 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == 2965 SATA_DTYPE_NONE) { 2966 /* 2967 * ahci_intr_phyrdy_change() may have rendered it to 2968 * AHCI_PORT_TYPE_NODEV. 2969 */ 2970 sd->satadev_type = SATA_DTYPE_NONE; 2971 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp, 2972 &addr); 2973 ahci_update_sata_registers(ahci_ctlp, port, sd); 2974 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2975 "ahci_tran_reset_dport returning SATA_FAILURE " 2976 "while no device attached: port: %d", port); 2977 mutex_exit(&ahci_portp->ahciport_mutex); 2978 ret = SATA_FAILURE; 2979 break; 2980 } 2981 2982 if (AHCI_ADDR_IS_PORT(&addr)) { 2983 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2984 ahci_portp, &addr); 2985 } else { 2986 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp, 2987 ahci_portp, &addr); 2988 } 2989 2990 mutex_exit(&ahci_portp->ahciport_mutex); 2991 break; 2992 2993 case SATA_ADDR_CNTRL: 2994 /* Reset the whole controller */ 2995 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2996 "reset the whole hba", instance); 2997 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2998 break; 2999 3000 default: 3001 ret = SATA_FAILURE; 3002 } 3003 3004 return (ret); 3005 } 3006 3007 /* 3008 * Called by sata framework to activate a port as part of hotplug. 3009 * (cfgadm -c connect satax/y) 3010 * Support port multiplier. 3011 */ 3012 static int 3013 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 3014 { 3015 ahci_ctl_t *ahci_ctlp; 3016 ahci_port_t *ahci_portp; 3017 ahci_addr_t addr; 3018 uint8_t cport = satadev->satadev_addr.cport; 3019 uint8_t pmport = satadev->satadev_addr.pmport; 3020 uint8_t port; 3021 int instance = ddi_get_instance(dip); 3022 3023 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3024 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3025 3026 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3027 "ahci_tran_hotplug_port_activate enter: cport %d", cport); 3028 3029 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3030 3031 mutex_enter(&ahci_portp->ahciport_mutex); 3032 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3033 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3034 3035 if (AHCI_ADDR_IS_PORT(&addr)) { 3036 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", 3037 instance, port); 3038 3039 /* Enable the interrupts on the port */ 3040 ahci_enable_port_intrs(ahci_ctlp, port); 3041 3042 /* 3043 * Reset the port so that the PHY communication would be 3044 * re-established. But this reset is an internal operation 3045 * and the sata module doesn't need to know about it. 3046 * Moreover, the port with a device attached will be started 3047 * too. 3048 */ 3049 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 3050 ahci_portp, port, 3051 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 3052 NULL); 3053 3054 /* 3055 * Need to check the link status and device status of the port 3056 * and consider raising power if the port was in D3 state 3057 */ 3058 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 3059 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 3060 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 3061 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3062 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated", 3063 instance, port, pmport); 3064 /* AHCI_ADDR_PMPORT */ 3065 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON; 3066 AHCIPORT_PMSTATE(ahci_portp, &addr) &= 3067 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN); 3068 } 3069 3070 satadev->satadev_state = ahci_portp->ahciport_port_state; 3071 3072 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3073 3074 mutex_exit(&ahci_portp->ahciport_mutex); 3075 return (SATA_SUCCESS); 3076 } 3077 3078 /* 3079 * Called by sata framework to deactivate a port as part of hotplug. 3080 * (cfgadm -c disconnect satax/y) 3081 * Support port multiplier. 3082 */ 3083 static int 3084 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 3085 { 3086 ahci_ctl_t *ahci_ctlp; 3087 ahci_port_t *ahci_portp; 3088 ahci_addr_t addr; 3089 uint8_t cport = satadev->satadev_addr.cport; 3090 uint8_t pmport = satadev->satadev_addr.pmport; 3091 uint8_t port; 3092 uint32_t port_scontrol; 3093 int instance = ddi_get_instance(dip); 3094 3095 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3096 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3097 3098 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3099 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport); 3100 3101 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3102 mutex_enter(&ahci_portp->ahciport_mutex); 3103 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3104 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3105 3106 if (AHCI_ADDR_IS_PORT(&addr)) { 3107 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 3108 instance, port); 3109 3110 /* Disable the interrupts on the port */ 3111 ahci_disable_port_intrs(ahci_ctlp, port); 3112 3113 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 3114 3115 /* First to abort all the pending commands */ 3116 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3117 3118 /* Then stop the port */ 3119 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3120 ahci_portp, port); 3121 } 3122 3123 /* Next put the PHY offline */ 3124 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3125 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3126 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 3127 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *) 3128 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3129 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3130 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated", 3131 instance, port, pmport); 3132 3133 ahci_disable_port_intrs(ahci_ctlp, port); 3134 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) 3135 != SATA_DTYPE_NONE) 3136 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3137 3138 /* Re-enable the interrupts for the other pmports */ 3139 ahci_enable_port_intrs(ahci_ctlp, port); 3140 } 3141 3142 /* Update port state */ 3143 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN); 3144 satadev->satadev_state = SATA_PSTATE_SHUTDOWN; 3145 3146 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3147 3148 mutex_exit(&ahci_portp->ahciport_mutex); 3149 return (SATA_SUCCESS); 3150 } 3151 3152 /* 3153 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 3154 * when a device is unplugged or a port is deactivated. 3155 * 3156 * WARNING!!! ahciport_mutex should be acquired before the function is called. 3157 */ 3158 static void 3159 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 3160 ahci_port_t *ahci_portp, uint8_t port) 3161 { 3162 uint32_t slot_status = 0; 3163 uint32_t abort_tags = 0; 3164 3165 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 3166 "ahci_reject_all_abort_pkts at port: %d", port); 3167 3168 /* Read/write port multiplier command takes highest priority */ 3169 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3170 slot_status = 0x1; 3171 abort_tags = 0x1; 3172 goto out; 3173 } 3174 3175 /* 3176 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 3177 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 3178 * to get the error data, if yes when the device is removed, the 3179 * command needs to be aborted too. 3180 */ 3181 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3182 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 3183 slot_status = 0x1; 3184 abort_tags = 0x1; 3185 goto out; 3186 } else { 3187 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3188 "ahci_reject_all_abort_pkts return directly " 3189 "port %d no needs to reject any outstanding " 3190 "commands", port); 3191 return; 3192 } 3193 } 3194 3195 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3196 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3197 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3198 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 3199 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3200 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3201 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3202 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 3203 } 3204 3205 out: 3206 /* No need to do mop when there is no outstanding commands */ 3207 if (slot_status != 0) { 3208 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3209 ahci_portp->ahciport_mop_in_progress++; 3210 3211 ahci_mop_commands(ahci_ctlp, 3212 ahci_portp, 3213 slot_status, 3214 0, /* failed tags */ 3215 0, /* timeout tags */ 3216 abort_tags, /* aborting tags */ 3217 0); /* reset tags */ 3218 } 3219 } 3220 3221 #if defined(__lock_lint) 3222 static int 3223 ahci_selftest(dev_info_t *dip, sata_device_t *device) 3224 { 3225 return (SATA_SUCCESS); 3226 } 3227 #endif 3228 3229 /* 3230 * Allocate the ports structure, only called by ahci_attach 3231 */ 3232 static int 3233 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 3234 { 3235 int port, cport = 0; 3236 3237 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3238 "ahci_alloc_ports_state enter", NULL); 3239 3240 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3241 3242 /* Allocate structures only for the implemented ports */ 3243 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3244 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3245 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3246 "hba port %d not implemented", port); 3247 continue; 3248 } 3249 3250 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 3251 ahci_ctlp->ahcictl_port_to_cport[port] = 3252 (uint8_t)cport++; 3253 3254 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 3255 goto err_out; 3256 } 3257 } 3258 3259 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3260 return (AHCI_SUCCESS); 3261 3262 err_out: 3263 for (port--; port >= 0; port--) { 3264 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3265 ahci_dealloc_port_state(ahci_ctlp, port); 3266 } 3267 } 3268 3269 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3270 return (AHCI_FAILURE); 3271 } 3272 3273 /* 3274 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 3275 */ 3276 static void 3277 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 3278 { 3279 int port; 3280 3281 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3282 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3283 /* if this port is implemented by the HBA */ 3284 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3285 ahci_dealloc_port_state(ahci_ctlp, port); 3286 } 3287 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3288 } 3289 3290 /* 3291 * Drain the taskq. 3292 */ 3293 static void 3294 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 3295 { 3296 ahci_port_t *ahci_portp; 3297 int port; 3298 3299 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3300 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3301 continue; 3302 } 3303 3304 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3305 3306 mutex_enter(&ahci_portp->ahciport_mutex); 3307 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 3308 mutex_exit(&ahci_portp->ahciport_mutex); 3309 } 3310 } 3311 3312 /* 3313 * Initialize the controller and all ports. And then try to start the ports 3314 * if there are devices attached. 3315 * 3316 * This routine can be called from three seperate cases: DDI_ATTACH, 3317 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 3318 * other two cases; device signature probing are attempted only during 3319 * DDI_ATTACH case. 3320 * 3321 * WARNING!!! Disable the whole controller's interrupts before calling and 3322 * the interrupts will be enabled upon successfully return. 3323 */ 3324 static int 3325 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 3326 { 3327 ahci_port_t *ahci_portp; 3328 ahci_addr_t addr; 3329 uint32_t ghc_control; 3330 int port; 3331 3332 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3333 "ahci_initialize_controller enter", NULL); 3334 3335 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3336 3337 /* 3338 * Indicate that system software is AHCI aware by setting 3339 * GHC.AE to 1 3340 */ 3341 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3342 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3343 3344 ghc_control |= AHCI_HBA_GHC_AE; 3345 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3346 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 3347 ghc_control); 3348 3349 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3350 3351 /* Initialize the implemented ports and structures */ 3352 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3353 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3354 continue; 3355 } 3356 3357 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3358 mutex_enter(&ahci_portp->ahciport_mutex); 3359 3360 /* 3361 * Ensure that the controller is not in the running state 3362 * by checking every implemented port's PxCMD register 3363 */ 3364 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port); 3365 3366 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr) 3367 != AHCI_SUCCESS) { 3368 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3369 "ahci_initialize_controller: failed to " 3370 "initialize port %d", port); 3371 /* 3372 * Set the port state to SATA_PSTATE_FAILED if 3373 * failed to initialize it. 3374 */ 3375 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3376 } 3377 3378 mutex_exit(&ahci_portp->ahciport_mutex); 3379 } 3380 3381 /* Enable the whole controller interrupts */ 3382 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3383 ahci_enable_all_intrs(ahci_ctlp); 3384 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3385 3386 return (AHCI_SUCCESS); 3387 } 3388 3389 /* 3390 * Reverse of ahci_initialize_controller() 3391 * 3392 * We only need to stop the ports and disable the interrupt. 3393 */ 3394 static void 3395 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 3396 { 3397 ahci_port_t *ahci_portp; 3398 int port; 3399 3400 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3401 "ahci_uninitialize_controller enter", NULL); 3402 3403 /* disable all the interrupts. */ 3404 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3405 ahci_disable_all_intrs(ahci_ctlp); 3406 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3407 3408 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3409 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3410 continue; 3411 } 3412 3413 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3414 3415 /* Stop the port by clearing PxCMD.ST */ 3416 mutex_enter(&ahci_portp->ahciport_mutex); 3417 3418 /* 3419 * Here we must disable the port interrupt because 3420 * ahci_disable_all_intrs only clear GHC.IE, and IS 3421 * register will be still set if PxIE is enabled. 3422 * When ahci shares one IRQ with other drivers, the 3423 * intr handler may claim the intr mistakenly. 3424 */ 3425 ahci_disable_port_intrs(ahci_ctlp, port); 3426 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3427 ahci_portp, port); 3428 mutex_exit(&ahci_portp->ahciport_mutex); 3429 } 3430 } 3431 3432 /* 3433 * ahci_alloc_pmult() 3434 * 1. Setting HBA port registers which are necessary for a port multiplier. 3435 * (Set PxCMD.PMA while PxCMD.ST is '0') 3436 * 2. Allocate ahci_pmult_info structure. 3437 * 3438 * NOTE: Must stop port before the function is called. 3439 * WARNING!!! ahciport_mutex should be acquired before the function is 3440 * called. 3441 */ 3442 static void 3443 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3444 { 3445 uint32_t port_cmd_status; 3446 uint8_t port = ahci_portp->ahciport_port_num; 3447 3448 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3449 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3450 3451 /* The port must have been stopped before. */ 3452 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST)); 3453 3454 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) { 3455 /* set PMA bit */ 3456 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3457 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3458 port_cmd_status|AHCI_CMD_STATUS_PMA); 3459 3460 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3461 "ahci_alloc_pmult: " 3462 "PxCMD.PMA bit set at port %d.", port); 3463 } 3464 3465 /* Allocate port multiplier information structure */ 3466 if (ahci_portp->ahciport_pmult_info == NULL) { 3467 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *) 3468 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP); 3469 } 3470 3471 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 3472 } 3473 3474 /* 3475 * ahci_dealloc_pmult() 3476 * 1. Clearing related registers when a port multiplier is detached. 3477 * (Clear PxCMD.PMA while PxCMD.ST is '0') 3478 * 2. Deallocate ahci_pmult_info structure. 3479 * 3480 * NOTE: Must stop port before the function is called. 3481 * WARNING!!! ahciport_mutex should be acquired before the function is 3482 * called. 3483 */ 3484 static void 3485 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3486 { 3487 uint32_t port_cmd_status; 3488 uint8_t port = ahci_portp->ahciport_port_num; 3489 3490 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3491 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3492 3493 if (port_cmd_status & AHCI_CMD_STATUS_PMA) { 3494 /* Clear PMA bit */ 3495 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3496 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3497 (port_cmd_status & (~AHCI_CMD_STATUS_PMA))); 3498 3499 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3500 "ahci_dealloc_pmult: " 3501 "PxCMD.PMA bit cleared at port %d.", port); 3502 } 3503 3504 /* Release port multiplier information structure */ 3505 if (ahci_portp->ahciport_pmult_info != NULL) { 3506 kmem_free(ahci_portp->ahciport_pmult_info, 3507 sizeof (ahci_pmult_info_t)); 3508 ahci_portp->ahciport_pmult_info = NULL; 3509 } 3510 } 3511 3512 /* 3513 * The routine is to initialize a port. First put the port in NOTRunning 3514 * state, then enable port interrupt and clear Serror register. And under 3515 * AHCI_ATTACH case, find device signature and then try to start the port. 3516 * 3517 * Called by 3518 * 1. ahci_initialize_controller 3519 * 2. ahci_intr_phyrdy_change (hotplug) 3520 * 3521 * WARNING!!! ahciport_mutex should be acquired before the function is called. 3522 */ 3523 static int 3524 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3525 ahci_addr_t *addrp) 3526 { 3527 uint32_t port_sstatus, port_task_file, port_cmd_status; 3528 uint8_t port = addrp->aa_port; 3529 boolean_t pm_mode = B_TRUE; /* Power Management mode */ 3530 int ret; 3531 3532 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 3533 3534 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */ 3535 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 3536 3537 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3538 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3539 3540 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3541 "ahci_initialize_port: port %d ", port); 3542 3543 /* 3544 * Check whether the port is in NotRunning state, if not, 3545 * put the port in NotRunning state 3546 */ 3547 if (port_cmd_status & 3548 (AHCI_CMD_STATUS_ST | 3549 AHCI_CMD_STATUS_CR | 3550 AHCI_CMD_STATUS_FRE | 3551 AHCI_CMD_STATUS_FR)) { 3552 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3553 ahci_portp, port); 3554 } 3555 3556 /* Disable interrupt */ 3557 ahci_disable_port_intrs(ahci_ctlp, port); 3558 3559 /* Device is unknown at first */ 3560 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 3561 3562 /* Disable the interface power management */ 3563 ahci_disable_interface_pm(ahci_ctlp, port); 3564 3565 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3566 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3567 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3568 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3569 3570 /* Check physcial link status */ 3571 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 3572 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 3573 3574 /* Check interface status */ 3575 port_task_file & AHCI_TFD_STS_BSY || 3576 port_task_file & AHCI_TFD_STS_DRQ || 3577 3578 /* Check whether port reset must be executed */ 3579 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET) { 3580 3581 /* Something went wrong, we need do some reset things */ 3582 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 3583 3584 /* Does port reset succeed on HBA port? */ 3585 if (ret != AHCI_SUCCESS) { 3586 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 3587 "ahci_initialize_port:" 3588 "port reset faild at port %d", port); 3589 return (AHCI_FAILURE); 3590 } 3591 3592 /* Is port failed? */ 3593 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 3594 SATA_PSTATE_FAILED) { 3595 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 3596 "ahci_initialize_port: port %d state 0x%x", 3597 port, ahci_portp->ahciport_port_state); 3598 return (AHCI_FAILURE); 3599 } 3600 } 3601 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 3602 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 3603 3604 /* 3605 * At the time being, only probe ports/devices and get the types of 3606 * attached devices during DDI_ATTACH. In fact, the device can be 3607 * changed during power state changes, but at the time being, we 3608 * don't support the situation. 3609 */ 3610 mutex_exit(&ahci_portp->ahciport_mutex); 3611 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3612 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) 3613 pm_mode = B_FALSE; 3614 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3615 mutex_enter(&ahci_portp->ahciport_mutex); 3616 3617 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) 3618 pm_mode = B_FALSE; 3619 3620 if (!pm_mode) { 3621 /* 3622 * Try to get the device signature if the port is 3623 * not empty. 3624 */ 3625 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 3626 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 3627 } else { 3628 3629 /* 3630 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 3631 * and PxFBU registers in case these registers were cleared 3632 * during the suspend. 3633 */ 3634 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3635 "ahci_initialize_port: port %d " 3636 "reset the port during resume", port); 3637 (void) ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 3638 3639 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3640 "ahci_initialize_port: port %d " 3641 "set PxCLB, PxCLBU, PxFB and PxFBU " 3642 "during resume", port); 3643 3644 /* Config Port Received FIS Base Address */ 3645 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 3646 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 3647 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 3648 3649 /* Config Port Command List Base Address */ 3650 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 3651 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 3652 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 3653 } 3654 3655 /* Return directly if no device connected */ 3656 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3657 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3658 "No device connected to port %d", port); 3659 goto out; 3660 } 3661 3662 /* If this is a port multiplier, we need do some initialization */ 3663 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 3664 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 3665 "Port multiplier found at port %d", port); 3666 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 3667 } 3668 3669 /* Try to start the port */ 3670 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 3671 != AHCI_SUCCESS) { 3672 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3673 "failed to start port %d", port); 3674 return (AHCI_FAILURE); 3675 } 3676 out: 3677 /* Enable port interrupts */ 3678 ahci_enable_port_intrs(ahci_ctlp, port); 3679 3680 return (AHCI_SUCCESS); 3681 } 3682 3683 /* 3684 * Handle hardware defect, and check the capabilities. For example, 3685 * power management capabilty and MSI capability. 3686 */ 3687 static int 3688 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 3689 { 3690 ushort_t caps_ptr, cap_count, cap; 3691 #if AHCI_DEBUG 3692 ushort_t pmcap, pmcsr; 3693 ushort_t msimc; 3694 #endif 3695 uint8_t revision; 3696 3697 ahci_ctlp->ahcictl_venid = 3698 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3699 PCI_CONF_VENID); 3700 3701 ahci_ctlp->ahcictl_devid = 3702 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3703 PCI_CONF_DEVID); 3704 3705 /* 3706 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 3707 * controllers with 0x00 revision id work on 4-byte aligned buffer, 3708 * which is a bug and was fixed after 0x00 revision id controllers. 3709 * 3710 * Moreover, VT8251 cannot use multiple command slots in the command 3711 * list for non-queued commands because the previous register content 3712 * of PxCI can be re-written in the register write, so a flag will be 3713 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 3714 */ 3715 if (ahci_ctlp->ahcictl_venid == VIA_VENID) { 3716 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3717 PCI_CONF_REVID); 3718 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3719 "revision id = 0x%x", revision); 3720 if (revision == 0x00) { 3721 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 3722 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3723 "change ddi_attr_align to 0x4", NULL); 3724 } 3725 3726 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 3727 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3728 "VT8251 cannot use multiple command lists for " 3729 "non-queued commands", NULL); 3730 } 3731 3732 /* 3733 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit 3734 * DMA addressing for communication memory descriptors though S64A bit 3735 * of CAP register declares it supports. Even though 64-bit DMA for 3736 * data buffer works on ASUS M2A-VM with newer BIOS, three other 3737 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and 3738 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller. 3739 * 3740 * Due to certain hardware issue, the chipset must do port reset during 3741 * initialization, otherwise, when retrieving device signature, 3742 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag 3743 * need to set. 3744 * 3745 * For this chipset software reset will get failure if the pmport of 3746 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port 3747 * multiplier is connected to the port. In order to fix the issue, 3748 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software 3749 * reset got failure, the driver will try to do another software reset 3750 * with pmport 0. 3751 */ 3752 if (ahci_ctlp->ahcictl_venid == 0x1002 && 3753 ahci_ctlp->ahcictl_devid == 0x4380) { 3754 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 3755 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3756 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 3757 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 3758 3759 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3760 "ATI SB600 cannot do 64-bit DMA for both data buffer and " 3761 "communication memory descriptors though CAP indicates " 3762 "support, so force it to use 32-bit DMA", NULL); 3763 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3764 "ATI SB600 need to do a port reset during initialization", 3765 NULL); 3766 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3767 "ATI SB600 will get software reset failure if pmport " 3768 "is set 0xf and no port multiplier is attached", NULL); 3769 } 3770 3771 /* 3772 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same 3773 * vendor ID and device ID (0x1002,0x4391). 3774 * 3775 * SB700/750 AHCI chipset on some boards doesn't support 64-bit 3776 * DMA addressing for communication memory descriptors though S64A bit 3777 * of CAP register declares the support. However, it does support 3778 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is 3779 * set for this controller. 3780 * 3781 * SB710 has the same initialization issue as SB600, so it also need 3782 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it. 3783 * 3784 * SB700 also has the same issue about software reset, and thus 3785 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed. 3786 */ 3787 if (ahci_ctlp->ahcictl_venid == 0x1002 && 3788 ahci_ctlp->ahcictl_devid == 0x4391) { 3789 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3790 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 3791 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 3792 3793 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3794 "ATI SB700/750 cannot do 64-bit DMA for communication " 3795 "memory descriptors though CAP indicates support, " 3796 "so force it to use 32-bit DMA", NULL); 3797 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3798 "ATI SB710 need to do a port reset during initialization", 3799 NULL); 3800 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3801 "ATI SB700 will get software reset failure if pmport " 3802 "is set 0xf and no port multiplier is attached", NULL); 3803 } 3804 3805 /* 3806 * Check if capabilities list is supported and if so, 3807 * get initial capabilities pointer and clear bits 0,1. 3808 */ 3809 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3810 PCI_CONF_STAT) & PCI_STAT_CAP) { 3811 caps_ptr = P2ALIGN(pci_config_get8( 3812 ahci_ctlp->ahcictl_pci_conf_handle, 3813 PCI_CONF_CAP_PTR), 4); 3814 } else { 3815 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3816 } 3817 3818 /* 3819 * Walk capabilities if supported. 3820 */ 3821 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3822 3823 /* 3824 * Check that we haven't exceeded the maximum number of 3825 * capabilities and that the pointer is in a valid range. 3826 */ 3827 if (++cap_count > PCI_CAP_MAX_PTR) { 3828 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3829 "too many device capabilities", NULL); 3830 return (AHCI_FAILURE); 3831 } 3832 if (caps_ptr < PCI_CAP_PTR_OFF) { 3833 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3834 "capabilities pointer 0x%x out of range", 3835 caps_ptr); 3836 return (AHCI_FAILURE); 3837 } 3838 3839 /* 3840 * Get next capability and check that it is valid. 3841 * For now, we only support power management. 3842 */ 3843 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3844 caps_ptr); 3845 switch (cap) { 3846 case PCI_CAP_ID_PM: 3847 3848 /* power management supported */ 3849 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3850 3851 /* Save PMCSR offset */ 3852 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3853 3854 #if AHCI_DEBUG 3855 pmcap = pci_config_get16( 3856 ahci_ctlp->ahcictl_pci_conf_handle, 3857 caps_ptr + PCI_PMCAP); 3858 pmcsr = pci_config_get16( 3859 ahci_ctlp->ahcictl_pci_conf_handle, 3860 ahci_ctlp->ahcictl_pmcsr_offset); 3861 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3862 "Power Management capability found PCI_PMCAP " 3863 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3864 if ((pmcap & 0x3) == 0x3) 3865 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3866 "PCI Power Management Interface " 3867 "spec 1.2 compliant", NULL); 3868 #endif 3869 break; 3870 3871 case PCI_CAP_ID_MSI: 3872 #if AHCI_DEBUG 3873 msimc = pci_config_get16( 3874 ahci_ctlp->ahcictl_pci_conf_handle, 3875 caps_ptr + PCI_MSI_CTRL); 3876 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3877 "Message Signaled Interrupt capability found " 3878 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3879 #endif 3880 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3881 "MSI capability found", NULL); 3882 break; 3883 3884 case PCI_CAP_ID_PCIX: 3885 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3886 "PCI-X capability found", NULL); 3887 break; 3888 3889 case PCI_CAP_ID_PCI_E: 3890 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3891 "PCI Express capability found", NULL); 3892 break; 3893 3894 case PCI_CAP_ID_MSI_X: 3895 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3896 "MSI-X capability found", NULL); 3897 break; 3898 3899 case PCI_CAP_ID_SATA: 3900 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3901 "SATA capability found", NULL); 3902 break; 3903 3904 case PCI_CAP_ID_VS: 3905 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3906 "Vendor Specific capability found", NULL); 3907 break; 3908 3909 default: 3910 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3911 "unrecognized capability 0x%x", cap); 3912 break; 3913 } 3914 3915 /* 3916 * Get next capabilities pointer and clear bits 0,1. 3917 */ 3918 caps_ptr = P2ALIGN(pci_config_get8( 3919 ahci_ctlp->ahcictl_pci_conf_handle, 3920 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3921 } 3922 3923 return (AHCI_SUCCESS); 3924 } 3925 3926 /* 3927 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE 3928 * PORTMULT command. SYNC & POLLING mode is used. 3929 * 3930 * WARNING!!! ahciport_mutex should be acquired before the function 3931 * is called. 3932 */ 3933 static int 3934 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 3935 uint8_t regn, uint32_t *pregv, uint8_t type) 3936 { 3937 ahci_port_t *ahci_portp; 3938 ahci_addr_t pmult_addr; 3939 sata_pkt_t *spkt; 3940 sata_cmd_t *scmd; 3941 sata_device_t sata_device; 3942 uint8_t port = addrp->aa_port; 3943 uint8_t pmport = addrp->aa_pmport; 3944 uint8_t cport; 3945 uint32_t intr_mask; 3946 int rval; 3947 char portstr[10]; 3948 3949 SET_PORTSTR(portstr, addrp); 3950 cport = ahci_ctlp->ahcictl_port_to_cport[port]; 3951 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3952 3953 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp)); 3954 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 3955 3956 /* Check the existence of the port multiplier */ 3957 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) 3958 return (AHCI_FAILURE); 3959 3960 /* Request a READ/WRITE PORTMULT sata packet. */ 3961 bzero(&sata_device, sizeof (sata_device_t)); 3962 sata_device.satadev_addr.cport = cport; 3963 sata_device.satadev_addr.pmport = pmport; 3964 sata_device.satadev_addr.qual = SATA_ADDR_PMULT; 3965 sata_device.satadev_rev = SATA_DEVICE_REV; 3966 3967 /* 3968 * Make sure no command is outstanding here. All R/W PMULT requests 3969 * come from 3970 * 3971 * 1. ahci_attach() 3972 * The port should be empty. 3973 * 3974 * 2. ahci_tran_probe_port() 3975 * Any request from SATA framework (via ahci_tran_start) should be 3976 * rejected if R/W PMULT command is outstanding. 3977 * 3978 * If we are doing mopping, do not check those flags because no 3979 * command will be actually outstanding. 3980 * 3981 * If the port has been occupied by any other commands, the probe 3982 * function will return a SATA_RETRY. SATA framework will retry 3983 * later. 3984 */ 3985 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3986 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 3987 "R/W PMULT failed: R/W PMULT in progress at port %d.", 3988 port, ahci_portp->ahciport_flags); 3989 return (AHCI_FAILURE); 3990 } 3991 3992 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && ( 3993 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 3994 NCQ_CMD_IN_PROGRESS(ahci_portp) || 3995 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) { 3996 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 3997 "R/W PMULT failed: port %d is occupied (flags 0x%x).", 3998 port, ahci_portp->ahciport_flags); 3999 return (AHCI_FAILURE); 4000 } 4001 4002 /* 4003 * The port multiplier is gone. This may happen when 4004 * 1. Cutting off the power of an enclosure. The device lose the power 4005 * before port multiplier. 4006 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive. 4007 * 4008 * The issued command should be aborted and the following command 4009 * should not be continued. 4010 */ 4011 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) { 4012 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4013 "READ/WRITE PMULT failed: " 4014 "port-mult is removed from port %d", port); 4015 return (AHCI_FAILURE); 4016 } 4017 4018 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT; 4019 4020 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip, 4021 &sata_device, regn, *pregv, type); 4022 4023 /* 4024 * READ/WRITE PORTMULT command is intended to sent to the control port 4025 * of the port multiplier. 4026 */ 4027 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port); 4028 4029 ahci_portp->ahciport_rdwr_pmult_pkt = spkt; 4030 4031 /* No interrupt here. Store the interrupt enable mask. */ 4032 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4033 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4034 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4035 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 4036 4037 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt); 4038 4039 if (rval == AHCI_SUCCESS && 4040 spkt->satapkt_reason == SATA_PKT_COMPLETED) { 4041 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) { 4042 scmd = &spkt->satapkt_cmd; 4043 *pregv = scmd->satacmd_lba_high_lsb << 24 | 4044 scmd->satacmd_lba_mid_lsb << 16 | 4045 scmd->satacmd_lba_low_lsb << 8 | 4046 scmd->satacmd_sec_count_lsb; 4047 } 4048 } else { 4049 /* Failed or not completed. */ 4050 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4051 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s", 4052 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write", 4053 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr); 4054 rval = AHCI_FAILURE; 4055 } 4056 out: 4057 /* Restore the interrupt mask */ 4058 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4059 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask); 4060 4061 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT; 4062 ahci_portp->ahciport_rdwr_pmult_pkt = NULL; 4063 sata_free_rdwr_pmult_pkt(spkt); 4064 return (rval); 4065 } 4066 4067 static int 4068 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4069 uint8_t regn, uint32_t *pregv) 4070 { 4071 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv, 4072 SATA_RDWR_PMULT_PKT_TYPE_READ); 4073 } 4074 4075 static int 4076 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4077 uint8_t regn, uint32_t regv) 4078 { 4079 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v, 4080 SATA_RDWR_PMULT_PKT_TYPE_WRITE); 4081 } 4082 4083 #define READ_PMULT(addrp, r, pv, out) \ 4084 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \ 4085 goto out; 4086 4087 #define WRITE_PMULT(addrp, r, v, out) \ 4088 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \ 4089 goto out; 4090 4091 /* 4092 * Update sata registers on port multiplier, including GSCR/PSCR registers. 4093 * ahci_update_pmult_gscr() 4094 * ahci_update_pmult_pscr() 4095 * 4096 * WARNING!!! ahciport_mutex should be acquired before those functions 4097 * get called. 4098 */ 4099 static int 4100 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4101 sata_pmult_gscr_t *sg) 4102 { 4103 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err); 4104 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err); 4105 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err); 4106 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err); 4107 4108 return (AHCI_SUCCESS); 4109 4110 err: /* R/W PMULT error */ 4111 return (AHCI_FAILURE); 4112 } 4113 4114 static int 4115 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4116 sata_device_t *sd) 4117 { 4118 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4119 4120 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err); 4121 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err); 4122 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err); 4123 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err); 4124 4125 return (AHCI_SUCCESS); 4126 4127 err: /* R/W PMULT error */ 4128 return (AHCI_FAILURE); 4129 } 4130 4131 /* 4132 * ahci_initialize_pmult() 4133 * 4134 * Initialize a port multiplier, including 4135 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16) 4136 * 2. Redefine MASK register. (SATA Chap 16.?) 4137 * 4138 * WARNING!!! ahciport_mutex should be acquired before the function 4139 * is called. 4140 */ 4141 static int 4142 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4143 ahci_addr_t *addrp, sata_device_t *sd) 4144 { 4145 sata_pmult_gscr_t sg; 4146 uint32_t gscr64; 4147 uint8_t port = addrp->aa_port; 4148 4149 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 4150 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4151 "[Initialize] Port-multiplier at port %d.", port); 4152 4153 /* 4154 * Enable features of port multiplier. Currently only 4155 * Asynchronous Notification is enabled. 4156 */ 4157 /* Check gscr64 for supported features. */ 4158 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err); 4159 4160 if (gscr64 & SATA_PMULT_CAP_SNOTIF) { 4161 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4162 "port %d: Port Multiplier supports " 4163 "Asynchronous Notification.", port); 4164 4165 /* Write to gscr96 to enabled features */ 4166 WRITE_PMULT(addrp, SATA_PMULT_GSCR96, 4167 SATA_PMULT_CAP_SNOTIF, err); 4168 4169 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4170 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4171 AHCI_SNOTIF_CLEAR_ALL); 4172 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4173 "port %d: PMult PxSNTF cleared.", port); 4174 4175 } 4176 4177 /* 4178 * Now we need to update gscr33 register to enable hot-plug interrupt 4179 * for sub devices behind port multiplier. 4180 */ 4181 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err); 4182 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4183 "port %d: gscr33 mask set to %x.", port, (0x1ffff)); 4184 4185 /* 4186 * Fetch the number of device ports of the port multiplier 4187 */ 4188 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS) 4189 return (AHCI_FAILURE); 4190 4191 /* Register the port multiplier to SATA Framework. */ 4192 mutex_exit(&ahci_portp->ahciport_mutex); 4193 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg); 4194 mutex_enter(&ahci_portp->ahciport_mutex); 4195 4196 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports = 4197 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK; 4198 4199 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4200 "port %d: pmult sub-port number updated to %x.", port, 4201 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports); 4202 4203 /* Till now port-mult is successfully initialized */ 4204 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT; 4205 return (AHCI_SUCCESS); 4206 4207 err: /* R/W PMULT error */ 4208 return (AHCI_FAILURE); 4209 } 4210 4211 /* 4212 * Initialize a port multiplier port. According to spec, firstly we need 4213 * issue a COMRESET, then a software reset to get its signature. 4214 * 4215 * NOTE: This function should only be called in ahci_probe_pmport() 4216 * WARNING!!! ahciport_mutex should be acquired before the function. 4217 */ 4218 static int 4219 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4220 ahci_addr_t *addrp) 4221 { 4222 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 4223 uint8_t port = addrp->aa_port; 4224 uint8_t pmport = addrp->aa_pmport; 4225 int ret = AHCI_FAILURE; 4226 4227 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4228 4229 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4230 "ahci_initialize_pmport: port %d:%d", port, pmport); 4231 4232 /* Check HBA port state */ 4233 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4234 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4235 "ahci_initialize_pmport:" 4236 "port %d:%d Port Multiplier is failed.", 4237 port, pmport); 4238 return (AHCI_FAILURE); 4239 } 4240 4241 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4242 return (AHCI_FAILURE); 4243 } 4244 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 4245 4246 /* Checking for outstanding commands */ 4247 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4248 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4249 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4250 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4251 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4252 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4253 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4254 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4255 } 4256 4257 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 4258 ahci_portp->ahciport_mop_in_progress++; 4259 4260 /* Clear status */ 4261 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN); 4262 4263 /* Firstly assume an unknown device */ 4264 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4265 4266 ahci_disable_port_intrs(ahci_ctlp, port); 4267 4268 /* port reset is necessary for port multiplier port */ 4269 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) { 4270 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4271 "ahci_initialize_pmport:" 4272 "port reset failed at port %d:%d", 4273 port, pmport); 4274 goto out; 4275 } 4276 4277 /* Is port failed? */ 4278 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4279 SATA_PSTATE_FAILED) { 4280 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4281 "ahci_initialize_pmport: port %d:%d failed. " 4282 "state = 0x%x", port, pmport, 4283 ahci_portp->ahciport_port_state); 4284 goto out; 4285 } 4286 4287 /* Is there any device attached? */ 4288 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp) 4289 == SATA_DTYPE_NONE) { 4290 /* Do not waste time on an empty port */ 4291 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4292 "ahci_initialize_pmport: No device is found " 4293 "at port %d:%d", port, pmport); 4294 ret = AHCI_SUCCESS; 4295 goto out; 4296 } 4297 4298 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4299 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4300 "port %d:%d is ready now.", port, pmport); 4301 4302 /* 4303 * Till now we can assure a device attached to that HBA port and work 4304 * correctly. Now try to get the device signature. This is an optional 4305 * step. If failed, unknown device is assumed, then SATA module will 4306 * continue to use IDENTIFY DEVICE to get the information of the 4307 * device. 4308 */ 4309 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4310 4311 ret = AHCI_SUCCESS; 4312 4313 out: 4314 /* Next try to mop the pending commands */ 4315 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 4316 finished_tags = ahci_portp->ahciport_pending_tags & 4317 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4318 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 4319 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4320 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4321 reset_tags &= ~finished_tags; 4322 4323 ahci_mop_commands(ahci_ctlp, 4324 ahci_portp, 4325 slot_status, 4326 0, /* failed tags */ 4327 0, /* timeout tags */ 4328 0, /* aborted tags */ 4329 reset_tags); /* reset tags */ 4330 4331 /* Clear PxSNTF register if supported. */ 4332 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 4333 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4334 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4335 AHCI_SNOTIF_CLEAR_ALL); 4336 } 4337 4338 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 4339 ahci_enable_port_intrs(ahci_ctlp, port); 4340 return (ret); 4341 } 4342 4343 /* 4344 * ahci_probe_pmult() 4345 * 4346 * This function will be called to probe a port multiplier, which will 4347 * handle hotplug events on port multiplier ports. 4348 * 4349 * NOTE: Only called from ahci_tran_probe_port() 4350 * WARNING!!! ahciport_mutex should be acquired before the function is called. 4351 */ 4352 static int 4353 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4354 ahci_addr_t *addrp) 4355 { 4356 sata_device_t sdevice; 4357 ahci_addr_t pmport_addr; 4358 uint32_t gscr32, port_hotplug_tags; 4359 uint32_t pmport_sstatus; 4360 int dev_exists_now = 0, dev_existed_previously = 0; 4361 uint8_t port = addrp->aa_port; 4362 int npmport; 4363 4364 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */ 4365 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err); 4366 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp); 4367 4368 do { 4369 npmport = ddi_ffs(port_hotplug_tags) - 1; 4370 if (npmport == -1) 4371 /* no pending hot plug events. */ 4372 return (AHCI_SUCCESS); 4373 4374 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4375 "hot-plug event at port %d:%d", port, npmport); 4376 4377 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport); 4378 4379 /* Check previous device at that port */ 4380 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr) 4381 != SATA_DTYPE_NONE) 4382 dev_existed_previously = 1; 4383 4384 /* PxSStatus tells the presence of device. */ 4385 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS, 4386 &pmport_sstatus, err); 4387 4388 if (SSTATUS_GET_DET(pmport_sstatus) == 4389 SSTATUS_DET_DEVPRE_PHYCOM) 4390 dev_exists_now = 1; 4391 4392 /* 4393 * Clear PxSERR is critical. The transition from 0 to 1 will 4394 * emit a FIS which generates an asynchronous notification 4395 * event at controller. If we fail to clear the PxSERR, the 4396 * Async Notif events will no longer be activated on this 4397 * pmport. 4398 */ 4399 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR, 4400 AHCI_SERROR_CLEAR_ALL, err); 4401 4402 bzero((void *)&sdevice, sizeof (sata_device_t)); 4403 sdevice.satadev_addr.cport = ahci_ctlp-> 4404 ahcictl_port_to_cport[port]; 4405 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT; 4406 sdevice.satadev_addr.pmport = (uint8_t)npmport; 4407 sdevice.satadev_state = SATA_PSTATE_PWRON; 4408 4409 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4410 "[Existence] %d -> %d", dev_existed_previously, 4411 dev_exists_now); 4412 4413 if (dev_exists_now) { 4414 if (dev_existed_previously) { 4415 /* Link (may) not change: Exist -> Exist * */ 4416 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 4417 "ahci_probe_pmult: port %d:%d " 4418 "device link lost/established", 4419 port, npmport); 4420 4421 mutex_exit(&ahci_portp->ahciport_mutex); 4422 sata_hba_event_notify( 4423 ahci_ctlp->ahcictl_sata_hba_tran-> 4424 sata_tran_hba_dip, 4425 &sdevice, 4426 SATA_EVNT_LINK_LOST| 4427 SATA_EVNT_LINK_ESTABLISHED); 4428 mutex_enter(&ahci_portp->ahciport_mutex); 4429 } else { 4430 /* Link change: None -> Exist */ 4431 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4432 "ahci_probe_pmult: port %d:%d " 4433 "device link established", port, npmport); 4434 4435 /* Clear port state */ 4436 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4437 SATA_STATE_UNKNOWN); 4438 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4439 "ahci_probe_pmult: port %d " 4440 "ahciport_port_state [Cleared].", port); 4441 4442 mutex_exit(&ahci_portp->ahciport_mutex); 4443 sata_hba_event_notify( 4444 ahci_ctlp->ahcictl_sata_hba_tran-> 4445 sata_tran_hba_dip, 4446 &sdevice, 4447 SATA_EVNT_LINK_ESTABLISHED); 4448 mutex_enter(&ahci_portp->ahciport_mutex); 4449 } 4450 } else { /* No device exists now */ 4451 if (dev_existed_previously) { 4452 4453 /* Link change: Exist -> None */ 4454 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4455 "ahci_probe_pmult: port %d:%d " 4456 "device link lost", port, npmport); 4457 4458 /* An existing device is lost. */ 4459 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4460 SATA_STATE_UNKNOWN); 4461 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr, 4462 SATA_DTYPE_NONE); 4463 4464 mutex_exit(&ahci_portp->ahciport_mutex); 4465 sata_hba_event_notify( 4466 ahci_ctlp->ahcictl_sata_hba_tran-> 4467 sata_tran_hba_dip, 4468 &sdevice, 4469 SATA_EVNT_LINK_LOST); 4470 mutex_enter(&ahci_portp->ahciport_mutex); 4471 } 4472 } 4473 4474 CLEAR_BIT(port_hotplug_tags, npmport); 4475 } while (port_hotplug_tags != 0); 4476 4477 return (AHCI_SUCCESS); 4478 4479 err: /* R/W PMULT error */ 4480 return (AHCI_FAILURE); 4481 } 4482 4483 /* 4484 * Probe and initialize a port multiplier port. 4485 * A port multiplier port could only be initilaizer here. 4486 * 4487 * WARNING!!! ahcictl_mutex should be acquired before the function 4488 * is called. 4489 */ 4490 static int 4491 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4492 ahci_addr_t *addrp, sata_device_t *sd) 4493 { 4494 uint32_t port_state; 4495 uint8_t port = addrp->aa_port; 4496 ahci_addr_t addr_pmult; 4497 4498 /* 4499 * Check the parent - port multiplier first. 4500 */ 4501 4502 /* 4503 * Parent port multiplier might have been removed. This event will be 4504 * ignored and failure. 4505 */ 4506 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 4507 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 4508 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4509 "ahci_tran_probe_port: " 4510 "parent device removed, ignore event.", NULL); 4511 4512 return (AHCI_FAILURE); 4513 } 4514 4515 /* The port is ready? */ 4516 port_state = ahci_portp->ahciport_port_state; 4517 if (!(port_state & SATA_STATE_READY)) { 4518 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4519 "ahci_tran_probe_port: " 4520 "parent port-mult is NOT ready.", NULL); 4521 4522 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 4523 ahci_portp, port, AHCI_PORT_RESET, NULL) != 4524 AHCI_SUCCESS) { 4525 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4526 "ahci_tran_probe_port: " 4527 "restart port-mult failed.", NULL); 4528 return (AHCI_FAILURE); 4529 } 4530 } 4531 4532 /* 4533 * If port-mult is restarted due to some reason, we need 4534 * re-initialized the PMult. 4535 */ 4536 if (!(port_state & SATA_DSTATE_PMULT_INIT)) { 4537 /* Initialize registers on a port multiplier */ 4538 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port); 4539 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp, 4540 &addr_pmult, sd) != AHCI_SUCCESS) 4541 return (AHCI_FAILURE); 4542 } 4543 4544 /* 4545 * Then we check the port-mult port 4546 */ 4547 /* Is this pmport initialized? */ 4548 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 4549 if (!(port_state & SATA_STATE_READY)) { 4550 4551 /* ahci_initialize_pmport() will set READY state */ 4552 if (ahci_initialize_pmport(ahci_ctlp, 4553 ahci_portp, addrp) != AHCI_SUCCESS) 4554 return (AHCI_FAILURE); 4555 } 4556 4557 return (AHCI_SUCCESS); 4558 } 4559 4560 /* 4561 * AHCI device reset ...; a single device on one of the ports is reset, 4562 * but the HBA and physical communication remain intact. This is the 4563 * least intrusive. 4564 * 4565 * When issuing a software reset sequence, there should not be other 4566 * commands in the command list, so we will first clear and then re-set 4567 * PxCMD.ST to clear PxCI. And before issuing the software reset, 4568 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 4569 * cleared unless command list override (PxCMD.CLO) is supported. 4570 * 4571 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 4572 * set before the function is called. 4573 */ 4574 static int 4575 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4576 ahci_addr_t *addrp) 4577 { 4578 ahci_fis_h2d_register_t *h2d_register_fisp; 4579 ahci_cmd_table_t *cmd_table; 4580 ahci_cmd_header_t *cmd_header; 4581 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 4582 int slot, loop_count; 4583 uint8_t port = addrp->aa_port; 4584 uint8_t pmport = addrp->aa_pmport; 4585 int rval = AHCI_FAILURE; 4586 4587 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 4588 "port %d:%d device software resetting (FIS)", port, pmport); 4589 4590 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 4591 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 4592 port) != AHCI_SUCCESS) { 4593 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4594 "ahci_software_reset: cannot stop HBA port %d.", port); 4595 goto out; 4596 } 4597 4598 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 4599 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4600 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4601 4602 if (port_task_file & AHCI_TFD_STS_BSY || 4603 port_task_file & AHCI_TFD_STS_DRQ) { 4604 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 4605 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4606 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 4607 "cannot issue a software reset.", port_task_file); 4608 goto out; 4609 } 4610 4611 /* 4612 * If HBA Support CLO, as Command List Override (CAP.SCLO is 4613 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 4614 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 4615 */ 4616 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4617 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 4618 4619 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4620 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4621 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4622 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4623 port_cmd_status|AHCI_CMD_STATUS_CLO); 4624 4625 /* Waiting till PxCMD.SCLO bit is cleared */ 4626 loop_count = 0; 4627 do { 4628 /* Wait for 10 millisec */ 4629 drv_usecwait(AHCI_10MS_USECS); 4630 4631 /* We are effectively timing out after 1 sec. */ 4632 if (loop_count++ > 100) { 4633 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4634 "SCLO time out. port %d is busy.", port); 4635 goto out; 4636 } 4637 4638 port_cmd_status = 4639 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4640 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4641 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 4642 4643 /* Re-check */ 4644 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4645 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4646 if (port_task_file & AHCI_TFD_STS_BSY || 4647 port_task_file & AHCI_TFD_STS_DRQ) { 4648 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4649 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 4650 port_task_file); 4651 goto out; 4652 } 4653 } 4654 4655 /* Then start port */ 4656 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 4657 != AHCI_SUCCESS) { 4658 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4659 "ahci_software_reset: cannot start AHCI port %d.", port); 4660 goto out; 4661 } 4662 4663 /* 4664 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 4665 * ahci_port.ahciport_pending_ncq_tags may fail 4666 * ahci_claim_free_slot(). Actually according to spec, by clearing 4667 * PxCMD.ST there is no command outstanding while executing software 4668 * reseting. Hence we directly use slot 0 instead of 4669 * ahci_claim_free_slot(). 4670 */ 4671 slot = 0; 4672 4673 /* Now send the first H2D Register FIS with SRST set to 1 */ 4674 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 4675 bzero((void *)cmd_table, ahci_cmd_table_size); 4676 4677 h2d_register_fisp = 4678 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 4679 4680 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 4681 SET_FIS_PMP(h2d_register_fisp, pmport); 4682 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 4683 4684 /* Set Command Header in Command List */ 4685 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 4686 BZERO_DESCR_INFO(cmd_header); 4687 BZERO_PRD_BYTE_COUNT(cmd_header); 4688 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 4689 SET_PORT_MULTI_PORT(cmd_header, pmport); 4690 4691 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 4692 SET_RESET(cmd_header, 1); 4693 SET_WRITE(cmd_header, 1); 4694 4695 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4696 0, 4697 ahci_cmd_table_size, 4698 DDI_DMA_SYNC_FORDEV); 4699 4700 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 4701 slot * sizeof (ahci_cmd_header_t), 4702 sizeof (ahci_cmd_header_t), 4703 DDI_DMA_SYNC_FORDEV); 4704 4705 /* Indicate to the HBA that a command is active. */ 4706 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4707 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 4708 (0x1 << slot)); 4709 4710 loop_count = 0; 4711 4712 /* Loop till the first command is finished */ 4713 do { 4714 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4715 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4716 4717 /* We are effectively timing out after 1 sec. */ 4718 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 4719 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4720 "the first SRST FIS is timed out, " 4721 "loop_count = %d", loop_count); 4722 goto out; 4723 } 4724 /* Wait for 10 millisec */ 4725 drv_usecwait(AHCI_10MS_USECS); 4726 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 4727 4728 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 4729 "ahci_software_reset: 1st loop count: %d, " 4730 "port_cmd_issue = 0x%x, slot = 0x%x", 4731 loop_count, port_cmd_issue, slot); 4732 4733 /* According to ATA spec, we need wait at least 5 microsecs here. */ 4734 drv_usecwait(AHCI_1MS_USECS); 4735 4736 /* Now send the second H2D Register FIS with SRST cleard to zero */ 4737 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 4738 bzero((void *)cmd_table, ahci_cmd_table_size); 4739 4740 h2d_register_fisp = 4741 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 4742 4743 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 4744 SET_FIS_PMP(h2d_register_fisp, pmport); 4745 4746 /* Set Command Header in Command List */ 4747 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 4748 BZERO_DESCR_INFO(cmd_header); 4749 BZERO_PRD_BYTE_COUNT(cmd_header); 4750 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 4751 SET_PORT_MULTI_PORT(cmd_header, pmport); 4752 4753 SET_WRITE(cmd_header, 1); 4754 4755 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4756 0, 4757 ahci_cmd_table_size, 4758 DDI_DMA_SYNC_FORDEV); 4759 4760 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 4761 slot * sizeof (ahci_cmd_header_t), 4762 sizeof (ahci_cmd_header_t), 4763 DDI_DMA_SYNC_FORDEV); 4764 4765 /* Indicate to the HBA that a command is active. */ 4766 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4767 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 4768 (0x1 << slot)); 4769 4770 loop_count = 0; 4771 4772 /* Loop till the second command is finished */ 4773 do { 4774 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4775 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4776 4777 /* We are effectively timing out after 1 sec. */ 4778 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 4779 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4780 "the second SRST FIS is timed out, " 4781 "loop_count = %d", loop_count); 4782 goto out; 4783 } 4784 4785 /* Wait for 10 millisec */ 4786 drv_usecwait(AHCI_10MS_USECS); 4787 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 4788 4789 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 4790 "ahci_software_reset: 2nd loop count: %d, " 4791 "port_cmd_issue = 0x%x, slot = 0x%x", 4792 loop_count, port_cmd_issue, slot); 4793 4794 rval = AHCI_SUCCESS; 4795 out: 4796 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4797 "ahci_software_reset: %s at port %d:%d", 4798 rval == AHCI_SUCCESS ? "succeed" : "failed", 4799 port, pmport); 4800 4801 return (rval); 4802 } 4803 4804 /* 4805 * AHCI port reset ...; the physical communication between the HBA and device 4806 * on a port are disabled. This is more intrusive. 4807 * 4808 * When an HBA or port reset occurs, Phy communication is going to 4809 * be re-established with the device through a COMRESET followed by the 4810 * normal out-of-band communication sequence defined in Serial ATA. AT 4811 * the end of reset, the device, if working properly, will send a D2H 4812 * Register FIS, which contains the device signature. When the HBA receives 4813 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 4814 * the PxSIG register with the signature. 4815 * 4816 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 4817 * to individually spin-up attached devices. Please refer to chapter 10.9 of 4818 * AHCI 1.0 spec. 4819 */ 4820 /* 4821 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 4822 * cleared before the function is called. 4823 */ 4824 static int 4825 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4826 ahci_addr_t *addrp) 4827 { 4828 ahci_addr_t pmult_addr; 4829 uint32_t cap_status, port_cmd_status; 4830 uint32_t port_scontrol, port_sstatus, port_serror; 4831 uint32_t port_intr_status, port_task_file; 4832 uint32_t port_state; 4833 uint8_t port = addrp->aa_port; 4834 4835 int loop_count; 4836 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 4837 4838 /* Target is a port multiplier port? */ 4839 if (AHCI_ADDR_IS_PMPORT(addrp)) 4840 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp)); 4841 4842 /* Otherwise it must be an HBA port. */ 4843 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 4844 4845 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4846 "Port %d port resetting...", port); 4847 ahci_portp->ahciport_port_state = 0; 4848 4849 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4850 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4851 4852 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4853 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4854 4855 if (cap_status & AHCI_HBA_CAP_SSS) { 4856 /* 4857 * HBA support staggered spin-up, if the port has 4858 * not spin up yet, then force it to do spin-up 4859 */ 4860 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 4861 if (!(ahci_portp->ahciport_flags 4862 & AHCI_PORT_FLAG_SPINUP)) { 4863 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4864 "Port %d PxCMD.SUD is zero, force " 4865 "it to do spin-up", port); 4866 ahci_portp->ahciport_flags |= 4867 AHCI_PORT_FLAG_SPINUP; 4868 } 4869 } 4870 } else { 4871 /* 4872 * HBA doesn't support stagger spin-up, force it 4873 * to do normal COMRESET 4874 */ 4875 if (ahci_portp->ahciport_flags & 4876 AHCI_PORT_FLAG_SPINUP) { 4877 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4878 "HBA does not support staggered spin-up " 4879 "force it to do normal COMRESET", NULL); 4880 ahci_portp->ahciport_flags &= 4881 ~AHCI_PORT_FLAG_SPINUP; 4882 } 4883 } 4884 4885 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 4886 /* Do normal COMRESET */ 4887 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4888 "ahci_port_reset: do normal COMRESET", port); 4889 4890 /* 4891 * According to the spec, SUD bit should be set here, 4892 * but JMicron JMB363 doesn't follow it, so remove 4893 * the assertion, and just print a debug message. 4894 */ 4895 #if AHCI_DEBUG 4896 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 4897 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4898 "port %d SUD bit not set", port) 4899 #endif 4900 4901 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4902 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4903 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 4904 4905 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4906 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4907 port_scontrol); 4908 4909 /* Enable PxCMD.FRE to read device */ 4910 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4911 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4912 port_cmd_status|AHCI_CMD_STATUS_FRE); 4913 4914 /* 4915 * Give time for COMRESET to percolate, according to the AHCI 4916 * spec, software shall wait at least 1 millisecond before 4917 * clearing PxSCTL.DET 4918 */ 4919 drv_usecwait(AHCI_1MS_USECS*2); 4920 4921 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 4922 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4923 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4924 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 4925 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4926 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4927 port_scontrol); 4928 } else { 4929 /* Do staggered spin-up */ 4930 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4931 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4932 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 4933 4934 /* PxSCTL.DET must be 0 */ 4935 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4936 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4937 port_scontrol); 4938 4939 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 4940 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4941 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4942 port_cmd_status); 4943 4944 /* 0 -> 1 edge */ 4945 drv_usecwait(AHCI_1MS_USECS*2); 4946 4947 /* Set PxCMD.SUD to 1 */ 4948 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4949 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4950 port_cmd_status |= AHCI_CMD_STATUS_SUD; 4951 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4952 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4953 port_cmd_status); 4954 4955 /* Enable PxCMD.FRE to read device */ 4956 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4957 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4958 port_cmd_status|AHCI_CMD_STATUS_FRE); 4959 } 4960 4961 /* 4962 * The port enters P:StartComm state, and HBA tells link layer to 4963 * start communication, which involves sending COMRESET to device. 4964 * And the HBA resets PxTFD.STS to 7Fh. 4965 * 4966 * When a COMINIT is received from the device, then the port enters 4967 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 4968 * PxSSTS.DET to 1h to indicate a device is detected but communication 4969 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 4970 * a COMINIT has been received. 4971 */ 4972 /* 4973 * The DET field is valid only if IPM field indicates 4974 * that the interface is in active state. 4975 */ 4976 loop_count = 0; 4977 do { 4978 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4979 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 4980 4981 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 4982 /* 4983 * If the interface is not active, the DET field 4984 * is considered not accurate. So we want to 4985 * continue looping. 4986 */ 4987 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 4988 } 4989 4990 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 4991 /* 4992 * We are effectively timing out after 0.1 sec. 4993 */ 4994 break; 4995 } 4996 4997 /* Wait for 10 millisec */ 4998 drv_usecwait(AHCI_10MS_USECS); 4999 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5000 5001 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5002 "ahci_port_reset: 1st loop count: %d, " 5003 "port_sstatus = 0x%x port %d", 5004 loop_count, port_sstatus, port); 5005 5006 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5007 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5008 /* 5009 * Either the port is not active or there 5010 * is no device present. 5011 */ 5012 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5013 return (AHCI_SUCCESS); 5014 } 5015 5016 /* Now we can make sure there is a device connected to the port */ 5017 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5018 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5019 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5020 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5021 5022 /* 5023 * A COMINIT signal is supposed to be received 5024 * PxSERR.DIAG.X or PxIS.PCS should be set 5025 */ 5026 if (!(port_intr_status & AHCI_INTR_STATUS_PCS) && 5027 !(port_serror & SERROR_EXCHANGED_ERR)) { 5028 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5029 "COMINIT signal from the device not received", 5030 instance, port); 5031 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5032 return (AHCI_FAILURE); 5033 } 5034 5035 /* 5036 * According to the spec, when PxSCTL.DET is set to 0h, upon 5037 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 5038 * shall be set to '1' by the HBA. 5039 * 5040 * However, we found JMicron JMB363 doesn't follow this, so 5041 * remove this check, and just print a debug message. 5042 */ 5043 #if AHCI_DEBUG 5044 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5045 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5046 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 5047 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5048 "port %d BSY bit is not set after COMINIT signal " 5049 "is received", port); 5050 } 5051 #endif 5052 5053 /* 5054 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 5055 * the D2H FIS received by HBA. 5056 */ 5057 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5058 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5059 SERROR_EXCHANGED_ERR); 5060 5061 /* 5062 * Devices should return a FIS contains its signature to HBA after 5063 * COMINIT signal. Check whether a D2H FIS is received by polling 5064 * PxTFD.STS.ERR bit. 5065 */ 5066 loop_count = 0; 5067 do { 5068 /* Wait for 10 millisec */ 5069 drv_usecwait(AHCI_10MS_USECS); 5070 5071 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5072 /* 5073 * We are effectively timing out after 11 sec. 5074 */ 5075 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5076 "the device hardware has been initialized and " 5077 "the power-up diagnostics failed", 5078 instance, port); 5079 5080 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5081 "port %d PxTFD.STS.ERR is not set, we need another " 5082 "software reset.", port); 5083 5084 /* Clear port serror register for the port */ 5085 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5086 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5087 AHCI_SERROR_CLEAR_ALL); 5088 5089 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 5090 5091 /* Try another software reset. */ 5092 if (ahci_software_reset(ahci_ctlp, ahci_portp, 5093 &pmult_addr) != AHCI_SUCCESS) { 5094 AHCIPORT_SET_STATE(ahci_portp, addrp, 5095 SATA_PSTATE_FAILED); 5096 return (AHCI_FAILURE); 5097 } 5098 break; 5099 } 5100 5101 /* Wait for 10 millisec */ 5102 drv_usecwait(AHCI_10MS_USECS); 5103 5104 /* 5105 * The Error bit '1' means COMRESET is finished successfully 5106 * The device hardware has been initialized and the power-up 5107 * diagnostics successfully completed. The device requests 5108 * that the Transport layer transmit a Register - D2H FIS to 5109 * the host. (SATA spec 11.5, v2.6) 5110 */ 5111 port_task_file = 5112 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5113 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5114 } while (((port_task_file & AHCI_TFD_ERR_MASK) 5115 >> AHCI_TFD_ERR_SHIFT) != AHCI_TFD_ERR_SGS); 5116 5117 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5118 "ahci_port_reset: 2nd loop count: %d, " 5119 "port_task_file = 0x%x port %d", 5120 loop_count, port_task_file, port); 5121 5122 /* Clear port serror register for the port */ 5123 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5124 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5125 AHCI_SERROR_CLEAR_ALL); 5126 5127 /* Set port as ready */ 5128 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5129 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5130 5131 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5132 "ahci_port_reset: succeed at port %d.", port); 5133 return (AHCI_SUCCESS); 5134 } 5135 5136 /* 5137 * COMRESET on a port multiplier port. 5138 * 5139 * NOTE: Only called in ahci_port_reset() 5140 */ 5141 static int 5142 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5143 ahci_addr_t *addrp) 5144 { 5145 uint32_t port_scontrol, port_sstatus, port_serror; 5146 uint32_t port_cmd_status, port_intr_status; 5147 uint32_t port_state; 5148 uint8_t port = addrp->aa_port; 5149 uint8_t pmport = addrp->aa_pmport; 5150 int loop_count; 5151 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5152 5153 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5154 "port %d:%d: pmport resetting", port, pmport); 5155 5156 /* Initialize pmport state */ 5157 AHCIPORT_SET_STATE(ahci_portp, addrp, 0); 5158 5159 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5160 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5161 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5162 5163 /* PxCMD.FRE should be set before. */ 5164 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5165 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5166 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE); 5167 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) 5168 return (AHCI_FAILURE); 5169 5170 /* 5171 * Give time for COMRESET to percolate, according to the AHCI 5172 * spec, software shall wait at least 1 millisecond before 5173 * clearing PxSCTL.DET 5174 */ 5175 drv_usecwait(AHCI_1MS_USECS*2); 5176 5177 /* 5178 * Fetch the SCONTROL again and rewrite the DET part with 0 5179 * This will generate an Asychronous Notification events. 5180 */ 5181 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5182 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5183 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5184 5185 /* 5186 * The port enters P:StartComm state, and HBA tells link layer to 5187 * start communication, which involves sending COMRESET to device. 5188 * And the HBA resets PxTFD.STS to 7Fh. 5189 * 5190 * When a COMINIT is received from the device, then the port enters 5191 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5192 * PxSSTS.DET to 1h to indicate a device is detected but communication 5193 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5194 * a COMINIT has been received. 5195 */ 5196 /* 5197 * The DET field is valid only if IPM field indicates 5198 * that the interface is in active state. 5199 */ 5200 loop_count = 0; 5201 do { 5202 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err); 5203 5204 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5205 /* 5206 * If the interface is not active, the DET field 5207 * is considered not accurate. So we want to 5208 * continue looping. 5209 */ 5210 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5211 } 5212 5213 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5214 /* 5215 * We are effectively timing out after 0.1 sec. 5216 */ 5217 break; 5218 } 5219 5220 /* Wait for 10 millisec */ 5221 drv_usecwait(AHCI_10MS_USECS); 5222 5223 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5224 5225 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5226 "ahci_pmport_reset: 1st loop count: %d, " 5227 "port_sstatus = 0x%x port %d:%d", 5228 loop_count, port_sstatus, port, pmport); 5229 5230 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5231 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5232 /* 5233 * Either the port is not active or there 5234 * is no device present. 5235 */ 5236 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5237 "ahci_pmport_reset: " 5238 "no device attached to port %d:%d", 5239 port, pmport); 5240 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5241 return (AHCI_SUCCESS); 5242 } 5243 5244 /* Now we can make sure there is a device connected to the port */ 5245 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */ 5246 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err); 5247 5248 if (!(port_serror & (1 << 26))) { 5249 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: " 5250 "COMINIT signal from the device not received port %d:%d", 5251 instance, port, pmport); 5252 5253 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5254 return (AHCI_FAILURE); 5255 } 5256 5257 /* 5258 * After clear PxSERR register, we will receive a D2H FIS. 5259 * Normally this FIS will cause a IPMS error according to AHCI spec 5260 * v1.2 because there is no command outstanding for it. So we need 5261 * to ignore this error. 5262 */ 5263 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS; 5264 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err); 5265 5266 /* Now we need to check the D2H FIS by checking IPMS error. */ 5267 loop_count = 0; 5268 do { 5269 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5270 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5271 5272 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5273 /* 5274 * No D2H FIS received. This is possible according 5275 * to SATA 2.6 spec. 5276 */ 5277 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d " 5278 "PxIS.IPMS is not set, we need another " 5279 "software reset.", port, pmport); 5280 5281 break; 5282 } 5283 5284 /* Wait for 10 millisec */ 5285 mutex_exit(&ahci_portp->ahciport_mutex); 5286 delay(AHCI_10MS_TICKS); 5287 mutex_enter(&ahci_portp->ahciport_mutex); 5288 5289 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS)); 5290 5291 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5292 "ahci_pmport_reset: 2st loop count: %d, " 5293 "port_sstatus = 0x%x port %d:%d", 5294 loop_count, port_sstatus, port, pmport); 5295 5296 /* Clear IPMS */ 5297 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5298 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5299 AHCI_INTR_STATUS_IPMS); 5300 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5301 5302 /* This pmport is now ready for ahci_tran_start() */ 5303 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5304 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5305 5306 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5307 "ahci_pmport_reset: succeed at port %d:%d", port, pmport); 5308 return (AHCI_SUCCESS); 5309 5310 err: /* R/W PMULT error */ 5311 /* IPMS flags might be set before. */ 5312 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5313 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5314 "ahci_pmport_reset: failed at port %d:%d", port, pmport); 5315 5316 return (AHCI_FAILURE); 5317 } 5318 5319 /* 5320 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 5321 * This is the most intrusive. 5322 * 5323 * When an HBA reset occurs, Phy communication will be re-established with 5324 * the device through a COMRESET followed by the normal out-of-band 5325 * communication sequence defined in Serial ATA. AT the end of reset, the 5326 * device, if working properly, will send a D2H Register FIS, which contains 5327 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 5328 * and PxTFD.ERR register fields, and updates the PxSIG register with the 5329 * signature. 5330 * 5331 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 5332 */ 5333 static int 5334 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 5335 { 5336 ahci_port_t *ahci_portp; 5337 ahci_addr_t addr; 5338 uint32_t ghc_control; 5339 uint8_t port; 5340 int loop_count; 5341 int rval = AHCI_SUCCESS; 5342 5343 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 5344 NULL); 5345 5346 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5347 5348 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5349 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5350 5351 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 5352 ghc_control |= AHCI_HBA_GHC_HR; 5353 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5354 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5355 5356 /* 5357 * Wait until HBA Reset complete or timeout 5358 */ 5359 loop_count = 0; 5360 do { 5361 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5362 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5363 5364 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 5365 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5366 "ahci hba reset is timing out, " 5367 "ghc_control = 0x%x", ghc_control); 5368 /* We are effectively timing out after 1 sec. */ 5369 break; 5370 } 5371 5372 /* Wait for 10 millisec */ 5373 drv_usecwait(AHCI_10MS_USECS); 5374 } while (ghc_control & AHCI_HBA_GHC_HR); 5375 5376 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5377 "ahci_hba_reset: 1st loop count: %d, " 5378 "ghc_control = 0x%x", loop_count, ghc_control); 5379 5380 if (ghc_control & AHCI_HBA_GHC_HR) { 5381 /* The hba is not reset for some reasons */ 5382 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5383 "hba reset failed: HBA in a hung or locked state", NULL); 5384 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5385 return (AHCI_FAILURE); 5386 } 5387 5388 /* 5389 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE 5390 */ 5391 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5392 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5393 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE); 5394 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5395 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5396 5397 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5398 5399 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 5400 /* Only check implemented ports */ 5401 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 5402 continue; 5403 } 5404 5405 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5406 mutex_enter(&ahci_portp->ahciport_mutex); 5407 5408 AHCI_ADDR_SET_PORT(&addr, port); 5409 5410 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 5411 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) != 5412 AHCI_SUCCESS) { 5413 rval = AHCI_FAILURE; 5414 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5415 "ahci_hba_reset: port %d failed", port); 5416 /* 5417 * Set the port state to SATA_PSTATE_FAILED if 5418 * failed to initialize it. 5419 */ 5420 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5421 } 5422 5423 mutex_exit(&ahci_portp->ahciport_mutex); 5424 } 5425 5426 return (rval); 5427 } 5428 5429 /* 5430 * This routine is only called from AHCI_ATTACH or phyrdy change 5431 * case. It first calls software reset, then stop the port and try to 5432 * read PxSIG register to find the type of device attached to the port. 5433 * 5434 * The caller should make sure a valid device exists on specified port and 5435 * physical communication has been established so that the signature could 5436 * be retrieved by software reset. 5437 * 5438 * WARNING!!! ahciport_mutex should be acquired before the function 5439 * is called. And the port interrupt is disabled. 5440 */ 5441 static void 5442 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5443 ahci_addr_t *addrp) 5444 { 5445 ahci_addr_t dev_addr; 5446 uint32_t signature; 5447 uint8_t port = addrp->aa_port; 5448 uint8_t pmport = addrp->aa_pmport; 5449 ASSERT(AHCI_ADDR_IS_VALID(addrp)); 5450 int rval; 5451 5452 if (AHCI_ADDR_IS_PORT(addrp)) { 5453 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5454 "ahci_find_dev_signature enter: port %d", port); 5455 5456 /* 5457 * NOTE: when the ahci address is a HBA port, we do not know 5458 * it is a device or a port multiplier that attached. we need 5459 * try a software reset at port multiplier address (0xf 5460 * pmport) 5461 */ 5462 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port); 5463 } else { 5464 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5465 "ahci_find_dev_signature enter: port %d:%d", 5466 port, pmport); 5467 dev_addr = *addrp; 5468 } 5469 5470 /* Assume it is unknown. */ 5471 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5472 5473 /* Issue a software reset to get the signature */ 5474 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr); 5475 if (rval != AHCI_SUCCESS) { 5476 5477 /* 5478 * Try to do software reset again with pmport set with 0 if 5479 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and 5480 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf) 5481 */ 5482 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) && 5483 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) { 5484 dev_addr.aa_pmport = 0; 5485 rval = ahci_software_reset(ahci_ctlp, ahci_portp, 5486 &dev_addr); 5487 } 5488 5489 if (rval != AHCI_SUCCESS) { 5490 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5491 "ahci_find_dev_signature: software reset failed " 5492 "at port %d:%d, cannot get signature.", 5493 port, pmport); 5494 5495 AHCIPORT_SET_STATE(ahci_portp, addrp, 5496 SATA_PSTATE_FAILED); 5497 return; 5498 } 5499 } 5500 5501 /* 5502 * ahci_software_reset has started the port, so we need manually stop 5503 * the port again. 5504 */ 5505 if (AHCI_ADDR_IS_PORT(addrp)) { 5506 if (ahci_put_port_into_notrunning_state(ahci_ctlp, 5507 ahci_portp, port) != AHCI_SUCCESS) { 5508 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5509 "ahci_find_dev_signature: cannot stop port %d.", 5510 port); 5511 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5512 return; 5513 } 5514 } 5515 5516 /* Now we can make sure that a valid signature is received. */ 5517 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5518 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 5519 5520 #ifdef AHCI_DEBUG 5521 if (AHCI_ADDR_IS_PMPORT(addrp)) { 5522 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 5523 "ahci_find_dev_signature: signature = 0x%x at port %d:%d", 5524 signature, port, pmport); 5525 } else { 5526 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5527 "ahci_find_dev_signature: signature = 0x%x at port %d", 5528 signature, port); 5529 } 5530 #endif 5531 5532 /* NOTE: Only support ATAPI device at controller port. */ 5533 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp)) 5534 signature = SATA_DTYPE_UNKNOWN; 5535 5536 switch (signature) { 5537 5538 case AHCI_SIGNATURE_DISK: 5539 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK); 5540 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5541 "Disk is found at port: %d", port); 5542 break; 5543 5544 case AHCI_SIGNATURE_ATAPI: 5545 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI); 5546 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5547 "ATAPI device is found at port: %d", port); 5548 break; 5549 5550 case AHCI_SIGNATURE_PORT_MULTIPLIER: 5551 /* Port Multiplier cannot recursively attached. */ 5552 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5553 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT); 5554 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5555 "Port Multiplier is found at port: %d", port); 5556 break; 5557 5558 default: 5559 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5560 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5561 "Unknown device is found at port: %d", port); 5562 } 5563 } 5564 5565 /* 5566 * According to the spec, to reliably detect hot plug removals, software 5567 * must disable interface power management. Software should perform the 5568 * following initialization on a port after a device is attached: 5569 * Set PxSCTL.IPM to 3h to disable interface state transitions 5570 * Set PxCMD.ALPE to '0' to disable aggressive power management 5571 * Disable device initiated interface power management by SET FEATURE 5572 * 5573 * We can ignore the last item because by default the feature is disabled 5574 */ 5575 static void 5576 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 5577 { 5578 uint32_t port_scontrol, port_cmd_status; 5579 5580 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5581 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5582 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 5583 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5584 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 5585 5586 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5587 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5588 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 5589 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5590 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5591 } 5592 5593 /* 5594 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 5595 * to 1, then set it firstly. 5596 * 5597 * Each port contains two major DMA engines. One DMA engine walks through 5598 * the command list, and is controlled by PxCMD.ST. The second DMA engine 5599 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 5600 * 5601 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 5602 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 5603 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 5604 * 5605 * Software shall not set PxCMD.ST to '1' unless a functional device is 5606 * present on the port(as determined by PxTFD.STS.BSY = '0', 5607 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 5608 * 5609 * WARNING!!! ahciport_mutex should be acquired before the function 5610 * is called. 5611 */ 5612 static int 5613 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 5614 { 5615 uint32_t port_cmd_status; 5616 5617 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 5618 5619 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 5620 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5621 "the state for port %d is 0x%x", 5622 port, ahci_portp->ahciport_port_state); 5623 return (AHCI_FAILURE); 5624 } 5625 5626 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5627 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5628 "no device is attached at port %d", port); 5629 return (AHCI_FAILURE); 5630 } 5631 5632 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 5633 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5634 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5635 5636 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 5637 port_cmd_status |= AHCI_CMD_STATUS_FRE; 5638 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5639 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5640 port_cmd_status); 5641 } 5642 5643 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5644 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5645 5646 port_cmd_status |= AHCI_CMD_STATUS_ST; 5647 5648 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5649 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5650 port_cmd_status); 5651 5652 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 5653 5654 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 5655 "PxCMD.ST set to '1' at port %d", port); 5656 5657 return (AHCI_SUCCESS); 5658 } 5659 5660 /* 5661 * Allocate the ahci_port_t including Received FIS and Command List. 5662 * The argument - port is the physical port number, and not logical 5663 * port number seen by the SATA framework. 5664 * 5665 * WARNING!!! ahcictl_mutex should be acquired before the function 5666 * is called. 5667 */ 5668 static int 5669 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 5670 { 5671 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5672 ahci_port_t *ahci_portp; 5673 char taskq_name[64] = "event_handle_taskq"; 5674 5675 ahci_portp = 5676 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 5677 5678 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 5679 ahci_portp->ahciport_port_num = port; 5680 5681 /* Initialize the port condition variable */ 5682 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 5683 5684 /* Initialize the port mutex */ 5685 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 5686 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 5687 5688 mutex_enter(&ahci_portp->ahciport_mutex); 5689 5690 /* 5691 * Allocate memory for received FIS structure and 5692 * command list for this port 5693 */ 5694 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 5695 goto err_case1; 5696 } 5697 5698 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 5699 goto err_case2; 5700 } 5701 5702 (void) snprintf(taskq_name + strlen(taskq_name), 5703 sizeof (taskq_name) - strlen(taskq_name), 5704 "_port%d", port); 5705 5706 /* Create the taskq for the port */ 5707 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 5708 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 5709 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 5710 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 5711 goto err_case3; 5712 } 5713 5714 /* Allocate the argument for the taskq */ 5715 ahci_portp->ahciport_event_args = 5716 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 5717 5718 ahci_portp->ahciport_event_args->ahciea_addrp = 5719 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP); 5720 5721 if (ahci_portp->ahciport_event_args == NULL) 5722 goto err_case4; 5723 5724 mutex_exit(&ahci_portp->ahciport_mutex); 5725 5726 return (AHCI_SUCCESS); 5727 5728 err_case4: 5729 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 5730 5731 err_case3: 5732 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 5733 5734 err_case2: 5735 ahci_dealloc_rcvd_fis(ahci_portp); 5736 5737 err_case1: 5738 mutex_exit(&ahci_portp->ahciport_mutex); 5739 mutex_destroy(&ahci_portp->ahciport_mutex); 5740 cv_destroy(&ahci_portp->ahciport_cv); 5741 5742 kmem_free(ahci_portp, sizeof (ahci_port_t)); 5743 5744 return (AHCI_FAILURE); 5745 } 5746 5747 /* 5748 * Reverse of ahci_dealloc_port_state(). 5749 * 5750 * WARNING!!! ahcictl_mutex should be acquired before the function 5751 * is called. 5752 */ 5753 static void 5754 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 5755 { 5756 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5757 5758 ASSERT(ahci_portp != NULL); 5759 5760 mutex_enter(&ahci_portp->ahciport_mutex); 5761 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp, 5762 sizeof (ahci_addr_t)); 5763 ahci_portp->ahciport_event_args->ahciea_addrp = NULL; 5764 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 5765 ahci_portp->ahciport_event_args = NULL; 5766 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 5767 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 5768 ahci_dealloc_rcvd_fis(ahci_portp); 5769 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 5770 mutex_exit(&ahci_portp->ahciport_mutex); 5771 5772 mutex_destroy(&ahci_portp->ahciport_mutex); 5773 cv_destroy(&ahci_portp->ahciport_cv); 5774 5775 kmem_free(ahci_portp, sizeof (ahci_port_t)); 5776 5777 ahci_ctlp->ahcictl_ports[port] = NULL; 5778 } 5779 5780 /* 5781 * Allocates memory for the Received FIS Structure 5782 * 5783 * WARNING!!! ahciport_mutex should be acquired before the function 5784 * is called. 5785 */ 5786 static int 5787 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5788 uint8_t port) 5789 { 5790 size_t rcvd_fis_size; 5791 size_t ret_len; 5792 uint_t cookie_count; 5793 5794 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 5795 5796 /* allocate rcvd FIS dma handle. */ 5797 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 5798 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 5799 DDI_DMA_SLEEP, 5800 NULL, 5801 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 5802 DDI_SUCCESS) { 5803 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5804 "rcvd FIS dma handle alloc failed", NULL); 5805 5806 return (AHCI_FAILURE); 5807 } 5808 5809 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 5810 rcvd_fis_size, 5811 &accattr, 5812 DDI_DMA_CONSISTENT, 5813 DDI_DMA_SLEEP, 5814 NULL, 5815 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 5816 &ret_len, 5817 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 5818 5819 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5820 "rcvd FIS dma mem alloc fail", NULL); 5821 /* error.. free the dma handle. */ 5822 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5823 return (AHCI_FAILURE); 5824 } 5825 5826 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 5827 NULL, 5828 (caddr_t)ahci_portp->ahciport_rcvd_fis, 5829 rcvd_fis_size, 5830 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 5831 DDI_DMA_SLEEP, 5832 NULL, 5833 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 5834 &cookie_count) != DDI_DMA_MAPPED) { 5835 5836 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5837 "rcvd FIS dma handle bind fail", NULL); 5838 /* error.. free the dma handle & free the memory. */ 5839 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 5840 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5841 return (AHCI_FAILURE); 5842 } 5843 5844 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 5845 5846 /* Config Port Received FIS Base Address */ 5847 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 5848 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 5849 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 5850 5851 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 5852 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 5853 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 5854 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 5855 5856 return (AHCI_SUCCESS); 5857 } 5858 5859 /* 5860 * Deallocates the Received FIS Structure 5861 * 5862 * WARNING!!! ahciport_mutex should be acquired before the function 5863 * is called. 5864 */ 5865 static void 5866 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 5867 { 5868 /* Unbind the cmd list dma handle first. */ 5869 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 5870 5871 /* Then free the underlying memory. */ 5872 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 5873 5874 /* Now free the handle itself. */ 5875 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5876 } 5877 5878 /* 5879 * Allocates memory for the Command List, which contains up to 32 entries. 5880 * Each entry contains a command header, which is a 32-byte structure that 5881 * includes the pointer to the command table. 5882 * 5883 * WARNING!!! ahciport_mutex should be acquired before the function 5884 * is called. 5885 */ 5886 static int 5887 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5888 uint8_t port) 5889 { 5890 size_t cmd_list_size; 5891 size_t ret_len; 5892 uint_t cookie_count; 5893 5894 cmd_list_size = 5895 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 5896 5897 /* allocate cmd list dma handle. */ 5898 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 5899 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 5900 DDI_DMA_SLEEP, 5901 NULL, 5902 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 5903 5904 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5905 "cmd list dma handle alloc failed", NULL); 5906 return (AHCI_FAILURE); 5907 } 5908 5909 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 5910 cmd_list_size, 5911 &accattr, 5912 DDI_DMA_CONSISTENT, 5913 DDI_DMA_SLEEP, 5914 NULL, 5915 (caddr_t *)&ahci_portp->ahciport_cmd_list, 5916 &ret_len, 5917 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 5918 5919 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5920 "cmd list dma mem alloc fail", NULL); 5921 /* error.. free the dma handle. */ 5922 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5923 return (AHCI_FAILURE); 5924 } 5925 5926 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 5927 NULL, 5928 (caddr_t)ahci_portp->ahciport_cmd_list, 5929 cmd_list_size, 5930 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 5931 DDI_DMA_SLEEP, 5932 NULL, 5933 &ahci_portp->ahciport_cmd_list_dma_cookie, 5934 &cookie_count) != DDI_DMA_MAPPED) { 5935 5936 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5937 "cmd list dma handle bind fail", NULL); 5938 /* error.. free the dma handle & free the memory. */ 5939 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 5940 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5941 return (AHCI_FAILURE); 5942 } 5943 5944 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 5945 5946 /* Config Port Command List Base Address */ 5947 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 5948 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 5949 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 5950 5951 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 5952 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 5953 5954 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 5955 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 5956 5957 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 5958 goto err_out; 5959 } 5960 5961 return (AHCI_SUCCESS); 5962 5963 err_out: 5964 /* Unbind the cmd list dma handle first. */ 5965 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 5966 5967 /* Then free the underlying memory. */ 5968 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 5969 5970 /* Now free the handle itself. */ 5971 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5972 5973 return (AHCI_FAILURE); 5974 } 5975 5976 /* 5977 * Deallocates the Command List 5978 * 5979 * WARNING!!! ahciport_mutex should be acquired before the function 5980 * is called. 5981 */ 5982 static void 5983 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 5984 { 5985 /* First dealloc command table */ 5986 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 5987 5988 /* Unbind the cmd list dma handle first. */ 5989 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 5990 5991 /* Then free the underlying memory. */ 5992 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 5993 5994 /* Now free the handle itself. */ 5995 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5996 } 5997 5998 /* 5999 * Allocates memory for all Command Tables, which contains Command FIS, 6000 * ATAPI Command and Physical Region Descriptor Table. 6001 * 6002 * WARNING!!! ahciport_mutex should be acquired before the function 6003 * is called. 6004 */ 6005 static int 6006 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6007 { 6008 size_t ret_len; 6009 ddi_dma_cookie_t cmd_table_dma_cookie; 6010 uint_t cookie_count; 6011 int slot; 6012 6013 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 6014 "ahci_alloc_cmd_tables: port %d enter", 6015 ahci_portp->ahciport_port_num); 6016 6017 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6018 /* Allocate cmd table dma handle. */ 6019 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6020 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 6021 DDI_DMA_SLEEP, 6022 NULL, 6023 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 6024 DDI_SUCCESS) { 6025 6026 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6027 "cmd table dma handle alloc failed", NULL); 6028 6029 goto err_out; 6030 } 6031 6032 if (ddi_dma_mem_alloc( 6033 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6034 ahci_cmd_table_size, 6035 &accattr, 6036 DDI_DMA_CONSISTENT, 6037 DDI_DMA_SLEEP, 6038 NULL, 6039 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 6040 &ret_len, 6041 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 6042 NULL) { 6043 6044 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6045 "cmd table dma mem alloc fail", NULL); 6046 6047 /* error.. free the dma handle. */ 6048 ddi_dma_free_handle( 6049 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6050 goto err_out; 6051 } 6052 6053 if (ddi_dma_addr_bind_handle( 6054 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6055 NULL, 6056 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 6057 ahci_cmd_table_size, 6058 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6059 DDI_DMA_SLEEP, 6060 NULL, 6061 &cmd_table_dma_cookie, 6062 &cookie_count) != DDI_DMA_MAPPED) { 6063 6064 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6065 "cmd table dma handle bind fail", NULL); 6066 /* error.. free the dma handle & free the memory. */ 6067 ddi_dma_mem_free( 6068 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6069 ddi_dma_free_handle( 6070 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6071 goto err_out; 6072 } 6073 6074 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 6075 ahci_cmd_table_size); 6076 6077 /* Config Port Command Table Base Address */ 6078 SET_COMMAND_TABLE_BASE_ADDR( 6079 (&ahci_portp->ahciport_cmd_list[slot]), 6080 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 6081 6082 #ifndef __lock_lint 6083 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 6084 (&ahci_portp->ahciport_cmd_list[slot]), 6085 cmd_table_dma_cookie.dmac_laddress >> 32); 6086 #endif 6087 } 6088 6089 return (AHCI_SUCCESS); 6090 err_out: 6091 6092 for (slot--; slot >= 0; slot--) { 6093 /* Unbind the cmd table dma handle first */ 6094 (void) ddi_dma_unbind_handle( 6095 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6096 6097 /* Then free the underlying memory */ 6098 ddi_dma_mem_free( 6099 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6100 6101 /* Now free the handle itself */ 6102 ddi_dma_free_handle( 6103 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6104 } 6105 6106 return (AHCI_FAILURE); 6107 } 6108 6109 /* 6110 * Deallocates memory for all Command Tables. 6111 * 6112 * WARNING!!! ahciport_mutex should be acquired before the function 6113 * is called. 6114 */ 6115 static void 6116 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6117 { 6118 int slot; 6119 6120 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6121 "ahci_dealloc_cmd_tables: %d enter", 6122 ahci_portp->ahciport_port_num); 6123 6124 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6125 /* Unbind the cmd table dma handle first. */ 6126 (void) ddi_dma_unbind_handle( 6127 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6128 6129 /* Then free the underlying memory. */ 6130 ddi_dma_mem_free( 6131 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6132 6133 /* Now free the handle itself. */ 6134 ddi_dma_free_handle( 6135 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6136 } 6137 } 6138 6139 /* 6140 * Update SATA registers at controller ports 6141 * 6142 * WARNING!!! ahciport_mutex should be acquired before those functions 6143 * get called. 6144 */ 6145 static void 6146 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 6147 sata_device_t *sd) 6148 { 6149 sd->satadev_scr.sstatus = 6150 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6151 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 6152 sd->satadev_scr.serror = 6153 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6154 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 6155 sd->satadev_scr.scontrol = 6156 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6157 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 6158 sd->satadev_scr.sactive = 6159 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6160 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 6161 } 6162 6163 /* 6164 * For poll mode, ahci_port_intr will be called to emulate the interrupt 6165 */ 6166 static void 6167 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 6168 { 6169 uint32_t port_intr_status; 6170 uint32_t port_intr_enable; 6171 6172 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6173 "ahci_port_intr enter: port %d", port); 6174 6175 mutex_enter(&ahci_portp->ahciport_mutex); 6176 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 6177 /* For SATA_OPMODE_POLLING commands */ 6178 port_intr_enable = 6179 (AHCI_INTR_STATUS_DHRS | 6180 AHCI_INTR_STATUS_PSS | 6181 AHCI_INTR_STATUS_SDBS | 6182 AHCI_INTR_STATUS_UFS | 6183 AHCI_INTR_STATUS_PCS | 6184 AHCI_INTR_STATUS_PRCS | 6185 AHCI_INTR_STATUS_OFS | 6186 AHCI_INTR_STATUS_INFS | 6187 AHCI_INTR_STATUS_IFS | 6188 AHCI_INTR_STATUS_HBDS | 6189 AHCI_INTR_STATUS_HBFS | 6190 AHCI_INTR_STATUS_TFES); 6191 } else { 6192 /* 6193 * port_intr_enable indicates that the corresponding interrrupt 6194 * reporting is enabled. 6195 */ 6196 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6197 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 6198 } 6199 6200 /* IPMS error in port reset should be ignored according AHCI spec. */ 6201 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS)) 6202 port_intr_enable |= AHCI_INTR_STATUS_IPMS; 6203 mutex_exit(&ahci_portp->ahciport_mutex); 6204 6205 /* 6206 * port_intr_stats indicates that the corresponding interrupt 6207 * condition is active. 6208 */ 6209 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6210 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 6211 6212 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6213 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 6214 "port_intr_enable = 0x%x", 6215 port, port_intr_status, port_intr_enable); 6216 6217 port_intr_status &= port_intr_enable; 6218 6219 /* First clear the port interrupts status */ 6220 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6221 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 6222 port_intr_status); 6223 6224 /* Check the completed non-queued commands */ 6225 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 6226 AHCI_INTR_STATUS_PSS)) { 6227 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 6228 ahci_portp, port); 6229 } 6230 6231 /* Check the completed queued commands */ 6232 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 6233 (void) ahci_intr_set_device_bits(ahci_ctlp, 6234 ahci_portp, port); 6235 } 6236 6237 /* Check the port connect change status interrupt bit */ 6238 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 6239 (void) ahci_intr_port_connect_change(ahci_ctlp, 6240 ahci_portp, port); 6241 } 6242 6243 /* Check the device mechanical presence status interrupt bit */ 6244 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 6245 (void) ahci_intr_device_mechanical_presence_status( 6246 ahci_ctlp, ahci_portp, port); 6247 } 6248 6249 /* Check the PhyRdy change status interrupt bit */ 6250 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 6251 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 6252 port); 6253 } 6254 6255 /* 6256 * Check the non-fatal error interrupt bits, there are four 6257 * kinds of non-fatal errors at the time being: 6258 * 6259 * PxIS.UFS - Unknown FIS Error 6260 * PxIS.OFS - Overflow Error 6261 * PxIS.INFS - Interface Non-Fatal Error 6262 * PxIS.IPMS - Incorrect Port Multiplier Status Error 6263 * 6264 * For these non-fatal errors, the HBA can continue to operate, 6265 * so the driver just log the error messages. 6266 */ 6267 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 6268 AHCI_INTR_STATUS_OFS | 6269 AHCI_INTR_STATUS_IPMS | 6270 AHCI_INTR_STATUS_INFS)) { 6271 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 6272 port, port_intr_status); 6273 } 6274 6275 /* 6276 * Check the fatal error interrupt bits, there are four kinds 6277 * of fatal errors for AHCI controllers: 6278 * 6279 * PxIS.HBFS - Host Bus Fatal Error 6280 * PxIS.HBDS - Host Bus Data Error 6281 * PxIS.IFS - Interface Fatal Error 6282 * PxIS.TFES - Task File Error 6283 * 6284 * The fatal error means the HBA can not recover from it by 6285 * itself, and it will try to abort the transfer, and the software 6286 * must intervene to restart the port. 6287 */ 6288 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 6289 AHCI_INTR_STATUS_HBDS | 6290 AHCI_INTR_STATUS_HBFS | 6291 AHCI_INTR_STATUS_TFES)) 6292 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 6293 port, port_intr_status); 6294 6295 /* Check the cold port detect interrupt bit */ 6296 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 6297 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 6298 } 6299 6300 /* Second clear the corresponding bit in IS.IPS */ 6301 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6302 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 6303 } 6304 6305 /* 6306 * Interrupt service handler 6307 */ 6308 static uint_t 6309 ahci_intr(caddr_t arg1, caddr_t arg2) 6310 { 6311 #ifndef __lock_lint 6312 _NOTE(ARGUNUSED(arg2)) 6313 #endif 6314 /* LINTED */ 6315 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 6316 ahci_port_t *ahci_portp; 6317 int32_t global_intr_status; 6318 uint8_t port; 6319 6320 /* 6321 * global_intr_status indicates that the corresponding port has 6322 * an interrupt pending. 6323 */ 6324 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6325 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 6326 6327 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 6328 /* The interrupt is not ours */ 6329 return (DDI_INTR_UNCLAIMED); 6330 } 6331 6332 /* Loop for all the ports */ 6333 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6334 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 6335 continue; 6336 } 6337 if (!((0x1 << port) & global_intr_status)) { 6338 continue; 6339 } 6340 6341 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6342 6343 /* Call ahci_port_intr */ 6344 ahci_port_intr(ahci_ctlp, ahci_portp, port); 6345 } 6346 6347 return (DDI_INTR_CLAIMED); 6348 } 6349 6350 /* 6351 * For non-queued commands, when the corresponding bit in the PxCI register 6352 * is cleared, it means the command is completed successfully. And according 6353 * to the HBA state machine, there are three conditions which possibly will 6354 * try to clear the PxCI register bit. 6355 * 1. Receive one D2H Register FIS which is with 'I' bit set 6356 * 2. Update PIO Setup FIS 6357 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 6358 * 6359 * Process completed non-queued commands when the interrupt status bit - 6360 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 6361 * 6362 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 6363 * with the 'I' bit set. And the following commands will send thus 6364 * FIS with 'I' bit set upon the successful completion: 6365 * 1. Non-data commands 6366 * 2. DMA data-in command 6367 * 3. DMA data-out command 6368 * 4. PIO data-out command 6369 * 5. PACKET non-data commands 6370 * 6. PACKET PIO data-in command 6371 * 7. PACKET PIO data-out command 6372 * 8. PACKET DMA data-in command 6373 * 9. PACKET DMA data-out command 6374 * 6375 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 6376 * with the 'I' bit set. And the following commands will send this 6377 * FIS upon the successful completion: 6378 * 1. PIO data-in command 6379 */ 6380 static int 6381 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 6382 ahci_port_t *ahci_portp, uint8_t port) 6383 { 6384 uint32_t port_cmd_issue = 0; 6385 uint32_t finished_tags; 6386 int finished_slot; 6387 sata_pkt_t *satapkt; 6388 ahci_fis_d2h_register_t *rcvd_fisp; 6389 #if AHCI_DEBUG 6390 ahci_cmd_header_t *cmd_header; 6391 uint32_t cmd_dmacount; 6392 #endif 6393 6394 mutex_enter(&ahci_portp->ahciport_mutex); 6395 6396 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6397 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6398 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6399 /* 6400 * Spurious interrupt. Nothing to be done. 6401 */ 6402 mutex_exit(&ahci_portp->ahciport_mutex); 6403 return (AHCI_SUCCESS); 6404 } 6405 6406 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6407 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6408 6409 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6410 /* Slot 0 is always used during error recovery */ 6411 finished_tags = 0x1 & ~port_cmd_issue; 6412 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6413 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 6414 "retrieval is finished, and finished_tags = 0x%x", 6415 port, finished_tags); 6416 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6417 finished_tags = 0x1 & ~port_cmd_issue; 6418 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 6419 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w " 6420 "port multiplier is finished, and finished_tags = 0x%x", 6421 port, finished_tags); 6422 6423 } else { 6424 6425 finished_tags = ahci_portp->ahciport_pending_tags & 6426 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6427 } 6428 6429 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6430 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 6431 "port_cmd_issue = 0x%x finished_tags = 0x%x", 6432 ahci_portp->ahciport_pending_tags, port_cmd_issue, 6433 finished_tags); 6434 6435 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6436 (finished_tags == 0x1)) { 6437 satapkt = ahci_portp->ahciport_err_retri_pkt; 6438 ASSERT(satapkt != NULL); 6439 6440 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6441 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6442 "with SATA_PKT_COMPLETED", (void *)satapkt); 6443 6444 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6445 goto out; 6446 } 6447 6448 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6449 (finished_tags == 0x1)) { 6450 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 6451 ASSERT(satapkt != NULL); 6452 6453 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6454 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6455 "with SATA_PKT_COMPLETED", (void *)satapkt); 6456 6457 /* READ PORTMULT need copy out FIS content. */ 6458 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6459 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6460 ahcirf_d2h_register_fis); 6461 satapkt->satapkt_cmd.satacmd_status_reg = 6462 GET_RFIS_STATUS(rcvd_fisp); 6463 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6464 } 6465 6466 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6467 goto out; 6468 } 6469 6470 while (finished_tags) { 6471 finished_slot = ddi_ffs(finished_tags) - 1; 6472 if (finished_slot == -1) { 6473 goto out; 6474 } 6475 6476 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6477 ASSERT(satapkt != NULL); 6478 #if AHCI_DEBUG 6479 /* 6480 * For non-native queued commands, the PRD byte count field 6481 * shall contain an accurate count of the number of bytes 6482 * transferred for the command before the PxCI bit is cleared 6483 * to '0' for the command. 6484 * 6485 * The purpose of this field is to let software know how many 6486 * bytes transferred for a given operation in order to 6487 * determine if underflow occurred. When issuing native command 6488 * queuing commands, this field should not be used and is not 6489 * required to be valid since in this case underflow is always 6490 * illegal. 6491 * 6492 * For data reads, the HBA will update its PRD byte count with 6493 * the total number of bytes received from the last FIS, and 6494 * may be able to continue normally. For data writes, the 6495 * device will detect an error, and HBA most likely will get 6496 * a fatal error. 6497 * 6498 * Therefore, here just put code to debug part. And please 6499 * refer to the comment above ahci_intr_fatal_error for the 6500 * definition of underflow error. 6501 */ 6502 cmd_dmacount = 6503 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 6504 if (cmd_dmacount) { 6505 cmd_header = 6506 &ahci_portp->ahciport_cmd_list[finished_slot]; 6507 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 6508 "ahci_intr_cmd_cmplt: port %d, " 6509 "PRD Byte Count = 0x%x, " 6510 "ahciport_prd_bytecounts = 0x%x", port, 6511 cmd_header->ahcich_prd_byte_count, 6512 cmd_dmacount); 6513 6514 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 6515 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 6516 "ahci_intr_cmd_cmplt: port %d, " 6517 "an underflow occurred", port); 6518 } 6519 } 6520 #endif 6521 6522 /* 6523 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 6524 * feature, sata_special_regs flag will be set, and the 6525 * driver should copy the status and the other corresponding 6526 * register values in the D2H Register FIS received (It's 6527 * working on Non-data protocol) from the device back to 6528 * the sata_cmd. 6529 * 6530 * For every AHCI port, there is only one Received FIS 6531 * structure, which contains the FISes received from the 6532 * device, So we're trying to copy the content of D2H 6533 * Register FIS in the Received FIS structure back to 6534 * the sata_cmd. 6535 */ 6536 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6537 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6538 ahcirf_d2h_register_fis); 6539 satapkt->satapkt_cmd.satacmd_status_reg = 6540 GET_RFIS_STATUS(rcvd_fisp); 6541 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6542 } 6543 6544 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6545 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6546 "with SATA_PKT_COMPLETED", (void *)satapkt); 6547 6548 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 6549 CLEAR_BIT(finished_tags, finished_slot); 6550 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 6551 6552 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6553 } 6554 out: 6555 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 6556 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 6557 ahci_portp->ahciport_pending_tags); 6558 6559 mutex_exit(&ahci_portp->ahciport_mutex); 6560 6561 return (AHCI_SUCCESS); 6562 } 6563 6564 /* 6565 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 6566 * with the 'I' bit set and has been copied into system memory. It will 6567 * be sent under the following situations: 6568 * 6569 * 1. NCQ command is completed 6570 * 6571 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 6572 * via the Set Device Bits FIS. When such event is generated, the software 6573 * needs to read PxSACT register and compares the current value to the 6574 * list of commands previously issue by software. ahciport_pending_ncq_tags 6575 * keeps the tags of previously issued commands. 6576 * 6577 * 2. Asynchronous Notification 6578 * 6579 * Asynchronous Notification is a feature in SATA spec 2.6. 6580 * 6581 * 1) ATAPI device will send a signal to the host when media is inserted or 6582 * removed and avoids polling the device for media changes. The signal 6583 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 6584 * set to '1'. At the moment, it's not supported yet. 6585 * 6586 * 2) Port multiplier will send a signal to the host when a hot plug event 6587 * has occured on a port multiplier port. It is used when command based 6588 * switching is employed. This is handled by ahci_intr_pmult_sntf_events() 6589 */ 6590 static int 6591 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 6592 ahci_port_t *ahci_portp, uint8_t port) 6593 { 6594 ahci_addr_t addr; 6595 6596 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6597 "ahci_intr_set_device_bits enter: port %d", port); 6598 6599 /* Initialize HBA port address */ 6600 AHCI_ADDR_SET_PORT(&addr, port); 6601 6602 /* NCQ plug handler */ 6603 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr); 6604 6605 /* Check port multiplier's asynchronous notification events */ 6606 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 6607 (void) ahci_intr_pmult_sntf_events(ahci_ctlp, 6608 ahci_portp, port); 6609 } 6610 6611 /* ATAPI events is not supported yet */ 6612 6613 return (AHCI_SUCCESS); 6614 } 6615 /* 6616 * NCQ interrupt handler. Called upon a NCQ command is completed. 6617 * Only be called from ahci_intr_set_device_bits(). 6618 * 6619 * WARNING!!! ahciport_mutex should be acquired before the function is 6620 * called. 6621 */ 6622 static int 6623 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp, 6624 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 6625 { 6626 uint32_t port_sactive; 6627 uint32_t port_cmd_issue; 6628 uint32_t issued_tags; 6629 int issued_slot; 6630 uint32_t finished_tags; 6631 int finished_slot; 6632 uint8_t port = addrp->aa_port; 6633 sata_pkt_t *satapkt; 6634 6635 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6636 "ahci_intr_set_device_bits enter: port %d", port); 6637 6638 mutex_enter(&ahci_portp->ahciport_mutex); 6639 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6640 mutex_exit(&ahci_portp->ahciport_mutex); 6641 return (AHCI_SUCCESS); 6642 } 6643 6644 /* 6645 * First the handler got which commands are finished by checking 6646 * PxSACT register 6647 */ 6648 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6649 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6650 6651 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6652 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 6653 6654 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6655 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 6656 "port_sactive = 0x%x", port, 6657 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 6658 6659 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6660 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 6661 6662 /* 6663 * For NCQ commands, the software can determine which command has 6664 * already been transmitted to the device by checking PxCI register. 6665 */ 6666 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6667 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6668 6669 issued_tags = ahci_portp->ahciport_pending_tags & 6670 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6671 6672 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6673 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 6674 "port_cmd_issue = 0x%x", port, 6675 ahci_portp->ahciport_pending_tags, port_cmd_issue); 6676 6677 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6678 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 6679 6680 /* 6681 * Clear ahciport_pending_tags bit when the corresponding command 6682 * is already sent down to the device. 6683 */ 6684 while (issued_tags) { 6685 issued_slot = ddi_ffs(issued_tags) - 1; 6686 if (issued_slot == -1) { 6687 goto next; 6688 } 6689 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 6690 CLEAR_BIT(issued_tags, issued_slot); 6691 } 6692 6693 next: 6694 while (finished_tags) { 6695 finished_slot = ddi_ffs(finished_tags) - 1; 6696 if (finished_slot == -1) { 6697 goto out; 6698 } 6699 6700 /* The command is certainly transmitted to the device */ 6701 ASSERT(!(ahci_portp->ahciport_pending_tags & 6702 (0x1 << finished_slot))); 6703 6704 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6705 ASSERT(satapkt != NULL); 6706 6707 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6708 "ahci_intr_set_device_bits: sending up pkt 0x%p " 6709 "with SATA_PKT_COMPLETED", (void *)satapkt); 6710 6711 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 6712 CLEAR_BIT(finished_tags, finished_slot); 6713 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 6714 6715 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6716 } 6717 out: 6718 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 6719 "ahci_intr_set_device_bits: port %d " 6720 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 6721 port, ahci_portp->ahciport_pending_ncq_tags, 6722 ahci_portp->ahciport_pending_tags); 6723 6724 mutex_exit(&ahci_portp->ahciport_mutex); 6725 6726 return (AHCI_SUCCESS); 6727 } 6728 6729 /* 6730 * Port multiplier asynchronous notification event handler. Called upon a 6731 * device is hot plugged/pulled. 6732 * 6733 * The async-notification event will only be recorded by ahcipmi_snotif_tags 6734 * here and will be handled by ahci_probe_pmult(). 6735 * 6736 * NOTE: called only from ahci_port_intr(). 6737 */ 6738 static int 6739 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6740 uint8_t port) 6741 { 6742 sata_device_t sdevice; 6743 6744 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6745 "ahci_intr_pmult_sntf_events enter: port %d ", port); 6746 6747 /* no hot-plug while attaching process */ 6748 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6749 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 6750 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6751 return (AHCI_SUCCESS); 6752 } 6753 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6754 6755 mutex_enter(&ahci_portp->ahciport_mutex); 6756 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 6757 mutex_exit(&ahci_portp->ahciport_mutex); 6758 return (AHCI_SUCCESS); 6759 } 6760 6761 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 6762 6763 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags = 6764 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6765 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port)); 6766 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6767 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 6768 AHCI_SNOTIF_CLEAR_ALL); 6769 6770 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) { 6771 mutex_exit(&ahci_portp->ahciport_mutex); 6772 return (AHCI_SUCCESS); 6773 } 6774 6775 /* Port Multiplier sub-device hot-plug handler */ 6776 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6777 mutex_exit(&ahci_portp->ahciport_mutex); 6778 return (AHCI_SUCCESS); 6779 } 6780 6781 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) { 6782 /* Not allowed to re-enter. */ 6783 mutex_exit(&ahci_portp->ahciport_mutex); 6784 return (AHCI_SUCCESS); 6785 } 6786 6787 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF; 6788 6789 /* 6790 * NOTE: 6791 * Even if Asynchronous Notification is supported (and enabled) by 6792 * both controller and the port multiplier, the content of PxSNTF 6793 * register is always set to 0x8000 by async notification event. We 6794 * need to check GSCR[32] on the port multiplier to find out the 6795 * owner of this event. 6796 * This is not accord with SATA spec 2.6 and needs further 6797 * clarification. 6798 */ 6799 /* hot-plug will not reported while reseting. */ 6800 if (ahci_portp->ahciport_reset_in_progress == 1) { 6801 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 6802 "port %d snotif event ignored", port); 6803 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 6804 mutex_exit(&ahci_portp->ahciport_mutex); 6805 return (AHCI_SUCCESS); 6806 } 6807 6808 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 6809 "PxSNTF is set to 0x%x by port multiplier", 6810 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags); 6811 6812 /* 6813 * Now we need do some necessary operation and inform SATA framework 6814 * that link/device events has happened. 6815 */ 6816 bzero((void *)&sdevice, sizeof (sata_device_t)); 6817 sdevice.satadev_addr.cport = ahci_ctlp-> 6818 ahcictl_port_to_cport[port]; 6819 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 6820 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 6821 sdevice.satadev_state = SATA_PSTATE_PWRON; 6822 6823 /* Just reject packets, do not stop that port. */ 6824 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 6825 6826 mutex_exit(&ahci_portp->ahciport_mutex); 6827 sata_hba_event_notify( 6828 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6829 &sdevice, 6830 SATA_EVNT_PMULT_LINK_CHANGED); 6831 mutex_enter(&ahci_portp->ahciport_mutex); 6832 6833 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 6834 mutex_exit(&ahci_portp->ahciport_mutex); 6835 6836 return (AHCI_SUCCESS); 6837 } 6838 6839 /* 6840 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 6841 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 6842 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 6843 * indicates a COMINIT signal was received. 6844 * 6845 * Hot plug insertion is detected by reception of a COMINIT signal from the 6846 * device. On reception of unsolicited COMINIT, the HBA shall generate a 6847 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 6848 * begin the normal communication negotiation sequence as outlined in the 6849 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 6850 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 6851 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 6852 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 6853 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 6854 * to handle hot plug insertion. In this interrupt handler, just do nothing 6855 * but print some log message and clear the bit. 6856 */ 6857 static int 6858 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 6859 ahci_port_t *ahci_portp, uint8_t port) 6860 { 6861 #if AHCI_DEBUG 6862 uint32_t port_serror; 6863 #endif 6864 6865 mutex_enter(&ahci_portp->ahciport_mutex); 6866 6867 #if AHCI_DEBUG 6868 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6869 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 6870 6871 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6872 "ahci_intr_port_connect_change: port %d, " 6873 "port_serror = 0x%x", port, port_serror); 6874 #endif 6875 6876 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 6877 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6878 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6879 SERROR_EXCHANGED_ERR); 6880 6881 mutex_exit(&ahci_portp->ahciport_mutex); 6882 6883 return (AHCI_SUCCESS); 6884 } 6885 6886 /* 6887 * Hot Plug Operation for platforms that support Mechanical Presence 6888 * Switches. 6889 * 6890 * When set, it indicates that a mechanical presence switch attached to this 6891 * port has been opened or closed, which may lead to a change in the connection 6892 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 6893 * are set to '1'. 6894 * 6895 * At the moment, this interrupt is not needed and disabled and we just log 6896 * the debug message. 6897 */ 6898 static int 6899 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 6900 ahci_port_t *ahci_portp, uint8_t port) 6901 { 6902 uint32_t cap_status, port_cmd_status; 6903 6904 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6905 "ahci_intr_device_mechanical_presence_status enter, " 6906 "port %d", port); 6907 6908 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6909 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 6910 6911 mutex_enter(&ahci_portp->ahciport_mutex); 6912 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6913 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6914 6915 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 6916 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 6917 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6918 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 6919 "the interrupt: cap_status = 0x%x, " 6920 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 6921 mutex_exit(&ahci_portp->ahciport_mutex); 6922 6923 return (AHCI_SUCCESS); 6924 } 6925 6926 #if AHCI_DEBUG 6927 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 6928 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6929 "The mechanical presence switch is open: " 6930 "port %d, port_cmd_status = 0x%x", 6931 port, port_cmd_status); 6932 } else { 6933 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6934 "The mechanical presence switch is close: " 6935 "port %d, port_cmd_status = 0x%x", 6936 port, port_cmd_status); 6937 } 6938 #endif 6939 6940 mutex_exit(&ahci_portp->ahciport_mutex); 6941 6942 return (AHCI_SUCCESS); 6943 } 6944 6945 /* 6946 * Native Hot Plug Support. 6947 * 6948 * When set, it indicates that the internal PHYRDY signal changed state. 6949 * This bit reflects the state of PxSERR.DIAG.N. 6950 * 6951 * There are three kinds of conditions to generate this interrupt event: 6952 * 1. a device is inserted 6953 * 2. a device is disconnected 6954 * 3. when the link enters/exits a Partial or Slumber interface power 6955 * management state 6956 * 6957 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 6958 * bit may be set due to the link entering the Partial or Slumber power 6959 * management state, rather than due to a hot plug insertion or removal 6960 * event. So far, the interface power management is disabled, so the 6961 * driver can reliably get removal detection notification via the 6962 * PxSERR.DIAG.N bit. 6963 */ 6964 static int 6965 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 6966 ahci_port_t *ahci_portp, uint8_t port) 6967 { 6968 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 6969 sata_device_t sdevice; 6970 int dev_exists_now = 0; 6971 int dev_existed_previously = 0; 6972 ahci_addr_t port_addr; 6973 6974 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6975 "ahci_intr_phyrdy_change enter, port %d", port); 6976 6977 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 6978 mutex_enter(&ahci_portp->ahciport_mutex); 6979 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6980 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6981 SERROR_PHY_RDY_CHG); 6982 mutex_exit(&ahci_portp->ahciport_mutex); 6983 6984 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6985 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 6986 (ahci_portp == NULL)) { 6987 /* The whole controller setup is not yet done. */ 6988 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6989 return (AHCI_SUCCESS); 6990 } 6991 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6992 6993 mutex_enter(&ahci_portp->ahciport_mutex); 6994 6995 /* SStatus tells the presence of device. */ 6996 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6997 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6998 6999 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 7000 dev_exists_now = 1; 7001 } 7002 7003 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 7004 dev_existed_previously = 1; 7005 } 7006 7007 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 7008 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 7009 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7010 "ahci_intr_phyrdy_change: port %d " 7011 "AHCI_PORT_FLAG_NODEV is cleared", port); 7012 if (dev_exists_now == 0) 7013 dev_existed_previously = 1; 7014 } 7015 7016 bzero((void *)&sdevice, sizeof (sata_device_t)); 7017 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7018 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7019 sdevice.satadev_addr.pmport = 0; 7020 sdevice.satadev_state = SATA_PSTATE_PWRON; 7021 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 7022 7023 AHCI_ADDR_SET_PORT(&port_addr, port); 7024 7025 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 7026 if (dev_exists_now) { 7027 if (dev_existed_previously) { /* 1 -> 1 */ 7028 /* Things are fine now. The loss was temporary. */ 7029 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7030 "ahci_intr_phyrdy_change port %d " 7031 "device link lost/established", port); 7032 7033 mutex_exit(&ahci_portp->ahciport_mutex); 7034 sata_hba_event_notify( 7035 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7036 &sdevice, 7037 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 7038 mutex_enter(&ahci_portp->ahciport_mutex); 7039 7040 } else { /* 0 -> 1 */ 7041 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7042 "ahci_intr_phyrdy_change: port %d " 7043 "device link established", port); 7044 7045 /* 7046 * A new device has been detected. The new device 7047 * might be a port multiplier instead of a drive, so 7048 * we cannot update the signature directly. 7049 */ 7050 (void) ahci_initialize_port(ahci_ctlp, 7051 ahci_portp, &port_addr); 7052 7053 /* Try to start the port */ 7054 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 7055 != AHCI_SUCCESS) { 7056 sdevice.satadev_state |= SATA_PSTATE_FAILED; 7057 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7058 "ahci_intr_phyrdy_change: port %d failed " 7059 "at start port", port); 7060 } 7061 7062 /* Clear the max queue depth for inserted device */ 7063 ahci_portp->ahciport_max_ncq_tags = 0; 7064 7065 mutex_exit(&ahci_portp->ahciport_mutex); 7066 sata_hba_event_notify( 7067 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7068 &sdevice, 7069 SATA_EVNT_LINK_ESTABLISHED); 7070 mutex_enter(&ahci_portp->ahciport_mutex); 7071 7072 } 7073 } else { /* No device exists now */ 7074 7075 if (dev_existed_previously) { /* 1 -> 0 */ 7076 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7077 "ahci_intr_phyrdy_change: port %d " 7078 "device link lost", port); 7079 7080 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7081 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7082 ahci_portp, port); 7083 7084 if (ahci_portp->ahciport_device_type == 7085 SATA_DTYPE_PMULT) { 7086 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 7087 } 7088 7089 /* An existing device is lost. */ 7090 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 7091 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN; 7092 7093 mutex_exit(&ahci_portp->ahciport_mutex); 7094 sata_hba_event_notify( 7095 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7096 &sdevice, 7097 SATA_EVNT_LINK_LOST); 7098 mutex_enter(&ahci_portp->ahciport_mutex); 7099 } 7100 } 7101 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 7102 7103 mutex_exit(&ahci_portp->ahciport_mutex); 7104 7105 return (AHCI_SUCCESS); 7106 } 7107 7108 /* 7109 * PxIS.UFS - Unknown FIS Error 7110 * 7111 * This interrupt event means an unknown FIS was received and has been 7112 * copied into system memory. An unknown FIS is not considered an illegal 7113 * FIS, unless the length received is more than 64 bytes. If an unknown 7114 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 7115 * normal operation. If the unknown FIS is more than 64 bytes, then it 7116 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 7117 * a fatal error. 7118 * 7119 * PxIS.IPMS - Incorrect Port Multiplier Status 7120 * 7121 * IPMS Indicates that the HBA received a FIS from a device that did not 7122 * have a command outstanding. The IPMS bit may be set during enumeration 7123 * of devices on a Port Multiplier due to the normal Port Multiplier 7124 * enumeration process. It is recommended that IPMS only be used after 7125 * enumeration is complete on the Port Multiplier (copied from spec). 7126 * 7127 * PxIS.OFS - Overflow Error 7128 * 7129 * Command list overflow is defined as software building a command table 7130 * that has fewer total bytes than the transaction given to the device. 7131 * On device writes, the HBA will run out of data, and on reads, there 7132 * will be no room to put the data. 7133 * 7134 * For an overflow on data read, either PIO or DMA, the HBA will set 7135 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 7136 * non-fatal error when the HBA can continues. Sometimes, it will cause 7137 * a fatal error and need the software to do something. 7138 * 7139 * For an overflow on data write, setting PxIS.OFS is optional for both 7140 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 7141 * software to clean up from this serious error. 7142 * 7143 * PxIS.INFS - Interface Non-Fatal Error 7144 * 7145 * This interrupt event indicates that the HBA encountered an error on 7146 * the Serial ATA interface but was able to continue operation. The kind 7147 * of error usually occurred during a non-Data FIS, and under this condition 7148 * the FIS will be re-transmitted by HBA automatically. 7149 * 7150 * When the FMA is implemented, there should be a stat structure to 7151 * record how many every kind of error happens. 7152 */ 7153 static int 7154 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7155 uint8_t port, uint32_t intr_status) 7156 { 7157 uint32_t port_serror; 7158 #if AHCI_DEBUG 7159 uint32_t port_cmd_status; 7160 uint32_t port_cmd_issue; 7161 uint32_t port_sactive; 7162 int current_slot; 7163 uint32_t current_tags; 7164 sata_pkt_t *satapkt; 7165 ahci_cmd_header_t *cmd_header; 7166 uint32_t cmd_dmacount; 7167 #endif 7168 7169 mutex_enter(&ahci_portp->ahciport_mutex); 7170 7171 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7172 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7173 7174 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 7175 "ahci_intr_non_fatal_error: port %d, " 7176 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status); 7177 7178 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 7179 7180 if (intr_status & AHCI_INTR_STATUS_UFS) { 7181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7182 "ahci port %d has unknown FIS error", port); 7183 7184 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 7185 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7186 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7187 SERROR_FIS_TYPE); 7188 } 7189 7190 #if AHCI_DEBUG 7191 if (intr_status & AHCI_INTR_STATUS_IPMS) { 7192 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d " 7193 "has Incorrect Port Multiplier Status error", port); 7194 } 7195 7196 if (intr_status & AHCI_INTR_STATUS_OFS) { 7197 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7198 "ahci port %d has overflow error", port); 7199 } 7200 7201 if (intr_status & AHCI_INTR_STATUS_INFS) { 7202 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7203 "ahci port %d has interface non fatal error", port); 7204 } 7205 7206 /* 7207 * Record the error occurred command's slot. 7208 */ 7209 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7210 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7211 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7212 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7213 7214 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7215 AHCI_CMD_STATUS_CCS_SHIFT; 7216 7217 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7218 satapkt = ahci_portp->ahciport_err_retri_pkt; 7219 ASSERT(satapkt != NULL); 7220 ASSERT(current_slot == 0); 7221 } else { 7222 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7223 } 7224 7225 if (satapkt != NULL) { 7226 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7227 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 7228 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 7229 satapkt->satapkt_cmd.satacmd_cmd_reg); 7230 7231 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7232 "ahci_intr_non_fatal_error: port %d, " 7233 "satapkt 0x%p is being processed when error occurs", 7234 port, (void *)satapkt); 7235 7236 /* 7237 * PRD Byte Count field of command header is not 7238 * required to reflect the total number of bytes 7239 * transferred when an overflow occurs, so here 7240 * just log the value. 7241 */ 7242 cmd_dmacount = 7243 ahci_portp->ahciport_prd_bytecounts[current_slot]; 7244 if (cmd_dmacount) { 7245 cmd_header = &ahci_portp-> 7246 ahciport_cmd_list[current_slot]; 7247 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7248 "ahci_intr_non_fatal_error: port %d, " 7249 "PRD Byte Count = 0x%x, " 7250 "ahciport_prd_bytecounts = 0x%x", port, 7251 cmd_header->ahcich_prd_byte_count, 7252 cmd_dmacount); 7253 } 7254 } 7255 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7256 /* 7257 * For queued command, list those command which have already 7258 * been transmitted to the device and still not completed. 7259 */ 7260 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7261 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7262 7263 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7264 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7265 7266 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 7267 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 7268 "port_sactive = 0x%x port_cmd_issue = 0x%x", 7269 ahci_portp->ahciport_pending_ncq_tags, 7270 port_sactive, port_cmd_issue); 7271 7272 current_tags = ahci_portp->ahciport_pending_ncq_tags & 7273 port_sactive & ~port_cmd_issue & 7274 AHCI_NCQ_SLOT_MASK(ahci_portp); 7275 7276 while (current_tags) { 7277 current_slot = ddi_ffs(current_tags) - 1; 7278 if (current_slot == -1) { 7279 goto out; 7280 } 7281 7282 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7283 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 7284 ahci_ctlp, "ahci_intr_non_fatal_error: " 7285 "port %d, satapkt 0x%p is outstanding when " 7286 "error occurs", port, (void *)satapkt); 7287 7288 CLEAR_BIT(current_tags, current_slot); 7289 } 7290 } 7291 out: 7292 #endif 7293 mutex_exit(&ahci_portp->ahciport_mutex); 7294 7295 return (AHCI_SUCCESS); 7296 } 7297 7298 /* 7299 * According to the AHCI spec, the error types include system memory 7300 * errors, interface errors, port multiplier errors, device errors, 7301 * command list overflow, command list underflow, native command 7302 * queuing tag errors and pio data transfer errors. 7303 * 7304 * System memory errors such as target abort, master abort, and parity 7305 * may cause the host to stop, and they are serious errors and needed 7306 * to be recovered with software intervention. When system software 7307 * has given a pointer to the HBA that doesn't exist in physical memory, 7308 * a master/target abort error occurs, and PxIS.HBFS will be set. A 7309 * data error such as CRC or parity occurs, the HBA aborts the transfer 7310 * (if necessary) and PxIS.HBDS will be set. 7311 * 7312 * Interface errors are errors that occur due to electrical issues on 7313 * the interface, or protocol miscommunication between the device and 7314 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 7315 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 7316 * causes PxIS.IFS/PxIS.INFS to be set are 7317 * 1. in PxSERR.ERR, P bit is set to '1' 7318 * 2. in PxSERR.DIAG, C or H bit is set to '1' 7319 * 3. PhyRdy drop unexpectly, N bit is set to '1' 7320 * If the error occurred during a non-data FIS, the FIS must be 7321 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 7322 * the error occurred during a data FIS, the transfer will stop, so 7323 * the error is fatal and PxIS.IFS is set. 7324 * 7325 * When a FIS arrives that updates the taskfile, the HBA checks to see 7326 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 7327 * stops processing any more commands. 7328 * 7329 * Command list overflow is defined as software building a command table 7330 * that has fewer total bytes than the transaction given to the device. 7331 * On device writes, the HBA will run out of data, and on reads, there 7332 * will be no room to put the data. For an overflow on data read, either 7333 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 7334 * For an overflow on data write, setting PxIS.OFS is optional for both 7335 * DMA and PIO, and a COMRESET is required by software to clean up from 7336 * this serious error. 7337 * 7338 * Command list underflow is defined as software building a command 7339 * table that has more total bytes than the transaction given to the 7340 * device. For data writes, both PIO and DMA, the device will detect 7341 * an error and end the transfer. And these errors are most likely going 7342 * to be fatal errors that will cause the port to be restarted. For 7343 * data reads, the HBA updates its PRD byte count, and may be 7344 * able to continue normally, but is not required to. And The HBA is 7345 * not required to detect underflow conditions for native command 7346 * queuing command. 7347 * 7348 * The HBA does not actively check incoming DMA Setup FISes to ensure 7349 * that the PxSACT register bit for that slot is set. Existing error 7350 * mechanisms, such as host bus failure, or bad protocol, are used to 7351 * recover from this case. 7352 * 7353 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 7354 * DATA FIS must be an integral number of Dwords. If the HBA receives 7355 * a request which is not an integral number of Dwords, the HBA 7356 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 7357 * software restarts the port. And the HBA ensures that the size 7358 * of the DATA FIS received during a PIO command matches the size in 7359 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 7360 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 7361 * stop running until software restarts the port. 7362 */ 7363 /* 7364 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 7365 * 7366 * PxIS.IFS indicates that the hba encountered an error on the serial ata 7367 * interface which caused the transfer to stop. 7368 * 7369 * PxIS.HBDS indicates that the hba encountered a data error 7370 * (uncorrectable ecc/parity) when reading from or writing to system memory. 7371 * 7372 * PxIS.HBFS indicates that the hba encountered a host bus error that it 7373 * cannot recover from, such as a bad software pointer. 7374 * 7375 * PxIS.TFES is set whenever the status register is updated by the device 7376 * and the error bit (bit 0) is set. 7377 */ 7378 static int 7379 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 7380 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 7381 { 7382 uint32_t port_cmd_status; 7383 uint32_t port_serror; 7384 uint32_t task_file_status; 7385 int failed_slot; 7386 sata_pkt_t *spkt = NULL; 7387 uint8_t err_byte; 7388 ahci_event_arg_t *args; 7389 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7390 7391 mutex_enter(&ahci_portp->ahciport_mutex); 7392 7393 /* 7394 * ahci_intr_phyrdy_change() may have rendered it to 7395 * SATA_DTYPE_NONE. 7396 */ 7397 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7398 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7399 "ahci_intr_fatal_error: port %d no device attached, " 7400 "and just return without doing anything", port); 7401 goto out0; 7402 } 7403 7404 if (intr_status & AHCI_INTR_STATUS_TFES) { 7405 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7406 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 7407 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7408 "ahci_intr_fatal_error: port %d " 7409 "task_file_status = 0x%x", port, task_file_status); 7410 } 7411 7412 /* 7413 * Here we just log the fatal error info in interrupt context. 7414 * Misc recovery processing will be handled in task queue. 7415 */ 7416 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7417 /* 7418 * Read PxCMD.CCS to determine the slot that the HBA 7419 * was processing when the error occurred. 7420 */ 7421 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7422 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7423 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7424 AHCI_CMD_STATUS_CCS_SHIFT; 7425 7426 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 7427 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7428 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 7429 "fatal error occurred for port %d", spkt, port); 7430 7431 /* A Task File Data error. */ 7432 if (intr_status & AHCI_INTR_STATUS_TFES) { 7433 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 7434 >> AHCI_TFD_ERR_SHIFT; 7435 7436 /* 7437 * Won't emit the error message if it is an IDENTIFY 7438 * DEVICE command sent to an ATAPI device. 7439 */ 7440 if ((spkt != NULL) && 7441 (spkt->satapkt_cmd.satacmd_cmd_reg == 7442 SATAC_ID_DEVICE) && 7443 (err_byte == SATA_ERROR_ABORT)) 7444 goto out1; 7445 7446 /* 7447 * Won't emit the error message if it is an ATAPI PACKET 7448 * command 7449 */ 7450 if ((spkt != NULL) && 7451 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 7452 goto out1; 7453 } 7454 } 7455 7456 /* print the fatal error type */ 7457 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 7458 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7459 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7460 7461 /* print PxSERR related error message */ 7462 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 7463 7464 /* print task file register value */ 7465 if (intr_status & AHCI_INTR_STATUS_TFES) { 7466 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 7467 "= 0x%x", instance, port, task_file_status); 7468 } 7469 7470 out1: 7471 /* Prepare the argument for the taskq */ 7472 args = ahci_portp->ahciport_event_args; 7473 args->ahciea_ctlp = (void *)ahci_ctlp; 7474 args->ahciea_portp = (void *)ahci_portp; 7475 args->ahciea_event = intr_status; 7476 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port); 7477 7478 /* Start the taskq to handle error recovery */ 7479 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 7480 ahci_events_handler, 7481 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 7482 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 7483 "failed", instance); 7484 } 7485 out0: 7486 mutex_exit(&ahci_portp->ahciport_mutex); 7487 7488 return (AHCI_SUCCESS); 7489 } 7490 7491 /* 7492 * Hot Plug Operation for platforms that support Cold Presence Detect. 7493 * 7494 * When set, a device status has changed as detected by the cold presence 7495 * detect logic. This bit can either be set due to a non-connected port 7496 * receiving a device, or a connected port having its device removed. 7497 * This bit is only valid if the port supports cold presence detect as 7498 * indicated by PxCMD.CPD set to '1'. 7499 * 7500 * At the moment, this interrupt is not needed and disabled and we just 7501 * log the debug message. 7502 */ 7503 static int 7504 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 7505 ahci_port_t *ahci_portp, uint8_t port) 7506 { 7507 uint32_t port_cmd_status; 7508 sata_device_t sdevice; 7509 7510 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7511 "ahci_intr_cold_port_detect enter, port %d", port); 7512 7513 mutex_enter(&ahci_portp->ahciport_mutex); 7514 7515 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7516 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7517 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 7518 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7519 "port %d does not support cold presence detect, so " 7520 "we just ignore this interrupt", port); 7521 mutex_exit(&ahci_portp->ahciport_mutex); 7522 return (AHCI_SUCCESS); 7523 } 7524 7525 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7526 "port %d device status has changed", port); 7527 7528 bzero((void *)&sdevice, sizeof (sata_device_t)); 7529 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7530 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7531 sdevice.satadev_addr.pmport = 0; 7532 sdevice.satadev_state = SATA_PSTATE_PWRON; 7533 7534 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 7535 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7536 "port %d: a device is hot plugged", port); 7537 mutex_exit(&ahci_portp->ahciport_mutex); 7538 sata_hba_event_notify( 7539 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7540 &sdevice, 7541 SATA_EVNT_DEVICE_ATTACHED); 7542 mutex_enter(&ahci_portp->ahciport_mutex); 7543 7544 } else { 7545 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7546 "port %d: a device is hot unplugged", port); 7547 mutex_exit(&ahci_portp->ahciport_mutex); 7548 sata_hba_event_notify( 7549 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7550 &sdevice, 7551 SATA_EVNT_DEVICE_DETACHED); 7552 mutex_enter(&ahci_portp->ahciport_mutex); 7553 } 7554 7555 mutex_exit(&ahci_portp->ahciport_mutex); 7556 7557 return (AHCI_SUCCESS); 7558 } 7559 7560 /* 7561 * Enable the interrupts for a particular port. 7562 * 7563 * WARNING!!! ahciport_mutex should be acquired before the function 7564 * is called. 7565 */ 7566 static void 7567 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 7568 { 7569 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7570 "ahci_enable_port_intrs enter, port %d", port); 7571 7572 /* 7573 * Clear port interrupt status before enabling interrupt 7574 */ 7575 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7576 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 7577 AHCI_PORT_INTR_MASK); 7578 7579 /* 7580 * Clear the pending bit from IS.IPS 7581 */ 7582 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7583 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 7584 7585 /* 7586 * Enable the following interrupts: 7587 * Device to Host Register FIS Interrupt (DHRS) 7588 * PIO Setup FIS Interrupt (PSS) 7589 * Set Device Bits Interrupt (SDBS) 7590 * Unknown FIS Interrupt (UFS) 7591 * Port Connect Change Status (PCS) 7592 * PhyRdy Change Status (PRCS) 7593 * Overflow Status (OFS) 7594 * Interface Non-fatal Error Status (INFS) 7595 * Interface Fatal Error Status (IFS) 7596 * Host Bus Data Error Status (HBDS) 7597 * Host Bus Fatal Error Status (HBFS) 7598 * Task File Error Status (TFES) 7599 */ 7600 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7601 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 7602 (AHCI_INTR_STATUS_DHRS | 7603 AHCI_INTR_STATUS_PSS | 7604 AHCI_INTR_STATUS_SDBS | 7605 AHCI_INTR_STATUS_UFS | 7606 AHCI_INTR_STATUS_DPS | 7607 AHCI_INTR_STATUS_PCS | 7608 AHCI_INTR_STATUS_PRCS | 7609 AHCI_INTR_STATUS_OFS | 7610 AHCI_INTR_STATUS_INFS | 7611 AHCI_INTR_STATUS_IFS | 7612 AHCI_INTR_STATUS_HBDS | 7613 AHCI_INTR_STATUS_HBFS | 7614 AHCI_INTR_STATUS_TFES)); 7615 } 7616 7617 /* 7618 * Enable interrupts for all the ports. 7619 * 7620 * WARNING!!! ahcictl_mutex should be acquired before the function 7621 * is called. 7622 */ 7623 static void 7624 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 7625 { 7626 uint32_t ghc_control; 7627 7628 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 7629 7630 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7631 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 7632 7633 ghc_control |= AHCI_HBA_GHC_IE; 7634 7635 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7636 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 7637 } 7638 7639 /* 7640 * Disable interrupts for a particular port. 7641 * 7642 * WARNING!!! ahciport_mutex should be acquired before the function 7643 * is called. 7644 */ 7645 static void 7646 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 7647 { 7648 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7649 "ahci_disable_port_intrs enter, port %d", port); 7650 7651 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7652 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 7653 } 7654 7655 /* 7656 * Disable interrupts for the whole HBA. 7657 * 7658 * The global bit is cleared, then all interrupt sources from all 7659 * ports are disabled. 7660 * 7661 * WARNING!!! ahcictl_mutex should be acquired before the function 7662 * is called. 7663 */ 7664 static void 7665 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 7666 { 7667 uint32_t ghc_control; 7668 7669 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 7670 NULL); 7671 7672 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7673 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 7674 7675 ghc_control &= ~ AHCI_HBA_GHC_IE; 7676 7677 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7678 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 7679 } 7680 7681 /* 7682 * Handle FIXED or MSI interrupts. 7683 */ 7684 /* 7685 * According to AHCI spec, the HBA may support several interrupt modes: 7686 * * pin based interrupts (FIXED) 7687 * * single MSI message interrupts 7688 * * multiple MSI based message interrupts 7689 * 7690 * For pin based interrupts, the software interrupt handler need to check IS 7691 * register to find out which port has pending interrupts. And then check 7692 * PxIS register to find out which interrupt events happened on that port. 7693 * 7694 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 7695 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 7696 * in that software interrupt handler need to check IS register to determine 7697 * which port triggered the interrupts since it uses a single message for all 7698 * port interrupts. 7699 * 7700 * HBA may optionally support multiple MSI message for better performance. In 7701 * this mode, each port may have its own interrupt message, and thus generation 7702 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 7703 * represents a power-of-2 wrapper on the number of implemented ports, and 7704 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 7705 * maximum number of assigned interrupts. When the number of MSI messages 7706 * allocated is less than the number requested, then hardware may have two 7707 * implementation behaviors: 7708 * * assign each ports its own interrupt and then force all additional 7709 * ports to share the last interrupt message, and this condition is 7710 * indicated by clearing GHC.MRSM to '0' 7711 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 7712 * When multiple-message MSI is enabled, hardware will still set IS register 7713 * as single message case. And this IS register may be used by software when 7714 * fewer than the requested number of messages is granted in order to determine 7715 * which port had the interrupt. 7716 * 7717 * Note: The current ahci driver only supports the first two interrupt modes: 7718 * pin based interrupts and single MSI message interrupts, and the reason 7719 * is indicated in below code. 7720 */ 7721 static int 7722 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 7723 { 7724 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 7725 int count, avail, actual; 7726 int i, rc; 7727 7728 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 7729 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 7730 7731 /* get number of interrupts. */ 7732 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 7733 if ((rc != DDI_SUCCESS) || (count == 0)) { 7734 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7735 "ddi_intr_get_nintrs() failed, " 7736 "rc %d count %d\n", rc, count); 7737 return (DDI_FAILURE); 7738 } 7739 7740 /* get number of available interrupts. */ 7741 rc = ddi_intr_get_navail(dip, intr_type, &avail); 7742 if ((rc != DDI_SUCCESS) || (avail == 0)) { 7743 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7744 "ddi_intr_get_navail() failed, " 7745 "rc %d avail %d\n", rc, avail); 7746 return (DDI_FAILURE); 7747 } 7748 7749 #if AHCI_DEBUG 7750 if (avail < count) { 7751 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7752 "ddi_intr_get_nintrs returned %d, navail() returned %d", 7753 count, avail); 7754 } 7755 #endif 7756 7757 /* 7758 * Note: So far Solaris restricts the maximum number of messages for 7759 * x86 to 2, that is avail is 2, so here we set the count with 1 to 7760 * force the driver to use single MSI message interrupt. In future if 7761 * Solaris remove the restriction, then we need to delete the below 7762 * code and try to use multiple interrupt routine to gain better 7763 * performance. 7764 */ 7765 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 7766 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7767 "force to use one interrupt routine though the " 7768 "HBA supports %d interrupt", count); 7769 count = 1; 7770 } 7771 7772 /* Allocate an array of interrupt handles. */ 7773 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 7774 ahci_ctlp->ahcictl_intr_htable = 7775 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 7776 7777 /* call ddi_intr_alloc(). */ 7778 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 7779 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 7780 7781 if ((rc != DDI_SUCCESS) || (actual == 0)) { 7782 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7783 "ddi_intr_alloc() failed, rc %d count %d actual %d " 7784 "avail %d\n", rc, count, actual, avail); 7785 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7786 ahci_ctlp->ahcictl_intr_size); 7787 return (DDI_FAILURE); 7788 } 7789 7790 /* use interrupt count returned */ 7791 #if AHCI_DEBUG 7792 if (actual < count) { 7793 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7794 "Requested: %d, Received: %d", count, actual); 7795 } 7796 #endif 7797 7798 ahci_ctlp->ahcictl_intr_cnt = actual; 7799 7800 /* 7801 * Get priority for first, assume remaining are all the same. 7802 */ 7803 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 7804 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 7805 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7806 "ddi_intr_get_pri() failed", NULL); 7807 7808 /* Free already allocated intr. */ 7809 for (i = 0; i < actual; i++) { 7810 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 7811 } 7812 7813 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7814 ahci_ctlp->ahcictl_intr_size); 7815 return (DDI_FAILURE); 7816 } 7817 7818 /* Test for high level interrupt. */ 7819 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 7820 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7821 "ahci_add_intrs: Hi level intr not supported", NULL); 7822 7823 /* Free already allocated intr. */ 7824 for (i = 0; i < actual; i++) { 7825 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 7826 } 7827 7828 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7829 sizeof (ddi_intr_handle_t)); 7830 7831 return (DDI_FAILURE); 7832 } 7833 7834 /* Call ddi_intr_add_handler(). */ 7835 for (i = 0; i < actual; i++) { 7836 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 7837 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 7838 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7839 "ddi_intr_add_handler() failed", NULL); 7840 7841 /* Free already allocated intr. */ 7842 for (i = 0; i < actual; i++) { 7843 (void) ddi_intr_free( 7844 ahci_ctlp->ahcictl_intr_htable[i]); 7845 } 7846 7847 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7848 ahci_ctlp->ahcictl_intr_size); 7849 return (DDI_FAILURE); 7850 } 7851 } 7852 7853 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 7854 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 7855 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7856 "ddi_intr_get_cap() failed", NULL); 7857 7858 /* Free already allocated intr. */ 7859 for (i = 0; i < actual; i++) { 7860 (void) ddi_intr_free( 7861 ahci_ctlp->ahcictl_intr_htable[i]); 7862 } 7863 7864 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7865 ahci_ctlp->ahcictl_intr_size); 7866 return (DDI_FAILURE); 7867 } 7868 7869 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 7870 /* Call ddi_intr_block_enable() for MSI. */ 7871 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 7872 ahci_ctlp->ahcictl_intr_cnt); 7873 } else { 7874 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 7875 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 7876 (void) ddi_intr_enable( 7877 ahci_ctlp->ahcictl_intr_htable[i]); 7878 } 7879 } 7880 7881 return (DDI_SUCCESS); 7882 } 7883 7884 /* 7885 * Removes the registered interrupts irrespective of whether they 7886 * were legacy or MSI. 7887 * 7888 * WARNING!!! The controller interrupts must be disabled before calling 7889 * this routine. 7890 */ 7891 static void 7892 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 7893 { 7894 int x; 7895 7896 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 7897 7898 /* Disable all interrupts. */ 7899 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 7900 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 7901 /* Call ddi_intr_block_disable(). */ 7902 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 7903 ahci_ctlp->ahcictl_intr_cnt); 7904 } else { 7905 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 7906 (void) ddi_intr_disable( 7907 ahci_ctlp->ahcictl_intr_htable[x]); 7908 } 7909 } 7910 7911 /* Call ddi_intr_remove_handler(). */ 7912 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 7913 (void) ddi_intr_remove_handler( 7914 ahci_ctlp->ahcictl_intr_htable[x]); 7915 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 7916 } 7917 7918 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 7919 } 7920 7921 /* 7922 * This routine tries to put port into P:NotRunning state by clearing 7923 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 7924 * and PxCMD.CR to '0'. 7925 * 7926 * WARNING!!! ahciport_mutex should be acquired before the function 7927 * is called. 7928 */ 7929 static int 7930 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 7931 ahci_port_t *ahci_portp, uint8_t port) 7932 { 7933 uint32_t port_cmd_status; 7934 int loop_count; 7935 7936 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7937 "ahci_put_port_into_notrunning_state enter: port %d", port); 7938 7939 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7940 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7941 7942 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 7943 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7944 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 7945 7946 /* Wait until PxCMD.CR is cleared */ 7947 loop_count = 0; 7948 do { 7949 port_cmd_status = 7950 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7951 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7952 7953 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 7954 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 7955 "clearing port %d CMD.CR timeout, " 7956 "port_cmd_status = 0x%x", port, 7957 port_cmd_status); 7958 /* 7959 * We are effectively timing out after 0.5 sec. 7960 * This value is specified in AHCI spec. 7961 */ 7962 break; 7963 } 7964 7965 /* Wait for 10 millisec */ 7966 drv_usecwait(AHCI_10MS_USECS); 7967 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 7968 7969 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 7970 7971 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 7972 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 7973 "ahci_put_port_into_notrunning_state: failed to clear " 7974 "PxCMD.CR to '0' after loop count: %d, and " 7975 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 7976 return (AHCI_FAILURE); 7977 } else { 7978 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 7979 "ahci_put_port_into_notrunning_state: succeeded to clear " 7980 "PxCMD.CR to '0' after loop count: %d, and " 7981 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 7982 return (AHCI_SUCCESS); 7983 } 7984 } 7985 7986 /* 7987 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 7988 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 7989 * stable state, then set PxCMD.ST to '1' to start the port directly. 7990 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 7991 * COMRESET to the device to put it in an idle state. 7992 * 7993 * The fifth argument returns whether the port reset is involved during 7994 * the process. 7995 * 7996 * The routine will be called under following scenarios: 7997 * + To reset the HBA 7998 * + To abort the packet(s) 7999 * + To reset the port 8000 * + To activate the port 8001 * + Fatal error recovery 8002 * + To abort the timeout packet(s) 8003 * 8004 * WARNING!!! ahciport_mutex should be acquired before the function 8005 * is called. And ahciport_mutex will be released before the reset 8006 * event is reported to sata module by calling sata_hba_event_notify, 8007 * and then be acquired again later. 8008 * 8009 * NOTES!!! During this procedure, PxSERR register will be cleared, and 8010 * according to the spec, the clearance of three bits will also clear 8011 * three interrupt status bits. 8012 * 1. PxSERR.DIAG.F will clear PxIS.UFS 8013 * 2. PxSERR.DIAG.X will clear PxIS.PCS 8014 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 8015 * 8016 * Among these three interrupt events, the driver needs to take care of 8017 * PxIS.PRCS, which is the hot plug event. When the driver found out 8018 * a device was unplugged, it will call the interrupt handler. 8019 */ 8020 static int 8021 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 8022 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 8023 { 8024 uint32_t port_sstatus; 8025 uint32_t task_file_status; 8026 sata_device_t sdevice; 8027 int rval; 8028 ahci_addr_t addr_port; 8029 ahci_pmult_info_t *pminfo = NULL; 8030 int dev_exists_begin = 0; 8031 int dev_exists_end = 0; 8032 uint32_t previous_dev_type = ahci_portp->ahciport_device_type; 8033 int npmport = 0; 8034 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8035 8036 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8037 "ahci_restart_port_wait_till_ready: port %d enter", port); 8038 8039 AHCI_ADDR_SET_PORT(&addr_port, port); 8040 8041 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 8042 dev_exists_begin = 1; 8043 8044 /* First clear PxCMD.ST */ 8045 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 8046 port); 8047 if (rval != AHCI_SUCCESS) 8048 /* 8049 * If PxCMD.CR does not clear within a reasonable time, it 8050 * may assume the interface is in a hung condition and may 8051 * continue with issuing the port reset. 8052 */ 8053 goto reset; 8054 8055 /* Then clear PxSERR */ 8056 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8057 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 8058 AHCI_SERROR_CLEAR_ALL); 8059 8060 /* Then get PxTFD */ 8061 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8062 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 8063 8064 /* 8065 * Check whether the device is in a stable status, if yes, 8066 * then start the port directly. However for ahci_tran_reset_dport, 8067 * we may have to perform a port reset. 8068 */ 8069 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 8070 !(flag & AHCI_PORT_RESET)) 8071 goto out; 8072 8073 reset: 8074 /* 8075 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 8076 * a COMRESET to the device 8077 */ 8078 ahci_disable_port_intrs(ahci_ctlp, port); 8079 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port); 8080 ahci_enable_port_intrs(ahci_ctlp, port); 8081 8082 #ifdef AHCI_DEBUG 8083 if (rval != AHCI_SUCCESS) 8084 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8085 "ahci_restart_port_wait_till_ready: port %d failed", 8086 port); 8087 #endif 8088 8089 if (reset_flag != NULL) 8090 *reset_flag = 1; 8091 8092 /* Indicate to the framework that a reset has happened. */ 8093 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 8094 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) && 8095 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 8096 /* Set the reset in progress flag */ 8097 ahci_portp->ahciport_reset_in_progress = 1; 8098 8099 bzero((void *)&sdevice, sizeof (sata_device_t)); 8100 sdevice.satadev_addr.cport = 8101 ahci_ctlp->ahcictl_port_to_cport[port]; 8102 sdevice.satadev_addr.pmport = 0; 8103 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8104 8105 sdevice.satadev_state = SATA_DSTATE_RESET | 8106 SATA_DSTATE_PWR_ACTIVE; 8107 if (ahci_ctlp->ahcictl_sata_hba_tran) { 8108 mutex_exit(&ahci_portp->ahciport_mutex); 8109 sata_hba_event_notify( 8110 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8111 &sdevice, 8112 SATA_EVNT_DEVICE_RESET); 8113 mutex_enter(&ahci_portp->ahciport_mutex); 8114 } 8115 8116 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8117 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8118 } else { 8119 ahci_portp->ahciport_reset_in_progress = 0; 8120 } 8121 8122 out: 8123 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8124 8125 /* SStatus tells the presence of device. */ 8126 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8127 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 8128 8129 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 8130 dev_exists_end = 1; 8131 } 8132 8133 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */ 8134 return (rval); 8135 8136 /* Check whether a hot plug event happened */ 8137 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */ 8138 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8139 "ahci_restart_port_wait_till_ready: port %d " 8140 "device is removed", port); 8141 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 8142 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8143 "ahci_restart_port_wait_till_ready: port %d " 8144 "AHCI_PORT_FLAG_NODEV flag is set", port); 8145 mutex_exit(&ahci_portp->ahciport_mutex); 8146 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 8147 mutex_enter(&ahci_portp->ahciport_mutex); 8148 8149 return (rval); 8150 } 8151 8152 8153 /* 0/1 -> 1 : device may change */ 8154 /* 8155 * May be called by ahci_fatal_error_recovery_handler, so 8156 * don't issue software if the previous device is ATAPI. 8157 */ 8158 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 8159 return (rval); 8160 8161 /* 8162 * The COMRESET will make port multiplier enter legacy mode. 8163 * Issue a software reset to make it work again. 8164 */ 8165 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port); 8166 8167 /* 8168 * Following codes are specific for the port multiplier 8169 */ 8170 if (previous_dev_type != SATA_DTYPE_PMULT && 8171 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8172 /* in case previous_dev_type is corrupt */ 8173 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8174 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8175 return (rval); 8176 } 8177 8178 /* Device change: PMult -> Non-PMult */ 8179 if (previous_dev_type == SATA_DTYPE_PMULT && 8180 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8181 /* 8182 * This might happen because 8183 * 1. Software reset failed. Port multiplier is not correctly 8184 * enumerated. 8185 * 2. Another non-port-multiplier device is attached. Perhaps 8186 * the port multiplier was replaced by another device by 8187 * whatever reason, but AHCI driver missed hot-plug event. 8188 * 8189 * Now that the port has been initialized, we just need to 8190 * update the port structure according new device, then report 8191 * and wait SATA framework to probe new device. 8192 */ 8193 8194 /* Force to release pmult resource */ 8195 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8196 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8197 8198 bzero((void *)&sdevice, sizeof (sata_device_t)); 8199 sdevice.satadev_addr.cport = 8200 ahci_ctlp->ahcictl_port_to_cport[port]; 8201 sdevice.satadev_addr.pmport = 0; 8202 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8203 8204 sdevice.satadev_state = SATA_DSTATE_RESET | 8205 SATA_DSTATE_PWR_ACTIVE; 8206 8207 mutex_exit(&ahci_portp->ahciport_mutex); 8208 sata_hba_event_notify( 8209 ahci_ctlp->ahcictl_dip, 8210 &sdevice, 8211 SATA_EVNT_DEVICE_RESET); 8212 mutex_enter(&ahci_portp->ahciport_mutex); 8213 8214 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8215 "Port multiplier is [Gone] at port %d ", port); 8216 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8217 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8218 8219 return (AHCI_SUCCESS); 8220 } 8221 8222 /* Device change: Non-PMult -> PMult */ 8223 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8224 8225 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */ 8226 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 8227 8228 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8229 } 8230 pminfo = ahci_portp->ahciport_pmult_info; 8231 ASSERT(pminfo != NULL); 8232 8233 /* Device (may) change: PMult -> PMult */ 8234 /* 8235 * First initialize port multiplier. Set state to READY and wait for 8236 * probe entry point to initialize it 8237 */ 8238 ahci_portp->ahciport_port_state = SATA_STATE_READY; 8239 8240 /* 8241 * It's a little complicated while target is a port multiplier. we 8242 * need to COMRESET all pmports behind that PMult otherwise those 8243 * sub-links between the PMult and the sub-devices will be in an 8244 * inactive state (indicated by PSCR0/PxSSTS) and the following access 8245 * to those sub-devices will be rejected by Link-Fatal-Error. 8246 */ 8247 /* 8248 * The PxSNTF will be set soon after the pmult is plugged. While the 8249 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we 8250 * simply mark every sub-port as 'unknown', then ahci_probe_pmport 8251 * will initialized it. 8252 */ 8253 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++) 8254 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN; 8255 8256 /* Report reset event. */ 8257 ahci_portp->ahciport_reset_in_progress = 1; 8258 8259 bzero((void *)&sdevice, sizeof (sata_device_t)); 8260 sdevice.satadev_addr.cport = cport; 8261 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 8262 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 8263 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE; 8264 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice, 8265 SATA_EVNT_DEVICE_RESET); 8266 8267 return (rval); 8268 } 8269 8270 /* 8271 * This routine may be called under four scenarios: 8272 * a) do the recovery from fatal error 8273 * b) or we need to timeout some commands 8274 * c) or we need to abort some commands 8275 * d) or we need reset device/port/controller 8276 * 8277 * In all these scenarios, we need to send any pending unfinished 8278 * commands up to sata framework. 8279 * 8280 * WARNING!!! ahciport_mutex should be acquired before the function is called. 8281 */ 8282 static void 8283 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 8284 ahci_port_t *ahci_portp, 8285 uint32_t slot_status, 8286 uint32_t failed_tags, 8287 uint32_t timeout_tags, 8288 uint32_t aborted_tags, 8289 uint32_t reset_tags) 8290 { 8291 uint32_t finished_tags = 0; 8292 uint32_t unfinished_tags = 0; 8293 int tmp_slot; 8294 sata_pkt_t *satapkt; 8295 int ncq_cmd_in_progress = 0; 8296 int err_retri_cmd_in_progress = 0; 8297 int rdwr_pmult_cmd_in_progress = 0; 8298 8299 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8300 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 8301 ahci_portp->ahciport_port_num, slot_status); 8302 8303 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8304 "ahci_mop_commands: failed_tags: 0x%x, " 8305 "timeout_tags: 0x%x aborted_tags: 0x%x, " 8306 "reset_tags: 0x%x", failed_tags, 8307 timeout_tags, aborted_tags, reset_tags); 8308 8309 #ifdef AHCI_DEBUG 8310 if (ahci_debug_flags & AHCIDBG_ERRS) { 8311 int i; 8312 char msg_buf[200] = {0, }; 8313 for (i = 0x1f; i >= 0; i--) { 8314 if (ahci_portp->ahciport_slot_pkts[i] != NULL) 8315 msg_buf[i] = 'X'; 8316 else 8317 msg_buf[i] = '.'; 8318 } 8319 msg_buf[0x20] = '\0'; 8320 cmn_err(CE_NOTE, "port[%d] slots: %s", 8321 ahci_portp->ahciport_port_num, msg_buf); 8322 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ", 8323 (void *)ahci_portp->ahciport_err_retri_pkt, 8324 (void *)ahci_portp->ahciport_rdwr_pmult_pkt); 8325 } 8326 #endif 8327 8328 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8329 finished_tags = ahci_portp->ahciport_pending_tags & 8330 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 8331 8332 unfinished_tags = slot_status & 8333 AHCI_SLOT_MASK(ahci_ctlp) & 8334 ~failed_tags & 8335 ~aborted_tags & 8336 ~reset_tags & 8337 ~timeout_tags; 8338 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8339 ncq_cmd_in_progress = 1; 8340 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 8341 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 8342 8343 unfinished_tags = slot_status & 8344 AHCI_NCQ_SLOT_MASK(ahci_portp) & 8345 ~failed_tags & 8346 ~aborted_tags & 8347 ~reset_tags & 8348 ~timeout_tags; 8349 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8350 8351 /* 8352 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 8353 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 8354 * complete successfully due to one of the following three 8355 * conditions: 8356 * 8357 * 1. Fatal error - failed_tags includes its slot 8358 * 2. Timed out - timeout_tags includes its slot 8359 * 3. Aborted when hot unplug - aborted_tags includes its 8360 * slot 8361 * 8362 * Please note that the command is always sent down in Slot 0 8363 */ 8364 err_retri_cmd_in_progress = 1; 8365 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 8366 "ahci_mop_commands is called for port %d while " 8367 "REQUEST SENSE or READ LOG EXT for error retrieval " 8368 "is being executed slot_status = 0x%x", 8369 ahci_portp->ahciport_port_num, slot_status); 8370 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 8371 ASSERT(slot_status == 0x1); 8372 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 8373 rdwr_pmult_cmd_in_progress = 1; 8374 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 8375 "ahci_mop_commands is called for port %d while " 8376 "READ/WRITE PORTMULT command is being executed", 8377 ahci_portp->ahciport_port_num); 8378 8379 ASSERT(slot_status == 0x1); 8380 } 8381 8382 #ifdef AHCI_DEBUG 8383 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8384 "ahci_mop_commands: finished_tags: 0x%x, " 8385 "unfinished_tags 0x%x", finished_tags, unfinished_tags); 8386 #endif 8387 8388 /* Send up finished packets with SATA_PKT_COMPLETED */ 8389 while (finished_tags) { 8390 tmp_slot = ddi_ffs(finished_tags) - 1; 8391 if (tmp_slot == -1) { 8392 break; 8393 } 8394 8395 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8396 ASSERT(satapkt != NULL); 8397 8398 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 8399 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 8400 (void *)satapkt); 8401 8402 /* 8403 * Cannot fetch the return register content since the port 8404 * was restarted, so the corresponding tag will be set to 8405 * aborted tags. 8406 */ 8407 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 8408 CLEAR_BIT(finished_tags, tmp_slot); 8409 aborted_tags |= tmp_slot; 8410 continue; 8411 } 8412 8413 if (ncq_cmd_in_progress) 8414 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8415 tmp_slot); 8416 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8417 CLEAR_BIT(finished_tags, tmp_slot); 8418 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8419 8420 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 8421 } 8422 8423 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 8424 while (failed_tags) { 8425 if (err_retri_cmd_in_progress) { 8426 satapkt = ahci_portp->ahciport_err_retri_pkt; 8427 ASSERT(satapkt != NULL); 8428 ASSERT(failed_tags == 0x1); 8429 8430 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8431 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8432 (void *)satapkt); 8433 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8434 break; 8435 } 8436 if (rdwr_pmult_cmd_in_progress) { 8437 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8438 ASSERT(satapkt != NULL); 8439 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8440 "ahci_mop_commands: sending up " 8441 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR", 8442 (void *)satapkt); 8443 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8444 break; 8445 } 8446 8447 tmp_slot = ddi_ffs(failed_tags) - 1; 8448 if (tmp_slot == -1) { 8449 break; 8450 } 8451 8452 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8453 ASSERT(satapkt != NULL); 8454 8455 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8456 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8457 (void *)satapkt); 8458 8459 if (ncq_cmd_in_progress) 8460 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8461 tmp_slot); 8462 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8463 CLEAR_BIT(failed_tags, tmp_slot); 8464 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8465 8466 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8467 } 8468 8469 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 8470 while (timeout_tags) { 8471 if (err_retri_cmd_in_progress) { 8472 satapkt = ahci_portp->ahciport_err_retri_pkt; 8473 ASSERT(satapkt != NULL); 8474 ASSERT(timeout_tags == 0x1); 8475 8476 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8477 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8478 (void *)satapkt); 8479 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8480 break; 8481 } 8482 if (rdwr_pmult_cmd_in_progress) { 8483 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8484 ASSERT(satapkt != NULL); 8485 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8486 "ahci_mop_commands: sending up " 8487 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT", 8488 (void *)satapkt); 8489 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8490 break; 8491 } 8492 8493 tmp_slot = ddi_ffs(timeout_tags) - 1; 8494 if (tmp_slot == -1) { 8495 break; 8496 } 8497 8498 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8499 ASSERT(satapkt != NULL); 8500 8501 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8502 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8503 (void *)satapkt); 8504 8505 if (ncq_cmd_in_progress) 8506 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8507 tmp_slot); 8508 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8509 CLEAR_BIT(timeout_tags, tmp_slot); 8510 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8511 8512 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8513 } 8514 8515 /* Send up aborted packets with SATA_PKT_ABORTED */ 8516 while (aborted_tags) { 8517 if (err_retri_cmd_in_progress) { 8518 satapkt = ahci_portp->ahciport_err_retri_pkt; 8519 ASSERT(satapkt != NULL); 8520 ASSERT(aborted_tags == 0x1); 8521 8522 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8523 "sending up pkt 0x%p with SATA_PKT_ABORTED", 8524 (void *)satapkt); 8525 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8526 break; 8527 } 8528 if (rdwr_pmult_cmd_in_progress) { 8529 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8530 ASSERT(satapkt != NULL); 8531 ASSERT(aborted_tags == 0x1); 8532 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8533 "ahci_mop_commands: sending up " 8534 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED", 8535 (void *)satapkt); 8536 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8537 break; 8538 } 8539 8540 tmp_slot = ddi_ffs(aborted_tags) - 1; 8541 if (tmp_slot == -1) { 8542 break; 8543 } 8544 8545 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8546 ASSERT(satapkt != NULL); 8547 8548 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8549 "sending up pkt 0x%p with SATA_PKT_ABORTED", 8550 (void *)satapkt); 8551 8552 if (ncq_cmd_in_progress) 8553 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8554 tmp_slot); 8555 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8556 CLEAR_BIT(aborted_tags, tmp_slot); 8557 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8558 8559 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8560 } 8561 8562 /* Send up reset packets with SATA_PKT_RESET. */ 8563 while (reset_tags) { 8564 if (rdwr_pmult_cmd_in_progress) { 8565 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8566 ASSERT(satapkt != NULL); 8567 ASSERT(aborted_tags == 0x1); 8568 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8569 "ahci_mop_commands: sending up " 8570 "rdwr pmult pkt 0x%p with SATA_PKT_RESET", 8571 (void *)satapkt); 8572 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8573 break; 8574 } 8575 8576 tmp_slot = ddi_ffs(reset_tags) - 1; 8577 if (tmp_slot == -1) { 8578 break; 8579 } 8580 8581 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8582 ASSERT(satapkt != NULL); 8583 8584 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8585 "sending up pkt 0x%p with SATA_PKT_RESET", 8586 (void *)satapkt); 8587 8588 if (ncq_cmd_in_progress) 8589 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8590 tmp_slot); 8591 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8592 CLEAR_BIT(reset_tags, tmp_slot); 8593 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8594 8595 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8596 } 8597 8598 /* Send up unfinished packets with SATA_PKT_RESET */ 8599 while (unfinished_tags) { 8600 tmp_slot = ddi_ffs(unfinished_tags) - 1; 8601 if (tmp_slot == -1) { 8602 break; 8603 } 8604 8605 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8606 ASSERT(satapkt != NULL); 8607 8608 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8609 "sending up pkt 0x%p with SATA_PKT_RESET", 8610 (void *)satapkt); 8611 8612 if (ncq_cmd_in_progress) 8613 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8614 tmp_slot); 8615 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8616 CLEAR_BIT(unfinished_tags, tmp_slot); 8617 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8618 8619 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8620 } 8621 8622 ahci_portp->ahciport_mop_in_progress--; 8623 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 8624 8625 if (ahci_portp->ahciport_mop_in_progress == 0) 8626 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 8627 } 8628 8629 /* 8630 * This routine is going to first request a READ LOG EXT sata pkt from sata 8631 * module, and then deliver it to the HBA to get the ncq failure context. 8632 * The return value is the exactly failed tags. 8633 */ 8634 static uint32_t 8635 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 8636 uint8_t port) 8637 { 8638 sata_device_t sdevice; 8639 sata_pkt_t *rdlog_spkt, *spkt; 8640 ddi_dma_handle_t buf_dma_handle; 8641 ahci_addr_t addr; 8642 int loop_count; 8643 int rval; 8644 int failed_slot; 8645 uint32_t failed_tags = 0; 8646 struct sata_ncq_error_recovery_page *ncq_err_page; 8647 8648 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 8649 "ahci_get_rdlogext_data enter: port %d", port); 8650 8651 /* Prepare the sdevice data */ 8652 bzero((void *)&sdevice, sizeof (sata_device_t)); 8653 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8654 8655 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8656 sdevice.satadev_addr.pmport = 0; 8657 8658 /* Translate sata_device.satadev_addr -> ahci_addr */ 8659 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 8660 8661 /* 8662 * Call the sata hba interface to get a rdlog spkt 8663 */ 8664 loop_count = 0; 8665 loop: 8666 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 8667 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 8668 if (rdlog_spkt == NULL) { 8669 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 8670 /* Sleep for a while */ 8671 drv_usecwait(AHCI_10MS_USECS); 8672 goto loop; 8673 } 8674 /* Timed out after 1s */ 8675 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8676 "failed to get rdlog spkt for port %d", port); 8677 return (failed_tags); 8678 } 8679 8680 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 8681 8682 /* 8683 * This flag is used to handle the specific error recovery when the 8684 * READ LOG EXT command gets a failure (fatal error or time-out). 8685 */ 8686 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 8687 8688 /* 8689 * This start is not supposed to fail because after port is restarted, 8690 * the whole command list is empty. 8691 */ 8692 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 8693 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt); 8694 ahci_portp->ahciport_err_retri_pkt = NULL; 8695 8696 /* Remove the flag after READ LOG EXT command is completed */ 8697 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 8698 8699 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 8700 /* Update the request log data */ 8701 buf_dma_handle = *(ddi_dma_handle_t *) 8702 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 8703 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 8704 DDI_DMA_SYNC_FORKERNEL); 8705 if (rval == DDI_SUCCESS) { 8706 ncq_err_page = 8707 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 8708 satapkt_cmd.satacmd_bp->b_un.b_addr; 8709 8710 /* Get the failed tag */ 8711 failed_slot = ncq_err_page->ncq_tag; 8712 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8713 "ahci_get_rdlogext_data: port %d " 8714 "failed slot %d", port, failed_slot); 8715 if (failed_slot & NQ) { 8716 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8717 "the failed slot is not a valid tag", NULL); 8718 goto out; 8719 } 8720 8721 failed_slot &= NCQ_TAG_MASK; 8722 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 8723 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8724 "ahci_get_rdlogext_data: failed spkt 0x%p", 8725 (void *)spkt); 8726 if (spkt == NULL) { 8727 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8728 "the failed slot spkt is NULL", NULL); 8729 goto out; 8730 } 8731 8732 failed_tags = 0x1 << failed_slot; 8733 8734 /* Fill out the error context */ 8735 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 8736 ncq_err_page); 8737 ahci_update_sata_registers(ahci_ctlp, port, 8738 &spkt->satapkt_device); 8739 } 8740 } 8741 out: 8742 sata_free_error_retrieval_pkt(rdlog_spkt); 8743 8744 return (failed_tags); 8745 } 8746 8747 /* 8748 * This routine is going to first request a REQUEST SENSE sata pkt from sata 8749 * module, and then deliver it to the HBA to get the sense data and copy 8750 * the sense data back to the orignal failed sata pkt, and free the REQUEST 8751 * SENSE sata pkt later. 8752 */ 8753 static void 8754 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 8755 uint8_t port, sata_pkt_t *spkt) 8756 { 8757 sata_device_t sdevice; 8758 sata_pkt_t *rs_spkt; 8759 sata_cmd_t *sata_cmd; 8760 ddi_dma_handle_t buf_dma_handle; 8761 ahci_addr_t addr; 8762 int loop_count; 8763 #if AHCI_DEBUG 8764 struct scsi_extended_sense *rqsense; 8765 #endif 8766 8767 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8768 "ahci_get_rqsense_data enter: port %d", port); 8769 8770 /* Prepare the sdevice data */ 8771 bzero((void *)&sdevice, sizeof (sata_device_t)); 8772 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8773 8774 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8775 sdevice.satadev_addr.pmport = 0; 8776 8777 /* Translate sata_device.satadev_addr -> ahci_addr */ 8778 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 8779 8780 sata_cmd = &spkt->satapkt_cmd; 8781 8782 /* 8783 * Call the sata hba interface to get a rs spkt 8784 */ 8785 loop_count = 0; 8786 loop: 8787 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 8788 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 8789 if (rs_spkt == NULL) { 8790 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 8791 /* Sleep for a while */ 8792 drv_usecwait(AHCI_10MS_USECS); 8793 goto loop; 8794 8795 } 8796 /* Timed out after 1s */ 8797 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8798 "failed to get rs spkt for port %d", port); 8799 return; 8800 } 8801 8802 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 8803 8804 /* 8805 * This flag is used to handle the specific error recovery when the 8806 * REQUEST SENSE command gets a faiure (fatal error or time-out). 8807 */ 8808 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 8809 8810 /* 8811 * This start is not supposed to fail because after port is restarted, 8812 * the whole command list is empty. 8813 */ 8814 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 8815 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt); 8816 ahci_portp->ahciport_err_retri_pkt = NULL; 8817 8818 /* Remove the flag after REQUEST SENSE command is completed */ 8819 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 8820 8821 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 8822 /* Update the request sense data */ 8823 buf_dma_handle = *(ddi_dma_handle_t *) 8824 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 8825 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 8826 DDI_DMA_SYNC_FORKERNEL); 8827 /* Copy the request sense data */ 8828 bcopy(rs_spkt-> 8829 satapkt_cmd.satacmd_bp->b_un.b_addr, 8830 &sata_cmd->satacmd_rqsense, 8831 SATA_ATAPI_MIN_RQSENSE_LEN); 8832 #if AHCI_DEBUG 8833 rqsense = (struct scsi_extended_sense *) 8834 sata_cmd->satacmd_rqsense; 8835 8836 /* Dump the sense data */ 8837 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 8838 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 8839 "Sense data for satapkt %p ATAPI cmd 0x%x", 8840 spkt, sata_cmd->satacmd_acdb[0]); 8841 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 8842 " es_code 0x%x es_class 0x%x " 8843 "es_key 0x%x es_add_code 0x%x " 8844 "es_qual_code 0x%x", 8845 rqsense->es_code, rqsense->es_class, 8846 rqsense->es_key, rqsense->es_add_code, 8847 rqsense->es_qual_code); 8848 #endif 8849 } 8850 8851 sata_free_error_retrieval_pkt(rs_spkt); 8852 } 8853 8854 /* 8855 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 8856 * the port must be restarted. When the HBA detects thus error, it may try 8857 * to abort a transfer. And if the transfer was aborted, the device is 8858 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 8859 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 8860 * that the device is in a stable status and transfers may be restarted without 8861 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 8862 * then the software will send the COMRESET to do the port reset. 8863 * 8864 * Software should perform the appropriate error recovery actions based on 8865 * whether non-queued commands were being issued or natived command queuing 8866 * commands were being issued. 8867 * 8868 * And software will complete the command that had the error with error mark 8869 * to higher level software. 8870 * 8871 * Fatal errors include the following: 8872 * PxIS.IFS - Interface Fatal Error Status 8873 * PxIS.HBDS - Host Bus Data Error Status 8874 * PxIS.HBFS - Host Bus Fatal Error Status 8875 * PxIS.TFES - Task File Error Status 8876 * 8877 * WARNING!!! ahciport_mutex should be acquired before the function is called. 8878 */ 8879 static void 8880 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 8881 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status) 8882 { 8883 uint32_t port_cmd_status; 8884 uint32_t slot_status = 0; 8885 uint32_t failed_tags = 0; 8886 int failed_slot; 8887 int reset_flag = 0, flag = 0; 8888 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 8889 sata_cmd_t *sata_cmd = NULL; 8890 sata_pkt_t *spkt = NULL; 8891 #if AHCI_DEBUG 8892 ahci_cmd_header_t *cmd_header; 8893 #endif 8894 uint8_t port = addrp->aa_port; 8895 8896 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8897 "ahci_fatal_error_recovery_handler enter: port %d", port); 8898 8899 /* Port multiplier error */ 8900 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8901 /* FBS code is neither completed nor tested. */ 8902 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp, 8903 port, intr_status); 8904 8905 /* Force a port reset */ 8906 flag = AHCI_PORT_RESET; 8907 } 8908 8909 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 8910 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8911 8912 /* Read PxCI to see which commands are still outstanding */ 8913 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8914 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 8915 8916 /* 8917 * Read PxCMD.CCS to determine the slot that the HBA 8918 * was processing when the error occurred. 8919 */ 8920 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8921 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8922 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 8923 AHCI_CMD_STATUS_CCS_SHIFT; 8924 8925 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8926 spkt = ahci_portp->ahciport_err_retri_pkt; 8927 ASSERT(spkt != NULL); 8928 } else { 8929 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 8930 if (spkt == NULL) { 8931 /* May happen when interface errors occur? */ 8932 goto next; 8933 } 8934 } 8935 8936 #if AHCI_DEBUG 8937 /* 8938 * Debugging purpose... 8939 */ 8940 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 8941 cmd_header = 8942 &ahci_portp->ahciport_cmd_list[failed_slot]; 8943 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 8944 "ahci_fatal_error_recovery_handler: port %d, " 8945 "PRD Byte Count = 0x%x, " 8946 "ahciport_prd_bytecounts = 0x%x", port, 8947 cmd_header->ahcich_prd_byte_count, 8948 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 8949 } 8950 #endif 8951 8952 sata_cmd = &spkt->satapkt_cmd; 8953 8954 /* Fill out the status and error registers for PxIS.TFES */ 8955 if (intr_status & AHCI_INTR_STATUS_TFES) { 8956 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 8957 ahcirf_d2h_register_fis); 8958 8959 /* Copy the error context back to the sata_cmd */ 8960 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 8961 } 8962 8963 /* The failed command must be one of the outstanding commands */ 8964 failed_tags = 0x1 << failed_slot; 8965 ASSERT(failed_tags & slot_status); 8966 8967 /* Update the sata registers, especially PxSERR register */ 8968 ahci_update_sata_registers(ahci_ctlp, port, 8969 &spkt->satapkt_device); 8970 8971 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8972 /* Read PxSACT to see which commands are still outstanding */ 8973 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8974 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 8975 } 8976 next: 8977 8978 #if AHCI_DEBUG 8979 /* 8980 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 8981 * set, it means a fatal error happened after REQUEST SENSE command 8982 * or READ LOG EXT command is delivered to the HBA during the error 8983 * recovery process. At this time, the only outstanding command is 8984 * supposed to be REQUEST SENSE command or READ LOG EXT command. 8985 */ 8986 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8987 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8988 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 8989 "command or READ LOG EXT command for error data retrieval " 8990 "failed", port); 8991 ASSERT(slot_status == 0x1); 8992 ASSERT(failed_slot == 0); 8993 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 8994 SCMD_REQUEST_SENSE || 8995 spkt->satapkt_cmd.satacmd_cmd_reg == 8996 SATAC_READ_LOG_EXT); 8997 } 8998 #endif 8999 9000 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9001 ahci_portp->ahciport_mop_in_progress++; 9002 9003 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9004 port, flag, &reset_flag); 9005 9006 /* 9007 * Won't retrieve error information: 9008 * 1. Port reset was involved to recover 9009 * 2. Device is gone 9010 * 3. IDENTIFY DEVICE command sent to ATAPI device 9011 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 9012 */ 9013 if (reset_flag || 9014 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 9015 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 9016 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9017 goto out; 9018 9019 /* 9020 * Deliver READ LOG EXT to gather information about the error when 9021 * a COMRESET has not been performed as part of the error recovery 9022 * during NCQ command processing. 9023 */ 9024 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9025 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 9026 ahci_portp, port); 9027 goto out; 9028 } 9029 9030 /* 9031 * Deliver REQUEST SENSE for ATAPI command to gather information about 9032 * the error when a COMRESET has not been performed as part of the 9033 * error recovery. 9034 */ 9035 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 9036 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 9037 out: 9038 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9039 "ahci_fatal_error_recovery_handler: port %d fatal error " 9040 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 9041 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 9042 port, slot_status, ahci_portp->ahciport_pending_tags, 9043 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 9044 9045 ahci_mop_commands(ahci_ctlp, 9046 ahci_portp, 9047 slot_status, 9048 failed_tags, /* failed tags */ 9049 0, /* timeout tags */ 9050 0, /* aborted tags */ 9051 0); /* reset tags */ 9052 } 9053 9054 /* 9055 * Used to recovery a PMULT pmport fatal error under FIS-based switching. 9056 * 1. device specific.PxFBS.SDE=1 9057 * 2. Non-Deivce specific. 9058 * Nothing will be done when Command-based switching is employed. 9059 * 9060 * Currently code is neither completed nor tested. 9061 * 9062 * WARNING!!! ahciport_mutex should be acquired before the function 9063 * is called. 9064 */ 9065 static void 9066 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9067 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 9068 { 9069 #ifndef __lock_lint 9070 _NOTE(ARGUNUSED(intr_status)) 9071 #endif 9072 uint32_t port_fbs_ctrl; 9073 int loop_count = 0; 9074 ahci_addr_t addr; 9075 9076 /* Nothing will be done under Command-based switching. */ 9077 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS)) 9078 return; 9079 9080 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9081 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9082 9083 if (!(port_fbs_ctrl & AHCI_FBS_EN)) 9084 /* FBS is not enabled. */ 9085 return; 9086 9087 /* Problem's getting complicated now. */ 9088 /* 9089 * If FIS-based switching is used, we need to check 9090 * the PxFBS to see the error type. 9091 */ 9092 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9093 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9094 9095 /* Refer to spec(v1.2) 9.3.6.1 */ 9096 if (port_fbs_ctrl & AHCI_FBS_SDE) { 9097 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9098 "A Device Sepcific Error: port %d", port); 9099 /* 9100 * Controller has paused commands for all other 9101 * sub-devices until PxFBS.DEC is set. 9102 */ 9103 ahci_reject_all_abort_pkts(ahci_ctlp, 9104 ahci_portp, 0); 9105 9106 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 9107 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port), 9108 port_fbs_ctrl | AHCI_FBS_DEC); 9109 9110 /* 9111 * Wait controller clear PxFBS.DEC, 9112 * then we can continue. 9113 */ 9114 loop_count = 0; 9115 do { 9116 port_fbs_ctrl = ddi_get32(ahci_ctlp-> 9117 ahcictl_ahci_acc_handle, (uint32_t *) 9118 AHCI_PORT_PxFBS(ahci_ctlp, port)); 9119 9120 if (loop_count++ > 1000) 9121 /* 9122 * Esclate the error. Follow 9123 * non-device specific error 9124 * procedure. 9125 */ 9126 return; 9127 9128 drv_usecwait(AHCI_100US_USECS); 9129 } while (port_fbs_ctrl & AHCI_FBS_DEC); 9130 9131 /* 9132 * Issue a software reset to ensure drive is in 9133 * a known state. 9134 */ 9135 (void) ahci_software_reset(ahci_ctlp, 9136 ahci_portp, &addr); 9137 9138 } else { 9139 9140 /* Process Non-Device Specific Error. */ 9141 /* This will be handled later on. */ 9142 cmn_err(CE_NOTE, "!FBS is not supported now."); 9143 } 9144 } 9145 /* 9146 * Handle events - fatal error recovery 9147 */ 9148 static void 9149 ahci_events_handler(void *args) 9150 { 9151 ahci_event_arg_t *ahci_event_arg; 9152 ahci_ctl_t *ahci_ctlp; 9153 ahci_port_t *ahci_portp; 9154 ahci_addr_t *addrp; 9155 uint32_t event; 9156 9157 ahci_event_arg = (ahci_event_arg_t *)args; 9158 9159 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 9160 ahci_portp = ahci_event_arg->ahciea_portp; 9161 addrp = ahci_event_arg->ahciea_addrp; 9162 event = ahci_event_arg->ahciea_event; 9163 9164 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9165 "ahci_events_handler enter: port %d intr_status = 0x%x", 9166 ahci_portp->ahciport_port_num, event); 9167 9168 mutex_enter(&ahci_portp->ahciport_mutex); 9169 9170 /* 9171 * ahci_intr_phyrdy_change() may have rendered it to 9172 * SATA_DTYPE_NONE. 9173 */ 9174 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9175 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 9176 "ahci_events_handler: port %d no device attached, " 9177 "and just return without doing anything", 9178 ahci_portp->ahciport_port_num); 9179 goto out; 9180 } 9181 9182 if (event & (AHCI_INTR_STATUS_IFS | 9183 AHCI_INTR_STATUS_HBDS | 9184 AHCI_INTR_STATUS_HBFS | 9185 AHCI_INTR_STATUS_TFES)) 9186 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 9187 addrp, event); 9188 9189 out: 9190 mutex_exit(&ahci_portp->ahciport_mutex); 9191 } 9192 9193 /* 9194 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 9195 * detect there are some commands which are timed out. 9196 */ 9197 static void 9198 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9199 uint8_t port, uint32_t tmp_timeout_tags) 9200 { 9201 uint32_t slot_status = 0; 9202 uint32_t finished_tags = 0; 9203 uint32_t timeout_tags = 0; 9204 9205 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 9206 "ahci_timeout_pkts enter: port %d", port); 9207 9208 mutex_enter(&ahci_portp->ahciport_mutex); 9209 9210 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9211 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) || 9212 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9213 /* Read PxCI to see which commands are still outstanding */ 9214 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9215 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9216 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9217 /* Read PxSACT to see which commands are still outstanding */ 9218 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9219 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9220 } 9221 9222 #if AHCI_DEBUG 9223 /* 9224 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9225 * set, it means a fatal error happened after REQUEST SENSE command 9226 * or READ LOG EXT command is delivered to the HBA during the error 9227 * recovery process. At this time, the only outstanding command is 9228 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9229 */ 9230 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9231 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9232 "ahci_timeout_pkts called while REQUEST SENSE " 9233 "command or READ LOG EXT command for error recovery " 9234 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 9235 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 9236 tmp_timeout_tags, slot_status, 9237 ahci_portp->ahciport_pending_tags, 9238 ahci_portp->ahciport_pending_ncq_tags); 9239 ASSERT(slot_status == 0x1); 9240 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9241 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9242 "ahci_timeout_pkts called while executing R/W PMULT " 9243 "command timeout_tags = 0x%x, slot_status = 0x%x", 9244 tmp_timeout_tags, slot_status); 9245 ASSERT(slot_status == 0x1); 9246 } 9247 #endif 9248 9249 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9250 ahci_portp->ahciport_mop_in_progress++; 9251 9252 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9253 port, AHCI_PORT_RESET, NULL); 9254 9255 /* 9256 * Re-identify timeout tags because some previously checked commands 9257 * could already complete. 9258 */ 9259 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9260 finished_tags = ahci_portp->ahciport_pending_tags & 9261 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 9262 timeout_tags = tmp_timeout_tags & ~finished_tags; 9263 9264 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 9265 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9266 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 9267 "pending_tags = 0x%x ", 9268 port, finished_tags, timeout_tags, 9269 slot_status, ahci_portp->ahciport_pending_tags); 9270 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9271 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 9272 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 9273 timeout_tags = tmp_timeout_tags & ~finished_tags; 9274 9275 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 9276 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9277 "timeout_tags = 0x%x, port_sactive = 0x%x, " 9278 "pending_ncq_tags = 0x%x ", 9279 port, finished_tags, timeout_tags, 9280 slot_status, ahci_portp->ahciport_pending_ncq_tags); 9281 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9282 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9283 timeout_tags = tmp_timeout_tags; 9284 } 9285 9286 ahci_mop_commands(ahci_ctlp, 9287 ahci_portp, 9288 slot_status, 9289 0, /* failed tags */ 9290 timeout_tags, /* timeout tags */ 9291 0, /* aborted tags */ 9292 0); /* reset tags */ 9293 9294 mutex_exit(&ahci_portp->ahciport_mutex); 9295 } 9296 9297 /* 9298 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 9299 * for long time. 9300 */ 9301 static void 9302 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 9303 { 9304 ahci_port_t *ahci_portp; 9305 sata_pkt_t *spkt; 9306 uint32_t pending_tags; 9307 uint32_t timeout_tags; 9308 uint32_t port_cmd_status; 9309 uint32_t port_sactive; 9310 uint8_t port; 9311 int tmp_slot; 9312 int current_slot; 9313 uint32_t current_tags; 9314 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9315 /* max number of cycles this packet should survive */ 9316 int max_life_cycles; 9317 9318 /* how many cycles this packet survived so far */ 9319 int watched_cycles; 9320 9321 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9322 9323 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9324 "ahci_watchdog_handler entered", NULL); 9325 9326 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9327 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9328 continue; 9329 } 9330 9331 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9332 9333 mutex_enter(&ahci_portp->ahciport_mutex); 9334 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9335 mutex_exit(&ahci_portp->ahciport_mutex); 9336 continue; 9337 } 9338 9339 /* Skip the check for those ports in error recovery */ 9340 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 9341 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 9342 mutex_exit(&ahci_portp->ahciport_mutex); 9343 continue; 9344 } 9345 9346 pending_tags = 0; 9347 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9348 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9349 9350 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9351 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9352 current_slot = 0; 9353 pending_tags = 0x1; 9354 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9355 current_slot = 9356 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9357 AHCI_CMD_STATUS_CCS_SHIFT; 9358 pending_tags = ahci_portp->ahciport_pending_tags; 9359 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9360 port_sactive = ddi_get32( 9361 ahci_ctlp->ahcictl_ahci_acc_handle, 9362 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9363 current_tags = port_sactive & 9364 ~port_cmd_status & 9365 AHCI_NCQ_SLOT_MASK(ahci_portp); 9366 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 9367 } 9368 9369 timeout_tags = 0; 9370 while (pending_tags) { 9371 tmp_slot = ddi_ffs(pending_tags) - 1; 9372 if (tmp_slot == -1) { 9373 break; 9374 } 9375 9376 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9377 spkt = ahci_portp->ahciport_err_retri_pkt; 9378 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 9379 spkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9380 else 9381 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9382 9383 if ((spkt != NULL) && spkt->satapkt_time && 9384 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 9385 /* 9386 * We are overloading satapkt_hba_driver_private 9387 * with watched_cycle count. 9388 * 9389 * If a packet has survived for more than it's 9390 * max life cycles, it is a candidate for time 9391 * out. 9392 */ 9393 watched_cycles = (int)(intptr_t) 9394 spkt->satapkt_hba_driver_private; 9395 watched_cycles++; 9396 max_life_cycles = (spkt->satapkt_time + 9397 ahci_watchdog_timeout - 1) / 9398 ahci_watchdog_timeout; 9399 9400 spkt->satapkt_hba_driver_private = 9401 (void *)(intptr_t)watched_cycles; 9402 9403 if (watched_cycles <= max_life_cycles) 9404 goto next; 9405 9406 #if AHCI_DEBUG 9407 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9408 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9409 ahci_ctlp, "watchdog: the current " 9410 "tags is 0x%x", current_tags); 9411 } else { 9412 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9413 ahci_ctlp, "watchdog: the current " 9414 "slot is %d", current_slot); 9415 } 9416 #endif 9417 9418 /* 9419 * We need to check whether the HBA has 9420 * begun to execute the command, if not, 9421 * then re-set the timer of the command. 9422 */ 9423 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 9424 (tmp_slot != current_slot) || 9425 NCQ_CMD_IN_PROGRESS(ahci_portp) && 9426 ((0x1 << tmp_slot) & current_tags)) { 9427 spkt->satapkt_hba_driver_private = 9428 (void *)(intptr_t)0; 9429 } else { 9430 timeout_tags |= (0x1 << tmp_slot); 9431 cmn_err(CE_WARN, "!ahci%d: watchdog " 9432 "port %d satapkt 0x%p timed out\n", 9433 instance, port, (void *)spkt); 9434 } 9435 } 9436 next: 9437 CLEAR_BIT(pending_tags, tmp_slot); 9438 } 9439 9440 if (timeout_tags) { 9441 mutex_exit(&ahci_portp->ahciport_mutex); 9442 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9443 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 9444 port, timeout_tags); 9445 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9446 mutex_enter(&ahci_portp->ahciport_mutex); 9447 } 9448 9449 mutex_exit(&ahci_portp->ahciport_mutex); 9450 } 9451 9452 /* Re-install the watchdog timeout handler */ 9453 if (ahci_ctlp->ahcictl_timeout_id != 0) { 9454 ahci_ctlp->ahcictl_timeout_id = 9455 timeout((void (*)(void *))ahci_watchdog_handler, 9456 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 9457 } 9458 9459 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9460 } 9461 9462 /* 9463 * Fill the error context into sata_cmd for non-queued command error. 9464 */ 9465 static void 9466 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9467 { 9468 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 9469 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9470 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9471 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9472 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9473 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9474 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9475 9476 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 9477 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9478 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9479 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9480 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9481 } 9482 } 9483 9484 /* 9485 * Fill the ncq error page into sata_cmd for queued command error. 9486 */ 9487 static void 9488 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 9489 struct sata_ncq_error_recovery_page *ncq_err_page) 9490 { 9491 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 9492 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 9493 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 9494 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 9495 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 9496 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 9497 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 9498 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 9499 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 9500 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 9501 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 9502 } 9503 9504 /* 9505 * Put the respective register value to sata_cmd_t for satacmd_flags. 9506 */ 9507 static void 9508 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9509 { 9510 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 9511 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9512 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 9513 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9514 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 9515 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9516 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 9517 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9518 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 9519 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9520 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 9521 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9522 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 9523 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9524 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 9525 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9526 if (scmd->satacmd_flags.sata_copy_out_device_reg) 9527 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9528 if (scmd->satacmd_flags.sata_copy_out_error_reg) 9529 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9530 } 9531 9532 static void 9533 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 9534 uint32_t intr_status) 9535 { 9536 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9537 9538 if (intr_status & AHCI_INTR_STATUS_IFS) 9539 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 9540 "error", instance, port); 9541 9542 if (intr_status & AHCI_INTR_STATUS_HBDS) 9543 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 9544 instance, port); 9545 9546 if (intr_status & AHCI_INTR_STATUS_HBFS) 9547 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 9548 instance, port); 9549 9550 if (intr_status & AHCI_INTR_STATUS_TFES) 9551 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 9552 instance, port); 9553 9554 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 9555 "recovery", instance, port); 9556 } 9557 9558 /* 9559 * Dump the serror message to the log. 9560 */ 9561 static void 9562 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 9563 uint32_t port_serror, int debug_only) 9564 { 9565 static char err_buf[512]; 9566 static char err_msg_header[16]; 9567 char *err_msg = err_buf; 9568 9569 *err_buf = '\0'; 9570 *err_msg_header = '\0'; 9571 9572 if (port_serror & SERROR_DATA_ERR_FIXED) { 9573 err_msg = strcat(err_msg, 9574 "\tRecovered Data Integrity Error (I)\n"); 9575 } 9576 9577 if (port_serror & SERROR_COMM_ERR_FIXED) { 9578 err_msg = strcat(err_msg, 9579 "\tRecovered Communication Error (M)\n"); 9580 } 9581 9582 if (port_serror & SERROR_DATA_ERR) { 9583 err_msg = strcat(err_msg, 9584 "\tTransient Data Integrity Error (T)\n"); 9585 } 9586 9587 if (port_serror & SERROR_PERSISTENT_ERR) { 9588 err_msg = strcat(err_msg, 9589 "\tPersistent Communication or Data Integrity Error (C)\n"); 9590 } 9591 9592 if (port_serror & SERROR_PROTOCOL_ERR) { 9593 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 9594 } 9595 9596 if (port_serror & SERROR_INT_ERR) { 9597 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 9598 } 9599 9600 if (port_serror & SERROR_PHY_RDY_CHG) { 9601 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 9602 } 9603 9604 if (port_serror & SERROR_PHY_INT_ERR) { 9605 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 9606 } 9607 9608 if (port_serror & SERROR_COMM_WAKE) { 9609 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 9610 } 9611 9612 if (port_serror & SERROR_10B_TO_8B_ERR) { 9613 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 9614 } 9615 9616 if (port_serror & SERROR_DISPARITY_ERR) { 9617 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 9618 } 9619 9620 if (port_serror & SERROR_CRC_ERR) { 9621 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 9622 } 9623 9624 if (port_serror & SERROR_HANDSHAKE_ERR) { 9625 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 9626 } 9627 9628 if (port_serror & SERROR_LINK_SEQ_ERR) { 9629 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 9630 } 9631 9632 if (port_serror & SERROR_TRANS_ERR) { 9633 err_msg = strcat(err_msg, 9634 "\tTransport state transition error (T)\n"); 9635 } 9636 9637 if (port_serror & SERROR_FIS_TYPE) { 9638 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 9639 } 9640 9641 if (port_serror & SERROR_EXCHANGED_ERR) { 9642 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 9643 } 9644 9645 if (err_msg == NULL) 9646 return; 9647 9648 if (debug_only) { 9649 (void) sprintf(err_msg_header, "port %d", port); 9650 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 9651 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 9652 } else if (ahci_ctlp) { 9653 cmn_err(CE_WARN, "!ahci%d: %s %s", 9654 ddi_get_instance(ahci_ctlp->ahcictl_dip), 9655 err_msg_header, err_msg); 9656 9657 /* sata trace debug */ 9658 sata_trace_debug(ahci_ctlp->ahcictl_dip, 9659 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 9660 err_msg_header, err_msg); 9661 } else { 9662 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 9663 9664 /* sata trace debug */ 9665 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 9666 } 9667 } 9668 9669 /* 9670 * Translate the sata_address_t type into the ahci_addr_t type. 9671 * sata_device.satadev_addr structure is used as source. 9672 */ 9673 static void 9674 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd, 9675 ahci_addr_t *ahci_addrp) 9676 { 9677 sata_address_t *sata_addrp = &sd->satadev_addr; 9678 ahci_addrp->aa_port = 9679 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport]; 9680 ahci_addrp->aa_pmport = sata_addrp->pmport; 9681 9682 switch (sata_addrp->qual) { 9683 case SATA_ADDR_DCPORT: 9684 case SATA_ADDR_CPORT: 9685 ahci_addrp->aa_qual = AHCI_ADDR_PORT; 9686 break; 9687 case SATA_ADDR_PMULT: 9688 case SATA_ADDR_PMULT_SPEC: 9689 ahci_addrp->aa_qual = AHCI_ADDR_PMULT; 9690 break; 9691 case SATA_ADDR_DPMPORT: 9692 case SATA_ADDR_PMPORT: 9693 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT; 9694 break; 9695 case SATA_ADDR_NULL: 9696 default: 9697 /* something went wrong */ 9698 ahci_addrp->aa_qual = AHCI_ADDR_NULL; 9699 break; 9700 } 9701 } 9702 9703 /* 9704 * This routine is to calculate the total number of ports implemented 9705 * by the HBA. 9706 */ 9707 static int 9708 ahci_get_num_implemented_ports(uint32_t ports_implemented) 9709 { 9710 uint8_t i; 9711 int num = 0; 9712 9713 for (i = 0; i < AHCI_MAX_PORTS; i++) { 9714 if (((uint32_t)0x1 << i) & ports_implemented) 9715 num++; 9716 } 9717 9718 return (num); 9719 } 9720 9721 #if AHCI_DEBUG 9722 static void 9723 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 9724 { 9725 static char name[16]; 9726 va_list ap; 9727 9728 mutex_enter(&ahci_log_mutex); 9729 9730 va_start(ap, fmt); 9731 if (ahci_ctlp) { 9732 (void) sprintf(name, "ahci%d: ", 9733 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 9734 } else { 9735 (void) sprintf(name, "ahci: "); 9736 } 9737 9738 (void) vsprintf(ahci_log_buf, fmt, ap); 9739 va_end(ap); 9740 9741 cmn_err(level, "%s%s", name, ahci_log_buf); 9742 9743 mutex_exit(&ahci_log_mutex); 9744 } 9745 #endif 9746 9747 /* 9748 * quiesce(9E) entry point. 9749 * 9750 * This function is called when the system is single-threaded at high 9751 * PIL with preemption disabled. Therefore, this function must not be 9752 * blocked. 9753 * 9754 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 9755 * DDI_FAILURE indicates an error condition and should almost never happen. 9756 */ 9757 static int 9758 ahci_quiesce(dev_info_t *dip) 9759 { 9760 ahci_ctl_t *ahci_ctlp; 9761 ahci_port_t *ahci_portp; 9762 int instance, port; 9763 9764 instance = ddi_get_instance(dip); 9765 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 9766 9767 if (ahci_ctlp == NULL) 9768 return (DDI_FAILURE); 9769 9770 #if AHCI_DEBUG 9771 ahci_debug_flags = 0; 9772 #endif 9773 9774 /* disable all the interrupts. */ 9775 ahci_disable_all_intrs(ahci_ctlp); 9776 9777 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9778 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9779 continue; 9780 } 9781 9782 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9783 9784 /* 9785 * Stop the port by clearing PxCMD.ST 9786 * 9787 * Here we must disable the port interrupt because 9788 * ahci_disable_all_intrs only clear GHC.IE, and IS 9789 * register will be still set if PxIE is enabled. 9790 * When ahci shares one IRQ with other drivers, the 9791 * intr handler may claim the intr mistakenly. 9792 */ 9793 ahci_disable_port_intrs(ahci_ctlp, port); 9794 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 9795 ahci_portp, port); 9796 } 9797 9798 return (DDI_SUCCESS); 9799 } 9800