1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_drain_ports_taskq(ahci_ctl_t *); 104 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 105 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 107 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 108 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 109 uint8_t, sata_pkt_t *); 110 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 111 uint8_t, sata_pkt_t *); 112 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 113 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 114 static void ahci_copy_ncq_err_page(sata_cmd_t *, 115 struct sata_ncq_error_recovery_page *); 116 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 117 118 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 119 static int ahci_hba_reset(ahci_ctl_t *); 120 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 124 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 125 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 126 uint8_t); 127 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 128 uint8_t, int, int *); 129 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 130 uint32_t, uint32_t, uint32_t, uint32_t); 131 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 132 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 133 uint8_t, sata_pkt_t *); 134 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 135 uint8_t, uint32_t); 136 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 137 uint8_t, uint32_t); 138 static void ahci_events_handler(void *); 139 static void ahci_watchdog_handler(ahci_ctl_t *); 140 141 static uint_t ahci_intr(caddr_t, caddr_t); 142 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 143 static int ahci_add_legacy_intrs(ahci_ctl_t *); 144 static int ahci_add_msi_intrs(ahci_ctl_t *); 145 static void ahci_rem_intrs(ahci_ctl_t *); 146 static void ahci_enable_all_intrs(ahci_ctl_t *); 147 static void ahci_disable_all_intrs(ahci_ctl_t *); 148 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 149 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 150 151 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 154 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 155 ahci_port_t *, uint8_t); 156 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 157 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 158 uint8_t, uint32_t); 159 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 160 uint8_t, uint32_t); 161 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 162 163 static int ahci_get_num_implemented_ports(uint32_t); 164 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 165 uint32_t); 166 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 167 #if AHCI_DEBUG 168 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 169 #endif 170 171 172 /* 173 * DMA attributes for the data buffer 174 * 175 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 176 * does not support 64-bit addressing 177 */ 178 static ddi_dma_attr_t buffer_dma_attr = { 179 DMA_ATTR_V0, /* dma_attr_version */ 180 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 181 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 182 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 183 0x2ull, /* dma_attr_align: word aligned */ 184 1, /* dma_attr_burstsizes */ 185 1, /* dma_attr_minxfer */ 186 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 187 0xffffffffull, /* dma_attr_seg */ 188 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 189 512, /* dma_attr_granular */ 190 0, /* dma_attr_flags */ 191 }; 192 193 /* 194 * DMA attributes for the rcvd FIS 195 * 196 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 197 * does not support 64-bit addressing 198 */ 199 static ddi_dma_attr_t rcvd_fis_dma_attr = { 200 DMA_ATTR_V0, /* dma_attr_version */ 201 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 202 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 203 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 204 0x100ull, /* dma_attr_align: 256-byte aligned */ 205 1, /* dma_attr_burstsizes */ 206 1, /* dma_attr_minxfer */ 207 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 208 0xffffffffull, /* dma_attr_seg */ 209 1, /* dma_attr_sgllen */ 210 1, /* dma_attr_granular */ 211 0, /* dma_attr_flags */ 212 }; 213 214 /* 215 * DMA attributes for the command list 216 * 217 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 218 * does not support 64-bit addressing 219 */ 220 static ddi_dma_attr_t cmd_list_dma_attr = { 221 DMA_ATTR_V0, /* dma_attr_version */ 222 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 223 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 224 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 225 0x400ull, /* dma_attr_align: 1K-byte aligned */ 226 1, /* dma_attr_burstsizes */ 227 1, /* dma_attr_minxfer */ 228 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 229 0xffffffffull, /* dma_attr_seg */ 230 1, /* dma_attr_sgllen */ 231 1, /* dma_attr_granular */ 232 0, /* dma_attr_flags */ 233 }; 234 235 /* 236 * DMA attributes for cmd tables 237 * 238 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 239 * does not support 64-bit addressing 240 */ 241 static ddi_dma_attr_t cmd_table_dma_attr = { 242 DMA_ATTR_V0, /* dma_attr_version */ 243 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 244 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 245 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 246 0x80ull, /* dma_attr_align: 128-byte aligned */ 247 1, /* dma_attr_burstsizes */ 248 1, /* dma_attr_minxfer */ 249 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 250 0xffffffffull, /* dma_attr_seg */ 251 1, /* dma_attr_sgllen */ 252 1, /* dma_attr_granular */ 253 0, /* dma_attr_flags */ 254 }; 255 256 257 /* Device access attributes */ 258 static ddi_device_acc_attr_t accattr = { 259 DDI_DEVICE_ATTR_V0, 260 DDI_STRUCTURE_LE_ACC, 261 DDI_STRICTORDER_ACC 262 }; 263 264 265 static struct dev_ops ahcictl_dev_ops = { 266 DEVO_REV, /* devo_rev */ 267 0, /* refcnt */ 268 ahci_getinfo, /* info */ 269 nulldev, /* identify */ 270 nulldev, /* probe */ 271 ahci_attach, /* attach */ 272 ahci_detach, /* detach */ 273 nodev, /* no reset */ 274 (struct cb_ops *)0, /* driver operations */ 275 NULL, /* bus operations */ 276 NULL, /* power */ 277 ahci_quiesce, /* quiesce */ 278 }; 279 280 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 281 SATA_TRAN_HOTPLUG_OPS_REV_1, 282 ahci_tran_hotplug_port_activate, 283 ahci_tran_hotplug_port_deactivate 284 }; 285 286 extern struct mod_ops mod_driverops; 287 288 static struct modldrv modldrv = { 289 &mod_driverops, /* driverops */ 290 ahci_ident, /* short description */ 291 &ahcictl_dev_ops, /* driver ops */ 292 }; 293 294 static struct modlinkage modlinkage = { 295 MODREV_1, 296 &modldrv, 297 NULL 298 }; 299 300 /* The following variables are watchdog handler related */ 301 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 302 static int ahci_watchdog_tick; 303 304 /* 305 * This static variable indicates the size of command table, 306 * and it's changeable with prdt number, which ahci_dma_prdt_number 307 * indicates. 308 */ 309 static size_t ahci_cmd_table_size; 310 311 /* 312 * ahci_dma_prdt_number, ahci_msi_enabled and ahci_64bit_dma_addressed 313 * are global variables, and can be changed via /etc/system file. 314 */ 315 316 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 317 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 318 319 /* AHCI MSI is tunable */ 320 boolean_t ahci_msi_enabled = B_FALSE; 321 322 /* 64-bit dma addressing is tunable */ 323 boolean_t ahci_64bit_dma_addressed = B_TRUE; 324 325 #if AHCI_DEBUG 326 uint32_t ahci_debug_flags = 0; 327 328 /* The following is needed for ahci_log() */ 329 static kmutex_t ahci_log_mutex; 330 static char ahci_log_buf[512]; 331 #endif 332 333 /* Opaque state pointer initialized by ddi_soft_state_init() */ 334 static void *ahci_statep = NULL; 335 336 /* 337 * ahci module initialization. 338 */ 339 int 340 _init(void) 341 { 342 int ret; 343 344 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 345 if (ret != 0) { 346 goto err_out; 347 } 348 349 #if AHCI_DEBUG 350 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 351 #endif 352 353 if ((ret = sata_hba_init(&modlinkage)) != 0) { 354 #if AHCI_DEBUG 355 mutex_destroy(&ahci_log_mutex); 356 #endif 357 ddi_soft_state_fini(&ahci_statep); 358 goto err_out; 359 } 360 361 ret = mod_install(&modlinkage); 362 if (ret != 0) { 363 sata_hba_fini(&modlinkage); 364 #if AHCI_DEBUG 365 mutex_destroy(&ahci_log_mutex); 366 #endif 367 ddi_soft_state_fini(&ahci_statep); 368 goto err_out; 369 } 370 371 /* watchdog tick */ 372 ahci_watchdog_tick = drv_usectohz( 373 (clock_t)ahci_watchdog_timeout * 1000000); 374 return (ret); 375 376 err_out: 377 cmn_err(CE_WARN, "!ahci: Module init failed"); 378 return (ret); 379 } 380 381 /* 382 * ahci module uninitialize. 383 */ 384 int 385 _fini(void) 386 { 387 int ret; 388 389 ret = mod_remove(&modlinkage); 390 if (ret != 0) { 391 return (ret); 392 } 393 394 /* Remove the resources allocated in _init(). */ 395 sata_hba_fini(&modlinkage); 396 #if AHCI_DEBUG 397 mutex_destroy(&ahci_log_mutex); 398 #endif 399 ddi_soft_state_fini(&ahci_statep); 400 401 return (ret); 402 } 403 404 /* 405 * _info entry point 406 */ 407 int 408 _info(struct modinfo *modinfop) 409 { 410 return (mod_info(&modlinkage, modinfop)); 411 } 412 413 /* 414 * The attach entry point for dev_ops. 415 */ 416 static int 417 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 418 { 419 ahci_ctl_t *ahci_ctlp; 420 int instance = ddi_get_instance(dip); 421 int status; 422 int attach_state; 423 uint32_t cap_status, ahci_version; 424 int intr_types; 425 int i; 426 pci_regspec_t *regs; 427 int regs_length; 428 int rnumber; 429 #if AHCI_DEBUG 430 int speed; 431 #endif 432 433 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter"); 434 435 switch (cmd) { 436 case DDI_ATTACH: 437 break; 438 439 case DDI_RESUME: 440 441 /* 442 * During DDI_RESUME, the hardware state of the device 443 * (power may have been removed from the device) must be 444 * restored, allow pending requests to continue, and 445 * service new requests. 446 */ 447 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 448 mutex_enter(&ahci_ctlp->ahcictl_mutex); 449 450 /* Restart watch thread */ 451 if (ahci_ctlp->ahcictl_timeout_id == 0) 452 ahci_ctlp->ahcictl_timeout_id = timeout( 453 (void (*)(void *))ahci_watchdog_handler, 454 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 455 456 mutex_exit(&ahci_ctlp->ahcictl_mutex); 457 458 /* 459 * Re-initialize the controller and enable the interrupts and 460 * restart all the ports. 461 * 462 * Note that so far we don't support hot-plug during 463 * suspend/resume. 464 */ 465 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 466 AHCIDBG0(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 467 "Failed to initialize the controller " 468 "during DDI_RESUME"); 469 return (DDI_FAILURE); 470 } 471 472 mutex_enter(&ahci_ctlp->ahcictl_mutex); 473 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 474 mutex_exit(&ahci_ctlp->ahcictl_mutex); 475 476 return (DDI_SUCCESS); 477 478 default: 479 return (DDI_FAILURE); 480 } 481 482 attach_state = AHCI_ATTACH_STATE_NONE; 483 484 /* Allocate soft state */ 485 status = ddi_soft_state_zalloc(ahci_statep, instance); 486 if (status != DDI_SUCCESS) { 487 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 488 instance); 489 goto err_out; 490 } 491 492 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 493 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 494 ahci_ctlp->ahcictl_dip = dip; 495 496 /* Initialize the cport/port mapping */ 497 for (i = 0; i < AHCI_MAX_PORTS; i++) { 498 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 499 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 500 } 501 502 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 503 504 /* 505 * Now map the AHCI base address; which includes global 506 * registers and port control registers 507 * 508 * According to the spec, the AHCI Base Address is BAR5, 509 * but BAR0-BAR4 are optional, so we need to check which 510 * rnumber is used for BAR5. 511 */ 512 513 /* 514 * search through DDI "reg" property for the AHCI register set 515 */ 516 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 517 DDI_PROP_DONTPASS, "reg", (int **)®s, 518 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 519 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 520 instance); 521 goto err_out; 522 } 523 524 /* AHCI Base Address is located at 0x24 offset */ 525 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 526 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 527 == AHCI_PCI_RNUM) 528 break; 529 } 530 531 ddi_prop_free(regs); 532 533 if (rnumber == regs_length) { 534 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 535 instance); 536 goto err_out; 537 } 538 539 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 540 541 status = ddi_regs_map_setup(dip, 542 rnumber, 543 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 544 0, 545 0, 546 &accattr, 547 &ahci_ctlp->ahcictl_ahci_acc_handle); 548 if (status != DDI_SUCCESS) { 549 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 550 instance); 551 goto err_out; 552 } 553 554 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 555 556 /* Get the AHCI version information */ 557 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 558 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 559 560 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 561 (ahci_version & 0xffff0000) >> 16, 562 ((ahci_version & 0x0000ff00) >> 4 | 563 (ahci_version & 0x000000ff))); 564 565 /* We don't support controllers whose versions are lower than 1.0 */ 566 if (!(ahci_version & 0xffff0000)) { 567 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 568 "than version 1.0", instance); 569 goto err_out; 570 } 571 572 /* Get the HBA capabilities information */ 573 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 574 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 575 576 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 577 cap_status); 578 579 #if AHCI_DEBUG 580 /* Get the interface speed supported by the HBA */ 581 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 582 if (speed == 0x01) { 583 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 584 "hba interface speed support: Gen 1 (1.5Gbps)"); 585 } else if (speed == 0x10) { 586 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 587 "hba interface speed support: Gen 2 (3 Gbps)"); 588 } else if (speed == 0x11) { 589 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 590 "hba interface speed support: Gen 3 (6 Gbps)"); 591 } 592 #endif 593 594 /* Get the number of command slots supported by the HBA */ 595 ahci_ctlp->ahcictl_num_cmd_slots = 596 ((cap_status & AHCI_HBA_CAP_NCS) >> 597 AHCI_HBA_CAP_NCS_SHIFT) + 1; 598 599 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 600 ahci_ctlp->ahcictl_num_cmd_slots); 601 602 /* Get the bit map which indicates ports implemented by the HBA */ 603 ahci_ctlp->ahcictl_ports_implemented = 604 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 605 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 606 607 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 608 ahci_ctlp->ahcictl_ports_implemented); 609 610 /* 611 * According to the AHCI spec, CAP.NP should indicate the maximum 612 * number of ports supported by the HBA silicon, but we found 613 * this value of ICH8 chipset only indicates the number of ports 614 * implemented (exposed) by it. Therefore, the driver should calculate 615 * the potential maximum value by checking PI register, and use 616 * the maximum of this value and CAP.NP. 617 */ 618 ahci_ctlp->ahcictl_num_ports = max( 619 (cap_status & AHCI_HBA_CAP_NP) + 1, 620 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 621 622 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 623 ahci_ctlp->ahcictl_num_ports); 624 625 /* Get the number of implemented ports by the HBA */ 626 ahci_ctlp->ahcictl_num_implemented_ports = 627 ahci_get_num_implemented_ports( 628 ahci_ctlp->ahcictl_ports_implemented); 629 630 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 631 "hba number of implemented ports: %d", 632 ahci_ctlp->ahcictl_num_implemented_ports); 633 634 /* Check whether HBA supports 64bit DMA addressing */ 635 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 636 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 637 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 638 "hba does not support 64-bit addressing"); 639 } 640 641 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 642 != DDI_SUCCESS) { 643 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 644 instance); 645 goto err_out; 646 } 647 648 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 649 650 /* 651 * Check the pci configuration space, and set caps. We also 652 * handle the hardware defect in this function. 653 * 654 * For example, force ATI SB600/SB700/SB750 to use 32-bit dma 655 * addressing since it doesn't support 64-bit dma though their 656 * registers declare they support. 657 */ 658 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 659 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 660 instance); 661 goto err_out; 662 } 663 664 /* 665 * Disable the whole controller interrupts before adding 666 * interrupt handlers(s). 667 */ 668 ahci_disable_all_intrs(ahci_ctlp); 669 670 /* Get supported interrupt types */ 671 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 672 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 673 instance); 674 goto err_out; 675 } 676 677 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 678 "ddi_intr_get_supported_types() returned: 0x%x", 679 intr_types); 680 681 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 682 /* 683 * Try MSI first, but fall back to FIXED if failed 684 */ 685 if (ahci_add_msi_intrs(ahci_ctlp) == DDI_SUCCESS) { 686 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 687 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 688 "Using MSI interrupt type"); 689 goto intr_done; 690 } 691 692 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 693 "MSI registration failed, " 694 "trying FIXED interrupts"); 695 } 696 697 if (intr_types & DDI_INTR_TYPE_FIXED) { 698 if (ahci_add_legacy_intrs(ahci_ctlp) == DDI_SUCCESS) { 699 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 700 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 701 "Using FIXED interrupt type"); 702 goto intr_done; 703 } 704 705 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 706 "FIXED interrupt registration failed"); 707 } 708 709 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 710 711 goto err_out; 712 713 intr_done: 714 715 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 716 717 /* Initialize the controller mutex */ 718 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 719 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 720 721 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 722 723 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 724 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 725 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 726 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 727 } 728 729 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 730 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 731 sizeof (ahci_prdt_item_t)); 732 733 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 734 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 735 " ahci_cmd_table_size is 0x%x", 736 ahci_dma_prdt_number, ahci_cmd_table_size); 737 738 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 739 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 740 ahci_dma_prdt_number; 741 742 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 743 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 744 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 745 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 746 747 if ((ahci_64bit_dma_addressed == B_FALSE) || 748 (ahci_ctlp->ahcictl_cap & AHCI_CAP_32BIT_DMA)) { 749 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 750 0xffffffffull; 751 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 752 0xffffffffull; 753 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 754 0xffffffffull; 755 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 756 0xffffffffull; 757 } 758 759 /* Allocate the ports structure */ 760 status = ahci_alloc_ports_state(ahci_ctlp); 761 if (status != AHCI_SUCCESS) { 762 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 763 instance); 764 goto err_out; 765 } 766 767 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 768 769 /* 770 * Initialize the controller and ports. 771 */ 772 status = ahci_initialize_controller(ahci_ctlp); 773 if (status != AHCI_SUCCESS) { 774 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 775 instance); 776 goto err_out; 777 } 778 779 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 780 781 /* Start one thread to check packet timeouts */ 782 ahci_ctlp->ahcictl_timeout_id = timeout( 783 (void (*)(void *))ahci_watchdog_handler, 784 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 785 786 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 787 788 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 789 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 790 instance); 791 goto err_out; 792 } 793 794 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 795 796 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!"); 797 798 return (DDI_SUCCESS); 799 800 err_out: 801 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 802 mutex_enter(&ahci_ctlp->ahcictl_mutex); 803 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 804 ahci_ctlp->ahcictl_timeout_id = 0; 805 mutex_exit(&ahci_ctlp->ahcictl_mutex); 806 } 807 808 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 809 ahci_uninitialize_controller(ahci_ctlp); 810 } 811 812 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 813 ahci_dealloc_ports_state(ahci_ctlp); 814 } 815 816 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 817 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 818 } 819 820 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 821 ahci_rem_intrs(ahci_ctlp); 822 } 823 824 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 825 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 826 } 827 828 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 829 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 830 } 831 832 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 833 ddi_soft_state_free(ahci_statep, instance); 834 } 835 836 return (DDI_FAILURE); 837 } 838 839 /* 840 * The detach entry point for dev_ops. 841 */ 842 static int 843 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 844 { 845 ahci_ctl_t *ahci_ctlp; 846 int instance; 847 int ret; 848 849 instance = ddi_get_instance(dip); 850 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 851 852 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter"); 853 854 switch (cmd) { 855 case DDI_DETACH: 856 857 /* disable the interrupts for an uninterrupted detach */ 858 mutex_enter(&ahci_ctlp->ahcictl_mutex); 859 ahci_disable_all_intrs(ahci_ctlp); 860 mutex_exit(&ahci_ctlp->ahcictl_mutex); 861 862 /* unregister from the sata framework. */ 863 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 864 if (ret != AHCI_SUCCESS) { 865 mutex_enter(&ahci_ctlp->ahcictl_mutex); 866 ahci_enable_all_intrs(ahci_ctlp); 867 mutex_exit(&ahci_ctlp->ahcictl_mutex); 868 return (DDI_FAILURE); 869 } 870 871 mutex_enter(&ahci_ctlp->ahcictl_mutex); 872 873 /* stop the watchdog handler */ 874 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 875 ahci_ctlp->ahcictl_timeout_id = 0; 876 877 mutex_exit(&ahci_ctlp->ahcictl_mutex); 878 879 /* uninitialize the controller */ 880 ahci_uninitialize_controller(ahci_ctlp); 881 882 /* remove the interrupts */ 883 ahci_rem_intrs(ahci_ctlp); 884 885 /* deallocate the ports structures */ 886 ahci_dealloc_ports_state(ahci_ctlp); 887 888 /* destroy mutex */ 889 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 890 891 /* teardown the pci config */ 892 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 893 894 /* remove the reg maps. */ 895 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 896 897 /* free the soft state. */ 898 ddi_soft_state_free(ahci_statep, instance); 899 900 return (DDI_SUCCESS); 901 902 case DDI_SUSPEND: 903 904 /* 905 * The steps associated with suspension must include putting 906 * the underlying device into a quiescent state so that it 907 * will not generate interrupts or modify or access memory. 908 */ 909 mutex_enter(&ahci_ctlp->ahcictl_mutex); 910 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 911 mutex_exit(&ahci_ctlp->ahcictl_mutex); 912 return (DDI_SUCCESS); 913 } 914 915 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 916 917 /* stop the watchdog handler */ 918 if (ahci_ctlp->ahcictl_timeout_id) { 919 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 920 ahci_ctlp->ahcictl_timeout_id = 0; 921 } 922 923 mutex_exit(&ahci_ctlp->ahcictl_mutex); 924 925 /* 926 * drain the taskq 927 */ 928 ahci_drain_ports_taskq(ahci_ctlp); 929 930 /* 931 * Disable the interrupts and stop all the ports. 932 */ 933 ahci_uninitialize_controller(ahci_ctlp); 934 935 return (DDI_SUCCESS); 936 937 default: 938 return (DDI_FAILURE); 939 } 940 } 941 942 /* 943 * The info entry point for dev_ops. 944 * 945 */ 946 static int 947 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 948 void *arg, void **result) 949 { 950 #ifndef __lock_lint 951 _NOTE(ARGUNUSED(dip)) 952 #endif /* __lock_lint */ 953 954 ahci_ctl_t *ahci_ctlp; 955 int instance; 956 dev_t dev; 957 958 dev = (dev_t)arg; 959 instance = getminor(dev); 960 961 switch (infocmd) { 962 case DDI_INFO_DEVT2DEVINFO: 963 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 964 if (ahci_ctlp != NULL) { 965 *result = ahci_ctlp->ahcictl_dip; 966 return (DDI_SUCCESS); 967 } else { 968 *result = NULL; 969 return (DDI_FAILURE); 970 } 971 case DDI_INFO_DEVT2INSTANCE: 972 *(int *)result = instance; 973 break; 974 default: 975 break; 976 } 977 978 return (DDI_SUCCESS); 979 } 980 981 /* 982 * Registers the ahci with sata framework. 983 */ 984 static int 985 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 986 { 987 struct sata_hba_tran *sata_hba_tran; 988 989 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 990 "ahci_register_sata_hba_tran enter"); 991 992 mutex_enter(&ahci_ctlp->ahcictl_mutex); 993 994 /* Allocate memory for the sata_hba_tran */ 995 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 996 997 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 998 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 999 sata_hba_tran->sata_tran_hba_dma_attr = 1000 &ahci_ctlp->ahcictl_buffer_dma_attr; 1001 1002 /* Report the number of implemented ports */ 1003 sata_hba_tran->sata_tran_hba_num_cports = 1004 ahci_ctlp->ahcictl_num_implemented_ports; 1005 1006 /* Support ATAPI device */ 1007 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1008 1009 /* Get the data transfer capability for PIO command by the HBA */ 1010 if (cap_status & AHCI_HBA_CAP_PMD) { 1011 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1012 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1013 "DRQ block data transfer for PIO command protocol"); 1014 } 1015 1016 /* 1017 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1018 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1019 * QUEUED (EXT), and SERVICE commands). Queued operations are 1020 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1021 * QUEUED commands when the HBA and device support native command 1022 * queuing(NCQ). 1023 * 1024 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1025 * CAP register of the HBA indicates NCQ is supported. 1026 * 1027 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1028 * set because the previous register content of PxCI can be re-written 1029 * in the register write. 1030 */ 1031 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1032 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1033 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1034 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1035 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1036 "Command Queuing"); 1037 } 1038 1039 /* Report the number of command slots */ 1040 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1041 1042 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1043 sata_hba_tran->sata_tran_start = ahci_tran_start; 1044 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1045 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1046 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1047 #ifdef __lock_lint 1048 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1049 #endif 1050 /* 1051 * When SATA framework adds support for pwrmgt the 1052 * pwrmgt_ops needs to be updated 1053 */ 1054 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1055 sata_hba_tran->sata_tran_ioctl = NULL; 1056 1057 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1058 1059 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1060 1061 /* Attach it to SATA framework */ 1062 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1063 != DDI_SUCCESS) { 1064 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1065 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1066 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1067 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1068 return (AHCI_FAILURE); 1069 } 1070 1071 return (AHCI_SUCCESS); 1072 } 1073 1074 /* 1075 * Unregisters the ahci with sata framework. 1076 */ 1077 static int 1078 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1079 { 1080 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 1081 "ahci_unregister_sata_hba_tran enter"); 1082 1083 /* Detach from the SATA framework. */ 1084 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1085 DDI_SUCCESS) { 1086 return (AHCI_FAILURE); 1087 } 1088 1089 /* Deallocate sata_hba_tran. */ 1090 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1091 sizeof (sata_hba_tran_t)); 1092 1093 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1094 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1095 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1096 1097 return (AHCI_SUCCESS); 1098 } 1099 1100 /* 1101 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1102 * port status registers and an attached device type via sata_device 1103 * structure. 1104 * 1105 * We return the cached information from a previous hardware probe. The 1106 * actual hardware probing itself was done either from within 1107 * ahci_initialize_controller() during the driver attach or from a phy 1108 * ready change interrupt handler. 1109 */ 1110 static int 1111 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1112 { 1113 ahci_ctl_t *ahci_ctlp; 1114 ahci_port_t *ahci_portp; 1115 uint8_t cport = sd->satadev_addr.cport; 1116 #if AHCI_DEBUG 1117 uint8_t pmport = sd->satadev_addr.pmport; 1118 uint8_t qual = sd->satadev_addr.qual; 1119 #endif 1120 uint8_t device_type; 1121 uint32_t port_state; 1122 uint8_t port; 1123 1124 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1125 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1126 1127 AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp, 1128 "ahci_tran_probe_port enter: cport: %d, " 1129 "pmport: %d, qual: %d", cport, pmport, qual); 1130 1131 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1132 1133 mutex_enter(&ahci_portp->ahciport_mutex); 1134 1135 port_state = ahci_portp->ahciport_port_state; 1136 switch (port_state) { 1137 1138 case SATA_PSTATE_FAILED: 1139 sd->satadev_state = SATA_PSTATE_FAILED; 1140 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1141 "ahci_tran_probe_port: port %d PORT FAILED", port); 1142 goto out; 1143 1144 case SATA_PSTATE_SHUTDOWN: 1145 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1146 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1147 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1148 goto out; 1149 1150 case SATA_PSTATE_PWROFF: 1151 sd->satadev_state = SATA_PSTATE_PWROFF; 1152 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1153 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1154 goto out; 1155 1156 case SATA_PSTATE_PWRON: 1157 sd->satadev_state = SATA_PSTATE_PWRON; 1158 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1159 "ahci_tran_probe_port: port %d PORT PWRON", port); 1160 break; 1161 1162 default: 1163 sd->satadev_state = port_state; 1164 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 1165 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1166 port, port_state); 1167 break; 1168 } 1169 1170 device_type = ahci_portp->ahciport_device_type; 1171 1172 switch (device_type) { 1173 1174 case SATA_DTYPE_ATADISK: 1175 sd->satadev_type = SATA_DTYPE_ATADISK; 1176 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1177 "ahci_tran_probe_port: port %d DISK found", port); 1178 break; 1179 1180 case SATA_DTYPE_ATAPI: 1181 /* 1182 * HBA driver only knows it's an ATAPI device, and don't know 1183 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1184 * type need to be determined by checking IDENTIFY PACKET 1185 * DEVICE data 1186 */ 1187 sd->satadev_type = SATA_DTYPE_ATAPI; 1188 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1189 "ahci_tran_probe_port: port %d ATAPI found", port); 1190 break; 1191 1192 case SATA_DTYPE_PMULT: 1193 sd->satadev_type = SATA_DTYPE_PMULT; 1194 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1195 "ahci_tran_probe_port: port %d Port Multiplier found", 1196 port); 1197 break; 1198 1199 case SATA_DTYPE_UNKNOWN: 1200 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1201 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1202 "ahci_tran_probe_port: port %d Unknown device found", port); 1203 break; 1204 1205 default: 1206 /* we don't support any other device types */ 1207 sd->satadev_type = SATA_DTYPE_NONE; 1208 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1209 "ahci_tran_probe_port: port %d No device found", port); 1210 break; 1211 } 1212 1213 out: 1214 ahci_update_sata_registers(ahci_ctlp, port, sd); 1215 mutex_exit(&ahci_portp->ahciport_mutex); 1216 1217 return (SATA_SUCCESS); 1218 } 1219 1220 /* 1221 * There are four operation modes in sata framework: 1222 * SATA_OPMODE_INTERRUPTS 1223 * SATA_OPMODE_POLLING 1224 * SATA_OPMODE_ASYNCH 1225 * SATA_OPMODE_SYNCH 1226 * 1227 * Their combined meanings as following: 1228 * 1229 * SATA_OPMODE_SYNCH 1230 * The command has to be completed before sata_tran_start functions returns. 1231 * Either interrupts or polling could be used - it's up to the driver. 1232 * Mode used currently for internal, sata-module initiated operations. 1233 * 1234 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1235 * It is the same as the one above. 1236 * 1237 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1238 * The command has to be completed before sata_tran_start function returns. 1239 * No interrupt used, polling only. This should be the mode used for scsi 1240 * packets with FLAG_NOINTR. 1241 * 1242 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1243 * The command may be queued (callback function specified). Interrupts could 1244 * be used. It's normal operation mode. 1245 */ 1246 /* 1247 * Called by sata framework to transport a sata packet down stream. 1248 */ 1249 static int 1250 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1251 { 1252 ahci_ctl_t *ahci_ctlp; 1253 ahci_port_t *ahci_portp; 1254 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1255 uint8_t port; 1256 1257 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1258 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1259 1260 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, 1261 "ahci_tran_start enter: cport %d satapkt 0x%p", 1262 cport, (void *)spkt); 1263 1264 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1265 1266 mutex_enter(&ahci_portp->ahciport_mutex); 1267 1268 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1269 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1270 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1271 /* 1272 * In case the targer driver would send the packet before 1273 * sata framework can have the opportunity to process those 1274 * event reports. 1275 */ 1276 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1277 spkt->satapkt_device.satadev_state = 1278 ahci_portp->ahciport_port_state; 1279 ahci_update_sata_registers(ahci_ctlp, port, 1280 &spkt->satapkt_device); 1281 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1282 "ahci_tran_start returning PORT_ERROR while " 1283 "port in FAILED/SHUTDOWN/PWROFF state: " 1284 "port: %d", port); 1285 mutex_exit(&ahci_portp->ahciport_mutex); 1286 return (SATA_TRAN_PORT_ERROR); 1287 } 1288 1289 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1290 /* 1291 * ahci_intr_phyrdy_change() may have rendered it to 1292 * SATA_DTYPE_NONE. 1293 */ 1294 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1295 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1296 spkt->satapkt_device.satadev_state = 1297 ahci_portp->ahciport_port_state; 1298 ahci_update_sata_registers(ahci_ctlp, port, 1299 &spkt->satapkt_device); 1300 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1301 "ahci_tran_start returning PORT_ERROR while " 1302 "no device attached: port: %d", port); 1303 mutex_exit(&ahci_portp->ahciport_mutex); 1304 return (SATA_TRAN_PORT_ERROR); 1305 } 1306 1307 /* 1308 * SATA HBA driver should remember that a device was reset and it 1309 * is supposed to reject any packets which do not specify either 1310 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1311 * 1312 * This is to prevent a race condition when a device was arbitrarily 1313 * reset by the HBA driver (and lost it's setting) and a target 1314 * driver sending some commands to a device before the sata framework 1315 * has a chance to restore the device setting (such as cache enable/ 1316 * disable or other resettable stuff). 1317 */ 1318 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1319 ahci_portp->ahciport_reset_in_progress = 0; 1320 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1321 "ahci_tran_start clearing the " 1322 "reset_in_progress for port: %d", port); 1323 } 1324 1325 if (ahci_portp->ahciport_reset_in_progress && 1326 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1327 ! ddi_in_panic()) { 1328 spkt->satapkt_reason = SATA_PKT_BUSY; 1329 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1330 "ahci_tran_start returning BUSY while " 1331 "reset in progress: port: %d", port); 1332 mutex_exit(&ahci_portp->ahciport_mutex); 1333 return (SATA_TRAN_BUSY); 1334 } 1335 1336 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1337 spkt->satapkt_reason = SATA_PKT_BUSY; 1338 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1339 "ahci_tran_start returning BUSY while " 1340 "mopping in progress: port: %d", port); 1341 mutex_exit(&ahci_portp->ahciport_mutex); 1342 return (SATA_TRAN_BUSY); 1343 } 1344 1345 if (spkt->satapkt_op_mode & 1346 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1347 /* 1348 * If a SYNC command to be executed in interrupt context, 1349 * bounce it back to sata module. 1350 */ 1351 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1352 servicing_interrupt()) { 1353 spkt->satapkt_reason = SATA_PKT_BUSY; 1354 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1355 "ahci_tran_start returning BUSY while " 1356 "sending SYNC mode under interrupt context: " 1357 "port : %d", port); 1358 mutex_exit(&ahci_portp->ahciport_mutex); 1359 return (SATA_TRAN_BUSY); 1360 } 1361 1362 /* We need to do the sync start now */ 1363 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1364 spkt) == AHCI_FAILURE) { 1365 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1366 "return QUEUE_FULL: port %d", port); 1367 mutex_exit(&ahci_portp->ahciport_mutex); 1368 return (SATA_TRAN_QUEUE_FULL); 1369 } 1370 } else { 1371 /* Async start, using interrupt */ 1372 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1373 == AHCI_FAILURE) { 1374 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1375 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1376 "returning QUEUE_FULL: port %d", port); 1377 mutex_exit(&ahci_portp->ahciport_mutex); 1378 return (SATA_TRAN_QUEUE_FULL); 1379 } 1380 } 1381 1382 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1383 "sata tran accepted: port %d", port); 1384 1385 mutex_exit(&ahci_portp->ahciport_mutex); 1386 return (SATA_TRAN_ACCEPTED); 1387 } 1388 1389 /* 1390 * SATA_OPMODE_SYNCH flag is set 1391 * 1392 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1393 * without interrupt, otherwise we can still use the interrupt. 1394 * 1395 * WARNING!!! ahciport_mutex should be acquired before the function 1396 * is called. 1397 */ 1398 static int 1399 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1400 uint8_t port, sata_pkt_t *spkt) 1401 { 1402 int pkt_timeout_ticks; 1403 uint32_t timeout_tags; 1404 int rval; 1405 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1406 1407 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1408 "port %d spkt 0x%p", port, spkt); 1409 1410 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1411 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1412 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1413 port, spkt)) == AHCI_FAILURE) { 1414 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1415 return (rval); 1416 } 1417 1418 pkt_timeout_ticks = 1419 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1420 1421 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1422 mutex_exit(&ahci_portp->ahciport_mutex); 1423 1424 /* Simulate the interrupt */ 1425 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1426 1427 drv_usecwait(AHCI_1MS_USECS); 1428 1429 mutex_enter(&ahci_portp->ahciport_mutex); 1430 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1431 if (pkt_timeout_ticks < 0) { 1432 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1433 "port %d satapkt 0x%p timed out\n", 1434 instance, port, (void *)spkt); 1435 timeout_tags = (0x1 << rval); 1436 mutex_exit(&ahci_portp->ahciport_mutex); 1437 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1438 port, timeout_tags); 1439 mutex_enter(&ahci_portp->ahciport_mutex); 1440 } 1441 } 1442 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1443 return (AHCI_SUCCESS); 1444 1445 } else { 1446 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1447 port, spkt)) == AHCI_FAILURE) 1448 return (rval); 1449 1450 #if AHCI_DEBUG 1451 /* 1452 * Note that the driver always uses the slot 0 to deliver 1453 * REQUEST SENSE or READ LOG EXT command 1454 */ 1455 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1456 ASSERT(rval == 0); 1457 #endif 1458 1459 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1460 cv_wait(&ahci_portp->ahciport_cv, 1461 &ahci_portp->ahciport_mutex); 1462 1463 return (AHCI_SUCCESS); 1464 } 1465 } 1466 1467 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1468 if (satapkt) { \ 1469 satapkt->satapkt_reason = reason; \ 1470 /* \ 1471 * We set the satapkt_reason in both sync and \ 1472 * non-sync cases. \ 1473 */ \ 1474 } \ 1475 if (satapkt && \ 1476 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1477 satapkt->satapkt_comp) { \ 1478 mutex_exit(&ahci_portp->ahciport_mutex); \ 1479 (*satapkt->satapkt_comp)(satapkt); \ 1480 mutex_enter(&ahci_portp->ahciport_mutex); \ 1481 } else { \ 1482 if (satapkt && \ 1483 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1484 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1485 cv_broadcast(&ahci_portp->ahciport_cv); \ 1486 } 1487 1488 /* 1489 * Searches for and claims a free command slot. 1490 * 1491 * Returns: 1492 * 1493 * AHCI_FAILURE if failed 1494 * 1. if no empty slot left 1495 * 2. non-queued command requested while queued command(s) is outstanding 1496 * 3. queued command requested whild non-queued command(s) is outstanding 1497 * 4. HBA doesn't support multiple-use of command list while already a 1498 * non-queued command is oustanding 1499 * 1500 * claimed slot number if succeeded 1501 * 1502 * NOTE: it will always return slot 0 during error recovery process for 1503 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1504 * 1505 * WARNING!!! ahciport_mutex should be acquired before the function 1506 * is called. 1507 */ 1508 static int 1509 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1510 ahci_port_t *ahci_portp, int command_type) 1511 { 1512 uint32_t free_slots; 1513 int slot; 1514 1515 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1516 "ahciport_pending_tags = 0x%x " 1517 "ahciport_pending_ncq_tags = 0x%x", 1518 ahci_portp->ahciport_pending_tags, 1519 ahci_portp->ahciport_pending_ncq_tags); 1520 1521 /* 1522 * According to the AHCI spec, system software is responsible to 1523 * ensure that queued and non-queued commands are not mixed in 1524 * the command list. 1525 */ 1526 if (command_type == AHCI_NON_NCQ_CMD) { 1527 /* Non-NCQ command request */ 1528 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1529 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1530 "ahci_claim_free_slot: there is still pending " 1531 "queued command(s) in the command list, " 1532 "so no available slot for the non-queued " 1533 "command"); 1534 return (AHCI_FAILURE); 1535 } 1536 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1537 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1538 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 1539 "ahci_claim_free_slot: HBA cannot support multiple-" 1540 "use of the command list for non-queued commands"); 1541 return (AHCI_FAILURE); 1542 } 1543 free_slots = (~ahci_portp->ahciport_pending_tags) & 1544 AHCI_SLOT_MASK(ahci_ctlp); 1545 } else if (command_type == AHCI_NCQ_CMD) { 1546 /* NCQ command request */ 1547 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1548 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1549 "ahci_claim_free_slot: there is still pending " 1550 "non-queued command(s) in the command list, " 1551 "so no available slot for the queued command"); 1552 return (AHCI_FAILURE); 1553 } 1554 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1555 AHCI_NCQ_SLOT_MASK(ahci_portp); 1556 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1557 /* Error retrieval command request */ 1558 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 1559 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1560 "SENSE or READ LOG EXT command"); 1561 slot = 0; 1562 goto out; 1563 } 1564 1565 slot = ddi_ffs(free_slots) - 1; 1566 if (slot == -1) { 1567 AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp, 1568 "ahci_claim_free_slot: no empty slots"); 1569 return (AHCI_FAILURE); 1570 } 1571 1572 /* 1573 * According to the AHCI spec, to allow a simple mechanism for the 1574 * HBA to map command list slots to queue entries, software must 1575 * match the tag number it uses to the slot it is placing the command 1576 * in. For example, if a queued command is placed in slot 5, the tag 1577 * for that command must be 5. 1578 */ 1579 if (command_type == AHCI_NCQ_CMD) { 1580 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1581 } 1582 1583 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1584 1585 out: 1586 AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp, 1587 "ahci_claim_free_slot: found slot: 0x%x", slot); 1588 1589 return (slot); 1590 } 1591 1592 /* 1593 * Builds the Command Table for the sata packet and delivers it to controller. 1594 * 1595 * Returns: 1596 * slot number if we can obtain a slot successfully 1597 * otherwise, return AHCI_FAILURE 1598 * 1599 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1600 */ 1601 static int 1602 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1603 uint8_t port, sata_pkt_t *spkt) 1604 { 1605 int cmd_slot; 1606 sata_cmd_t *scmd; 1607 ahci_fis_h2d_register_t *h2d_register_fisp; 1608 ahci_cmd_table_t *cmd_table; 1609 ahci_cmd_header_t *cmd_header; 1610 int ncookies; 1611 int i; 1612 int command_type = AHCI_NON_NCQ_CMD; 1613 int ncq_qdepth; 1614 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1615 #if AHCI_DEBUG 1616 uint32_t *ptr; 1617 uint8_t *ptr2; 1618 #endif 1619 1620 spkt->satapkt_reason = SATA_PKT_BUSY; 1621 1622 scmd = &spkt->satapkt_cmd; 1623 1624 /* Check if the command is a NCQ command */ 1625 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1626 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1627 command_type = AHCI_NCQ_CMD; 1628 1629 /* 1630 * When NCQ is support, system software must determine the 1631 * maximum tag allowed by the device and the HBA, and it 1632 * must use a value not beyond of the lower bound of the two. 1633 * 1634 * Sata module is going to calculate the qdepth and send 1635 * down to HBA driver via sata_cmd. 1636 */ 1637 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1638 1639 /* 1640 * At the moment, the driver doesn't support the dynamic 1641 * setting of the maximum ncq depth, and the value can be 1642 * set either during the attach or after hot-plug insertion. 1643 */ 1644 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1645 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1646 AHCIDBG2(AHCIDBG_NCQ, ahci_ctlp, 1647 "ahci_deliver_satapkt: port %d the max tags for " 1648 "NCQ command is %d", port, ncq_qdepth); 1649 } else { 1650 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1651 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1652 " port %d the max tag for NCQ command is " 1653 "requested to change from %d to %d, at the" 1654 " moment the driver doesn't support the " 1655 "dynamic change so it's going to " 1656 "still use the previous tag value", 1657 instance, port, 1658 ahci_portp->ahciport_max_ncq_tags, 1659 ncq_qdepth); 1660 } 1661 } 1662 } 1663 1664 /* Check if the command is an error retrieval command */ 1665 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1666 command_type = AHCI_ERR_RETRI_CMD; 1667 1668 /* Check if there is an empty command slot */ 1669 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1670 if (cmd_slot == AHCI_FAILURE) { 1671 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free command slot"); 1672 return (AHCI_FAILURE); 1673 } 1674 1675 AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1676 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1677 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1678 cmd_slot, port, (void *)spkt); 1679 1680 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1681 bzero((void *)cmd_table, ahci_cmd_table_size); 1682 1683 /* For data transfer operations, it is the H2D Register FIS */ 1684 h2d_register_fisp = 1685 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1686 1687 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1688 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1689 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1690 SET_FIS_PMP(h2d_register_fisp, 1691 spkt->satapkt_device.satadev_addr.pmport); 1692 } 1693 1694 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1695 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1696 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1697 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1698 1699 switch (scmd->satacmd_addr_type) { 1700 1701 case 0: 1702 /* 1703 * satacmd_addr_type will be 0 for the commands below: 1704 * ATAPI command 1705 * SATAC_IDLE_IM 1706 * SATAC_STANDBY_IM 1707 * SATAC_DOWNLOAD_MICROCODE 1708 * SATAC_FLUSH_CACHE 1709 * SATAC_SET_FEATURES 1710 * SATAC_SMART 1711 * SATAC_ID_PACKET_DEVICE 1712 * SATAC_ID_DEVICE 1713 */ 1714 /* FALLTHRU */ 1715 1716 case ATA_ADDR_LBA: 1717 /* FALLTHRU */ 1718 1719 case ATA_ADDR_LBA28: 1720 /* LBA[7:0] */ 1721 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1722 1723 /* LBA[15:8] */ 1724 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1725 1726 /* LBA[23:16] */ 1727 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1728 1729 /* LBA [27:24] (also called dev_head) */ 1730 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1731 1732 break; 1733 1734 case ATA_ADDR_LBA48: 1735 /* LBA[7:0] */ 1736 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1737 1738 /* LBA[15:8] */ 1739 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1740 1741 /* LBA[23:16] */ 1742 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1743 1744 /* LBA [31:24] */ 1745 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1746 scmd->satacmd_lba_low_msb); 1747 1748 /* LBA [39:32] */ 1749 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1750 scmd->satacmd_lba_mid_msb); 1751 1752 /* LBA [47:40] */ 1753 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1754 scmd->satacmd_lba_high_msb); 1755 1756 /* Set dev_head */ 1757 SET_FIS_DEV_HEAD(h2d_register_fisp, 1758 scmd->satacmd_device_reg); 1759 1760 /* Set the extended sector count and features */ 1761 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1762 scmd->satacmd_sec_count_msb); 1763 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1764 scmd->satacmd_features_reg_ext); 1765 break; 1766 } 1767 1768 /* 1769 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1770 * filled into features field, and sector count 8:15 is filled into 1771 * features (exp) field. TAG is filled into sector field. 1772 */ 1773 if (command_type == AHCI_NCQ_CMD) { 1774 SET_FIS_FEATURES(h2d_register_fisp, 1775 scmd->satacmd_sec_count_lsb); 1776 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1777 scmd->satacmd_sec_count_msb); 1778 1779 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1780 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1781 } 1782 1783 ncookies = scmd->satacmd_num_dma_cookies; 1784 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1785 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1786 ncookies, ahci_dma_prdt_number); 1787 1788 ASSERT(ncookies <= ahci_dma_prdt_number); 1789 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1790 1791 /* *** now fill the scatter gather list ******* */ 1792 for (i = 0; i < ncookies; i++) { 1793 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1794 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1795 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1796 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1797 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1798 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1799 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1800 scmd->satacmd_dma_cookie_list[i].dmac_size; 1801 } 1802 1803 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1804 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1805 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1806 1807 /* The ACMD field is filled in for ATAPI command */ 1808 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1809 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1810 SATA_ATAPI_MAX_CDB_LEN); 1811 } 1812 1813 /* Set Command Header in Command List */ 1814 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1815 BZERO_DESCR_INFO(cmd_header); 1816 BZERO_PRD_BYTE_COUNT(cmd_header); 1817 1818 /* Set the number of entries in the PRD table */ 1819 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1820 1821 /* Set the length of the command in the CFIS area */ 1822 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1823 1824 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1825 "sata_data_direction = 0x%x", 1826 scmd->satacmd_flags.sata_data_direction); 1827 1828 /* Set A bit if it is an ATAPI command */ 1829 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1830 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1831 1832 /* Set W bit if data is going to the device */ 1833 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1834 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1835 1836 /* 1837 * Set the prefetchable bit - this bit is only valid if the PRDTL 1838 * field is non-zero or the ATAPI 'A' bit is set in the command 1839 * header. This bit cannot be set when using native command 1840 * queuing commands or when using FIS-based switching with a Port 1841 * multiplier. 1842 */ 1843 if (command_type != AHCI_NCQ_CMD) 1844 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1845 1846 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1847 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1848 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1849 1850 /* 1851 * We are overloading satapkt_hba_driver_private with 1852 * watched_cycle count. 1853 */ 1854 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1855 1856 #if AHCI_DEBUG 1857 if (ahci_debug_flags & AHCIDBG_ATACMD && 1858 scmd->satacmd_cmd_reg != SATAC_PACKET || 1859 ahci_debug_flags & AHCIDBG_ATAPICMD && 1860 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1861 1862 /* Dump the command header and table */ 1863 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1864 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1865 "0x%p cmd_reg 0x%x port %d", spkt, 1866 scmd->satacmd_cmd_reg, port); 1867 ptr = (uint32_t *)cmd_header; 1868 ahci_log(ahci_ctlp, CE_WARN, 1869 " Command Header:%8x %8x %8x %8x", 1870 ptr[0], ptr[1], ptr[2], ptr[3]); 1871 1872 /* Dump the H2D register FIS */ 1873 ptr = (uint32_t *)h2d_register_fisp; 1874 ahci_log(ahci_ctlp, CE_WARN, 1875 " Command FIS: %8x %8x %8x %8x", 1876 ptr[0], ptr[1], ptr[2], ptr[3]); 1877 1878 /* Dump the ACMD register FIS */ 1879 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1880 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1881 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1882 ahci_log(ahci_ctlp, CE_WARN, 1883 " ATAPI command: %2x %2x %2x %2x " 1884 "%2x %2x %2x %2x", 1885 ptr2[8 * i], ptr2[8 * i + 1], 1886 ptr2[8 * i + 2], ptr2[8 * i + 3], 1887 ptr2[8 * i + 4], ptr2[8 * i + 5], 1888 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1889 1890 /* Dump the PRDT */ 1891 for (i = 0; i < ncookies; i++) { 1892 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1893 ahci_log(ahci_ctlp, CE_WARN, 1894 " Cookie %d: %8x %8x %8x %8x", 1895 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1896 } 1897 } 1898 #endif 1899 1900 (void) ddi_dma_sync( 1901 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1902 0, 1903 ahci_cmd_table_size, 1904 DDI_DMA_SYNC_FORDEV); 1905 1906 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1907 cmd_slot * sizeof (ahci_cmd_header_t), 1908 sizeof (ahci_cmd_header_t), 1909 DDI_DMA_SYNC_FORDEV); 1910 1911 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1912 if (command_type == AHCI_NCQ_CMD) { 1913 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1914 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1915 (0x1 << cmd_slot)); 1916 } 1917 1918 /* Indicate to the HBA that a command is active. */ 1919 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1920 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1921 (0x1 << cmd_slot)); 1922 1923 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1924 "exit: port %d", port); 1925 1926 return (cmd_slot); 1927 } 1928 1929 /* 1930 * Called by the sata framework to abort the previously sent packet(s). 1931 * 1932 * Reset device to abort commands. 1933 */ 1934 static int 1935 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1936 { 1937 ahci_ctl_t *ahci_ctlp; 1938 ahci_port_t *ahci_portp; 1939 uint32_t slot_status = 0; 1940 uint32_t aborted_tags = 0; 1941 uint32_t finished_tags = 0; 1942 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1943 uint8_t port; 1944 int tmp_slot; 1945 int instance = ddi_get_instance(dip); 1946 1947 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1948 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1949 1950 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 1951 "ahci_tran_abort enter: port %d", port); 1952 1953 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1954 mutex_enter(&ahci_portp->ahciport_mutex); 1955 1956 /* 1957 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 1958 * commands are being mopped, therefore there is nothing else to do 1959 */ 1960 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1961 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1962 "ahci_tran_abort: port %d is in " 1963 "mopping process, so just return directly ", port); 1964 mutex_exit(&ahci_portp->ahciport_mutex); 1965 return (SATA_SUCCESS); 1966 } 1967 1968 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1969 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1970 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1971 /* 1972 * In case the targer driver would send the request before 1973 * sata framework can have the opportunity to process those 1974 * event reports. 1975 */ 1976 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1977 spkt->satapkt_device.satadev_state = 1978 ahci_portp->ahciport_port_state; 1979 ahci_update_sata_registers(ahci_ctlp, port, 1980 &spkt->satapkt_device); 1981 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1982 "ahci_tran_abort returning SATA_FAILURE while " 1983 "port in FAILED/SHUTDOWN/PWROFF state: " 1984 "port: %d", port); 1985 mutex_exit(&ahci_portp->ahciport_mutex); 1986 return (SATA_FAILURE); 1987 } 1988 1989 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1990 /* 1991 * ahci_intr_phyrdy_change() may have rendered it to 1992 * AHCI_PORT_TYPE_NODEV. 1993 */ 1994 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1995 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1996 spkt->satapkt_device.satadev_state = 1997 ahci_portp->ahciport_port_state; 1998 ahci_update_sata_registers(ahci_ctlp, port, 1999 &spkt->satapkt_device); 2000 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2001 "ahci_tran_abort returning SATA_FAILURE while " 2002 "no device attached: port: %d", port); 2003 mutex_exit(&ahci_portp->ahciport_mutex); 2004 return (SATA_FAILURE); 2005 } 2006 2007 if (flag == SATA_ABORT_ALL_PACKETS) { 2008 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2009 aborted_tags = ahci_portp->ahciport_pending_tags; 2010 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2011 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2012 2013 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2014 instance, port); 2015 } else { 2016 aborted_tags = 0xffffffff; 2017 /* 2018 * Aborting one specific packet, first search the 2019 * ahciport_slot_pkts[] list for matching spkt. 2020 */ 2021 for (tmp_slot = 0; 2022 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2023 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2024 aborted_tags = (0x1 << tmp_slot); 2025 break; 2026 } 2027 } 2028 2029 if (aborted_tags == 0xffffffff) { 2030 /* request packet is not on the pending list */ 2031 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 2032 "Cannot find the aborting pkt 0x%p on the " 2033 "pending list", (void *)spkt); 2034 ahci_update_sata_registers(ahci_ctlp, port, 2035 &spkt->satapkt_device); 2036 mutex_exit(&ahci_portp->ahciport_mutex); 2037 return (SATA_FAILURE); 2038 } 2039 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2040 instance, port, (void *)spkt); 2041 } 2042 2043 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2044 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2045 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2046 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2047 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2048 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2049 2050 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2051 ahci_portp->ahciport_mop_in_progress++; 2052 2053 /* 2054 * To abort the packet(s), first we are trying to clear PxCMD.ST 2055 * to stop the port, and if the port can be stopped 2056 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2057 * then we just send back the aborted packet(s) with ABORTED flag 2058 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2059 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2060 * perform a COMRESET. 2061 */ 2062 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2063 ahci_portp, port, NULL, NULL); 2064 2065 /* 2066 * Compute which have finished and which need to be retried. 2067 * 2068 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2069 * minus the slot_status. The aborted_tags has to be deducted by 2070 * finished_tags since we can't possibly abort a tag which had finished 2071 * already. 2072 */ 2073 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2074 finished_tags = ahci_portp->ahciport_pending_tags & 2075 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2076 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2077 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2078 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2079 2080 aborted_tags &= ~finished_tags; 2081 2082 ahci_mop_commands(ahci_ctlp, 2083 ahci_portp, 2084 slot_status, 2085 0, /* failed tags */ 2086 0, /* timeout tags */ 2087 aborted_tags, 2088 0); /* reset tags */ 2089 2090 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2091 mutex_exit(&ahci_portp->ahciport_mutex); 2092 2093 return (SATA_SUCCESS); 2094 } 2095 2096 /* 2097 * Used to do device reset and reject all the pending packets on a device 2098 * during the reset operation. 2099 * 2100 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2101 */ 2102 static int 2103 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2104 ahci_port_t *ahci_portp, uint8_t port) 2105 { 2106 uint32_t slot_status = 0; 2107 uint32_t reset_tags = 0; 2108 uint32_t finished_tags = 0; 2109 sata_device_t sdevice; 2110 int ret; 2111 2112 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2113 "ahci_reset_device_reject_pkts on port: %d", port); 2114 2115 /* 2116 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2117 * commands are being mopped, therefore there is nothing else to do 2118 */ 2119 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2120 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2121 "ahci_reset_device_reject_pkts: port %d is in " 2122 "mopping process, so return directly ", port); 2123 return (SATA_SUCCESS); 2124 } 2125 2126 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2127 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2128 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2129 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2130 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2131 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2132 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2133 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2134 } 2135 2136 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2137 != AHCI_SUCCESS) { 2138 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2139 "Try to do a port reset after software " 2140 "reset failed", port); 2141 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2142 if (ret != AHCI_SUCCESS) { 2143 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2144 "ahci_reset_device_reject_pkts: port %d " 2145 "failed", port); 2146 return (SATA_FAILURE); 2147 } 2148 } 2149 /* Set the reset in progress flag */ 2150 ahci_portp->ahciport_reset_in_progress = 1; 2151 2152 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2153 ahci_portp->ahciport_mop_in_progress++; 2154 2155 /* Indicate to the framework that a reset has happened */ 2156 bzero((void *)&sdevice, sizeof (sata_device_t)); 2157 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2158 sdevice.satadev_addr.pmport = 0; 2159 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2160 2161 sdevice.satadev_state = SATA_DSTATE_RESET | 2162 SATA_DSTATE_PWR_ACTIVE; 2163 mutex_exit(&ahci_portp->ahciport_mutex); 2164 sata_hba_event_notify( 2165 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2166 &sdevice, 2167 SATA_EVNT_DEVICE_RESET); 2168 mutex_enter(&ahci_portp->ahciport_mutex); 2169 2170 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 2171 "port %d sending event up: SATA_EVNT_RESET", port); 2172 2173 /* Next try to mop the pending commands */ 2174 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2175 finished_tags = ahci_portp->ahciport_pending_tags & 2176 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2177 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2178 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2179 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2180 2181 reset_tags &= ~finished_tags; 2182 2183 ahci_mop_commands(ahci_ctlp, 2184 ahci_portp, 2185 slot_status, 2186 0, /* failed tags */ 2187 0, /* timeout tags */ 2188 0, /* aborted tags */ 2189 reset_tags); /* reset tags */ 2190 2191 return (SATA_SUCCESS); 2192 } 2193 2194 /* 2195 * Used to do port reset and reject all the pending packets on a port during 2196 * the reset operation. 2197 * 2198 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2199 */ 2200 static int 2201 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2202 ahci_port_t *ahci_portp, uint8_t port) 2203 { 2204 uint32_t slot_status = 0; 2205 uint32_t reset_tags = 0; 2206 uint32_t finished_tags = 0; 2207 2208 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2209 "ahci_reset_port_reject_pkts on port: %d", port); 2210 2211 /* 2212 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2213 * commands are being mopped, therefore there is nothing else to do 2214 */ 2215 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2216 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2217 "ahci_reset_port_reject_pkts: port %d is in " 2218 "mopping process, so return directly ", port); 2219 return (SATA_SUCCESS); 2220 } 2221 2222 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2223 ahci_portp->ahciport_mop_in_progress++; 2224 2225 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2226 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2227 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2228 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2229 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2230 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2231 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2232 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2233 } 2234 2235 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2236 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2237 NULL) != AHCI_SUCCESS) 2238 return (SATA_FAILURE); 2239 2240 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2241 finished_tags = ahci_portp->ahciport_pending_tags & 2242 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2243 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2244 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2245 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2246 2247 reset_tags &= ~finished_tags; 2248 2249 ahci_mop_commands(ahci_ctlp, 2250 ahci_portp, 2251 slot_status, 2252 0, /* failed tags */ 2253 0, /* timeout tags */ 2254 0, /* aborted tags */ 2255 reset_tags); /* reset tags */ 2256 2257 return (SATA_SUCCESS); 2258 } 2259 2260 /* 2261 * Used to do hba reset and reject all the pending packets on all ports 2262 * during the reset operation. 2263 */ 2264 static int 2265 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2266 { 2267 ahci_port_t *ahci_portp; 2268 uint32_t slot_status[AHCI_MAX_PORTS]; 2269 uint32_t reset_tags[AHCI_MAX_PORTS]; 2270 uint32_t finished_tags[AHCI_MAX_PORTS]; 2271 uint8_t port; 2272 int ret = SATA_SUCCESS; 2273 2274 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 2275 "ahci_reset_hba_reject_pkts enter"); 2276 2277 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2278 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2279 continue; 2280 } 2281 2282 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2283 2284 mutex_enter(&ahci_portp->ahciport_mutex); 2285 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2286 slot_status[port] = ddi_get32( 2287 ahci_ctlp->ahcictl_ahci_acc_handle, 2288 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2289 reset_tags[port] = slot_status[port] & 2290 AHCI_SLOT_MASK(ahci_ctlp); 2291 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2292 slot_status[port] = ddi_get32( 2293 ahci_ctlp->ahcictl_ahci_acc_handle, 2294 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2295 reset_tags[port] = slot_status[port] & 2296 AHCI_NCQ_SLOT_MASK(ahci_portp); 2297 } 2298 mutex_exit(&ahci_portp->ahciport_mutex); 2299 } 2300 2301 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2302 ret = SATA_FAILURE; 2303 goto out; 2304 } 2305 2306 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2307 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2308 continue; 2309 } 2310 2311 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2312 2313 mutex_enter(&ahci_portp->ahciport_mutex); 2314 /* 2315 * To prevent recursive enter to ahci_mop_commands, we need 2316 * check AHCI_PORT_FLAG_MOPPING flag. 2317 */ 2318 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2319 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2320 "ahci_reset_hba_reject_pkts: port %d is in " 2321 "mopping process, so return directly ", port); 2322 mutex_exit(&ahci_portp->ahciport_mutex); 2323 continue; 2324 } 2325 2326 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2327 ahci_portp->ahciport_mop_in_progress++; 2328 2329 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2330 finished_tags[port] = 2331 ahci_portp->ahciport_pending_tags & 2332 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2333 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2334 finished_tags[port] = 2335 ahci_portp->ahciport_pending_ncq_tags & 2336 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2337 2338 reset_tags[port] &= ~finished_tags[port]; 2339 2340 ahci_mop_commands(ahci_ctlp, 2341 ahci_portp, 2342 slot_status[port], 2343 0, /* failed tags */ 2344 0, /* timeout tags */ 2345 0, /* aborted tags */ 2346 reset_tags[port]); /* reset tags */ 2347 mutex_exit(&ahci_portp->ahciport_mutex); 2348 } 2349 out: 2350 return (ret); 2351 } 2352 2353 /* 2354 * Called by sata framework to reset a port(s) or device. 2355 */ 2356 static int 2357 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2358 { 2359 ahci_ctl_t *ahci_ctlp; 2360 ahci_port_t *ahci_portp; 2361 uint8_t cport = sd->satadev_addr.cport; 2362 uint8_t port; 2363 int ret = SATA_SUCCESS; 2364 int instance = ddi_get_instance(dip); 2365 2366 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2367 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2368 2369 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2370 "ahci_tran_reset_port enter: cport: %d", cport); 2371 2372 switch (sd->satadev_addr.qual) { 2373 case SATA_ADDR_CPORT: 2374 /* Port reset */ 2375 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2376 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2377 "reset port", instance, port); 2378 2379 mutex_enter(&ahci_portp->ahciport_mutex); 2380 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2381 mutex_exit(&ahci_portp->ahciport_mutex); 2382 2383 break; 2384 2385 case SATA_ADDR_DCPORT: 2386 /* Device reset */ 2387 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2388 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2389 "reset device", instance, port); 2390 2391 mutex_enter(&ahci_portp->ahciport_mutex); 2392 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2393 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2394 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2395 /* 2396 * In case the targer driver would send the request 2397 * before sata framework can have the opportunity to 2398 * process those event reports. 2399 */ 2400 sd->satadev_state = ahci_portp->ahciport_port_state; 2401 ahci_update_sata_registers(ahci_ctlp, port, sd); 2402 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2403 "ahci_tran_reset_dport returning SATA_FAILURE " 2404 "while port in FAILED/SHUTDOWN/PWROFF state: " 2405 "port: %d", port); 2406 mutex_exit(&ahci_portp->ahciport_mutex); 2407 ret = SATA_FAILURE; 2408 break; 2409 } 2410 2411 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2412 /* 2413 * ahci_intr_phyrdy_change() may have rendered it to 2414 * AHCI_PORT_TYPE_NODEV. 2415 */ 2416 sd->satadev_type = SATA_DTYPE_NONE; 2417 sd->satadev_state = ahci_portp->ahciport_port_state; 2418 ahci_update_sata_registers(ahci_ctlp, port, sd); 2419 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2420 "ahci_tran_reset_dport returning SATA_FAILURE " 2421 "while no device attached: port: %d", port); 2422 mutex_exit(&ahci_portp->ahciport_mutex); 2423 ret = SATA_FAILURE; 2424 break; 2425 } 2426 2427 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2428 ahci_portp, port); 2429 mutex_exit(&ahci_portp->ahciport_mutex); 2430 break; 2431 2432 case SATA_ADDR_CNTRL: 2433 /* Reset the whole controller */ 2434 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2435 "reset the whole hba", instance, port); 2436 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2437 break; 2438 2439 case SATA_ADDR_PMPORT: 2440 case SATA_ADDR_DPMPORT: 2441 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 2442 "ahci_tran_reset_dport: port multiplier will be " 2443 "supported later"); 2444 /* FALLTHRU */ 2445 default: 2446 ret = SATA_FAILURE; 2447 } 2448 2449 return (ret); 2450 } 2451 2452 /* 2453 * Called by sata framework to activate a port as part of hotplug. 2454 * (cfgadm -c connect satax/y) 2455 * Note: Not port-mult aware. 2456 */ 2457 static int 2458 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2459 { 2460 ahci_ctl_t *ahci_ctlp; 2461 ahci_port_t *ahci_portp; 2462 uint8_t cport = satadev->satadev_addr.cport; 2463 uint8_t port; 2464 int instance = ddi_get_instance(dip); 2465 2466 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2467 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2468 2469 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2470 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2471 2472 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2473 2474 mutex_enter(&ahci_portp->ahciport_mutex); 2475 ahci_enable_port_intrs(ahci_ctlp, port); 2476 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2477 2478 /* 2479 * Reset the port so that the PHY communication would be re-established. 2480 * But this reset is an internal operation and the sata module doesn't 2481 * need to know about it. Moreover, the port with a device attached will 2482 * be started too. 2483 */ 2484 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2485 ahci_portp, port, 2486 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2487 NULL); 2488 2489 /* 2490 * Need to check the link status and device status of the port 2491 * and consider raising power if the port was in D3 state 2492 */ 2493 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2494 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2495 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2496 2497 satadev->satadev_state = ahci_portp->ahciport_port_state; 2498 2499 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2500 2501 mutex_exit(&ahci_portp->ahciport_mutex); 2502 return (SATA_SUCCESS); 2503 } 2504 2505 /* 2506 * Called by sata framework to deactivate a port as part of hotplug. 2507 * (cfgadm -c disconnect satax/y) 2508 * Note: Not port-mult aware. 2509 */ 2510 static int 2511 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2512 { 2513 ahci_ctl_t *ahci_ctlp; 2514 ahci_port_t *ahci_portp; 2515 uint8_t cport = satadev->satadev_addr.cport; 2516 uint8_t port; 2517 uint32_t port_scontrol; 2518 int instance = ddi_get_instance(dip); 2519 2520 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2521 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2522 2523 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2524 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2525 2526 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2527 2528 mutex_enter(&ahci_portp->ahciport_mutex); 2529 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2530 instance, port); 2531 2532 /* Disable the interrupts on the port */ 2533 ahci_disable_port_intrs(ahci_ctlp, port); 2534 2535 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2536 goto phy_offline; 2537 } 2538 2539 /* First to abort all the pending commands */ 2540 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2541 2542 /* Then stop the port */ 2543 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2544 ahci_portp, port); 2545 2546 /* Next put the PHY offline */ 2547 2548 phy_offline: 2549 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2550 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2551 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2552 2553 /* Update ahciport_port_state */ 2554 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2555 satadev->satadev_state = ahci_portp->ahciport_port_state; 2556 2557 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2558 2559 mutex_exit(&ahci_portp->ahciport_mutex); 2560 return (SATA_SUCCESS); 2561 } 2562 2563 /* 2564 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2565 * when a device is unplugged or a port is deactivated. 2566 * 2567 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2568 */ 2569 static void 2570 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2571 ahci_port_t *ahci_portp, uint8_t port) 2572 { 2573 uint32_t slot_status = 0; 2574 uint32_t abort_tags = 0; 2575 2576 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2577 "ahci_reject_all_abort_pkts on port: %d", port); 2578 2579 /* 2580 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2581 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2582 * to get the error data, if yes when the device is removed, the 2583 * command needs to be aborted too. 2584 */ 2585 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2586 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2587 slot_status = 0x1; 2588 abort_tags = 0x1; 2589 goto out; 2590 } else { 2591 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2592 "ahci_reject_all_abort_pkts return directly " 2593 "port %d no needs to reject any outstanding " 2594 "commands", port); 2595 return; 2596 } 2597 } 2598 2599 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2600 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2601 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2602 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2603 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2604 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2605 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2606 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2607 } 2608 2609 out: 2610 /* No need to do mop when there is no outstanding commands */ 2611 if (slot_status != 0) { 2612 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2613 ahci_portp->ahciport_mop_in_progress++; 2614 2615 ahci_mop_commands(ahci_ctlp, 2616 ahci_portp, 2617 slot_status, 2618 0, /* failed tags */ 2619 0, /* timeout tags */ 2620 abort_tags, /* aborting tags */ 2621 0); /* reset tags */ 2622 } 2623 } 2624 2625 #if defined(__lock_lint) 2626 static int 2627 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2628 { 2629 return (SATA_SUCCESS); 2630 } 2631 #endif 2632 2633 /* 2634 * Allocate the ports structure, only called by ahci_attach 2635 */ 2636 static int 2637 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2638 { 2639 int port, cport = 0; 2640 2641 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2642 "ahci_alloc_ports_state enter"); 2643 2644 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2645 2646 /* Allocate structures only for the implemented ports */ 2647 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2648 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2649 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2650 "hba port %d not implemented", port); 2651 continue; 2652 } 2653 2654 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2655 ahci_ctlp->ahcictl_port_to_cport[port] = 2656 (uint8_t)cport++; 2657 2658 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2659 goto err_out; 2660 } 2661 } 2662 2663 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2664 return (AHCI_SUCCESS); 2665 2666 err_out: 2667 for (port--; port >= 0; port--) { 2668 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2669 ahci_dealloc_port_state(ahci_ctlp, port); 2670 } 2671 } 2672 2673 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2674 return (AHCI_FAILURE); 2675 } 2676 2677 /* 2678 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2679 */ 2680 static void 2681 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2682 { 2683 int port; 2684 2685 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2686 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2687 /* if this port is implemented by the HBA */ 2688 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2689 ahci_dealloc_port_state(ahci_ctlp, port); 2690 } 2691 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2692 } 2693 2694 /* 2695 * Drain the taskq. 2696 */ 2697 static void 2698 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 2699 { 2700 ahci_port_t *ahci_portp; 2701 int port; 2702 2703 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2704 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2705 continue; 2706 } 2707 2708 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2709 2710 mutex_enter(&ahci_portp->ahciport_mutex); 2711 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 2712 mutex_exit(&ahci_portp->ahciport_mutex); 2713 } 2714 } 2715 2716 /* 2717 * Initialize the controller and all ports. And then try to start the ports 2718 * if there are devices attached. 2719 * 2720 * This routine can be called from three seperate cases: DDI_ATTACH, 2721 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2722 * other two cases; device signature probing are attempted only during 2723 * DDI_ATTACH case. 2724 * 2725 * WARNING!!! Disable the whole controller's interrupts before calling and 2726 * the interrupts will be enabled upon successfully return. 2727 */ 2728 static int 2729 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2730 { 2731 ahci_port_t *ahci_portp; 2732 uint32_t ghc_control; 2733 int port; 2734 2735 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2736 "ahci_initialize_controller enter"); 2737 2738 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2739 2740 /* 2741 * Indicate that system software is AHCI aware by setting 2742 * GHC.AE to 1 2743 */ 2744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2746 2747 ghc_control |= AHCI_HBA_GHC_AE; 2748 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2749 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2750 ghc_control); 2751 2752 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2753 2754 /* Initialize the implemented ports and structures */ 2755 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2756 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2757 continue; 2758 } 2759 2760 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2761 mutex_enter(&ahci_portp->ahciport_mutex); 2762 2763 /* 2764 * Ensure that the controller is not in the running state 2765 * by checking every implemented port's PxCMD register 2766 */ 2767 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2768 != AHCI_SUCCESS) { 2769 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2770 "ahci_initialize_controller: failed to " 2771 "initialize port %d", port); 2772 /* 2773 * Set the port state to SATA_PSTATE_FAILED if 2774 * failed to initialize it. 2775 */ 2776 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2777 } 2778 2779 mutex_exit(&ahci_portp->ahciport_mutex); 2780 } 2781 2782 /* Enable the whole controller interrupts */ 2783 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2784 ahci_enable_all_intrs(ahci_ctlp); 2785 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2786 2787 return (AHCI_SUCCESS); 2788 } 2789 2790 /* 2791 * Reverse of ahci_initialize_controller() 2792 * 2793 * We only need to stop the ports and disable the interrupt. 2794 */ 2795 static void 2796 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2797 { 2798 ahci_port_t *ahci_portp; 2799 int port; 2800 2801 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2802 "ahci_uninitialize_controller enter"); 2803 2804 /* disable all the interrupts. */ 2805 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2806 ahci_disable_all_intrs(ahci_ctlp); 2807 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2808 2809 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2810 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2811 continue; 2812 } 2813 2814 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2815 2816 /* Stop the port by clearing PxCMD.ST */ 2817 mutex_enter(&ahci_portp->ahciport_mutex); 2818 2819 /* 2820 * Here we must disable the port interrupt because 2821 * ahci_disable_all_intrs only clear GHC.IE, and IS 2822 * register will be still set if PxIE is enabled. 2823 * When ahci shares one IRQ with other drivers, the 2824 * intr handler may claim the intr mistakenly. 2825 */ 2826 ahci_disable_port_intrs(ahci_ctlp, port); 2827 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2828 ahci_portp, port); 2829 mutex_exit(&ahci_portp->ahciport_mutex); 2830 } 2831 } 2832 2833 /* 2834 * The routine is to initialize the port. First put the port in NOTRunning 2835 * state, then enable port interrupt and clear Serror register. And under 2836 * AHCI_ATTACH case, find device signature and then try to start the port. 2837 * 2838 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2839 */ 2840 static int 2841 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2842 ahci_port_t *ahci_portp, uint8_t port) 2843 { 2844 uint32_t port_cmd_status; 2845 2846 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2847 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2848 2849 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2850 "ahci_initialize_port: port %d " 2851 "port_cmd_status = 0x%x", port, port_cmd_status); 2852 /* 2853 * Check whether the port is in NotRunning state, if not, 2854 * put the port in NotRunning state 2855 */ 2856 if (!(port_cmd_status & 2857 (AHCI_CMD_STATUS_ST | 2858 AHCI_CMD_STATUS_CR | 2859 AHCI_CMD_STATUS_FRE | 2860 AHCI_CMD_STATUS_FR))) { 2861 2862 goto next; 2863 } 2864 2865 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 2866 port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT, NULL) != AHCI_SUCCESS) 2867 return (AHCI_FAILURE); 2868 2869 next: 2870 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2871 "port %d is in NotRunning state now", port); 2872 2873 /* 2874 * At the time being, only probe ports/devices and get the types of 2875 * attached devices during DDI_ATTACH. In fact, the device can be 2876 * changed during power state changes, but at the time being, we 2877 * don't support the situation. 2878 */ 2879 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2880 /* Try to get the device signature */ 2881 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2882 } else { 2883 2884 /* 2885 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2886 * and PxFBU registers in case these registers were cleared 2887 * during the suspend. 2888 */ 2889 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2890 "ahci_initialize_port: port %d " 2891 "reset the port during resume", port); 2892 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2893 2894 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2895 "ahci_initialize_port: port %d " 2896 "set PxCLB, PxCLBU, PxFB and PxFBU " 2897 "during resume", port); 2898 2899 /* Config Port Received FIS Base Address */ 2900 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2901 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2902 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2903 2904 /* Config Port Command List Base Address */ 2905 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2906 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2907 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2908 } 2909 2910 /* Disable the interface power management */ 2911 ahci_disable_interface_pm(ahci_ctlp, port); 2912 2913 /* Return directly if no device connected */ 2914 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2915 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2916 "No device connected to port %d", port); 2917 goto out; 2918 } 2919 2920 /* Try to start the port */ 2921 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 2922 != AHCI_SUCCESS) { 2923 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2924 "failed to start port %d", port); 2925 return (AHCI_FAILURE); 2926 } 2927 out: 2928 /* Enable port interrupts */ 2929 ahci_enable_port_intrs(ahci_ctlp, port); 2930 2931 return (AHCI_SUCCESS); 2932 } 2933 2934 /* 2935 * Check the hardware defects and the power management capability. 2936 */ 2937 static int 2938 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 2939 { 2940 ushort_t venid, devid; 2941 ushort_t caps_ptr, cap_count, cap; 2942 #if AHCI_DEBUG 2943 ushort_t pmcap, pmcsr; 2944 #endif 2945 uint8_t revision; 2946 2947 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2948 PCI_CONF_VENID); 2949 2950 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2951 PCI_CONF_DEVID); 2952 2953 /* 2954 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 2955 * controllers with 0x00 revision id work on 4-byte aligned buffer, 2956 * which is a bug and was fixed after 0x00 revision id controllers. 2957 * 2958 * Moreover, VT8251 cannot use multiple command slots in the command 2959 * list for non-queued commands because the previous register content 2960 * of PxCI can be re-written in the register write, so a flag will be 2961 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 2962 */ 2963 if (venid == VIA_VENID) { 2964 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 2965 PCI_CONF_REVID); 2966 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2967 "revision id = 0x%x", revision); 2968 if (revision == 0x00) { 2969 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 2970 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2971 "change ddi_attr_align to 0x4"); 2972 } 2973 2974 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 2975 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2976 "VT8251 cannot use multiple command lists for " 2977 "non-queued commands"); 2978 } 2979 2980 /* 2981 * ATI SB600 (1002,4380) and SB700/750 (1002,4391) AHCI chipsets don't 2982 * support 64-bit DMA addressing though they declare the support, 2983 * so we need to set AHCI_CAP_32BIT_DMA flag to force 32-bit DMA. 2984 */ 2985 if (venid == 0x1002 && (devid == 0x4380 || devid == 0x4391)) { 2986 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2987 "ATI SB600/700/750 cannot do 64-bit DMA though CAP " 2988 "indicates support, so force it to use 32-bit DMA"); 2989 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 2990 } 2991 2992 /* 2993 * Check if capabilities list is supported and if so, 2994 * get initial capabilities pointer and clear bits 0,1. 2995 */ 2996 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2997 PCI_CONF_STAT) & PCI_STAT_CAP) { 2998 caps_ptr = P2ALIGN(pci_config_get8( 2999 ahci_ctlp->ahcictl_pci_conf_handle, 3000 PCI_CONF_CAP_PTR), 4); 3001 } else { 3002 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3003 } 3004 3005 /* 3006 * Walk capabilities if supported. 3007 */ 3008 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3009 3010 /* 3011 * Check that we haven't exceeded the maximum number of 3012 * capabilities and that the pointer is in a valid range. 3013 */ 3014 if (++cap_count > PCI_CAP_MAX_PTR) { 3015 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3016 "too many device capabilities"); 3017 return (AHCI_FAILURE); 3018 } 3019 if (caps_ptr < PCI_CAP_PTR_OFF) { 3020 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3021 "capabilities pointer 0x%x out of range", 3022 caps_ptr); 3023 return (AHCI_FAILURE); 3024 } 3025 3026 /* 3027 * Get next capability and check that it is valid. 3028 * For now, we only support power management. 3029 */ 3030 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3031 caps_ptr); 3032 switch (cap) { 3033 case PCI_CAP_ID_PM: 3034 3035 /* power management supported */ 3036 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3037 3038 /* Save PMCSR offset */ 3039 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3040 3041 #if AHCI_DEBUG 3042 pmcap = pci_config_get16( 3043 ahci_ctlp->ahcictl_pci_conf_handle, 3044 caps_ptr + PCI_PMCAP); 3045 pmcsr = pci_config_get16( 3046 ahci_ctlp->ahcictl_pci_conf_handle, 3047 ahci_ctlp->ahcictl_pmcsr_offset); 3048 AHCIDBG2(AHCIDBG_PM, ahci_ctlp, 3049 "Power Management capability found PCI_PMCAP " 3050 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3051 if ((pmcap & 0x3) == 0x3) 3052 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3053 "PCI Power Management Interface " 3054 "spec 1.2 compliant"); 3055 #endif 3056 break; 3057 3058 case PCI_CAP_ID_MSI: 3059 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, "MSI capability found"); 3060 break; 3061 3062 case PCI_CAP_ID_PCIX: 3063 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3064 "PCI-X capability found"); 3065 break; 3066 3067 case PCI_CAP_ID_PCI_E: 3068 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3069 "PCI Express capability found"); 3070 break; 3071 3072 case PCI_CAP_ID_MSI_X: 3073 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3074 "MSI-X capability found"); 3075 break; 3076 3077 case PCI_CAP_ID_SATA: 3078 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3079 "SATA capability found"); 3080 break; 3081 3082 case PCI_CAP_ID_VS: 3083 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3084 "Vendor Specific capability found"); 3085 break; 3086 3087 default: 3088 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 3089 "unrecognized capability 0x%x", cap); 3090 break; 3091 } 3092 3093 /* 3094 * Get next capabilities pointer and clear bits 0,1. 3095 */ 3096 caps_ptr = P2ALIGN(pci_config_get8( 3097 ahci_ctlp->ahcictl_pci_conf_handle, 3098 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3099 } 3100 3101 return (AHCI_SUCCESS); 3102 } 3103 3104 /* 3105 * AHCI device reset ...; a single device on one of the ports is reset, 3106 * but the HBA and physical communication remain intact. This is the 3107 * least intrusive. 3108 * 3109 * When issuing a software reset sequence, there should not be other 3110 * commands in the command list, so we will first clear and then re-set 3111 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3112 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3113 * cleared unless command list override (PxCMD.CLO) is supported. 3114 * 3115 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3116 * set before the function is called. 3117 */ 3118 static int 3119 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3120 uint8_t port) 3121 { 3122 ahci_fis_h2d_register_t *h2d_register_fisp; 3123 ahci_cmd_table_t *cmd_table; 3124 ahci_cmd_header_t *cmd_header; 3125 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3126 int slot, loop_count; 3127 3128 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 3129 "Port %d device resetting", port); 3130 3131 /* First clear PxCMD.ST */ 3132 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3133 ahci_portp, port); 3134 3135 /* Then re-set PxCMD.ST */ 3136 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 3137 3138 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3139 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3140 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3141 3142 if (port_task_file & AHCI_TFD_STS_BSY || 3143 port_task_file & AHCI_TFD_STS_DRQ) { 3144 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3145 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3146 if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) { 3147 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3148 "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, " 3149 "but PxCMD.CLO isn't supported, so a port " 3150 "reset is needed."); 3151 return (AHCI_FAILURE); 3152 } 3153 } 3154 3155 slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, AHCI_NON_NCQ_CMD); 3156 if (slot == AHCI_FAILURE) { 3157 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 3158 "ahci_software_reset: no free slot"); 3159 return (AHCI_FAILURE); 3160 } 3161 3162 /* Now send the first H2D Register FIS with SRST set to 1 */ 3163 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3164 bzero((void *)cmd_table, ahci_cmd_table_size); 3165 3166 h2d_register_fisp = 3167 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3168 3169 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3170 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3171 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3172 3173 /* Set Command Header in Command List */ 3174 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3175 BZERO_DESCR_INFO(cmd_header); 3176 BZERO_PRD_BYTE_COUNT(cmd_header); 3177 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3178 3179 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3180 SET_RESET(cmd_header, 1); 3181 SET_WRITE(cmd_header, 1); 3182 3183 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3184 0, 3185 ahci_cmd_table_size, 3186 DDI_DMA_SYNC_FORDEV); 3187 3188 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3189 slot * sizeof (ahci_cmd_header_t), 3190 sizeof (ahci_cmd_header_t), 3191 DDI_DMA_SYNC_FORDEV); 3192 3193 /* Indicate to the HBA that a command is active. */ 3194 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3195 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3196 (0x1 << slot)); 3197 3198 loop_count = 0; 3199 3200 /* Loop till the first command is finished */ 3201 do { 3202 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3203 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3204 3205 /* We are effectively timing out after 1 sec. */ 3206 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3207 break; 3208 } 3209 /* Wait for 10 millisec */ 3210 delay(AHCI_10MS_TICKS); 3211 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3212 3213 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3214 "ahci_software_reset: 1st loop count: %d, " 3215 "port_cmd_issue = 0x%x, slot = 0x%x", 3216 loop_count, port_cmd_issue, slot); 3217 3218 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3219 3220 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3221 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3222 bzero((void *)cmd_table, ahci_cmd_table_size); 3223 3224 h2d_register_fisp = 3225 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3226 3227 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3228 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3229 3230 /* Set Command Header in Command List */ 3231 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3232 BZERO_DESCR_INFO(cmd_header); 3233 BZERO_PRD_BYTE_COUNT(cmd_header); 3234 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3235 3236 SET_WRITE(cmd_header, 1); 3237 3238 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3239 0, 3240 ahci_cmd_table_size, 3241 DDI_DMA_SYNC_FORDEV); 3242 3243 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3244 slot * sizeof (ahci_cmd_header_t), 3245 sizeof (ahci_cmd_header_t), 3246 DDI_DMA_SYNC_FORDEV); 3247 3248 /* Indicate to the HBA that a command is active. */ 3249 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3250 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3251 (0x1 << slot)); 3252 3253 loop_count = 0; 3254 3255 /* Loop till the second command is finished */ 3256 do { 3257 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3258 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3259 3260 /* We are effectively timing out after 1 sec. */ 3261 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3262 break; 3263 } 3264 /* Wait for 10 millisec */ 3265 delay(AHCI_10MS_TICKS); 3266 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3267 3268 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3269 "ahci_software_reset: 2nd loop count: %d, " 3270 "port_cmd_issue = 0x%x, slot = 0x%x", 3271 loop_count, port_cmd_issue, slot); 3272 3273 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3274 3275 return (AHCI_SUCCESS); 3276 } 3277 3278 /* 3279 * AHCI port reset ...; the physical communication between the HBA and device 3280 * on a port are disabled. This is more intrusive. 3281 * 3282 * When an HBA or port reset occurs, Phy communication is going to 3283 * be re-established with the device through a COMRESET followed by the 3284 * normal out-of-band communication sequence defined in Serial ATA. AT 3285 * the end of reset, the device, if working properly, will send a D2H 3286 * Register FIS, which contains the device signature. When the HBA receives 3287 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3288 * the PxSIG register with the signature. 3289 * 3290 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3291 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3292 * AHCI 1.0 spec. 3293 */ 3294 /* 3295 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3296 * cleared before the function is called. 3297 */ 3298 static int 3299 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3300 { 3301 uint32_t cap_status, port_cmd_status; 3302 uint32_t port_scontrol, port_sstatus; 3303 uint32_t port_intr_status, port_task_file; 3304 #if AHCI_DEBUG 3305 uint32_t port_signature; 3306 #endif 3307 int loop_count; 3308 int rval = AHCI_SUCCESS; 3309 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3310 3311 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3312 "Port %d port resetting...", port); 3313 ahci_portp->ahciport_port_state = 0; 3314 3315 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3316 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3317 3318 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3319 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3320 3321 if (cap_status & AHCI_HBA_CAP_SSS) { 3322 /* 3323 * HBA support staggered spin-up, if the port has 3324 * not spin up yet, then force it to do spin-up 3325 */ 3326 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3327 if (!(ahci_portp->ahciport_flags 3328 & AHCI_PORT_FLAG_SPINUP)) { 3329 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3330 "Port %d PxCMD.SUD is zero, force " 3331 "it to do spin-up", port); 3332 ahci_portp->ahciport_flags |= 3333 AHCI_PORT_FLAG_SPINUP; 3334 } 3335 } 3336 } else { 3337 /* 3338 * HBA doesn't support stagger spin-up, force it 3339 * to do normal COMRESET 3340 */ 3341 if (ahci_portp->ahciport_flags & 3342 AHCI_PORT_FLAG_SPINUP) { 3343 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3344 "HBA does not support staggered spin-up " 3345 "force it to do normal COMRESET"); 3346 ahci_portp->ahciport_flags &= 3347 ~AHCI_PORT_FLAG_SPINUP; 3348 } 3349 } 3350 3351 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3352 /* Do normal COMRESET */ 3353 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3354 "ahci_port_reset: do normal COMRESET", port); 3355 3356 /* 3357 * According to the spec, SUD bit should be set here, 3358 * but JMicron JMB363 doesn't follow it, so remove 3359 * the assertion, and just print a debug message. 3360 */ 3361 #if AHCI_DEBUG 3362 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3363 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3364 "port %d SUD bit not set", port) 3365 #endif 3366 3367 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3368 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3369 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3370 3371 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3372 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3373 port_scontrol); 3374 3375 /* Enable PxCMD.FRE to read device */ 3376 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3377 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3378 port_cmd_status|AHCI_CMD_STATUS_FRE); 3379 3380 /* 3381 * Give time for COMRESET to percolate, according to the AHCI 3382 * spec, software shall wait at least 1 millisecond before 3383 * clearing PxSCTL.DET 3384 */ 3385 delay(AHCI_1MS_TICKS*2); 3386 3387 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3388 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3389 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3390 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3391 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3392 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3393 port_scontrol); 3394 } else { 3395 /* Do staggered spin-up */ 3396 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3397 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3398 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3399 3400 /* PxSCTL.DET must be 0 */ 3401 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3402 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3403 port_scontrol); 3404 3405 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3406 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3407 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3408 port_cmd_status); 3409 3410 /* 0 -> 1 edge */ 3411 delay(AHCI_1MS_TICKS*2); 3412 3413 /* Set PxCMD.SUD to 1 */ 3414 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3415 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3416 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3417 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3418 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3419 port_cmd_status); 3420 3421 /* Enable PxCMD.FRE to read device */ 3422 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3423 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3424 port_cmd_status|AHCI_CMD_STATUS_FRE); 3425 } 3426 3427 /* 3428 * The port enters P:StartComm state, and HBA tells link layer to 3429 * start communication, which involves sending COMRESET to device. 3430 * And the HBA resets PxTFD.STS to 7Fh. 3431 * 3432 * When a COMINIT is received from the device, then the port enters 3433 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3434 * PxSSTS.DET to 1h to indicate a device is detected but communication 3435 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3436 * a COMINIT has been received. 3437 */ 3438 /* 3439 * The DET field is valid only if IPM field indicates 3440 * that the interface is in active state. 3441 */ 3442 loop_count = 0; 3443 do { 3444 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3445 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3446 3447 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3448 /* 3449 * If the interface is not active, the DET field 3450 * is considered not accurate. So we want to 3451 * continue looping. 3452 */ 3453 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3454 } 3455 3456 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3457 /* 3458 * We are effectively timing out after 0.1 sec. 3459 */ 3460 break; 3461 } 3462 3463 /* Wait for 10 millisec */ 3464 delay(AHCI_10MS_TICKS); 3465 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3466 3467 AHCIDBG3(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3468 "ahci_port_reset: 1st loop count: %d, " 3469 "port_sstatus = 0x%x port %d", 3470 loop_count, port_sstatus, port); 3471 3472 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3473 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3474 /* 3475 * Either the port is not active or there 3476 * is no device present. 3477 */ 3478 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3479 goto out; 3480 } 3481 3482 /* Now we can make sure there is a device connected to the port */ 3483 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3484 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3485 3486 /* a COMINIT signal is supposed to be received */ 3487 if (!(port_intr_status & AHCI_INTR_STATUS_PCS)) { 3488 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3489 "COMINIT signal from the device not received", 3490 instance, port); 3491 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3492 rval = AHCI_FAILURE; 3493 goto out; 3494 } 3495 3496 /* 3497 * According to the spec, when PxSCTL.DET is set to 0h, upon 3498 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3499 * shall be set to '1' by the HBA. 3500 * 3501 * However, we found JMicron JMB363 doesn't follow this, so 3502 * remove this check, and just print a debug message. 3503 */ 3504 #if AHCI_DEBUG 3505 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3506 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3507 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3508 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3509 "port %d BSY bit is not set after COMINIT signal " 3510 "is received", port); 3511 } 3512 #endif 3513 3514 /* 3515 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3516 * the D2H FIS received by HBA. 3517 */ 3518 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3519 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3520 SERROR_EXCHANGED_ERR); 3521 3522 /* 3523 * Next check whether COMRESET is completed successfully 3524 */ 3525 loop_count = 0; 3526 do { 3527 port_task_file = 3528 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3529 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3530 3531 /* 3532 * The Error bit '1' means COMRESET is finished successfully 3533 * The device hardware has been initialized and the power-up 3534 * diagnostics successfully completed. 3535 */ 3536 if (((port_task_file & AHCI_TFD_ERR_MASK) 3537 >> AHCI_TFD_ERR_SHIFT) == 0x1) { 3538 #if AHCI_DEBUG 3539 port_signature = ddi_get32( 3540 ahci_ctlp->ahcictl_ahci_acc_handle, 3541 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3542 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 3543 "COMRESET success, D2H register FIS " 3544 "post to received FIS structure " 3545 "port %d signature = 0x%x", 3546 port, port_signature); 3547 #endif 3548 goto out_check; 3549 } 3550 3551 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3552 /* 3553 * We are effectively timing out after 11 sec. 3554 */ 3555 break; 3556 } 3557 3558 /* Wait for 10 millisec */ 3559 delay(AHCI_10MS_TICKS); 3560 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3561 >> AHCI_TFD_ERR_SHIFT) != 0x1); 3562 3563 AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: 2nd loop " 3564 "count: %d, port_task_file = 0x%x port %d", 3565 loop_count, port_task_file, port); 3566 3567 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d the device hardware " 3568 "has been initialized and the power-up diagnostics failed", 3569 instance, port); 3570 3571 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3572 rval = AHCI_FAILURE; 3573 3574 out: 3575 /* Clear port serror register for the port */ 3576 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3577 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3578 AHCI_SERROR_CLEAR_ALL); 3579 3580 return (rval); 3581 3582 out_check: 3583 /* 3584 * Check device status, if keep busy or COMRESET error 3585 * do device reset to patch some SATA disks' issue 3586 * 3587 * For VT8251, sometimes need to do the device reset 3588 */ 3589 if ((port_task_file & AHCI_TFD_STS_BSY) || 3590 (port_task_file & AHCI_TFD_STS_DRQ)) { 3591 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "port %d keep BSY/DRQ set " 3592 "need to do device reset", port); 3593 3594 (void) ahci_software_reset(ahci_ctlp, ahci_portp, port); 3595 3596 port_task_file = 3597 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3598 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3599 3600 if (port_task_file & AHCI_TFD_STS_BSY || 3601 port_task_file & AHCI_TFD_STS_DRQ) { 3602 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset: port %d " 3603 "BSY/DRQ still set after device reset " 3604 "port_task_file = 0x%x", instance, 3605 port, port_task_file); 3606 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3607 rval = AHCI_FAILURE; 3608 } 3609 } 3610 3611 goto out; 3612 } 3613 3614 /* 3615 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3616 * This is the most intrusive. 3617 * 3618 * When an HBA reset occurs, Phy communication will be re-established with 3619 * the device through a COMRESET followed by the normal out-of-band 3620 * communication sequence defined in Serial ATA. AT the end of reset, the 3621 * device, if working properly, will send a D2H Register FIS, which contains 3622 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3623 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3624 * signature. 3625 * 3626 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3627 */ 3628 static int 3629 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3630 { 3631 ahci_port_t *ahci_portp; 3632 uint32_t ghc_control; 3633 uint8_t port; 3634 int loop_count; 3635 int rval = AHCI_SUCCESS; 3636 3637 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting"); 3638 3639 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3640 3641 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3642 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3643 3644 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3645 ghc_control |= AHCI_HBA_GHC_HR; 3646 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3647 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3648 3649 /* 3650 * Wait until HBA Reset complete or timeout 3651 */ 3652 loop_count = 0; 3653 do { 3654 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3655 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3656 3657 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3658 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3659 "ahci hba reset is timing out, " 3660 "ghc_control = 0x%x", ghc_control); 3661 /* We are effectively timing out after 1 sec. */ 3662 break; 3663 } 3664 3665 /* Wait for 10 millisec */ 3666 delay(AHCI_10MS_TICKS); 3667 } while (ghc_control & AHCI_HBA_GHC_HR); 3668 3669 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3670 "ahci_hba_reset: 1st loop count: %d, " 3671 "ghc_control = 0x%x", loop_count, ghc_control); 3672 3673 if (ghc_control & AHCI_HBA_GHC_HR) { 3674 /* The hba is not reset for some reasons */ 3675 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3676 "hba reset failed: HBA in a hung or locked state"); 3677 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3678 return (AHCI_FAILURE); 3679 } 3680 3681 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3682 /* Only check implemented ports */ 3683 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3684 continue; 3685 } 3686 3687 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3688 mutex_enter(&ahci_portp->ahciport_mutex); 3689 3690 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3691 != AHCI_SUCCESS) { 3692 rval = AHCI_FAILURE; 3693 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3694 "ahci_hba_reset: port %d failed", port); 3695 } 3696 3697 mutex_exit(&ahci_portp->ahciport_mutex); 3698 } 3699 3700 /* 3701 * Indicate that system software is AHCI aware by setting 3702 * GHC.AE to 1 3703 */ 3704 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3705 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3706 3707 ghc_control |= AHCI_HBA_GHC_AE; 3708 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3709 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3710 3711 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3712 3713 return (rval); 3714 } 3715 3716 /* 3717 * This routine is only called from AHCI_ATTACH or phyrdy change 3718 * case. It first calls port reset to initialize port, probe port and probe 3719 * device, then try to read PxSIG register to find the type of device 3720 * attached to the port. 3721 * 3722 * WARNING!!! ahciport_mutex should be acquired before the function 3723 * is called. And the port interrupt is disabled. 3724 */ 3725 static void 3726 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3727 ahci_port_t *ahci_portp, uint8_t port) 3728 { 3729 uint32_t signature; 3730 3731 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3732 "ahci_find_dev_signature enter: port %d", port); 3733 3734 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3735 3736 /* Call port reset to check link status and get device signature */ 3737 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 3738 3739 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3740 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3741 "ahci_find_dev_signature: No device is found " 3742 "at port %d", port); 3743 return; 3744 } 3745 3746 /* Check the port state */ 3747 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3748 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 3749 "ahci_find_dev_signature: port %d state 0x%x", 3750 port, ahci_portp->ahciport_port_state); 3751 return; 3752 } 3753 3754 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3755 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3756 3757 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3758 "ahci_find_dev_signature: port %d signature = 0x%x", 3759 port, signature); 3760 3761 switch (signature) { 3762 3763 case AHCI_SIGNATURE_DISK: 3764 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3765 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3766 "Disk is found at port: %d", port); 3767 break; 3768 3769 case AHCI_SIGNATURE_ATAPI: 3770 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3771 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3772 "ATAPI device is found at port: %d", port); 3773 break; 3774 3775 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3776 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3777 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3778 "Port Multiplier is found at port: %d", port); 3779 break; 3780 3781 default: 3782 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3783 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3784 "Unknown device is found at port: %d", port); 3785 } 3786 } 3787 3788 /* 3789 * According to the spec, to reliably detect hot plug removals, software 3790 * must disable interface power management. Software should perform the 3791 * following initialization on a port after a device is attached: 3792 * Set PxSCTL.IPM to 3h to disable interface state transitions 3793 * Set PxCMD.ALPE to '0' to disable aggressive power management 3794 * Disable device initiated interface power management by SET FEATURE 3795 * 3796 * We can ignore the last item because by default the feature is disabled 3797 */ 3798 static void 3799 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3800 { 3801 uint32_t port_scontrol, port_cmd_status; 3802 3803 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3804 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3805 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3806 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3807 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3808 3809 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3810 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3811 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3812 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3813 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3814 } 3815 3816 /* 3817 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3818 * to 1, then set it firstly. 3819 * 3820 * Each port contains two major DMA engines. One DMA engine walks through 3821 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3822 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3823 * 3824 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3825 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 3826 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 3827 * 3828 * Software shall not set PxCMD.ST to '1' unless a functional device is 3829 * present on the port(as determined by PxTFD.STS.BSY = '0', 3830 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 3831 * 3832 * WARNING!!! ahciport_mutex should be acquired before the function 3833 * is called. 3834 */ 3835 static int 3836 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3837 { 3838 uint32_t port_cmd_status; 3839 3840 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 3841 3842 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3843 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3844 "the state for port %d is 0x%x", 3845 port, ahci_portp->ahciport_port_state); 3846 return (AHCI_FAILURE); 3847 } 3848 3849 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3850 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3851 "no device is attached at port %d", port); 3852 return (AHCI_FAILURE); 3853 } 3854 3855 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 3856 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3857 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3858 3859 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 3860 port_cmd_status |= AHCI_CMD_STATUS_FRE; 3861 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3862 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3863 port_cmd_status); 3864 } 3865 3866 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3867 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3868 3869 port_cmd_status |= AHCI_CMD_STATUS_ST; 3870 3871 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3872 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3873 port_cmd_status); 3874 3875 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 3876 3877 return (AHCI_SUCCESS); 3878 } 3879 3880 /* 3881 * Allocate the ahci_port_t including Received FIS and Command List. 3882 * The argument - port is the physical port number, and not logical 3883 * port number seen by the SATA framework. 3884 * 3885 * WARNING!!! ahcictl_mutex should be acquired before the function 3886 * is called. 3887 */ 3888 static int 3889 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3890 { 3891 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 3892 ahci_port_t *ahci_portp; 3893 char taskq_name[64] = "event_handle_taskq"; 3894 3895 ahci_portp = 3896 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 3897 3898 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 3899 ahci_portp->ahciport_port_num = port; 3900 3901 /* Intialize the port condition variable */ 3902 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 3903 3904 /* Initialize the port mutex */ 3905 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 3906 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 3907 3908 mutex_enter(&ahci_portp->ahciport_mutex); 3909 3910 /* 3911 * Allocate memory for received FIS structure and 3912 * command list for this port 3913 */ 3914 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3915 goto err_case1; 3916 } 3917 3918 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3919 goto err_case2; 3920 } 3921 3922 (void) snprintf(taskq_name + strlen(taskq_name), 3923 sizeof (taskq_name) - strlen(taskq_name), 3924 "_port%d", port); 3925 3926 /* Create the taskq for the port */ 3927 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 3928 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 3929 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 3930 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 3931 goto err_case3; 3932 } 3933 3934 /* Allocate the argument for the taskq */ 3935 ahci_portp->ahciport_event_args = 3936 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 3937 3938 if (ahci_portp->ahciport_event_args == NULL) 3939 goto err_case4; 3940 3941 mutex_exit(&ahci_portp->ahciport_mutex); 3942 3943 return (AHCI_SUCCESS); 3944 3945 err_case4: 3946 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 3947 3948 err_case3: 3949 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3950 3951 err_case2: 3952 ahci_dealloc_rcvd_fis(ahci_portp); 3953 3954 err_case1: 3955 mutex_exit(&ahci_portp->ahciport_mutex); 3956 mutex_destroy(&ahci_portp->ahciport_mutex); 3957 cv_destroy(&ahci_portp->ahciport_cv); 3958 3959 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3960 3961 return (AHCI_FAILURE); 3962 } 3963 3964 /* 3965 * Reverse of ahci_dealloc_port_state(). 3966 * 3967 * WARNING!!! ahcictl_mutex should be acquired before the function 3968 * is called. 3969 */ 3970 static void 3971 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3972 { 3973 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3974 3975 ASSERT(ahci_portp != NULL); 3976 3977 mutex_enter(&ahci_portp->ahciport_mutex); 3978 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 3979 ahci_portp->ahciport_event_args = NULL; 3980 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 3981 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3982 ahci_dealloc_rcvd_fis(ahci_portp); 3983 mutex_exit(&ahci_portp->ahciport_mutex); 3984 3985 mutex_destroy(&ahci_portp->ahciport_mutex); 3986 cv_destroy(&ahci_portp->ahciport_cv); 3987 3988 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3989 3990 ahci_ctlp->ahcictl_ports[port] = NULL; 3991 } 3992 3993 /* 3994 * Allocates memory for the Received FIS Structure 3995 * 3996 * WARNING!!! ahciport_mutex should be acquired before the function 3997 * is called. 3998 */ 3999 static int 4000 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4001 uint8_t port) 4002 { 4003 size_t rcvd_fis_size; 4004 size_t ret_len; 4005 uint_t cookie_count; 4006 4007 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 4008 4009 /* allocate rcvd FIS dma handle. */ 4010 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4011 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 4012 DDI_DMA_SLEEP, 4013 NULL, 4014 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 4015 DDI_SUCCESS) { 4016 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4017 "rcvd FIS dma handle alloc failed"); 4018 4019 return (AHCI_FAILURE); 4020 } 4021 4022 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 4023 rcvd_fis_size, 4024 &accattr, 4025 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4026 DDI_DMA_SLEEP, 4027 NULL, 4028 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4029 &ret_len, 4030 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4031 4032 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4033 "rcvd FIS dma mem alloc fail"); 4034 /* error.. free the dma handle. */ 4035 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4036 return (AHCI_FAILURE); 4037 } 4038 4039 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4040 NULL, 4041 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4042 rcvd_fis_size, 4043 DDI_DMA_CONSISTENT, 4044 DDI_DMA_SLEEP, 4045 NULL, 4046 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4047 &cookie_count) != DDI_DMA_MAPPED) { 4048 4049 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4050 "rcvd FIS dma handle bind fail"); 4051 /* error.. free the dma handle & free the memory. */ 4052 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4053 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4054 return (AHCI_FAILURE); 4055 } 4056 4057 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4058 4059 /* Config Port Received FIS Base Address */ 4060 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4061 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4062 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4063 4064 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4065 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4066 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4067 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4068 4069 return (AHCI_SUCCESS); 4070 } 4071 4072 /* 4073 * Deallocates the Received FIS Structure 4074 * 4075 * WARNING!!! ahciport_mutex should be acquired before the function 4076 * is called. 4077 */ 4078 static void 4079 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4080 { 4081 /* Unbind the cmd list dma handle first. */ 4082 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4083 4084 /* Then free the underlying memory. */ 4085 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4086 4087 /* Now free the handle itself. */ 4088 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4089 } 4090 4091 /* 4092 * Allocates memory for the Command List, which contains up to 32 entries. 4093 * Each entry contains a command header, which is a 32-byte structure that 4094 * includes the pointer to the command table. 4095 * 4096 * WARNING!!! ahciport_mutex should be acquired before the function 4097 * is called. 4098 */ 4099 static int 4100 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4101 uint8_t port) 4102 { 4103 size_t cmd_list_size; 4104 size_t ret_len; 4105 uint_t cookie_count; 4106 4107 cmd_list_size = 4108 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4109 4110 /* allocate cmd list dma handle. */ 4111 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4112 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4113 DDI_DMA_SLEEP, 4114 NULL, 4115 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4116 4117 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4118 "cmd list dma handle alloc failed"); 4119 return (AHCI_FAILURE); 4120 } 4121 4122 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4123 cmd_list_size, 4124 &accattr, 4125 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4126 DDI_DMA_SLEEP, 4127 NULL, 4128 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4129 &ret_len, 4130 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4131 4132 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4133 "cmd list dma mem alloc fail"); 4134 /* error.. free the dma handle. */ 4135 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4136 return (AHCI_FAILURE); 4137 } 4138 4139 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4140 NULL, 4141 (caddr_t)ahci_portp->ahciport_cmd_list, 4142 cmd_list_size, 4143 DDI_DMA_CONSISTENT, 4144 DDI_DMA_SLEEP, 4145 NULL, 4146 &ahci_portp->ahciport_cmd_list_dma_cookie, 4147 &cookie_count) != DDI_DMA_MAPPED) { 4148 4149 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4150 "cmd list dma handle bind fail"); 4151 /* error.. free the dma handle & free the memory. */ 4152 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4153 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4154 return (AHCI_FAILURE); 4155 } 4156 4157 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4158 4159 /* Config Port Command List Base Address */ 4160 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4161 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4162 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4163 4164 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4165 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4166 4167 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4168 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4169 4170 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4171 goto err_out; 4172 } 4173 4174 return (AHCI_SUCCESS); 4175 4176 err_out: 4177 /* Unbind the cmd list dma handle first. */ 4178 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4179 4180 /* Then free the underlying memory. */ 4181 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4182 4183 /* Now free the handle itself. */ 4184 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4185 4186 return (AHCI_FAILURE); 4187 } 4188 4189 /* 4190 * Deallocates the Command List 4191 * 4192 * WARNING!!! ahciport_mutex should be acquired before the function 4193 * is called. 4194 */ 4195 static void 4196 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4197 { 4198 /* First dealloc command table */ 4199 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4200 4201 /* Unbind the cmd list dma handle first. */ 4202 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4203 4204 /* Then free the underlying memory. */ 4205 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4206 4207 /* Now free the handle itself. */ 4208 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4209 } 4210 4211 /* 4212 * Allocates memory for all Command Tables, which contains Command FIS, 4213 * ATAPI Command and Physical Region Descriptor Table. 4214 * 4215 * WARNING!!! ahciport_mutex should be acquired before the function 4216 * is called. 4217 */ 4218 static int 4219 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4220 { 4221 size_t ret_len; 4222 ddi_dma_cookie_t cmd_table_dma_cookie; 4223 uint_t cookie_count; 4224 int slot; 4225 4226 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4227 "ahci_alloc_cmd_tables: port %d enter", 4228 ahci_portp->ahciport_port_num); 4229 4230 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4231 /* Allocate cmd table dma handle. */ 4232 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4233 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4234 DDI_DMA_SLEEP, 4235 NULL, 4236 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4237 DDI_SUCCESS) { 4238 4239 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4240 "cmd table dma handle alloc failed"); 4241 4242 goto err_out; 4243 } 4244 4245 if (ddi_dma_mem_alloc( 4246 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4247 ahci_cmd_table_size, 4248 &accattr, 4249 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4250 DDI_DMA_SLEEP, 4251 NULL, 4252 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4253 &ret_len, 4254 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4255 NULL) { 4256 4257 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4258 "cmd table dma mem alloc fail"); 4259 4260 /* error.. free the dma handle. */ 4261 ddi_dma_free_handle( 4262 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4263 goto err_out; 4264 } 4265 4266 if (ddi_dma_addr_bind_handle( 4267 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4268 NULL, 4269 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4270 ahci_cmd_table_size, 4271 DDI_DMA_CONSISTENT, 4272 DDI_DMA_SLEEP, 4273 NULL, 4274 &cmd_table_dma_cookie, 4275 &cookie_count) != DDI_DMA_MAPPED) { 4276 4277 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4278 "cmd table dma handle bind fail"); 4279 /* error.. free the dma handle & free the memory. */ 4280 ddi_dma_mem_free( 4281 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4282 ddi_dma_free_handle( 4283 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4284 goto err_out; 4285 } 4286 4287 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4288 ahci_cmd_table_size); 4289 4290 /* Config Port Command Table Base Address */ 4291 SET_COMMAND_TABLE_BASE_ADDR( 4292 (&ahci_portp->ahciport_cmd_list[slot]), 4293 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4294 4295 #ifndef __lock_lint 4296 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4297 (&ahci_portp->ahciport_cmd_list[slot]), 4298 cmd_table_dma_cookie.dmac_laddress >> 32); 4299 #endif 4300 } 4301 4302 return (AHCI_SUCCESS); 4303 err_out: 4304 4305 for (slot--; slot >= 0; slot--) { 4306 /* Unbind the cmd table dma handle first */ 4307 (void) ddi_dma_unbind_handle( 4308 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4309 4310 /* Then free the underlying memory */ 4311 ddi_dma_mem_free( 4312 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4313 4314 /* Now free the handle itself */ 4315 ddi_dma_free_handle( 4316 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4317 } 4318 4319 return (AHCI_FAILURE); 4320 } 4321 4322 /* 4323 * Deallocates memory for all Command Tables. 4324 * 4325 * WARNING!!! ahciport_mutex should be acquired before the function 4326 * is called. 4327 */ 4328 static void 4329 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4330 { 4331 int slot; 4332 4333 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 4334 "ahci_dealloc_cmd_tables: %d enter", 4335 ahci_portp->ahciport_port_num); 4336 4337 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4338 /* Unbind the cmd table dma handle first. */ 4339 (void) ddi_dma_unbind_handle( 4340 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4341 4342 /* Then free the underlying memory. */ 4343 ddi_dma_mem_free( 4344 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4345 4346 /* Now free the handle itself. */ 4347 ddi_dma_free_handle( 4348 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4349 } 4350 } 4351 4352 /* 4353 * WARNING!!! ahciport_mutex should be acquired before the function 4354 * is called. 4355 */ 4356 static void 4357 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4358 sata_device_t *sd) 4359 { 4360 sd->satadev_scr.sstatus = 4361 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4362 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4363 sd->satadev_scr.serror = 4364 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4365 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4366 sd->satadev_scr.scontrol = 4367 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4368 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4369 sd->satadev_scr.sactive = 4370 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4371 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4372 } 4373 4374 /* 4375 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4376 */ 4377 static void 4378 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4379 { 4380 uint32_t port_intr_status; 4381 uint32_t port_intr_enable; 4382 4383 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4384 "ahci_port_intr enter: port %d", port); 4385 4386 mutex_enter(&ahci_portp->ahciport_mutex); 4387 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4388 /* For SATA_OPMODE_POLLING commands */ 4389 port_intr_enable = 4390 (AHCI_INTR_STATUS_DHRS | 4391 AHCI_INTR_STATUS_PSS | 4392 AHCI_INTR_STATUS_SDBS | 4393 AHCI_INTR_STATUS_UFS | 4394 AHCI_INTR_STATUS_PCS | 4395 AHCI_INTR_STATUS_PRCS | 4396 AHCI_INTR_STATUS_OFS | 4397 AHCI_INTR_STATUS_INFS | 4398 AHCI_INTR_STATUS_IFS | 4399 AHCI_INTR_STATUS_HBDS | 4400 AHCI_INTR_STATUS_HBFS | 4401 AHCI_INTR_STATUS_TFES); 4402 mutex_exit(&ahci_portp->ahciport_mutex); 4403 goto next; 4404 } 4405 mutex_exit(&ahci_portp->ahciport_mutex); 4406 4407 /* 4408 * port_intr_enable indicates that the corresponding interrrupt 4409 * reporting is enabled. 4410 */ 4411 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4412 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4413 next: 4414 /* 4415 * port_intr_stats indicates that the corresponding interrupt 4416 * condition is active. 4417 */ 4418 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4419 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4420 4421 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4422 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4423 "port_intr_enable = 0x%x", 4424 port, port_intr_status, port_intr_enable); 4425 4426 port_intr_status &= port_intr_enable; 4427 4428 /* First clear the port interrupts status */ 4429 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4430 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4431 port_intr_status); 4432 4433 /* Check the completed non-queued commands */ 4434 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4435 AHCI_INTR_STATUS_PSS)) { 4436 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4437 ahci_portp, port); 4438 } 4439 4440 /* Check the completed queued commands */ 4441 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4442 (void) ahci_intr_set_device_bits(ahci_ctlp, 4443 ahci_portp, port); 4444 } 4445 4446 /* Check the port connect change status interrupt bit */ 4447 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4448 (void) ahci_intr_port_connect_change(ahci_ctlp, 4449 ahci_portp, port); 4450 } 4451 4452 /* Check the device mechanical presence status interrupt bit */ 4453 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4454 (void) ahci_intr_device_mechanical_presence_status( 4455 ahci_ctlp, ahci_portp, port); 4456 } 4457 4458 /* Check the PhyRdy change status interrupt bit */ 4459 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4460 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4461 port); 4462 } 4463 4464 /* 4465 * Check the non-fatal error interrupt bits, there are three 4466 * kinds of non-fatal errors at the time being: 4467 * 4468 * PxIS.UFS - Unknown FIS Error 4469 * PxIS.OFS - Overflow Error 4470 * PxIS.INFS - Interface Non-Fatal Error 4471 * 4472 * For these non-fatal errors, the HBA can continue to operate, 4473 * so the driver just log the error messages. 4474 */ 4475 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4476 AHCI_INTR_STATUS_OFS | 4477 AHCI_INTR_STATUS_INFS)) { 4478 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4479 port, port_intr_status); 4480 } 4481 4482 /* 4483 * Check the fatal error interrupt bits, there are four kinds 4484 * of fatal errors for AHCI controllers: 4485 * 4486 * PxIS.HBFS - Host Bus Fatal Error 4487 * PxIS.HBDS - Host Bus Data Error 4488 * PxIS.IFS - Interface Fatal Error 4489 * PxIS.TFES - Task File Error 4490 * 4491 * The fatal error means the HBA can not recover from it by 4492 * itself, and it will try to abort the transfer, and the software 4493 * must intervene to restart the port. 4494 */ 4495 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4496 AHCI_INTR_STATUS_HBDS | 4497 AHCI_INTR_STATUS_HBFS | 4498 AHCI_INTR_STATUS_TFES)) 4499 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4500 port, port_intr_status); 4501 4502 /* Check the cold port detect interrupt bit */ 4503 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4504 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4505 } 4506 4507 /* Second clear the corresponding bit in IS.IPS */ 4508 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4509 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4510 } 4511 4512 /* 4513 * Interrupt service handler 4514 */ 4515 static uint_t 4516 ahci_intr(caddr_t arg1, caddr_t arg2) 4517 { 4518 #ifndef __lock_lint 4519 _NOTE(ARGUNUSED(arg2)) 4520 #endif 4521 /* LINTED */ 4522 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4523 ahci_port_t *ahci_portp; 4524 int32_t global_intr_status; 4525 uint8_t port; 4526 4527 /* 4528 * global_intr_status indicates that the corresponding port has 4529 * an interrupt pending. 4530 */ 4531 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4532 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4533 4534 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4535 /* The interrupt is not ours */ 4536 return (DDI_INTR_UNCLAIMED); 4537 } 4538 4539 /* Loop for all the ports */ 4540 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4541 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4542 continue; 4543 } 4544 if (!((0x1 << port) & global_intr_status)) { 4545 continue; 4546 } 4547 4548 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4549 4550 /* Call ahci_port_intr */ 4551 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4552 } 4553 4554 return (DDI_INTR_CLAIMED); 4555 } 4556 4557 /* 4558 * For non-queued commands, when the corresponding bit in the PxCI register 4559 * is cleared, it means the command is completed successfully. And according 4560 * to the HBA state machine, there are three conditions which possibly will 4561 * try to clear the PxCI register bit. 4562 * 1. Receive one D2H Register FIS which is with 'I' bit set 4563 * 2. Update PIO Setup FIS 4564 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4565 * 4566 * Process completed non-queued commands when the interrupt status bit - 4567 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4568 * 4569 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4570 * with the 'I' bit set. And the following commands will send thus 4571 * FIS with 'I' bit set upon the successful completion: 4572 * 1. Non-data commands 4573 * 2. DMA data-in command 4574 * 3. DMA data-out command 4575 * 4. PIO data-out command 4576 * 5. PACKET non-data commands 4577 * 6. PACKET PIO data-in command 4578 * 7. PACKET PIO data-out command 4579 * 8. PACKET DMA data-in command 4580 * 9. PACKET DMA data-out command 4581 * 4582 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4583 * with the 'I' bit set. And the following commands will send this 4584 * FIS upon the successful completion: 4585 * 1. PIO data-in command 4586 */ 4587 static int 4588 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4589 ahci_port_t *ahci_portp, uint8_t port) 4590 { 4591 uint32_t port_cmd_issue = 0; 4592 uint32_t finished_tags; 4593 int finished_slot; 4594 sata_pkt_t *satapkt; 4595 ahci_fis_d2h_register_t *rcvd_fisp; 4596 #if AHCI_DEBUG 4597 ahci_cmd_header_t *cmd_header; 4598 uint32_t cmd_dmacount; 4599 #endif 4600 4601 mutex_enter(&ahci_portp->ahciport_mutex); 4602 4603 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4604 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4605 /* 4606 * Spurious interrupt. Nothing to be done. 4607 */ 4608 mutex_exit(&ahci_portp->ahciport_mutex); 4609 return (AHCI_SUCCESS); 4610 } 4611 4612 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4613 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4614 4615 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4616 /* Slot 0 is always used during error recovery */ 4617 finished_tags = 0x1 & ~port_cmd_issue; 4618 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 4619 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4620 "retrieval is finished, and finished_tags = 0x%x", 4621 port, finished_tags); 4622 } else { 4623 finished_tags = ahci_portp->ahciport_pending_tags & 4624 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4625 } 4626 4627 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4628 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4629 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4630 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4631 finished_tags); 4632 4633 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4634 (finished_tags == 0x1)) { 4635 satapkt = ahci_portp->ahciport_err_retri_pkt; 4636 ASSERT(satapkt != NULL); 4637 4638 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4639 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4640 "with SATA_PKT_COMPLETED", (void *)satapkt); 4641 4642 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4643 goto out; 4644 } 4645 4646 while (finished_tags) { 4647 finished_slot = ddi_ffs(finished_tags) - 1; 4648 if (finished_slot == -1) { 4649 goto out; 4650 } 4651 4652 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4653 ASSERT(satapkt != NULL); 4654 #if AHCI_DEBUG 4655 /* 4656 * For non-native queued commands, the PRD byte count field 4657 * shall contain an accurate count of the number of bytes 4658 * transferred for the command before the PxCI bit is cleared 4659 * to '0' for the command. 4660 * 4661 * The purpose of this field is to let software know how many 4662 * bytes transferred for a given operation in order to 4663 * determine if underflow occurred. When issuing native command 4664 * queuing commands, this field should not be used and is not 4665 * required to be valid since in this case underflow is always 4666 * illegal. 4667 * 4668 * For data reads, the HBA will update its PRD byte count with 4669 * the total number of bytes received from the last FIS, and 4670 * may be able to continue normally. For data writes, the 4671 * device will detect an error, and HBA most likely will get 4672 * a fatal error. 4673 * 4674 * Therefore, here just put code to debug part. And please 4675 * refer to the comment above ahci_intr_fatal_error for the 4676 * definition of underflow error. 4677 */ 4678 cmd_dmacount = 4679 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4680 if (cmd_dmacount) { 4681 cmd_header = 4682 &ahci_portp->ahciport_cmd_list[finished_slot]; 4683 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4684 "ahci_intr_cmd_cmplt: port %d, " 4685 "PRD Byte Count = 0x%x, " 4686 "ahciport_prd_bytecounts = 0x%x", port, 4687 cmd_header->ahcich_prd_byte_count, 4688 cmd_dmacount); 4689 4690 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4691 AHCIDBG1(AHCIDBG_UNDERFLOW, ahci_ctlp, 4692 "ahci_intr_cmd_cmplt: port %d, " 4693 "an underflow occurred", port); 4694 } 4695 } 4696 #endif 4697 4698 /* 4699 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4700 * feature, sata_special_regs flag will be set, and the 4701 * driver should copy the status and the other corresponding 4702 * register values in the D2H Register FIS received (It's 4703 * working on Non-data protocol) from the device back to 4704 * the sata_cmd. 4705 * 4706 * For every AHCI port, there is only one Received FIS 4707 * structure, which contains the FISes received from the 4708 * device, So we're trying to copy the content of D2H 4709 * Register FIS in the Received FIS structure back to 4710 * the sata_cmd. 4711 */ 4712 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4713 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4714 ahcirf_d2h_register_fis); 4715 satapkt->satapkt_cmd.satacmd_status_reg = 4716 GET_RFIS_STATUS(rcvd_fisp); 4717 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4718 } 4719 4720 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4721 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4722 "with SATA_PKT_COMPLETED", (void *)satapkt); 4723 4724 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4725 CLEAR_BIT(finished_tags, finished_slot); 4726 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4727 4728 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4729 } 4730 out: 4731 AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp, 4732 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4733 ahci_portp->ahciport_pending_tags); 4734 4735 mutex_exit(&ahci_portp->ahciport_mutex); 4736 4737 return (AHCI_SUCCESS); 4738 } 4739 4740 /* 4741 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4742 * with the 'I' bit set and has been copied into system memory. It will 4743 * be sent under the following situations: 4744 * 4745 * 1. NCQ command is completed 4746 * 2. Asynchronous notification 4747 * 4748 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4749 * via the Set Device Bits FIS. When such event is generated, the software 4750 * needs to read PxSACT register and compares the current value to the 4751 * list of commands previously issue by software. ahciport_pending_ncq_tags 4752 * keeps the tags of previously issued commands. 4753 * 4754 * Asynchronous Notification is a feature in SATA II, which allows an 4755 * ATAPI device to send a signal to the host when media is inserted or 4756 * removed and avoids polling the device for media changes. The signal 4757 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4758 * set to '1'. At the moment, it's not supported yet. 4759 */ 4760 static int 4761 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4762 ahci_port_t *ahci_portp, uint8_t port) 4763 { 4764 uint32_t port_sactive; 4765 uint32_t port_cmd_issue; 4766 uint32_t issued_tags; 4767 int issued_slot; 4768 uint32_t finished_tags; 4769 int finished_slot; 4770 sata_pkt_t *satapkt; 4771 4772 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4773 "ahci_intr_set_device_bits enter: port %d", port); 4774 4775 mutex_enter(&ahci_portp->ahciport_mutex); 4776 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4777 mutex_exit(&ahci_portp->ahciport_mutex); 4778 return (AHCI_SUCCESS); 4779 } 4780 4781 /* 4782 * First the handler got which commands are finished by checking 4783 * PxSACT register 4784 */ 4785 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4786 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4787 4788 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4789 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4790 4791 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4792 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4793 "port_sactive = 0x%x", port, 4794 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4795 4796 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4797 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4798 4799 /* 4800 * For NCQ commands, the software can determine which command has 4801 * already been transmitted to the device by checking PxCI register. 4802 */ 4803 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4804 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4805 4806 issued_tags = ahci_portp->ahciport_pending_tags & 4807 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4808 4809 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4810 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4811 "port_cmd_issue = 0x%x", port, 4812 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4813 4814 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4815 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4816 4817 /* 4818 * Clear ahciport_pending_tags bit when the corresponding command 4819 * is already sent down to the device. 4820 */ 4821 while (issued_tags) { 4822 issued_slot = ddi_ffs(issued_tags) - 1; 4823 if (issued_slot == -1) { 4824 goto next; 4825 } 4826 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 4827 CLEAR_BIT(issued_tags, issued_slot); 4828 } 4829 4830 next: 4831 while (finished_tags) { 4832 finished_slot = ddi_ffs(finished_tags) - 1; 4833 if (finished_slot == -1) { 4834 goto out; 4835 } 4836 4837 /* The command is certainly transmitted to the device */ 4838 ASSERT(!(ahci_portp->ahciport_pending_tags & 4839 (0x1 << finished_slot))); 4840 4841 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4842 ASSERT(satapkt != NULL); 4843 4844 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4845 "ahci_intr_set_device_bits: sending up pkt 0x%p " 4846 "with SATA_PKT_COMPLETED", (void *)satapkt); 4847 4848 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 4849 CLEAR_BIT(finished_tags, finished_slot); 4850 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4851 4852 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4853 } 4854 out: 4855 AHCIDBG3(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 4856 "ahci_intr_set_device_bits: port %d " 4857 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 4858 port, ahci_portp->ahciport_pending_ncq_tags, 4859 ahci_portp->ahciport_pending_tags); 4860 4861 mutex_exit(&ahci_portp->ahciport_mutex); 4862 4863 return (AHCI_SUCCESS); 4864 } 4865 4866 /* 4867 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 4868 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 4869 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 4870 * indicates a COMINIT signal was received. 4871 * 4872 * Hot plug insertion is detected by reception of a COMINIT signal from the 4873 * device. On reception of unsolicited COMINIT, the HBA shall generate a 4874 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 4875 * begin the normal communication negotiation sequence as outlined in the 4876 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 4877 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 4878 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 4879 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 4880 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 4881 * to handle hot plug insertion. In this interrupt handler, just do nothing 4882 * but print some log message and clear the bit. 4883 */ 4884 static int 4885 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 4886 ahci_port_t *ahci_portp, uint8_t port) 4887 { 4888 #if AHCI_DEBUG 4889 uint32_t port_serror; 4890 #endif 4891 4892 mutex_enter(&ahci_portp->ahciport_mutex); 4893 4894 #if AHCI_DEBUG 4895 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4896 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 4897 4898 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4899 "ahci_intr_port_connect_change: port %d, " 4900 "port_serror = 0x%x", port, port_serror); 4901 #endif 4902 4903 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 4904 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4905 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 4906 SERROR_EXCHANGED_ERR); 4907 4908 mutex_exit(&ahci_portp->ahciport_mutex); 4909 4910 return (AHCI_SUCCESS); 4911 } 4912 4913 /* 4914 * Hot Plug Operation for platforms that support Mechanical Presence 4915 * Switches. 4916 * 4917 * When set, it indicates that a mechanical presence switch attached to this 4918 * port has been opened or closed, which may lead to a change in the connection 4919 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 4920 * are set to '1'. 4921 * 4922 * At the moment, this interrupt is not needed and disabled and we just log 4923 * the debug message. 4924 */ 4925 static int 4926 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 4927 ahci_port_t *ahci_portp, uint8_t port) 4928 { 4929 uint32_t cap_status, port_cmd_status; 4930 4931 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4932 "ahci_intr_device_mechanical_presence_status enter, " 4933 "port %d", port); 4934 4935 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4936 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4937 4938 mutex_enter(&ahci_portp->ahciport_mutex); 4939 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4940 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4941 4942 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 4943 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 4944 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4945 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 4946 "the interrupt: cap_status = 0x%x, " 4947 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 4948 mutex_exit(&ahci_portp->ahciport_mutex); 4949 4950 return (AHCI_SUCCESS); 4951 } 4952 4953 #if AHCI_DEBUG 4954 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 4955 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4956 "The mechanical presence switch is open: " 4957 "port %d, port_cmd_status = 0x%x", 4958 port, port_cmd_status); 4959 } else { 4960 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4961 "The mechanical presence switch is close: " 4962 "port %d, port_cmd_status = 0x%x", 4963 port, port_cmd_status); 4964 } 4965 #endif 4966 4967 mutex_exit(&ahci_portp->ahciport_mutex); 4968 4969 return (AHCI_SUCCESS); 4970 } 4971 4972 /* 4973 * Native Hot Plug Support. 4974 * 4975 * When set, it indicates that the internal PHYRDY signal changed state. 4976 * This bit reflects the state of PxSERR.DIAG.N. 4977 * 4978 * There are three kinds of conditions to generate this interrupt event: 4979 * 1. a device is inserted 4980 * 2. a device is disconnected 4981 * 3. when the link enters/exits a Partial or Slumber interface power 4982 * management state 4983 * 4984 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 4985 * bit may be set due to the link entering the Partial or Slumber power 4986 * management state, rather than due to a hot plug insertion or removal 4987 * event. So far, the interface power management is disabled, so the 4988 * driver can reliably get removal detection notification via the 4989 * PxSERR.DIAG.N bit. 4990 */ 4991 static int 4992 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 4993 ahci_port_t *ahci_portp, uint8_t port) 4994 { 4995 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 4996 sata_device_t sdevice; 4997 int dev_exists_now = 0; 4998 int dev_existed_previously = 0; 4999 5000 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5001 "ahci_intr_phyrdy_change enter, port %d", port); 5002 5003 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 5004 mutex_enter(&ahci_portp->ahciport_mutex); 5005 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5006 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5007 SERROR_PHY_RDY_CHG); 5008 mutex_exit(&ahci_portp->ahciport_mutex); 5009 5010 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5011 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 5012 (ahci_portp == NULL)) { 5013 /* The whole controller setup is not yet done. */ 5014 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5015 return (AHCI_SUCCESS); 5016 } 5017 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5018 5019 mutex_enter(&ahci_portp->ahciport_mutex); 5020 5021 /* SStatus tells the presence of device. */ 5022 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5023 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5024 5025 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5026 dev_exists_now = 1; 5027 } 5028 5029 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5030 dev_existed_previously = 1; 5031 } 5032 5033 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5034 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5035 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5036 "ahci_intr_phyrdy_change: port %d " 5037 "AHCI_PORT_FLAG_NODEV is cleared", port); 5038 if (dev_exists_now == 0) 5039 dev_existed_previously = 1; 5040 } 5041 5042 bzero((void *)&sdevice, sizeof (sata_device_t)); 5043 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5044 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5045 sdevice.satadev_addr.pmport = 0; 5046 sdevice.satadev_state = SATA_PSTATE_PWRON; 5047 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5048 5049 if (dev_exists_now) { 5050 if (dev_existed_previously) { 5051 /* Things are fine now. The loss was temporary. */ 5052 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5053 "ahci_intr_phyrdy_change port %d " 5054 "device link lost/established", port); 5055 5056 mutex_exit(&ahci_portp->ahciport_mutex); 5057 sata_hba_event_notify( 5058 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5059 &sdevice, 5060 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5061 mutex_enter(&ahci_portp->ahciport_mutex); 5062 5063 } else { 5064 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5065 "ahci_intr_phyrdy_change: port %d " 5066 "device link established", port); 5067 5068 /* A new device has been detected. */ 5069 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5070 5071 /* Try to start the port */ 5072 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5073 != AHCI_SUCCESS) { 5074 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5075 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5076 "ahci_intr_phyrdy_change: port %d failed " 5077 "at start port", port); 5078 } 5079 5080 /* Clear the max queue depth for inserted device */ 5081 ahci_portp->ahciport_max_ncq_tags = 0; 5082 5083 mutex_exit(&ahci_portp->ahciport_mutex); 5084 sata_hba_event_notify( 5085 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5086 &sdevice, 5087 SATA_EVNT_LINK_ESTABLISHED); 5088 mutex_enter(&ahci_portp->ahciport_mutex); 5089 5090 } 5091 } else { /* No device exists now */ 5092 5093 if (dev_existed_previously) { 5094 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5095 "ahci_intr_phyrdy_change: port %d " 5096 "device link lost", port); 5097 5098 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5099 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5100 ahci_portp, port); 5101 5102 /* An existing device is lost. */ 5103 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5104 5105 mutex_exit(&ahci_portp->ahciport_mutex); 5106 sata_hba_event_notify( 5107 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5108 &sdevice, 5109 SATA_EVNT_LINK_LOST); 5110 mutex_enter(&ahci_portp->ahciport_mutex); 5111 } 5112 } 5113 5114 mutex_exit(&ahci_portp->ahciport_mutex); 5115 5116 return (AHCI_SUCCESS); 5117 } 5118 5119 /* 5120 * PxIS.UFS - Unknown FIS Error 5121 * 5122 * This interrupt event means an unknown FIS was received and has been 5123 * copied into system memory. An unknown FIS is not considered an illegal 5124 * FIS, unless the length received is more than 64 bytes. If an unknown 5125 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5126 * normal operation. If the unknown FIS is more than 64 bytes, then it 5127 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5128 * a fatal error. 5129 * 5130 * PxIS.OFS - Overflow Error 5131 * 5132 * Command list overflow is defined as software building a command table 5133 * that has fewer total bytes than the transaction given to the device. 5134 * On device writes, the HBA will run out of data, and on reads, there 5135 * will be no room to put the data. 5136 * 5137 * For an overflow on data read, either PIO or DMA, the HBA will set 5138 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5139 * non-fatal error when the HBA can continues. Sometimes, it will cause 5140 * a fatal error and need the software to do something. 5141 * 5142 * For an overflow on data write, setting PxIS.OFS is optional for both 5143 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5144 * software to clean up from this serious error. 5145 * 5146 * PxIS.INFS - Interface Non-Fatal Error 5147 * 5148 * This interrupt event indicates that the HBA encountered an error on 5149 * the Serial ATA interface but was able to continue operation. The kind 5150 * of error usually occurred during a non-Data FIS, and under this condition 5151 * the FIS will be re-transmitted by HBA automatically. 5152 * 5153 * When the FMA is implemented, there should be a stat structure to 5154 * record how many every kind of error happens. 5155 */ 5156 static int 5157 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5158 uint8_t port, uint32_t intr_status) 5159 { 5160 uint32_t port_serror; 5161 #if AHCI_DEBUG 5162 uint32_t port_cmd_status; 5163 uint32_t port_cmd_issue; 5164 uint32_t port_sactive; 5165 int current_slot; 5166 uint32_t current_tags; 5167 sata_pkt_t *satapkt; 5168 ahci_cmd_header_t *cmd_header; 5169 uint32_t cmd_dmacount; 5170 #endif 5171 5172 mutex_enter(&ahci_portp->ahciport_mutex); 5173 5174 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5175 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5176 5177 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5178 "ahci_intr_non_fatal_error: port %d, " 5179 "port_serror = 0x%x", port, port_serror); 5180 5181 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5182 5183 if (intr_status & AHCI_INTR_STATUS_UFS) { 5184 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5185 "ahci port %d has unknown FIS error", port); 5186 5187 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5188 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5189 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5190 SERROR_FIS_TYPE); 5191 } 5192 5193 #if AHCI_DEBUG 5194 if (intr_status & AHCI_INTR_STATUS_OFS) { 5195 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5196 "ahci port %d has overflow error", port); 5197 } 5198 5199 if (intr_status & AHCI_INTR_STATUS_INFS) { 5200 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5201 "ahci port %d has interface non fatal error", port); 5202 } 5203 5204 /* 5205 * Record the error occurred command's slot. 5206 */ 5207 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5208 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5209 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5210 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5211 5212 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5213 AHCI_CMD_STATUS_CCS_SHIFT; 5214 5215 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5216 satapkt = ahci_portp->ahciport_err_retri_pkt; 5217 ASSERT(satapkt != NULL); 5218 ASSERT(current_slot == 0); 5219 } else { 5220 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5221 } 5222 5223 if (satapkt != NULL) { 5224 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5225 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5226 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5227 satapkt->satapkt_cmd.satacmd_cmd_reg); 5228 5229 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5230 "ahci_intr_non_fatal_error: port %d, " 5231 "satapkt 0x%p is being processed when error occurs", 5232 port, (void *)satapkt); 5233 5234 /* 5235 * PRD Byte Count field of command header is not 5236 * required to reflect the total number of bytes 5237 * transferred when an overflow occurs, so here 5238 * just log the value. 5239 */ 5240 cmd_dmacount = 5241 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5242 if (cmd_dmacount) { 5243 cmd_header = &ahci_portp-> 5244 ahciport_cmd_list[current_slot]; 5245 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5246 "ahci_intr_non_fatal_error: port %d, " 5247 "PRD Byte Count = 0x%x, " 5248 "ahciport_prd_bytecounts = 0x%x", port, 5249 cmd_header->ahcich_prd_byte_count, 5250 cmd_dmacount); 5251 } 5252 } 5253 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5254 /* 5255 * For queued command, list those command which have already 5256 * been transmitted to the device and still not completed. 5257 */ 5258 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5259 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5260 5261 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5262 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5263 5264 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5265 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5266 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5267 ahci_portp->ahciport_pending_ncq_tags, 5268 port_sactive, port_cmd_issue); 5269 5270 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5271 port_sactive & ~port_cmd_issue & 5272 AHCI_NCQ_SLOT_MASK(ahci_portp); 5273 5274 while (current_tags) { 5275 current_slot = ddi_ffs(current_tags) - 1; 5276 if (current_slot == -1) { 5277 goto out; 5278 } 5279 5280 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5281 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5282 ahci_ctlp, "ahci_intr_non_fatal_error: " 5283 "port %d, satapkt 0x%p is outstanding when " 5284 "error occurs", port, (void *)satapkt); 5285 } 5286 } 5287 out: 5288 #endif 5289 mutex_exit(&ahci_portp->ahciport_mutex); 5290 5291 return (AHCI_SUCCESS); 5292 } 5293 5294 /* 5295 * According to the AHCI spec, the error types include system memory 5296 * errors, interface errors, port multiplier errors, device errors, 5297 * command list overflow, command list underflow, native command 5298 * queuing tag errors and pio data transfer errors. 5299 * 5300 * System memory errors such as target abort, master abort, and parity 5301 * may cause the host to stop, and they are serious errors and needed 5302 * to be recovered with software intervention. When system software 5303 * has given a pointer to the HBA that doesn't exist in physical memory, 5304 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5305 * data error such as CRC or parity occurs, the HBA aborts the transfer 5306 * (if necessary) and PxIS.HBDS will be set. 5307 * 5308 * Interface errors are errors that occur due to electrical issues on 5309 * the interface, or protocol miscommunication between the device and 5310 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5311 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5312 * causes PxIS.IFS/PxIS.INFS to be set are 5313 * 1. in PxSERR.ERR, P bit is set to '1' 5314 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5315 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5316 * If the error occurred during a non-data FIS, the FIS must be 5317 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5318 * the error occurred during a data FIS, the transfer will stop, so 5319 * the error is fatal and PxIS.IFS is set. 5320 * 5321 * When a FIS arrives that updates the taskfile, the HBA checks to see 5322 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5323 * stops processing any more commands. 5324 * 5325 * Command list overflow is defined as software building a command table 5326 * that has fewer total bytes than the transaction given to the device. 5327 * On device writes, the HBA will run out of data, and on reads, there 5328 * will be no room to put the data. For an overflow on data read, either 5329 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5330 * For an overflow on data write, setting PxIS.OFS is optional for both 5331 * DMA and PIO, and a COMRESET is required by software to clean up from 5332 * this serious error. 5333 * 5334 * Command list underflow is defined as software building a command 5335 * table that has more total bytes than the transaction given to the 5336 * device. For data writes, both PIO and DMA, the device will detect 5337 * an error and end the transfer. And these errors are most likely going 5338 * to be fatal errors that will cause the port to be restarted. For 5339 * data reads, the HBA updates its PRD byte count, and may be 5340 * able to continue normally, but is not required to. And The HBA is 5341 * not required to detect underflow conditions for native command 5342 * queuing command. 5343 * 5344 * The HBA does not actively check incoming DMA Setup FISes to ensure 5345 * that the PxSACT register bit for that slot is set. Existing error 5346 * mechanisms, such as host bus failure, or bad protocol, are used to 5347 * recover from this case. 5348 * 5349 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5350 * DATA FIS must be an integral number of Dwords. If the HBA receives 5351 * a request which is not an integral number of Dwords, the HBA 5352 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5353 * software restarts the port. And the HBA ensures that the size 5354 * of the DATA FIS received during a PIO command matches the size in 5355 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5356 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5357 * stop running until software restarts the port. 5358 */ 5359 /* 5360 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5361 * 5362 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5363 * interface which caused the transfer to stop. 5364 * 5365 * PxIS.HBDS indicates that the hba encountered a data error 5366 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5367 * 5368 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5369 * cannot recover from, such as a bad software pointer. 5370 * 5371 * PxIS.TFES is set whenever the status register is updated by the device 5372 * and the error bit (bit 0) is set. 5373 */ 5374 static int 5375 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5376 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5377 { 5378 uint32_t port_cmd_status; 5379 uint32_t port_serror; 5380 uint32_t task_file_status; 5381 int failed_slot; 5382 sata_pkt_t *spkt = NULL; 5383 uint8_t err_byte; 5384 ahci_event_arg_t *args; 5385 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5386 5387 mutex_enter(&ahci_portp->ahciport_mutex); 5388 5389 /* 5390 * ahci_intr_phyrdy_change() may have rendered it to 5391 * SATA_DTYPE_NONE. 5392 */ 5393 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5394 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5395 "ahci_intr_fatal_error: port %d no device attached, " 5396 "and just return without doing anything", port); 5397 goto out0; 5398 } 5399 5400 if (intr_status & AHCI_INTR_STATUS_TFES) { 5401 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5402 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5403 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5404 "ahci_intr_fatal_error: port %d " 5405 "task_file_status = 0x%x", port, task_file_status); 5406 } 5407 5408 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5409 /* 5410 * Read PxCMD.CCS to determine the slot that the HBA 5411 * was processing when the error occurred. 5412 */ 5413 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5414 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5415 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5416 AHCI_CMD_STATUS_CCS_SHIFT; 5417 5418 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5419 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5420 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5421 "fatal error occurred for port %d", spkt, port); 5422 5423 if (intr_status & AHCI_INTR_STATUS_TFES) { 5424 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5425 >> AHCI_TFD_ERR_SHIFT; 5426 5427 /* 5428 * Won't emit the error message if it is an IDENTIFY 5429 * DEVICE command sent to an ATAPI device. 5430 */ 5431 if ((spkt != NULL) && 5432 (spkt->satapkt_cmd.satacmd_cmd_reg == 5433 SATAC_ID_DEVICE) && 5434 (err_byte == SATA_ERROR_ABORT)) 5435 goto out1; 5436 5437 /* 5438 * Won't emit the error message if it is an ATAPI PACKET 5439 * command 5440 */ 5441 if ((spkt != NULL) && 5442 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5443 goto out1; 5444 } 5445 } 5446 5447 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5448 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5449 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5450 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5451 out1: 5452 /* Prepare the argument for the taskq */ 5453 args = ahci_portp->ahciport_event_args; 5454 args->ahciea_ctlp = (void *)ahci_ctlp; 5455 args->ahciea_portp = (void *)ahci_portp; 5456 args->ahciea_event = intr_status; 5457 5458 /* Start the taskq to handle error recovery */ 5459 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 5460 ahci_events_handler, 5461 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5462 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5463 "failed", instance); 5464 } 5465 out0: 5466 mutex_exit(&ahci_portp->ahciport_mutex); 5467 5468 return (AHCI_SUCCESS); 5469 } 5470 5471 /* 5472 * Hot Plug Operation for platforms that support Cold Presence Detect. 5473 * 5474 * When set, a device status has changed as detected by the cold presence 5475 * detect logic. This bit can either be set due to a non-connected port 5476 * receiving a device, or a connected port having its device removed. 5477 * This bit is only valid if the port supports cold presence detect as 5478 * indicated by PxCMD.CPD set to '1'. 5479 * 5480 * At the moment, this interrupt is not needed and disabled and we just 5481 * log the debug message. 5482 */ 5483 static int 5484 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5485 ahci_port_t *ahci_portp, uint8_t port) 5486 { 5487 uint32_t port_cmd_status; 5488 sata_device_t sdevice; 5489 5490 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5491 "ahci_intr_cold_port_detect enter, port %d", port); 5492 5493 mutex_enter(&ahci_portp->ahciport_mutex); 5494 5495 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5496 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5497 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5498 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5499 "port %d does not support cold presence detect, so " 5500 "we just ignore this interrupt", port); 5501 mutex_exit(&ahci_portp->ahciport_mutex); 5502 return (AHCI_SUCCESS); 5503 } 5504 5505 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5506 "port %d device status has changed", port); 5507 5508 bzero((void *)&sdevice, sizeof (sata_device_t)); 5509 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5510 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5511 sdevice.satadev_addr.pmport = 0; 5512 sdevice.satadev_state = SATA_PSTATE_PWRON; 5513 5514 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5515 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5516 "port %d: a device is hot plugged", port); 5517 mutex_exit(&ahci_portp->ahciport_mutex); 5518 sata_hba_event_notify( 5519 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5520 &sdevice, 5521 SATA_EVNT_DEVICE_ATTACHED); 5522 mutex_enter(&ahci_portp->ahciport_mutex); 5523 5524 } else { 5525 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5526 "port %d: a device is hot unplugged", port); 5527 mutex_exit(&ahci_portp->ahciport_mutex); 5528 sata_hba_event_notify( 5529 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5530 &sdevice, 5531 SATA_EVNT_DEVICE_DETACHED); 5532 mutex_enter(&ahci_portp->ahciport_mutex); 5533 } 5534 5535 mutex_exit(&ahci_portp->ahciport_mutex); 5536 5537 return (AHCI_SUCCESS); 5538 } 5539 5540 /* 5541 * Enable the interrupts for a particular port. 5542 * 5543 * WARNING!!! ahciport_mutex should be acquired before the function 5544 * is called. 5545 */ 5546 static void 5547 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5548 { 5549 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5550 "ahci_enable_port_intrs enter, port %d", port); 5551 5552 /* 5553 * Clear port interrupt status before enabling interrupt 5554 */ 5555 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5556 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5557 AHCI_PORT_INTR_MASK); 5558 5559 /* 5560 * Clear the pending bit from IS.IPS 5561 */ 5562 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5563 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5564 5565 /* 5566 * Enable the following interrupts: 5567 * Device to Host Register FIS Interrupt (DHRS) 5568 * PIO Setup FIS Interrupt (PSS) 5569 * Set Device Bits Interrupt (SDBS) 5570 * Unknown FIS Interrupt (UFS) 5571 * Port Connect Change Status (PCS) 5572 * PhyRdy Change Status (PRCS) 5573 * Overflow Status (OFS) 5574 * Interface Non-fatal Error Status (INFS) 5575 * Interface Fatal Error Status (IFS) 5576 * Host Bus Data Error Status (HBDS) 5577 * Host Bus Fatal Error Status (HBFS) 5578 * Task File Error Status (TFES) 5579 */ 5580 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5581 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5582 (AHCI_INTR_STATUS_DHRS | 5583 AHCI_INTR_STATUS_PSS | 5584 AHCI_INTR_STATUS_SDBS | 5585 AHCI_INTR_STATUS_UFS | 5586 AHCI_INTR_STATUS_DPS | 5587 AHCI_INTR_STATUS_PCS | 5588 AHCI_INTR_STATUS_PRCS | 5589 AHCI_INTR_STATUS_OFS | 5590 AHCI_INTR_STATUS_INFS | 5591 AHCI_INTR_STATUS_IFS | 5592 AHCI_INTR_STATUS_HBDS | 5593 AHCI_INTR_STATUS_HBFS | 5594 AHCI_INTR_STATUS_TFES)); 5595 } 5596 5597 /* 5598 * Enable interrupts for all the ports. 5599 * 5600 * WARNING!!! ahcictl_mutex should be acquired before the function 5601 * is called. 5602 */ 5603 static void 5604 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5605 { 5606 uint32_t ghc_control; 5607 5608 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter"); 5609 5610 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5611 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5612 5613 ghc_control |= AHCI_HBA_GHC_IE; 5614 5615 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5616 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5617 } 5618 5619 /* 5620 * Disable interrupts for a particular port. 5621 * 5622 * WARNING!!! ahciport_mutex should be acquired before the function 5623 * is called. 5624 */ 5625 static void 5626 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5627 { 5628 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5629 "ahci_disable_port_intrs enter, port %d", port); 5630 5631 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5632 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5633 } 5634 5635 /* 5636 * Disable interrupts for the whole HBA. 5637 * 5638 * The global bit is cleared, then all interrupt sources from all 5639 * ports are disabled. 5640 * 5641 * WARNING!!! ahcictl_mutex should be acquired before the function 5642 * is called. 5643 */ 5644 static void 5645 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5646 { 5647 uint32_t ghc_control; 5648 5649 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter"); 5650 5651 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5652 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5653 5654 ghc_control &= ~ AHCI_HBA_GHC_IE; 5655 5656 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5657 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5658 } 5659 5660 /* 5661 * Handle INTx and legacy interrupts. 5662 */ 5663 static int 5664 ahci_add_legacy_intrs(ahci_ctl_t *ahci_ctlp) 5665 { 5666 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5667 int actual, count = 0; 5668 int x, y, rc, inum = 0; 5669 5670 AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp, 5671 "ahci_add_legacy_intrs enter"); 5672 5673 /* get number of interrupts. */ 5674 rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_FIXED, &count); 5675 if ((rc != DDI_SUCCESS) || (count == 0)) { 5676 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5677 "ddi_intr_get_nintrs() failed, " 5678 "rc %d count %d\n", rc, count); 5679 return (DDI_FAILURE); 5680 } 5681 5682 /* Allocate an array of interrupt handles. */ 5683 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5684 ahci_ctlp->ahcictl_intr_htable = 5685 kmem_zalloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5686 5687 /* call ddi_intr_alloc(). */ 5688 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5689 DDI_INTR_TYPE_FIXED, 5690 inum, count, &actual, DDI_INTR_ALLOC_STRICT); 5691 5692 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5693 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5694 "ddi_intr_alloc() failed, rc %d\n", rc); 5695 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5696 ahci_ctlp->ahcictl_intr_size); 5697 return (DDI_FAILURE); 5698 } 5699 5700 if (actual < count) { 5701 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5702 "Requested: %d, Received: %d", count, actual); 5703 5704 for (x = 0; x < actual; x++) { 5705 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5706 } 5707 5708 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5709 ahci_ctlp->ahcictl_intr_size); 5710 return (DDI_FAILURE); 5711 } 5712 5713 ahci_ctlp->ahcictl_intr_cnt = actual; 5714 5715 /* Get intr priority. */ 5716 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5717 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5718 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5719 "ddi_intr_get_pri() failed"); 5720 5721 for (x = 0; x < actual; x++) { 5722 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5723 } 5724 5725 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5726 ahci_ctlp->ahcictl_intr_size); 5727 return (DDI_FAILURE); 5728 } 5729 5730 /* Test for high level interrupt. */ 5731 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5732 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5733 "ahci_add_legacy_intrs: Hi level intr not supported"); 5734 5735 for (x = 0; x < actual; x++) { 5736 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5737 } 5738 5739 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5740 sizeof (ddi_intr_handle_t)); 5741 5742 return (DDI_FAILURE); 5743 } 5744 5745 /* Call ddi_intr_add_handler(). */ 5746 for (x = 0; x < actual; x++) { 5747 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x], 5748 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5749 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5750 "ddi_intr_add_handler() failed"); 5751 5752 for (y = 0; y < actual; y++) { 5753 (void) ddi_intr_free( 5754 ahci_ctlp->ahcictl_intr_htable[y]); 5755 } 5756 5757 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5758 ahci_ctlp->ahcictl_intr_size); 5759 return (DDI_FAILURE); 5760 } 5761 } 5762 5763 /* Call ddi_intr_enable() for legacy interrupts. */ 5764 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5765 (void) ddi_intr_enable(ahci_ctlp->ahcictl_intr_htable[x]); 5766 } 5767 5768 return (DDI_SUCCESS); 5769 } 5770 5771 /* 5772 * Handle MSI interrupts. 5773 */ 5774 static int 5775 ahci_add_msi_intrs(ahci_ctl_t *ahci_ctlp) 5776 { 5777 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5778 int count, avail, actual; 5779 int x, y, rc, inum = 0; 5780 5781 AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp, 5782 "ahci_add_msi_intrs enter"); 5783 5784 /* get number of interrupts. */ 5785 rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &count); 5786 if ((rc != DDI_SUCCESS) || (count == 0)) { 5787 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5788 "ddi_intr_get_nintrs() failed, " 5789 "rc %d count %d\n", rc, count); 5790 return (DDI_FAILURE); 5791 } 5792 5793 /* get number of available interrupts. */ 5794 rc = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail); 5795 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5796 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5797 "ddi_intr_get_navail() failed, " 5798 "rc %d avail %d\n", rc, avail); 5799 return (DDI_FAILURE); 5800 } 5801 5802 #if AHCI_DEBUG 5803 if (avail < count) { 5804 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5805 "ddi_intr_get_nvail returned %d, navail() returned %d", 5806 count, avail); 5807 } 5808 #endif 5809 5810 /* Allocate an array of interrupt handles. */ 5811 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5812 ahci_ctlp->ahcictl_intr_htable = 5813 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5814 5815 /* call ddi_intr_alloc(). */ 5816 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5817 DDI_INTR_TYPE_MSI, inum, count, &actual, DDI_INTR_ALLOC_NORMAL); 5818 5819 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5820 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5821 "ddi_intr_alloc() failed, rc %d\n", rc); 5822 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5823 ahci_ctlp->ahcictl_intr_size); 5824 return (DDI_FAILURE); 5825 } 5826 5827 /* use interrupt count returned */ 5828 #if AHCI_DEBUG 5829 if (actual < count) { 5830 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5831 "Requested: %d, Received: %d", count, actual); 5832 } 5833 #endif 5834 5835 ahci_ctlp->ahcictl_intr_cnt = actual; 5836 5837 /* 5838 * Get priority for first msi, assume remaining are all the same. 5839 */ 5840 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5841 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5842 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5843 "ddi_intr_get_pri() failed"); 5844 5845 /* Free already allocated intr. */ 5846 for (y = 0; y < actual; y++) { 5847 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]); 5848 } 5849 5850 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5851 ahci_ctlp->ahcictl_intr_size); 5852 return (DDI_FAILURE); 5853 } 5854 5855 /* Test for high level interrupt. */ 5856 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5857 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5858 "ahci_add_msi_intrs: Hi level intr not supported"); 5859 5860 /* Free already allocated intr. */ 5861 for (y = 0; y < actual; y++) { 5862 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]); 5863 } 5864 5865 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5866 sizeof (ddi_intr_handle_t)); 5867 5868 return (DDI_FAILURE); 5869 } 5870 5871 /* Call ddi_intr_add_handler(). */ 5872 for (x = 0; x < actual; x++) { 5873 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x], 5874 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5875 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5876 "ddi_intr_add_handler() failed"); 5877 5878 /* Free already allocated intr. */ 5879 for (y = 0; y < actual; y++) { 5880 (void) ddi_intr_free( 5881 ahci_ctlp->ahcictl_intr_htable[y]); 5882 } 5883 5884 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5885 ahci_ctlp->ahcictl_intr_size); 5886 return (DDI_FAILURE); 5887 } 5888 } 5889 5890 5891 (void) ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 5892 &ahci_ctlp->ahcictl_intr_cap); 5893 5894 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 5895 /* Call ddi_intr_block_enable() for MSI. */ 5896 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 5897 ahci_ctlp->ahcictl_intr_cnt); 5898 } else { 5899 /* Call ddi_intr_enable() for MSI non block enable. */ 5900 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5901 (void) ddi_intr_enable( 5902 ahci_ctlp->ahcictl_intr_htable[x]); 5903 } 5904 } 5905 5906 return (DDI_SUCCESS); 5907 } 5908 5909 /* 5910 * Removes the registered interrupts irrespective of whether they 5911 * were legacy or MSI. 5912 * 5913 * WARNING!!! The controller interrupts must be disabled before calling 5914 * this routine. 5915 */ 5916 static void 5917 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 5918 { 5919 int x; 5920 5921 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered"); 5922 5923 /* Disable all interrupts. */ 5924 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 5925 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 5926 /* Call ddi_intr_block_disable(). */ 5927 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 5928 ahci_ctlp->ahcictl_intr_cnt); 5929 } else { 5930 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5931 (void) ddi_intr_disable( 5932 ahci_ctlp->ahcictl_intr_htable[x]); 5933 } 5934 } 5935 5936 /* Call ddi_intr_remove_handler(). */ 5937 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5938 (void) ddi_intr_remove_handler( 5939 ahci_ctlp->ahcictl_intr_htable[x]); 5940 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5941 } 5942 5943 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 5944 } 5945 5946 /* 5947 * This routine tries to put port into P:NotRunning state by clearing 5948 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 5949 * and PxCMD.CR to '0'. 5950 * 5951 * WARNING!!! ahciport_mutex should be acquired before the function 5952 * is called. 5953 */ 5954 static int 5955 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 5956 ahci_port_t *ahci_portp, uint8_t port) 5957 { 5958 uint32_t port_cmd_status; 5959 int loop_count; 5960 5961 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5962 "ahci_put_port_into_notrunning_state enter: port %d", port); 5963 5964 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5965 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5966 5967 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 5968 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5969 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5970 5971 /* Wait until PxCMD.CR is cleared */ 5972 loop_count = 0; 5973 do { 5974 port_cmd_status = 5975 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5976 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5977 5978 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 5979 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 5980 "clearing port %d CMD.CR timeout, " 5981 "port_cmd_status = 0x%x", port, 5982 port_cmd_status); 5983 /* 5984 * We are effectively timing out after 0.5 sec. 5985 * This value is specified in AHCI spec. 5986 */ 5987 break; 5988 } 5989 5990 /* Wait for 10 millisec */ 5991 drv_usecwait(10000); 5992 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 5993 5994 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 5995 5996 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 5997 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5998 "ahci_put_port_into_notrunning_state: failed to clear " 5999 "PxCMD.CR to '0' after loop count: %d, and " 6000 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6001 return (AHCI_FAILURE); 6002 } else { 6003 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 6004 "ahci_put_port_into_notrunning_state: succeeded to clear " 6005 "PxCMD.CR to '0' after loop count: %d, and " 6006 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6007 return (AHCI_SUCCESS); 6008 } 6009 } 6010 6011 /* 6012 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 6013 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 6014 * stable state, then set PxCMD.ST to '1' to start the port directly. 6015 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 6016 * COMRESET to the device to put it in an idle state. 6017 * 6018 * The fifth argument returns whether the port reset is involved during 6019 * the process. 6020 * 6021 * The routine will be called under six scenarios: 6022 * 1. Initialize the port 6023 * 2. To abort the packet(s) 6024 * 3. To reset the port 6025 * 4. To activate the port 6026 * 5. Fatal error recovery 6027 * 6. To abort the timeout packet(s) 6028 * 6029 * WARNING!!! ahciport_mutex should be acquired before the function 6030 * is called. And ahciport_mutex will be released before the reset 6031 * event is reported to sata module by calling sata_hba_event_notify, 6032 * and then be acquired again later. 6033 * 6034 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6035 * according to the spec, the clearance of three bits will also clear 6036 * three interrupt status bits. 6037 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6038 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6039 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6040 * 6041 * Among these three interrupt events, the driver needs to take care of 6042 * PxIS.PRCS, which is the hot plug event. When the driver found out 6043 * a device was unplugged, it will call the interrupt handler. 6044 */ 6045 static int 6046 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6047 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6048 { 6049 uint32_t port_sstatus; 6050 uint32_t task_file_status; 6051 sata_device_t sdevice; 6052 int rval; 6053 int dev_exists_begin = 0; 6054 int dev_exists_end = 0; 6055 6056 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6057 "ahci_restart_port_wait_till_ready: port %d enter", port); 6058 6059 if ((flag != AHCI_PORT_INIT) && 6060 (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)) 6061 dev_exists_begin = 1; 6062 6063 /* First clear PxCMD.ST */ 6064 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6065 port); 6066 if (rval != AHCI_SUCCESS) 6067 /* 6068 * If PxCMD.CR does not clear within a reasonable time, it 6069 * may assume the interface is in a hung condition and may 6070 * continue with issuing the port reset. 6071 */ 6072 goto reset; 6073 6074 /* Then clear PxSERR */ 6075 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6076 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6077 AHCI_SERROR_CLEAR_ALL); 6078 6079 /* Then get PxTFD */ 6080 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6081 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6082 6083 /* 6084 * Check whether the device is in a stable status, if yes, 6085 * then start the port directly. However for ahci_tran_dport_reset, 6086 * we may have to perform a port reset. 6087 */ 6088 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6089 !(flag & AHCI_PORT_RESET)) 6090 goto out; 6091 6092 reset: 6093 /* 6094 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6095 * a COMRESET to the device 6096 */ 6097 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6098 6099 if (reset_flag != NULL) 6100 *reset_flag = 1; 6101 6102 /* Indicate to the framework that a reset has happened. */ 6103 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6104 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6105 /* Set the reset in progress flag */ 6106 ahci_portp->ahciport_reset_in_progress = 1; 6107 6108 bzero((void *)&sdevice, sizeof (sata_device_t)); 6109 sdevice.satadev_addr.cport = 6110 ahci_ctlp->ahcictl_port_to_cport[port]; 6111 sdevice.satadev_addr.pmport = 0; 6112 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6113 6114 sdevice.satadev_state = SATA_DSTATE_RESET | 6115 SATA_DSTATE_PWR_ACTIVE; 6116 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6117 mutex_exit(&ahci_portp->ahciport_mutex); 6118 sata_hba_event_notify( 6119 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6120 &sdevice, 6121 SATA_EVNT_DEVICE_RESET); 6122 mutex_enter(&ahci_portp->ahciport_mutex); 6123 } 6124 6125 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 6126 "port %d sending event up: SATA_EVNT_RESET", port); 6127 } else { 6128 ahci_portp->ahciport_reset_in_progress = 0; 6129 } 6130 6131 out: 6132 /* Start the port if not in initialization phase */ 6133 if (flag & AHCI_PORT_INIT) 6134 return (rval); 6135 6136 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6137 6138 /* SStatus tells the presence of device. */ 6139 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6140 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6141 6142 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6143 dev_exists_end = 1; 6144 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6145 } 6146 6147 /* Check whether a hot plug event happened */ 6148 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6149 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6150 "ahci_restart_port_wait_till_ready: port %d " 6151 "device is removed", port); 6152 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6153 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6154 "ahci_restart_port_wait_till_ready: port %d " 6155 "AHCI_PORT_FLAG_NODEV flag is set", port); 6156 mutex_exit(&ahci_portp->ahciport_mutex); 6157 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6158 mutex_enter(&ahci_portp->ahciport_mutex); 6159 } 6160 6161 return (rval); 6162 } 6163 6164 /* 6165 * This routine may be called under four scenarios: 6166 * a) do the recovery from fatal error 6167 * b) or we need to timeout some commands 6168 * c) or we need to abort some commands 6169 * d) or we need reset device/port/controller 6170 * 6171 * In all these scenarios, we need to send any pending unfinished 6172 * commands up to sata framework. 6173 * 6174 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6175 */ 6176 static void 6177 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6178 ahci_port_t *ahci_portp, 6179 uint32_t slot_status, 6180 uint32_t failed_tags, 6181 uint32_t timeout_tags, 6182 uint32_t aborted_tags, 6183 uint32_t reset_tags) 6184 { 6185 uint32_t finished_tags = 0; 6186 uint32_t unfinished_tags = 0; 6187 int tmp_slot; 6188 sata_pkt_t *satapkt; 6189 int ncq_cmd_in_progress = 0; 6190 int err_retri_cmd_in_progress = 0; 6191 6192 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6193 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6194 ahci_portp->ahciport_port_num, slot_status); 6195 6196 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6197 "ahci_mop_commands: failed_tags: 0x%x, " 6198 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6199 "reset_tags: 0x%x", failed_tags, 6200 timeout_tags, aborted_tags, reset_tags); 6201 6202 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6203 finished_tags = ahci_portp->ahciport_pending_tags & 6204 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6205 6206 unfinished_tags = slot_status & 6207 AHCI_SLOT_MASK(ahci_ctlp) & 6208 ~failed_tags & 6209 ~aborted_tags & 6210 ~reset_tags & 6211 ~timeout_tags; 6212 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6213 ncq_cmd_in_progress = 1; 6214 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6215 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6216 6217 unfinished_tags = slot_status & 6218 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6219 ~failed_tags & 6220 ~aborted_tags & 6221 ~reset_tags & 6222 ~timeout_tags; 6223 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6224 6225 /* 6226 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6227 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6228 * complete successfully due to one of the following three 6229 * conditions: 6230 * 6231 * 1. Fatal error - failed_tags includes its slot 6232 * 2. Timed out - timeout_tags includes its slot 6233 * 3. Aborted when hot unplug - aborted_tags includes its 6234 * slot 6235 * 6236 * Please note that the command is always sent down in Slot 0 6237 */ 6238 err_retri_cmd_in_progress = 1; 6239 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6240 "ahci_mop_commands is called for port %d while " 6241 "REQUEST SENSE or READ LOG EXT for error retrieval " 6242 "is being executed slot_status = 0x%x", 6243 ahci_portp->ahciport_port_num, slot_status); 6244 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6245 ASSERT(slot_status == 0x1); 6246 } 6247 6248 /* Send up finished packets with SATA_PKT_COMPLETED */ 6249 while (finished_tags) { 6250 tmp_slot = ddi_ffs(finished_tags) - 1; 6251 if (tmp_slot == -1) { 6252 break; 6253 } 6254 6255 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6256 ASSERT(satapkt != NULL); 6257 6258 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6259 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6260 (void *)satapkt); 6261 6262 /* 6263 * Cannot fetch the return register content since the port 6264 * was restarted, so the corresponding tag will be set to 6265 * aborted tags. 6266 */ 6267 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6268 CLEAR_BIT(finished_tags, tmp_slot); 6269 aborted_tags |= tmp_slot; 6270 continue; 6271 } 6272 6273 if (ncq_cmd_in_progress) 6274 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6275 tmp_slot); 6276 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6277 CLEAR_BIT(finished_tags, tmp_slot); 6278 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6279 6280 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6281 } 6282 6283 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6284 while (failed_tags) { 6285 if (err_retri_cmd_in_progress) { 6286 satapkt = ahci_portp->ahciport_err_retri_pkt; 6287 ASSERT(satapkt != NULL); 6288 ASSERT(failed_tags == 0x1); 6289 6290 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6291 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6292 (void *)satapkt); 6293 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6294 break; 6295 } 6296 6297 tmp_slot = ddi_ffs(failed_tags) - 1; 6298 if (tmp_slot == -1) { 6299 break; 6300 } 6301 6302 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6303 ASSERT(satapkt != NULL); 6304 6305 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6306 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6307 (void *)satapkt); 6308 6309 if (ncq_cmd_in_progress) 6310 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6311 tmp_slot); 6312 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6313 CLEAR_BIT(failed_tags, tmp_slot); 6314 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6315 6316 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6317 } 6318 6319 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6320 while (timeout_tags) { 6321 if (err_retri_cmd_in_progress) { 6322 satapkt = ahci_portp->ahciport_err_retri_pkt; 6323 ASSERT(satapkt != NULL); 6324 ASSERT(timeout_tags == 0x1); 6325 6326 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6327 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6328 (void *)satapkt); 6329 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6330 break; 6331 } 6332 6333 tmp_slot = ddi_ffs(timeout_tags) - 1; 6334 if (tmp_slot == -1) { 6335 break; 6336 } 6337 6338 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6339 ASSERT(satapkt != NULL); 6340 6341 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6342 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6343 (void *)satapkt); 6344 6345 if (ncq_cmd_in_progress) 6346 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6347 tmp_slot); 6348 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6349 CLEAR_BIT(timeout_tags, tmp_slot); 6350 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6351 6352 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6353 } 6354 6355 /* Send up aborted packets with SATA_PKT_ABORTED */ 6356 while (aborted_tags) { 6357 if (err_retri_cmd_in_progress) { 6358 satapkt = ahci_portp->ahciport_err_retri_pkt; 6359 ASSERT(satapkt != NULL); 6360 ASSERT(aborted_tags == 0x1); 6361 6362 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6363 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6364 (void *)satapkt); 6365 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6366 break; 6367 } 6368 6369 tmp_slot = ddi_ffs(aborted_tags) - 1; 6370 if (tmp_slot == -1) { 6371 break; 6372 } 6373 6374 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6375 ASSERT(satapkt != NULL); 6376 6377 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6378 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6379 (void *)satapkt); 6380 6381 if (ncq_cmd_in_progress) 6382 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6383 tmp_slot); 6384 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6385 CLEAR_BIT(aborted_tags, tmp_slot); 6386 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6387 6388 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6389 } 6390 6391 /* Send up reset packets with SATA_PKT_RESET. */ 6392 while (reset_tags) { 6393 tmp_slot = ddi_ffs(reset_tags) - 1; 6394 if (tmp_slot == -1) { 6395 break; 6396 } 6397 6398 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6399 ASSERT(satapkt != NULL); 6400 6401 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6402 "sending up pkt 0x%p with SATA_PKT_RESET", 6403 (void *)satapkt); 6404 6405 if (ncq_cmd_in_progress) 6406 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6407 tmp_slot); 6408 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6409 CLEAR_BIT(reset_tags, tmp_slot); 6410 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6411 6412 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6413 } 6414 6415 /* Send up unfinished packets with SATA_PKT_RESET */ 6416 while (unfinished_tags) { 6417 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6418 if (tmp_slot == -1) { 6419 break; 6420 } 6421 6422 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6423 ASSERT(satapkt != NULL); 6424 6425 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6426 "sending up pkt 0x%p with SATA_PKT_RESET", 6427 (void *)satapkt); 6428 6429 if (ncq_cmd_in_progress) 6430 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6431 tmp_slot); 6432 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6433 CLEAR_BIT(unfinished_tags, tmp_slot); 6434 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6435 6436 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6437 } 6438 6439 ahci_portp->ahciport_mop_in_progress--; 6440 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6441 6442 if (ahci_portp->ahciport_mop_in_progress == 0) 6443 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6444 } 6445 6446 /* 6447 * This routine is going to first request a READ LOG EXT sata pkt from sata 6448 * module, and then deliver it to the HBA to get the ncq failure context. 6449 * The return value is the exactly failed tags. 6450 */ 6451 static uint32_t 6452 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6453 uint8_t port) 6454 { 6455 sata_device_t sdevice; 6456 sata_pkt_t *rdlog_spkt, *spkt; 6457 ddi_dma_handle_t buf_dma_handle; 6458 int loop_count; 6459 int rval; 6460 int failed_slot; 6461 uint32_t failed_tags = 0; 6462 struct sata_ncq_error_recovery_page *ncq_err_page; 6463 6464 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6465 "ahci_get_rdlogext_data enter: port %d", port); 6466 6467 /* Prepare the sdevice data */ 6468 bzero((void *)&sdevice, sizeof (sata_device_t)); 6469 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6470 6471 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6472 sdevice.satadev_addr.pmport = 0; 6473 6474 /* 6475 * Call the sata hba interface to get a rdlog spkt 6476 */ 6477 loop_count = 0; 6478 loop: 6479 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6480 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6481 if (rdlog_spkt == NULL) { 6482 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6483 /* Sleep for a while */ 6484 delay(AHCI_10MS_TICKS); 6485 goto loop; 6486 } 6487 /* Timed out after 1s */ 6488 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6489 "failed to get rdlog spkt for port %d", port); 6490 return (failed_tags); 6491 } 6492 6493 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6494 6495 /* 6496 * This flag is used to handle the specific error recovery when the 6497 * READ LOG EXT command gets a failure (fatal error or time-out). 6498 */ 6499 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6500 6501 /* 6502 * This start is not supposed to fail because after port is restarted, 6503 * the whole command list is empty. 6504 */ 6505 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6506 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6507 ahci_portp->ahciport_err_retri_pkt = NULL; 6508 6509 /* Remove the flag after READ LOG EXT command is completed */ 6510 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6511 6512 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6513 /* Update the request log data */ 6514 buf_dma_handle = *(ddi_dma_handle_t *) 6515 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6516 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6517 DDI_DMA_SYNC_FORKERNEL); 6518 if (rval == DDI_SUCCESS) { 6519 ncq_err_page = 6520 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6521 satapkt_cmd.satacmd_bp->b_un.b_addr; 6522 6523 /* Get the failed tag */ 6524 failed_slot = ncq_err_page->ncq_tag; 6525 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 6526 "ahci_get_rdlogext_data: port %d " 6527 "failed slot %d", port, failed_slot); 6528 if (failed_slot & NQ) { 6529 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6530 "the failed slot is not a valid tag"); 6531 goto out; 6532 } 6533 6534 failed_slot &= NCQ_TAG_MASK; 6535 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6536 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6537 "ahci_get_rdlogext_data: failed spkt 0x%p", 6538 (void *)spkt); 6539 if (spkt == NULL) { 6540 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6541 "the failed slot spkt is NULL") 6542 goto out; 6543 } 6544 6545 failed_tags = 0x1 << failed_slot; 6546 6547 /* Fill out the error context */ 6548 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6549 ncq_err_page); 6550 ahci_update_sata_registers(ahci_ctlp, port, 6551 &spkt->satapkt_device); 6552 } 6553 } 6554 out: 6555 sata_free_error_retrieval_pkt(rdlog_spkt); 6556 6557 return (failed_tags); 6558 } 6559 6560 /* 6561 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6562 * module, and then deliver it to the HBA to get the sense data and copy 6563 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6564 * SENSE sata pkt later. 6565 */ 6566 static void 6567 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6568 uint8_t port, sata_pkt_t *spkt) 6569 { 6570 sata_device_t sdevice; 6571 sata_pkt_t *rs_spkt; 6572 sata_cmd_t *sata_cmd; 6573 ddi_dma_handle_t buf_dma_handle; 6574 int loop_count; 6575 #if AHCI_DEBUG 6576 struct scsi_extended_sense *rqsense; 6577 #endif 6578 6579 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6580 "ahci_get_rqsense_data enter: port %d", port); 6581 6582 /* Prepare the sdevice data */ 6583 bzero((void *)&sdevice, sizeof (sata_device_t)); 6584 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6585 6586 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6587 sdevice.satadev_addr.pmport = 0; 6588 6589 sata_cmd = &spkt->satapkt_cmd; 6590 6591 /* 6592 * Call the sata hba interface to get a rs spkt 6593 */ 6594 loop_count = 0; 6595 loop: 6596 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6597 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6598 if (rs_spkt == NULL) { 6599 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6600 /* Sleep for a while */ 6601 delay(AHCI_10MS_TICKS); 6602 goto loop; 6603 6604 } 6605 /* Timed out after 1s */ 6606 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6607 "failed to get rs spkt for port %d", port); 6608 return; 6609 } 6610 6611 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6612 6613 /* 6614 * This flag is used to handle the specific error recovery when the 6615 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6616 */ 6617 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6618 6619 /* 6620 * This start is not supposed to fail because after port is restarted, 6621 * the whole command list is empty. 6622 */ 6623 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6624 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6625 ahci_portp->ahciport_err_retri_pkt = NULL; 6626 6627 /* Remove the flag after REQUEST SENSE command is completed */ 6628 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6629 6630 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6631 /* Update the request sense data */ 6632 buf_dma_handle = *(ddi_dma_handle_t *) 6633 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6634 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6635 DDI_DMA_SYNC_FORKERNEL); 6636 /* Copy the request sense data */ 6637 bcopy(rs_spkt-> 6638 satapkt_cmd.satacmd_bp->b_un.b_addr, 6639 &sata_cmd->satacmd_rqsense, 6640 SATA_ATAPI_MIN_RQSENSE_LEN); 6641 #if AHCI_DEBUG 6642 rqsense = (struct scsi_extended_sense *) 6643 sata_cmd->satacmd_rqsense; 6644 6645 /* Dump the sense data */ 6646 AHCIDBG0(AHCIDBG_SENSEDATA, ahci_ctlp, "\n"); 6647 AHCIDBG2(AHCIDBG_SENSEDATA, ahci_ctlp, 6648 "Sense data for satapkt %p ATAPI cmd 0x%x", 6649 spkt, sata_cmd->satacmd_acdb[0]); 6650 AHCIDBG5(AHCIDBG_SENSEDATA, ahci_ctlp, 6651 " es_code 0x%x es_class 0x%x " 6652 "es_key 0x%x es_add_code 0x%x " 6653 "es_qual_code 0x%x", 6654 rqsense->es_code, rqsense->es_class, 6655 rqsense->es_key, rqsense->es_add_code, 6656 rqsense->es_qual_code); 6657 #endif 6658 } 6659 6660 sata_free_error_retrieval_pkt(rs_spkt); 6661 } 6662 6663 /* 6664 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6665 * the port must be restarted. When the HBA detects thus error, it may try 6666 * to abort a transfer. And if the transfer was aborted, the device is 6667 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6668 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6669 * that the device is in a stable status and transfers may be restarted without 6670 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6671 * then the software will send the COMRESET to do the port reset. 6672 * 6673 * Software should perform the appropriate error recovery actions based on 6674 * whether non-queued commands were being issued or natived command queuing 6675 * commands were being issued. 6676 * 6677 * And software will complete the command that had the error with error mark 6678 * to higher level software. 6679 * 6680 * Fatal errors include the following: 6681 * PxIS.IFS - Interface Fatal Error Status 6682 * PxIS.HBDS - Host Bus Data Error Status 6683 * PxIS.HBFS - Host Bus Fatal Error Status 6684 * PxIS.TFES - Task File Error Status 6685 * 6686 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6687 */ 6688 static void 6689 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6690 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6691 { 6692 uint32_t port_cmd_status; 6693 uint32_t slot_status = 0; 6694 uint32_t failed_tags = 0; 6695 int failed_slot; 6696 int reset_flag = 0; 6697 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6698 sata_cmd_t *sata_cmd = NULL; 6699 sata_pkt_t *spkt = NULL; 6700 #if AHCI_DEBUG 6701 ahci_cmd_header_t *cmd_header; 6702 #endif 6703 6704 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6705 "ahci_fatal_error_recovery_handler enter: port %d", port); 6706 6707 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6708 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6709 6710 /* Read PxCI to see which commands are still outstanding */ 6711 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6712 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6713 6714 /* 6715 * Read PxCMD.CCS to determine the slot that the HBA 6716 * was processing when the error occurred. 6717 */ 6718 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6719 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6720 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6721 AHCI_CMD_STATUS_CCS_SHIFT; 6722 6723 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6724 spkt = ahci_portp->ahciport_err_retri_pkt; 6725 ASSERT(spkt != NULL); 6726 } else { 6727 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6728 if (spkt == NULL) { 6729 /* May happen when interface errors occur? */ 6730 goto next; 6731 } 6732 } 6733 6734 #if AHCI_DEBUG 6735 /* 6736 * Debugging purpose... 6737 */ 6738 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6739 cmd_header = 6740 &ahci_portp->ahciport_cmd_list[failed_slot]; 6741 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6742 "ahci_fatal_error_recovery_handler: port %d, " 6743 "PRD Byte Count = 0x%x, " 6744 "ahciport_prd_bytecounts = 0x%x", port, 6745 cmd_header->ahcich_prd_byte_count, 6746 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6747 } 6748 #endif 6749 6750 sata_cmd = &spkt->satapkt_cmd; 6751 6752 /* Fill out the status and error registers for PxIS.TFES */ 6753 if (intr_status & AHCI_INTR_STATUS_TFES) { 6754 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6755 ahcirf_d2h_register_fis); 6756 6757 /* Copy the error context back to the sata_cmd */ 6758 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6759 } 6760 6761 /* The failed command must be one of the outstanding commands */ 6762 failed_tags = 0x1 << failed_slot; 6763 ASSERT(failed_tags & slot_status); 6764 6765 /* Update the sata registers, especially PxSERR register */ 6766 ahci_update_sata_registers(ahci_ctlp, port, 6767 &spkt->satapkt_device); 6768 6769 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6770 /* Read PxSACT to see which commands are still outstanding */ 6771 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6772 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6773 } 6774 next: 6775 6776 #if AHCI_DEBUG 6777 /* 6778 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6779 * set, it means a fatal error happened after REQUEST SENSE command 6780 * or READ LOG EXT command is delivered to the HBA during the error 6781 * recovery process. At this time, the only outstanding command is 6782 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6783 */ 6784 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6785 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6786 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6787 "command or READ LOG EXT command for error data retrieval " 6788 "failed", port); 6789 ASSERT(slot_status == 0x1); 6790 ASSERT(failed_slot == 0); 6791 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6792 SCMD_REQUEST_SENSE || 6793 spkt->satapkt_cmd.satacmd_cmd_reg == 6794 SATAC_READ_LOG_EXT); 6795 } 6796 #endif 6797 6798 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6799 ahci_portp->ahciport_mop_in_progress++; 6800 6801 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6802 port, NULL, &reset_flag); 6803 6804 /* 6805 * Won't retrieve error information: 6806 * 1. Port reset was involved to recover 6807 * 2. Device is gone 6808 * 3. IDENTIFY DEVICE command sent to ATAPI device 6809 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6810 */ 6811 if (reset_flag || 6812 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6813 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6814 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6815 goto out; 6816 6817 /* 6818 * Deliver READ LOG EXT to gather information about the error when 6819 * a COMRESET has not been performed as part of the error recovery 6820 * during NCQ command processing. 6821 */ 6822 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6823 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6824 ahci_portp, port); 6825 goto out; 6826 } 6827 6828 /* 6829 * Deliver REQUEST SENSE for ATAPI command to gather information about 6830 * the error when a COMRESET has not been performed as part of the 6831 * error recovery. 6832 */ 6833 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6834 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6835 out: 6836 AHCIDBG5(AHCIDBG_ERRS, ahci_ctlp, 6837 "ahci_fatal_error_recovery_handler: port %d fatal error " 6838 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6839 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6840 port, slot_status, ahci_portp->ahciport_pending_tags, 6841 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6842 6843 ahci_mop_commands(ahci_ctlp, 6844 ahci_portp, 6845 slot_status, 6846 failed_tags, /* failed tags */ 6847 0, /* timeout tags */ 6848 0, /* aborted tags */ 6849 0); /* reset tags */ 6850 } 6851 6852 /* 6853 * Handle events - fatal error recovery 6854 */ 6855 static void 6856 ahci_events_handler(void *args) 6857 { 6858 ahci_event_arg_t *ahci_event_arg; 6859 ahci_ctl_t *ahci_ctlp; 6860 ahci_port_t *ahci_portp; 6861 uint32_t event; 6862 uint8_t port; 6863 6864 ahci_event_arg = (ahci_event_arg_t *)args; 6865 6866 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 6867 ahci_portp = ahci_event_arg->ahciea_portp; 6868 event = ahci_event_arg->ahciea_event; 6869 port = ahci_portp->ahciport_port_num; 6870 6871 AHCIDBG2(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6872 "ahci_events_handler enter: port %d intr_status = 0x%x", 6873 port, event); 6874 6875 mutex_enter(&ahci_portp->ahciport_mutex); 6876 6877 /* 6878 * ahci_intr_phyrdy_change() may have rendered it to 6879 * SATA_DTYPE_NONE. 6880 */ 6881 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 6882 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6883 "ahci_events_handler: port %d no device attached, " 6884 "and just return without doing anything", port); 6885 goto out; 6886 } 6887 6888 if (event & (AHCI_INTR_STATUS_IFS | 6889 AHCI_INTR_STATUS_HBDS | 6890 AHCI_INTR_STATUS_HBFS | 6891 AHCI_INTR_STATUS_TFES)) 6892 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 6893 port, event); 6894 6895 out: 6896 mutex_exit(&ahci_portp->ahciport_mutex); 6897 } 6898 6899 /* 6900 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 6901 * detect there are some commands which are timed out. 6902 */ 6903 static void 6904 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6905 uint8_t port, uint32_t tmp_timeout_tags) 6906 { 6907 uint32_t slot_status = 0; 6908 uint32_t finished_tags = 0; 6909 uint32_t timeout_tags = 0; 6910 6911 AHCIDBG1(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6912 "ahci_timeout_pkts enter: port %d", port); 6913 6914 mutex_enter(&ahci_portp->ahciport_mutex); 6915 6916 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6917 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6918 /* Read PxCI to see which commands are still outstanding */ 6919 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6920 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6921 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6922 /* Read PxSACT to see which commands are still outstanding */ 6923 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6924 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6925 } 6926 6927 #if AHCI_DEBUG 6928 /* 6929 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6930 * set, it means a fatal error happened after REQUEST SENSE command 6931 * or READ LOG EXT command is delivered to the HBA during the error 6932 * recovery process. At this time, the only outstanding command is 6933 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6934 */ 6935 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6936 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 6937 "ahci_timeout_pkts called while REQUEST SENSE " 6938 "command or READ LOG EXT command for error recovery " 6939 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 6940 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 6941 tmp_timeout_tags, slot_status, 6942 ahci_portp->ahciport_pending_tags, 6943 ahci_portp->ahciport_pending_ncq_tags); 6944 ASSERT(slot_status == 0x1); 6945 } 6946 #endif 6947 6948 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6949 ahci_portp->ahciport_mop_in_progress++; 6950 6951 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6952 port, NULL, NULL); 6953 6954 /* 6955 * Re-identify timeout tags because some previously checked commands 6956 * could already complete. 6957 */ 6958 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6959 finished_tags = ahci_portp->ahciport_pending_tags & 6960 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6961 timeout_tags = tmp_timeout_tags & ~finished_tags; 6962 6963 AHCIDBG5(AHCIDBG_TIMEOUT, ahci_ctlp, 6964 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6965 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 6966 "pending_tags = 0x%x ", 6967 port, finished_tags, timeout_tags, 6968 slot_status, ahci_portp->ahciport_pending_tags); 6969 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6970 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6971 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6972 timeout_tags = tmp_timeout_tags & ~finished_tags; 6973 6974 AHCIDBG5(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 6975 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6976 "timeout_tags = 0x%x, port_sactive = 0x%x, " 6977 "pending_ncq_tags = 0x%x ", 6978 port, finished_tags, timeout_tags, 6979 slot_status, ahci_portp->ahciport_pending_ncq_tags); 6980 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6981 timeout_tags = tmp_timeout_tags; 6982 } 6983 6984 ahci_mop_commands(ahci_ctlp, 6985 ahci_portp, 6986 slot_status, 6987 0, /* failed tags */ 6988 timeout_tags, /* timeout tags */ 6989 0, /* aborted tags */ 6990 0); /* reset tags */ 6991 6992 mutex_exit(&ahci_portp->ahciport_mutex); 6993 } 6994 6995 /* 6996 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 6997 * for long time. 6998 */ 6999 static void 7000 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 7001 { 7002 ahci_port_t *ahci_portp; 7003 sata_pkt_t *spkt; 7004 uint32_t pending_tags; 7005 uint32_t timeout_tags; 7006 uint32_t port_cmd_status; 7007 uint32_t port_sactive; 7008 uint8_t port; 7009 int tmp_slot; 7010 int current_slot; 7011 uint32_t current_tags; 7012 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7013 /* max number of cycles this packet should survive */ 7014 int max_life_cycles; 7015 7016 /* how many cycles this packet survived so far */ 7017 int watched_cycles; 7018 7019 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7020 7021 AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 7022 "ahci_watchdog_handler entered"); 7023 7024 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7025 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7026 continue; 7027 } 7028 7029 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7030 7031 mutex_enter(&ahci_portp->ahciport_mutex); 7032 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7033 mutex_exit(&ahci_portp->ahciport_mutex); 7034 continue; 7035 } 7036 7037 /* Skip the check for those ports in error recovery */ 7038 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7039 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7040 mutex_exit(&ahci_portp->ahciport_mutex); 7041 continue; 7042 } 7043 7044 pending_tags = 0; 7045 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7046 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7047 7048 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7049 current_slot = 0; 7050 pending_tags = 0x1; 7051 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7052 current_slot = 7053 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7054 AHCI_CMD_STATUS_CCS_SHIFT; 7055 pending_tags = ahci_portp->ahciport_pending_tags; 7056 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7057 port_sactive = ddi_get32( 7058 ahci_ctlp->ahcictl_ahci_acc_handle, 7059 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7060 current_tags = port_sactive & 7061 ~port_cmd_status & 7062 AHCI_NCQ_SLOT_MASK(ahci_portp); 7063 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7064 } 7065 7066 timeout_tags = 0; 7067 while (pending_tags) { 7068 tmp_slot = ddi_ffs(pending_tags) - 1; 7069 if (tmp_slot == -1) { 7070 break; 7071 } 7072 7073 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7074 spkt = ahci_portp->ahciport_err_retri_pkt; 7075 else 7076 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7077 7078 if ((spkt != NULL) && spkt->satapkt_time && 7079 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7080 /* 7081 * We are overloading satapkt_hba_driver_private 7082 * with watched_cycle count. 7083 * 7084 * If a packet has survived for more than it's 7085 * max life cycles, it is a candidate for time 7086 * out. 7087 */ 7088 watched_cycles = (int)(intptr_t) 7089 spkt->satapkt_hba_driver_private; 7090 watched_cycles++; 7091 max_life_cycles = (spkt->satapkt_time + 7092 ahci_watchdog_timeout - 1) / 7093 ahci_watchdog_timeout; 7094 7095 spkt->satapkt_hba_driver_private = 7096 (void *)(intptr_t)watched_cycles; 7097 7098 if (watched_cycles <= max_life_cycles) 7099 goto next; 7100 7101 AHCIDBG1(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7102 ahci_ctlp, "the current slot is %d", 7103 current_slot); 7104 /* 7105 * We need to check whether the HBA has 7106 * begun to execute the command, if not, 7107 * then re-set the timer of the command. 7108 */ 7109 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7110 (tmp_slot != current_slot) || 7111 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7112 ((0x1 << tmp_slot) & current_tags)) { 7113 spkt->satapkt_hba_driver_private = 7114 (void *)(intptr_t)0; 7115 } else { 7116 timeout_tags |= (0x1 << tmp_slot); 7117 cmn_err(CE_WARN, "!ahci%d: watchdog " 7118 "port %d satapkt 0x%p timed out\n", 7119 instance, port, (void *)spkt); 7120 } 7121 } 7122 next: 7123 CLEAR_BIT(pending_tags, tmp_slot); 7124 } 7125 7126 if (timeout_tags) { 7127 mutex_exit(&ahci_portp->ahciport_mutex); 7128 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7129 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7130 port, timeout_tags); 7131 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7132 mutex_enter(&ahci_portp->ahciport_mutex); 7133 } 7134 7135 mutex_exit(&ahci_portp->ahciport_mutex); 7136 } 7137 7138 /* Re-install the watchdog timeout handler */ 7139 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7140 ahci_ctlp->ahcictl_timeout_id = 7141 timeout((void (*)(void *))ahci_watchdog_handler, 7142 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7143 } 7144 7145 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7146 } 7147 7148 /* 7149 * Fill the error context into sata_cmd for non-queued command error. 7150 */ 7151 static void 7152 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7153 { 7154 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7155 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7156 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7157 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7158 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7159 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7160 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7161 7162 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7163 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7164 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7165 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7166 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7167 } 7168 } 7169 7170 /* 7171 * Fill the ncq error page into sata_cmd for queued command error. 7172 */ 7173 static void 7174 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7175 struct sata_ncq_error_recovery_page *ncq_err_page) 7176 { 7177 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7178 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7179 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7180 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7181 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7182 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7183 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7184 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7185 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7186 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7187 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7188 } 7189 7190 /* 7191 * Put the respective register value to sata_cmd_t for satacmd_flags. 7192 */ 7193 static void 7194 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7195 { 7196 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7197 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7198 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7199 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7200 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7201 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7202 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7203 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7204 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7205 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7206 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7207 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7208 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7209 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7210 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7211 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7212 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7213 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7214 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7215 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7216 } 7217 7218 static void 7219 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7220 uint32_t intr_status) 7221 { 7222 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7223 7224 if (intr_status & AHCI_INTR_STATUS_IFS) 7225 cmn_err(CE_WARN, "ahci%d: ahci port %d has interface fatal " 7226 "error", instance, port); 7227 7228 if (intr_status & AHCI_INTR_STATUS_HBDS) 7229 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus data error", 7230 instance, port); 7231 7232 if (intr_status & AHCI_INTR_STATUS_HBFS) 7233 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus fatal error", 7234 instance, port); 7235 7236 if (intr_status & AHCI_INTR_STATUS_TFES) 7237 cmn_err(CE_WARN, "ahci%d: ahci port %d has task file error", 7238 instance, port); 7239 7240 cmn_err(CE_WARN, "ahci%d: ahci port %d is trying to do error " 7241 "recovery", instance, port); 7242 } 7243 7244 /* 7245 * Dump the serror message to the log. 7246 */ 7247 static void 7248 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7249 uint32_t port_serror, int debug_only) 7250 { 7251 static char err_buf[512]; 7252 static char err_msg_header[16]; 7253 char *err_msg = err_buf; 7254 7255 *err_buf = '\0'; 7256 *err_msg_header = '\0'; 7257 7258 if (port_serror & SERROR_DATA_ERR_FIXED) { 7259 err_msg = strcat(err_msg, 7260 "\tRecovered Data Integrity Error (I)\n"); 7261 } 7262 7263 if (port_serror & SERROR_COMM_ERR_FIXED) { 7264 err_msg = strcat(err_msg, 7265 "\tRecovered Communication Error (M)\n"); 7266 } 7267 7268 if (port_serror & SERROR_DATA_ERR) { 7269 err_msg = strcat(err_msg, 7270 "\tTransient Data Integrity Error (T)\n"); 7271 } 7272 7273 if (port_serror & SERROR_PERSISTENT_ERR) { 7274 err_msg = strcat(err_msg, 7275 "\tPersistent Communication or Data Integrity Error (C)\n"); 7276 } 7277 7278 if (port_serror & SERROR_PROTOCOL_ERR) { 7279 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7280 } 7281 7282 if (port_serror & SERROR_INT_ERR) { 7283 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7284 } 7285 7286 if (port_serror & SERROR_PHY_RDY_CHG) { 7287 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7288 } 7289 7290 if (port_serror & SERROR_PHY_INT_ERR) { 7291 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7292 } 7293 7294 if (port_serror & SERROR_COMM_WAKE) { 7295 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7296 } 7297 7298 if (port_serror & SERROR_10B_TO_8B_ERR) { 7299 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7300 } 7301 7302 if (port_serror & SERROR_DISPARITY_ERR) { 7303 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7304 } 7305 7306 if (port_serror & SERROR_CRC_ERR) { 7307 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7308 } 7309 7310 if (port_serror & SERROR_HANDSHAKE_ERR) { 7311 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7312 } 7313 7314 if (port_serror & SERROR_LINK_SEQ_ERR) { 7315 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7316 } 7317 7318 if (port_serror & SERROR_TRANS_ERR) { 7319 err_msg = strcat(err_msg, 7320 "\tTransport state transition error (T)\n"); 7321 } 7322 7323 if (port_serror & SERROR_FIS_TYPE) { 7324 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7325 } 7326 7327 if (port_serror & SERROR_EXCHANGED_ERR) { 7328 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7329 } 7330 7331 if (err_msg == NULL) 7332 return; 7333 7334 if (debug_only) { 7335 (void) sprintf(err_msg_header, "port %d", port); 7336 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg_header); 7337 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg); 7338 } else if (ahci_ctlp) { 7339 cmn_err(CE_WARN, "ahci%d: %s %s", 7340 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7341 err_msg_header, err_msg); 7342 } else { 7343 cmn_err(CE_WARN, "ahci: %s %s", err_msg_header, err_msg); 7344 } 7345 } 7346 7347 /* 7348 * This routine is to calculate the total number of ports implemented 7349 * by the HBA. 7350 */ 7351 static int 7352 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7353 { 7354 uint8_t i; 7355 int num = 0; 7356 7357 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7358 if (((uint32_t)0x1 << i) & ports_implemented) 7359 num++; 7360 } 7361 7362 return (num); 7363 } 7364 7365 #if AHCI_DEBUG 7366 static void 7367 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7368 { 7369 static char name[16]; 7370 va_list ap; 7371 7372 mutex_enter(&ahci_log_mutex); 7373 7374 va_start(ap, fmt); 7375 if (ahci_ctlp) { 7376 (void) sprintf(name, "ahci%d: ", 7377 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7378 } else { 7379 (void) sprintf(name, "ahci: "); 7380 } 7381 7382 (void) vsprintf(ahci_log_buf, fmt, ap); 7383 va_end(ap); 7384 7385 cmn_err(level, "%s%s", name, ahci_log_buf); 7386 7387 mutex_exit(&ahci_log_mutex); 7388 } 7389 #endif 7390 7391 /* 7392 * quiesce(9E) entry point. 7393 * 7394 * This function is called when the system is single-threaded at high 7395 * PIL with preemption disabled. Therefore, this function must not be 7396 * blocked. 7397 * 7398 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7399 * DDI_FAILURE indicates an error condition and should almost never happen. 7400 */ 7401 static int 7402 ahci_quiesce(dev_info_t *dip) 7403 { 7404 ahci_ctl_t *ahci_ctlp; 7405 ahci_port_t *ahci_portp; 7406 int instance, port; 7407 7408 instance = ddi_get_instance(dip); 7409 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7410 7411 if (ahci_ctlp == NULL) 7412 return (DDI_FAILURE); 7413 7414 #if AHCI_DEBUG 7415 ahci_debug_flags = 0; 7416 #endif 7417 7418 /* disable all the interrupts. */ 7419 ahci_disable_all_intrs(ahci_ctlp); 7420 7421 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7422 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7423 continue; 7424 } 7425 7426 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7427 7428 /* 7429 * Stop the port by clearing PxCMD.ST 7430 * 7431 * Here we must disable the port interrupt because 7432 * ahci_disable_all_intrs only clear GHC.IE, and IS 7433 * register will be still set if PxIE is enabled. 7434 * When ahci shares one IRQ with other drivers, the 7435 * intr handler may claim the intr mistakenly. 7436 */ 7437 ahci_disable_port_intrs(ahci_ctlp, port); 7438 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7439 ahci_portp, port); 7440 } 7441 7442 return (DDI_SUCCESS); 7443 } 7444