1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_drain_ports_taskq(ahci_ctl_t *); 104 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 105 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 107 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 108 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 109 uint8_t, sata_pkt_t *); 110 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 111 uint8_t, sata_pkt_t *); 112 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 113 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 114 static void ahci_copy_ncq_err_page(sata_cmd_t *, 115 struct sata_ncq_error_recovery_page *); 116 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 117 118 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 119 static int ahci_hba_reset(ahci_ctl_t *); 120 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 124 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 125 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 126 uint8_t); 127 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 128 uint8_t, int, int *); 129 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 130 uint32_t, uint32_t, uint32_t, uint32_t); 131 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 132 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 133 uint8_t, sata_pkt_t *); 134 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 135 uint8_t, uint32_t); 136 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 137 uint8_t, uint32_t); 138 static void ahci_events_handler(void *); 139 static void ahci_watchdog_handler(ahci_ctl_t *); 140 141 static uint_t ahci_intr(caddr_t, caddr_t); 142 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 143 static int ahci_add_intrs(ahci_ctl_t *, int); 144 static void ahci_rem_intrs(ahci_ctl_t *); 145 static void ahci_enable_all_intrs(ahci_ctl_t *); 146 static void ahci_disable_all_intrs(ahci_ctl_t *); 147 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 148 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 149 150 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 151 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 154 ahci_port_t *, uint8_t); 155 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, uint32_t); 158 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 159 uint8_t, uint32_t); 160 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 161 162 static int ahci_get_num_implemented_ports(uint32_t); 163 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 164 uint32_t); 165 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 166 #if AHCI_DEBUG 167 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 168 #endif 169 170 171 /* 172 * DMA attributes for the data buffer 173 * 174 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 175 * does not support 64-bit addressing 176 */ 177 static ddi_dma_attr_t buffer_dma_attr = { 178 DMA_ATTR_V0, /* dma_attr_version */ 179 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 180 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 181 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 182 0x2ull, /* dma_attr_align: word aligned */ 183 1, /* dma_attr_burstsizes */ 184 1, /* dma_attr_minxfer */ 185 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 186 0xffffffffull, /* dma_attr_seg */ 187 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 188 512, /* dma_attr_granular */ 189 0, /* dma_attr_flags */ 190 }; 191 192 /* 193 * DMA attributes for the rcvd FIS 194 * 195 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 196 * does not support 64-bit addressing 197 */ 198 static ddi_dma_attr_t rcvd_fis_dma_attr = { 199 DMA_ATTR_V0, /* dma_attr_version */ 200 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 201 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 202 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 203 0x100ull, /* dma_attr_align: 256-byte aligned */ 204 1, /* dma_attr_burstsizes */ 205 1, /* dma_attr_minxfer */ 206 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 207 0xffffffffull, /* dma_attr_seg */ 208 1, /* dma_attr_sgllen */ 209 1, /* dma_attr_granular */ 210 0, /* dma_attr_flags */ 211 }; 212 213 /* 214 * DMA attributes for the command list 215 * 216 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 217 * does not support 64-bit addressing 218 */ 219 static ddi_dma_attr_t cmd_list_dma_attr = { 220 DMA_ATTR_V0, /* dma_attr_version */ 221 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 222 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 223 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 224 0x400ull, /* dma_attr_align: 1K-byte aligned */ 225 1, /* dma_attr_burstsizes */ 226 1, /* dma_attr_minxfer */ 227 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 228 0xffffffffull, /* dma_attr_seg */ 229 1, /* dma_attr_sgllen */ 230 1, /* dma_attr_granular */ 231 0, /* dma_attr_flags */ 232 }; 233 234 /* 235 * DMA attributes for cmd tables 236 * 237 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 238 * does not support 64-bit addressing 239 */ 240 static ddi_dma_attr_t cmd_table_dma_attr = { 241 DMA_ATTR_V0, /* dma_attr_version */ 242 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 243 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 244 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 245 0x80ull, /* dma_attr_align: 128-byte aligned */ 246 1, /* dma_attr_burstsizes */ 247 1, /* dma_attr_minxfer */ 248 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 249 0xffffffffull, /* dma_attr_seg */ 250 1, /* dma_attr_sgllen */ 251 1, /* dma_attr_granular */ 252 0, /* dma_attr_flags */ 253 }; 254 255 256 /* Device access attributes */ 257 static ddi_device_acc_attr_t accattr = { 258 DDI_DEVICE_ATTR_V0, 259 DDI_STRUCTURE_LE_ACC, 260 DDI_STRICTORDER_ACC 261 }; 262 263 264 static struct dev_ops ahcictl_dev_ops = { 265 DEVO_REV, /* devo_rev */ 266 0, /* refcnt */ 267 ahci_getinfo, /* info */ 268 nulldev, /* identify */ 269 nulldev, /* probe */ 270 ahci_attach, /* attach */ 271 ahci_detach, /* detach */ 272 nodev, /* no reset */ 273 (struct cb_ops *)0, /* driver operations */ 274 NULL, /* bus operations */ 275 NULL, /* power */ 276 ahci_quiesce, /* quiesce */ 277 }; 278 279 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 280 SATA_TRAN_HOTPLUG_OPS_REV_1, 281 ahci_tran_hotplug_port_activate, 282 ahci_tran_hotplug_port_deactivate 283 }; 284 285 extern struct mod_ops mod_driverops; 286 287 static struct modldrv modldrv = { 288 &mod_driverops, /* driverops */ 289 ahci_ident, /* short description */ 290 &ahcictl_dev_ops, /* driver ops */ 291 }; 292 293 static struct modlinkage modlinkage = { 294 MODREV_1, 295 &modldrv, 296 NULL 297 }; 298 299 /* The following variables are watchdog handler related */ 300 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 301 static int ahci_watchdog_tick; 302 303 /* 304 * This static variable indicates the size of command table, 305 * and it's changeable with prdt number, which ahci_dma_prdt_number 306 * indicates. 307 */ 308 static size_t ahci_cmd_table_size; 309 310 /* 311 * The below global variables are tunable via /etc/system 312 * 313 * ahci_dma_prdt_number 314 * ahci_msi_enabled 315 * ahci_buf_64bit_dma 316 * ahci_commu_64bit_dma 317 */ 318 319 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 320 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 321 322 /* AHCI MSI is tunable */ 323 boolean_t ahci_msi_enabled = B_TRUE; 324 325 /* 326 * 64-bit dma addressing for data buffer is tunable 327 * 328 * The variable controls only the below value: 329 * DBAU (upper 32-bits physical address of data block) 330 */ 331 boolean_t ahci_buf_64bit_dma = B_TRUE; 332 333 /* 334 * 64-bit dma addressing for communication system descriptors is tunable 335 * 336 * The variable controls the below three values: 337 * 338 * PxCLBU (upper 32-bits for the command list base physical address) 339 * PxFBU (upper 32-bits for the received FIS base physical address) 340 * CTBAU (upper 32-bits of command table base) 341 */ 342 boolean_t ahci_commu_64bit_dma = B_TRUE; 343 344 /* 345 * End of global tunable variable definition 346 */ 347 348 #if AHCI_DEBUG 349 uint32_t ahci_debug_flags = 0; 350 #else 351 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 352 #endif 353 354 355 #if AHCI_DEBUG 356 /* The following is needed for ahci_log() */ 357 static kmutex_t ahci_log_mutex; 358 static char ahci_log_buf[512]; 359 #endif 360 361 /* Opaque state pointer initialized by ddi_soft_state_init() */ 362 static void *ahci_statep = NULL; 363 364 /* 365 * ahci module initialization. 366 */ 367 int 368 _init(void) 369 { 370 int ret; 371 372 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 373 if (ret != 0) { 374 goto err_out; 375 } 376 377 #if AHCI_DEBUG 378 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 379 #endif 380 381 if ((ret = sata_hba_init(&modlinkage)) != 0) { 382 #if AHCI_DEBUG 383 mutex_destroy(&ahci_log_mutex); 384 #endif 385 ddi_soft_state_fini(&ahci_statep); 386 goto err_out; 387 } 388 389 ret = mod_install(&modlinkage); 390 if (ret != 0) { 391 sata_hba_fini(&modlinkage); 392 #if AHCI_DEBUG 393 mutex_destroy(&ahci_log_mutex); 394 #endif 395 ddi_soft_state_fini(&ahci_statep); 396 goto err_out; 397 } 398 399 /* watchdog tick */ 400 ahci_watchdog_tick = drv_usectohz( 401 (clock_t)ahci_watchdog_timeout * 1000000); 402 return (ret); 403 404 err_out: 405 cmn_err(CE_WARN, "!ahci: Module init failed"); 406 return (ret); 407 } 408 409 /* 410 * ahci module uninitialize. 411 */ 412 int 413 _fini(void) 414 { 415 int ret; 416 417 ret = mod_remove(&modlinkage); 418 if (ret != 0) { 419 return (ret); 420 } 421 422 /* Remove the resources allocated in _init(). */ 423 sata_hba_fini(&modlinkage); 424 #if AHCI_DEBUG 425 mutex_destroy(&ahci_log_mutex); 426 #endif 427 ddi_soft_state_fini(&ahci_statep); 428 429 return (ret); 430 } 431 432 /* 433 * _info entry point 434 */ 435 int 436 _info(struct modinfo *modinfop) 437 { 438 return (mod_info(&modlinkage, modinfop)); 439 } 440 441 /* 442 * The attach entry point for dev_ops. 443 */ 444 static int 445 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 446 { 447 ahci_ctl_t *ahci_ctlp = NULL; 448 int instance = ddi_get_instance(dip); 449 int status; 450 int attach_state; 451 uint32_t cap_status, ahci_version; 452 int intr_types; 453 int i; 454 pci_regspec_t *regs; 455 int regs_length; 456 int rnumber; 457 #if AHCI_DEBUG 458 int speed; 459 #endif 460 461 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 462 NULL); 463 464 switch (cmd) { 465 case DDI_ATTACH: 466 break; 467 468 case DDI_RESUME: 469 470 /* 471 * During DDI_RESUME, the hardware state of the device 472 * (power may have been removed from the device) must be 473 * restored, allow pending requests to continue, and 474 * service new requests. 475 */ 476 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 477 mutex_enter(&ahci_ctlp->ahcictl_mutex); 478 479 /* Restart watch thread */ 480 if (ahci_ctlp->ahcictl_timeout_id == 0) 481 ahci_ctlp->ahcictl_timeout_id = timeout( 482 (void (*)(void *))ahci_watchdog_handler, 483 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 484 485 mutex_exit(&ahci_ctlp->ahcictl_mutex); 486 487 /* 488 * Re-initialize the controller and enable the interrupts and 489 * restart all the ports. 490 * 491 * Note that so far we don't support hot-plug during 492 * suspend/resume. 493 */ 494 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 495 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 496 "Failed to initialize the controller " 497 "during DDI_RESUME", NULL); 498 return (DDI_FAILURE); 499 } 500 501 mutex_enter(&ahci_ctlp->ahcictl_mutex); 502 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 503 mutex_exit(&ahci_ctlp->ahcictl_mutex); 504 505 return (DDI_SUCCESS); 506 507 default: 508 return (DDI_FAILURE); 509 } 510 511 attach_state = AHCI_ATTACH_STATE_NONE; 512 513 /* Allocate soft state */ 514 status = ddi_soft_state_zalloc(ahci_statep, instance); 515 if (status != DDI_SUCCESS) { 516 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 517 instance); 518 goto err_out; 519 } 520 521 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 522 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 523 ahci_ctlp->ahcictl_dip = dip; 524 525 /* Initialize the cport/port mapping */ 526 for (i = 0; i < AHCI_MAX_PORTS; i++) { 527 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 528 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 529 } 530 531 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 532 533 /* 534 * Now map the AHCI base address; which includes global 535 * registers and port control registers 536 * 537 * According to the spec, the AHCI Base Address is BAR5, 538 * but BAR0-BAR4 are optional, so we need to check which 539 * rnumber is used for BAR5. 540 */ 541 542 /* 543 * search through DDI "reg" property for the AHCI register set 544 */ 545 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 546 DDI_PROP_DONTPASS, "reg", (int **)®s, 547 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 548 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 549 instance); 550 goto err_out; 551 } 552 553 /* AHCI Base Address is located at 0x24 offset */ 554 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 555 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 556 == AHCI_PCI_RNUM) 557 break; 558 } 559 560 ddi_prop_free(regs); 561 562 if (rnumber == regs_length) { 563 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 564 instance); 565 goto err_out; 566 } 567 568 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 569 570 status = ddi_regs_map_setup(dip, 571 rnumber, 572 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 573 0, 574 0, 575 &accattr, 576 &ahci_ctlp->ahcictl_ahci_acc_handle); 577 if (status != DDI_SUCCESS) { 578 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 579 instance); 580 goto err_out; 581 } 582 583 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 584 585 /* Get the AHCI version information */ 586 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 587 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 588 589 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 590 (ahci_version & 0xffff0000) >> 16, 591 ((ahci_version & 0x0000ff00) >> 4 | 592 (ahci_version & 0x000000ff))); 593 594 /* We don't support controllers whose versions are lower than 1.0 */ 595 if (!(ahci_version & 0xffff0000)) { 596 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 597 "than version 1.0", instance); 598 goto err_out; 599 } 600 601 /* Get the HBA capabilities information */ 602 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 603 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 604 605 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 606 cap_status); 607 608 #if AHCI_DEBUG 609 /* Get the interface speed supported by the HBA */ 610 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 611 if (speed == 0x01) { 612 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 613 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 614 } else if (speed == 0x10) { 615 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 616 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 617 } else if (speed == 0x11) { 618 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 619 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 620 } 621 #endif 622 623 /* Get the number of command slots supported by the HBA */ 624 ahci_ctlp->ahcictl_num_cmd_slots = 625 ((cap_status & AHCI_HBA_CAP_NCS) >> 626 AHCI_HBA_CAP_NCS_SHIFT) + 1; 627 628 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 629 ahci_ctlp->ahcictl_num_cmd_slots); 630 631 /* Get the bit map which indicates ports implemented by the HBA */ 632 ahci_ctlp->ahcictl_ports_implemented = 633 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 634 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 635 636 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 637 ahci_ctlp->ahcictl_ports_implemented); 638 639 /* 640 * According to the AHCI spec, CAP.NP should indicate the maximum 641 * number of ports supported by the HBA silicon, but we found 642 * this value of ICH8 chipset only indicates the number of ports 643 * implemented (exposed) by it. Therefore, the driver should calculate 644 * the potential maximum value by checking PI register, and use 645 * the maximum of this value and CAP.NP. 646 */ 647 ahci_ctlp->ahcictl_num_ports = max( 648 (cap_status & AHCI_HBA_CAP_NP) + 1, 649 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 650 651 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 652 ahci_ctlp->ahcictl_num_ports); 653 654 /* Get the number of implemented ports by the HBA */ 655 ahci_ctlp->ahcictl_num_implemented_ports = 656 ahci_get_num_implemented_ports( 657 ahci_ctlp->ahcictl_ports_implemented); 658 659 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 660 "hba number of implemented ports: %d", 661 ahci_ctlp->ahcictl_num_implemented_ports); 662 663 /* Check whether HBA supports 64bit DMA addressing */ 664 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 665 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 666 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 667 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 668 "hba does not support 64-bit addressing", NULL); 669 } 670 671 /* Checking for Support Command List Override */ 672 if (cap_status & AHCI_HBA_CAP_SCLO) { 673 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 674 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 675 "hba supports command list override.", NULL); 676 } 677 678 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 679 != DDI_SUCCESS) { 680 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 681 instance); 682 goto err_out; 683 } 684 685 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 686 687 /* 688 * Check the pci configuration space, and set caps. We also 689 * handle the hardware defect in this function. 690 * 691 * For example, force ATI SB600 to use 32-bit dma addressing 692 * since it doesn't support 64-bit dma though its CAP register 693 * declares it support. 694 */ 695 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 696 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 697 instance); 698 goto err_out; 699 } 700 701 /* 702 * Disable the whole controller interrupts before adding 703 * interrupt handlers(s). 704 */ 705 ahci_disable_all_intrs(ahci_ctlp); 706 707 /* Get supported interrupt types */ 708 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 709 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 710 instance); 711 goto err_out; 712 } 713 714 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 715 "ddi_intr_get_supported_types() returned: 0x%x", 716 intr_types); 717 718 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 719 /* 720 * Try MSI first, but fall back to FIXED if failed 721 */ 722 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 723 DDI_SUCCESS) { 724 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 725 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 726 "Using MSI interrupt type", NULL); 727 goto intr_done; 728 } 729 730 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 731 "MSI registration failed, " 732 "trying FIXED interrupts", NULL); 733 } 734 735 if (intr_types & DDI_INTR_TYPE_FIXED) { 736 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 737 DDI_SUCCESS) { 738 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 739 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 740 "Using FIXED interrupt type", NULL); 741 goto intr_done; 742 } 743 744 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 745 "FIXED interrupt registration failed", NULL); 746 } 747 748 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 749 750 goto err_out; 751 752 intr_done: 753 754 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 755 756 /* Initialize the controller mutex */ 757 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 758 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 759 760 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 761 762 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 763 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 764 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 765 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 766 } 767 768 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 769 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 770 sizeof (ahci_prdt_item_t)); 771 772 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 773 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 774 " ahci_cmd_table_size is 0x%x", 775 ahci_dma_prdt_number, ahci_cmd_table_size); 776 777 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 778 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 779 ahci_dma_prdt_number; 780 781 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 782 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 783 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 784 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 785 786 if ((ahci_buf_64bit_dma == B_FALSE) || 787 (ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA)) { 788 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 789 0xffffffffull; 790 } 791 792 if ((ahci_commu_64bit_dma == B_FALSE) || 793 (ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA)) { 794 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 795 0xffffffffull; 796 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 797 0xffffffffull; 798 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 799 0xffffffffull; 800 } 801 802 /* Allocate the ports structure */ 803 status = ahci_alloc_ports_state(ahci_ctlp); 804 if (status != AHCI_SUCCESS) { 805 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 806 instance); 807 goto err_out; 808 } 809 810 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 811 812 /* 813 * Initialize the controller and ports. 814 */ 815 status = ahci_initialize_controller(ahci_ctlp); 816 if (status != AHCI_SUCCESS) { 817 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 818 instance); 819 goto err_out; 820 } 821 822 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 823 824 /* Start one thread to check packet timeouts */ 825 ahci_ctlp->ahcictl_timeout_id = timeout( 826 (void (*)(void *))ahci_watchdog_handler, 827 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 828 829 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 830 831 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 832 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 833 instance); 834 goto err_out; 835 } 836 837 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 838 839 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 840 841 return (DDI_SUCCESS); 842 843 err_out: 844 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 845 mutex_enter(&ahci_ctlp->ahcictl_mutex); 846 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 847 ahci_ctlp->ahcictl_timeout_id = 0; 848 mutex_exit(&ahci_ctlp->ahcictl_mutex); 849 } 850 851 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 852 ahci_uninitialize_controller(ahci_ctlp); 853 } 854 855 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 856 ahci_dealloc_ports_state(ahci_ctlp); 857 } 858 859 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 860 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 861 } 862 863 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 864 ahci_rem_intrs(ahci_ctlp); 865 } 866 867 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 868 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 869 } 870 871 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 872 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 873 } 874 875 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 876 ddi_soft_state_free(ahci_statep, instance); 877 } 878 879 return (DDI_FAILURE); 880 } 881 882 /* 883 * The detach entry point for dev_ops. 884 */ 885 static int 886 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 887 { 888 ahci_ctl_t *ahci_ctlp; 889 int instance; 890 int ret; 891 892 instance = ddi_get_instance(dip); 893 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 894 895 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 896 897 switch (cmd) { 898 case DDI_DETACH: 899 900 /* disable the interrupts for an uninterrupted detach */ 901 mutex_enter(&ahci_ctlp->ahcictl_mutex); 902 ahci_disable_all_intrs(ahci_ctlp); 903 mutex_exit(&ahci_ctlp->ahcictl_mutex); 904 905 /* unregister from the sata framework. */ 906 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 907 if (ret != AHCI_SUCCESS) { 908 mutex_enter(&ahci_ctlp->ahcictl_mutex); 909 ahci_enable_all_intrs(ahci_ctlp); 910 mutex_exit(&ahci_ctlp->ahcictl_mutex); 911 return (DDI_FAILURE); 912 } 913 914 mutex_enter(&ahci_ctlp->ahcictl_mutex); 915 916 /* stop the watchdog handler */ 917 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 918 ahci_ctlp->ahcictl_timeout_id = 0; 919 920 mutex_exit(&ahci_ctlp->ahcictl_mutex); 921 922 /* uninitialize the controller */ 923 ahci_uninitialize_controller(ahci_ctlp); 924 925 /* remove the interrupts */ 926 ahci_rem_intrs(ahci_ctlp); 927 928 /* deallocate the ports structures */ 929 ahci_dealloc_ports_state(ahci_ctlp); 930 931 /* destroy mutex */ 932 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 933 934 /* teardown the pci config */ 935 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 936 937 /* remove the reg maps. */ 938 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 939 940 /* free the soft state. */ 941 ddi_soft_state_free(ahci_statep, instance); 942 943 return (DDI_SUCCESS); 944 945 case DDI_SUSPEND: 946 947 /* 948 * The steps associated with suspension must include putting 949 * the underlying device into a quiescent state so that it 950 * will not generate interrupts or modify or access memory. 951 */ 952 mutex_enter(&ahci_ctlp->ahcictl_mutex); 953 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 954 mutex_exit(&ahci_ctlp->ahcictl_mutex); 955 return (DDI_SUCCESS); 956 } 957 958 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 959 960 /* stop the watchdog handler */ 961 if (ahci_ctlp->ahcictl_timeout_id) { 962 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 963 ahci_ctlp->ahcictl_timeout_id = 0; 964 } 965 966 mutex_exit(&ahci_ctlp->ahcictl_mutex); 967 968 /* 969 * drain the taskq 970 */ 971 ahci_drain_ports_taskq(ahci_ctlp); 972 973 /* 974 * Disable the interrupts and stop all the ports. 975 */ 976 ahci_uninitialize_controller(ahci_ctlp); 977 978 return (DDI_SUCCESS); 979 980 default: 981 return (DDI_FAILURE); 982 } 983 } 984 985 /* 986 * The info entry point for dev_ops. 987 * 988 */ 989 static int 990 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 991 void *arg, void **result) 992 { 993 #ifndef __lock_lint 994 _NOTE(ARGUNUSED(dip)) 995 #endif /* __lock_lint */ 996 997 ahci_ctl_t *ahci_ctlp; 998 int instance; 999 dev_t dev; 1000 1001 dev = (dev_t)arg; 1002 instance = getminor(dev); 1003 1004 switch (infocmd) { 1005 case DDI_INFO_DEVT2DEVINFO: 1006 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1007 if (ahci_ctlp != NULL) { 1008 *result = ahci_ctlp->ahcictl_dip; 1009 return (DDI_SUCCESS); 1010 } else { 1011 *result = NULL; 1012 return (DDI_FAILURE); 1013 } 1014 case DDI_INFO_DEVT2INSTANCE: 1015 *(int *)result = instance; 1016 break; 1017 default: 1018 break; 1019 } 1020 1021 return (DDI_SUCCESS); 1022 } 1023 1024 /* 1025 * Registers the ahci with sata framework. 1026 */ 1027 static int 1028 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1029 { 1030 struct sata_hba_tran *sata_hba_tran; 1031 1032 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1033 "ahci_register_sata_hba_tran enter", NULL); 1034 1035 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1036 1037 /* Allocate memory for the sata_hba_tran */ 1038 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1039 1040 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 1041 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1042 sata_hba_tran->sata_tran_hba_dma_attr = 1043 &ahci_ctlp->ahcictl_buffer_dma_attr; 1044 1045 /* Report the number of implemented ports */ 1046 sata_hba_tran->sata_tran_hba_num_cports = 1047 ahci_ctlp->ahcictl_num_implemented_ports; 1048 1049 /* Support ATAPI device */ 1050 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1051 1052 /* Get the data transfer capability for PIO command by the HBA */ 1053 if (cap_status & AHCI_HBA_CAP_PMD) { 1054 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1055 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1056 "DRQ block data transfer for PIO command protocol", NULL); 1057 } 1058 1059 /* 1060 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1061 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1062 * QUEUED (EXT), and SERVICE commands). Queued operations are 1063 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1064 * QUEUED commands when the HBA and device support native command 1065 * queuing(NCQ). 1066 * 1067 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1068 * CAP register of the HBA indicates NCQ is supported. 1069 * 1070 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1071 * set because the previous register content of PxCI can be re-written 1072 * in the register write. 1073 */ 1074 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1075 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1076 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1077 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1078 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1079 "Command Queuing", NULL); 1080 } 1081 1082 /* Report the number of command slots */ 1083 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1084 1085 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1086 sata_hba_tran->sata_tran_start = ahci_tran_start; 1087 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1088 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1089 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1090 #ifdef __lock_lint 1091 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1092 #endif 1093 /* 1094 * When SATA framework adds support for pwrmgt the 1095 * pwrmgt_ops needs to be updated 1096 */ 1097 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1098 sata_hba_tran->sata_tran_ioctl = NULL; 1099 1100 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1101 1102 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1103 1104 /* Attach it to SATA framework */ 1105 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1106 != DDI_SUCCESS) { 1107 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1108 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1109 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1110 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1111 return (AHCI_FAILURE); 1112 } 1113 1114 return (AHCI_SUCCESS); 1115 } 1116 1117 /* 1118 * Unregisters the ahci with sata framework. 1119 */ 1120 static int 1121 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1122 { 1123 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1124 "ahci_unregister_sata_hba_tran enter", NULL); 1125 1126 /* Detach from the SATA framework. */ 1127 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1128 DDI_SUCCESS) { 1129 return (AHCI_FAILURE); 1130 } 1131 1132 /* Deallocate sata_hba_tran. */ 1133 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1134 sizeof (sata_hba_tran_t)); 1135 1136 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1137 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1138 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1139 1140 return (AHCI_SUCCESS); 1141 } 1142 1143 /* 1144 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1145 * port status registers and an attached device type via sata_device 1146 * structure. 1147 * 1148 * We return the cached information from a previous hardware probe. The 1149 * actual hardware probing itself was done either from within 1150 * ahci_initialize_controller() during the driver attach or from a phy 1151 * ready change interrupt handler. 1152 */ 1153 static int 1154 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1155 { 1156 ahci_ctl_t *ahci_ctlp; 1157 ahci_port_t *ahci_portp; 1158 uint8_t cport = sd->satadev_addr.cport; 1159 uint8_t pmport = sd->satadev_addr.pmport; 1160 uint8_t qual = sd->satadev_addr.qual; 1161 uint8_t device_type; 1162 uint32_t port_state; 1163 uint8_t port; 1164 1165 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1166 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1167 1168 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1169 "ahci_tran_probe_port enter: cport: %d, " 1170 "pmport: %d, qual: %d", cport, pmport, qual); 1171 1172 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1173 1174 mutex_enter(&ahci_portp->ahciport_mutex); 1175 1176 port_state = ahci_portp->ahciport_port_state; 1177 switch (port_state) { 1178 1179 case SATA_PSTATE_FAILED: 1180 sd->satadev_state = SATA_PSTATE_FAILED; 1181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1182 "ahci_tran_probe_port: port %d PORT FAILED", port); 1183 goto out; 1184 1185 case SATA_PSTATE_SHUTDOWN: 1186 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1187 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1188 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1189 goto out; 1190 1191 case SATA_PSTATE_PWROFF: 1192 sd->satadev_state = SATA_PSTATE_PWROFF; 1193 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1194 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1195 goto out; 1196 1197 case SATA_PSTATE_PWRON: 1198 sd->satadev_state = SATA_PSTATE_PWRON; 1199 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1200 "ahci_tran_probe_port: port %d PORT PWRON", port); 1201 break; 1202 1203 default: 1204 sd->satadev_state = port_state; 1205 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1206 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1207 port, port_state); 1208 break; 1209 } 1210 1211 device_type = ahci_portp->ahciport_device_type; 1212 1213 switch (device_type) { 1214 1215 case SATA_DTYPE_ATADISK: 1216 sd->satadev_type = SATA_DTYPE_ATADISK; 1217 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1218 "ahci_tran_probe_port: port %d DISK found", port); 1219 break; 1220 1221 case SATA_DTYPE_ATAPI: 1222 /* 1223 * HBA driver only knows it's an ATAPI device, and don't know 1224 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1225 * type need to be determined by checking IDENTIFY PACKET 1226 * DEVICE data 1227 */ 1228 sd->satadev_type = SATA_DTYPE_ATAPI; 1229 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1230 "ahci_tran_probe_port: port %d ATAPI found", port); 1231 break; 1232 1233 case SATA_DTYPE_PMULT: 1234 sd->satadev_type = SATA_DTYPE_PMULT; 1235 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1236 "ahci_tran_probe_port: port %d Port Multiplier found", 1237 port); 1238 break; 1239 1240 case SATA_DTYPE_UNKNOWN: 1241 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1242 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1243 "ahci_tran_probe_port: port %d Unknown device found", port); 1244 break; 1245 1246 default: 1247 /* we don't support any other device types */ 1248 sd->satadev_type = SATA_DTYPE_NONE; 1249 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1250 "ahci_tran_probe_port: port %d No device found", port); 1251 break; 1252 } 1253 1254 out: 1255 ahci_update_sata_registers(ahci_ctlp, port, sd); 1256 mutex_exit(&ahci_portp->ahciport_mutex); 1257 1258 return (SATA_SUCCESS); 1259 } 1260 1261 /* 1262 * There are four operation modes in sata framework: 1263 * SATA_OPMODE_INTERRUPTS 1264 * SATA_OPMODE_POLLING 1265 * SATA_OPMODE_ASYNCH 1266 * SATA_OPMODE_SYNCH 1267 * 1268 * Their combined meanings as following: 1269 * 1270 * SATA_OPMODE_SYNCH 1271 * The command has to be completed before sata_tran_start functions returns. 1272 * Either interrupts or polling could be used - it's up to the driver. 1273 * Mode used currently for internal, sata-module initiated operations. 1274 * 1275 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1276 * It is the same as the one above. 1277 * 1278 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1279 * The command has to be completed before sata_tran_start function returns. 1280 * No interrupt used, polling only. This should be the mode used for scsi 1281 * packets with FLAG_NOINTR. 1282 * 1283 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1284 * The command may be queued (callback function specified). Interrupts could 1285 * be used. It's normal operation mode. 1286 */ 1287 /* 1288 * Called by sata framework to transport a sata packet down stream. 1289 */ 1290 static int 1291 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1292 { 1293 ahci_ctl_t *ahci_ctlp; 1294 ahci_port_t *ahci_portp; 1295 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1296 uint8_t port; 1297 1298 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1299 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1300 1301 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1302 "ahci_tran_start enter: cport %d satapkt 0x%p", 1303 cport, (void *)spkt); 1304 1305 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1306 1307 mutex_enter(&ahci_portp->ahciport_mutex); 1308 1309 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1310 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1311 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1312 /* 1313 * In case the targer driver would send the packet before 1314 * sata framework can have the opportunity to process those 1315 * event reports. 1316 */ 1317 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1318 spkt->satapkt_device.satadev_state = 1319 ahci_portp->ahciport_port_state; 1320 ahci_update_sata_registers(ahci_ctlp, port, 1321 &spkt->satapkt_device); 1322 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1323 "ahci_tran_start returning PORT_ERROR while " 1324 "port in FAILED/SHUTDOWN/PWROFF state: " 1325 "port: %d", port); 1326 mutex_exit(&ahci_portp->ahciport_mutex); 1327 return (SATA_TRAN_PORT_ERROR); 1328 } 1329 1330 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1331 /* 1332 * ahci_intr_phyrdy_change() may have rendered it to 1333 * SATA_DTYPE_NONE. 1334 */ 1335 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1336 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1337 spkt->satapkt_device.satadev_state = 1338 ahci_portp->ahciport_port_state; 1339 ahci_update_sata_registers(ahci_ctlp, port, 1340 &spkt->satapkt_device); 1341 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1342 "ahci_tran_start returning PORT_ERROR while " 1343 "no device attached: port: %d", port); 1344 mutex_exit(&ahci_portp->ahciport_mutex); 1345 return (SATA_TRAN_PORT_ERROR); 1346 } 1347 1348 /* 1349 * SATA HBA driver should remember that a device was reset and it 1350 * is supposed to reject any packets which do not specify either 1351 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1352 * 1353 * This is to prevent a race condition when a device was arbitrarily 1354 * reset by the HBA driver (and lost it's setting) and a target 1355 * driver sending some commands to a device before the sata framework 1356 * has a chance to restore the device setting (such as cache enable/ 1357 * disable or other resettable stuff). 1358 */ 1359 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1360 ahci_portp->ahciport_reset_in_progress = 0; 1361 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1362 "ahci_tran_start clearing the " 1363 "reset_in_progress for port: %d", port); 1364 } 1365 1366 if (ahci_portp->ahciport_reset_in_progress && 1367 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1368 ! ddi_in_panic()) { 1369 spkt->satapkt_reason = SATA_PKT_BUSY; 1370 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1371 "ahci_tran_start returning BUSY while " 1372 "reset in progress: port: %d", port); 1373 mutex_exit(&ahci_portp->ahciport_mutex); 1374 return (SATA_TRAN_BUSY); 1375 } 1376 1377 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1378 spkt->satapkt_reason = SATA_PKT_BUSY; 1379 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1380 "ahci_tran_start returning BUSY while " 1381 "mopping in progress: port: %d", port); 1382 mutex_exit(&ahci_portp->ahciport_mutex); 1383 return (SATA_TRAN_BUSY); 1384 } 1385 1386 if (spkt->satapkt_op_mode & 1387 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1388 /* 1389 * If a SYNC command to be executed in interrupt context, 1390 * bounce it back to sata module. 1391 */ 1392 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1393 servicing_interrupt()) { 1394 spkt->satapkt_reason = SATA_PKT_BUSY; 1395 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1396 "ahci_tran_start returning BUSY while " 1397 "sending SYNC mode under interrupt context: " 1398 "port : %d", port); 1399 mutex_exit(&ahci_portp->ahciport_mutex); 1400 return (SATA_TRAN_BUSY); 1401 } 1402 1403 /* We need to do the sync start now */ 1404 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1405 spkt) == AHCI_FAILURE) { 1406 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1407 "return QUEUE_FULL: port %d", port); 1408 mutex_exit(&ahci_portp->ahciport_mutex); 1409 return (SATA_TRAN_QUEUE_FULL); 1410 } 1411 } else { 1412 /* Async start, using interrupt */ 1413 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1414 == AHCI_FAILURE) { 1415 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1416 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1417 "returning QUEUE_FULL: port %d", port); 1418 mutex_exit(&ahci_portp->ahciport_mutex); 1419 return (SATA_TRAN_QUEUE_FULL); 1420 } 1421 } 1422 1423 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1424 "sata tran accepted: port %d", port); 1425 1426 mutex_exit(&ahci_portp->ahciport_mutex); 1427 return (SATA_TRAN_ACCEPTED); 1428 } 1429 1430 /* 1431 * SATA_OPMODE_SYNCH flag is set 1432 * 1433 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1434 * without interrupt, otherwise we can still use the interrupt. 1435 * 1436 * WARNING!!! ahciport_mutex should be acquired before the function 1437 * is called. 1438 */ 1439 static int 1440 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1441 uint8_t port, sata_pkt_t *spkt) 1442 { 1443 int pkt_timeout_ticks; 1444 uint32_t timeout_tags; 1445 int rval; 1446 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1447 1448 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1449 "port %d spkt 0x%p", port, spkt); 1450 1451 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1452 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1453 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1454 port, spkt)) == AHCI_FAILURE) { 1455 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1456 return (rval); 1457 } 1458 1459 pkt_timeout_ticks = 1460 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1461 1462 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1463 mutex_exit(&ahci_portp->ahciport_mutex); 1464 1465 /* Simulate the interrupt */ 1466 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1467 1468 drv_usecwait(AHCI_1MS_USECS); 1469 1470 mutex_enter(&ahci_portp->ahciport_mutex); 1471 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1472 if (pkt_timeout_ticks < 0) { 1473 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1474 "port %d satapkt 0x%p timed out\n", 1475 instance, port, (void *)spkt); 1476 timeout_tags = (0x1 << rval); 1477 mutex_exit(&ahci_portp->ahciport_mutex); 1478 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1479 port, timeout_tags); 1480 mutex_enter(&ahci_portp->ahciport_mutex); 1481 } 1482 } 1483 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1484 return (AHCI_SUCCESS); 1485 1486 } else { 1487 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1488 port, spkt)) == AHCI_FAILURE) 1489 return (rval); 1490 1491 #if AHCI_DEBUG 1492 /* 1493 * Note that the driver always uses the slot 0 to deliver 1494 * REQUEST SENSE or READ LOG EXT command 1495 */ 1496 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1497 ASSERT(rval == 0); 1498 #endif 1499 1500 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1501 cv_wait(&ahci_portp->ahciport_cv, 1502 &ahci_portp->ahciport_mutex); 1503 1504 return (AHCI_SUCCESS); 1505 } 1506 } 1507 1508 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1509 if (satapkt) { \ 1510 satapkt->satapkt_reason = reason; \ 1511 /* \ 1512 * We set the satapkt_reason in both sync and \ 1513 * non-sync cases. \ 1514 */ \ 1515 } \ 1516 if (satapkt && \ 1517 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1518 satapkt->satapkt_comp) { \ 1519 mutex_exit(&ahci_portp->ahciport_mutex); \ 1520 (*satapkt->satapkt_comp)(satapkt); \ 1521 mutex_enter(&ahci_portp->ahciport_mutex); \ 1522 } else { \ 1523 if (satapkt && \ 1524 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1525 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1526 cv_broadcast(&ahci_portp->ahciport_cv); \ 1527 } 1528 1529 /* 1530 * Searches for and claims a free command slot. 1531 * 1532 * Returns: 1533 * 1534 * AHCI_FAILURE if failed 1535 * 1. if no empty slot left 1536 * 2. non-queued command requested while queued command(s) is outstanding 1537 * 3. queued command requested whild non-queued command(s) is outstanding 1538 * 4. HBA doesn't support multiple-use of command list while already a 1539 * non-queued command is oustanding 1540 * 1541 * claimed slot number if succeeded 1542 * 1543 * NOTE: it will always return slot 0 during error recovery process for 1544 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1545 * 1546 * WARNING!!! ahciport_mutex should be acquired before the function 1547 * is called. 1548 */ 1549 static int 1550 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1551 ahci_port_t *ahci_portp, int command_type) 1552 { 1553 uint32_t free_slots; 1554 int slot; 1555 1556 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1557 "ahciport_pending_tags = 0x%x " 1558 "ahciport_pending_ncq_tags = 0x%x", 1559 ahci_portp->ahciport_pending_tags, 1560 ahci_portp->ahciport_pending_ncq_tags); 1561 1562 /* 1563 * According to the AHCI spec, system software is responsible to 1564 * ensure that queued and non-queued commands are not mixed in 1565 * the command list. 1566 */ 1567 if (command_type == AHCI_NON_NCQ_CMD) { 1568 /* Non-NCQ command request */ 1569 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1570 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1571 "ahci_claim_free_slot: there is still pending " 1572 "queued command(s) in the command list, " 1573 "so no available slot for the non-queued " 1574 "command", NULL); 1575 return (AHCI_FAILURE); 1576 } 1577 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1578 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1579 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1580 "ahci_claim_free_slot: HBA cannot support multiple-" 1581 "use of the command list for non-queued commands", 1582 NULL); 1583 return (AHCI_FAILURE); 1584 } 1585 free_slots = (~ahci_portp->ahciport_pending_tags) & 1586 AHCI_SLOT_MASK(ahci_ctlp); 1587 } else if (command_type == AHCI_NCQ_CMD) { 1588 /* NCQ command request */ 1589 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1590 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1591 "ahci_claim_free_slot: there is still pending " 1592 "non-queued command(s) in the command list, " 1593 "so no available slot for the queued command", 1594 NULL); 1595 return (AHCI_FAILURE); 1596 } 1597 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1598 AHCI_NCQ_SLOT_MASK(ahci_portp); 1599 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1600 /* Error retrieval command request */ 1601 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1602 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1603 "SENSE or READ LOG EXT command", NULL); 1604 slot = 0; 1605 goto out; 1606 } 1607 1608 slot = ddi_ffs(free_slots) - 1; 1609 if (slot == -1) { 1610 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1611 "ahci_claim_free_slot: no empty slots", NULL); 1612 return (AHCI_FAILURE); 1613 } 1614 1615 /* 1616 * According to the AHCI spec, to allow a simple mechanism for the 1617 * HBA to map command list slots to queue entries, software must 1618 * match the tag number it uses to the slot it is placing the command 1619 * in. For example, if a queued command is placed in slot 5, the tag 1620 * for that command must be 5. 1621 */ 1622 if (command_type == AHCI_NCQ_CMD) { 1623 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1624 } 1625 1626 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1627 1628 out: 1629 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1630 "ahci_claim_free_slot: found slot: 0x%x", slot); 1631 1632 return (slot); 1633 } 1634 1635 /* 1636 * Builds the Command Table for the sata packet and delivers it to controller. 1637 * 1638 * Returns: 1639 * slot number if we can obtain a slot successfully 1640 * otherwise, return AHCI_FAILURE 1641 * 1642 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1643 */ 1644 static int 1645 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1646 uint8_t port, sata_pkt_t *spkt) 1647 { 1648 int cmd_slot; 1649 sata_cmd_t *scmd; 1650 ahci_fis_h2d_register_t *h2d_register_fisp; 1651 ahci_cmd_table_t *cmd_table; 1652 ahci_cmd_header_t *cmd_header; 1653 int ncookies; 1654 int i; 1655 int command_type = AHCI_NON_NCQ_CMD; 1656 int ncq_qdepth; 1657 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1658 #if AHCI_DEBUG 1659 uint32_t *ptr; 1660 uint8_t *ptr2; 1661 #endif 1662 1663 spkt->satapkt_reason = SATA_PKT_BUSY; 1664 1665 scmd = &spkt->satapkt_cmd; 1666 1667 /* Check if the command is a NCQ command */ 1668 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1669 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1670 command_type = AHCI_NCQ_CMD; 1671 1672 /* 1673 * When NCQ is support, system software must determine the 1674 * maximum tag allowed by the device and the HBA, and it 1675 * must use a value not beyond of the lower bound of the two. 1676 * 1677 * Sata module is going to calculate the qdepth and send 1678 * down to HBA driver via sata_cmd. 1679 */ 1680 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1681 1682 /* 1683 * At the moment, the driver doesn't support the dynamic 1684 * setting of the maximum ncq depth, and the value can be 1685 * set either during the attach or after hot-plug insertion. 1686 */ 1687 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1688 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1689 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 1690 "ahci_deliver_satapkt: port %d the max tags for " 1691 "NCQ command is %d", port, ncq_qdepth); 1692 } else { 1693 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1694 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1695 " port %d the max tag for NCQ command is " 1696 "requested to change from %d to %d, at the" 1697 " moment the driver doesn't support the " 1698 "dynamic change so it's going to " 1699 "still use the previous tag value", 1700 instance, port, 1701 ahci_portp->ahciport_max_ncq_tags, 1702 ncq_qdepth); 1703 } 1704 } 1705 } 1706 1707 /* Check if the command is an error retrieval command */ 1708 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1709 command_type = AHCI_ERR_RETRI_CMD; 1710 1711 /* Check if there is an empty command slot */ 1712 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1713 if (cmd_slot == AHCI_FAILURE) { 1714 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 1715 return (AHCI_FAILURE); 1716 } 1717 1718 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1719 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1720 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1721 cmd_slot, port, (void *)spkt); 1722 1723 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1724 bzero((void *)cmd_table, ahci_cmd_table_size); 1725 1726 /* For data transfer operations, it is the H2D Register FIS */ 1727 h2d_register_fisp = 1728 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1729 1730 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1731 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1732 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1733 SET_FIS_PMP(h2d_register_fisp, 1734 spkt->satapkt_device.satadev_addr.pmport); 1735 } 1736 1737 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1738 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1739 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1740 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1741 1742 switch (scmd->satacmd_addr_type) { 1743 1744 case 0: 1745 /* 1746 * satacmd_addr_type will be 0 for the commands below: 1747 * ATAPI command 1748 * SATAC_IDLE_IM 1749 * SATAC_STANDBY_IM 1750 * SATAC_DOWNLOAD_MICROCODE 1751 * SATAC_FLUSH_CACHE 1752 * SATAC_SET_FEATURES 1753 * SATAC_SMART 1754 * SATAC_ID_PACKET_DEVICE 1755 * SATAC_ID_DEVICE 1756 */ 1757 /* FALLTHRU */ 1758 1759 case ATA_ADDR_LBA: 1760 /* FALLTHRU */ 1761 1762 case ATA_ADDR_LBA28: 1763 /* LBA[7:0] */ 1764 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1765 1766 /* LBA[15:8] */ 1767 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1768 1769 /* LBA[23:16] */ 1770 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1771 1772 /* LBA [27:24] (also called dev_head) */ 1773 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1774 1775 break; 1776 1777 case ATA_ADDR_LBA48: 1778 /* LBA[7:0] */ 1779 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1780 1781 /* LBA[15:8] */ 1782 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1783 1784 /* LBA[23:16] */ 1785 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1786 1787 /* LBA [31:24] */ 1788 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1789 scmd->satacmd_lba_low_msb); 1790 1791 /* LBA [39:32] */ 1792 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1793 scmd->satacmd_lba_mid_msb); 1794 1795 /* LBA [47:40] */ 1796 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1797 scmd->satacmd_lba_high_msb); 1798 1799 /* Set dev_head */ 1800 SET_FIS_DEV_HEAD(h2d_register_fisp, 1801 scmd->satacmd_device_reg); 1802 1803 /* Set the extended sector count and features */ 1804 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1805 scmd->satacmd_sec_count_msb); 1806 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1807 scmd->satacmd_features_reg_ext); 1808 break; 1809 } 1810 1811 /* 1812 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1813 * filled into features field, and sector count 8:15 is filled into 1814 * features (exp) field. TAG is filled into sector field. 1815 */ 1816 if (command_type == AHCI_NCQ_CMD) { 1817 SET_FIS_FEATURES(h2d_register_fisp, 1818 scmd->satacmd_sec_count_lsb); 1819 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1820 scmd->satacmd_sec_count_msb); 1821 1822 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1823 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1824 } 1825 1826 ncookies = scmd->satacmd_num_dma_cookies; 1827 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 1828 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1829 ncookies, ahci_dma_prdt_number); 1830 1831 ASSERT(ncookies <= ahci_dma_prdt_number); 1832 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1833 1834 /* *** now fill the scatter gather list ******* */ 1835 for (i = 0; i < ncookies; i++) { 1836 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1837 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1838 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1839 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1840 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1841 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1842 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1843 scmd->satacmd_dma_cookie_list[i].dmac_size; 1844 } 1845 1846 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 1847 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1848 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1849 1850 /* The ACMD field is filled in for ATAPI command */ 1851 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1852 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1853 SATA_ATAPI_MAX_CDB_LEN); 1854 } 1855 1856 /* Set Command Header in Command List */ 1857 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1858 BZERO_DESCR_INFO(cmd_header); 1859 BZERO_PRD_BYTE_COUNT(cmd_header); 1860 1861 /* Set the number of entries in the PRD table */ 1862 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1863 1864 /* Set the length of the command in the CFIS area */ 1865 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1866 1867 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1868 "sata_data_direction = 0x%x", 1869 scmd->satacmd_flags.sata_data_direction); 1870 1871 /* Set A bit if it is an ATAPI command */ 1872 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1873 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1874 1875 /* Set W bit if data is going to the device */ 1876 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1877 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1878 1879 /* 1880 * Set the prefetchable bit - this bit is only valid if the PRDTL 1881 * field is non-zero or the ATAPI 'A' bit is set in the command 1882 * header. This bit cannot be set when using native command 1883 * queuing commands or when using FIS-based switching with a Port 1884 * multiplier. 1885 */ 1886 if (command_type != AHCI_NCQ_CMD) 1887 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1888 1889 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1890 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1891 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1892 1893 /* 1894 * We are overloading satapkt_hba_driver_private with 1895 * watched_cycle count. 1896 */ 1897 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1898 1899 #if AHCI_DEBUG 1900 if (ahci_debug_flags & AHCIDBG_ATACMD && 1901 scmd->satacmd_cmd_reg != SATAC_PACKET || 1902 ahci_debug_flags & AHCIDBG_ATAPICMD && 1903 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1904 1905 /* Dump the command header and table */ 1906 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1907 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1908 "0x%p cmd_reg 0x%x port %d", spkt, 1909 scmd->satacmd_cmd_reg, port); 1910 ptr = (uint32_t *)cmd_header; 1911 ahci_log(ahci_ctlp, CE_WARN, 1912 " Command Header:%8x %8x %8x %8x", 1913 ptr[0], ptr[1], ptr[2], ptr[3]); 1914 1915 /* Dump the H2D register FIS */ 1916 ptr = (uint32_t *)h2d_register_fisp; 1917 ahci_log(ahci_ctlp, CE_WARN, 1918 " Command FIS: %8x %8x %8x %8x", 1919 ptr[0], ptr[1], ptr[2], ptr[3]); 1920 1921 /* Dump the ACMD register FIS */ 1922 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1923 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1924 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1925 ahci_log(ahci_ctlp, CE_WARN, 1926 " ATAPI command: %2x %2x %2x %2x " 1927 "%2x %2x %2x %2x", 1928 ptr2[8 * i], ptr2[8 * i + 1], 1929 ptr2[8 * i + 2], ptr2[8 * i + 3], 1930 ptr2[8 * i + 4], ptr2[8 * i + 5], 1931 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1932 1933 /* Dump the PRDT */ 1934 for (i = 0; i < ncookies; i++) { 1935 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1936 ahci_log(ahci_ctlp, CE_WARN, 1937 " Cookie %d: %8x %8x %8x %8x", 1938 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1939 } 1940 } 1941 #endif 1942 1943 (void) ddi_dma_sync( 1944 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1945 0, 1946 ahci_cmd_table_size, 1947 DDI_DMA_SYNC_FORDEV); 1948 1949 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1950 cmd_slot * sizeof (ahci_cmd_header_t), 1951 sizeof (ahci_cmd_header_t), 1952 DDI_DMA_SYNC_FORDEV); 1953 1954 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1955 if (command_type == AHCI_NCQ_CMD) { 1956 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1957 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1958 (0x1 << cmd_slot)); 1959 } 1960 1961 /* Indicate to the HBA that a command is active. */ 1962 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1963 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1964 (0x1 << cmd_slot)); 1965 1966 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1967 "exit: port %d", port); 1968 1969 return (cmd_slot); 1970 } 1971 1972 /* 1973 * Called by the sata framework to abort the previously sent packet(s). 1974 * 1975 * Reset device to abort commands. 1976 */ 1977 static int 1978 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1979 { 1980 ahci_ctl_t *ahci_ctlp; 1981 ahci_port_t *ahci_portp; 1982 uint32_t slot_status = 0; 1983 uint32_t aborted_tags = 0; 1984 uint32_t finished_tags = 0; 1985 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1986 uint8_t port; 1987 int tmp_slot; 1988 int instance = ddi_get_instance(dip); 1989 1990 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1991 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1992 1993 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1994 "ahci_tran_abort enter: port %d", port); 1995 1996 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1997 mutex_enter(&ahci_portp->ahciport_mutex); 1998 1999 /* 2000 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2001 * commands are being mopped, therefore there is nothing else to do 2002 */ 2003 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2004 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2005 "ahci_tran_abort: port %d is in " 2006 "mopping process, so just return directly ", port); 2007 mutex_exit(&ahci_portp->ahciport_mutex); 2008 return (SATA_SUCCESS); 2009 } 2010 2011 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2012 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2013 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2014 /* 2015 * In case the targer driver would send the request before 2016 * sata framework can have the opportunity to process those 2017 * event reports. 2018 */ 2019 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2020 spkt->satapkt_device.satadev_state = 2021 ahci_portp->ahciport_port_state; 2022 ahci_update_sata_registers(ahci_ctlp, port, 2023 &spkt->satapkt_device); 2024 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2025 "ahci_tran_abort returning SATA_FAILURE while " 2026 "port in FAILED/SHUTDOWN/PWROFF state: " 2027 "port: %d", port); 2028 mutex_exit(&ahci_portp->ahciport_mutex); 2029 return (SATA_FAILURE); 2030 } 2031 2032 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2033 /* 2034 * ahci_intr_phyrdy_change() may have rendered it to 2035 * AHCI_PORT_TYPE_NODEV. 2036 */ 2037 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2038 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2039 spkt->satapkt_device.satadev_state = 2040 ahci_portp->ahciport_port_state; 2041 ahci_update_sata_registers(ahci_ctlp, port, 2042 &spkt->satapkt_device); 2043 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2044 "ahci_tran_abort returning SATA_FAILURE while " 2045 "no device attached: port: %d", port); 2046 mutex_exit(&ahci_portp->ahciport_mutex); 2047 return (SATA_FAILURE); 2048 } 2049 2050 if (flag == SATA_ABORT_ALL_PACKETS) { 2051 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2052 aborted_tags = ahci_portp->ahciport_pending_tags; 2053 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2054 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2055 2056 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2057 instance, port); 2058 } else { 2059 aborted_tags = 0xffffffff; 2060 /* 2061 * Aborting one specific packet, first search the 2062 * ahciport_slot_pkts[] list for matching spkt. 2063 */ 2064 for (tmp_slot = 0; 2065 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2066 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2067 aborted_tags = (0x1 << tmp_slot); 2068 break; 2069 } 2070 } 2071 2072 if (aborted_tags == 0xffffffff) { 2073 /* request packet is not on the pending list */ 2074 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2075 "Cannot find the aborting pkt 0x%p on the " 2076 "pending list", (void *)spkt); 2077 ahci_update_sata_registers(ahci_ctlp, port, 2078 &spkt->satapkt_device); 2079 mutex_exit(&ahci_portp->ahciport_mutex); 2080 return (SATA_FAILURE); 2081 } 2082 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2083 instance, port, (void *)spkt); 2084 } 2085 2086 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2087 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2088 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2089 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2090 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2091 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2092 2093 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2094 ahci_portp->ahciport_mop_in_progress++; 2095 2096 /* 2097 * To abort the packet(s), first we are trying to clear PxCMD.ST 2098 * to stop the port, and if the port can be stopped 2099 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2100 * then we just send back the aborted packet(s) with ABORTED flag 2101 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2102 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2103 * perform a COMRESET. 2104 */ 2105 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2106 ahci_portp, port, NULL, NULL); 2107 2108 /* 2109 * Compute which have finished and which need to be retried. 2110 * 2111 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2112 * minus the slot_status. The aborted_tags has to be deducted by 2113 * finished_tags since we can't possibly abort a tag which had finished 2114 * already. 2115 */ 2116 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2117 finished_tags = ahci_portp->ahciport_pending_tags & 2118 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2119 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2120 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2121 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2122 2123 aborted_tags &= ~finished_tags; 2124 2125 ahci_mop_commands(ahci_ctlp, 2126 ahci_portp, 2127 slot_status, 2128 0, /* failed tags */ 2129 0, /* timeout tags */ 2130 aborted_tags, 2131 0); /* reset tags */ 2132 2133 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2134 mutex_exit(&ahci_portp->ahciport_mutex); 2135 2136 return (SATA_SUCCESS); 2137 } 2138 2139 /* 2140 * Used to do device reset and reject all the pending packets on a device 2141 * during the reset operation. 2142 * 2143 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2144 */ 2145 static int 2146 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2147 ahci_port_t *ahci_portp, uint8_t port) 2148 { 2149 uint32_t slot_status = 0; 2150 uint32_t reset_tags = 0; 2151 uint32_t finished_tags = 0; 2152 sata_device_t sdevice; 2153 int ret; 2154 2155 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2156 "ahci_reset_device_reject_pkts on port: %d", port); 2157 2158 /* 2159 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2160 * commands are being mopped, therefore there is nothing else to do 2161 */ 2162 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2163 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2164 "ahci_reset_device_reject_pkts: port %d is in " 2165 "mopping process, so return directly ", port); 2166 return (SATA_SUCCESS); 2167 } 2168 2169 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2170 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2171 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2172 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2173 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2174 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2175 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2176 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2177 } 2178 2179 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2180 != AHCI_SUCCESS) { 2181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2182 "Try to do a port reset after software " 2183 "reset failed", port); 2184 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2185 if (ret != AHCI_SUCCESS) { 2186 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2187 "ahci_reset_device_reject_pkts: port %d " 2188 "failed", port); 2189 return (SATA_FAILURE); 2190 } 2191 } 2192 /* Set the reset in progress flag */ 2193 ahci_portp->ahciport_reset_in_progress = 1; 2194 2195 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2196 ahci_portp->ahciport_mop_in_progress++; 2197 2198 /* Indicate to the framework that a reset has happened */ 2199 bzero((void *)&sdevice, sizeof (sata_device_t)); 2200 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2201 sdevice.satadev_addr.pmport = 0; 2202 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2203 2204 sdevice.satadev_state = SATA_DSTATE_RESET | 2205 SATA_DSTATE_PWR_ACTIVE; 2206 mutex_exit(&ahci_portp->ahciport_mutex); 2207 sata_hba_event_notify( 2208 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2209 &sdevice, 2210 SATA_EVNT_DEVICE_RESET); 2211 mutex_enter(&ahci_portp->ahciport_mutex); 2212 2213 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2214 "port %d sending event up: SATA_EVNT_RESET", port); 2215 2216 /* Next try to mop the pending commands */ 2217 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2218 finished_tags = ahci_portp->ahciport_pending_tags & 2219 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2220 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2221 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2222 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2223 2224 reset_tags &= ~finished_tags; 2225 2226 ahci_mop_commands(ahci_ctlp, 2227 ahci_portp, 2228 slot_status, 2229 0, /* failed tags */ 2230 0, /* timeout tags */ 2231 0, /* aborted tags */ 2232 reset_tags); /* reset tags */ 2233 2234 return (SATA_SUCCESS); 2235 } 2236 2237 /* 2238 * Used to do port reset and reject all the pending packets on a port during 2239 * the reset operation. 2240 * 2241 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2242 */ 2243 static int 2244 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2245 ahci_port_t *ahci_portp, uint8_t port) 2246 { 2247 uint32_t slot_status = 0; 2248 uint32_t reset_tags = 0; 2249 uint32_t finished_tags = 0; 2250 2251 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2252 "ahci_reset_port_reject_pkts on port: %d", port); 2253 2254 /* 2255 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2256 * commands are being mopped, therefore there is nothing else to do 2257 */ 2258 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2259 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2260 "ahci_reset_port_reject_pkts: port %d is in " 2261 "mopping process, so return directly ", port); 2262 return (SATA_SUCCESS); 2263 } 2264 2265 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2266 ahci_portp->ahciport_mop_in_progress++; 2267 2268 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2269 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2270 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2271 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2272 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2273 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2274 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2275 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2276 } 2277 2278 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2279 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2280 NULL) != AHCI_SUCCESS) 2281 return (SATA_FAILURE); 2282 2283 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2284 finished_tags = ahci_portp->ahciport_pending_tags & 2285 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2286 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2287 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2288 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2289 2290 reset_tags &= ~finished_tags; 2291 2292 ahci_mop_commands(ahci_ctlp, 2293 ahci_portp, 2294 slot_status, 2295 0, /* failed tags */ 2296 0, /* timeout tags */ 2297 0, /* aborted tags */ 2298 reset_tags); /* reset tags */ 2299 2300 return (SATA_SUCCESS); 2301 } 2302 2303 /* 2304 * Used to do hba reset and reject all the pending packets on all ports 2305 * during the reset operation. 2306 */ 2307 static int 2308 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2309 { 2310 ahci_port_t *ahci_portp; 2311 uint32_t slot_status[AHCI_MAX_PORTS]; 2312 uint32_t reset_tags[AHCI_MAX_PORTS]; 2313 uint32_t finished_tags[AHCI_MAX_PORTS]; 2314 uint8_t port; 2315 int ret = SATA_SUCCESS; 2316 2317 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2318 "ahci_reset_hba_reject_pkts enter", NULL); 2319 2320 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2321 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2322 continue; 2323 } 2324 2325 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2326 2327 mutex_enter(&ahci_portp->ahciport_mutex); 2328 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2329 slot_status[port] = ddi_get32( 2330 ahci_ctlp->ahcictl_ahci_acc_handle, 2331 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2332 reset_tags[port] = slot_status[port] & 2333 AHCI_SLOT_MASK(ahci_ctlp); 2334 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2335 slot_status[port] = ddi_get32( 2336 ahci_ctlp->ahcictl_ahci_acc_handle, 2337 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2338 reset_tags[port] = slot_status[port] & 2339 AHCI_NCQ_SLOT_MASK(ahci_portp); 2340 } 2341 mutex_exit(&ahci_portp->ahciport_mutex); 2342 } 2343 2344 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2345 ret = SATA_FAILURE; 2346 goto out; 2347 } 2348 2349 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2350 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2351 continue; 2352 } 2353 2354 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2355 2356 mutex_enter(&ahci_portp->ahciport_mutex); 2357 /* 2358 * To prevent recursive enter to ahci_mop_commands, we need 2359 * check AHCI_PORT_FLAG_MOPPING flag. 2360 */ 2361 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2362 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2363 "ahci_reset_hba_reject_pkts: port %d is in " 2364 "mopping process, so return directly ", port); 2365 mutex_exit(&ahci_portp->ahciport_mutex); 2366 continue; 2367 } 2368 2369 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2370 ahci_portp->ahciport_mop_in_progress++; 2371 2372 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2373 finished_tags[port] = 2374 ahci_portp->ahciport_pending_tags & 2375 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2376 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2377 finished_tags[port] = 2378 ahci_portp->ahciport_pending_ncq_tags & 2379 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2380 2381 reset_tags[port] &= ~finished_tags[port]; 2382 2383 ahci_mop_commands(ahci_ctlp, 2384 ahci_portp, 2385 slot_status[port], 2386 0, /* failed tags */ 2387 0, /* timeout tags */ 2388 0, /* aborted tags */ 2389 reset_tags[port]); /* reset tags */ 2390 mutex_exit(&ahci_portp->ahciport_mutex); 2391 } 2392 out: 2393 return (ret); 2394 } 2395 2396 /* 2397 * Called by sata framework to reset a port(s) or device. 2398 */ 2399 static int 2400 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2401 { 2402 ahci_ctl_t *ahci_ctlp; 2403 ahci_port_t *ahci_portp; 2404 uint8_t cport = sd->satadev_addr.cport; 2405 uint8_t port; 2406 int ret = SATA_SUCCESS; 2407 int instance = ddi_get_instance(dip); 2408 2409 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2410 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2411 2412 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2413 "ahci_tran_reset_port enter: cport: %d", cport); 2414 2415 switch (sd->satadev_addr.qual) { 2416 case SATA_ADDR_CPORT: 2417 /* Port reset */ 2418 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2419 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2420 "reset port", instance, port); 2421 2422 mutex_enter(&ahci_portp->ahciport_mutex); 2423 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2424 mutex_exit(&ahci_portp->ahciport_mutex); 2425 2426 break; 2427 2428 case SATA_ADDR_DCPORT: 2429 /* Device reset */ 2430 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2431 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2432 "reset device", instance, port); 2433 2434 mutex_enter(&ahci_portp->ahciport_mutex); 2435 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2436 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2437 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2438 /* 2439 * In case the targer driver would send the request 2440 * before sata framework can have the opportunity to 2441 * process those event reports. 2442 */ 2443 sd->satadev_state = ahci_portp->ahciport_port_state; 2444 ahci_update_sata_registers(ahci_ctlp, port, sd); 2445 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2446 "ahci_tran_reset_dport returning SATA_FAILURE " 2447 "while port in FAILED/SHUTDOWN/PWROFF state: " 2448 "port: %d", port); 2449 mutex_exit(&ahci_portp->ahciport_mutex); 2450 ret = SATA_FAILURE; 2451 break; 2452 } 2453 2454 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2455 /* 2456 * ahci_intr_phyrdy_change() may have rendered it to 2457 * AHCI_PORT_TYPE_NODEV. 2458 */ 2459 sd->satadev_type = SATA_DTYPE_NONE; 2460 sd->satadev_state = ahci_portp->ahciport_port_state; 2461 ahci_update_sata_registers(ahci_ctlp, port, sd); 2462 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2463 "ahci_tran_reset_dport returning SATA_FAILURE " 2464 "while no device attached: port: %d", port); 2465 mutex_exit(&ahci_portp->ahciport_mutex); 2466 ret = SATA_FAILURE; 2467 break; 2468 } 2469 2470 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2471 ahci_portp, port); 2472 mutex_exit(&ahci_portp->ahciport_mutex); 2473 break; 2474 2475 case SATA_ADDR_CNTRL: 2476 /* Reset the whole controller */ 2477 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2478 "reset the whole hba", instance, port); 2479 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2480 break; 2481 2482 case SATA_ADDR_PMPORT: 2483 case SATA_ADDR_DPMPORT: 2484 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2485 "ahci_tran_reset_dport: port multiplier will be " 2486 "supported later", NULL); 2487 /* FALLTHRU */ 2488 default: 2489 ret = SATA_FAILURE; 2490 } 2491 2492 return (ret); 2493 } 2494 2495 /* 2496 * Called by sata framework to activate a port as part of hotplug. 2497 * (cfgadm -c connect satax/y) 2498 * Note: Not port-mult aware. 2499 */ 2500 static int 2501 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2502 { 2503 ahci_ctl_t *ahci_ctlp; 2504 ahci_port_t *ahci_portp; 2505 uint8_t cport = satadev->satadev_addr.cport; 2506 uint8_t port; 2507 int instance = ddi_get_instance(dip); 2508 2509 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2510 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2511 2512 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2513 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2514 2515 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2516 2517 mutex_enter(&ahci_portp->ahciport_mutex); 2518 ahci_enable_port_intrs(ahci_ctlp, port); 2519 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2520 2521 /* 2522 * Reset the port so that the PHY communication would be re-established. 2523 * But this reset is an internal operation and the sata module doesn't 2524 * need to know about it. Moreover, the port with a device attached will 2525 * be started too. 2526 */ 2527 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2528 ahci_portp, port, 2529 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2530 NULL); 2531 2532 /* 2533 * Need to check the link status and device status of the port 2534 * and consider raising power if the port was in D3 state 2535 */ 2536 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2537 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2538 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2539 2540 satadev->satadev_state = ahci_portp->ahciport_port_state; 2541 2542 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2543 2544 mutex_exit(&ahci_portp->ahciport_mutex); 2545 return (SATA_SUCCESS); 2546 } 2547 2548 /* 2549 * Called by sata framework to deactivate a port as part of hotplug. 2550 * (cfgadm -c disconnect satax/y) 2551 * Note: Not port-mult aware. 2552 */ 2553 static int 2554 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2555 { 2556 ahci_ctl_t *ahci_ctlp; 2557 ahci_port_t *ahci_portp; 2558 uint8_t cport = satadev->satadev_addr.cport; 2559 uint8_t port; 2560 uint32_t port_scontrol; 2561 int instance = ddi_get_instance(dip); 2562 2563 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2564 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2565 2566 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2567 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2568 2569 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2570 2571 mutex_enter(&ahci_portp->ahciport_mutex); 2572 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2573 instance, port); 2574 2575 /* Disable the interrupts on the port */ 2576 ahci_disable_port_intrs(ahci_ctlp, port); 2577 2578 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2579 goto phy_offline; 2580 } 2581 2582 /* First to abort all the pending commands */ 2583 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2584 2585 /* Then stop the port */ 2586 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2587 ahci_portp, port); 2588 2589 /* Next put the PHY offline */ 2590 2591 phy_offline: 2592 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2593 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2594 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2595 2596 /* Update ahciport_port_state */ 2597 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2598 satadev->satadev_state = ahci_portp->ahciport_port_state; 2599 2600 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2601 2602 mutex_exit(&ahci_portp->ahciport_mutex); 2603 return (SATA_SUCCESS); 2604 } 2605 2606 /* 2607 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2608 * when a device is unplugged or a port is deactivated. 2609 * 2610 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2611 */ 2612 static void 2613 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2614 ahci_port_t *ahci_portp, uint8_t port) 2615 { 2616 uint32_t slot_status = 0; 2617 uint32_t abort_tags = 0; 2618 2619 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2620 "ahci_reject_all_abort_pkts on port: %d", port); 2621 2622 /* 2623 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2624 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2625 * to get the error data, if yes when the device is removed, the 2626 * command needs to be aborted too. 2627 */ 2628 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2629 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2630 slot_status = 0x1; 2631 abort_tags = 0x1; 2632 goto out; 2633 } else { 2634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2635 "ahci_reject_all_abort_pkts return directly " 2636 "port %d no needs to reject any outstanding " 2637 "commands", port); 2638 return; 2639 } 2640 } 2641 2642 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2643 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2644 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2645 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2646 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2647 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2648 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2649 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2650 } 2651 2652 out: 2653 /* No need to do mop when there is no outstanding commands */ 2654 if (slot_status != 0) { 2655 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2656 ahci_portp->ahciport_mop_in_progress++; 2657 2658 ahci_mop_commands(ahci_ctlp, 2659 ahci_portp, 2660 slot_status, 2661 0, /* failed tags */ 2662 0, /* timeout tags */ 2663 abort_tags, /* aborting tags */ 2664 0); /* reset tags */ 2665 } 2666 } 2667 2668 #if defined(__lock_lint) 2669 static int 2670 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2671 { 2672 return (SATA_SUCCESS); 2673 } 2674 #endif 2675 2676 /* 2677 * Allocate the ports structure, only called by ahci_attach 2678 */ 2679 static int 2680 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2681 { 2682 int port, cport = 0; 2683 2684 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2685 "ahci_alloc_ports_state enter", NULL); 2686 2687 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2688 2689 /* Allocate structures only for the implemented ports */ 2690 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2691 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2692 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2693 "hba port %d not implemented", port); 2694 continue; 2695 } 2696 2697 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2698 ahci_ctlp->ahcictl_port_to_cport[port] = 2699 (uint8_t)cport++; 2700 2701 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2702 goto err_out; 2703 } 2704 } 2705 2706 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2707 return (AHCI_SUCCESS); 2708 2709 err_out: 2710 for (port--; port >= 0; port--) { 2711 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2712 ahci_dealloc_port_state(ahci_ctlp, port); 2713 } 2714 } 2715 2716 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2717 return (AHCI_FAILURE); 2718 } 2719 2720 /* 2721 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2722 */ 2723 static void 2724 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2725 { 2726 int port; 2727 2728 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2729 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2730 /* if this port is implemented by the HBA */ 2731 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2732 ahci_dealloc_port_state(ahci_ctlp, port); 2733 } 2734 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2735 } 2736 2737 /* 2738 * Drain the taskq. 2739 */ 2740 static void 2741 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 2742 { 2743 ahci_port_t *ahci_portp; 2744 int port; 2745 2746 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2747 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2748 continue; 2749 } 2750 2751 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2752 2753 mutex_enter(&ahci_portp->ahciport_mutex); 2754 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 2755 mutex_exit(&ahci_portp->ahciport_mutex); 2756 } 2757 } 2758 2759 /* 2760 * Initialize the controller and all ports. And then try to start the ports 2761 * if there are devices attached. 2762 * 2763 * This routine can be called from three seperate cases: DDI_ATTACH, 2764 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2765 * other two cases; device signature probing are attempted only during 2766 * DDI_ATTACH case. 2767 * 2768 * WARNING!!! Disable the whole controller's interrupts before calling and 2769 * the interrupts will be enabled upon successfully return. 2770 */ 2771 static int 2772 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2773 { 2774 ahci_port_t *ahci_portp; 2775 uint32_t ghc_control; 2776 int port; 2777 2778 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2779 "ahci_initialize_controller enter", NULL); 2780 2781 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2782 2783 /* 2784 * Indicate that system software is AHCI aware by setting 2785 * GHC.AE to 1 2786 */ 2787 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2788 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2789 2790 ghc_control |= AHCI_HBA_GHC_AE; 2791 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2792 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2793 ghc_control); 2794 2795 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2796 2797 /* Initialize the implemented ports and structures */ 2798 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2799 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2800 continue; 2801 } 2802 2803 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2804 mutex_enter(&ahci_portp->ahciport_mutex); 2805 2806 /* 2807 * Ensure that the controller is not in the running state 2808 * by checking every implemented port's PxCMD register 2809 */ 2810 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2811 != AHCI_SUCCESS) { 2812 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2813 "ahci_initialize_controller: failed to " 2814 "initialize port %d", port); 2815 /* 2816 * Set the port state to SATA_PSTATE_FAILED if 2817 * failed to initialize it. 2818 */ 2819 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2820 } 2821 2822 mutex_exit(&ahci_portp->ahciport_mutex); 2823 } 2824 2825 /* Enable the whole controller interrupts */ 2826 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2827 ahci_enable_all_intrs(ahci_ctlp); 2828 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2829 2830 return (AHCI_SUCCESS); 2831 } 2832 2833 /* 2834 * Reverse of ahci_initialize_controller() 2835 * 2836 * We only need to stop the ports and disable the interrupt. 2837 */ 2838 static void 2839 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2840 { 2841 ahci_port_t *ahci_portp; 2842 int port; 2843 2844 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2845 "ahci_uninitialize_controller enter", NULL); 2846 2847 /* disable all the interrupts. */ 2848 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2849 ahci_disable_all_intrs(ahci_ctlp); 2850 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2851 2852 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2853 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2854 continue; 2855 } 2856 2857 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2858 2859 /* Stop the port by clearing PxCMD.ST */ 2860 mutex_enter(&ahci_portp->ahciport_mutex); 2861 2862 /* 2863 * Here we must disable the port interrupt because 2864 * ahci_disable_all_intrs only clear GHC.IE, and IS 2865 * register will be still set if PxIE is enabled. 2866 * When ahci shares one IRQ with other drivers, the 2867 * intr handler may claim the intr mistakenly. 2868 */ 2869 ahci_disable_port_intrs(ahci_ctlp, port); 2870 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2871 ahci_portp, port); 2872 mutex_exit(&ahci_portp->ahciport_mutex); 2873 } 2874 } 2875 2876 /* 2877 * The routine is to initialize the port. First put the port in NOTRunning 2878 * state, then enable port interrupt and clear Serror register. And under 2879 * AHCI_ATTACH case, find device signature and then try to start the port. 2880 * 2881 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2882 */ 2883 static int 2884 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2885 ahci_port_t *ahci_portp, uint8_t port) 2886 { 2887 uint32_t port_sstatus, port_task_file, port_cmd_status; 2888 int ret; 2889 2890 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2891 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2892 2893 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2894 "ahci_initialize_port: port %d " 2895 "port_cmd_status = 0x%x", port, port_cmd_status); 2896 /* 2897 * Check whether the port is in NotRunning state, if not, 2898 * put the port in NotRunning state 2899 */ 2900 if (port_cmd_status & 2901 (AHCI_CMD_STATUS_ST | 2902 AHCI_CMD_STATUS_CR | 2903 AHCI_CMD_STATUS_FRE | 2904 AHCI_CMD_STATUS_FR)) { 2905 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2906 ahci_portp, port); 2907 } 2908 2909 /* Device is unknown at first */ 2910 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 2911 2912 /* Disable the interface power management */ 2913 ahci_disable_interface_pm(ahci_ctlp, port); 2914 2915 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2916 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 2917 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2918 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 2919 2920 /* Check physcial link status */ 2921 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 2922 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 2923 2924 /* Check interface status */ 2925 port_task_file & AHCI_TFD_STS_BSY || 2926 port_task_file & AHCI_TFD_STS_DRQ) { 2927 2928 /* Incorrect task file state, we need to reset port */ 2929 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2930 2931 /* Does port reset succeed on HBA port? */ 2932 if (ret != AHCI_SUCCESS) { 2933 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 2934 "ahci_initialize_port:" 2935 "port reset faild at port %d", port); 2936 return (AHCI_FAILURE); 2937 } 2938 2939 /* Is port failed? */ 2940 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 2941 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 2942 "ahci_initialize_port: port %d state 0x%x", 2943 port, ahci_portp->ahciport_port_state); 2944 return (AHCI_FAILURE); 2945 } 2946 } 2947 2948 ahci_portp->ahciport_port_state = SATA_STATE_READY; 2949 2950 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 2951 2952 /* 2953 * At the time being, only probe ports/devices and get the types of 2954 * attached devices during DDI_ATTACH. In fact, the device can be 2955 * changed during power state changes, but at the time being, we 2956 * don't support the situation. 2957 */ 2958 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2959 /* 2960 * Try to get the device signature if the port is 2961 * not empty. 2962 */ 2963 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 2964 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2965 } else { 2966 2967 /* 2968 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2969 * and PxFBU registers in case these registers were cleared 2970 * during the suspend. 2971 */ 2972 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 2973 "ahci_initialize_port: port %d " 2974 "reset the port during resume", port); 2975 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2976 2977 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 2978 "ahci_initialize_port: port %d " 2979 "set PxCLB, PxCLBU, PxFB and PxFBU " 2980 "during resume", port); 2981 2982 /* Config Port Received FIS Base Address */ 2983 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2984 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2985 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2986 2987 /* Config Port Command List Base Address */ 2988 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2989 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2990 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2991 } 2992 2993 /* Return directly if no device connected */ 2994 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2995 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2996 "No device connected to port %d", port); 2997 goto out; 2998 } 2999 3000 /* Try to start the port */ 3001 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 3002 != AHCI_SUCCESS) { 3003 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3004 "failed to start port %d", port); 3005 return (AHCI_FAILURE); 3006 } 3007 out: 3008 /* Enable port interrupts */ 3009 ahci_enable_port_intrs(ahci_ctlp, port); 3010 3011 return (AHCI_SUCCESS); 3012 } 3013 3014 /* 3015 * Handle hardware defect, and check the capabilities. For example, 3016 * power management capabilty and MSI capability. 3017 */ 3018 static int 3019 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 3020 { 3021 ushort_t venid, devid; 3022 ushort_t caps_ptr, cap_count, cap; 3023 #if AHCI_DEBUG 3024 ushort_t pmcap, pmcsr; 3025 ushort_t msimc; 3026 #endif 3027 uint8_t revision; 3028 3029 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3030 PCI_CONF_VENID); 3031 3032 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3033 PCI_CONF_DEVID); 3034 3035 /* 3036 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 3037 * controllers with 0x00 revision id work on 4-byte aligned buffer, 3038 * which is a bug and was fixed after 0x00 revision id controllers. 3039 * 3040 * Moreover, VT8251 cannot use multiple command slots in the command 3041 * list for non-queued commands because the previous register content 3042 * of PxCI can be re-written in the register write, so a flag will be 3043 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 3044 */ 3045 if (venid == VIA_VENID) { 3046 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3047 PCI_CONF_REVID); 3048 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3049 "revision id = 0x%x", revision); 3050 if (revision == 0x00) { 3051 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 3052 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3053 "change ddi_attr_align to 0x4", NULL); 3054 } 3055 3056 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 3057 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3058 "VT8251 cannot use multiple command lists for " 3059 "non-queued commands", NULL); 3060 } 3061 3062 /* 3063 * AMD/ATI SB600 (1002,4380) AHCI chipset doesn't support 64-bit DMA 3064 * addressing for both data buffer and communication memory descriptors 3065 * though S64A bit of CAP register declares the support. 3066 */ 3067 if (venid == 0x1002 && devid == 0x4380) { 3068 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3069 "ATI SB600 cannot do 64-bit DMA for both data buffer " 3070 "and communication memory descriptors though CAP indicates " 3071 "support, so force it to use 32-bit DMA", NULL); 3072 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 3073 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3074 } 3075 3076 /* 3077 * AMD/ATI SB700/750 (1002,4391) AHCI chipset doesn't support 64-bit 3078 * DMA addressing for communication memory descriptors though S64A bit 3079 * of CAP register declares the support. However, it does support 3080 * 64-bit DMA for data buffer. 3081 */ 3082 if (venid == 0x1002 && devid == 0x4391) { 3083 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3084 "ATI SB700/750 cannot do 64-bit DMA for communication " 3085 "memory descriptors though CAP indicates support, " 3086 "so force it to use 32-bit DMA", NULL); 3087 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3088 } 3089 3090 /* 3091 * nVidia MCP78 AHCI controller (pci10de,0ad4) will be forced to use 3092 * Fixed interrupt until the known CR 6766472 (MSIs do not function 3093 * on most Nvidia boards) is fixed. 3094 */ 3095 if (venid == 0x10de && devid == 0x0ad4) { 3096 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3097 "Force nVidia MCP78 AHCI controller to use " 3098 "fixed interrupts", NULL); 3099 ahci_msi_enabled = B_FALSE; 3100 } 3101 3102 /* 3103 * Check if capabilities list is supported and if so, 3104 * get initial capabilities pointer and clear bits 0,1. 3105 */ 3106 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3107 PCI_CONF_STAT) & PCI_STAT_CAP) { 3108 caps_ptr = P2ALIGN(pci_config_get8( 3109 ahci_ctlp->ahcictl_pci_conf_handle, 3110 PCI_CONF_CAP_PTR), 4); 3111 } else { 3112 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3113 } 3114 3115 /* 3116 * Walk capabilities if supported. 3117 */ 3118 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3119 3120 /* 3121 * Check that we haven't exceeded the maximum number of 3122 * capabilities and that the pointer is in a valid range. 3123 */ 3124 if (++cap_count > PCI_CAP_MAX_PTR) { 3125 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3126 "too many device capabilities", NULL); 3127 return (AHCI_FAILURE); 3128 } 3129 if (caps_ptr < PCI_CAP_PTR_OFF) { 3130 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3131 "capabilities pointer 0x%x out of range", 3132 caps_ptr); 3133 return (AHCI_FAILURE); 3134 } 3135 3136 /* 3137 * Get next capability and check that it is valid. 3138 * For now, we only support power management. 3139 */ 3140 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3141 caps_ptr); 3142 switch (cap) { 3143 case PCI_CAP_ID_PM: 3144 3145 /* power management supported */ 3146 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3147 3148 /* Save PMCSR offset */ 3149 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3150 3151 #if AHCI_DEBUG 3152 pmcap = pci_config_get16( 3153 ahci_ctlp->ahcictl_pci_conf_handle, 3154 caps_ptr + PCI_PMCAP); 3155 pmcsr = pci_config_get16( 3156 ahci_ctlp->ahcictl_pci_conf_handle, 3157 ahci_ctlp->ahcictl_pmcsr_offset); 3158 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3159 "Power Management capability found PCI_PMCAP " 3160 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3161 if ((pmcap & 0x3) == 0x3) 3162 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3163 "PCI Power Management Interface " 3164 "spec 1.2 compliant", NULL); 3165 #endif 3166 break; 3167 3168 case PCI_CAP_ID_MSI: 3169 #if AHCI_DEBUG 3170 msimc = pci_config_get16( 3171 ahci_ctlp->ahcictl_pci_conf_handle, 3172 caps_ptr + PCI_MSI_CTRL); 3173 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3174 "Message Signaled Interrupt capability found " 3175 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3176 #endif 3177 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3178 "MSI capability found", NULL); 3179 break; 3180 3181 case PCI_CAP_ID_PCIX: 3182 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3183 "PCI-X capability found", NULL); 3184 break; 3185 3186 case PCI_CAP_ID_PCI_E: 3187 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3188 "PCI Express capability found", NULL); 3189 break; 3190 3191 case PCI_CAP_ID_MSI_X: 3192 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3193 "MSI-X capability found", NULL); 3194 break; 3195 3196 case PCI_CAP_ID_SATA: 3197 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3198 "SATA capability found", NULL); 3199 break; 3200 3201 case PCI_CAP_ID_VS: 3202 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3203 "Vendor Specific capability found", NULL); 3204 break; 3205 3206 default: 3207 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3208 "unrecognized capability 0x%x", cap); 3209 break; 3210 } 3211 3212 /* 3213 * Get next capabilities pointer and clear bits 0,1. 3214 */ 3215 caps_ptr = P2ALIGN(pci_config_get8( 3216 ahci_ctlp->ahcictl_pci_conf_handle, 3217 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3218 } 3219 3220 return (AHCI_SUCCESS); 3221 } 3222 3223 /* 3224 * AHCI device reset ...; a single device on one of the ports is reset, 3225 * but the HBA and physical communication remain intact. This is the 3226 * least intrusive. 3227 * 3228 * When issuing a software reset sequence, there should not be other 3229 * commands in the command list, so we will first clear and then re-set 3230 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3231 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3232 * cleared unless command list override (PxCMD.CLO) is supported. 3233 * 3234 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3235 * set before the function is called. 3236 */ 3237 static int 3238 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3239 uint8_t port) 3240 { 3241 ahci_fis_h2d_register_t *h2d_register_fisp; 3242 ahci_cmd_table_t *cmd_table; 3243 ahci_cmd_header_t *cmd_header; 3244 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3245 int slot, loop_count; 3246 int rval = AHCI_FAILURE; 3247 3248 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3249 "Port %d device resetting", port); 3250 3251 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 3252 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 3253 port) != AHCI_SUCCESS) { 3254 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3255 "ahci_software_reset: cannot stop HBA port %d.", port); 3256 goto out; 3257 } 3258 3259 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3260 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3261 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3262 3263 if (port_task_file & AHCI_TFD_STS_BSY || 3264 port_task_file & AHCI_TFD_STS_DRQ) { 3265 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 3266 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3267 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 3268 "cannot issue a software reset.", port_task_file); 3269 goto out; 3270 } 3271 3272 /* 3273 * If HBA Support CLO, as Command List Override (CAP.SCLO is 3274 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 3275 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 3276 */ 3277 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3278 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 3279 3280 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3281 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3282 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3283 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3284 port_cmd_status|AHCI_CMD_STATUS_CLO); 3285 3286 /* Waiting till PxCMD.SCLO bit is cleared */ 3287 loop_count = 0; 3288 do { 3289 /* Wait for 10 millisec */ 3290 drv_usecwait(AHCI_10MS_USECS); 3291 3292 /* We are effectively timing out after 1 sec. */ 3293 if (loop_count++ > 100) { 3294 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3295 "SCLO time out. port %d is busy.", port); 3296 goto out; 3297 } 3298 3299 port_cmd_status = 3300 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3301 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3302 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 3303 3304 /* Re-check */ 3305 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3306 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3307 if (port_task_file & AHCI_TFD_STS_BSY || 3308 port_task_file & AHCI_TFD_STS_DRQ) { 3309 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3310 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 3311 port_task_file); 3312 goto out; 3313 } 3314 } 3315 3316 /* Then start port */ 3317 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 3318 != AHCI_SUCCESS) { 3319 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3320 "ahci_software_reset: cannot start AHCI port %d.", port); 3321 goto out; 3322 } 3323 3324 /* 3325 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 3326 * ahci_port.ahciport_pending_ncq_tags may fail 3327 * ahci_claim_free_slot(). Actually according to spec, by clearing 3328 * PxCMD.ST there is no command outstanding while executing software 3329 * reseting. Hence we directly use slot 0 instead of 3330 * ahci_claim_free_slot(). 3331 */ 3332 slot = 0; 3333 3334 /* Now send the first H2D Register FIS with SRST set to 1 */ 3335 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3336 bzero((void *)cmd_table, ahci_cmd_table_size); 3337 3338 h2d_register_fisp = 3339 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3340 3341 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3342 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3343 3344 /* Set Command Header in Command List */ 3345 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3346 BZERO_DESCR_INFO(cmd_header); 3347 BZERO_PRD_BYTE_COUNT(cmd_header); 3348 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3349 3350 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3351 SET_RESET(cmd_header, 1); 3352 SET_WRITE(cmd_header, 1); 3353 3354 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3355 0, 3356 ahci_cmd_table_size, 3357 DDI_DMA_SYNC_FORDEV); 3358 3359 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3360 slot * sizeof (ahci_cmd_header_t), 3361 sizeof (ahci_cmd_header_t), 3362 DDI_DMA_SYNC_FORDEV); 3363 3364 /* Indicate to the HBA that a command is active. */ 3365 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3366 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3367 (0x1 << slot)); 3368 3369 loop_count = 0; 3370 3371 /* Loop till the first command is finished */ 3372 do { 3373 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3374 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3375 3376 /* We are effectively timing out after 1 sec. */ 3377 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3378 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3379 "the first SRST FIS is timed out, " 3380 "loop_count = %d", loop_count); 3381 goto out; 3382 } 3383 3384 /* Wait for 10 millisec */ 3385 delay(AHCI_10MS_TICKS); 3386 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3387 3388 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 3389 "ahci_software_reset: 1st loop count: %d, " 3390 "port_cmd_issue = 0x%x, slot = 0x%x", 3391 loop_count, port_cmd_issue, slot); 3392 3393 /* According to ATA spec, we need wait at least 5 microsecs here. */ 3394 drv_usecwait(AHCI_1MS_USECS); 3395 3396 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3397 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3398 bzero((void *)cmd_table, ahci_cmd_table_size); 3399 3400 h2d_register_fisp = 3401 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3402 3403 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3404 3405 /* Set Command Header in Command List */ 3406 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3407 BZERO_DESCR_INFO(cmd_header); 3408 BZERO_PRD_BYTE_COUNT(cmd_header); 3409 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3410 3411 SET_WRITE(cmd_header, 1); 3412 3413 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3414 0, 3415 ahci_cmd_table_size, 3416 DDI_DMA_SYNC_FORDEV); 3417 3418 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3419 slot * sizeof (ahci_cmd_header_t), 3420 sizeof (ahci_cmd_header_t), 3421 DDI_DMA_SYNC_FORDEV); 3422 3423 /* Indicate to the HBA that a command is active. */ 3424 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3425 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3426 (0x1 << slot)); 3427 3428 loop_count = 0; 3429 3430 /* Loop till the second command is finished */ 3431 do { 3432 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3433 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3434 3435 /* We are effectively timing out after 1 sec. */ 3436 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3437 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3438 "the second SRST FIS is timed out, " 3439 "loop_count = %d", loop_count); 3440 goto out; 3441 } 3442 3443 /* Wait for 10 millisec */ 3444 delay(AHCI_10MS_TICKS); 3445 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3446 3447 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 3448 "ahci_software_reset: 2nd loop count: %d, " 3449 "port_cmd_issue = 0x%x, slot = 0x%x", 3450 loop_count, port_cmd_issue, slot); 3451 3452 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3453 3454 rval = AHCI_SUCCESS; 3455 out: 3456 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3457 "ahci_software_reset: %s at port %d", 3458 rval == AHCI_SUCCESS ? "succeed" : "failed", 3459 port); 3460 3461 return (rval); 3462 } 3463 3464 /* 3465 * AHCI port reset ...; the physical communication between the HBA and device 3466 * on a port are disabled. This is more intrusive. 3467 * 3468 * When an HBA or port reset occurs, Phy communication is going to 3469 * be re-established with the device through a COMRESET followed by the 3470 * normal out-of-band communication sequence defined in Serial ATA. AT 3471 * the end of reset, the device, if working properly, will send a D2H 3472 * Register FIS, which contains the device signature. When the HBA receives 3473 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3474 * the PxSIG register with the signature. 3475 * 3476 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3477 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3478 * AHCI 1.0 spec. 3479 */ 3480 /* 3481 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3482 * cleared before the function is called. 3483 */ 3484 static int 3485 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3486 { 3487 uint32_t cap_status, port_cmd_status; 3488 uint32_t port_scontrol, port_sstatus, port_serror; 3489 uint32_t port_intr_status, port_task_file; 3490 3491 int loop_count; 3492 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3493 3494 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3495 "Port %d port resetting...", port); 3496 ahci_portp->ahciport_port_state = 0; 3497 3498 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3499 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3500 3501 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3502 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3503 3504 if (cap_status & AHCI_HBA_CAP_SSS) { 3505 /* 3506 * HBA support staggered spin-up, if the port has 3507 * not spin up yet, then force it to do spin-up 3508 */ 3509 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3510 if (!(ahci_portp->ahciport_flags 3511 & AHCI_PORT_FLAG_SPINUP)) { 3512 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3513 "Port %d PxCMD.SUD is zero, force " 3514 "it to do spin-up", port); 3515 ahci_portp->ahciport_flags |= 3516 AHCI_PORT_FLAG_SPINUP; 3517 } 3518 } 3519 } else { 3520 /* 3521 * HBA doesn't support stagger spin-up, force it 3522 * to do normal COMRESET 3523 */ 3524 if (ahci_portp->ahciport_flags & 3525 AHCI_PORT_FLAG_SPINUP) { 3526 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3527 "HBA does not support staggered spin-up " 3528 "force it to do normal COMRESET", NULL); 3529 ahci_portp->ahciport_flags &= 3530 ~AHCI_PORT_FLAG_SPINUP; 3531 } 3532 } 3533 3534 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3535 /* Do normal COMRESET */ 3536 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3537 "ahci_port_reset: do normal COMRESET", port); 3538 3539 /* 3540 * According to the spec, SUD bit should be set here, 3541 * but JMicron JMB363 doesn't follow it, so remove 3542 * the assertion, and just print a debug message. 3543 */ 3544 #if AHCI_DEBUG 3545 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3546 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3547 "port %d SUD bit not set", port) 3548 #endif 3549 3550 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3551 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3552 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3553 3554 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3555 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3556 port_scontrol); 3557 3558 /* Enable PxCMD.FRE to read device */ 3559 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3560 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3561 port_cmd_status|AHCI_CMD_STATUS_FRE); 3562 3563 /* 3564 * Give time for COMRESET to percolate, according to the AHCI 3565 * spec, software shall wait at least 1 millisecond before 3566 * clearing PxSCTL.DET 3567 */ 3568 delay(AHCI_1MS_TICKS*2); 3569 3570 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3571 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3572 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3573 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3574 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3575 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3576 port_scontrol); 3577 } else { 3578 /* Do staggered spin-up */ 3579 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3580 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3581 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3582 3583 /* PxSCTL.DET must be 0 */ 3584 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3585 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3586 port_scontrol); 3587 3588 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3589 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3590 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3591 port_cmd_status); 3592 3593 /* 0 -> 1 edge */ 3594 delay(AHCI_1MS_TICKS*2); 3595 3596 /* Set PxCMD.SUD to 1 */ 3597 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3598 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3599 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3600 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3601 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3602 port_cmd_status); 3603 3604 /* Enable PxCMD.FRE to read device */ 3605 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3606 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3607 port_cmd_status|AHCI_CMD_STATUS_FRE); 3608 } 3609 3610 /* 3611 * The port enters P:StartComm state, and HBA tells link layer to 3612 * start communication, which involves sending COMRESET to device. 3613 * And the HBA resets PxTFD.STS to 7Fh. 3614 * 3615 * When a COMINIT is received from the device, then the port enters 3616 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3617 * PxSSTS.DET to 1h to indicate a device is detected but communication 3618 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3619 * a COMINIT has been received. 3620 */ 3621 /* 3622 * The DET field is valid only if IPM field indicates 3623 * that the interface is in active state. 3624 */ 3625 loop_count = 0; 3626 do { 3627 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3628 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3629 3630 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3631 /* 3632 * If the interface is not active, the DET field 3633 * is considered not accurate. So we want to 3634 * continue looping. 3635 */ 3636 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3637 } 3638 3639 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3640 /* 3641 * We are effectively timing out after 0.1 sec. 3642 */ 3643 break; 3644 } 3645 3646 /* Wait for 10 millisec */ 3647 delay(AHCI_10MS_TICKS); 3648 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3649 3650 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3651 "ahci_port_reset: 1st loop count: %d, " 3652 "port_sstatus = 0x%x port %d", 3653 loop_count, port_sstatus, port); 3654 3655 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3656 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3657 /* 3658 * Either the port is not active or there 3659 * is no device present. 3660 */ 3661 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3662 return (AHCI_SUCCESS); 3663 } 3664 3665 /* Now we can make sure there is a device connected to the port */ 3666 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3667 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3668 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3669 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 3670 3671 /* 3672 * A COMINIT signal is supposed to be received 3673 * PxSERR.DIAG.X or PxIS.PCS should be set 3674 */ 3675 if (!(port_intr_status & AHCI_INTR_STATUS_PCS) && 3676 !(port_serror & SERROR_EXCHANGED_ERR)) { 3677 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3678 "COMINIT signal from the device not received", 3679 instance, port); 3680 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3681 return (AHCI_FAILURE); 3682 } 3683 3684 /* 3685 * According to the spec, when PxSCTL.DET is set to 0h, upon 3686 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3687 * shall be set to '1' by the HBA. 3688 * 3689 * However, we found JMicron JMB363 doesn't follow this, so 3690 * remove this check, and just print a debug message. 3691 */ 3692 #if AHCI_DEBUG 3693 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3694 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3695 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3696 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3697 "port %d BSY bit is not set after COMINIT signal " 3698 "is received", port); 3699 } 3700 #endif 3701 3702 /* 3703 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3704 * the D2H FIS received by HBA. 3705 */ 3706 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3707 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3708 SERROR_EXCHANGED_ERR); 3709 3710 /* 3711 * Devices should return a FIS contains its signature to HBA after 3712 * COMINIT signal. Check whether a D2H FIS is received by polling 3713 * PxTFD.STS.ERR bit. 3714 */ 3715 loop_count = 0; 3716 do { 3717 /* Wait for 10 millisec */ 3718 delay(AHCI_10MS_TICKS); 3719 3720 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3721 /* 3722 * We are effectively timing out after 11 sec. 3723 */ 3724 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3725 "the device hardware has been initialized and " 3726 "the power-up diagnostics failed", 3727 instance, port); 3728 3729 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3730 "port %d PxTFD.STS.ERR is not set, we need another " 3731 "software reset.", port); 3732 3733 /* Clear port serror register for the port */ 3734 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3735 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3736 AHCI_SERROR_CLEAR_ALL); 3737 3738 /* Try another software reset. */ 3739 if (ahci_software_reset(ahci_ctlp, ahci_portp, 3740 port) != AHCI_SUCCESS) { 3741 ahci_portp->ahciport_port_state |= 3742 SATA_PSTATE_FAILED; 3743 return (AHCI_FAILURE); 3744 } 3745 break; 3746 } 3747 3748 /* 3749 * The Error bit '1' means COMRESET is finished successfully 3750 * The device hardware has been initialized and the power-up 3751 * diagnostics successfully completed. The device requests 3752 * that the Transport layer transmit a Register - D2H FIS to 3753 * the host. (SATA spec 11.5, v2.6) 3754 */ 3755 port_task_file = 3756 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3757 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3758 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3759 >> AHCI_TFD_ERR_SHIFT) != AHCI_TFD_ERR_SGS); 3760 3761 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3762 "ahci_port_reset: 2nd loop count: %d, " 3763 "port_task_file = 0x%x port %d", 3764 loop_count, port_task_file, port); 3765 3766 /* Clear port serror register for the port */ 3767 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3768 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3769 AHCI_SERROR_CLEAR_ALL); 3770 3771 /* Set port as ready */ 3772 ahci_portp->ahciport_port_state |= SATA_STATE_READY; 3773 3774 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 3775 "ahci_port_reset: succeed at port %d.", port); 3776 return (AHCI_SUCCESS); 3777 } 3778 3779 /* 3780 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3781 * This is the most intrusive. 3782 * 3783 * When an HBA reset occurs, Phy communication will be re-established with 3784 * the device through a COMRESET followed by the normal out-of-band 3785 * communication sequence defined in Serial ATA. AT the end of reset, the 3786 * device, if working properly, will send a D2H Register FIS, which contains 3787 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3788 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3789 * signature. 3790 * 3791 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3792 */ 3793 static int 3794 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3795 { 3796 ahci_port_t *ahci_portp; 3797 uint32_t ghc_control; 3798 uint8_t port; 3799 int loop_count; 3800 int rval = AHCI_SUCCESS; 3801 3802 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 3803 NULL); 3804 3805 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3806 3807 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3808 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3809 3810 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3811 ghc_control |= AHCI_HBA_GHC_HR; 3812 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3813 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3814 3815 /* 3816 * Wait until HBA Reset complete or timeout 3817 */ 3818 loop_count = 0; 3819 do { 3820 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3821 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3822 3823 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3824 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3825 "ahci hba reset is timing out, " 3826 "ghc_control = 0x%x", ghc_control); 3827 /* We are effectively timing out after 1 sec. */ 3828 break; 3829 } 3830 3831 /* Wait for 10 millisec */ 3832 delay(AHCI_10MS_TICKS); 3833 } while (ghc_control & AHCI_HBA_GHC_HR); 3834 3835 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3836 "ahci_hba_reset: 1st loop count: %d, " 3837 "ghc_control = 0x%x", loop_count, ghc_control); 3838 3839 if (ghc_control & AHCI_HBA_GHC_HR) { 3840 /* The hba is not reset for some reasons */ 3841 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3842 "hba reset failed: HBA in a hung or locked state", NULL); 3843 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3844 return (AHCI_FAILURE); 3845 } 3846 3847 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3848 /* Only check implemented ports */ 3849 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3850 continue; 3851 } 3852 3853 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3854 mutex_enter(&ahci_portp->ahciport_mutex); 3855 3856 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3857 != AHCI_SUCCESS) { 3858 rval = AHCI_FAILURE; 3859 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3860 "ahci_hba_reset: port %d failed", port); 3861 } 3862 3863 mutex_exit(&ahci_portp->ahciport_mutex); 3864 } 3865 3866 /* 3867 * Indicate that system software is AHCI aware by setting 3868 * GHC.AE to 1 3869 */ 3870 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3871 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3872 3873 ghc_control |= AHCI_HBA_GHC_AE; 3874 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3875 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3876 3877 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3878 3879 return (rval); 3880 } 3881 3882 /* 3883 * This routine is only called from AHCI_ATTACH or phyrdy change 3884 * case. It first calls software reset, then stop the port and try to 3885 * read PxSIG register to find the type of device attached to the port. 3886 * 3887 * The caller should make sure a valid device exists on specified port and 3888 * physical communication has been established so that the signature could 3889 * be retrieved by software reset. 3890 * 3891 * WARNING!!! ahciport_mutex should be acquired before the function 3892 * is called. And the port interrupt is disabled. 3893 */ 3894 static void 3895 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3896 ahci_port_t *ahci_portp, uint8_t port) 3897 { 3898 uint32_t signature; 3899 3900 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3901 "ahci_find_dev_signature enter: port %d", port); 3902 3903 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3904 3905 /* Issue a software reset to get the signature */ 3906 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 3907 != AHCI_SUCCESS) { 3908 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3909 "ahci_find_dev_signature: software reset failed " 3910 "at port %d, cannot get signature.", port); 3911 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3912 return; 3913 } 3914 3915 /* 3916 * ahci_software_reset has started the port, so we need manually stop 3917 * the port again. 3918 */ 3919 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, port) 3920 != AHCI_SUCCESS) { 3921 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3922 "ahci_find_dev_signature: cannot stop port %d.", port); 3923 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3924 return; 3925 } 3926 3927 /* Now we can make sure that a valid signature is received. */ 3928 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3929 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3930 3931 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3932 "ahci_find_dev_signature: port %d signature = 0x%x", 3933 port, signature); 3934 3935 switch (signature) { 3936 3937 case AHCI_SIGNATURE_DISK: 3938 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3939 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3940 "Disk is found at port: %d", port); 3941 break; 3942 3943 case AHCI_SIGNATURE_ATAPI: 3944 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3945 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3946 "ATAPI device is found at port: %d", port); 3947 break; 3948 3949 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3950 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3951 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3952 "Port Multiplier is found at port: %d", port); 3953 break; 3954 3955 default: 3956 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3957 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3958 "Unknown device is found at port: %d", port); 3959 } 3960 } 3961 3962 /* 3963 * According to the spec, to reliably detect hot plug removals, software 3964 * must disable interface power management. Software should perform the 3965 * following initialization on a port after a device is attached: 3966 * Set PxSCTL.IPM to 3h to disable interface state transitions 3967 * Set PxCMD.ALPE to '0' to disable aggressive power management 3968 * Disable device initiated interface power management by SET FEATURE 3969 * 3970 * We can ignore the last item because by default the feature is disabled 3971 */ 3972 static void 3973 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3974 { 3975 uint32_t port_scontrol, port_cmd_status; 3976 3977 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3978 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3979 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3980 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3981 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3982 3983 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3984 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3985 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3986 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3987 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3988 } 3989 3990 /* 3991 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3992 * to 1, then set it firstly. 3993 * 3994 * Each port contains two major DMA engines. One DMA engine walks through 3995 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3996 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3997 * 3998 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3999 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 4000 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 4001 * 4002 * Software shall not set PxCMD.ST to '1' unless a functional device is 4003 * present on the port(as determined by PxTFD.STS.BSY = '0', 4004 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 4005 * 4006 * WARNING!!! ahciport_mutex should be acquired before the function 4007 * is called. 4008 */ 4009 static int 4010 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4011 { 4012 uint32_t port_cmd_status; 4013 4014 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 4015 4016 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4017 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 4018 "the state for port %d is 0x%x", 4019 port, ahci_portp->ahciport_port_state); 4020 return (AHCI_FAILURE); 4021 } 4022 4023 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 4024 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 4025 "no device is attached at port %d", port); 4026 return (AHCI_FAILURE); 4027 } 4028 4029 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 4030 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4031 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4032 4033 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 4034 port_cmd_status |= AHCI_CMD_STATUS_FRE; 4035 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4036 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4037 port_cmd_status); 4038 } 4039 4040 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4041 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4042 4043 port_cmd_status |= AHCI_CMD_STATUS_ST; 4044 4045 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4046 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4047 port_cmd_status); 4048 4049 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 4050 4051 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 4052 "PxCMD.ST set to '1' at port %d", port); 4053 4054 return (AHCI_SUCCESS); 4055 } 4056 4057 /* 4058 * Allocate the ahci_port_t including Received FIS and Command List. 4059 * The argument - port is the physical port number, and not logical 4060 * port number seen by the SATA framework. 4061 * 4062 * WARNING!!! ahcictl_mutex should be acquired before the function 4063 * is called. 4064 */ 4065 static int 4066 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 4067 { 4068 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 4069 ahci_port_t *ahci_portp; 4070 char taskq_name[64] = "event_handle_taskq"; 4071 4072 ahci_portp = 4073 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 4074 4075 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 4076 ahci_portp->ahciport_port_num = port; 4077 4078 /* Intialize the port condition variable */ 4079 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 4080 4081 /* Initialize the port mutex */ 4082 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 4083 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 4084 4085 mutex_enter(&ahci_portp->ahciport_mutex); 4086 4087 /* 4088 * Allocate memory for received FIS structure and 4089 * command list for this port 4090 */ 4091 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 4092 goto err_case1; 4093 } 4094 4095 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 4096 goto err_case2; 4097 } 4098 4099 (void) snprintf(taskq_name + strlen(taskq_name), 4100 sizeof (taskq_name) - strlen(taskq_name), 4101 "_port%d", port); 4102 4103 /* Create the taskq for the port */ 4104 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 4105 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 4106 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 4107 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 4108 goto err_case3; 4109 } 4110 4111 /* Allocate the argument for the taskq */ 4112 ahci_portp->ahciport_event_args = 4113 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 4114 4115 if (ahci_portp->ahciport_event_args == NULL) 4116 goto err_case4; 4117 4118 mutex_exit(&ahci_portp->ahciport_mutex); 4119 4120 return (AHCI_SUCCESS); 4121 4122 err_case4: 4123 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 4124 4125 err_case3: 4126 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 4127 4128 err_case2: 4129 ahci_dealloc_rcvd_fis(ahci_portp); 4130 4131 err_case1: 4132 mutex_exit(&ahci_portp->ahciport_mutex); 4133 mutex_destroy(&ahci_portp->ahciport_mutex); 4134 cv_destroy(&ahci_portp->ahciport_cv); 4135 4136 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4137 4138 return (AHCI_FAILURE); 4139 } 4140 4141 /* 4142 * Reverse of ahci_dealloc_port_state(). 4143 * 4144 * WARNING!!! ahcictl_mutex should be acquired before the function 4145 * is called. 4146 */ 4147 static void 4148 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 4149 { 4150 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4151 4152 ASSERT(ahci_portp != NULL); 4153 4154 mutex_enter(&ahci_portp->ahciport_mutex); 4155 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 4156 ahci_portp->ahciport_event_args = NULL; 4157 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 4158 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 4159 ahci_dealloc_rcvd_fis(ahci_portp); 4160 mutex_exit(&ahci_portp->ahciport_mutex); 4161 4162 mutex_destroy(&ahci_portp->ahciport_mutex); 4163 cv_destroy(&ahci_portp->ahciport_cv); 4164 4165 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4166 4167 ahci_ctlp->ahcictl_ports[port] = NULL; 4168 } 4169 4170 /* 4171 * Allocates memory for the Received FIS Structure 4172 * 4173 * WARNING!!! ahciport_mutex should be acquired before the function 4174 * is called. 4175 */ 4176 static int 4177 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4178 uint8_t port) 4179 { 4180 size_t rcvd_fis_size; 4181 size_t ret_len; 4182 uint_t cookie_count; 4183 4184 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 4185 4186 /* allocate rcvd FIS dma handle. */ 4187 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4188 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 4189 DDI_DMA_SLEEP, 4190 NULL, 4191 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 4192 DDI_SUCCESS) { 4193 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4194 "rcvd FIS dma handle alloc failed", NULL); 4195 4196 return (AHCI_FAILURE); 4197 } 4198 4199 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 4200 rcvd_fis_size, 4201 &accattr, 4202 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4203 DDI_DMA_SLEEP, 4204 NULL, 4205 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4206 &ret_len, 4207 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4208 4209 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4210 "rcvd FIS dma mem alloc fail", NULL); 4211 /* error.. free the dma handle. */ 4212 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4213 return (AHCI_FAILURE); 4214 } 4215 4216 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4217 NULL, 4218 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4219 rcvd_fis_size, 4220 DDI_DMA_CONSISTENT, 4221 DDI_DMA_SLEEP, 4222 NULL, 4223 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4224 &cookie_count) != DDI_DMA_MAPPED) { 4225 4226 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4227 "rcvd FIS dma handle bind fail", NULL); 4228 /* error.. free the dma handle & free the memory. */ 4229 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4230 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4231 return (AHCI_FAILURE); 4232 } 4233 4234 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4235 4236 /* Config Port Received FIS Base Address */ 4237 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4238 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4239 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4240 4241 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4242 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4243 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4244 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4245 4246 return (AHCI_SUCCESS); 4247 } 4248 4249 /* 4250 * Deallocates the Received FIS Structure 4251 * 4252 * WARNING!!! ahciport_mutex should be acquired before the function 4253 * is called. 4254 */ 4255 static void 4256 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4257 { 4258 /* Unbind the cmd list dma handle first. */ 4259 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4260 4261 /* Then free the underlying memory. */ 4262 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4263 4264 /* Now free the handle itself. */ 4265 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4266 } 4267 4268 /* 4269 * Allocates memory for the Command List, which contains up to 32 entries. 4270 * Each entry contains a command header, which is a 32-byte structure that 4271 * includes the pointer to the command table. 4272 * 4273 * WARNING!!! ahciport_mutex should be acquired before the function 4274 * is called. 4275 */ 4276 static int 4277 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4278 uint8_t port) 4279 { 4280 size_t cmd_list_size; 4281 size_t ret_len; 4282 uint_t cookie_count; 4283 4284 cmd_list_size = 4285 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4286 4287 /* allocate cmd list dma handle. */ 4288 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4289 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4290 DDI_DMA_SLEEP, 4291 NULL, 4292 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4293 4294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4295 "cmd list dma handle alloc failed", NULL); 4296 return (AHCI_FAILURE); 4297 } 4298 4299 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4300 cmd_list_size, 4301 &accattr, 4302 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4303 DDI_DMA_SLEEP, 4304 NULL, 4305 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4306 &ret_len, 4307 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4308 4309 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4310 "cmd list dma mem alloc fail", NULL); 4311 /* error.. free the dma handle. */ 4312 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4313 return (AHCI_FAILURE); 4314 } 4315 4316 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4317 NULL, 4318 (caddr_t)ahci_portp->ahciport_cmd_list, 4319 cmd_list_size, 4320 DDI_DMA_CONSISTENT, 4321 DDI_DMA_SLEEP, 4322 NULL, 4323 &ahci_portp->ahciport_cmd_list_dma_cookie, 4324 &cookie_count) != DDI_DMA_MAPPED) { 4325 4326 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4327 "cmd list dma handle bind fail", NULL); 4328 /* error.. free the dma handle & free the memory. */ 4329 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4330 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4331 return (AHCI_FAILURE); 4332 } 4333 4334 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4335 4336 /* Config Port Command List Base Address */ 4337 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4338 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4339 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4340 4341 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4342 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4343 4344 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4345 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4346 4347 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4348 goto err_out; 4349 } 4350 4351 return (AHCI_SUCCESS); 4352 4353 err_out: 4354 /* Unbind the cmd list dma handle first. */ 4355 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4356 4357 /* Then free the underlying memory. */ 4358 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4359 4360 /* Now free the handle itself. */ 4361 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4362 4363 return (AHCI_FAILURE); 4364 } 4365 4366 /* 4367 * Deallocates the Command List 4368 * 4369 * WARNING!!! ahciport_mutex should be acquired before the function 4370 * is called. 4371 */ 4372 static void 4373 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4374 { 4375 /* First dealloc command table */ 4376 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4377 4378 /* Unbind the cmd list dma handle first. */ 4379 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4380 4381 /* Then free the underlying memory. */ 4382 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4383 4384 /* Now free the handle itself. */ 4385 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4386 } 4387 4388 /* 4389 * Allocates memory for all Command Tables, which contains Command FIS, 4390 * ATAPI Command and Physical Region Descriptor Table. 4391 * 4392 * WARNING!!! ahciport_mutex should be acquired before the function 4393 * is called. 4394 */ 4395 static int 4396 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4397 { 4398 size_t ret_len; 4399 ddi_dma_cookie_t cmd_table_dma_cookie; 4400 uint_t cookie_count; 4401 int slot; 4402 4403 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4404 "ahci_alloc_cmd_tables: port %d enter", 4405 ahci_portp->ahciport_port_num); 4406 4407 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4408 /* Allocate cmd table dma handle. */ 4409 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4410 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4411 DDI_DMA_SLEEP, 4412 NULL, 4413 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4414 DDI_SUCCESS) { 4415 4416 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4417 "cmd table dma handle alloc failed", NULL); 4418 4419 goto err_out; 4420 } 4421 4422 if (ddi_dma_mem_alloc( 4423 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4424 ahci_cmd_table_size, 4425 &accattr, 4426 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4427 DDI_DMA_SLEEP, 4428 NULL, 4429 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4430 &ret_len, 4431 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4432 NULL) { 4433 4434 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4435 "cmd table dma mem alloc fail", NULL); 4436 4437 /* error.. free the dma handle. */ 4438 ddi_dma_free_handle( 4439 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4440 goto err_out; 4441 } 4442 4443 if (ddi_dma_addr_bind_handle( 4444 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4445 NULL, 4446 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4447 ahci_cmd_table_size, 4448 DDI_DMA_CONSISTENT, 4449 DDI_DMA_SLEEP, 4450 NULL, 4451 &cmd_table_dma_cookie, 4452 &cookie_count) != DDI_DMA_MAPPED) { 4453 4454 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4455 "cmd table dma handle bind fail", NULL); 4456 /* error.. free the dma handle & free the memory. */ 4457 ddi_dma_mem_free( 4458 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4459 ddi_dma_free_handle( 4460 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4461 goto err_out; 4462 } 4463 4464 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4465 ahci_cmd_table_size); 4466 4467 /* Config Port Command Table Base Address */ 4468 SET_COMMAND_TABLE_BASE_ADDR( 4469 (&ahci_portp->ahciport_cmd_list[slot]), 4470 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4471 4472 #ifndef __lock_lint 4473 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4474 (&ahci_portp->ahciport_cmd_list[slot]), 4475 cmd_table_dma_cookie.dmac_laddress >> 32); 4476 #endif 4477 } 4478 4479 return (AHCI_SUCCESS); 4480 err_out: 4481 4482 for (slot--; slot >= 0; slot--) { 4483 /* Unbind the cmd table dma handle first */ 4484 (void) ddi_dma_unbind_handle( 4485 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4486 4487 /* Then free the underlying memory */ 4488 ddi_dma_mem_free( 4489 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4490 4491 /* Now free the handle itself */ 4492 ddi_dma_free_handle( 4493 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4494 } 4495 4496 return (AHCI_FAILURE); 4497 } 4498 4499 /* 4500 * Deallocates memory for all Command Tables. 4501 * 4502 * WARNING!!! ahciport_mutex should be acquired before the function 4503 * is called. 4504 */ 4505 static void 4506 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4507 { 4508 int slot; 4509 4510 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 4511 "ahci_dealloc_cmd_tables: %d enter", 4512 ahci_portp->ahciport_port_num); 4513 4514 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4515 /* Unbind the cmd table dma handle first. */ 4516 (void) ddi_dma_unbind_handle( 4517 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4518 4519 /* Then free the underlying memory. */ 4520 ddi_dma_mem_free( 4521 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4522 4523 /* Now free the handle itself. */ 4524 ddi_dma_free_handle( 4525 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4526 } 4527 } 4528 4529 /* 4530 * WARNING!!! ahciport_mutex should be acquired before the function 4531 * is called. 4532 */ 4533 static void 4534 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4535 sata_device_t *sd) 4536 { 4537 sd->satadev_scr.sstatus = 4538 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4539 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4540 sd->satadev_scr.serror = 4541 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4542 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4543 sd->satadev_scr.scontrol = 4544 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4545 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4546 sd->satadev_scr.sactive = 4547 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4548 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4549 } 4550 4551 /* 4552 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4553 */ 4554 static void 4555 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4556 { 4557 uint32_t port_intr_status; 4558 uint32_t port_intr_enable; 4559 4560 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4561 "ahci_port_intr enter: port %d", port); 4562 4563 mutex_enter(&ahci_portp->ahciport_mutex); 4564 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4565 /* For SATA_OPMODE_POLLING commands */ 4566 port_intr_enable = 4567 (AHCI_INTR_STATUS_DHRS | 4568 AHCI_INTR_STATUS_PSS | 4569 AHCI_INTR_STATUS_SDBS | 4570 AHCI_INTR_STATUS_UFS | 4571 AHCI_INTR_STATUS_PCS | 4572 AHCI_INTR_STATUS_PRCS | 4573 AHCI_INTR_STATUS_OFS | 4574 AHCI_INTR_STATUS_INFS | 4575 AHCI_INTR_STATUS_IFS | 4576 AHCI_INTR_STATUS_HBDS | 4577 AHCI_INTR_STATUS_HBFS | 4578 AHCI_INTR_STATUS_TFES); 4579 mutex_exit(&ahci_portp->ahciport_mutex); 4580 goto next; 4581 } 4582 mutex_exit(&ahci_portp->ahciport_mutex); 4583 4584 /* 4585 * port_intr_enable indicates that the corresponding interrrupt 4586 * reporting is enabled. 4587 */ 4588 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4589 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4590 next: 4591 /* 4592 * port_intr_stats indicates that the corresponding interrupt 4593 * condition is active. 4594 */ 4595 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4596 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4597 4598 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4599 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4600 "port_intr_enable = 0x%x", 4601 port, port_intr_status, port_intr_enable); 4602 4603 port_intr_status &= port_intr_enable; 4604 4605 /* First clear the port interrupts status */ 4606 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4607 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4608 port_intr_status); 4609 4610 /* Check the completed non-queued commands */ 4611 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4612 AHCI_INTR_STATUS_PSS)) { 4613 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4614 ahci_portp, port); 4615 } 4616 4617 /* Check the completed queued commands */ 4618 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4619 (void) ahci_intr_set_device_bits(ahci_ctlp, 4620 ahci_portp, port); 4621 } 4622 4623 /* Check the port connect change status interrupt bit */ 4624 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4625 (void) ahci_intr_port_connect_change(ahci_ctlp, 4626 ahci_portp, port); 4627 } 4628 4629 /* Check the device mechanical presence status interrupt bit */ 4630 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4631 (void) ahci_intr_device_mechanical_presence_status( 4632 ahci_ctlp, ahci_portp, port); 4633 } 4634 4635 /* Check the PhyRdy change status interrupt bit */ 4636 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4637 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4638 port); 4639 } 4640 4641 /* 4642 * Check the non-fatal error interrupt bits, there are three 4643 * kinds of non-fatal errors at the time being: 4644 * 4645 * PxIS.UFS - Unknown FIS Error 4646 * PxIS.OFS - Overflow Error 4647 * PxIS.INFS - Interface Non-Fatal Error 4648 * 4649 * For these non-fatal errors, the HBA can continue to operate, 4650 * so the driver just log the error messages. 4651 */ 4652 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4653 AHCI_INTR_STATUS_OFS | 4654 AHCI_INTR_STATUS_INFS)) { 4655 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4656 port, port_intr_status); 4657 } 4658 4659 /* 4660 * Check the fatal error interrupt bits, there are four kinds 4661 * of fatal errors for AHCI controllers: 4662 * 4663 * PxIS.HBFS - Host Bus Fatal Error 4664 * PxIS.HBDS - Host Bus Data Error 4665 * PxIS.IFS - Interface Fatal Error 4666 * PxIS.TFES - Task File Error 4667 * 4668 * The fatal error means the HBA can not recover from it by 4669 * itself, and it will try to abort the transfer, and the software 4670 * must intervene to restart the port. 4671 */ 4672 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4673 AHCI_INTR_STATUS_HBDS | 4674 AHCI_INTR_STATUS_HBFS | 4675 AHCI_INTR_STATUS_TFES)) 4676 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4677 port, port_intr_status); 4678 4679 /* Check the cold port detect interrupt bit */ 4680 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4681 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4682 } 4683 4684 /* Second clear the corresponding bit in IS.IPS */ 4685 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4686 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4687 } 4688 4689 /* 4690 * Interrupt service handler 4691 */ 4692 static uint_t 4693 ahci_intr(caddr_t arg1, caddr_t arg2) 4694 { 4695 #ifndef __lock_lint 4696 _NOTE(ARGUNUSED(arg2)) 4697 #endif 4698 /* LINTED */ 4699 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4700 ahci_port_t *ahci_portp; 4701 int32_t global_intr_status; 4702 uint8_t port; 4703 4704 /* 4705 * global_intr_status indicates that the corresponding port has 4706 * an interrupt pending. 4707 */ 4708 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4709 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4710 4711 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4712 /* The interrupt is not ours */ 4713 return (DDI_INTR_UNCLAIMED); 4714 } 4715 4716 /* Loop for all the ports */ 4717 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4718 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4719 continue; 4720 } 4721 if (!((0x1 << port) & global_intr_status)) { 4722 continue; 4723 } 4724 4725 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4726 4727 /* Call ahci_port_intr */ 4728 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4729 } 4730 4731 return (DDI_INTR_CLAIMED); 4732 } 4733 4734 /* 4735 * For non-queued commands, when the corresponding bit in the PxCI register 4736 * is cleared, it means the command is completed successfully. And according 4737 * to the HBA state machine, there are three conditions which possibly will 4738 * try to clear the PxCI register bit. 4739 * 1. Receive one D2H Register FIS which is with 'I' bit set 4740 * 2. Update PIO Setup FIS 4741 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4742 * 4743 * Process completed non-queued commands when the interrupt status bit - 4744 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4745 * 4746 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4747 * with the 'I' bit set. And the following commands will send thus 4748 * FIS with 'I' bit set upon the successful completion: 4749 * 1. Non-data commands 4750 * 2. DMA data-in command 4751 * 3. DMA data-out command 4752 * 4. PIO data-out command 4753 * 5. PACKET non-data commands 4754 * 6. PACKET PIO data-in command 4755 * 7. PACKET PIO data-out command 4756 * 8. PACKET DMA data-in command 4757 * 9. PACKET DMA data-out command 4758 * 4759 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4760 * with the 'I' bit set. And the following commands will send this 4761 * FIS upon the successful completion: 4762 * 1. PIO data-in command 4763 */ 4764 static int 4765 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4766 ahci_port_t *ahci_portp, uint8_t port) 4767 { 4768 uint32_t port_cmd_issue = 0; 4769 uint32_t finished_tags; 4770 int finished_slot; 4771 sata_pkt_t *satapkt; 4772 ahci_fis_d2h_register_t *rcvd_fisp; 4773 #if AHCI_DEBUG 4774 ahci_cmd_header_t *cmd_header; 4775 uint32_t cmd_dmacount; 4776 #endif 4777 4778 mutex_enter(&ahci_portp->ahciport_mutex); 4779 4780 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4781 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4782 /* 4783 * Spurious interrupt. Nothing to be done. 4784 */ 4785 mutex_exit(&ahci_portp->ahciport_mutex); 4786 return (AHCI_SUCCESS); 4787 } 4788 4789 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4790 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4791 4792 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4793 /* Slot 0 is always used during error recovery */ 4794 finished_tags = 0x1 & ~port_cmd_issue; 4795 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4796 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4797 "retrieval is finished, and finished_tags = 0x%x", 4798 port, finished_tags); 4799 } else { 4800 finished_tags = ahci_portp->ahciport_pending_tags & 4801 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4802 } 4803 4804 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4805 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4806 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4807 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4808 finished_tags); 4809 4810 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4811 (finished_tags == 0x1)) { 4812 satapkt = ahci_portp->ahciport_err_retri_pkt; 4813 ASSERT(satapkt != NULL); 4814 4815 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4816 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4817 "with SATA_PKT_COMPLETED", (void *)satapkt); 4818 4819 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4820 goto out; 4821 } 4822 4823 while (finished_tags) { 4824 finished_slot = ddi_ffs(finished_tags) - 1; 4825 if (finished_slot == -1) { 4826 goto out; 4827 } 4828 4829 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4830 ASSERT(satapkt != NULL); 4831 #if AHCI_DEBUG 4832 /* 4833 * For non-native queued commands, the PRD byte count field 4834 * shall contain an accurate count of the number of bytes 4835 * transferred for the command before the PxCI bit is cleared 4836 * to '0' for the command. 4837 * 4838 * The purpose of this field is to let software know how many 4839 * bytes transferred for a given operation in order to 4840 * determine if underflow occurred. When issuing native command 4841 * queuing commands, this field should not be used and is not 4842 * required to be valid since in this case underflow is always 4843 * illegal. 4844 * 4845 * For data reads, the HBA will update its PRD byte count with 4846 * the total number of bytes received from the last FIS, and 4847 * may be able to continue normally. For data writes, the 4848 * device will detect an error, and HBA most likely will get 4849 * a fatal error. 4850 * 4851 * Therefore, here just put code to debug part. And please 4852 * refer to the comment above ahci_intr_fatal_error for the 4853 * definition of underflow error. 4854 */ 4855 cmd_dmacount = 4856 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4857 if (cmd_dmacount) { 4858 cmd_header = 4859 &ahci_portp->ahciport_cmd_list[finished_slot]; 4860 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4861 "ahci_intr_cmd_cmplt: port %d, " 4862 "PRD Byte Count = 0x%x, " 4863 "ahciport_prd_bytecounts = 0x%x", port, 4864 cmd_header->ahcich_prd_byte_count, 4865 cmd_dmacount); 4866 4867 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4868 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 4869 "ahci_intr_cmd_cmplt: port %d, " 4870 "an underflow occurred", port); 4871 } 4872 } 4873 #endif 4874 4875 /* 4876 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4877 * feature, sata_special_regs flag will be set, and the 4878 * driver should copy the status and the other corresponding 4879 * register values in the D2H Register FIS received (It's 4880 * working on Non-data protocol) from the device back to 4881 * the sata_cmd. 4882 * 4883 * For every AHCI port, there is only one Received FIS 4884 * structure, which contains the FISes received from the 4885 * device, So we're trying to copy the content of D2H 4886 * Register FIS in the Received FIS structure back to 4887 * the sata_cmd. 4888 */ 4889 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4890 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4891 ahcirf_d2h_register_fis); 4892 satapkt->satapkt_cmd.satacmd_status_reg = 4893 GET_RFIS_STATUS(rcvd_fisp); 4894 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4895 } 4896 4897 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4898 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4899 "with SATA_PKT_COMPLETED", (void *)satapkt); 4900 4901 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4902 CLEAR_BIT(finished_tags, finished_slot); 4903 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4904 4905 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4906 } 4907 out: 4908 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 4909 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4910 ahci_portp->ahciport_pending_tags); 4911 4912 mutex_exit(&ahci_portp->ahciport_mutex); 4913 4914 return (AHCI_SUCCESS); 4915 } 4916 4917 /* 4918 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4919 * with the 'I' bit set and has been copied into system memory. It will 4920 * be sent under the following situations: 4921 * 4922 * 1. NCQ command is completed 4923 * 2. Asynchronous notification 4924 * 4925 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4926 * via the Set Device Bits FIS. When such event is generated, the software 4927 * needs to read PxSACT register and compares the current value to the 4928 * list of commands previously issue by software. ahciport_pending_ncq_tags 4929 * keeps the tags of previously issued commands. 4930 * 4931 * Asynchronous Notification is a feature in SATA II, which allows an 4932 * ATAPI device to send a signal to the host when media is inserted or 4933 * removed and avoids polling the device for media changes. The signal 4934 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4935 * set to '1'. At the moment, it's not supported yet. 4936 */ 4937 static int 4938 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4939 ahci_port_t *ahci_portp, uint8_t port) 4940 { 4941 uint32_t port_sactive; 4942 uint32_t port_cmd_issue; 4943 uint32_t issued_tags; 4944 int issued_slot; 4945 uint32_t finished_tags; 4946 int finished_slot; 4947 sata_pkt_t *satapkt; 4948 4949 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4950 "ahci_intr_set_device_bits enter: port %d", port); 4951 4952 mutex_enter(&ahci_portp->ahciport_mutex); 4953 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4954 mutex_exit(&ahci_portp->ahciport_mutex); 4955 return (AHCI_SUCCESS); 4956 } 4957 4958 /* 4959 * First the handler got which commands are finished by checking 4960 * PxSACT register 4961 */ 4962 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4963 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4964 4965 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4966 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4967 4968 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4969 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4970 "port_sactive = 0x%x", port, 4971 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4972 4973 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4974 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4975 4976 /* 4977 * For NCQ commands, the software can determine which command has 4978 * already been transmitted to the device by checking PxCI register. 4979 */ 4980 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4981 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4982 4983 issued_tags = ahci_portp->ahciport_pending_tags & 4984 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4985 4986 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4987 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4988 "port_cmd_issue = 0x%x", port, 4989 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4990 4991 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4992 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4993 4994 /* 4995 * Clear ahciport_pending_tags bit when the corresponding command 4996 * is already sent down to the device. 4997 */ 4998 while (issued_tags) { 4999 issued_slot = ddi_ffs(issued_tags) - 1; 5000 if (issued_slot == -1) { 5001 goto next; 5002 } 5003 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 5004 CLEAR_BIT(issued_tags, issued_slot); 5005 } 5006 5007 next: 5008 while (finished_tags) { 5009 finished_slot = ddi_ffs(finished_tags) - 1; 5010 if (finished_slot == -1) { 5011 goto out; 5012 } 5013 5014 /* The command is certainly transmitted to the device */ 5015 ASSERT(!(ahci_portp->ahciport_pending_tags & 5016 (0x1 << finished_slot))); 5017 5018 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 5019 ASSERT(satapkt != NULL); 5020 5021 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 5022 "ahci_intr_set_device_bits: sending up pkt 0x%p " 5023 "with SATA_PKT_COMPLETED", (void *)satapkt); 5024 5025 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 5026 CLEAR_BIT(finished_tags, finished_slot); 5027 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 5028 5029 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 5030 } 5031 out: 5032 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 5033 "ahci_intr_set_device_bits: port %d " 5034 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 5035 port, ahci_portp->ahciport_pending_ncq_tags, 5036 ahci_portp->ahciport_pending_tags); 5037 5038 mutex_exit(&ahci_portp->ahciport_mutex); 5039 5040 return (AHCI_SUCCESS); 5041 } 5042 5043 /* 5044 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 5045 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 5046 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 5047 * indicates a COMINIT signal was received. 5048 * 5049 * Hot plug insertion is detected by reception of a COMINIT signal from the 5050 * device. On reception of unsolicited COMINIT, the HBA shall generate a 5051 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 5052 * begin the normal communication negotiation sequence as outlined in the 5053 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 5054 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 5055 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 5056 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 5057 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 5058 * to handle hot plug insertion. In this interrupt handler, just do nothing 5059 * but print some log message and clear the bit. 5060 */ 5061 static int 5062 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 5063 ahci_port_t *ahci_portp, uint8_t port) 5064 { 5065 #if AHCI_DEBUG 5066 uint32_t port_serror; 5067 #endif 5068 5069 mutex_enter(&ahci_portp->ahciport_mutex); 5070 5071 #if AHCI_DEBUG 5072 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5073 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5074 5075 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5076 "ahci_intr_port_connect_change: port %d, " 5077 "port_serror = 0x%x", port, port_serror); 5078 #endif 5079 5080 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 5081 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5082 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5083 SERROR_EXCHANGED_ERR); 5084 5085 mutex_exit(&ahci_portp->ahciport_mutex); 5086 5087 return (AHCI_SUCCESS); 5088 } 5089 5090 /* 5091 * Hot Plug Operation for platforms that support Mechanical Presence 5092 * Switches. 5093 * 5094 * When set, it indicates that a mechanical presence switch attached to this 5095 * port has been opened or closed, which may lead to a change in the connection 5096 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 5097 * are set to '1'. 5098 * 5099 * At the moment, this interrupt is not needed and disabled and we just log 5100 * the debug message. 5101 */ 5102 static int 5103 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 5104 ahci_port_t *ahci_portp, uint8_t port) 5105 { 5106 uint32_t cap_status, port_cmd_status; 5107 5108 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5109 "ahci_intr_device_mechanical_presence_status enter, " 5110 "port %d", port); 5111 5112 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5113 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 5114 5115 mutex_enter(&ahci_portp->ahciport_mutex); 5116 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5117 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5118 5119 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 5120 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 5121 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5122 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 5123 "the interrupt: cap_status = 0x%x, " 5124 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 5125 mutex_exit(&ahci_portp->ahciport_mutex); 5126 5127 return (AHCI_SUCCESS); 5128 } 5129 5130 #if AHCI_DEBUG 5131 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 5132 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5133 "The mechanical presence switch is open: " 5134 "port %d, port_cmd_status = 0x%x", 5135 port, port_cmd_status); 5136 } else { 5137 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5138 "The mechanical presence switch is close: " 5139 "port %d, port_cmd_status = 0x%x", 5140 port, port_cmd_status); 5141 } 5142 #endif 5143 5144 mutex_exit(&ahci_portp->ahciport_mutex); 5145 5146 return (AHCI_SUCCESS); 5147 } 5148 5149 /* 5150 * Native Hot Plug Support. 5151 * 5152 * When set, it indicates that the internal PHYRDY signal changed state. 5153 * This bit reflects the state of PxSERR.DIAG.N. 5154 * 5155 * There are three kinds of conditions to generate this interrupt event: 5156 * 1. a device is inserted 5157 * 2. a device is disconnected 5158 * 3. when the link enters/exits a Partial or Slumber interface power 5159 * management state 5160 * 5161 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 5162 * bit may be set due to the link entering the Partial or Slumber power 5163 * management state, rather than due to a hot plug insertion or removal 5164 * event. So far, the interface power management is disabled, so the 5165 * driver can reliably get removal detection notification via the 5166 * PxSERR.DIAG.N bit. 5167 */ 5168 static int 5169 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 5170 ahci_port_t *ahci_portp, uint8_t port) 5171 { 5172 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 5173 sata_device_t sdevice; 5174 int dev_exists_now = 0; 5175 int dev_existed_previously = 0; 5176 5177 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5178 "ahci_intr_phyrdy_change enter, port %d", port); 5179 5180 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 5181 mutex_enter(&ahci_portp->ahciport_mutex); 5182 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5183 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5184 SERROR_PHY_RDY_CHG); 5185 mutex_exit(&ahci_portp->ahciport_mutex); 5186 5187 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5188 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 5189 (ahci_portp == NULL)) { 5190 /* The whole controller setup is not yet done. */ 5191 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5192 return (AHCI_SUCCESS); 5193 } 5194 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5195 5196 mutex_enter(&ahci_portp->ahciport_mutex); 5197 5198 /* SStatus tells the presence of device. */ 5199 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5200 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5201 5202 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5203 dev_exists_now = 1; 5204 } 5205 5206 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5207 dev_existed_previously = 1; 5208 } 5209 5210 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5211 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5212 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5213 "ahci_intr_phyrdy_change: port %d " 5214 "AHCI_PORT_FLAG_NODEV is cleared", port); 5215 if (dev_exists_now == 0) 5216 dev_existed_previously = 1; 5217 } 5218 5219 bzero((void *)&sdevice, sizeof (sata_device_t)); 5220 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5221 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5222 sdevice.satadev_addr.pmport = 0; 5223 sdevice.satadev_state = SATA_PSTATE_PWRON; 5224 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5225 5226 if (dev_exists_now) { 5227 if (dev_existed_previously) { 5228 /* Things are fine now. The loss was temporary. */ 5229 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5230 "ahci_intr_phyrdy_change port %d " 5231 "device link lost/established", port); 5232 5233 mutex_exit(&ahci_portp->ahciport_mutex); 5234 sata_hba_event_notify( 5235 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5236 &sdevice, 5237 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5238 mutex_enter(&ahci_portp->ahciport_mutex); 5239 5240 } else { 5241 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5242 "ahci_intr_phyrdy_change: port %d " 5243 "device link established", port); 5244 5245 /* A new device has been detected. */ 5246 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 5247 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5248 5249 /* Try to start the port */ 5250 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5251 != AHCI_SUCCESS) { 5252 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5253 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5254 "ahci_intr_phyrdy_change: port %d failed " 5255 "at start port", port); 5256 } 5257 5258 /* Clear the max queue depth for inserted device */ 5259 ahci_portp->ahciport_max_ncq_tags = 0; 5260 5261 mutex_exit(&ahci_portp->ahciport_mutex); 5262 sata_hba_event_notify( 5263 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5264 &sdevice, 5265 SATA_EVNT_LINK_ESTABLISHED); 5266 mutex_enter(&ahci_portp->ahciport_mutex); 5267 5268 } 5269 } else { /* No device exists now */ 5270 5271 if (dev_existed_previously) { 5272 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5273 "ahci_intr_phyrdy_change: port %d " 5274 "device link lost", port); 5275 5276 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5277 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5278 ahci_portp, port); 5279 5280 /* An existing device is lost. */ 5281 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5282 5283 mutex_exit(&ahci_portp->ahciport_mutex); 5284 sata_hba_event_notify( 5285 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5286 &sdevice, 5287 SATA_EVNT_LINK_LOST); 5288 mutex_enter(&ahci_portp->ahciport_mutex); 5289 } 5290 } 5291 5292 mutex_exit(&ahci_portp->ahciport_mutex); 5293 5294 return (AHCI_SUCCESS); 5295 } 5296 5297 /* 5298 * PxIS.UFS - Unknown FIS Error 5299 * 5300 * This interrupt event means an unknown FIS was received and has been 5301 * copied into system memory. An unknown FIS is not considered an illegal 5302 * FIS, unless the length received is more than 64 bytes. If an unknown 5303 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5304 * normal operation. If the unknown FIS is more than 64 bytes, then it 5305 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5306 * a fatal error. 5307 * 5308 * PxIS.OFS - Overflow Error 5309 * 5310 * Command list overflow is defined as software building a command table 5311 * that has fewer total bytes than the transaction given to the device. 5312 * On device writes, the HBA will run out of data, and on reads, there 5313 * will be no room to put the data. 5314 * 5315 * For an overflow on data read, either PIO or DMA, the HBA will set 5316 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5317 * non-fatal error when the HBA can continues. Sometimes, it will cause 5318 * a fatal error and need the software to do something. 5319 * 5320 * For an overflow on data write, setting PxIS.OFS is optional for both 5321 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5322 * software to clean up from this serious error. 5323 * 5324 * PxIS.INFS - Interface Non-Fatal Error 5325 * 5326 * This interrupt event indicates that the HBA encountered an error on 5327 * the Serial ATA interface but was able to continue operation. The kind 5328 * of error usually occurred during a non-Data FIS, and under this condition 5329 * the FIS will be re-transmitted by HBA automatically. 5330 * 5331 * When the FMA is implemented, there should be a stat structure to 5332 * record how many every kind of error happens. 5333 */ 5334 static int 5335 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5336 uint8_t port, uint32_t intr_status) 5337 { 5338 uint32_t port_serror; 5339 #if AHCI_DEBUG 5340 uint32_t port_cmd_status; 5341 uint32_t port_cmd_issue; 5342 uint32_t port_sactive; 5343 int current_slot; 5344 uint32_t current_tags; 5345 sata_pkt_t *satapkt; 5346 ahci_cmd_header_t *cmd_header; 5347 uint32_t cmd_dmacount; 5348 #endif 5349 5350 mutex_enter(&ahci_portp->ahciport_mutex); 5351 5352 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5353 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5354 5355 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5356 "ahci_intr_non_fatal_error: port %d, " 5357 "port_serror = 0x%x", port, port_serror); 5358 5359 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5360 5361 if (intr_status & AHCI_INTR_STATUS_UFS) { 5362 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5363 "ahci port %d has unknown FIS error", port); 5364 5365 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5366 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5367 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5368 SERROR_FIS_TYPE); 5369 } 5370 5371 #if AHCI_DEBUG 5372 if (intr_status & AHCI_INTR_STATUS_OFS) { 5373 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5374 "ahci port %d has overflow error", port); 5375 } 5376 5377 if (intr_status & AHCI_INTR_STATUS_INFS) { 5378 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5379 "ahci port %d has interface non fatal error", port); 5380 } 5381 5382 /* 5383 * Record the error occurred command's slot. 5384 */ 5385 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5386 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5387 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5388 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5389 5390 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5391 AHCI_CMD_STATUS_CCS_SHIFT; 5392 5393 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5394 satapkt = ahci_portp->ahciport_err_retri_pkt; 5395 ASSERT(satapkt != NULL); 5396 ASSERT(current_slot == 0); 5397 } else { 5398 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5399 } 5400 5401 if (satapkt != NULL) { 5402 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5403 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5404 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5405 satapkt->satapkt_cmd.satacmd_cmd_reg); 5406 5407 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5408 "ahci_intr_non_fatal_error: port %d, " 5409 "satapkt 0x%p is being processed when error occurs", 5410 port, (void *)satapkt); 5411 5412 /* 5413 * PRD Byte Count field of command header is not 5414 * required to reflect the total number of bytes 5415 * transferred when an overflow occurs, so here 5416 * just log the value. 5417 */ 5418 cmd_dmacount = 5419 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5420 if (cmd_dmacount) { 5421 cmd_header = &ahci_portp-> 5422 ahciport_cmd_list[current_slot]; 5423 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5424 "ahci_intr_non_fatal_error: port %d, " 5425 "PRD Byte Count = 0x%x, " 5426 "ahciport_prd_bytecounts = 0x%x", port, 5427 cmd_header->ahcich_prd_byte_count, 5428 cmd_dmacount); 5429 } 5430 } 5431 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5432 /* 5433 * For queued command, list those command which have already 5434 * been transmitted to the device and still not completed. 5435 */ 5436 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5437 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5438 5439 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5440 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5441 5442 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5443 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5444 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5445 ahci_portp->ahciport_pending_ncq_tags, 5446 port_sactive, port_cmd_issue); 5447 5448 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5449 port_sactive & ~port_cmd_issue & 5450 AHCI_NCQ_SLOT_MASK(ahci_portp); 5451 5452 while (current_tags) { 5453 current_slot = ddi_ffs(current_tags) - 1; 5454 if (current_slot == -1) { 5455 goto out; 5456 } 5457 5458 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5459 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5460 ahci_ctlp, "ahci_intr_non_fatal_error: " 5461 "port %d, satapkt 0x%p is outstanding when " 5462 "error occurs", port, (void *)satapkt); 5463 5464 CLEAR_BIT(current_tags, current_slot); 5465 } 5466 } 5467 out: 5468 #endif 5469 mutex_exit(&ahci_portp->ahciport_mutex); 5470 5471 return (AHCI_SUCCESS); 5472 } 5473 5474 /* 5475 * According to the AHCI spec, the error types include system memory 5476 * errors, interface errors, port multiplier errors, device errors, 5477 * command list overflow, command list underflow, native command 5478 * queuing tag errors and pio data transfer errors. 5479 * 5480 * System memory errors such as target abort, master abort, and parity 5481 * may cause the host to stop, and they are serious errors and needed 5482 * to be recovered with software intervention. When system software 5483 * has given a pointer to the HBA that doesn't exist in physical memory, 5484 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5485 * data error such as CRC or parity occurs, the HBA aborts the transfer 5486 * (if necessary) and PxIS.HBDS will be set. 5487 * 5488 * Interface errors are errors that occur due to electrical issues on 5489 * the interface, or protocol miscommunication between the device and 5490 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5491 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5492 * causes PxIS.IFS/PxIS.INFS to be set are 5493 * 1. in PxSERR.ERR, P bit is set to '1' 5494 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5495 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5496 * If the error occurred during a non-data FIS, the FIS must be 5497 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5498 * the error occurred during a data FIS, the transfer will stop, so 5499 * the error is fatal and PxIS.IFS is set. 5500 * 5501 * When a FIS arrives that updates the taskfile, the HBA checks to see 5502 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5503 * stops processing any more commands. 5504 * 5505 * Command list overflow is defined as software building a command table 5506 * that has fewer total bytes than the transaction given to the device. 5507 * On device writes, the HBA will run out of data, and on reads, there 5508 * will be no room to put the data. For an overflow on data read, either 5509 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5510 * For an overflow on data write, setting PxIS.OFS is optional for both 5511 * DMA and PIO, and a COMRESET is required by software to clean up from 5512 * this serious error. 5513 * 5514 * Command list underflow is defined as software building a command 5515 * table that has more total bytes than the transaction given to the 5516 * device. For data writes, both PIO and DMA, the device will detect 5517 * an error and end the transfer. And these errors are most likely going 5518 * to be fatal errors that will cause the port to be restarted. For 5519 * data reads, the HBA updates its PRD byte count, and may be 5520 * able to continue normally, but is not required to. And The HBA is 5521 * not required to detect underflow conditions for native command 5522 * queuing command. 5523 * 5524 * The HBA does not actively check incoming DMA Setup FISes to ensure 5525 * that the PxSACT register bit for that slot is set. Existing error 5526 * mechanisms, such as host bus failure, or bad protocol, are used to 5527 * recover from this case. 5528 * 5529 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5530 * DATA FIS must be an integral number of Dwords. If the HBA receives 5531 * a request which is not an integral number of Dwords, the HBA 5532 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5533 * software restarts the port. And the HBA ensures that the size 5534 * of the DATA FIS received during a PIO command matches the size in 5535 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5536 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5537 * stop running until software restarts the port. 5538 */ 5539 /* 5540 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5541 * 5542 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5543 * interface which caused the transfer to stop. 5544 * 5545 * PxIS.HBDS indicates that the hba encountered a data error 5546 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5547 * 5548 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5549 * cannot recover from, such as a bad software pointer. 5550 * 5551 * PxIS.TFES is set whenever the status register is updated by the device 5552 * and the error bit (bit 0) is set. 5553 */ 5554 static int 5555 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5556 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5557 { 5558 uint32_t port_cmd_status; 5559 uint32_t port_serror; 5560 uint32_t task_file_status; 5561 int failed_slot; 5562 sata_pkt_t *spkt = NULL; 5563 uint8_t err_byte; 5564 ahci_event_arg_t *args; 5565 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5566 5567 mutex_enter(&ahci_portp->ahciport_mutex); 5568 5569 /* 5570 * ahci_intr_phyrdy_change() may have rendered it to 5571 * SATA_DTYPE_NONE. 5572 */ 5573 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5574 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5575 "ahci_intr_fatal_error: port %d no device attached, " 5576 "and just return without doing anything", port); 5577 goto out0; 5578 } 5579 5580 if (intr_status & AHCI_INTR_STATUS_TFES) { 5581 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5582 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5583 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5584 "ahci_intr_fatal_error: port %d " 5585 "task_file_status = 0x%x", port, task_file_status); 5586 } 5587 5588 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5589 /* 5590 * Read PxCMD.CCS to determine the slot that the HBA 5591 * was processing when the error occurred. 5592 */ 5593 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5594 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5595 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5596 AHCI_CMD_STATUS_CCS_SHIFT; 5597 5598 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5599 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5600 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5601 "fatal error occurred for port %d", spkt, port); 5602 5603 if (intr_status & AHCI_INTR_STATUS_TFES) { 5604 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5605 >> AHCI_TFD_ERR_SHIFT; 5606 5607 /* 5608 * Won't emit the error message if it is an IDENTIFY 5609 * DEVICE command sent to an ATAPI device. 5610 */ 5611 if ((spkt != NULL) && 5612 (spkt->satapkt_cmd.satacmd_cmd_reg == 5613 SATAC_ID_DEVICE) && 5614 (err_byte == SATA_ERROR_ABORT)) 5615 goto out1; 5616 5617 /* 5618 * Won't emit the error message if it is an ATAPI PACKET 5619 * command 5620 */ 5621 if ((spkt != NULL) && 5622 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5623 goto out1; 5624 } 5625 } 5626 5627 /* print the fatal error type */ 5628 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5629 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5630 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5631 5632 /* print PxSERR related error message */ 5633 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5634 5635 /* print task file register value */ 5636 if (intr_status & AHCI_INTR_STATUS_TFES) { 5637 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 5638 "= 0x%x", instance, port, task_file_status); 5639 } 5640 5641 out1: 5642 /* Prepare the argument for the taskq */ 5643 args = ahci_portp->ahciport_event_args; 5644 args->ahciea_ctlp = (void *)ahci_ctlp; 5645 args->ahciea_portp = (void *)ahci_portp; 5646 args->ahciea_event = intr_status; 5647 5648 /* Start the taskq to handle error recovery */ 5649 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 5650 ahci_events_handler, 5651 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5652 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5653 "failed", instance); 5654 } 5655 out0: 5656 mutex_exit(&ahci_portp->ahciport_mutex); 5657 5658 return (AHCI_SUCCESS); 5659 } 5660 5661 /* 5662 * Hot Plug Operation for platforms that support Cold Presence Detect. 5663 * 5664 * When set, a device status has changed as detected by the cold presence 5665 * detect logic. This bit can either be set due to a non-connected port 5666 * receiving a device, or a connected port having its device removed. 5667 * This bit is only valid if the port supports cold presence detect as 5668 * indicated by PxCMD.CPD set to '1'. 5669 * 5670 * At the moment, this interrupt is not needed and disabled and we just 5671 * log the debug message. 5672 */ 5673 static int 5674 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5675 ahci_port_t *ahci_portp, uint8_t port) 5676 { 5677 uint32_t port_cmd_status; 5678 sata_device_t sdevice; 5679 5680 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5681 "ahci_intr_cold_port_detect enter, port %d", port); 5682 5683 mutex_enter(&ahci_portp->ahciport_mutex); 5684 5685 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5686 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5687 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5688 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5689 "port %d does not support cold presence detect, so " 5690 "we just ignore this interrupt", port); 5691 mutex_exit(&ahci_portp->ahciport_mutex); 5692 return (AHCI_SUCCESS); 5693 } 5694 5695 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5696 "port %d device status has changed", port); 5697 5698 bzero((void *)&sdevice, sizeof (sata_device_t)); 5699 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5700 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5701 sdevice.satadev_addr.pmport = 0; 5702 sdevice.satadev_state = SATA_PSTATE_PWRON; 5703 5704 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5705 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5706 "port %d: a device is hot plugged", port); 5707 mutex_exit(&ahci_portp->ahciport_mutex); 5708 sata_hba_event_notify( 5709 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5710 &sdevice, 5711 SATA_EVNT_DEVICE_ATTACHED); 5712 mutex_enter(&ahci_portp->ahciport_mutex); 5713 5714 } else { 5715 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5716 "port %d: a device is hot unplugged", port); 5717 mutex_exit(&ahci_portp->ahciport_mutex); 5718 sata_hba_event_notify( 5719 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5720 &sdevice, 5721 SATA_EVNT_DEVICE_DETACHED); 5722 mutex_enter(&ahci_portp->ahciport_mutex); 5723 } 5724 5725 mutex_exit(&ahci_portp->ahciport_mutex); 5726 5727 return (AHCI_SUCCESS); 5728 } 5729 5730 /* 5731 * Enable the interrupts for a particular port. 5732 * 5733 * WARNING!!! ahciport_mutex should be acquired before the function 5734 * is called. 5735 */ 5736 static void 5737 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5738 { 5739 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5740 "ahci_enable_port_intrs enter, port %d", port); 5741 5742 /* 5743 * Clear port interrupt status before enabling interrupt 5744 */ 5745 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5746 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5747 AHCI_PORT_INTR_MASK); 5748 5749 /* 5750 * Clear the pending bit from IS.IPS 5751 */ 5752 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5753 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5754 5755 /* 5756 * Enable the following interrupts: 5757 * Device to Host Register FIS Interrupt (DHRS) 5758 * PIO Setup FIS Interrupt (PSS) 5759 * Set Device Bits Interrupt (SDBS) 5760 * Unknown FIS Interrupt (UFS) 5761 * Port Connect Change Status (PCS) 5762 * PhyRdy Change Status (PRCS) 5763 * Overflow Status (OFS) 5764 * Interface Non-fatal Error Status (INFS) 5765 * Interface Fatal Error Status (IFS) 5766 * Host Bus Data Error Status (HBDS) 5767 * Host Bus Fatal Error Status (HBFS) 5768 * Task File Error Status (TFES) 5769 */ 5770 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5771 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5772 (AHCI_INTR_STATUS_DHRS | 5773 AHCI_INTR_STATUS_PSS | 5774 AHCI_INTR_STATUS_SDBS | 5775 AHCI_INTR_STATUS_UFS | 5776 AHCI_INTR_STATUS_DPS | 5777 AHCI_INTR_STATUS_PCS | 5778 AHCI_INTR_STATUS_PRCS | 5779 AHCI_INTR_STATUS_OFS | 5780 AHCI_INTR_STATUS_INFS | 5781 AHCI_INTR_STATUS_IFS | 5782 AHCI_INTR_STATUS_HBDS | 5783 AHCI_INTR_STATUS_HBFS | 5784 AHCI_INTR_STATUS_TFES)); 5785 } 5786 5787 /* 5788 * Enable interrupts for all the ports. 5789 * 5790 * WARNING!!! ahcictl_mutex should be acquired before the function 5791 * is called. 5792 */ 5793 static void 5794 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5795 { 5796 uint32_t ghc_control; 5797 5798 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 5799 5800 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5801 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5802 5803 ghc_control |= AHCI_HBA_GHC_IE; 5804 5805 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5806 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5807 } 5808 5809 /* 5810 * Disable interrupts for a particular port. 5811 * 5812 * WARNING!!! ahciport_mutex should be acquired before the function 5813 * is called. 5814 */ 5815 static void 5816 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5817 { 5818 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5819 "ahci_disable_port_intrs enter, port %d", port); 5820 5821 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5822 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5823 } 5824 5825 /* 5826 * Disable interrupts for the whole HBA. 5827 * 5828 * The global bit is cleared, then all interrupt sources from all 5829 * ports are disabled. 5830 * 5831 * WARNING!!! ahcictl_mutex should be acquired before the function 5832 * is called. 5833 */ 5834 static void 5835 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5836 { 5837 uint32_t ghc_control; 5838 5839 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 5840 NULL); 5841 5842 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5843 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5844 5845 ghc_control &= ~ AHCI_HBA_GHC_IE; 5846 5847 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5848 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5849 } 5850 5851 /* 5852 * Handle FIXED or MSI interrupts. 5853 */ 5854 /* 5855 * According to AHCI spec, the HBA may support several interrupt modes: 5856 * * pin based interrupts (FIXED) 5857 * * single MSI message interrupts 5858 * * multiple MSI based message interrupts 5859 * 5860 * For pin based interrupts, the software interrupt handler need to check IS 5861 * register to find out which port has pending interrupts. And then check 5862 * PxIS register to find out which interrupt events happened on that port. 5863 * 5864 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 5865 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 5866 * in that software interrupt handler need to check IS register to determine 5867 * which port triggered the interrupts since it uses a single message for all 5868 * port interrupts. 5869 * 5870 * HBA may optionally support multiple MSI message for better performance. In 5871 * this mode, each port may have its own interrupt message, and thus generation 5872 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 5873 * represents a power-of-2 wrapper on the number of implemented ports, and 5874 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 5875 * maximum number of assigned interrupts. When the number of MSI messages 5876 * allocated is less than the number requested, then hardware may have two 5877 * implementation behaviors: 5878 * * assign each ports its own interrupt and then force all additional 5879 * ports to share the last interrupt message, and this condition is 5880 * indicated by clearing GHC.MRSM to '0' 5881 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 5882 * When multiple-message MSI is enabled, hardware will still set IS register 5883 * as single message case. And this IS register may be used by software when 5884 * fewer than the requested number of messages is granted in order to determine 5885 * which port had the interrupt. 5886 * 5887 * Note: The current ahci driver only supports the first two interrupt modes: 5888 * pin based interrupts and single MSI message interrupts, and the reason 5889 * is indicated in below code. 5890 */ 5891 static int 5892 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 5893 { 5894 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5895 int count, avail, actual; 5896 int i, rc; 5897 5898 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 5899 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 5900 5901 /* get number of interrupts. */ 5902 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 5903 if ((rc != DDI_SUCCESS) || (count == 0)) { 5904 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5905 "ddi_intr_get_nintrs() failed, " 5906 "rc %d count %d\n", rc, count); 5907 return (DDI_FAILURE); 5908 } 5909 5910 /* get number of available interrupts. */ 5911 rc = ddi_intr_get_navail(dip, intr_type, &avail); 5912 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5913 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5914 "ddi_intr_get_navail() failed, " 5915 "rc %d avail %d\n", rc, avail); 5916 return (DDI_FAILURE); 5917 } 5918 5919 #if AHCI_DEBUG 5920 if (avail < count) { 5921 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5922 "ddi_intr_get_nintrs returned %d, navail() returned %d", 5923 count, avail); 5924 } 5925 #endif 5926 5927 /* 5928 * Note: So far Solaris restricts the maximum number of messages for 5929 * x86 to 2, that is avail is 2, so here we set the count with 1 to 5930 * force the driver to use single MSI message interrupt. In future if 5931 * Solaris remove the restriction, then we need to delete the below 5932 * code and try to use multiple interrupt routine to gain better 5933 * performance. 5934 */ 5935 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 5936 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5937 "force to use one interrupt routine though the " 5938 "HBA supports %d interrupt", count); 5939 count = 1; 5940 } 5941 5942 /* Allocate an array of interrupt handles. */ 5943 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5944 ahci_ctlp->ahcictl_intr_htable = 5945 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5946 5947 /* call ddi_intr_alloc(). */ 5948 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5949 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 5950 5951 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5952 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5953 "ddi_intr_alloc() failed, rc %d count %d actual %d " 5954 "avail %d\n", rc, count, actual, avail); 5955 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5956 ahci_ctlp->ahcictl_intr_size); 5957 return (DDI_FAILURE); 5958 } 5959 5960 /* use interrupt count returned */ 5961 #if AHCI_DEBUG 5962 if (actual < count) { 5963 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5964 "Requested: %d, Received: %d", count, actual); 5965 } 5966 #endif 5967 5968 ahci_ctlp->ahcictl_intr_cnt = actual; 5969 5970 /* 5971 * Get priority for first, assume remaining are all the same. 5972 */ 5973 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5974 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5975 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5976 "ddi_intr_get_pri() failed", NULL); 5977 5978 /* Free already allocated intr. */ 5979 for (i = 0; i < actual; i++) { 5980 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5981 } 5982 5983 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5984 ahci_ctlp->ahcictl_intr_size); 5985 return (DDI_FAILURE); 5986 } 5987 5988 /* Test for high level interrupt. */ 5989 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5990 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5991 "ahci_add_intrs: Hi level intr not supported", NULL); 5992 5993 /* Free already allocated intr. */ 5994 for (i = 0; i < actual; i++) { 5995 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5996 } 5997 5998 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5999 sizeof (ddi_intr_handle_t)); 6000 6001 return (DDI_FAILURE); 6002 } 6003 6004 /* Call ddi_intr_add_handler(). */ 6005 for (i = 0; i < actual; i++) { 6006 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 6007 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 6008 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 6009 "ddi_intr_add_handler() failed", NULL); 6010 6011 /* Free already allocated intr. */ 6012 for (i = 0; i < actual; i++) { 6013 (void) ddi_intr_free( 6014 ahci_ctlp->ahcictl_intr_htable[i]); 6015 } 6016 6017 kmem_free(ahci_ctlp->ahcictl_intr_htable, 6018 ahci_ctlp->ahcictl_intr_size); 6019 return (DDI_FAILURE); 6020 } 6021 } 6022 6023 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 6024 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 6025 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 6026 "ddi_intr_get_cap() failed", NULL); 6027 6028 /* Free already allocated intr. */ 6029 for (i = 0; i < actual; i++) { 6030 (void) ddi_intr_free( 6031 ahci_ctlp->ahcictl_intr_htable[i]); 6032 } 6033 6034 kmem_free(ahci_ctlp->ahcictl_intr_htable, 6035 ahci_ctlp->ahcictl_intr_size); 6036 return (DDI_FAILURE); 6037 } 6038 6039 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 6040 /* Call ddi_intr_block_enable() for MSI. */ 6041 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 6042 ahci_ctlp->ahcictl_intr_cnt); 6043 } else { 6044 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 6045 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 6046 (void) ddi_intr_enable( 6047 ahci_ctlp->ahcictl_intr_htable[i]); 6048 } 6049 } 6050 6051 return (DDI_SUCCESS); 6052 } 6053 6054 /* 6055 * Removes the registered interrupts irrespective of whether they 6056 * were legacy or MSI. 6057 * 6058 * WARNING!!! The controller interrupts must be disabled before calling 6059 * this routine. 6060 */ 6061 static void 6062 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 6063 { 6064 int x; 6065 6066 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 6067 6068 /* Disable all interrupts. */ 6069 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 6070 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 6071 /* Call ddi_intr_block_disable(). */ 6072 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 6073 ahci_ctlp->ahcictl_intr_cnt); 6074 } else { 6075 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 6076 (void) ddi_intr_disable( 6077 ahci_ctlp->ahcictl_intr_htable[x]); 6078 } 6079 } 6080 6081 /* Call ddi_intr_remove_handler(). */ 6082 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 6083 (void) ddi_intr_remove_handler( 6084 ahci_ctlp->ahcictl_intr_htable[x]); 6085 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 6086 } 6087 6088 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 6089 } 6090 6091 /* 6092 * This routine tries to put port into P:NotRunning state by clearing 6093 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 6094 * and PxCMD.CR to '0'. 6095 * 6096 * WARNING!!! ahciport_mutex should be acquired before the function 6097 * is called. 6098 */ 6099 static int 6100 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 6101 ahci_port_t *ahci_portp, uint8_t port) 6102 { 6103 uint32_t port_cmd_status; 6104 int loop_count; 6105 6106 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6107 "ahci_put_port_into_notrunning_state enter: port %d", port); 6108 6109 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6110 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6111 6112 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 6113 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6114 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 6115 6116 /* Wait until PxCMD.CR is cleared */ 6117 loop_count = 0; 6118 do { 6119 port_cmd_status = 6120 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6121 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6122 6123 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 6124 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6125 "clearing port %d CMD.CR timeout, " 6126 "port_cmd_status = 0x%x", port, 6127 port_cmd_status); 6128 /* 6129 * We are effectively timing out after 0.5 sec. 6130 * This value is specified in AHCI spec. 6131 */ 6132 break; 6133 } 6134 6135 /* Wait for 10 millisec */ 6136 drv_usecwait(10000); 6137 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 6138 6139 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 6140 6141 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 6142 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 6143 "ahci_put_port_into_notrunning_state: failed to clear " 6144 "PxCMD.CR to '0' after loop count: %d, and " 6145 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6146 return (AHCI_FAILURE); 6147 } else { 6148 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 6149 "ahci_put_port_into_notrunning_state: succeeded to clear " 6150 "PxCMD.CR to '0' after loop count: %d, and " 6151 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6152 return (AHCI_SUCCESS); 6153 } 6154 } 6155 6156 /* 6157 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 6158 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 6159 * stable state, then set PxCMD.ST to '1' to start the port directly. 6160 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 6161 * COMRESET to the device to put it in an idle state. 6162 * 6163 * The fifth argument returns whether the port reset is involved during 6164 * the process. 6165 * 6166 * The routine will be called under following scenarios: 6167 * + To abort the packet(s) 6168 * + To reset the port 6169 * + To activate the port 6170 * + Fatal error recovery 6171 * + To abort the timeout packet(s) 6172 * 6173 * WARNING!!! ahciport_mutex should be acquired before the function 6174 * is called. And ahciport_mutex will be released before the reset 6175 * event is reported to sata module by calling sata_hba_event_notify, 6176 * and then be acquired again later. 6177 * 6178 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6179 * according to the spec, the clearance of three bits will also clear 6180 * three interrupt status bits. 6181 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6182 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6183 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6184 * 6185 * Among these three interrupt events, the driver needs to take care of 6186 * PxIS.PRCS, which is the hot plug event. When the driver found out 6187 * a device was unplugged, it will call the interrupt handler. 6188 */ 6189 static int 6190 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6191 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6192 { 6193 uint32_t port_sstatus; 6194 uint32_t task_file_status; 6195 sata_device_t sdevice; 6196 int rval; 6197 int dev_exists_begin = 0; 6198 int dev_exists_end = 0; 6199 6200 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6201 "ahci_restart_port_wait_till_ready: port %d enter", port); 6202 6203 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 6204 dev_exists_begin = 1; 6205 6206 /* First clear PxCMD.ST */ 6207 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6208 port); 6209 if (rval != AHCI_SUCCESS) 6210 /* 6211 * If PxCMD.CR does not clear within a reasonable time, it 6212 * may assume the interface is in a hung condition and may 6213 * continue with issuing the port reset. 6214 */ 6215 goto reset; 6216 6217 /* Then clear PxSERR */ 6218 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6219 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6220 AHCI_SERROR_CLEAR_ALL); 6221 6222 /* Then get PxTFD */ 6223 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6224 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6225 6226 /* 6227 * Check whether the device is in a stable status, if yes, 6228 * then start the port directly. However for ahci_tran_dport_reset, 6229 * we may have to perform a port reset. 6230 */ 6231 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6232 !(flag & AHCI_PORT_RESET)) 6233 goto out; 6234 6235 reset: 6236 /* 6237 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6238 * a COMRESET to the device 6239 */ 6240 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6241 6242 if (reset_flag != NULL) 6243 *reset_flag = 1; 6244 6245 /* Indicate to the framework that a reset has happened. */ 6246 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6247 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6248 /* Set the reset in progress flag */ 6249 ahci_portp->ahciport_reset_in_progress = 1; 6250 6251 bzero((void *)&sdevice, sizeof (sata_device_t)); 6252 sdevice.satadev_addr.cport = 6253 ahci_ctlp->ahcictl_port_to_cport[port]; 6254 sdevice.satadev_addr.pmport = 0; 6255 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6256 6257 sdevice.satadev_state = SATA_DSTATE_RESET | 6258 SATA_DSTATE_PWR_ACTIVE; 6259 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6260 mutex_exit(&ahci_portp->ahciport_mutex); 6261 sata_hba_event_notify( 6262 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6263 &sdevice, 6264 SATA_EVNT_DEVICE_RESET); 6265 mutex_enter(&ahci_portp->ahciport_mutex); 6266 } 6267 6268 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 6269 "port %d sending event up: SATA_EVNT_RESET", port); 6270 } else { 6271 ahci_portp->ahciport_reset_in_progress = 0; 6272 } 6273 6274 out: 6275 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6276 6277 /* SStatus tells the presence of device. */ 6278 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6279 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6280 6281 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6282 dev_exists_end = 1; 6283 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6284 } 6285 6286 /* Check whether a hot plug event happened */ 6287 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6288 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6289 "ahci_restart_port_wait_till_ready: port %d " 6290 "device is removed", port); 6291 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6292 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6293 "ahci_restart_port_wait_till_ready: port %d " 6294 "AHCI_PORT_FLAG_NODEV flag is set", port); 6295 mutex_exit(&ahci_portp->ahciport_mutex); 6296 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6297 mutex_enter(&ahci_portp->ahciport_mutex); 6298 } 6299 6300 return (rval); 6301 } 6302 6303 /* 6304 * This routine may be called under four scenarios: 6305 * a) do the recovery from fatal error 6306 * b) or we need to timeout some commands 6307 * c) or we need to abort some commands 6308 * d) or we need reset device/port/controller 6309 * 6310 * In all these scenarios, we need to send any pending unfinished 6311 * commands up to sata framework. 6312 * 6313 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6314 */ 6315 static void 6316 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6317 ahci_port_t *ahci_portp, 6318 uint32_t slot_status, 6319 uint32_t failed_tags, 6320 uint32_t timeout_tags, 6321 uint32_t aborted_tags, 6322 uint32_t reset_tags) 6323 { 6324 uint32_t finished_tags = 0; 6325 uint32_t unfinished_tags = 0; 6326 int tmp_slot; 6327 sata_pkt_t *satapkt; 6328 int ncq_cmd_in_progress = 0; 6329 int err_retri_cmd_in_progress = 0; 6330 6331 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6332 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6333 ahci_portp->ahciport_port_num, slot_status); 6334 6335 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6336 "ahci_mop_commands: failed_tags: 0x%x, " 6337 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6338 "reset_tags: 0x%x", failed_tags, 6339 timeout_tags, aborted_tags, reset_tags); 6340 6341 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6342 finished_tags = ahci_portp->ahciport_pending_tags & 6343 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6344 6345 unfinished_tags = slot_status & 6346 AHCI_SLOT_MASK(ahci_ctlp) & 6347 ~failed_tags & 6348 ~aborted_tags & 6349 ~reset_tags & 6350 ~timeout_tags; 6351 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6352 ncq_cmd_in_progress = 1; 6353 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6354 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6355 6356 unfinished_tags = slot_status & 6357 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6358 ~failed_tags & 6359 ~aborted_tags & 6360 ~reset_tags & 6361 ~timeout_tags; 6362 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6363 6364 /* 6365 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6366 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6367 * complete successfully due to one of the following three 6368 * conditions: 6369 * 6370 * 1. Fatal error - failed_tags includes its slot 6371 * 2. Timed out - timeout_tags includes its slot 6372 * 3. Aborted when hot unplug - aborted_tags includes its 6373 * slot 6374 * 6375 * Please note that the command is always sent down in Slot 0 6376 */ 6377 err_retri_cmd_in_progress = 1; 6378 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6379 "ahci_mop_commands is called for port %d while " 6380 "REQUEST SENSE or READ LOG EXT for error retrieval " 6381 "is being executed slot_status = 0x%x", 6382 ahci_portp->ahciport_port_num, slot_status); 6383 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6384 ASSERT(slot_status == 0x1); 6385 } 6386 6387 /* Send up finished packets with SATA_PKT_COMPLETED */ 6388 while (finished_tags) { 6389 tmp_slot = ddi_ffs(finished_tags) - 1; 6390 if (tmp_slot == -1) { 6391 break; 6392 } 6393 6394 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6395 ASSERT(satapkt != NULL); 6396 6397 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6398 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6399 (void *)satapkt); 6400 6401 /* 6402 * Cannot fetch the return register content since the port 6403 * was restarted, so the corresponding tag will be set to 6404 * aborted tags. 6405 */ 6406 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6407 CLEAR_BIT(finished_tags, tmp_slot); 6408 aborted_tags |= tmp_slot; 6409 continue; 6410 } 6411 6412 if (ncq_cmd_in_progress) 6413 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6414 tmp_slot); 6415 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6416 CLEAR_BIT(finished_tags, tmp_slot); 6417 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6418 6419 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6420 } 6421 6422 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6423 while (failed_tags) { 6424 if (err_retri_cmd_in_progress) { 6425 satapkt = ahci_portp->ahciport_err_retri_pkt; 6426 ASSERT(satapkt != NULL); 6427 ASSERT(failed_tags == 0x1); 6428 6429 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6430 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6431 (void *)satapkt); 6432 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6433 break; 6434 } 6435 6436 tmp_slot = ddi_ffs(failed_tags) - 1; 6437 if (tmp_slot == -1) { 6438 break; 6439 } 6440 6441 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6442 ASSERT(satapkt != NULL); 6443 6444 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6445 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6446 (void *)satapkt); 6447 6448 if (ncq_cmd_in_progress) 6449 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6450 tmp_slot); 6451 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6452 CLEAR_BIT(failed_tags, tmp_slot); 6453 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6454 6455 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6456 } 6457 6458 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6459 while (timeout_tags) { 6460 if (err_retri_cmd_in_progress) { 6461 satapkt = ahci_portp->ahciport_err_retri_pkt; 6462 ASSERT(satapkt != NULL); 6463 ASSERT(timeout_tags == 0x1); 6464 6465 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6466 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6467 (void *)satapkt); 6468 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6469 break; 6470 } 6471 6472 tmp_slot = ddi_ffs(timeout_tags) - 1; 6473 if (tmp_slot == -1) { 6474 break; 6475 } 6476 6477 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6478 ASSERT(satapkt != NULL); 6479 6480 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6481 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6482 (void *)satapkt); 6483 6484 if (ncq_cmd_in_progress) 6485 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6486 tmp_slot); 6487 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6488 CLEAR_BIT(timeout_tags, tmp_slot); 6489 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6490 6491 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6492 } 6493 6494 /* Send up aborted packets with SATA_PKT_ABORTED */ 6495 while (aborted_tags) { 6496 if (err_retri_cmd_in_progress) { 6497 satapkt = ahci_portp->ahciport_err_retri_pkt; 6498 ASSERT(satapkt != NULL); 6499 ASSERT(aborted_tags == 0x1); 6500 6501 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6502 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6503 (void *)satapkt); 6504 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6505 break; 6506 } 6507 6508 tmp_slot = ddi_ffs(aborted_tags) - 1; 6509 if (tmp_slot == -1) { 6510 break; 6511 } 6512 6513 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6514 ASSERT(satapkt != NULL); 6515 6516 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6517 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6518 (void *)satapkt); 6519 6520 if (ncq_cmd_in_progress) 6521 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6522 tmp_slot); 6523 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6524 CLEAR_BIT(aborted_tags, tmp_slot); 6525 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6526 6527 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6528 } 6529 6530 /* Send up reset packets with SATA_PKT_RESET. */ 6531 while (reset_tags) { 6532 tmp_slot = ddi_ffs(reset_tags) - 1; 6533 if (tmp_slot == -1) { 6534 break; 6535 } 6536 6537 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6538 ASSERT(satapkt != NULL); 6539 6540 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6541 "sending up pkt 0x%p with SATA_PKT_RESET", 6542 (void *)satapkt); 6543 6544 if (ncq_cmd_in_progress) 6545 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6546 tmp_slot); 6547 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6548 CLEAR_BIT(reset_tags, tmp_slot); 6549 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6550 6551 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6552 } 6553 6554 /* Send up unfinished packets with SATA_PKT_RESET */ 6555 while (unfinished_tags) { 6556 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6557 if (tmp_slot == -1) { 6558 break; 6559 } 6560 6561 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6562 ASSERT(satapkt != NULL); 6563 6564 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6565 "sending up pkt 0x%p with SATA_PKT_RESET", 6566 (void *)satapkt); 6567 6568 if (ncq_cmd_in_progress) 6569 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6570 tmp_slot); 6571 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6572 CLEAR_BIT(unfinished_tags, tmp_slot); 6573 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6574 6575 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6576 } 6577 6578 ahci_portp->ahciport_mop_in_progress--; 6579 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6580 6581 if (ahci_portp->ahciport_mop_in_progress == 0) 6582 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6583 } 6584 6585 /* 6586 * This routine is going to first request a READ LOG EXT sata pkt from sata 6587 * module, and then deliver it to the HBA to get the ncq failure context. 6588 * The return value is the exactly failed tags. 6589 */ 6590 static uint32_t 6591 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6592 uint8_t port) 6593 { 6594 sata_device_t sdevice; 6595 sata_pkt_t *rdlog_spkt, *spkt; 6596 ddi_dma_handle_t buf_dma_handle; 6597 int loop_count; 6598 int rval; 6599 int failed_slot; 6600 uint32_t failed_tags = 0; 6601 struct sata_ncq_error_recovery_page *ncq_err_page; 6602 6603 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6604 "ahci_get_rdlogext_data enter: port %d", port); 6605 6606 /* Prepare the sdevice data */ 6607 bzero((void *)&sdevice, sizeof (sata_device_t)); 6608 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6609 6610 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6611 sdevice.satadev_addr.pmport = 0; 6612 6613 /* 6614 * Call the sata hba interface to get a rdlog spkt 6615 */ 6616 loop_count = 0; 6617 loop: 6618 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6619 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6620 if (rdlog_spkt == NULL) { 6621 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6622 /* Sleep for a while */ 6623 delay(AHCI_10MS_TICKS); 6624 goto loop; 6625 } 6626 /* Timed out after 1s */ 6627 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6628 "failed to get rdlog spkt for port %d", port); 6629 return (failed_tags); 6630 } 6631 6632 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6633 6634 /* 6635 * This flag is used to handle the specific error recovery when the 6636 * READ LOG EXT command gets a failure (fatal error or time-out). 6637 */ 6638 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6639 6640 /* 6641 * This start is not supposed to fail because after port is restarted, 6642 * the whole command list is empty. 6643 */ 6644 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6645 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6646 ahci_portp->ahciport_err_retri_pkt = NULL; 6647 6648 /* Remove the flag after READ LOG EXT command is completed */ 6649 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6650 6651 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6652 /* Update the request log data */ 6653 buf_dma_handle = *(ddi_dma_handle_t *) 6654 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6655 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6656 DDI_DMA_SYNC_FORKERNEL); 6657 if (rval == DDI_SUCCESS) { 6658 ncq_err_page = 6659 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6660 satapkt_cmd.satacmd_bp->b_un.b_addr; 6661 6662 /* Get the failed tag */ 6663 failed_slot = ncq_err_page->ncq_tag; 6664 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6665 "ahci_get_rdlogext_data: port %d " 6666 "failed slot %d", port, failed_slot); 6667 if (failed_slot & NQ) { 6668 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6669 "the failed slot is not a valid tag", NULL); 6670 goto out; 6671 } 6672 6673 failed_slot &= NCQ_TAG_MASK; 6674 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6675 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6676 "ahci_get_rdlogext_data: failed spkt 0x%p", 6677 (void *)spkt); 6678 if (spkt == NULL) { 6679 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6680 "the failed slot spkt is NULL", NULL); 6681 goto out; 6682 } 6683 6684 failed_tags = 0x1 << failed_slot; 6685 6686 /* Fill out the error context */ 6687 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6688 ncq_err_page); 6689 ahci_update_sata_registers(ahci_ctlp, port, 6690 &spkt->satapkt_device); 6691 } 6692 } 6693 out: 6694 sata_free_error_retrieval_pkt(rdlog_spkt); 6695 6696 return (failed_tags); 6697 } 6698 6699 /* 6700 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6701 * module, and then deliver it to the HBA to get the sense data and copy 6702 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6703 * SENSE sata pkt later. 6704 */ 6705 static void 6706 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6707 uint8_t port, sata_pkt_t *spkt) 6708 { 6709 sata_device_t sdevice; 6710 sata_pkt_t *rs_spkt; 6711 sata_cmd_t *sata_cmd; 6712 ddi_dma_handle_t buf_dma_handle; 6713 int loop_count; 6714 #if AHCI_DEBUG 6715 struct scsi_extended_sense *rqsense; 6716 #endif 6717 6718 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6719 "ahci_get_rqsense_data enter: port %d", port); 6720 6721 /* Prepare the sdevice data */ 6722 bzero((void *)&sdevice, sizeof (sata_device_t)); 6723 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6724 6725 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6726 sdevice.satadev_addr.pmport = 0; 6727 6728 sata_cmd = &spkt->satapkt_cmd; 6729 6730 /* 6731 * Call the sata hba interface to get a rs spkt 6732 */ 6733 loop_count = 0; 6734 loop: 6735 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6736 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6737 if (rs_spkt == NULL) { 6738 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6739 /* Sleep for a while */ 6740 delay(AHCI_10MS_TICKS); 6741 goto loop; 6742 6743 } 6744 /* Timed out after 1s */ 6745 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6746 "failed to get rs spkt for port %d", port); 6747 return; 6748 } 6749 6750 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6751 6752 /* 6753 * This flag is used to handle the specific error recovery when the 6754 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6755 */ 6756 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6757 6758 /* 6759 * This start is not supposed to fail because after port is restarted, 6760 * the whole command list is empty. 6761 */ 6762 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6763 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6764 ahci_portp->ahciport_err_retri_pkt = NULL; 6765 6766 /* Remove the flag after REQUEST SENSE command is completed */ 6767 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6768 6769 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6770 /* Update the request sense data */ 6771 buf_dma_handle = *(ddi_dma_handle_t *) 6772 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6773 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6774 DDI_DMA_SYNC_FORKERNEL); 6775 /* Copy the request sense data */ 6776 bcopy(rs_spkt-> 6777 satapkt_cmd.satacmd_bp->b_un.b_addr, 6778 &sata_cmd->satacmd_rqsense, 6779 SATA_ATAPI_MIN_RQSENSE_LEN); 6780 #if AHCI_DEBUG 6781 rqsense = (struct scsi_extended_sense *) 6782 sata_cmd->satacmd_rqsense; 6783 6784 /* Dump the sense data */ 6785 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 6786 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 6787 "Sense data for satapkt %p ATAPI cmd 0x%x", 6788 spkt, sata_cmd->satacmd_acdb[0]); 6789 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 6790 " es_code 0x%x es_class 0x%x " 6791 "es_key 0x%x es_add_code 0x%x " 6792 "es_qual_code 0x%x", 6793 rqsense->es_code, rqsense->es_class, 6794 rqsense->es_key, rqsense->es_add_code, 6795 rqsense->es_qual_code); 6796 #endif 6797 } 6798 6799 sata_free_error_retrieval_pkt(rs_spkt); 6800 } 6801 6802 /* 6803 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6804 * the port must be restarted. When the HBA detects thus error, it may try 6805 * to abort a transfer. And if the transfer was aborted, the device is 6806 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6807 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6808 * that the device is in a stable status and transfers may be restarted without 6809 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6810 * then the software will send the COMRESET to do the port reset. 6811 * 6812 * Software should perform the appropriate error recovery actions based on 6813 * whether non-queued commands were being issued or natived command queuing 6814 * commands were being issued. 6815 * 6816 * And software will complete the command that had the error with error mark 6817 * to higher level software. 6818 * 6819 * Fatal errors include the following: 6820 * PxIS.IFS - Interface Fatal Error Status 6821 * PxIS.HBDS - Host Bus Data Error Status 6822 * PxIS.HBFS - Host Bus Fatal Error Status 6823 * PxIS.TFES - Task File Error Status 6824 * 6825 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6826 */ 6827 static void 6828 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6829 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6830 { 6831 uint32_t port_cmd_status; 6832 uint32_t slot_status = 0; 6833 uint32_t failed_tags = 0; 6834 int failed_slot; 6835 int reset_flag = 0; 6836 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6837 sata_cmd_t *sata_cmd = NULL; 6838 sata_pkt_t *spkt = NULL; 6839 #if AHCI_DEBUG 6840 ahci_cmd_header_t *cmd_header; 6841 #endif 6842 6843 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6844 "ahci_fatal_error_recovery_handler enter: port %d", port); 6845 6846 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6847 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6848 6849 /* Read PxCI to see which commands are still outstanding */ 6850 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6851 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6852 6853 /* 6854 * Read PxCMD.CCS to determine the slot that the HBA 6855 * was processing when the error occurred. 6856 */ 6857 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6858 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6859 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6860 AHCI_CMD_STATUS_CCS_SHIFT; 6861 6862 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6863 spkt = ahci_portp->ahciport_err_retri_pkt; 6864 ASSERT(spkt != NULL); 6865 } else { 6866 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6867 if (spkt == NULL) { 6868 /* May happen when interface errors occur? */ 6869 goto next; 6870 } 6871 } 6872 6873 #if AHCI_DEBUG 6874 /* 6875 * Debugging purpose... 6876 */ 6877 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6878 cmd_header = 6879 &ahci_portp->ahciport_cmd_list[failed_slot]; 6880 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6881 "ahci_fatal_error_recovery_handler: port %d, " 6882 "PRD Byte Count = 0x%x, " 6883 "ahciport_prd_bytecounts = 0x%x", port, 6884 cmd_header->ahcich_prd_byte_count, 6885 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6886 } 6887 #endif 6888 6889 sata_cmd = &spkt->satapkt_cmd; 6890 6891 /* Fill out the status and error registers for PxIS.TFES */ 6892 if (intr_status & AHCI_INTR_STATUS_TFES) { 6893 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6894 ahcirf_d2h_register_fis); 6895 6896 /* Copy the error context back to the sata_cmd */ 6897 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6898 } 6899 6900 /* The failed command must be one of the outstanding commands */ 6901 failed_tags = 0x1 << failed_slot; 6902 ASSERT(failed_tags & slot_status); 6903 6904 /* Update the sata registers, especially PxSERR register */ 6905 ahci_update_sata_registers(ahci_ctlp, port, 6906 &spkt->satapkt_device); 6907 6908 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6909 /* Read PxSACT to see which commands are still outstanding */ 6910 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6911 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6912 } 6913 next: 6914 6915 #if AHCI_DEBUG 6916 /* 6917 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6918 * set, it means a fatal error happened after REQUEST SENSE command 6919 * or READ LOG EXT command is delivered to the HBA during the error 6920 * recovery process. At this time, the only outstanding command is 6921 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6922 */ 6923 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6924 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6925 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6926 "command or READ LOG EXT command for error data retrieval " 6927 "failed", port); 6928 ASSERT(slot_status == 0x1); 6929 ASSERT(failed_slot == 0); 6930 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6931 SCMD_REQUEST_SENSE || 6932 spkt->satapkt_cmd.satacmd_cmd_reg == 6933 SATAC_READ_LOG_EXT); 6934 } 6935 #endif 6936 6937 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6938 ahci_portp->ahciport_mop_in_progress++; 6939 6940 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6941 port, NULL, &reset_flag); 6942 6943 /* 6944 * Won't retrieve error information: 6945 * 1. Port reset was involved to recover 6946 * 2. Device is gone 6947 * 3. IDENTIFY DEVICE command sent to ATAPI device 6948 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6949 */ 6950 if (reset_flag || 6951 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6952 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6953 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6954 goto out; 6955 6956 /* 6957 * Deliver READ LOG EXT to gather information about the error when 6958 * a COMRESET has not been performed as part of the error recovery 6959 * during NCQ command processing. 6960 */ 6961 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6962 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6963 ahci_portp, port); 6964 goto out; 6965 } 6966 6967 /* 6968 * Deliver REQUEST SENSE for ATAPI command to gather information about 6969 * the error when a COMRESET has not been performed as part of the 6970 * error recovery. 6971 */ 6972 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6973 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6974 out: 6975 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6976 "ahci_fatal_error_recovery_handler: port %d fatal error " 6977 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6978 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6979 port, slot_status, ahci_portp->ahciport_pending_tags, 6980 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6981 6982 ahci_mop_commands(ahci_ctlp, 6983 ahci_portp, 6984 slot_status, 6985 failed_tags, /* failed tags */ 6986 0, /* timeout tags */ 6987 0, /* aborted tags */ 6988 0); /* reset tags */ 6989 } 6990 6991 /* 6992 * Handle events - fatal error recovery 6993 */ 6994 static void 6995 ahci_events_handler(void *args) 6996 { 6997 ahci_event_arg_t *ahci_event_arg; 6998 ahci_ctl_t *ahci_ctlp; 6999 ahci_port_t *ahci_portp; 7000 uint32_t event; 7001 uint8_t port; 7002 7003 ahci_event_arg = (ahci_event_arg_t *)args; 7004 7005 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 7006 ahci_portp = ahci_event_arg->ahciea_portp; 7007 event = ahci_event_arg->ahciea_event; 7008 port = ahci_portp->ahciport_port_num; 7009 7010 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7011 "ahci_events_handler enter: port %d intr_status = 0x%x", 7012 port, event); 7013 7014 mutex_enter(&ahci_portp->ahciport_mutex); 7015 7016 /* 7017 * ahci_intr_phyrdy_change() may have rendered it to 7018 * SATA_DTYPE_NONE. 7019 */ 7020 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7021 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7022 "ahci_events_handler: port %d no device attached, " 7023 "and just return without doing anything", port); 7024 goto out; 7025 } 7026 7027 if (event & (AHCI_INTR_STATUS_IFS | 7028 AHCI_INTR_STATUS_HBDS | 7029 AHCI_INTR_STATUS_HBFS | 7030 AHCI_INTR_STATUS_TFES)) 7031 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 7032 port, event); 7033 7034 out: 7035 mutex_exit(&ahci_portp->ahciport_mutex); 7036 } 7037 7038 /* 7039 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 7040 * detect there are some commands which are timed out. 7041 */ 7042 static void 7043 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7044 uint8_t port, uint32_t tmp_timeout_tags) 7045 { 7046 uint32_t slot_status = 0; 7047 uint32_t finished_tags = 0; 7048 uint32_t timeout_tags = 0; 7049 7050 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 7051 "ahci_timeout_pkts enter: port %d", port); 7052 7053 mutex_enter(&ahci_portp->ahciport_mutex); 7054 7055 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7056 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7057 /* Read PxCI to see which commands are still outstanding */ 7058 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7059 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7060 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7061 /* Read PxSACT to see which commands are still outstanding */ 7062 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7063 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7064 } 7065 7066 #if AHCI_DEBUG 7067 /* 7068 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 7069 * set, it means a fatal error happened after REQUEST SENSE command 7070 * or READ LOG EXT command is delivered to the HBA during the error 7071 * recovery process. At this time, the only outstanding command is 7072 * supposed to be REQUEST SENSE command or READ LOG EXT command. 7073 */ 7074 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7075 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 7076 "ahci_timeout_pkts called while REQUEST SENSE " 7077 "command or READ LOG EXT command for error recovery " 7078 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 7079 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 7080 tmp_timeout_tags, slot_status, 7081 ahci_portp->ahciport_pending_tags, 7082 ahci_portp->ahciport_pending_ncq_tags); 7083 ASSERT(slot_status == 0x1); 7084 } 7085 #endif 7086 7087 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 7088 ahci_portp->ahciport_mop_in_progress++; 7089 7090 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 7091 port, NULL, NULL); 7092 7093 /* 7094 * Re-identify timeout tags because some previously checked commands 7095 * could already complete. 7096 */ 7097 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7098 finished_tags = ahci_portp->ahciport_pending_tags & 7099 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 7100 timeout_tags = tmp_timeout_tags & ~finished_tags; 7101 7102 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 7103 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 7104 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 7105 "pending_tags = 0x%x ", 7106 port, finished_tags, timeout_tags, 7107 slot_status, ahci_portp->ahciport_pending_tags); 7108 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7109 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 7110 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 7111 timeout_tags = tmp_timeout_tags & ~finished_tags; 7112 7113 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 7114 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 7115 "timeout_tags = 0x%x, port_sactive = 0x%x, " 7116 "pending_ncq_tags = 0x%x ", 7117 port, finished_tags, timeout_tags, 7118 slot_status, ahci_portp->ahciport_pending_ncq_tags); 7119 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7120 timeout_tags = tmp_timeout_tags; 7121 } 7122 7123 ahci_mop_commands(ahci_ctlp, 7124 ahci_portp, 7125 slot_status, 7126 0, /* failed tags */ 7127 timeout_tags, /* timeout tags */ 7128 0, /* aborted tags */ 7129 0); /* reset tags */ 7130 7131 mutex_exit(&ahci_portp->ahciport_mutex); 7132 } 7133 7134 /* 7135 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 7136 * for long time. 7137 */ 7138 static void 7139 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 7140 { 7141 ahci_port_t *ahci_portp; 7142 sata_pkt_t *spkt; 7143 uint32_t pending_tags; 7144 uint32_t timeout_tags; 7145 uint32_t port_cmd_status; 7146 uint32_t port_sactive; 7147 uint8_t port; 7148 int tmp_slot; 7149 int current_slot; 7150 uint32_t current_tags; 7151 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7152 /* max number of cycles this packet should survive */ 7153 int max_life_cycles; 7154 7155 /* how many cycles this packet survived so far */ 7156 int watched_cycles; 7157 7158 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7159 7160 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7161 "ahci_watchdog_handler entered", NULL); 7162 7163 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7164 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7165 continue; 7166 } 7167 7168 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7169 7170 mutex_enter(&ahci_portp->ahciport_mutex); 7171 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7172 mutex_exit(&ahci_portp->ahciport_mutex); 7173 continue; 7174 } 7175 7176 /* Skip the check for those ports in error recovery */ 7177 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7178 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7179 mutex_exit(&ahci_portp->ahciport_mutex); 7180 continue; 7181 } 7182 7183 pending_tags = 0; 7184 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7185 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7186 7187 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7188 current_slot = 0; 7189 pending_tags = 0x1; 7190 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7191 current_slot = 7192 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7193 AHCI_CMD_STATUS_CCS_SHIFT; 7194 pending_tags = ahci_portp->ahciport_pending_tags; 7195 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7196 port_sactive = ddi_get32( 7197 ahci_ctlp->ahcictl_ahci_acc_handle, 7198 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7199 current_tags = port_sactive & 7200 ~port_cmd_status & 7201 AHCI_NCQ_SLOT_MASK(ahci_portp); 7202 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7203 } 7204 7205 timeout_tags = 0; 7206 while (pending_tags) { 7207 tmp_slot = ddi_ffs(pending_tags) - 1; 7208 if (tmp_slot == -1) { 7209 break; 7210 } 7211 7212 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7213 spkt = ahci_portp->ahciport_err_retri_pkt; 7214 else 7215 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7216 7217 if ((spkt != NULL) && spkt->satapkt_time && 7218 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7219 /* 7220 * We are overloading satapkt_hba_driver_private 7221 * with watched_cycle count. 7222 * 7223 * If a packet has survived for more than it's 7224 * max life cycles, it is a candidate for time 7225 * out. 7226 */ 7227 watched_cycles = (int)(intptr_t) 7228 spkt->satapkt_hba_driver_private; 7229 watched_cycles++; 7230 max_life_cycles = (spkt->satapkt_time + 7231 ahci_watchdog_timeout - 1) / 7232 ahci_watchdog_timeout; 7233 7234 spkt->satapkt_hba_driver_private = 7235 (void *)(intptr_t)watched_cycles; 7236 7237 if (watched_cycles <= max_life_cycles) 7238 goto next; 7239 7240 #if AHCI_DEBUG 7241 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7242 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7243 ahci_ctlp, "watchdog: the current " 7244 "tags is 0x%x", current_tags); 7245 } else { 7246 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7247 ahci_ctlp, "watchdog: the current " 7248 "slot is %d", current_slot); 7249 } 7250 #endif 7251 7252 /* 7253 * We need to check whether the HBA has 7254 * begun to execute the command, if not, 7255 * then re-set the timer of the command. 7256 */ 7257 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7258 (tmp_slot != current_slot) || 7259 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7260 ((0x1 << tmp_slot) & current_tags)) { 7261 spkt->satapkt_hba_driver_private = 7262 (void *)(intptr_t)0; 7263 } else { 7264 timeout_tags |= (0x1 << tmp_slot); 7265 cmn_err(CE_WARN, "!ahci%d: watchdog " 7266 "port %d satapkt 0x%p timed out\n", 7267 instance, port, (void *)spkt); 7268 } 7269 } 7270 next: 7271 CLEAR_BIT(pending_tags, tmp_slot); 7272 } 7273 7274 if (timeout_tags) { 7275 mutex_exit(&ahci_portp->ahciport_mutex); 7276 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7277 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7278 port, timeout_tags); 7279 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7280 mutex_enter(&ahci_portp->ahciport_mutex); 7281 } 7282 7283 mutex_exit(&ahci_portp->ahciport_mutex); 7284 } 7285 7286 /* Re-install the watchdog timeout handler */ 7287 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7288 ahci_ctlp->ahcictl_timeout_id = 7289 timeout((void (*)(void *))ahci_watchdog_handler, 7290 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7291 } 7292 7293 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7294 } 7295 7296 /* 7297 * Fill the error context into sata_cmd for non-queued command error. 7298 */ 7299 static void 7300 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7301 { 7302 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7303 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7304 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7305 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7306 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7307 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7308 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7309 7310 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7311 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7312 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7313 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7314 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7315 } 7316 } 7317 7318 /* 7319 * Fill the ncq error page into sata_cmd for queued command error. 7320 */ 7321 static void 7322 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7323 struct sata_ncq_error_recovery_page *ncq_err_page) 7324 { 7325 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7326 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7327 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7328 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7329 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7330 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7331 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7332 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7333 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7334 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7335 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7336 } 7337 7338 /* 7339 * Put the respective register value to sata_cmd_t for satacmd_flags. 7340 */ 7341 static void 7342 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7343 { 7344 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7345 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7346 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7347 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7348 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7349 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7350 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7351 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7352 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7353 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7354 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7355 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7356 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7357 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7358 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7359 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7360 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7361 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7362 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7363 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7364 } 7365 7366 static void 7367 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7368 uint32_t intr_status) 7369 { 7370 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7371 7372 if (intr_status & AHCI_INTR_STATUS_IFS) 7373 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 7374 "error", instance, port); 7375 7376 if (intr_status & AHCI_INTR_STATUS_HBDS) 7377 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 7378 instance, port); 7379 7380 if (intr_status & AHCI_INTR_STATUS_HBFS) 7381 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 7382 instance, port); 7383 7384 if (intr_status & AHCI_INTR_STATUS_TFES) 7385 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 7386 instance, port); 7387 7388 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 7389 "recovery", instance, port); 7390 } 7391 7392 /* 7393 * Dump the serror message to the log. 7394 */ 7395 static void 7396 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7397 uint32_t port_serror, int debug_only) 7398 { 7399 static char err_buf[512]; 7400 static char err_msg_header[16]; 7401 char *err_msg = err_buf; 7402 7403 *err_buf = '\0'; 7404 *err_msg_header = '\0'; 7405 7406 if (port_serror & SERROR_DATA_ERR_FIXED) { 7407 err_msg = strcat(err_msg, 7408 "\tRecovered Data Integrity Error (I)\n"); 7409 } 7410 7411 if (port_serror & SERROR_COMM_ERR_FIXED) { 7412 err_msg = strcat(err_msg, 7413 "\tRecovered Communication Error (M)\n"); 7414 } 7415 7416 if (port_serror & SERROR_DATA_ERR) { 7417 err_msg = strcat(err_msg, 7418 "\tTransient Data Integrity Error (T)\n"); 7419 } 7420 7421 if (port_serror & SERROR_PERSISTENT_ERR) { 7422 err_msg = strcat(err_msg, 7423 "\tPersistent Communication or Data Integrity Error (C)\n"); 7424 } 7425 7426 if (port_serror & SERROR_PROTOCOL_ERR) { 7427 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7428 } 7429 7430 if (port_serror & SERROR_INT_ERR) { 7431 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7432 } 7433 7434 if (port_serror & SERROR_PHY_RDY_CHG) { 7435 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7436 } 7437 7438 if (port_serror & SERROR_PHY_INT_ERR) { 7439 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7440 } 7441 7442 if (port_serror & SERROR_COMM_WAKE) { 7443 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7444 } 7445 7446 if (port_serror & SERROR_10B_TO_8B_ERR) { 7447 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7448 } 7449 7450 if (port_serror & SERROR_DISPARITY_ERR) { 7451 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7452 } 7453 7454 if (port_serror & SERROR_CRC_ERR) { 7455 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7456 } 7457 7458 if (port_serror & SERROR_HANDSHAKE_ERR) { 7459 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7460 } 7461 7462 if (port_serror & SERROR_LINK_SEQ_ERR) { 7463 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7464 } 7465 7466 if (port_serror & SERROR_TRANS_ERR) { 7467 err_msg = strcat(err_msg, 7468 "\tTransport state transition error (T)\n"); 7469 } 7470 7471 if (port_serror & SERROR_FIS_TYPE) { 7472 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7473 } 7474 7475 if (port_serror & SERROR_EXCHANGED_ERR) { 7476 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7477 } 7478 7479 if (err_msg == NULL) 7480 return; 7481 7482 if (debug_only) { 7483 (void) sprintf(err_msg_header, "port %d", port); 7484 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 7485 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 7486 } else if (ahci_ctlp) { 7487 cmn_err(CE_WARN, "!ahci%d: %s %s", 7488 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7489 err_msg_header, err_msg); 7490 7491 /* sata trace debug */ 7492 sata_trace_debug(ahci_ctlp->ahcictl_dip, 7493 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 7494 err_msg_header, err_msg); 7495 } else { 7496 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 7497 7498 /* sata trace debug */ 7499 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 7500 } 7501 } 7502 7503 /* 7504 * This routine is to calculate the total number of ports implemented 7505 * by the HBA. 7506 */ 7507 static int 7508 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7509 { 7510 uint8_t i; 7511 int num = 0; 7512 7513 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7514 if (((uint32_t)0x1 << i) & ports_implemented) 7515 num++; 7516 } 7517 7518 return (num); 7519 } 7520 7521 #if AHCI_DEBUG 7522 static void 7523 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7524 { 7525 static char name[16]; 7526 va_list ap; 7527 7528 mutex_enter(&ahci_log_mutex); 7529 7530 va_start(ap, fmt); 7531 if (ahci_ctlp) { 7532 (void) sprintf(name, "ahci%d: ", 7533 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7534 } else { 7535 (void) sprintf(name, "ahci: "); 7536 } 7537 7538 (void) vsprintf(ahci_log_buf, fmt, ap); 7539 va_end(ap); 7540 7541 cmn_err(level, "%s%s", name, ahci_log_buf); 7542 7543 mutex_exit(&ahci_log_mutex); 7544 } 7545 #endif 7546 7547 /* 7548 * quiesce(9E) entry point. 7549 * 7550 * This function is called when the system is single-threaded at high 7551 * PIL with preemption disabled. Therefore, this function must not be 7552 * blocked. 7553 * 7554 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7555 * DDI_FAILURE indicates an error condition and should almost never happen. 7556 */ 7557 static int 7558 ahci_quiesce(dev_info_t *dip) 7559 { 7560 ahci_ctl_t *ahci_ctlp; 7561 ahci_port_t *ahci_portp; 7562 int instance, port; 7563 7564 instance = ddi_get_instance(dip); 7565 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7566 7567 if (ahci_ctlp == NULL) 7568 return (DDI_FAILURE); 7569 7570 #if AHCI_DEBUG 7571 ahci_debug_flags = 0; 7572 #endif 7573 7574 /* disable all the interrupts. */ 7575 ahci_disable_all_intrs(ahci_ctlp); 7576 7577 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7578 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7579 continue; 7580 } 7581 7582 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7583 7584 /* 7585 * Stop the port by clearing PxCMD.ST 7586 * 7587 * Here we must disable the port interrupt because 7588 * ahci_disable_all_intrs only clear GHC.IE, and IS 7589 * register will be still set if PxIE is enabled. 7590 * When ahci shares one IRQ with other drivers, the 7591 * intr handler may claim the intr mistakenly. 7592 */ 7593 ahci_disable_port_intrs(ahci_ctlp, port); 7594 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7595 ahci_portp, port); 7596 } 7597 7598 return (DDI_SUCCESS); 7599 } 7600