1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_drain_ports_taskq(ahci_ctl_t *); 104 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 105 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 107 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 108 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 109 uint8_t, sata_pkt_t *); 110 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 111 uint8_t, sata_pkt_t *); 112 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 113 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 114 static void ahci_copy_ncq_err_page(sata_cmd_t *, 115 struct sata_ncq_error_recovery_page *); 116 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 117 118 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 119 static int ahci_hba_reset(ahci_ctl_t *); 120 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 124 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 125 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 126 uint8_t); 127 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 128 uint8_t, int, int *); 129 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 130 uint32_t, uint32_t, uint32_t, uint32_t); 131 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 132 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 133 uint8_t, sata_pkt_t *); 134 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 135 uint8_t, uint32_t); 136 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 137 uint8_t, uint32_t); 138 static void ahci_events_handler(void *); 139 static void ahci_watchdog_handler(ahci_ctl_t *); 140 141 static uint_t ahci_intr(caddr_t, caddr_t); 142 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 143 static int ahci_add_intrs(ahci_ctl_t *, int); 144 static void ahci_rem_intrs(ahci_ctl_t *); 145 static void ahci_enable_all_intrs(ahci_ctl_t *); 146 static void ahci_disable_all_intrs(ahci_ctl_t *); 147 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 148 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 149 150 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 151 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 154 ahci_port_t *, uint8_t); 155 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, uint32_t); 158 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 159 uint8_t, uint32_t); 160 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 161 162 static int ahci_get_num_implemented_ports(uint32_t); 163 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 164 uint32_t); 165 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 166 #if AHCI_DEBUG 167 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 168 #endif 169 170 171 /* 172 * DMA attributes for the data buffer 173 * 174 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 175 * does not support 64-bit addressing 176 */ 177 static ddi_dma_attr_t buffer_dma_attr = { 178 DMA_ATTR_V0, /* dma_attr_version */ 179 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 180 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 181 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 182 0x2ull, /* dma_attr_align: word aligned */ 183 1, /* dma_attr_burstsizes */ 184 1, /* dma_attr_minxfer */ 185 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 186 0xffffffffull, /* dma_attr_seg */ 187 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 188 512, /* dma_attr_granular */ 189 0, /* dma_attr_flags */ 190 }; 191 192 /* 193 * DMA attributes for the rcvd FIS 194 * 195 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 196 * does not support 64-bit addressing 197 */ 198 static ddi_dma_attr_t rcvd_fis_dma_attr = { 199 DMA_ATTR_V0, /* dma_attr_version */ 200 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 201 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 202 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 203 0x100ull, /* dma_attr_align: 256-byte aligned */ 204 1, /* dma_attr_burstsizes */ 205 1, /* dma_attr_minxfer */ 206 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 207 0xffffffffull, /* dma_attr_seg */ 208 1, /* dma_attr_sgllen */ 209 1, /* dma_attr_granular */ 210 0, /* dma_attr_flags */ 211 }; 212 213 /* 214 * DMA attributes for the command list 215 * 216 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 217 * does not support 64-bit addressing 218 */ 219 static ddi_dma_attr_t cmd_list_dma_attr = { 220 DMA_ATTR_V0, /* dma_attr_version */ 221 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 222 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 223 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 224 0x400ull, /* dma_attr_align: 1K-byte aligned */ 225 1, /* dma_attr_burstsizes */ 226 1, /* dma_attr_minxfer */ 227 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 228 0xffffffffull, /* dma_attr_seg */ 229 1, /* dma_attr_sgllen */ 230 1, /* dma_attr_granular */ 231 0, /* dma_attr_flags */ 232 }; 233 234 /* 235 * DMA attributes for cmd tables 236 * 237 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 238 * does not support 64-bit addressing 239 */ 240 static ddi_dma_attr_t cmd_table_dma_attr = { 241 DMA_ATTR_V0, /* dma_attr_version */ 242 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 243 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 244 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 245 0x80ull, /* dma_attr_align: 128-byte aligned */ 246 1, /* dma_attr_burstsizes */ 247 1, /* dma_attr_minxfer */ 248 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 249 0xffffffffull, /* dma_attr_seg */ 250 1, /* dma_attr_sgllen */ 251 1, /* dma_attr_granular */ 252 0, /* dma_attr_flags */ 253 }; 254 255 256 /* Device access attributes */ 257 static ddi_device_acc_attr_t accattr = { 258 DDI_DEVICE_ATTR_V0, 259 DDI_STRUCTURE_LE_ACC, 260 DDI_STRICTORDER_ACC 261 }; 262 263 264 static struct dev_ops ahcictl_dev_ops = { 265 DEVO_REV, /* devo_rev */ 266 0, /* refcnt */ 267 ahci_getinfo, /* info */ 268 nulldev, /* identify */ 269 nulldev, /* probe */ 270 ahci_attach, /* attach */ 271 ahci_detach, /* detach */ 272 nodev, /* no reset */ 273 (struct cb_ops *)0, /* driver operations */ 274 NULL, /* bus operations */ 275 NULL, /* power */ 276 ahci_quiesce, /* quiesce */ 277 }; 278 279 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 280 SATA_TRAN_HOTPLUG_OPS_REV_1, 281 ahci_tran_hotplug_port_activate, 282 ahci_tran_hotplug_port_deactivate 283 }; 284 285 extern struct mod_ops mod_driverops; 286 287 static struct modldrv modldrv = { 288 &mod_driverops, /* driverops */ 289 ahci_ident, /* short description */ 290 &ahcictl_dev_ops, /* driver ops */ 291 }; 292 293 static struct modlinkage modlinkage = { 294 MODREV_1, 295 &modldrv, 296 NULL 297 }; 298 299 /* The following variables are watchdog handler related */ 300 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 301 static int ahci_watchdog_tick; 302 303 /* 304 * This static variable indicates the size of command table, 305 * and it's changeable with prdt number, which ahci_dma_prdt_number 306 * indicates. 307 */ 308 static size_t ahci_cmd_table_size; 309 310 /* 311 * The below global variables are tunable via /etc/system 312 * 313 * ahci_dma_prdt_number 314 * ahci_msi_enabled 315 * ahci_buf_64bit_dma 316 * ahci_commu_64bit_dma 317 */ 318 319 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 320 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 321 322 /* AHCI MSI is tunable */ 323 boolean_t ahci_msi_enabled = B_TRUE; 324 325 /* 326 * 64-bit dma addressing for data buffer is tunable 327 * 328 * The variable controls only the below value: 329 * DBAU (upper 32-bits physical address of data block) 330 */ 331 boolean_t ahci_buf_64bit_dma = B_TRUE; 332 333 /* 334 * 64-bit dma addressing for communication system descriptors is tunable 335 * 336 * The variable controls the below three values: 337 * 338 * PxCLBU (upper 32-bits for the command list base physical address) 339 * PxFBU (upper 32-bits for the received FIS base physical address) 340 * CTBAU (upper 32-bits of command table base) 341 */ 342 boolean_t ahci_commu_64bit_dma = B_TRUE; 343 344 /* 345 * End of global tunable variable definition 346 */ 347 348 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 349 350 #if AHCI_DEBUG 351 /* The following is needed for ahci_log() */ 352 static kmutex_t ahci_log_mutex; 353 static char ahci_log_buf[512]; 354 #endif 355 356 /* Opaque state pointer initialized by ddi_soft_state_init() */ 357 static void *ahci_statep = NULL; 358 359 /* 360 * ahci module initialization. 361 */ 362 int 363 _init(void) 364 { 365 int ret; 366 367 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 368 if (ret != 0) { 369 goto err_out; 370 } 371 372 #if AHCI_DEBUG 373 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 374 #endif 375 376 if ((ret = sata_hba_init(&modlinkage)) != 0) { 377 #if AHCI_DEBUG 378 mutex_destroy(&ahci_log_mutex); 379 #endif 380 ddi_soft_state_fini(&ahci_statep); 381 goto err_out; 382 } 383 384 ret = mod_install(&modlinkage); 385 if (ret != 0) { 386 sata_hba_fini(&modlinkage); 387 #if AHCI_DEBUG 388 mutex_destroy(&ahci_log_mutex); 389 #endif 390 ddi_soft_state_fini(&ahci_statep); 391 goto err_out; 392 } 393 394 /* watchdog tick */ 395 ahci_watchdog_tick = drv_usectohz( 396 (clock_t)ahci_watchdog_timeout * 1000000); 397 return (ret); 398 399 err_out: 400 cmn_err(CE_WARN, "!ahci: Module init failed"); 401 return (ret); 402 } 403 404 /* 405 * ahci module uninitialize. 406 */ 407 int 408 _fini(void) 409 { 410 int ret; 411 412 ret = mod_remove(&modlinkage); 413 if (ret != 0) { 414 return (ret); 415 } 416 417 /* Remove the resources allocated in _init(). */ 418 sata_hba_fini(&modlinkage); 419 #if AHCI_DEBUG 420 mutex_destroy(&ahci_log_mutex); 421 #endif 422 ddi_soft_state_fini(&ahci_statep); 423 424 return (ret); 425 } 426 427 /* 428 * _info entry point 429 */ 430 int 431 _info(struct modinfo *modinfop) 432 { 433 return (mod_info(&modlinkage, modinfop)); 434 } 435 436 /* 437 * The attach entry point for dev_ops. 438 */ 439 static int 440 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 441 { 442 ahci_ctl_t *ahci_ctlp = NULL; 443 int instance = ddi_get_instance(dip); 444 int status; 445 int attach_state; 446 uint32_t cap_status, ahci_version; 447 int intr_types; 448 int i; 449 pci_regspec_t *regs; 450 int regs_length; 451 int rnumber; 452 #if AHCI_DEBUG 453 int speed; 454 #endif 455 456 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 457 NULL); 458 459 switch (cmd) { 460 case DDI_ATTACH: 461 break; 462 463 case DDI_RESUME: 464 465 /* 466 * During DDI_RESUME, the hardware state of the device 467 * (power may have been removed from the device) must be 468 * restored, allow pending requests to continue, and 469 * service new requests. 470 */ 471 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 472 mutex_enter(&ahci_ctlp->ahcictl_mutex); 473 474 /* Restart watch thread */ 475 if (ahci_ctlp->ahcictl_timeout_id == 0) 476 ahci_ctlp->ahcictl_timeout_id = timeout( 477 (void (*)(void *))ahci_watchdog_handler, 478 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 479 480 mutex_exit(&ahci_ctlp->ahcictl_mutex); 481 482 /* 483 * Re-initialize the controller and enable the interrupts and 484 * restart all the ports. 485 * 486 * Note that so far we don't support hot-plug during 487 * suspend/resume. 488 */ 489 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 490 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 491 "Failed to initialize the controller " 492 "during DDI_RESUME", NULL); 493 return (DDI_FAILURE); 494 } 495 496 mutex_enter(&ahci_ctlp->ahcictl_mutex); 497 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 498 mutex_exit(&ahci_ctlp->ahcictl_mutex); 499 500 return (DDI_SUCCESS); 501 502 default: 503 return (DDI_FAILURE); 504 } 505 506 attach_state = AHCI_ATTACH_STATE_NONE; 507 508 /* Allocate soft state */ 509 status = ddi_soft_state_zalloc(ahci_statep, instance); 510 if (status != DDI_SUCCESS) { 511 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 512 instance); 513 goto err_out; 514 } 515 516 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 517 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 518 ahci_ctlp->ahcictl_dip = dip; 519 520 /* Initialize the cport/port mapping */ 521 for (i = 0; i < AHCI_MAX_PORTS; i++) { 522 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 523 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 524 } 525 526 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 527 528 /* 529 * Now map the AHCI base address; which includes global 530 * registers and port control registers 531 * 532 * According to the spec, the AHCI Base Address is BAR5, 533 * but BAR0-BAR4 are optional, so we need to check which 534 * rnumber is used for BAR5. 535 */ 536 537 /* 538 * search through DDI "reg" property for the AHCI register set 539 */ 540 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 541 DDI_PROP_DONTPASS, "reg", (int **)®s, 542 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 543 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 544 instance); 545 goto err_out; 546 } 547 548 /* AHCI Base Address is located at 0x24 offset */ 549 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 550 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 551 == AHCI_PCI_RNUM) 552 break; 553 } 554 555 ddi_prop_free(regs); 556 557 if (rnumber == regs_length) { 558 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 559 instance); 560 goto err_out; 561 } 562 563 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 564 565 status = ddi_regs_map_setup(dip, 566 rnumber, 567 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 568 0, 569 0, 570 &accattr, 571 &ahci_ctlp->ahcictl_ahci_acc_handle); 572 if (status != DDI_SUCCESS) { 573 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 574 instance); 575 goto err_out; 576 } 577 578 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 579 580 /* Get the AHCI version information */ 581 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 582 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 583 584 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 585 (ahci_version & 0xffff0000) >> 16, 586 ((ahci_version & 0x0000ff00) >> 4 | 587 (ahci_version & 0x000000ff))); 588 589 /* We don't support controllers whose versions are lower than 1.0 */ 590 if (!(ahci_version & 0xffff0000)) { 591 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 592 "than version 1.0", instance); 593 goto err_out; 594 } 595 596 /* Get the HBA capabilities information */ 597 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 598 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 599 600 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 601 cap_status); 602 603 #if AHCI_DEBUG 604 /* Get the interface speed supported by the HBA */ 605 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 606 if (speed == 0x01) { 607 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 608 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 609 } else if (speed == 0x10) { 610 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 611 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 612 } else if (speed == 0x11) { 613 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 614 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 615 } 616 #endif 617 618 /* Get the number of command slots supported by the HBA */ 619 ahci_ctlp->ahcictl_num_cmd_slots = 620 ((cap_status & AHCI_HBA_CAP_NCS) >> 621 AHCI_HBA_CAP_NCS_SHIFT) + 1; 622 623 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 624 ahci_ctlp->ahcictl_num_cmd_slots); 625 626 /* Get the bit map which indicates ports implemented by the HBA */ 627 ahci_ctlp->ahcictl_ports_implemented = 628 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 629 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 630 631 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 632 ahci_ctlp->ahcictl_ports_implemented); 633 634 /* 635 * According to the AHCI spec, CAP.NP should indicate the maximum 636 * number of ports supported by the HBA silicon, but we found 637 * this value of ICH8 chipset only indicates the number of ports 638 * implemented (exposed) by it. Therefore, the driver should calculate 639 * the potential maximum value by checking PI register, and use 640 * the maximum of this value and CAP.NP. 641 */ 642 ahci_ctlp->ahcictl_num_ports = max( 643 (cap_status & AHCI_HBA_CAP_NP) + 1, 644 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 645 646 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 647 ahci_ctlp->ahcictl_num_ports); 648 649 /* Get the number of implemented ports by the HBA */ 650 ahci_ctlp->ahcictl_num_implemented_ports = 651 ahci_get_num_implemented_ports( 652 ahci_ctlp->ahcictl_ports_implemented); 653 654 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 655 "hba number of implemented ports: %d", 656 ahci_ctlp->ahcictl_num_implemented_ports); 657 658 /* Check whether HBA supports 64bit DMA addressing */ 659 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 660 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 661 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 662 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 663 "hba does not support 64-bit addressing", NULL); 664 } 665 666 /* Checking for Support Command List Override */ 667 if (cap_status & AHCI_HBA_CAP_SCLO) { 668 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 669 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 670 "hba supports command list override.", NULL); 671 } 672 673 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 674 != DDI_SUCCESS) { 675 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 676 instance); 677 goto err_out; 678 } 679 680 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 681 682 /* 683 * Check the pci configuration space, and set caps. We also 684 * handle the hardware defect in this function. 685 * 686 * For example, force ATI SB600 to use 32-bit dma addressing 687 * since it doesn't support 64-bit dma though its CAP register 688 * declares it support. 689 */ 690 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 691 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 692 instance); 693 goto err_out; 694 } 695 696 /* 697 * Disable the whole controller interrupts before adding 698 * interrupt handlers(s). 699 */ 700 ahci_disable_all_intrs(ahci_ctlp); 701 702 /* Get supported interrupt types */ 703 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 704 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 705 instance); 706 goto err_out; 707 } 708 709 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 710 "ddi_intr_get_supported_types() returned: 0x%x", 711 intr_types); 712 713 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 714 /* 715 * Try MSI first, but fall back to FIXED if failed 716 */ 717 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 718 DDI_SUCCESS) { 719 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 720 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 721 "Using MSI interrupt type", NULL); 722 goto intr_done; 723 } 724 725 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 726 "MSI registration failed, " 727 "trying FIXED interrupts", NULL); 728 } 729 730 if (intr_types & DDI_INTR_TYPE_FIXED) { 731 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 732 DDI_SUCCESS) { 733 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 734 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 735 "Using FIXED interrupt type", NULL); 736 goto intr_done; 737 } 738 739 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 740 "FIXED interrupt registration failed", NULL); 741 } 742 743 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 744 745 goto err_out; 746 747 intr_done: 748 749 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 750 751 /* Initialize the controller mutex */ 752 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 753 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 754 755 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 756 757 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 758 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 759 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 760 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 761 } 762 763 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 764 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 765 sizeof (ahci_prdt_item_t)); 766 767 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 768 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 769 " ahci_cmd_table_size is 0x%x", 770 ahci_dma_prdt_number, ahci_cmd_table_size); 771 772 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 773 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 774 ahci_dma_prdt_number; 775 776 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 777 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 778 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 779 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 780 781 if ((ahci_buf_64bit_dma == B_FALSE) || 782 (ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA)) { 783 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 784 0xffffffffull; 785 } 786 787 if ((ahci_commu_64bit_dma == B_FALSE) || 788 (ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA)) { 789 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 790 0xffffffffull; 791 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 792 0xffffffffull; 793 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 794 0xffffffffull; 795 } 796 797 /* Allocate the ports structure */ 798 status = ahci_alloc_ports_state(ahci_ctlp); 799 if (status != AHCI_SUCCESS) { 800 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 801 instance); 802 goto err_out; 803 } 804 805 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 806 807 /* 808 * Initialize the controller and ports. 809 */ 810 status = ahci_initialize_controller(ahci_ctlp); 811 if (status != AHCI_SUCCESS) { 812 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 813 instance); 814 goto err_out; 815 } 816 817 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 818 819 /* Start one thread to check packet timeouts */ 820 ahci_ctlp->ahcictl_timeout_id = timeout( 821 (void (*)(void *))ahci_watchdog_handler, 822 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 823 824 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 825 826 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 827 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 828 instance); 829 goto err_out; 830 } 831 832 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 833 834 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 835 836 return (DDI_SUCCESS); 837 838 err_out: 839 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 840 mutex_enter(&ahci_ctlp->ahcictl_mutex); 841 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 842 ahci_ctlp->ahcictl_timeout_id = 0; 843 mutex_exit(&ahci_ctlp->ahcictl_mutex); 844 } 845 846 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 847 ahci_uninitialize_controller(ahci_ctlp); 848 } 849 850 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 851 ahci_dealloc_ports_state(ahci_ctlp); 852 } 853 854 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 855 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 856 } 857 858 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 859 ahci_rem_intrs(ahci_ctlp); 860 } 861 862 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 863 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 864 } 865 866 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 867 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 868 } 869 870 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 871 ddi_soft_state_free(ahci_statep, instance); 872 } 873 874 return (DDI_FAILURE); 875 } 876 877 /* 878 * The detach entry point for dev_ops. 879 */ 880 static int 881 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 882 { 883 ahci_ctl_t *ahci_ctlp; 884 int instance; 885 int ret; 886 887 instance = ddi_get_instance(dip); 888 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 889 890 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 891 892 switch (cmd) { 893 case DDI_DETACH: 894 895 /* disable the interrupts for an uninterrupted detach */ 896 mutex_enter(&ahci_ctlp->ahcictl_mutex); 897 ahci_disable_all_intrs(ahci_ctlp); 898 mutex_exit(&ahci_ctlp->ahcictl_mutex); 899 900 /* unregister from the sata framework. */ 901 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 902 if (ret != AHCI_SUCCESS) { 903 mutex_enter(&ahci_ctlp->ahcictl_mutex); 904 ahci_enable_all_intrs(ahci_ctlp); 905 mutex_exit(&ahci_ctlp->ahcictl_mutex); 906 return (DDI_FAILURE); 907 } 908 909 mutex_enter(&ahci_ctlp->ahcictl_mutex); 910 911 /* stop the watchdog handler */ 912 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 913 ahci_ctlp->ahcictl_timeout_id = 0; 914 915 mutex_exit(&ahci_ctlp->ahcictl_mutex); 916 917 /* uninitialize the controller */ 918 ahci_uninitialize_controller(ahci_ctlp); 919 920 /* remove the interrupts */ 921 ahci_rem_intrs(ahci_ctlp); 922 923 /* deallocate the ports structures */ 924 ahci_dealloc_ports_state(ahci_ctlp); 925 926 /* destroy mutex */ 927 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 928 929 /* teardown the pci config */ 930 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 931 932 /* remove the reg maps. */ 933 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 934 935 /* free the soft state. */ 936 ddi_soft_state_free(ahci_statep, instance); 937 938 return (DDI_SUCCESS); 939 940 case DDI_SUSPEND: 941 942 /* 943 * The steps associated with suspension must include putting 944 * the underlying device into a quiescent state so that it 945 * will not generate interrupts or modify or access memory. 946 */ 947 mutex_enter(&ahci_ctlp->ahcictl_mutex); 948 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 949 mutex_exit(&ahci_ctlp->ahcictl_mutex); 950 return (DDI_SUCCESS); 951 } 952 953 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 954 955 /* stop the watchdog handler */ 956 if (ahci_ctlp->ahcictl_timeout_id) { 957 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 958 ahci_ctlp->ahcictl_timeout_id = 0; 959 } 960 961 mutex_exit(&ahci_ctlp->ahcictl_mutex); 962 963 /* 964 * drain the taskq 965 */ 966 ahci_drain_ports_taskq(ahci_ctlp); 967 968 /* 969 * Disable the interrupts and stop all the ports. 970 */ 971 ahci_uninitialize_controller(ahci_ctlp); 972 973 return (DDI_SUCCESS); 974 975 default: 976 return (DDI_FAILURE); 977 } 978 } 979 980 /* 981 * The info entry point for dev_ops. 982 * 983 */ 984 static int 985 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 986 void *arg, void **result) 987 { 988 #ifndef __lock_lint 989 _NOTE(ARGUNUSED(dip)) 990 #endif /* __lock_lint */ 991 992 ahci_ctl_t *ahci_ctlp; 993 int instance; 994 dev_t dev; 995 996 dev = (dev_t)arg; 997 instance = getminor(dev); 998 999 switch (infocmd) { 1000 case DDI_INFO_DEVT2DEVINFO: 1001 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1002 if (ahci_ctlp != NULL) { 1003 *result = ahci_ctlp->ahcictl_dip; 1004 return (DDI_SUCCESS); 1005 } else { 1006 *result = NULL; 1007 return (DDI_FAILURE); 1008 } 1009 case DDI_INFO_DEVT2INSTANCE: 1010 *(int *)result = instance; 1011 break; 1012 default: 1013 break; 1014 } 1015 1016 return (DDI_SUCCESS); 1017 } 1018 1019 /* 1020 * Registers the ahci with sata framework. 1021 */ 1022 static int 1023 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1024 { 1025 struct sata_hba_tran *sata_hba_tran; 1026 1027 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1028 "ahci_register_sata_hba_tran enter", NULL); 1029 1030 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1031 1032 /* Allocate memory for the sata_hba_tran */ 1033 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1034 1035 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 1036 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1037 sata_hba_tran->sata_tran_hba_dma_attr = 1038 &ahci_ctlp->ahcictl_buffer_dma_attr; 1039 1040 /* Report the number of implemented ports */ 1041 sata_hba_tran->sata_tran_hba_num_cports = 1042 ahci_ctlp->ahcictl_num_implemented_ports; 1043 1044 /* Support ATAPI device */ 1045 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1046 1047 /* Get the data transfer capability for PIO command by the HBA */ 1048 if (cap_status & AHCI_HBA_CAP_PMD) { 1049 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1050 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1051 "DRQ block data transfer for PIO command protocol", NULL); 1052 } 1053 1054 /* 1055 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1056 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1057 * QUEUED (EXT), and SERVICE commands). Queued operations are 1058 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1059 * QUEUED commands when the HBA and device support native command 1060 * queuing(NCQ). 1061 * 1062 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1063 * CAP register of the HBA indicates NCQ is supported. 1064 * 1065 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1066 * set because the previous register content of PxCI can be re-written 1067 * in the register write. 1068 */ 1069 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1070 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1071 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1072 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1073 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1074 "Command Queuing", NULL); 1075 } 1076 1077 /* Report the number of command slots */ 1078 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1079 1080 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1081 sata_hba_tran->sata_tran_start = ahci_tran_start; 1082 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1083 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1084 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1085 #ifdef __lock_lint 1086 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1087 #endif 1088 /* 1089 * When SATA framework adds support for pwrmgt the 1090 * pwrmgt_ops needs to be updated 1091 */ 1092 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1093 sata_hba_tran->sata_tran_ioctl = NULL; 1094 1095 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1096 1097 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1098 1099 /* Attach it to SATA framework */ 1100 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1101 != DDI_SUCCESS) { 1102 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1103 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1104 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1105 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1106 return (AHCI_FAILURE); 1107 } 1108 1109 return (AHCI_SUCCESS); 1110 } 1111 1112 /* 1113 * Unregisters the ahci with sata framework. 1114 */ 1115 static int 1116 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1117 { 1118 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1119 "ahci_unregister_sata_hba_tran enter", NULL); 1120 1121 /* Detach from the SATA framework. */ 1122 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1123 DDI_SUCCESS) { 1124 return (AHCI_FAILURE); 1125 } 1126 1127 /* Deallocate sata_hba_tran. */ 1128 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1129 sizeof (sata_hba_tran_t)); 1130 1131 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1132 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1133 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1134 1135 return (AHCI_SUCCESS); 1136 } 1137 1138 /* 1139 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1140 * port status registers and an attached device type via sata_device 1141 * structure. 1142 * 1143 * We return the cached information from a previous hardware probe. The 1144 * actual hardware probing itself was done either from within 1145 * ahci_initialize_controller() during the driver attach or from a phy 1146 * ready change interrupt handler. 1147 */ 1148 static int 1149 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1150 { 1151 ahci_ctl_t *ahci_ctlp; 1152 ahci_port_t *ahci_portp; 1153 uint8_t cport = sd->satadev_addr.cport; 1154 uint8_t pmport = sd->satadev_addr.pmport; 1155 uint8_t qual = sd->satadev_addr.qual; 1156 uint8_t device_type; 1157 uint32_t port_state; 1158 uint8_t port; 1159 1160 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1161 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1162 1163 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1164 "ahci_tran_probe_port enter: cport: %d, " 1165 "pmport: %d, qual: %d", cport, pmport, qual); 1166 1167 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1168 1169 mutex_enter(&ahci_portp->ahciport_mutex); 1170 1171 port_state = ahci_portp->ahciport_port_state; 1172 switch (port_state) { 1173 1174 case SATA_PSTATE_FAILED: 1175 sd->satadev_state = SATA_PSTATE_FAILED; 1176 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1177 "ahci_tran_probe_port: port %d PORT FAILED", port); 1178 goto out; 1179 1180 case SATA_PSTATE_SHUTDOWN: 1181 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1182 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1183 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1184 goto out; 1185 1186 case SATA_PSTATE_PWROFF: 1187 sd->satadev_state = SATA_PSTATE_PWROFF; 1188 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1189 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1190 goto out; 1191 1192 case SATA_PSTATE_PWRON: 1193 sd->satadev_state = SATA_PSTATE_PWRON; 1194 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1195 "ahci_tran_probe_port: port %d PORT PWRON", port); 1196 break; 1197 1198 default: 1199 sd->satadev_state = port_state; 1200 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1201 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1202 port, port_state); 1203 break; 1204 } 1205 1206 device_type = ahci_portp->ahciport_device_type; 1207 1208 switch (device_type) { 1209 1210 case SATA_DTYPE_ATADISK: 1211 sd->satadev_type = SATA_DTYPE_ATADISK; 1212 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1213 "ahci_tran_probe_port: port %d DISK found", port); 1214 break; 1215 1216 case SATA_DTYPE_ATAPI: 1217 /* 1218 * HBA driver only knows it's an ATAPI device, and don't know 1219 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1220 * type need to be determined by checking IDENTIFY PACKET 1221 * DEVICE data 1222 */ 1223 sd->satadev_type = SATA_DTYPE_ATAPI; 1224 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1225 "ahci_tran_probe_port: port %d ATAPI found", port); 1226 break; 1227 1228 case SATA_DTYPE_PMULT: 1229 sd->satadev_type = SATA_DTYPE_PMULT; 1230 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1231 "ahci_tran_probe_port: port %d Port Multiplier found", 1232 port); 1233 break; 1234 1235 case SATA_DTYPE_UNKNOWN: 1236 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1237 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1238 "ahci_tran_probe_port: port %d Unknown device found", port); 1239 break; 1240 1241 default: 1242 /* we don't support any other device types */ 1243 sd->satadev_type = SATA_DTYPE_NONE; 1244 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1245 "ahci_tran_probe_port: port %d No device found", port); 1246 break; 1247 } 1248 1249 out: 1250 ahci_update_sata_registers(ahci_ctlp, port, sd); 1251 mutex_exit(&ahci_portp->ahciport_mutex); 1252 1253 return (SATA_SUCCESS); 1254 } 1255 1256 /* 1257 * There are four operation modes in sata framework: 1258 * SATA_OPMODE_INTERRUPTS 1259 * SATA_OPMODE_POLLING 1260 * SATA_OPMODE_ASYNCH 1261 * SATA_OPMODE_SYNCH 1262 * 1263 * Their combined meanings as following: 1264 * 1265 * SATA_OPMODE_SYNCH 1266 * The command has to be completed before sata_tran_start functions returns. 1267 * Either interrupts or polling could be used - it's up to the driver. 1268 * Mode used currently for internal, sata-module initiated operations. 1269 * 1270 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1271 * It is the same as the one above. 1272 * 1273 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1274 * The command has to be completed before sata_tran_start function returns. 1275 * No interrupt used, polling only. This should be the mode used for scsi 1276 * packets with FLAG_NOINTR. 1277 * 1278 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1279 * The command may be queued (callback function specified). Interrupts could 1280 * be used. It's normal operation mode. 1281 */ 1282 /* 1283 * Called by sata framework to transport a sata packet down stream. 1284 */ 1285 static int 1286 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1287 { 1288 ahci_ctl_t *ahci_ctlp; 1289 ahci_port_t *ahci_portp; 1290 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1291 uint8_t port; 1292 1293 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1294 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1295 1296 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1297 "ahci_tran_start enter: cport %d satapkt 0x%p", 1298 cport, (void *)spkt); 1299 1300 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1301 1302 mutex_enter(&ahci_portp->ahciport_mutex); 1303 1304 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1305 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1306 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1307 /* 1308 * In case the targer driver would send the packet before 1309 * sata framework can have the opportunity to process those 1310 * event reports. 1311 */ 1312 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1313 spkt->satapkt_device.satadev_state = 1314 ahci_portp->ahciport_port_state; 1315 ahci_update_sata_registers(ahci_ctlp, port, 1316 &spkt->satapkt_device); 1317 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1318 "ahci_tran_start returning PORT_ERROR while " 1319 "port in FAILED/SHUTDOWN/PWROFF state: " 1320 "port: %d", port); 1321 mutex_exit(&ahci_portp->ahciport_mutex); 1322 return (SATA_TRAN_PORT_ERROR); 1323 } 1324 1325 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1326 /* 1327 * ahci_intr_phyrdy_change() may have rendered it to 1328 * SATA_DTYPE_NONE. 1329 */ 1330 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1331 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1332 spkt->satapkt_device.satadev_state = 1333 ahci_portp->ahciport_port_state; 1334 ahci_update_sata_registers(ahci_ctlp, port, 1335 &spkt->satapkt_device); 1336 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1337 "ahci_tran_start returning PORT_ERROR while " 1338 "no device attached: port: %d", port); 1339 mutex_exit(&ahci_portp->ahciport_mutex); 1340 return (SATA_TRAN_PORT_ERROR); 1341 } 1342 1343 /* 1344 * SATA HBA driver should remember that a device was reset and it 1345 * is supposed to reject any packets which do not specify either 1346 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1347 * 1348 * This is to prevent a race condition when a device was arbitrarily 1349 * reset by the HBA driver (and lost it's setting) and a target 1350 * driver sending some commands to a device before the sata framework 1351 * has a chance to restore the device setting (such as cache enable/ 1352 * disable or other resettable stuff). 1353 */ 1354 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1355 ahci_portp->ahciport_reset_in_progress = 0; 1356 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1357 "ahci_tran_start clearing the " 1358 "reset_in_progress for port: %d", port); 1359 } 1360 1361 if (ahci_portp->ahciport_reset_in_progress && 1362 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1363 ! ddi_in_panic()) { 1364 spkt->satapkt_reason = SATA_PKT_BUSY; 1365 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1366 "ahci_tran_start returning BUSY while " 1367 "reset in progress: port: %d", port); 1368 mutex_exit(&ahci_portp->ahciport_mutex); 1369 return (SATA_TRAN_BUSY); 1370 } 1371 1372 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1373 spkt->satapkt_reason = SATA_PKT_BUSY; 1374 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1375 "ahci_tran_start returning BUSY while " 1376 "mopping in progress: port: %d", port); 1377 mutex_exit(&ahci_portp->ahciport_mutex); 1378 return (SATA_TRAN_BUSY); 1379 } 1380 1381 if (spkt->satapkt_op_mode & 1382 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1383 /* 1384 * If a SYNC command to be executed in interrupt context, 1385 * bounce it back to sata module. 1386 */ 1387 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1388 servicing_interrupt()) { 1389 spkt->satapkt_reason = SATA_PKT_BUSY; 1390 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1391 "ahci_tran_start returning BUSY while " 1392 "sending SYNC mode under interrupt context: " 1393 "port : %d", port); 1394 mutex_exit(&ahci_portp->ahciport_mutex); 1395 return (SATA_TRAN_BUSY); 1396 } 1397 1398 /* We need to do the sync start now */ 1399 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1400 spkt) == AHCI_FAILURE) { 1401 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1402 "return QUEUE_FULL: port %d", port); 1403 mutex_exit(&ahci_portp->ahciport_mutex); 1404 return (SATA_TRAN_QUEUE_FULL); 1405 } 1406 } else { 1407 /* Async start, using interrupt */ 1408 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1409 == AHCI_FAILURE) { 1410 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1411 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1412 "returning QUEUE_FULL: port %d", port); 1413 mutex_exit(&ahci_portp->ahciport_mutex); 1414 return (SATA_TRAN_QUEUE_FULL); 1415 } 1416 } 1417 1418 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1419 "sata tran accepted: port %d", port); 1420 1421 mutex_exit(&ahci_portp->ahciport_mutex); 1422 return (SATA_TRAN_ACCEPTED); 1423 } 1424 1425 /* 1426 * SATA_OPMODE_SYNCH flag is set 1427 * 1428 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1429 * without interrupt, otherwise we can still use the interrupt. 1430 * 1431 * WARNING!!! ahciport_mutex should be acquired before the function 1432 * is called. 1433 */ 1434 static int 1435 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1436 uint8_t port, sata_pkt_t *spkt) 1437 { 1438 int pkt_timeout_ticks; 1439 uint32_t timeout_tags; 1440 int rval; 1441 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1442 1443 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1444 "port %d spkt 0x%p", port, spkt); 1445 1446 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1447 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1448 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1449 port, spkt)) == AHCI_FAILURE) { 1450 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1451 return (rval); 1452 } 1453 1454 pkt_timeout_ticks = 1455 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1456 1457 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1458 mutex_exit(&ahci_portp->ahciport_mutex); 1459 1460 /* Simulate the interrupt */ 1461 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1462 1463 drv_usecwait(AHCI_1MS_USECS); 1464 1465 mutex_enter(&ahci_portp->ahciport_mutex); 1466 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1467 if (pkt_timeout_ticks < 0) { 1468 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1469 "port %d satapkt 0x%p timed out\n", 1470 instance, port, (void *)spkt); 1471 timeout_tags = (0x1 << rval); 1472 mutex_exit(&ahci_portp->ahciport_mutex); 1473 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1474 port, timeout_tags); 1475 mutex_enter(&ahci_portp->ahciport_mutex); 1476 } 1477 } 1478 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1479 return (AHCI_SUCCESS); 1480 1481 } else { 1482 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1483 port, spkt)) == AHCI_FAILURE) 1484 return (rval); 1485 1486 #if AHCI_DEBUG 1487 /* 1488 * Note that the driver always uses the slot 0 to deliver 1489 * REQUEST SENSE or READ LOG EXT command 1490 */ 1491 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1492 ASSERT(rval == 0); 1493 #endif 1494 1495 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1496 cv_wait(&ahci_portp->ahciport_cv, 1497 &ahci_portp->ahciport_mutex); 1498 1499 return (AHCI_SUCCESS); 1500 } 1501 } 1502 1503 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1504 if (satapkt) { \ 1505 satapkt->satapkt_reason = reason; \ 1506 /* \ 1507 * We set the satapkt_reason in both sync and \ 1508 * non-sync cases. \ 1509 */ \ 1510 } \ 1511 if (satapkt && \ 1512 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1513 satapkt->satapkt_comp) { \ 1514 mutex_exit(&ahci_portp->ahciport_mutex); \ 1515 (*satapkt->satapkt_comp)(satapkt); \ 1516 mutex_enter(&ahci_portp->ahciport_mutex); \ 1517 } else { \ 1518 if (satapkt && \ 1519 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1520 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1521 cv_broadcast(&ahci_portp->ahciport_cv); \ 1522 } 1523 1524 /* 1525 * Searches for and claims a free command slot. 1526 * 1527 * Returns: 1528 * 1529 * AHCI_FAILURE if failed 1530 * 1. if no empty slot left 1531 * 2. non-queued command requested while queued command(s) is outstanding 1532 * 3. queued command requested whild non-queued command(s) is outstanding 1533 * 4. HBA doesn't support multiple-use of command list while already a 1534 * non-queued command is oustanding 1535 * 1536 * claimed slot number if succeeded 1537 * 1538 * NOTE: it will always return slot 0 during error recovery process for 1539 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1540 * 1541 * WARNING!!! ahciport_mutex should be acquired before the function 1542 * is called. 1543 */ 1544 static int 1545 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1546 ahci_port_t *ahci_portp, int command_type) 1547 { 1548 uint32_t free_slots; 1549 int slot; 1550 1551 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1552 "ahciport_pending_tags = 0x%x " 1553 "ahciport_pending_ncq_tags = 0x%x", 1554 ahci_portp->ahciport_pending_tags, 1555 ahci_portp->ahciport_pending_ncq_tags); 1556 1557 /* 1558 * According to the AHCI spec, system software is responsible to 1559 * ensure that queued and non-queued commands are not mixed in 1560 * the command list. 1561 */ 1562 if (command_type == AHCI_NON_NCQ_CMD) { 1563 /* Non-NCQ command request */ 1564 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1565 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1566 "ahci_claim_free_slot: there is still pending " 1567 "queued command(s) in the command list, " 1568 "so no available slot for the non-queued " 1569 "command", NULL); 1570 return (AHCI_FAILURE); 1571 } 1572 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1573 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1574 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1575 "ahci_claim_free_slot: HBA cannot support multiple-" 1576 "use of the command list for non-queued commands", 1577 NULL); 1578 return (AHCI_FAILURE); 1579 } 1580 free_slots = (~ahci_portp->ahciport_pending_tags) & 1581 AHCI_SLOT_MASK(ahci_ctlp); 1582 } else if (command_type == AHCI_NCQ_CMD) { 1583 /* NCQ command request */ 1584 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1585 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1586 "ahci_claim_free_slot: there is still pending " 1587 "non-queued command(s) in the command list, " 1588 "so no available slot for the queued command", 1589 NULL); 1590 return (AHCI_FAILURE); 1591 } 1592 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1593 AHCI_NCQ_SLOT_MASK(ahci_portp); 1594 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1595 /* Error retrieval command request */ 1596 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1597 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1598 "SENSE or READ LOG EXT command", NULL); 1599 slot = 0; 1600 goto out; 1601 } 1602 1603 slot = ddi_ffs(free_slots) - 1; 1604 if (slot == -1) { 1605 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1606 "ahci_claim_free_slot: no empty slots", NULL); 1607 return (AHCI_FAILURE); 1608 } 1609 1610 /* 1611 * According to the AHCI spec, to allow a simple mechanism for the 1612 * HBA to map command list slots to queue entries, software must 1613 * match the tag number it uses to the slot it is placing the command 1614 * in. For example, if a queued command is placed in slot 5, the tag 1615 * for that command must be 5. 1616 */ 1617 if (command_type == AHCI_NCQ_CMD) { 1618 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1619 } 1620 1621 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1622 1623 out: 1624 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1625 "ahci_claim_free_slot: found slot: 0x%x", slot); 1626 1627 return (slot); 1628 } 1629 1630 /* 1631 * Builds the Command Table for the sata packet and delivers it to controller. 1632 * 1633 * Returns: 1634 * slot number if we can obtain a slot successfully 1635 * otherwise, return AHCI_FAILURE 1636 * 1637 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1638 */ 1639 static int 1640 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1641 uint8_t port, sata_pkt_t *spkt) 1642 { 1643 int cmd_slot; 1644 sata_cmd_t *scmd; 1645 ahci_fis_h2d_register_t *h2d_register_fisp; 1646 ahci_cmd_table_t *cmd_table; 1647 ahci_cmd_header_t *cmd_header; 1648 int ncookies; 1649 int i; 1650 int command_type = AHCI_NON_NCQ_CMD; 1651 int ncq_qdepth; 1652 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1653 #if AHCI_DEBUG 1654 uint32_t *ptr; 1655 uint8_t *ptr2; 1656 #endif 1657 1658 spkt->satapkt_reason = SATA_PKT_BUSY; 1659 1660 scmd = &spkt->satapkt_cmd; 1661 1662 /* Check if the command is a NCQ command */ 1663 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1664 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1665 command_type = AHCI_NCQ_CMD; 1666 1667 /* 1668 * When NCQ is support, system software must determine the 1669 * maximum tag allowed by the device and the HBA, and it 1670 * must use a value not beyond of the lower bound of the two. 1671 * 1672 * Sata module is going to calculate the qdepth and send 1673 * down to HBA driver via sata_cmd. 1674 */ 1675 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1676 1677 /* 1678 * At the moment, the driver doesn't support the dynamic 1679 * setting of the maximum ncq depth, and the value can be 1680 * set either during the attach or after hot-plug insertion. 1681 */ 1682 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1683 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1684 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 1685 "ahci_deliver_satapkt: port %d the max tags for " 1686 "NCQ command is %d", port, ncq_qdepth); 1687 } else { 1688 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1689 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1690 " port %d the max tag for NCQ command is " 1691 "requested to change from %d to %d, at the" 1692 " moment the driver doesn't support the " 1693 "dynamic change so it's going to " 1694 "still use the previous tag value", 1695 instance, port, 1696 ahci_portp->ahciport_max_ncq_tags, 1697 ncq_qdepth); 1698 } 1699 } 1700 } 1701 1702 /* Check if the command is an error retrieval command */ 1703 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1704 command_type = AHCI_ERR_RETRI_CMD; 1705 1706 /* Check if there is an empty command slot */ 1707 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1708 if (cmd_slot == AHCI_FAILURE) { 1709 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 1710 return (AHCI_FAILURE); 1711 } 1712 1713 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1714 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1715 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1716 cmd_slot, port, (void *)spkt); 1717 1718 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1719 bzero((void *)cmd_table, ahci_cmd_table_size); 1720 1721 /* For data transfer operations, it is the H2D Register FIS */ 1722 h2d_register_fisp = 1723 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1724 1725 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1726 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1727 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1728 SET_FIS_PMP(h2d_register_fisp, 1729 spkt->satapkt_device.satadev_addr.pmport); 1730 } 1731 1732 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1733 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1734 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1735 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1736 1737 switch (scmd->satacmd_addr_type) { 1738 1739 case 0: 1740 /* 1741 * satacmd_addr_type will be 0 for the commands below: 1742 * ATAPI command 1743 * SATAC_IDLE_IM 1744 * SATAC_STANDBY_IM 1745 * SATAC_DOWNLOAD_MICROCODE 1746 * SATAC_FLUSH_CACHE 1747 * SATAC_SET_FEATURES 1748 * SATAC_SMART 1749 * SATAC_ID_PACKET_DEVICE 1750 * SATAC_ID_DEVICE 1751 */ 1752 /* FALLTHRU */ 1753 1754 case ATA_ADDR_LBA: 1755 /* FALLTHRU */ 1756 1757 case ATA_ADDR_LBA28: 1758 /* LBA[7:0] */ 1759 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1760 1761 /* LBA[15:8] */ 1762 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1763 1764 /* LBA[23:16] */ 1765 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1766 1767 /* LBA [27:24] (also called dev_head) */ 1768 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1769 1770 break; 1771 1772 case ATA_ADDR_LBA48: 1773 /* LBA[7:0] */ 1774 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1775 1776 /* LBA[15:8] */ 1777 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1778 1779 /* LBA[23:16] */ 1780 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1781 1782 /* LBA [31:24] */ 1783 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1784 scmd->satacmd_lba_low_msb); 1785 1786 /* LBA [39:32] */ 1787 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1788 scmd->satacmd_lba_mid_msb); 1789 1790 /* LBA [47:40] */ 1791 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1792 scmd->satacmd_lba_high_msb); 1793 1794 /* Set dev_head */ 1795 SET_FIS_DEV_HEAD(h2d_register_fisp, 1796 scmd->satacmd_device_reg); 1797 1798 /* Set the extended sector count and features */ 1799 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1800 scmd->satacmd_sec_count_msb); 1801 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1802 scmd->satacmd_features_reg_ext); 1803 break; 1804 } 1805 1806 /* 1807 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1808 * filled into features field, and sector count 8:15 is filled into 1809 * features (exp) field. TAG is filled into sector field. 1810 */ 1811 if (command_type == AHCI_NCQ_CMD) { 1812 SET_FIS_FEATURES(h2d_register_fisp, 1813 scmd->satacmd_sec_count_lsb); 1814 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1815 scmd->satacmd_sec_count_msb); 1816 1817 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1818 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1819 } 1820 1821 ncookies = scmd->satacmd_num_dma_cookies; 1822 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 1823 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1824 ncookies, ahci_dma_prdt_number); 1825 1826 ASSERT(ncookies <= ahci_dma_prdt_number); 1827 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1828 1829 /* *** now fill the scatter gather list ******* */ 1830 for (i = 0; i < ncookies; i++) { 1831 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1832 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1833 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1834 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1835 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1836 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1837 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1838 scmd->satacmd_dma_cookie_list[i].dmac_size; 1839 } 1840 1841 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 1842 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1843 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1844 1845 /* The ACMD field is filled in for ATAPI command */ 1846 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1847 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1848 SATA_ATAPI_MAX_CDB_LEN); 1849 } 1850 1851 /* Set Command Header in Command List */ 1852 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1853 BZERO_DESCR_INFO(cmd_header); 1854 BZERO_PRD_BYTE_COUNT(cmd_header); 1855 1856 /* Set the number of entries in the PRD table */ 1857 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1858 1859 /* Set the length of the command in the CFIS area */ 1860 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1861 1862 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1863 "sata_data_direction = 0x%x", 1864 scmd->satacmd_flags.sata_data_direction); 1865 1866 /* Set A bit if it is an ATAPI command */ 1867 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1868 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1869 1870 /* Set W bit if data is going to the device */ 1871 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1872 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1873 1874 /* 1875 * Set the prefetchable bit - this bit is only valid if the PRDTL 1876 * field is non-zero or the ATAPI 'A' bit is set in the command 1877 * header. This bit cannot be set when using native command 1878 * queuing commands or when using FIS-based switching with a Port 1879 * multiplier. 1880 */ 1881 if (command_type != AHCI_NCQ_CMD) 1882 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1883 1884 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1885 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1886 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1887 1888 /* 1889 * We are overloading satapkt_hba_driver_private with 1890 * watched_cycle count. 1891 */ 1892 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1893 1894 #if AHCI_DEBUG 1895 if (ahci_debug_flags & AHCIDBG_ATACMD && 1896 scmd->satacmd_cmd_reg != SATAC_PACKET || 1897 ahci_debug_flags & AHCIDBG_ATAPICMD && 1898 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1899 1900 /* Dump the command header and table */ 1901 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1902 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1903 "0x%p cmd_reg 0x%x port %d", spkt, 1904 scmd->satacmd_cmd_reg, port); 1905 ptr = (uint32_t *)cmd_header; 1906 ahci_log(ahci_ctlp, CE_WARN, 1907 " Command Header:%8x %8x %8x %8x", 1908 ptr[0], ptr[1], ptr[2], ptr[3]); 1909 1910 /* Dump the H2D register FIS */ 1911 ptr = (uint32_t *)h2d_register_fisp; 1912 ahci_log(ahci_ctlp, CE_WARN, 1913 " Command FIS: %8x %8x %8x %8x", 1914 ptr[0], ptr[1], ptr[2], ptr[3]); 1915 1916 /* Dump the ACMD register FIS */ 1917 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1918 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1919 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1920 ahci_log(ahci_ctlp, CE_WARN, 1921 " ATAPI command: %2x %2x %2x %2x " 1922 "%2x %2x %2x %2x", 1923 ptr2[8 * i], ptr2[8 * i + 1], 1924 ptr2[8 * i + 2], ptr2[8 * i + 3], 1925 ptr2[8 * i + 4], ptr2[8 * i + 5], 1926 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1927 1928 /* Dump the PRDT */ 1929 for (i = 0; i < ncookies; i++) { 1930 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1931 ahci_log(ahci_ctlp, CE_WARN, 1932 " Cookie %d: %8x %8x %8x %8x", 1933 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1934 } 1935 } 1936 #endif 1937 1938 (void) ddi_dma_sync( 1939 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1940 0, 1941 ahci_cmd_table_size, 1942 DDI_DMA_SYNC_FORDEV); 1943 1944 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1945 cmd_slot * sizeof (ahci_cmd_header_t), 1946 sizeof (ahci_cmd_header_t), 1947 DDI_DMA_SYNC_FORDEV); 1948 1949 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1950 if (command_type == AHCI_NCQ_CMD) { 1951 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1952 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1953 (0x1 << cmd_slot)); 1954 } 1955 1956 /* Indicate to the HBA that a command is active. */ 1957 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1958 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1959 (0x1 << cmd_slot)); 1960 1961 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1962 "exit: port %d", port); 1963 1964 return (cmd_slot); 1965 } 1966 1967 /* 1968 * Called by the sata framework to abort the previously sent packet(s). 1969 * 1970 * Reset device to abort commands. 1971 */ 1972 static int 1973 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1974 { 1975 ahci_ctl_t *ahci_ctlp; 1976 ahci_port_t *ahci_portp; 1977 uint32_t slot_status = 0; 1978 uint32_t aborted_tags = 0; 1979 uint32_t finished_tags = 0; 1980 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1981 uint8_t port; 1982 int tmp_slot; 1983 int instance = ddi_get_instance(dip); 1984 1985 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1986 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1987 1988 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1989 "ahci_tran_abort enter: port %d", port); 1990 1991 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1992 mutex_enter(&ahci_portp->ahciport_mutex); 1993 1994 /* 1995 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 1996 * commands are being mopped, therefore there is nothing else to do 1997 */ 1998 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1999 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2000 "ahci_tran_abort: port %d is in " 2001 "mopping process, so just return directly ", port); 2002 mutex_exit(&ahci_portp->ahciport_mutex); 2003 return (SATA_SUCCESS); 2004 } 2005 2006 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2007 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2008 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2009 /* 2010 * In case the targer driver would send the request before 2011 * sata framework can have the opportunity to process those 2012 * event reports. 2013 */ 2014 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2015 spkt->satapkt_device.satadev_state = 2016 ahci_portp->ahciport_port_state; 2017 ahci_update_sata_registers(ahci_ctlp, port, 2018 &spkt->satapkt_device); 2019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2020 "ahci_tran_abort returning SATA_FAILURE while " 2021 "port in FAILED/SHUTDOWN/PWROFF state: " 2022 "port: %d", port); 2023 mutex_exit(&ahci_portp->ahciport_mutex); 2024 return (SATA_FAILURE); 2025 } 2026 2027 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2028 /* 2029 * ahci_intr_phyrdy_change() may have rendered it to 2030 * AHCI_PORT_TYPE_NODEV. 2031 */ 2032 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2033 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2034 spkt->satapkt_device.satadev_state = 2035 ahci_portp->ahciport_port_state; 2036 ahci_update_sata_registers(ahci_ctlp, port, 2037 &spkt->satapkt_device); 2038 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2039 "ahci_tran_abort returning SATA_FAILURE while " 2040 "no device attached: port: %d", port); 2041 mutex_exit(&ahci_portp->ahciport_mutex); 2042 return (SATA_FAILURE); 2043 } 2044 2045 if (flag == SATA_ABORT_ALL_PACKETS) { 2046 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2047 aborted_tags = ahci_portp->ahciport_pending_tags; 2048 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2049 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2050 2051 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2052 instance, port); 2053 } else { 2054 aborted_tags = 0xffffffff; 2055 /* 2056 * Aborting one specific packet, first search the 2057 * ahciport_slot_pkts[] list for matching spkt. 2058 */ 2059 for (tmp_slot = 0; 2060 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2061 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2062 aborted_tags = (0x1 << tmp_slot); 2063 break; 2064 } 2065 } 2066 2067 if (aborted_tags == 0xffffffff) { 2068 /* request packet is not on the pending list */ 2069 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2070 "Cannot find the aborting pkt 0x%p on the " 2071 "pending list", (void *)spkt); 2072 ahci_update_sata_registers(ahci_ctlp, port, 2073 &spkt->satapkt_device); 2074 mutex_exit(&ahci_portp->ahciport_mutex); 2075 return (SATA_FAILURE); 2076 } 2077 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2078 instance, port, (void *)spkt); 2079 } 2080 2081 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2082 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2083 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2084 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2085 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2086 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2087 2088 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2089 ahci_portp->ahciport_mop_in_progress++; 2090 2091 /* 2092 * To abort the packet(s), first we are trying to clear PxCMD.ST 2093 * to stop the port, and if the port can be stopped 2094 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2095 * then we just send back the aborted packet(s) with ABORTED flag 2096 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2097 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2098 * perform a COMRESET. 2099 */ 2100 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2101 ahci_portp, port, NULL, NULL); 2102 2103 /* 2104 * Compute which have finished and which need to be retried. 2105 * 2106 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2107 * minus the slot_status. The aborted_tags has to be deducted by 2108 * finished_tags since we can't possibly abort a tag which had finished 2109 * already. 2110 */ 2111 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2112 finished_tags = ahci_portp->ahciport_pending_tags & 2113 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2114 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2115 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2116 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2117 2118 aborted_tags &= ~finished_tags; 2119 2120 ahci_mop_commands(ahci_ctlp, 2121 ahci_portp, 2122 slot_status, 2123 0, /* failed tags */ 2124 0, /* timeout tags */ 2125 aborted_tags, 2126 0); /* reset tags */ 2127 2128 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2129 mutex_exit(&ahci_portp->ahciport_mutex); 2130 2131 return (SATA_SUCCESS); 2132 } 2133 2134 /* 2135 * Used to do device reset and reject all the pending packets on a device 2136 * during the reset operation. 2137 * 2138 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2139 */ 2140 static int 2141 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2142 ahci_port_t *ahci_portp, uint8_t port) 2143 { 2144 uint32_t slot_status = 0; 2145 uint32_t reset_tags = 0; 2146 uint32_t finished_tags = 0; 2147 sata_device_t sdevice; 2148 int ret; 2149 2150 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2151 "ahci_reset_device_reject_pkts on port: %d", port); 2152 2153 /* 2154 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2155 * commands are being mopped, therefore there is nothing else to do 2156 */ 2157 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2158 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2159 "ahci_reset_device_reject_pkts: port %d is in " 2160 "mopping process, so return directly ", port); 2161 return (SATA_SUCCESS); 2162 } 2163 2164 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2165 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2166 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2167 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2168 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2169 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2170 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2171 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2172 } 2173 2174 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2175 != AHCI_SUCCESS) { 2176 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2177 "Try to do a port reset after software " 2178 "reset failed", port); 2179 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2180 if (ret != AHCI_SUCCESS) { 2181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2182 "ahci_reset_device_reject_pkts: port %d " 2183 "failed", port); 2184 return (SATA_FAILURE); 2185 } 2186 } 2187 /* Set the reset in progress flag */ 2188 ahci_portp->ahciport_reset_in_progress = 1; 2189 2190 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2191 ahci_portp->ahciport_mop_in_progress++; 2192 2193 /* Indicate to the framework that a reset has happened */ 2194 bzero((void *)&sdevice, sizeof (sata_device_t)); 2195 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2196 sdevice.satadev_addr.pmport = 0; 2197 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2198 2199 sdevice.satadev_state = SATA_DSTATE_RESET | 2200 SATA_DSTATE_PWR_ACTIVE; 2201 mutex_exit(&ahci_portp->ahciport_mutex); 2202 sata_hba_event_notify( 2203 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2204 &sdevice, 2205 SATA_EVNT_DEVICE_RESET); 2206 mutex_enter(&ahci_portp->ahciport_mutex); 2207 2208 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2209 "port %d sending event up: SATA_EVNT_RESET", port); 2210 2211 /* Next try to mop the pending commands */ 2212 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2213 finished_tags = ahci_portp->ahciport_pending_tags & 2214 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2215 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2216 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2217 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2218 2219 reset_tags &= ~finished_tags; 2220 2221 ahci_mop_commands(ahci_ctlp, 2222 ahci_portp, 2223 slot_status, 2224 0, /* failed tags */ 2225 0, /* timeout tags */ 2226 0, /* aborted tags */ 2227 reset_tags); /* reset tags */ 2228 2229 return (SATA_SUCCESS); 2230 } 2231 2232 /* 2233 * Used to do port reset and reject all the pending packets on a port during 2234 * the reset operation. 2235 * 2236 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2237 */ 2238 static int 2239 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2240 ahci_port_t *ahci_portp, uint8_t port) 2241 { 2242 uint32_t slot_status = 0; 2243 uint32_t reset_tags = 0; 2244 uint32_t finished_tags = 0; 2245 2246 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2247 "ahci_reset_port_reject_pkts on port: %d", port); 2248 2249 /* 2250 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2251 * commands are being mopped, therefore there is nothing else to do 2252 */ 2253 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2254 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2255 "ahci_reset_port_reject_pkts: port %d is in " 2256 "mopping process, so return directly ", port); 2257 return (SATA_SUCCESS); 2258 } 2259 2260 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2261 ahci_portp->ahciport_mop_in_progress++; 2262 2263 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2264 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2265 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2266 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2267 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2268 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2269 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2270 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2271 } 2272 2273 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2274 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2275 NULL) != AHCI_SUCCESS) 2276 return (SATA_FAILURE); 2277 2278 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2279 finished_tags = ahci_portp->ahciport_pending_tags & 2280 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2281 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2282 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2283 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2284 2285 reset_tags &= ~finished_tags; 2286 2287 ahci_mop_commands(ahci_ctlp, 2288 ahci_portp, 2289 slot_status, 2290 0, /* failed tags */ 2291 0, /* timeout tags */ 2292 0, /* aborted tags */ 2293 reset_tags); /* reset tags */ 2294 2295 return (SATA_SUCCESS); 2296 } 2297 2298 /* 2299 * Used to do hba reset and reject all the pending packets on all ports 2300 * during the reset operation. 2301 */ 2302 static int 2303 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2304 { 2305 ahci_port_t *ahci_portp; 2306 uint32_t slot_status[AHCI_MAX_PORTS]; 2307 uint32_t reset_tags[AHCI_MAX_PORTS]; 2308 uint32_t finished_tags[AHCI_MAX_PORTS]; 2309 uint8_t port; 2310 int ret = SATA_SUCCESS; 2311 2312 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2313 "ahci_reset_hba_reject_pkts enter", NULL); 2314 2315 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2316 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2317 continue; 2318 } 2319 2320 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2321 2322 mutex_enter(&ahci_portp->ahciport_mutex); 2323 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2324 slot_status[port] = ddi_get32( 2325 ahci_ctlp->ahcictl_ahci_acc_handle, 2326 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2327 reset_tags[port] = slot_status[port] & 2328 AHCI_SLOT_MASK(ahci_ctlp); 2329 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2330 slot_status[port] = ddi_get32( 2331 ahci_ctlp->ahcictl_ahci_acc_handle, 2332 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2333 reset_tags[port] = slot_status[port] & 2334 AHCI_NCQ_SLOT_MASK(ahci_portp); 2335 } 2336 mutex_exit(&ahci_portp->ahciport_mutex); 2337 } 2338 2339 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2340 ret = SATA_FAILURE; 2341 goto out; 2342 } 2343 2344 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2345 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2346 continue; 2347 } 2348 2349 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2350 2351 mutex_enter(&ahci_portp->ahciport_mutex); 2352 /* 2353 * To prevent recursive enter to ahci_mop_commands, we need 2354 * check AHCI_PORT_FLAG_MOPPING flag. 2355 */ 2356 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2357 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2358 "ahci_reset_hba_reject_pkts: port %d is in " 2359 "mopping process, so return directly ", port); 2360 mutex_exit(&ahci_portp->ahciport_mutex); 2361 continue; 2362 } 2363 2364 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2365 ahci_portp->ahciport_mop_in_progress++; 2366 2367 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2368 finished_tags[port] = 2369 ahci_portp->ahciport_pending_tags & 2370 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2371 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2372 finished_tags[port] = 2373 ahci_portp->ahciport_pending_ncq_tags & 2374 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2375 2376 reset_tags[port] &= ~finished_tags[port]; 2377 2378 ahci_mop_commands(ahci_ctlp, 2379 ahci_portp, 2380 slot_status[port], 2381 0, /* failed tags */ 2382 0, /* timeout tags */ 2383 0, /* aborted tags */ 2384 reset_tags[port]); /* reset tags */ 2385 mutex_exit(&ahci_portp->ahciport_mutex); 2386 } 2387 out: 2388 return (ret); 2389 } 2390 2391 /* 2392 * Called by sata framework to reset a port(s) or device. 2393 */ 2394 static int 2395 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2396 { 2397 ahci_ctl_t *ahci_ctlp; 2398 ahci_port_t *ahci_portp; 2399 uint8_t cport = sd->satadev_addr.cport; 2400 uint8_t port; 2401 int ret = SATA_SUCCESS; 2402 int instance = ddi_get_instance(dip); 2403 2404 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2405 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2406 2407 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2408 "ahci_tran_reset_port enter: cport: %d", cport); 2409 2410 switch (sd->satadev_addr.qual) { 2411 case SATA_ADDR_CPORT: 2412 /* Port reset */ 2413 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2414 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2415 "reset port", instance, port); 2416 2417 mutex_enter(&ahci_portp->ahciport_mutex); 2418 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2419 mutex_exit(&ahci_portp->ahciport_mutex); 2420 2421 break; 2422 2423 case SATA_ADDR_DCPORT: 2424 /* Device reset */ 2425 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2426 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2427 "reset device", instance, port); 2428 2429 mutex_enter(&ahci_portp->ahciport_mutex); 2430 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2431 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2432 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2433 /* 2434 * In case the targer driver would send the request 2435 * before sata framework can have the opportunity to 2436 * process those event reports. 2437 */ 2438 sd->satadev_state = ahci_portp->ahciport_port_state; 2439 ahci_update_sata_registers(ahci_ctlp, port, sd); 2440 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2441 "ahci_tran_reset_dport returning SATA_FAILURE " 2442 "while port in FAILED/SHUTDOWN/PWROFF state: " 2443 "port: %d", port); 2444 mutex_exit(&ahci_portp->ahciport_mutex); 2445 ret = SATA_FAILURE; 2446 break; 2447 } 2448 2449 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2450 /* 2451 * ahci_intr_phyrdy_change() may have rendered it to 2452 * AHCI_PORT_TYPE_NODEV. 2453 */ 2454 sd->satadev_type = SATA_DTYPE_NONE; 2455 sd->satadev_state = ahci_portp->ahciport_port_state; 2456 ahci_update_sata_registers(ahci_ctlp, port, sd); 2457 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2458 "ahci_tran_reset_dport returning SATA_FAILURE " 2459 "while no device attached: port: %d", port); 2460 mutex_exit(&ahci_portp->ahciport_mutex); 2461 ret = SATA_FAILURE; 2462 break; 2463 } 2464 2465 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2466 ahci_portp, port); 2467 mutex_exit(&ahci_portp->ahciport_mutex); 2468 break; 2469 2470 case SATA_ADDR_CNTRL: 2471 /* Reset the whole controller */ 2472 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2473 "reset the whole hba", instance, port); 2474 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2475 break; 2476 2477 case SATA_ADDR_PMPORT: 2478 case SATA_ADDR_DPMPORT: 2479 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2480 "ahci_tran_reset_dport: port multiplier will be " 2481 "supported later", NULL); 2482 /* FALLTHRU */ 2483 default: 2484 ret = SATA_FAILURE; 2485 } 2486 2487 return (ret); 2488 } 2489 2490 /* 2491 * Called by sata framework to activate a port as part of hotplug. 2492 * (cfgadm -c connect satax/y) 2493 * Note: Not port-mult aware. 2494 */ 2495 static int 2496 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2497 { 2498 ahci_ctl_t *ahci_ctlp; 2499 ahci_port_t *ahci_portp; 2500 uint8_t cport = satadev->satadev_addr.cport; 2501 uint8_t port; 2502 int instance = ddi_get_instance(dip); 2503 2504 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2505 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2506 2507 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2508 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2509 2510 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2511 2512 mutex_enter(&ahci_portp->ahciport_mutex); 2513 ahci_enable_port_intrs(ahci_ctlp, port); 2514 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2515 2516 /* 2517 * Reset the port so that the PHY communication would be re-established. 2518 * But this reset is an internal operation and the sata module doesn't 2519 * need to know about it. Moreover, the port with a device attached will 2520 * be started too. 2521 */ 2522 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2523 ahci_portp, port, 2524 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2525 NULL); 2526 2527 /* 2528 * Need to check the link status and device status of the port 2529 * and consider raising power if the port was in D3 state 2530 */ 2531 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2532 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2533 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2534 2535 satadev->satadev_state = ahci_portp->ahciport_port_state; 2536 2537 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2538 2539 mutex_exit(&ahci_portp->ahciport_mutex); 2540 return (SATA_SUCCESS); 2541 } 2542 2543 /* 2544 * Called by sata framework to deactivate a port as part of hotplug. 2545 * (cfgadm -c disconnect satax/y) 2546 * Note: Not port-mult aware. 2547 */ 2548 static int 2549 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2550 { 2551 ahci_ctl_t *ahci_ctlp; 2552 ahci_port_t *ahci_portp; 2553 uint8_t cport = satadev->satadev_addr.cport; 2554 uint8_t port; 2555 uint32_t port_scontrol; 2556 int instance = ddi_get_instance(dip); 2557 2558 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2559 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2560 2561 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2562 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2563 2564 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2565 2566 mutex_enter(&ahci_portp->ahciport_mutex); 2567 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2568 instance, port); 2569 2570 /* Disable the interrupts on the port */ 2571 ahci_disable_port_intrs(ahci_ctlp, port); 2572 2573 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2574 goto phy_offline; 2575 } 2576 2577 /* First to abort all the pending commands */ 2578 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2579 2580 /* Then stop the port */ 2581 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2582 ahci_portp, port); 2583 2584 /* Next put the PHY offline */ 2585 2586 phy_offline: 2587 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2588 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2589 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2590 2591 /* Update ahciport_port_state */ 2592 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2593 satadev->satadev_state = ahci_portp->ahciport_port_state; 2594 2595 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2596 2597 mutex_exit(&ahci_portp->ahciport_mutex); 2598 return (SATA_SUCCESS); 2599 } 2600 2601 /* 2602 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2603 * when a device is unplugged or a port is deactivated. 2604 * 2605 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2606 */ 2607 static void 2608 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2609 ahci_port_t *ahci_portp, uint8_t port) 2610 { 2611 uint32_t slot_status = 0; 2612 uint32_t abort_tags = 0; 2613 2614 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2615 "ahci_reject_all_abort_pkts on port: %d", port); 2616 2617 /* 2618 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2619 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2620 * to get the error data, if yes when the device is removed, the 2621 * command needs to be aborted too. 2622 */ 2623 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2624 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2625 slot_status = 0x1; 2626 abort_tags = 0x1; 2627 goto out; 2628 } else { 2629 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2630 "ahci_reject_all_abort_pkts return directly " 2631 "port %d no needs to reject any outstanding " 2632 "commands", port); 2633 return; 2634 } 2635 } 2636 2637 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2638 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2639 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2640 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2641 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2642 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2643 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2644 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2645 } 2646 2647 out: 2648 /* No need to do mop when there is no outstanding commands */ 2649 if (slot_status != 0) { 2650 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2651 ahci_portp->ahciport_mop_in_progress++; 2652 2653 ahci_mop_commands(ahci_ctlp, 2654 ahci_portp, 2655 slot_status, 2656 0, /* failed tags */ 2657 0, /* timeout tags */ 2658 abort_tags, /* aborting tags */ 2659 0); /* reset tags */ 2660 } 2661 } 2662 2663 #if defined(__lock_lint) 2664 static int 2665 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2666 { 2667 return (SATA_SUCCESS); 2668 } 2669 #endif 2670 2671 /* 2672 * Allocate the ports structure, only called by ahci_attach 2673 */ 2674 static int 2675 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2676 { 2677 int port, cport = 0; 2678 2679 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2680 "ahci_alloc_ports_state enter", NULL); 2681 2682 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2683 2684 /* Allocate structures only for the implemented ports */ 2685 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2686 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2687 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2688 "hba port %d not implemented", port); 2689 continue; 2690 } 2691 2692 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2693 ahci_ctlp->ahcictl_port_to_cport[port] = 2694 (uint8_t)cport++; 2695 2696 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2697 goto err_out; 2698 } 2699 } 2700 2701 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2702 return (AHCI_SUCCESS); 2703 2704 err_out: 2705 for (port--; port >= 0; port--) { 2706 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2707 ahci_dealloc_port_state(ahci_ctlp, port); 2708 } 2709 } 2710 2711 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2712 return (AHCI_FAILURE); 2713 } 2714 2715 /* 2716 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2717 */ 2718 static void 2719 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2720 { 2721 int port; 2722 2723 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2724 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2725 /* if this port is implemented by the HBA */ 2726 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2727 ahci_dealloc_port_state(ahci_ctlp, port); 2728 } 2729 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2730 } 2731 2732 /* 2733 * Drain the taskq. 2734 */ 2735 static void 2736 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 2737 { 2738 ahci_port_t *ahci_portp; 2739 int port; 2740 2741 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2742 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2743 continue; 2744 } 2745 2746 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2747 2748 mutex_enter(&ahci_portp->ahciport_mutex); 2749 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 2750 mutex_exit(&ahci_portp->ahciport_mutex); 2751 } 2752 } 2753 2754 /* 2755 * Initialize the controller and all ports. And then try to start the ports 2756 * if there are devices attached. 2757 * 2758 * This routine can be called from three seperate cases: DDI_ATTACH, 2759 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2760 * other two cases; device signature probing are attempted only during 2761 * DDI_ATTACH case. 2762 * 2763 * WARNING!!! Disable the whole controller's interrupts before calling and 2764 * the interrupts will be enabled upon successfully return. 2765 */ 2766 static int 2767 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2768 { 2769 ahci_port_t *ahci_portp; 2770 uint32_t ghc_control; 2771 int port; 2772 2773 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2774 "ahci_initialize_controller enter", NULL); 2775 2776 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2777 2778 /* 2779 * Indicate that system software is AHCI aware by setting 2780 * GHC.AE to 1 2781 */ 2782 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2783 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2784 2785 ghc_control |= AHCI_HBA_GHC_AE; 2786 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2787 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2788 ghc_control); 2789 2790 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2791 2792 /* Initialize the implemented ports and structures */ 2793 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2794 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2795 continue; 2796 } 2797 2798 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2799 mutex_enter(&ahci_portp->ahciport_mutex); 2800 2801 /* 2802 * Ensure that the controller is not in the running state 2803 * by checking every implemented port's PxCMD register 2804 */ 2805 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2806 != AHCI_SUCCESS) { 2807 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2808 "ahci_initialize_controller: failed to " 2809 "initialize port %d", port); 2810 /* 2811 * Set the port state to SATA_PSTATE_FAILED if 2812 * failed to initialize it. 2813 */ 2814 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2815 } 2816 2817 mutex_exit(&ahci_portp->ahciport_mutex); 2818 } 2819 2820 /* Enable the whole controller interrupts */ 2821 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2822 ahci_enable_all_intrs(ahci_ctlp); 2823 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2824 2825 return (AHCI_SUCCESS); 2826 } 2827 2828 /* 2829 * Reverse of ahci_initialize_controller() 2830 * 2831 * We only need to stop the ports and disable the interrupt. 2832 */ 2833 static void 2834 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2835 { 2836 ahci_port_t *ahci_portp; 2837 int port; 2838 2839 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2840 "ahci_uninitialize_controller enter", NULL); 2841 2842 /* disable all the interrupts. */ 2843 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2844 ahci_disable_all_intrs(ahci_ctlp); 2845 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2846 2847 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2848 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2849 continue; 2850 } 2851 2852 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2853 2854 /* Stop the port by clearing PxCMD.ST */ 2855 mutex_enter(&ahci_portp->ahciport_mutex); 2856 2857 /* 2858 * Here we must disable the port interrupt because 2859 * ahci_disable_all_intrs only clear GHC.IE, and IS 2860 * register will be still set if PxIE is enabled. 2861 * When ahci shares one IRQ with other drivers, the 2862 * intr handler may claim the intr mistakenly. 2863 */ 2864 ahci_disable_port_intrs(ahci_ctlp, port); 2865 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2866 ahci_portp, port); 2867 mutex_exit(&ahci_portp->ahciport_mutex); 2868 } 2869 } 2870 2871 /* 2872 * The routine is to initialize the port. First put the port in NOTRunning 2873 * state, then enable port interrupt and clear Serror register. And under 2874 * AHCI_ATTACH case, find device signature and then try to start the port. 2875 * 2876 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2877 */ 2878 static int 2879 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2880 ahci_port_t *ahci_portp, uint8_t port) 2881 { 2882 uint32_t port_sstatus, port_task_file, port_cmd_status; 2883 int ret; 2884 2885 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2886 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2887 2888 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2889 "ahci_initialize_port: port %d " 2890 "port_cmd_status = 0x%x", port, port_cmd_status); 2891 /* 2892 * Check whether the port is in NotRunning state, if not, 2893 * put the port in NotRunning state 2894 */ 2895 if (port_cmd_status & 2896 (AHCI_CMD_STATUS_ST | 2897 AHCI_CMD_STATUS_CR | 2898 AHCI_CMD_STATUS_FRE | 2899 AHCI_CMD_STATUS_FR)) { 2900 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2901 ahci_portp, port); 2902 } 2903 2904 /* Device is unknown at first */ 2905 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 2906 2907 /* Disable the interface power management */ 2908 ahci_disable_interface_pm(ahci_ctlp, port); 2909 2910 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2911 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 2912 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2913 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 2914 2915 /* Check physcial link status */ 2916 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 2917 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 2918 2919 /* Check interface status */ 2920 port_task_file & AHCI_TFD_STS_BSY || 2921 port_task_file & AHCI_TFD_STS_DRQ) { 2922 2923 /* Incorrect task file state, we need to reset port */ 2924 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2925 2926 /* Does port reset succeed on HBA port? */ 2927 if (ret != AHCI_SUCCESS) { 2928 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 2929 "ahci_initialize_port:" 2930 "port reset faild at port %d", port); 2931 return (AHCI_FAILURE); 2932 } 2933 2934 /* Is port failed? */ 2935 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 2936 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 2937 "ahci_initialize_port: port %d state 0x%x", 2938 port, ahci_portp->ahciport_port_state); 2939 return (AHCI_FAILURE); 2940 } 2941 } 2942 2943 ahci_portp->ahciport_port_state = SATA_STATE_READY; 2944 2945 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 2946 2947 /* 2948 * At the time being, only probe ports/devices and get the types of 2949 * attached devices during DDI_ATTACH. In fact, the device can be 2950 * changed during power state changes, but at the time being, we 2951 * don't support the situation. 2952 */ 2953 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2954 /* 2955 * Try to get the device signature if the port is 2956 * not empty. 2957 */ 2958 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 2959 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2960 } else { 2961 2962 /* 2963 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2964 * and PxFBU registers in case these registers were cleared 2965 * during the suspend. 2966 */ 2967 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 2968 "ahci_initialize_port: port %d " 2969 "reset the port during resume", port); 2970 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2971 2972 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 2973 "ahci_initialize_port: port %d " 2974 "set PxCLB, PxCLBU, PxFB and PxFBU " 2975 "during resume", port); 2976 2977 /* Config Port Received FIS Base Address */ 2978 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2979 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2980 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2981 2982 /* Config Port Command List Base Address */ 2983 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2984 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2985 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2986 } 2987 2988 /* Return directly if no device connected */ 2989 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2990 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2991 "No device connected to port %d", port); 2992 goto out; 2993 } 2994 2995 /* Try to start the port */ 2996 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 2997 != AHCI_SUCCESS) { 2998 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2999 "failed to start port %d", port); 3000 return (AHCI_FAILURE); 3001 } 3002 out: 3003 /* Enable port interrupts */ 3004 ahci_enable_port_intrs(ahci_ctlp, port); 3005 3006 return (AHCI_SUCCESS); 3007 } 3008 3009 /* 3010 * Handle hardware defect, and check the capabilities. For example, 3011 * power management capabilty and MSI capability. 3012 */ 3013 static int 3014 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 3015 { 3016 ushort_t venid, devid; 3017 ushort_t caps_ptr, cap_count, cap; 3018 #if AHCI_DEBUG 3019 ushort_t pmcap, pmcsr; 3020 ushort_t msimc; 3021 #endif 3022 uint8_t revision; 3023 3024 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3025 PCI_CONF_VENID); 3026 3027 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3028 PCI_CONF_DEVID); 3029 3030 /* 3031 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 3032 * controllers with 0x00 revision id work on 4-byte aligned buffer, 3033 * which is a bug and was fixed after 0x00 revision id controllers. 3034 * 3035 * Moreover, VT8251 cannot use multiple command slots in the command 3036 * list for non-queued commands because the previous register content 3037 * of PxCI can be re-written in the register write, so a flag will be 3038 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 3039 */ 3040 if (venid == VIA_VENID) { 3041 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3042 PCI_CONF_REVID); 3043 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3044 "revision id = 0x%x", revision); 3045 if (revision == 0x00) { 3046 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 3047 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3048 "change ddi_attr_align to 0x4", NULL); 3049 } 3050 3051 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 3052 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3053 "VT8251 cannot use multiple command lists for " 3054 "non-queued commands", NULL); 3055 } 3056 3057 /* 3058 * AMD/ATI SB600 (1002,4380) AHCI chipset doesn't support 64-bit DMA 3059 * addressing for both data buffer and communication memory descriptors 3060 * though S64A bit of CAP register declares the support. 3061 */ 3062 if (venid == 0x1002 && devid == 0x4380) { 3063 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3064 "ATI SB600 cannot do 64-bit DMA for both data buffer " 3065 "and communication memory descriptors though CAP indicates " 3066 "support, so force it to use 32-bit DMA", NULL); 3067 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 3068 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3069 } 3070 3071 /* 3072 * AMD/ATI SB700/750 (1002,4391) AHCI chipset doesn't support 64-bit 3073 * DMA addressing for communication memory descriptors though S64A bit 3074 * of CAP register declares the support. However, it does support 3075 * 64-bit DMA for data buffer. 3076 */ 3077 if (venid == 0x1002 && devid == 0x4391) { 3078 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3079 "ATI SB700/750 cannot do 64-bit DMA for communication " 3080 "memory descriptors though CAP indicates support, " 3081 "so force it to use 32-bit DMA", NULL); 3082 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3083 } 3084 3085 /* 3086 * nVidia MCP78 AHCI controller (pci10de,0ad4) will be forced to use 3087 * Fixed interrupt until the known CR 6766472 (MSIs do not function 3088 * on most Nvidia boards) is fixed. 3089 */ 3090 if (venid == 0x10de && devid == 0x0ad4) { 3091 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3092 "Force nVidia MCP78 AHCI controller to use " 3093 "fixed interrupts", NULL); 3094 ahci_msi_enabled = B_FALSE; 3095 } 3096 3097 /* 3098 * Check if capabilities list is supported and if so, 3099 * get initial capabilities pointer and clear bits 0,1. 3100 */ 3101 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3102 PCI_CONF_STAT) & PCI_STAT_CAP) { 3103 caps_ptr = P2ALIGN(pci_config_get8( 3104 ahci_ctlp->ahcictl_pci_conf_handle, 3105 PCI_CONF_CAP_PTR), 4); 3106 } else { 3107 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3108 } 3109 3110 /* 3111 * Walk capabilities if supported. 3112 */ 3113 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3114 3115 /* 3116 * Check that we haven't exceeded the maximum number of 3117 * capabilities and that the pointer is in a valid range. 3118 */ 3119 if (++cap_count > PCI_CAP_MAX_PTR) { 3120 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3121 "too many device capabilities", NULL); 3122 return (AHCI_FAILURE); 3123 } 3124 if (caps_ptr < PCI_CAP_PTR_OFF) { 3125 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3126 "capabilities pointer 0x%x out of range", 3127 caps_ptr); 3128 return (AHCI_FAILURE); 3129 } 3130 3131 /* 3132 * Get next capability and check that it is valid. 3133 * For now, we only support power management. 3134 */ 3135 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3136 caps_ptr); 3137 switch (cap) { 3138 case PCI_CAP_ID_PM: 3139 3140 /* power management supported */ 3141 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3142 3143 /* Save PMCSR offset */ 3144 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3145 3146 #if AHCI_DEBUG 3147 pmcap = pci_config_get16( 3148 ahci_ctlp->ahcictl_pci_conf_handle, 3149 caps_ptr + PCI_PMCAP); 3150 pmcsr = pci_config_get16( 3151 ahci_ctlp->ahcictl_pci_conf_handle, 3152 ahci_ctlp->ahcictl_pmcsr_offset); 3153 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3154 "Power Management capability found PCI_PMCAP " 3155 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3156 if ((pmcap & 0x3) == 0x3) 3157 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3158 "PCI Power Management Interface " 3159 "spec 1.2 compliant", NULL); 3160 #endif 3161 break; 3162 3163 case PCI_CAP_ID_MSI: 3164 #if AHCI_DEBUG 3165 msimc = pci_config_get16( 3166 ahci_ctlp->ahcictl_pci_conf_handle, 3167 caps_ptr + PCI_MSI_CTRL); 3168 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3169 "Message Signaled Interrupt capability found " 3170 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3171 #endif 3172 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3173 "MSI capability found", NULL); 3174 break; 3175 3176 case PCI_CAP_ID_PCIX: 3177 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3178 "PCI-X capability found", NULL); 3179 break; 3180 3181 case PCI_CAP_ID_PCI_E: 3182 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3183 "PCI Express capability found", NULL); 3184 break; 3185 3186 case PCI_CAP_ID_MSI_X: 3187 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3188 "MSI-X capability found", NULL); 3189 break; 3190 3191 case PCI_CAP_ID_SATA: 3192 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3193 "SATA capability found", NULL); 3194 break; 3195 3196 case PCI_CAP_ID_VS: 3197 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3198 "Vendor Specific capability found", NULL); 3199 break; 3200 3201 default: 3202 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3203 "unrecognized capability 0x%x", cap); 3204 break; 3205 } 3206 3207 /* 3208 * Get next capabilities pointer and clear bits 0,1. 3209 */ 3210 caps_ptr = P2ALIGN(pci_config_get8( 3211 ahci_ctlp->ahcictl_pci_conf_handle, 3212 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3213 } 3214 3215 return (AHCI_SUCCESS); 3216 } 3217 3218 /* 3219 * AHCI device reset ...; a single device on one of the ports is reset, 3220 * but the HBA and physical communication remain intact. This is the 3221 * least intrusive. 3222 * 3223 * When issuing a software reset sequence, there should not be other 3224 * commands in the command list, so we will first clear and then re-set 3225 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3226 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3227 * cleared unless command list override (PxCMD.CLO) is supported. 3228 * 3229 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3230 * set before the function is called. 3231 */ 3232 static int 3233 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3234 uint8_t port) 3235 { 3236 ahci_fis_h2d_register_t *h2d_register_fisp; 3237 ahci_cmd_table_t *cmd_table; 3238 ahci_cmd_header_t *cmd_header; 3239 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3240 int slot, loop_count; 3241 int rval = AHCI_FAILURE; 3242 3243 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3244 "Port %d device resetting", port); 3245 3246 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 3247 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 3248 port) != AHCI_SUCCESS) { 3249 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3250 "ahci_software_reset: cannot stop HBA port %d.", port); 3251 goto out; 3252 } 3253 3254 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3255 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3256 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3257 3258 if (port_task_file & AHCI_TFD_STS_BSY || 3259 port_task_file & AHCI_TFD_STS_DRQ) { 3260 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 3261 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3262 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 3263 "cannot issue a software reset.", port_task_file); 3264 goto out; 3265 } 3266 3267 /* 3268 * If HBA Support CLO, as Command List Override (CAP.SCLO is 3269 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 3270 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 3271 */ 3272 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3273 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 3274 3275 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3276 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3277 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3278 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3279 port_cmd_status|AHCI_CMD_STATUS_CLO); 3280 3281 /* Waiting till PxCMD.SCLO bit is cleared */ 3282 loop_count = 0; 3283 do { 3284 /* Wait for 10 millisec */ 3285 drv_usecwait(AHCI_10MS_USECS); 3286 3287 /* We are effectively timing out after 1 sec. */ 3288 if (loop_count++ > 100) { 3289 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3290 "SCLO time out. port %d is busy.", port); 3291 goto out; 3292 } 3293 3294 port_cmd_status = 3295 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3296 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3297 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 3298 3299 /* Re-check */ 3300 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3301 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3302 if (port_task_file & AHCI_TFD_STS_BSY || 3303 port_task_file & AHCI_TFD_STS_DRQ) { 3304 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3305 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 3306 port_task_file); 3307 goto out; 3308 } 3309 } 3310 3311 /* Then start port */ 3312 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 3313 != AHCI_SUCCESS) { 3314 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3315 "ahci_software_reset: cannot start AHCI port %d.", port); 3316 goto out; 3317 } 3318 3319 /* 3320 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 3321 * ahci_port.ahciport_pending_ncq_tags may fail 3322 * ahci_claim_free_slot(). Actually according to spec, by clearing 3323 * PxCMD.ST there is no command outstanding while executing software 3324 * reseting. Hence we directly use slot 0 instead of 3325 * ahci_claim_free_slot(). 3326 */ 3327 slot = 0; 3328 3329 /* Now send the first H2D Register FIS with SRST set to 1 */ 3330 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3331 bzero((void *)cmd_table, ahci_cmd_table_size); 3332 3333 h2d_register_fisp = 3334 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3335 3336 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3337 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3338 3339 /* Set Command Header in Command List */ 3340 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3341 BZERO_DESCR_INFO(cmd_header); 3342 BZERO_PRD_BYTE_COUNT(cmd_header); 3343 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3344 3345 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3346 SET_RESET(cmd_header, 1); 3347 SET_WRITE(cmd_header, 1); 3348 3349 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3350 0, 3351 ahci_cmd_table_size, 3352 DDI_DMA_SYNC_FORDEV); 3353 3354 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3355 slot * sizeof (ahci_cmd_header_t), 3356 sizeof (ahci_cmd_header_t), 3357 DDI_DMA_SYNC_FORDEV); 3358 3359 /* Indicate to the HBA that a command is active. */ 3360 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3361 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3362 (0x1 << slot)); 3363 3364 loop_count = 0; 3365 3366 /* Loop till the first command is finished */ 3367 do { 3368 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3369 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3370 3371 /* We are effectively timing out after 1 sec. */ 3372 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3373 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3374 "the first SRST FIS is timed out, " 3375 "loop_count = %d", loop_count); 3376 goto out; 3377 } 3378 3379 /* Wait for 10 millisec */ 3380 delay(AHCI_10MS_TICKS); 3381 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3382 3383 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 3384 "ahci_software_reset: 1st loop count: %d, " 3385 "port_cmd_issue = 0x%x, slot = 0x%x", 3386 loop_count, port_cmd_issue, slot); 3387 3388 /* According to ATA spec, we need wait at least 5 microsecs here. */ 3389 drv_usecwait(AHCI_1MS_USECS); 3390 3391 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3392 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3393 bzero((void *)cmd_table, ahci_cmd_table_size); 3394 3395 h2d_register_fisp = 3396 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3397 3398 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3399 3400 /* Set Command Header in Command List */ 3401 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3402 BZERO_DESCR_INFO(cmd_header); 3403 BZERO_PRD_BYTE_COUNT(cmd_header); 3404 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3405 3406 SET_WRITE(cmd_header, 1); 3407 3408 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3409 0, 3410 ahci_cmd_table_size, 3411 DDI_DMA_SYNC_FORDEV); 3412 3413 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3414 slot * sizeof (ahci_cmd_header_t), 3415 sizeof (ahci_cmd_header_t), 3416 DDI_DMA_SYNC_FORDEV); 3417 3418 /* Indicate to the HBA that a command is active. */ 3419 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3420 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3421 (0x1 << slot)); 3422 3423 loop_count = 0; 3424 3425 /* Loop till the second command is finished */ 3426 do { 3427 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3428 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3429 3430 /* We are effectively timing out after 1 sec. */ 3431 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3432 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3433 "the second SRST FIS is timed out, " 3434 "loop_count = %d", loop_count); 3435 goto out; 3436 } 3437 3438 /* Wait for 10 millisec */ 3439 delay(AHCI_10MS_TICKS); 3440 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3441 3442 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 3443 "ahci_software_reset: 2nd loop count: %d, " 3444 "port_cmd_issue = 0x%x, slot = 0x%x", 3445 loop_count, port_cmd_issue, slot); 3446 3447 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3448 3449 rval = AHCI_SUCCESS; 3450 out: 3451 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3452 "ahci_software_reset: %s at port %d", 3453 rval == AHCI_SUCCESS ? "succeed" : "failed", 3454 port); 3455 3456 return (rval); 3457 } 3458 3459 /* 3460 * AHCI port reset ...; the physical communication between the HBA and device 3461 * on a port are disabled. This is more intrusive. 3462 * 3463 * When an HBA or port reset occurs, Phy communication is going to 3464 * be re-established with the device through a COMRESET followed by the 3465 * normal out-of-band communication sequence defined in Serial ATA. AT 3466 * the end of reset, the device, if working properly, will send a D2H 3467 * Register FIS, which contains the device signature. When the HBA receives 3468 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3469 * the PxSIG register with the signature. 3470 * 3471 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3472 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3473 * AHCI 1.0 spec. 3474 */ 3475 /* 3476 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3477 * cleared before the function is called. 3478 */ 3479 static int 3480 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3481 { 3482 uint32_t cap_status, port_cmd_status; 3483 uint32_t port_scontrol, port_sstatus, port_serror; 3484 uint32_t port_intr_status, port_task_file; 3485 3486 int loop_count; 3487 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3488 3489 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3490 "Port %d port resetting...", port); 3491 ahci_portp->ahciport_port_state = 0; 3492 3493 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3494 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3495 3496 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3497 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3498 3499 if (cap_status & AHCI_HBA_CAP_SSS) { 3500 /* 3501 * HBA support staggered spin-up, if the port has 3502 * not spin up yet, then force it to do spin-up 3503 */ 3504 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3505 if (!(ahci_portp->ahciport_flags 3506 & AHCI_PORT_FLAG_SPINUP)) { 3507 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3508 "Port %d PxCMD.SUD is zero, force " 3509 "it to do spin-up", port); 3510 ahci_portp->ahciport_flags |= 3511 AHCI_PORT_FLAG_SPINUP; 3512 } 3513 } 3514 } else { 3515 /* 3516 * HBA doesn't support stagger spin-up, force it 3517 * to do normal COMRESET 3518 */ 3519 if (ahci_portp->ahciport_flags & 3520 AHCI_PORT_FLAG_SPINUP) { 3521 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3522 "HBA does not support staggered spin-up " 3523 "force it to do normal COMRESET", NULL); 3524 ahci_portp->ahciport_flags &= 3525 ~AHCI_PORT_FLAG_SPINUP; 3526 } 3527 } 3528 3529 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3530 /* Do normal COMRESET */ 3531 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3532 "ahci_port_reset: do normal COMRESET", port); 3533 3534 /* 3535 * According to the spec, SUD bit should be set here, 3536 * but JMicron JMB363 doesn't follow it, so remove 3537 * the assertion, and just print a debug message. 3538 */ 3539 #if AHCI_DEBUG 3540 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3541 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3542 "port %d SUD bit not set", port) 3543 #endif 3544 3545 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3546 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3547 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3548 3549 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3550 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3551 port_scontrol); 3552 3553 /* Enable PxCMD.FRE to read device */ 3554 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3555 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3556 port_cmd_status|AHCI_CMD_STATUS_FRE); 3557 3558 /* 3559 * Give time for COMRESET to percolate, according to the AHCI 3560 * spec, software shall wait at least 1 millisecond before 3561 * clearing PxSCTL.DET 3562 */ 3563 delay(AHCI_1MS_TICKS*2); 3564 3565 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3566 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3567 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3568 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3569 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3570 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3571 port_scontrol); 3572 } else { 3573 /* Do staggered spin-up */ 3574 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3575 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3576 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3577 3578 /* PxSCTL.DET must be 0 */ 3579 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3580 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3581 port_scontrol); 3582 3583 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3584 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3585 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3586 port_cmd_status); 3587 3588 /* 0 -> 1 edge */ 3589 delay(AHCI_1MS_TICKS*2); 3590 3591 /* Set PxCMD.SUD to 1 */ 3592 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3593 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3594 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3595 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3596 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3597 port_cmd_status); 3598 3599 /* Enable PxCMD.FRE to read device */ 3600 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3601 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3602 port_cmd_status|AHCI_CMD_STATUS_FRE); 3603 } 3604 3605 /* 3606 * The port enters P:StartComm state, and HBA tells link layer to 3607 * start communication, which involves sending COMRESET to device. 3608 * And the HBA resets PxTFD.STS to 7Fh. 3609 * 3610 * When a COMINIT is received from the device, then the port enters 3611 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3612 * PxSSTS.DET to 1h to indicate a device is detected but communication 3613 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3614 * a COMINIT has been received. 3615 */ 3616 /* 3617 * The DET field is valid only if IPM field indicates 3618 * that the interface is in active state. 3619 */ 3620 loop_count = 0; 3621 do { 3622 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3623 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3624 3625 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3626 /* 3627 * If the interface is not active, the DET field 3628 * is considered not accurate. So we want to 3629 * continue looping. 3630 */ 3631 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3632 } 3633 3634 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3635 /* 3636 * We are effectively timing out after 0.1 sec. 3637 */ 3638 break; 3639 } 3640 3641 /* Wait for 10 millisec */ 3642 delay(AHCI_10MS_TICKS); 3643 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3644 3645 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3646 "ahci_port_reset: 1st loop count: %d, " 3647 "port_sstatus = 0x%x port %d", 3648 loop_count, port_sstatus, port); 3649 3650 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3651 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3652 /* 3653 * Either the port is not active or there 3654 * is no device present. 3655 */ 3656 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3657 return (AHCI_SUCCESS); 3658 } 3659 3660 /* Now we can make sure there is a device connected to the port */ 3661 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3662 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3663 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3664 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 3665 3666 /* 3667 * A COMINIT signal is supposed to be received 3668 * PxSERR.DIAG.X or PxIS.PCS should be set 3669 */ 3670 if (!(port_intr_status & AHCI_INTR_STATUS_PCS) && 3671 !(port_serror & SERROR_EXCHANGED_ERR)) { 3672 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3673 "COMINIT signal from the device not received", 3674 instance, port); 3675 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3676 return (AHCI_FAILURE); 3677 } 3678 3679 /* 3680 * According to the spec, when PxSCTL.DET is set to 0h, upon 3681 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3682 * shall be set to '1' by the HBA. 3683 * 3684 * However, we found JMicron JMB363 doesn't follow this, so 3685 * remove this check, and just print a debug message. 3686 */ 3687 #if AHCI_DEBUG 3688 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3689 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3690 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3691 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3692 "port %d BSY bit is not set after COMINIT signal " 3693 "is received", port); 3694 } 3695 #endif 3696 3697 /* 3698 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3699 * the D2H FIS received by HBA. 3700 */ 3701 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3702 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3703 SERROR_EXCHANGED_ERR); 3704 3705 /* 3706 * Devices should return a FIS contains its signature to HBA after 3707 * COMINIT signal. Check whether a D2H FIS is received by polling 3708 * PxTFD.STS.ERR bit. 3709 */ 3710 loop_count = 0; 3711 do { 3712 /* Wait for 10 millisec */ 3713 delay(AHCI_10MS_TICKS); 3714 3715 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3716 /* 3717 * We are effectively timing out after 11 sec. 3718 */ 3719 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3720 "the device hardware has been initialized and " 3721 "the power-up diagnostics failed", 3722 instance, port); 3723 3724 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3725 "port %d PxTFD.STS.ERR is not set, we need another " 3726 "software reset.", port); 3727 3728 /* Clear port serror register for the port */ 3729 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3730 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3731 AHCI_SERROR_CLEAR_ALL); 3732 3733 /* Try another software reset. */ 3734 if (ahci_software_reset(ahci_ctlp, ahci_portp, 3735 port) != AHCI_SUCCESS) { 3736 ahci_portp->ahciport_port_state |= 3737 SATA_PSTATE_FAILED; 3738 return (AHCI_FAILURE); 3739 } 3740 break; 3741 } 3742 3743 /* 3744 * The Error bit '1' means COMRESET is finished successfully 3745 * The device hardware has been initialized and the power-up 3746 * diagnostics successfully completed. The device requests 3747 * that the Transport layer transmit a Register - D2H FIS to 3748 * the host. (SATA spec 11.5, v2.6) 3749 */ 3750 port_task_file = 3751 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3752 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3753 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3754 >> AHCI_TFD_ERR_SHIFT) != AHCI_TFD_ERR_SGS); 3755 3756 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3757 "ahci_port_reset: 2nd loop count: %d, " 3758 "port_task_file = 0x%x port %d", 3759 loop_count, port_task_file, port); 3760 3761 /* Clear port serror register for the port */ 3762 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3763 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3764 AHCI_SERROR_CLEAR_ALL); 3765 3766 /* Set port as ready */ 3767 ahci_portp->ahciport_port_state |= SATA_STATE_READY; 3768 3769 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 3770 "ahci_port_reset: succeed at port %d.", port); 3771 return (AHCI_SUCCESS); 3772 } 3773 3774 /* 3775 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3776 * This is the most intrusive. 3777 * 3778 * When an HBA reset occurs, Phy communication will be re-established with 3779 * the device through a COMRESET followed by the normal out-of-band 3780 * communication sequence defined in Serial ATA. AT the end of reset, the 3781 * device, if working properly, will send a D2H Register FIS, which contains 3782 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3783 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3784 * signature. 3785 * 3786 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3787 */ 3788 static int 3789 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3790 { 3791 ahci_port_t *ahci_portp; 3792 uint32_t ghc_control; 3793 uint8_t port; 3794 int loop_count; 3795 int rval = AHCI_SUCCESS; 3796 3797 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 3798 NULL); 3799 3800 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3801 3802 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3803 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3804 3805 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3806 ghc_control |= AHCI_HBA_GHC_HR; 3807 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3808 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3809 3810 /* 3811 * Wait until HBA Reset complete or timeout 3812 */ 3813 loop_count = 0; 3814 do { 3815 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3816 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3817 3818 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3819 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3820 "ahci hba reset is timing out, " 3821 "ghc_control = 0x%x", ghc_control); 3822 /* We are effectively timing out after 1 sec. */ 3823 break; 3824 } 3825 3826 /* Wait for 10 millisec */ 3827 delay(AHCI_10MS_TICKS); 3828 } while (ghc_control & AHCI_HBA_GHC_HR); 3829 3830 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3831 "ahci_hba_reset: 1st loop count: %d, " 3832 "ghc_control = 0x%x", loop_count, ghc_control); 3833 3834 if (ghc_control & AHCI_HBA_GHC_HR) { 3835 /* The hba is not reset for some reasons */ 3836 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3837 "hba reset failed: HBA in a hung or locked state", NULL); 3838 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3839 return (AHCI_FAILURE); 3840 } 3841 3842 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3843 /* Only check implemented ports */ 3844 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3845 continue; 3846 } 3847 3848 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3849 mutex_enter(&ahci_portp->ahciport_mutex); 3850 3851 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3852 != AHCI_SUCCESS) { 3853 rval = AHCI_FAILURE; 3854 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3855 "ahci_hba_reset: port %d failed", port); 3856 } 3857 3858 mutex_exit(&ahci_portp->ahciport_mutex); 3859 } 3860 3861 /* 3862 * Indicate that system software is AHCI aware by setting 3863 * GHC.AE to 1 3864 */ 3865 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3866 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3867 3868 ghc_control |= AHCI_HBA_GHC_AE; 3869 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3870 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3871 3872 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3873 3874 return (rval); 3875 } 3876 3877 /* 3878 * This routine is only called from AHCI_ATTACH or phyrdy change 3879 * case. It first calls software reset, then stop the port and try to 3880 * read PxSIG register to find the type of device attached to the port. 3881 * 3882 * The caller should make sure a valid device exists on specified port and 3883 * physical communication has been established so that the signature could 3884 * be retrieved by software reset. 3885 * 3886 * WARNING!!! ahciport_mutex should be acquired before the function 3887 * is called. And the port interrupt is disabled. 3888 */ 3889 static void 3890 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3891 ahci_port_t *ahci_portp, uint8_t port) 3892 { 3893 uint32_t signature; 3894 3895 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3896 "ahci_find_dev_signature enter: port %d", port); 3897 3898 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3899 3900 /* Issue a software reset to get the signature */ 3901 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 3902 != AHCI_SUCCESS) { 3903 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3904 "ahci_find_dev_signature: software reset failed " 3905 "at port %d, cannot get signature.", port); 3906 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3907 return; 3908 } 3909 3910 /* 3911 * ahci_software_reset has started the port, so we need manually stop 3912 * the port again. 3913 */ 3914 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, port) 3915 != AHCI_SUCCESS) { 3916 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3917 "ahci_find_dev_signature: cannot stop port %d.", port); 3918 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3919 return; 3920 } 3921 3922 /* Now we can make sure that a valid signature is received. */ 3923 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3924 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3925 3926 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3927 "ahci_find_dev_signature: port %d signature = 0x%x", 3928 port, signature); 3929 3930 switch (signature) { 3931 3932 case AHCI_SIGNATURE_DISK: 3933 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3934 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3935 "Disk is found at port: %d", port); 3936 break; 3937 3938 case AHCI_SIGNATURE_ATAPI: 3939 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3940 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3941 "ATAPI device is found at port: %d", port); 3942 break; 3943 3944 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3945 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3946 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3947 "Port Multiplier is found at port: %d", port); 3948 break; 3949 3950 default: 3951 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3952 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 3953 "Unknown device is found at port: %d", port); 3954 } 3955 } 3956 3957 /* 3958 * According to the spec, to reliably detect hot plug removals, software 3959 * must disable interface power management. Software should perform the 3960 * following initialization on a port after a device is attached: 3961 * Set PxSCTL.IPM to 3h to disable interface state transitions 3962 * Set PxCMD.ALPE to '0' to disable aggressive power management 3963 * Disable device initiated interface power management by SET FEATURE 3964 * 3965 * We can ignore the last item because by default the feature is disabled 3966 */ 3967 static void 3968 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3969 { 3970 uint32_t port_scontrol, port_cmd_status; 3971 3972 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3973 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3974 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3975 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3976 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3977 3978 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3979 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3980 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3981 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3982 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3983 } 3984 3985 /* 3986 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3987 * to 1, then set it firstly. 3988 * 3989 * Each port contains two major DMA engines. One DMA engine walks through 3990 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3991 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3992 * 3993 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3994 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 3995 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 3996 * 3997 * Software shall not set PxCMD.ST to '1' unless a functional device is 3998 * present on the port(as determined by PxTFD.STS.BSY = '0', 3999 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 4000 * 4001 * WARNING!!! ahciport_mutex should be acquired before the function 4002 * is called. 4003 */ 4004 static int 4005 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4006 { 4007 uint32_t port_cmd_status; 4008 4009 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 4010 4011 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4012 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 4013 "the state for port %d is 0x%x", 4014 port, ahci_portp->ahciport_port_state); 4015 return (AHCI_FAILURE); 4016 } 4017 4018 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 4019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 4020 "no device is attached at port %d", port); 4021 return (AHCI_FAILURE); 4022 } 4023 4024 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 4025 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4026 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4027 4028 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 4029 port_cmd_status |= AHCI_CMD_STATUS_FRE; 4030 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4031 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4032 port_cmd_status); 4033 } 4034 4035 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4036 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4037 4038 port_cmd_status |= AHCI_CMD_STATUS_ST; 4039 4040 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4041 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4042 port_cmd_status); 4043 4044 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 4045 4046 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 4047 "PxCMD.ST set to '1' at port %d", port); 4048 4049 return (AHCI_SUCCESS); 4050 } 4051 4052 /* 4053 * Allocate the ahci_port_t including Received FIS and Command List. 4054 * The argument - port is the physical port number, and not logical 4055 * port number seen by the SATA framework. 4056 * 4057 * WARNING!!! ahcictl_mutex should be acquired before the function 4058 * is called. 4059 */ 4060 static int 4061 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 4062 { 4063 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 4064 ahci_port_t *ahci_portp; 4065 char taskq_name[64] = "event_handle_taskq"; 4066 4067 ahci_portp = 4068 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 4069 4070 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 4071 ahci_portp->ahciport_port_num = port; 4072 4073 /* Intialize the port condition variable */ 4074 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 4075 4076 /* Initialize the port mutex */ 4077 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 4078 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 4079 4080 mutex_enter(&ahci_portp->ahciport_mutex); 4081 4082 /* 4083 * Allocate memory for received FIS structure and 4084 * command list for this port 4085 */ 4086 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 4087 goto err_case1; 4088 } 4089 4090 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 4091 goto err_case2; 4092 } 4093 4094 (void) snprintf(taskq_name + strlen(taskq_name), 4095 sizeof (taskq_name) - strlen(taskq_name), 4096 "_port%d", port); 4097 4098 /* Create the taskq for the port */ 4099 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 4100 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 4101 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 4102 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 4103 goto err_case3; 4104 } 4105 4106 /* Allocate the argument for the taskq */ 4107 ahci_portp->ahciport_event_args = 4108 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 4109 4110 if (ahci_portp->ahciport_event_args == NULL) 4111 goto err_case4; 4112 4113 mutex_exit(&ahci_portp->ahciport_mutex); 4114 4115 return (AHCI_SUCCESS); 4116 4117 err_case4: 4118 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 4119 4120 err_case3: 4121 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 4122 4123 err_case2: 4124 ahci_dealloc_rcvd_fis(ahci_portp); 4125 4126 err_case1: 4127 mutex_exit(&ahci_portp->ahciport_mutex); 4128 mutex_destroy(&ahci_portp->ahciport_mutex); 4129 cv_destroy(&ahci_portp->ahciport_cv); 4130 4131 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4132 4133 return (AHCI_FAILURE); 4134 } 4135 4136 /* 4137 * Reverse of ahci_dealloc_port_state(). 4138 * 4139 * WARNING!!! ahcictl_mutex should be acquired before the function 4140 * is called. 4141 */ 4142 static void 4143 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 4144 { 4145 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4146 4147 ASSERT(ahci_portp != NULL); 4148 4149 mutex_enter(&ahci_portp->ahciport_mutex); 4150 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 4151 ahci_portp->ahciport_event_args = NULL; 4152 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 4153 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 4154 ahci_dealloc_rcvd_fis(ahci_portp); 4155 mutex_exit(&ahci_portp->ahciport_mutex); 4156 4157 mutex_destroy(&ahci_portp->ahciport_mutex); 4158 cv_destroy(&ahci_portp->ahciport_cv); 4159 4160 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4161 4162 ahci_ctlp->ahcictl_ports[port] = NULL; 4163 } 4164 4165 /* 4166 * Allocates memory for the Received FIS Structure 4167 * 4168 * WARNING!!! ahciport_mutex should be acquired before the function 4169 * is called. 4170 */ 4171 static int 4172 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4173 uint8_t port) 4174 { 4175 size_t rcvd_fis_size; 4176 size_t ret_len; 4177 uint_t cookie_count; 4178 4179 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 4180 4181 /* allocate rcvd FIS dma handle. */ 4182 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4183 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 4184 DDI_DMA_SLEEP, 4185 NULL, 4186 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 4187 DDI_SUCCESS) { 4188 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4189 "rcvd FIS dma handle alloc failed", NULL); 4190 4191 return (AHCI_FAILURE); 4192 } 4193 4194 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 4195 rcvd_fis_size, 4196 &accattr, 4197 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4198 DDI_DMA_SLEEP, 4199 NULL, 4200 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4201 &ret_len, 4202 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4203 4204 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4205 "rcvd FIS dma mem alloc fail", NULL); 4206 /* error.. free the dma handle. */ 4207 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4208 return (AHCI_FAILURE); 4209 } 4210 4211 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4212 NULL, 4213 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4214 rcvd_fis_size, 4215 DDI_DMA_CONSISTENT, 4216 DDI_DMA_SLEEP, 4217 NULL, 4218 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4219 &cookie_count) != DDI_DMA_MAPPED) { 4220 4221 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4222 "rcvd FIS dma handle bind fail", NULL); 4223 /* error.. free the dma handle & free the memory. */ 4224 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4225 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4226 return (AHCI_FAILURE); 4227 } 4228 4229 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4230 4231 /* Config Port Received FIS Base Address */ 4232 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4233 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4234 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4235 4236 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4237 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4238 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4239 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4240 4241 return (AHCI_SUCCESS); 4242 } 4243 4244 /* 4245 * Deallocates the Received FIS Structure 4246 * 4247 * WARNING!!! ahciport_mutex should be acquired before the function 4248 * is called. 4249 */ 4250 static void 4251 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4252 { 4253 /* Unbind the cmd list dma handle first. */ 4254 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4255 4256 /* Then free the underlying memory. */ 4257 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4258 4259 /* Now free the handle itself. */ 4260 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4261 } 4262 4263 /* 4264 * Allocates memory for the Command List, which contains up to 32 entries. 4265 * Each entry contains a command header, which is a 32-byte structure that 4266 * includes the pointer to the command table. 4267 * 4268 * WARNING!!! ahciport_mutex should be acquired before the function 4269 * is called. 4270 */ 4271 static int 4272 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4273 uint8_t port) 4274 { 4275 size_t cmd_list_size; 4276 size_t ret_len; 4277 uint_t cookie_count; 4278 4279 cmd_list_size = 4280 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4281 4282 /* allocate cmd list dma handle. */ 4283 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4284 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4285 DDI_DMA_SLEEP, 4286 NULL, 4287 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4288 4289 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4290 "cmd list dma handle alloc failed", NULL); 4291 return (AHCI_FAILURE); 4292 } 4293 4294 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4295 cmd_list_size, 4296 &accattr, 4297 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4298 DDI_DMA_SLEEP, 4299 NULL, 4300 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4301 &ret_len, 4302 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4303 4304 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4305 "cmd list dma mem alloc fail", NULL); 4306 /* error.. free the dma handle. */ 4307 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4308 return (AHCI_FAILURE); 4309 } 4310 4311 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4312 NULL, 4313 (caddr_t)ahci_portp->ahciport_cmd_list, 4314 cmd_list_size, 4315 DDI_DMA_CONSISTENT, 4316 DDI_DMA_SLEEP, 4317 NULL, 4318 &ahci_portp->ahciport_cmd_list_dma_cookie, 4319 &cookie_count) != DDI_DMA_MAPPED) { 4320 4321 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4322 "cmd list dma handle bind fail", NULL); 4323 /* error.. free the dma handle & free the memory. */ 4324 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4325 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4326 return (AHCI_FAILURE); 4327 } 4328 4329 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4330 4331 /* Config Port Command List Base Address */ 4332 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4333 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4334 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4335 4336 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4337 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4338 4339 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4340 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4341 4342 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4343 goto err_out; 4344 } 4345 4346 return (AHCI_SUCCESS); 4347 4348 err_out: 4349 /* Unbind the cmd list dma handle first. */ 4350 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4351 4352 /* Then free the underlying memory. */ 4353 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4354 4355 /* Now free the handle itself. */ 4356 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4357 4358 return (AHCI_FAILURE); 4359 } 4360 4361 /* 4362 * Deallocates the Command List 4363 * 4364 * WARNING!!! ahciport_mutex should be acquired before the function 4365 * is called. 4366 */ 4367 static void 4368 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4369 { 4370 /* First dealloc command table */ 4371 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4372 4373 /* Unbind the cmd list dma handle first. */ 4374 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4375 4376 /* Then free the underlying memory. */ 4377 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4378 4379 /* Now free the handle itself. */ 4380 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4381 } 4382 4383 /* 4384 * Allocates memory for all Command Tables, which contains Command FIS, 4385 * ATAPI Command and Physical Region Descriptor Table. 4386 * 4387 * WARNING!!! ahciport_mutex should be acquired before the function 4388 * is called. 4389 */ 4390 static int 4391 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4392 { 4393 size_t ret_len; 4394 ddi_dma_cookie_t cmd_table_dma_cookie; 4395 uint_t cookie_count; 4396 int slot; 4397 4398 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4399 "ahci_alloc_cmd_tables: port %d enter", 4400 ahci_portp->ahciport_port_num); 4401 4402 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4403 /* Allocate cmd table dma handle. */ 4404 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4405 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4406 DDI_DMA_SLEEP, 4407 NULL, 4408 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4409 DDI_SUCCESS) { 4410 4411 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4412 "cmd table dma handle alloc failed", NULL); 4413 4414 goto err_out; 4415 } 4416 4417 if (ddi_dma_mem_alloc( 4418 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4419 ahci_cmd_table_size, 4420 &accattr, 4421 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4422 DDI_DMA_SLEEP, 4423 NULL, 4424 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4425 &ret_len, 4426 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4427 NULL) { 4428 4429 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4430 "cmd table dma mem alloc fail", NULL); 4431 4432 /* error.. free the dma handle. */ 4433 ddi_dma_free_handle( 4434 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4435 goto err_out; 4436 } 4437 4438 if (ddi_dma_addr_bind_handle( 4439 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4440 NULL, 4441 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4442 ahci_cmd_table_size, 4443 DDI_DMA_CONSISTENT, 4444 DDI_DMA_SLEEP, 4445 NULL, 4446 &cmd_table_dma_cookie, 4447 &cookie_count) != DDI_DMA_MAPPED) { 4448 4449 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4450 "cmd table dma handle bind fail", NULL); 4451 /* error.. free the dma handle & free the memory. */ 4452 ddi_dma_mem_free( 4453 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4454 ddi_dma_free_handle( 4455 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4456 goto err_out; 4457 } 4458 4459 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4460 ahci_cmd_table_size); 4461 4462 /* Config Port Command Table Base Address */ 4463 SET_COMMAND_TABLE_BASE_ADDR( 4464 (&ahci_portp->ahciport_cmd_list[slot]), 4465 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4466 4467 #ifndef __lock_lint 4468 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4469 (&ahci_portp->ahciport_cmd_list[slot]), 4470 cmd_table_dma_cookie.dmac_laddress >> 32); 4471 #endif 4472 } 4473 4474 return (AHCI_SUCCESS); 4475 err_out: 4476 4477 for (slot--; slot >= 0; slot--) { 4478 /* Unbind the cmd table dma handle first */ 4479 (void) ddi_dma_unbind_handle( 4480 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4481 4482 /* Then free the underlying memory */ 4483 ddi_dma_mem_free( 4484 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4485 4486 /* Now free the handle itself */ 4487 ddi_dma_free_handle( 4488 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4489 } 4490 4491 return (AHCI_FAILURE); 4492 } 4493 4494 /* 4495 * Deallocates memory for all Command Tables. 4496 * 4497 * WARNING!!! ahciport_mutex should be acquired before the function 4498 * is called. 4499 */ 4500 static void 4501 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4502 { 4503 int slot; 4504 4505 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 4506 "ahci_dealloc_cmd_tables: %d enter", 4507 ahci_portp->ahciport_port_num); 4508 4509 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4510 /* Unbind the cmd table dma handle first. */ 4511 (void) ddi_dma_unbind_handle( 4512 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4513 4514 /* Then free the underlying memory. */ 4515 ddi_dma_mem_free( 4516 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4517 4518 /* Now free the handle itself. */ 4519 ddi_dma_free_handle( 4520 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4521 } 4522 } 4523 4524 /* 4525 * WARNING!!! ahciport_mutex should be acquired before the function 4526 * is called. 4527 */ 4528 static void 4529 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4530 sata_device_t *sd) 4531 { 4532 sd->satadev_scr.sstatus = 4533 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4534 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4535 sd->satadev_scr.serror = 4536 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4537 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4538 sd->satadev_scr.scontrol = 4539 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4540 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4541 sd->satadev_scr.sactive = 4542 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4543 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4544 } 4545 4546 /* 4547 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4548 */ 4549 static void 4550 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4551 { 4552 uint32_t port_intr_status; 4553 uint32_t port_intr_enable; 4554 4555 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4556 "ahci_port_intr enter: port %d", port); 4557 4558 mutex_enter(&ahci_portp->ahciport_mutex); 4559 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4560 /* For SATA_OPMODE_POLLING commands */ 4561 port_intr_enable = 4562 (AHCI_INTR_STATUS_DHRS | 4563 AHCI_INTR_STATUS_PSS | 4564 AHCI_INTR_STATUS_SDBS | 4565 AHCI_INTR_STATUS_UFS | 4566 AHCI_INTR_STATUS_PCS | 4567 AHCI_INTR_STATUS_PRCS | 4568 AHCI_INTR_STATUS_OFS | 4569 AHCI_INTR_STATUS_INFS | 4570 AHCI_INTR_STATUS_IFS | 4571 AHCI_INTR_STATUS_HBDS | 4572 AHCI_INTR_STATUS_HBFS | 4573 AHCI_INTR_STATUS_TFES); 4574 mutex_exit(&ahci_portp->ahciport_mutex); 4575 goto next; 4576 } 4577 mutex_exit(&ahci_portp->ahciport_mutex); 4578 4579 /* 4580 * port_intr_enable indicates that the corresponding interrrupt 4581 * reporting is enabled. 4582 */ 4583 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4584 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4585 next: 4586 /* 4587 * port_intr_stats indicates that the corresponding interrupt 4588 * condition is active. 4589 */ 4590 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4591 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4592 4593 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4594 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4595 "port_intr_enable = 0x%x", 4596 port, port_intr_status, port_intr_enable); 4597 4598 port_intr_status &= port_intr_enable; 4599 4600 /* First clear the port interrupts status */ 4601 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4602 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4603 port_intr_status); 4604 4605 /* Check the completed non-queued commands */ 4606 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4607 AHCI_INTR_STATUS_PSS)) { 4608 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4609 ahci_portp, port); 4610 } 4611 4612 /* Check the completed queued commands */ 4613 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4614 (void) ahci_intr_set_device_bits(ahci_ctlp, 4615 ahci_portp, port); 4616 } 4617 4618 /* Check the port connect change status interrupt bit */ 4619 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4620 (void) ahci_intr_port_connect_change(ahci_ctlp, 4621 ahci_portp, port); 4622 } 4623 4624 /* Check the device mechanical presence status interrupt bit */ 4625 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4626 (void) ahci_intr_device_mechanical_presence_status( 4627 ahci_ctlp, ahci_portp, port); 4628 } 4629 4630 /* Check the PhyRdy change status interrupt bit */ 4631 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4632 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4633 port); 4634 } 4635 4636 /* 4637 * Check the non-fatal error interrupt bits, there are three 4638 * kinds of non-fatal errors at the time being: 4639 * 4640 * PxIS.UFS - Unknown FIS Error 4641 * PxIS.OFS - Overflow Error 4642 * PxIS.INFS - Interface Non-Fatal Error 4643 * 4644 * For these non-fatal errors, the HBA can continue to operate, 4645 * so the driver just log the error messages. 4646 */ 4647 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4648 AHCI_INTR_STATUS_OFS | 4649 AHCI_INTR_STATUS_INFS)) { 4650 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4651 port, port_intr_status); 4652 } 4653 4654 /* 4655 * Check the fatal error interrupt bits, there are four kinds 4656 * of fatal errors for AHCI controllers: 4657 * 4658 * PxIS.HBFS - Host Bus Fatal Error 4659 * PxIS.HBDS - Host Bus Data Error 4660 * PxIS.IFS - Interface Fatal Error 4661 * PxIS.TFES - Task File Error 4662 * 4663 * The fatal error means the HBA can not recover from it by 4664 * itself, and it will try to abort the transfer, and the software 4665 * must intervene to restart the port. 4666 */ 4667 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4668 AHCI_INTR_STATUS_HBDS | 4669 AHCI_INTR_STATUS_HBFS | 4670 AHCI_INTR_STATUS_TFES)) 4671 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4672 port, port_intr_status); 4673 4674 /* Check the cold port detect interrupt bit */ 4675 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4676 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4677 } 4678 4679 /* Second clear the corresponding bit in IS.IPS */ 4680 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4681 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4682 } 4683 4684 /* 4685 * Interrupt service handler 4686 */ 4687 static uint_t 4688 ahci_intr(caddr_t arg1, caddr_t arg2) 4689 { 4690 #ifndef __lock_lint 4691 _NOTE(ARGUNUSED(arg2)) 4692 #endif 4693 /* LINTED */ 4694 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4695 ahci_port_t *ahci_portp; 4696 int32_t global_intr_status; 4697 uint8_t port; 4698 4699 /* 4700 * global_intr_status indicates that the corresponding port has 4701 * an interrupt pending. 4702 */ 4703 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4704 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4705 4706 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4707 /* The interrupt is not ours */ 4708 return (DDI_INTR_UNCLAIMED); 4709 } 4710 4711 /* Loop for all the ports */ 4712 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4713 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4714 continue; 4715 } 4716 if (!((0x1 << port) & global_intr_status)) { 4717 continue; 4718 } 4719 4720 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4721 4722 /* Call ahci_port_intr */ 4723 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4724 } 4725 4726 return (DDI_INTR_CLAIMED); 4727 } 4728 4729 /* 4730 * For non-queued commands, when the corresponding bit in the PxCI register 4731 * is cleared, it means the command is completed successfully. And according 4732 * to the HBA state machine, there are three conditions which possibly will 4733 * try to clear the PxCI register bit. 4734 * 1. Receive one D2H Register FIS which is with 'I' bit set 4735 * 2. Update PIO Setup FIS 4736 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4737 * 4738 * Process completed non-queued commands when the interrupt status bit - 4739 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4740 * 4741 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4742 * with the 'I' bit set. And the following commands will send thus 4743 * FIS with 'I' bit set upon the successful completion: 4744 * 1. Non-data commands 4745 * 2. DMA data-in command 4746 * 3. DMA data-out command 4747 * 4. PIO data-out command 4748 * 5. PACKET non-data commands 4749 * 6. PACKET PIO data-in command 4750 * 7. PACKET PIO data-out command 4751 * 8. PACKET DMA data-in command 4752 * 9. PACKET DMA data-out command 4753 * 4754 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4755 * with the 'I' bit set. And the following commands will send this 4756 * FIS upon the successful completion: 4757 * 1. PIO data-in command 4758 */ 4759 static int 4760 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4761 ahci_port_t *ahci_portp, uint8_t port) 4762 { 4763 uint32_t port_cmd_issue = 0; 4764 uint32_t finished_tags; 4765 int finished_slot; 4766 sata_pkt_t *satapkt; 4767 ahci_fis_d2h_register_t *rcvd_fisp; 4768 #if AHCI_DEBUG 4769 ahci_cmd_header_t *cmd_header; 4770 uint32_t cmd_dmacount; 4771 #endif 4772 4773 mutex_enter(&ahci_portp->ahciport_mutex); 4774 4775 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4776 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4777 /* 4778 * Spurious interrupt. Nothing to be done. 4779 */ 4780 mutex_exit(&ahci_portp->ahciport_mutex); 4781 return (AHCI_SUCCESS); 4782 } 4783 4784 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4785 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4786 4787 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4788 /* Slot 0 is always used during error recovery */ 4789 finished_tags = 0x1 & ~port_cmd_issue; 4790 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4791 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4792 "retrieval is finished, and finished_tags = 0x%x", 4793 port, finished_tags); 4794 } else { 4795 finished_tags = ahci_portp->ahciport_pending_tags & 4796 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4797 } 4798 4799 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4800 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4801 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4802 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4803 finished_tags); 4804 4805 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4806 (finished_tags == 0x1)) { 4807 satapkt = ahci_portp->ahciport_err_retri_pkt; 4808 ASSERT(satapkt != NULL); 4809 4810 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4811 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4812 "with SATA_PKT_COMPLETED", (void *)satapkt); 4813 4814 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4815 goto out; 4816 } 4817 4818 while (finished_tags) { 4819 finished_slot = ddi_ffs(finished_tags) - 1; 4820 if (finished_slot == -1) { 4821 goto out; 4822 } 4823 4824 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4825 ASSERT(satapkt != NULL); 4826 #if AHCI_DEBUG 4827 /* 4828 * For non-native queued commands, the PRD byte count field 4829 * shall contain an accurate count of the number of bytes 4830 * transferred for the command before the PxCI bit is cleared 4831 * to '0' for the command. 4832 * 4833 * The purpose of this field is to let software know how many 4834 * bytes transferred for a given operation in order to 4835 * determine if underflow occurred. When issuing native command 4836 * queuing commands, this field should not be used and is not 4837 * required to be valid since in this case underflow is always 4838 * illegal. 4839 * 4840 * For data reads, the HBA will update its PRD byte count with 4841 * the total number of bytes received from the last FIS, and 4842 * may be able to continue normally. For data writes, the 4843 * device will detect an error, and HBA most likely will get 4844 * a fatal error. 4845 * 4846 * Therefore, here just put code to debug part. And please 4847 * refer to the comment above ahci_intr_fatal_error for the 4848 * definition of underflow error. 4849 */ 4850 cmd_dmacount = 4851 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4852 if (cmd_dmacount) { 4853 cmd_header = 4854 &ahci_portp->ahciport_cmd_list[finished_slot]; 4855 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4856 "ahci_intr_cmd_cmplt: port %d, " 4857 "PRD Byte Count = 0x%x, " 4858 "ahciport_prd_bytecounts = 0x%x", port, 4859 cmd_header->ahcich_prd_byte_count, 4860 cmd_dmacount); 4861 4862 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4863 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 4864 "ahci_intr_cmd_cmplt: port %d, " 4865 "an underflow occurred", port); 4866 } 4867 } 4868 #endif 4869 4870 /* 4871 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4872 * feature, sata_special_regs flag will be set, and the 4873 * driver should copy the status and the other corresponding 4874 * register values in the D2H Register FIS received (It's 4875 * working on Non-data protocol) from the device back to 4876 * the sata_cmd. 4877 * 4878 * For every AHCI port, there is only one Received FIS 4879 * structure, which contains the FISes received from the 4880 * device, So we're trying to copy the content of D2H 4881 * Register FIS in the Received FIS structure back to 4882 * the sata_cmd. 4883 */ 4884 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4885 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4886 ahcirf_d2h_register_fis); 4887 satapkt->satapkt_cmd.satacmd_status_reg = 4888 GET_RFIS_STATUS(rcvd_fisp); 4889 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4890 } 4891 4892 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 4893 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4894 "with SATA_PKT_COMPLETED", (void *)satapkt); 4895 4896 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4897 CLEAR_BIT(finished_tags, finished_slot); 4898 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4899 4900 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4901 } 4902 out: 4903 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 4904 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4905 ahci_portp->ahciport_pending_tags); 4906 4907 mutex_exit(&ahci_portp->ahciport_mutex); 4908 4909 return (AHCI_SUCCESS); 4910 } 4911 4912 /* 4913 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4914 * with the 'I' bit set and has been copied into system memory. It will 4915 * be sent under the following situations: 4916 * 4917 * 1. NCQ command is completed 4918 * 2. Asynchronous notification 4919 * 4920 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4921 * via the Set Device Bits FIS. When such event is generated, the software 4922 * needs to read PxSACT register and compares the current value to the 4923 * list of commands previously issue by software. ahciport_pending_ncq_tags 4924 * keeps the tags of previously issued commands. 4925 * 4926 * Asynchronous Notification is a feature in SATA II, which allows an 4927 * ATAPI device to send a signal to the host when media is inserted or 4928 * removed and avoids polling the device for media changes. The signal 4929 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4930 * set to '1'. At the moment, it's not supported yet. 4931 */ 4932 static int 4933 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4934 ahci_port_t *ahci_portp, uint8_t port) 4935 { 4936 uint32_t port_sactive; 4937 uint32_t port_cmd_issue; 4938 uint32_t issued_tags; 4939 int issued_slot; 4940 uint32_t finished_tags; 4941 int finished_slot; 4942 sata_pkt_t *satapkt; 4943 4944 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4945 "ahci_intr_set_device_bits enter: port %d", port); 4946 4947 mutex_enter(&ahci_portp->ahciport_mutex); 4948 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4949 mutex_exit(&ahci_portp->ahciport_mutex); 4950 return (AHCI_SUCCESS); 4951 } 4952 4953 /* 4954 * First the handler got which commands are finished by checking 4955 * PxSACT register 4956 */ 4957 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4958 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4959 4960 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4961 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4962 4963 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4964 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4965 "port_sactive = 0x%x", port, 4966 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4967 4968 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4969 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4970 4971 /* 4972 * For NCQ commands, the software can determine which command has 4973 * already been transmitted to the device by checking PxCI register. 4974 */ 4975 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4976 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4977 4978 issued_tags = ahci_portp->ahciport_pending_tags & 4979 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4980 4981 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4982 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4983 "port_cmd_issue = 0x%x", port, 4984 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4985 4986 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4987 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4988 4989 /* 4990 * Clear ahciport_pending_tags bit when the corresponding command 4991 * is already sent down to the device. 4992 */ 4993 while (issued_tags) { 4994 issued_slot = ddi_ffs(issued_tags) - 1; 4995 if (issued_slot == -1) { 4996 goto next; 4997 } 4998 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 4999 CLEAR_BIT(issued_tags, issued_slot); 5000 } 5001 5002 next: 5003 while (finished_tags) { 5004 finished_slot = ddi_ffs(finished_tags) - 1; 5005 if (finished_slot == -1) { 5006 goto out; 5007 } 5008 5009 /* The command is certainly transmitted to the device */ 5010 ASSERT(!(ahci_portp->ahciport_pending_tags & 5011 (0x1 << finished_slot))); 5012 5013 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 5014 ASSERT(satapkt != NULL); 5015 5016 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 5017 "ahci_intr_set_device_bits: sending up pkt 0x%p " 5018 "with SATA_PKT_COMPLETED", (void *)satapkt); 5019 5020 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 5021 CLEAR_BIT(finished_tags, finished_slot); 5022 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 5023 5024 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 5025 } 5026 out: 5027 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 5028 "ahci_intr_set_device_bits: port %d " 5029 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 5030 port, ahci_portp->ahciport_pending_ncq_tags, 5031 ahci_portp->ahciport_pending_tags); 5032 5033 mutex_exit(&ahci_portp->ahciport_mutex); 5034 5035 return (AHCI_SUCCESS); 5036 } 5037 5038 /* 5039 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 5040 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 5041 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 5042 * indicates a COMINIT signal was received. 5043 * 5044 * Hot plug insertion is detected by reception of a COMINIT signal from the 5045 * device. On reception of unsolicited COMINIT, the HBA shall generate a 5046 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 5047 * begin the normal communication negotiation sequence as outlined in the 5048 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 5049 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 5050 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 5051 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 5052 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 5053 * to handle hot plug insertion. In this interrupt handler, just do nothing 5054 * but print some log message and clear the bit. 5055 */ 5056 static int 5057 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 5058 ahci_port_t *ahci_portp, uint8_t port) 5059 { 5060 #if AHCI_DEBUG 5061 uint32_t port_serror; 5062 #endif 5063 5064 mutex_enter(&ahci_portp->ahciport_mutex); 5065 5066 #if AHCI_DEBUG 5067 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5068 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5069 5070 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5071 "ahci_intr_port_connect_change: port %d, " 5072 "port_serror = 0x%x", port, port_serror); 5073 #endif 5074 5075 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 5076 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5077 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5078 SERROR_EXCHANGED_ERR); 5079 5080 mutex_exit(&ahci_portp->ahciport_mutex); 5081 5082 return (AHCI_SUCCESS); 5083 } 5084 5085 /* 5086 * Hot Plug Operation for platforms that support Mechanical Presence 5087 * Switches. 5088 * 5089 * When set, it indicates that a mechanical presence switch attached to this 5090 * port has been opened or closed, which may lead to a change in the connection 5091 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 5092 * are set to '1'. 5093 * 5094 * At the moment, this interrupt is not needed and disabled and we just log 5095 * the debug message. 5096 */ 5097 static int 5098 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 5099 ahci_port_t *ahci_portp, uint8_t port) 5100 { 5101 uint32_t cap_status, port_cmd_status; 5102 5103 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5104 "ahci_intr_device_mechanical_presence_status enter, " 5105 "port %d", port); 5106 5107 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5108 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 5109 5110 mutex_enter(&ahci_portp->ahciport_mutex); 5111 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5112 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5113 5114 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 5115 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 5116 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5117 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 5118 "the interrupt: cap_status = 0x%x, " 5119 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 5120 mutex_exit(&ahci_portp->ahciport_mutex); 5121 5122 return (AHCI_SUCCESS); 5123 } 5124 5125 #if AHCI_DEBUG 5126 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 5127 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5128 "The mechanical presence switch is open: " 5129 "port %d, port_cmd_status = 0x%x", 5130 port, port_cmd_status); 5131 } else { 5132 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5133 "The mechanical presence switch is close: " 5134 "port %d, port_cmd_status = 0x%x", 5135 port, port_cmd_status); 5136 } 5137 #endif 5138 5139 mutex_exit(&ahci_portp->ahciport_mutex); 5140 5141 return (AHCI_SUCCESS); 5142 } 5143 5144 /* 5145 * Native Hot Plug Support. 5146 * 5147 * When set, it indicates that the internal PHYRDY signal changed state. 5148 * This bit reflects the state of PxSERR.DIAG.N. 5149 * 5150 * There are three kinds of conditions to generate this interrupt event: 5151 * 1. a device is inserted 5152 * 2. a device is disconnected 5153 * 3. when the link enters/exits a Partial or Slumber interface power 5154 * management state 5155 * 5156 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 5157 * bit may be set due to the link entering the Partial or Slumber power 5158 * management state, rather than due to a hot plug insertion or removal 5159 * event. So far, the interface power management is disabled, so the 5160 * driver can reliably get removal detection notification via the 5161 * PxSERR.DIAG.N bit. 5162 */ 5163 static int 5164 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 5165 ahci_port_t *ahci_portp, uint8_t port) 5166 { 5167 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 5168 sata_device_t sdevice; 5169 int dev_exists_now = 0; 5170 int dev_existed_previously = 0; 5171 5172 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5173 "ahci_intr_phyrdy_change enter, port %d", port); 5174 5175 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 5176 mutex_enter(&ahci_portp->ahciport_mutex); 5177 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5178 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5179 SERROR_PHY_RDY_CHG); 5180 mutex_exit(&ahci_portp->ahciport_mutex); 5181 5182 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5183 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 5184 (ahci_portp == NULL)) { 5185 /* The whole controller setup is not yet done. */ 5186 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5187 return (AHCI_SUCCESS); 5188 } 5189 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5190 5191 mutex_enter(&ahci_portp->ahciport_mutex); 5192 5193 /* SStatus tells the presence of device. */ 5194 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5195 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5196 5197 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5198 dev_exists_now = 1; 5199 } 5200 5201 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5202 dev_existed_previously = 1; 5203 } 5204 5205 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5206 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5207 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5208 "ahci_intr_phyrdy_change: port %d " 5209 "AHCI_PORT_FLAG_NODEV is cleared", port); 5210 if (dev_exists_now == 0) 5211 dev_existed_previously = 1; 5212 } 5213 5214 bzero((void *)&sdevice, sizeof (sata_device_t)); 5215 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5216 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5217 sdevice.satadev_addr.pmport = 0; 5218 sdevice.satadev_state = SATA_PSTATE_PWRON; 5219 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5220 5221 if (dev_exists_now) { 5222 if (dev_existed_previously) { 5223 /* Things are fine now. The loss was temporary. */ 5224 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5225 "ahci_intr_phyrdy_change port %d " 5226 "device link lost/established", port); 5227 5228 mutex_exit(&ahci_portp->ahciport_mutex); 5229 sata_hba_event_notify( 5230 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5231 &sdevice, 5232 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5233 mutex_enter(&ahci_portp->ahciport_mutex); 5234 5235 } else { 5236 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5237 "ahci_intr_phyrdy_change: port %d " 5238 "device link established", port); 5239 5240 /* A new device has been detected. */ 5241 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 5242 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5243 5244 /* Try to start the port */ 5245 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5246 != AHCI_SUCCESS) { 5247 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5248 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5249 "ahci_intr_phyrdy_change: port %d failed " 5250 "at start port", port); 5251 } 5252 5253 /* Clear the max queue depth for inserted device */ 5254 ahci_portp->ahciport_max_ncq_tags = 0; 5255 5256 mutex_exit(&ahci_portp->ahciport_mutex); 5257 sata_hba_event_notify( 5258 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5259 &sdevice, 5260 SATA_EVNT_LINK_ESTABLISHED); 5261 mutex_enter(&ahci_portp->ahciport_mutex); 5262 5263 } 5264 } else { /* No device exists now */ 5265 5266 if (dev_existed_previously) { 5267 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 5268 "ahci_intr_phyrdy_change: port %d " 5269 "device link lost", port); 5270 5271 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5272 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5273 ahci_portp, port); 5274 5275 /* An existing device is lost. */ 5276 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5277 5278 mutex_exit(&ahci_portp->ahciport_mutex); 5279 sata_hba_event_notify( 5280 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5281 &sdevice, 5282 SATA_EVNT_LINK_LOST); 5283 mutex_enter(&ahci_portp->ahciport_mutex); 5284 } 5285 } 5286 5287 mutex_exit(&ahci_portp->ahciport_mutex); 5288 5289 return (AHCI_SUCCESS); 5290 } 5291 5292 /* 5293 * PxIS.UFS - Unknown FIS Error 5294 * 5295 * This interrupt event means an unknown FIS was received and has been 5296 * copied into system memory. An unknown FIS is not considered an illegal 5297 * FIS, unless the length received is more than 64 bytes. If an unknown 5298 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5299 * normal operation. If the unknown FIS is more than 64 bytes, then it 5300 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5301 * a fatal error. 5302 * 5303 * PxIS.OFS - Overflow Error 5304 * 5305 * Command list overflow is defined as software building a command table 5306 * that has fewer total bytes than the transaction given to the device. 5307 * On device writes, the HBA will run out of data, and on reads, there 5308 * will be no room to put the data. 5309 * 5310 * For an overflow on data read, either PIO or DMA, the HBA will set 5311 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5312 * non-fatal error when the HBA can continues. Sometimes, it will cause 5313 * a fatal error and need the software to do something. 5314 * 5315 * For an overflow on data write, setting PxIS.OFS is optional for both 5316 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5317 * software to clean up from this serious error. 5318 * 5319 * PxIS.INFS - Interface Non-Fatal Error 5320 * 5321 * This interrupt event indicates that the HBA encountered an error on 5322 * the Serial ATA interface but was able to continue operation. The kind 5323 * of error usually occurred during a non-Data FIS, and under this condition 5324 * the FIS will be re-transmitted by HBA automatically. 5325 * 5326 * When the FMA is implemented, there should be a stat structure to 5327 * record how many every kind of error happens. 5328 */ 5329 static int 5330 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5331 uint8_t port, uint32_t intr_status) 5332 { 5333 uint32_t port_serror; 5334 #if AHCI_DEBUG 5335 uint32_t port_cmd_status; 5336 uint32_t port_cmd_issue; 5337 uint32_t port_sactive; 5338 int current_slot; 5339 uint32_t current_tags; 5340 sata_pkt_t *satapkt; 5341 ahci_cmd_header_t *cmd_header; 5342 uint32_t cmd_dmacount; 5343 #endif 5344 5345 mutex_enter(&ahci_portp->ahciport_mutex); 5346 5347 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5348 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5349 5350 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5351 "ahci_intr_non_fatal_error: port %d, " 5352 "port_serror = 0x%x", port, port_serror); 5353 5354 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5355 5356 if (intr_status & AHCI_INTR_STATUS_UFS) { 5357 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5358 "ahci port %d has unknown FIS error", port); 5359 5360 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5361 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5362 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5363 SERROR_FIS_TYPE); 5364 } 5365 5366 #if AHCI_DEBUG 5367 if (intr_status & AHCI_INTR_STATUS_OFS) { 5368 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5369 "ahci port %d has overflow error", port); 5370 } 5371 5372 if (intr_status & AHCI_INTR_STATUS_INFS) { 5373 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5374 "ahci port %d has interface non fatal error", port); 5375 } 5376 5377 /* 5378 * Record the error occurred command's slot. 5379 */ 5380 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5381 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5382 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5383 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5384 5385 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5386 AHCI_CMD_STATUS_CCS_SHIFT; 5387 5388 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5389 satapkt = ahci_portp->ahciport_err_retri_pkt; 5390 ASSERT(satapkt != NULL); 5391 ASSERT(current_slot == 0); 5392 } else { 5393 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5394 } 5395 5396 if (satapkt != NULL) { 5397 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5398 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5399 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5400 satapkt->satapkt_cmd.satacmd_cmd_reg); 5401 5402 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5403 "ahci_intr_non_fatal_error: port %d, " 5404 "satapkt 0x%p is being processed when error occurs", 5405 port, (void *)satapkt); 5406 5407 /* 5408 * PRD Byte Count field of command header is not 5409 * required to reflect the total number of bytes 5410 * transferred when an overflow occurs, so here 5411 * just log the value. 5412 */ 5413 cmd_dmacount = 5414 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5415 if (cmd_dmacount) { 5416 cmd_header = &ahci_portp-> 5417 ahciport_cmd_list[current_slot]; 5418 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5419 "ahci_intr_non_fatal_error: port %d, " 5420 "PRD Byte Count = 0x%x, " 5421 "ahciport_prd_bytecounts = 0x%x", port, 5422 cmd_header->ahcich_prd_byte_count, 5423 cmd_dmacount); 5424 } 5425 } 5426 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5427 /* 5428 * For queued command, list those command which have already 5429 * been transmitted to the device and still not completed. 5430 */ 5431 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5432 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5433 5434 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5435 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5436 5437 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5438 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5439 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5440 ahci_portp->ahciport_pending_ncq_tags, 5441 port_sactive, port_cmd_issue); 5442 5443 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5444 port_sactive & ~port_cmd_issue & 5445 AHCI_NCQ_SLOT_MASK(ahci_portp); 5446 5447 while (current_tags) { 5448 current_slot = ddi_ffs(current_tags) - 1; 5449 if (current_slot == -1) { 5450 goto out; 5451 } 5452 5453 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5454 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5455 ahci_ctlp, "ahci_intr_non_fatal_error: " 5456 "port %d, satapkt 0x%p is outstanding when " 5457 "error occurs", port, (void *)satapkt); 5458 5459 CLEAR_BIT(current_tags, current_slot); 5460 } 5461 } 5462 out: 5463 #endif 5464 mutex_exit(&ahci_portp->ahciport_mutex); 5465 5466 return (AHCI_SUCCESS); 5467 } 5468 5469 /* 5470 * According to the AHCI spec, the error types include system memory 5471 * errors, interface errors, port multiplier errors, device errors, 5472 * command list overflow, command list underflow, native command 5473 * queuing tag errors and pio data transfer errors. 5474 * 5475 * System memory errors such as target abort, master abort, and parity 5476 * may cause the host to stop, and they are serious errors and needed 5477 * to be recovered with software intervention. When system software 5478 * has given a pointer to the HBA that doesn't exist in physical memory, 5479 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5480 * data error such as CRC or parity occurs, the HBA aborts the transfer 5481 * (if necessary) and PxIS.HBDS will be set. 5482 * 5483 * Interface errors are errors that occur due to electrical issues on 5484 * the interface, or protocol miscommunication between the device and 5485 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5486 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5487 * causes PxIS.IFS/PxIS.INFS to be set are 5488 * 1. in PxSERR.ERR, P bit is set to '1' 5489 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5490 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5491 * If the error occurred during a non-data FIS, the FIS must be 5492 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5493 * the error occurred during a data FIS, the transfer will stop, so 5494 * the error is fatal and PxIS.IFS is set. 5495 * 5496 * When a FIS arrives that updates the taskfile, the HBA checks to see 5497 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5498 * stops processing any more commands. 5499 * 5500 * Command list overflow is defined as software building a command table 5501 * that has fewer total bytes than the transaction given to the device. 5502 * On device writes, the HBA will run out of data, and on reads, there 5503 * will be no room to put the data. For an overflow on data read, either 5504 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5505 * For an overflow on data write, setting PxIS.OFS is optional for both 5506 * DMA and PIO, and a COMRESET is required by software to clean up from 5507 * this serious error. 5508 * 5509 * Command list underflow is defined as software building a command 5510 * table that has more total bytes than the transaction given to the 5511 * device. For data writes, both PIO and DMA, the device will detect 5512 * an error and end the transfer. And these errors are most likely going 5513 * to be fatal errors that will cause the port to be restarted. For 5514 * data reads, the HBA updates its PRD byte count, and may be 5515 * able to continue normally, but is not required to. And The HBA is 5516 * not required to detect underflow conditions for native command 5517 * queuing command. 5518 * 5519 * The HBA does not actively check incoming DMA Setup FISes to ensure 5520 * that the PxSACT register bit for that slot is set. Existing error 5521 * mechanisms, such as host bus failure, or bad protocol, are used to 5522 * recover from this case. 5523 * 5524 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5525 * DATA FIS must be an integral number of Dwords. If the HBA receives 5526 * a request which is not an integral number of Dwords, the HBA 5527 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5528 * software restarts the port. And the HBA ensures that the size 5529 * of the DATA FIS received during a PIO command matches the size in 5530 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5531 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5532 * stop running until software restarts the port. 5533 */ 5534 /* 5535 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5536 * 5537 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5538 * interface which caused the transfer to stop. 5539 * 5540 * PxIS.HBDS indicates that the hba encountered a data error 5541 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5542 * 5543 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5544 * cannot recover from, such as a bad software pointer. 5545 * 5546 * PxIS.TFES is set whenever the status register is updated by the device 5547 * and the error bit (bit 0) is set. 5548 */ 5549 static int 5550 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5551 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5552 { 5553 uint32_t port_cmd_status; 5554 uint32_t port_serror; 5555 uint32_t task_file_status; 5556 int failed_slot; 5557 sata_pkt_t *spkt = NULL; 5558 uint8_t err_byte; 5559 ahci_event_arg_t *args; 5560 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5561 5562 mutex_enter(&ahci_portp->ahciport_mutex); 5563 5564 /* 5565 * ahci_intr_phyrdy_change() may have rendered it to 5566 * SATA_DTYPE_NONE. 5567 */ 5568 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5569 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5570 "ahci_intr_fatal_error: port %d no device attached, " 5571 "and just return without doing anything", port); 5572 goto out0; 5573 } 5574 5575 if (intr_status & AHCI_INTR_STATUS_TFES) { 5576 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5577 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5578 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5579 "ahci_intr_fatal_error: port %d " 5580 "task_file_status = 0x%x", port, task_file_status); 5581 } 5582 5583 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5584 /* 5585 * Read PxCMD.CCS to determine the slot that the HBA 5586 * was processing when the error occurred. 5587 */ 5588 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5589 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5590 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5591 AHCI_CMD_STATUS_CCS_SHIFT; 5592 5593 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5594 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5595 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5596 "fatal error occurred for port %d", spkt, port); 5597 5598 if (intr_status & AHCI_INTR_STATUS_TFES) { 5599 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5600 >> AHCI_TFD_ERR_SHIFT; 5601 5602 /* 5603 * Won't emit the error message if it is an IDENTIFY 5604 * DEVICE command sent to an ATAPI device. 5605 */ 5606 if ((spkt != NULL) && 5607 (spkt->satapkt_cmd.satacmd_cmd_reg == 5608 SATAC_ID_DEVICE) && 5609 (err_byte == SATA_ERROR_ABORT)) 5610 goto out1; 5611 5612 /* 5613 * Won't emit the error message if it is an ATAPI PACKET 5614 * command 5615 */ 5616 if ((spkt != NULL) && 5617 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5618 goto out1; 5619 } 5620 } 5621 5622 /* print the fatal error type */ 5623 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5624 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5625 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5626 5627 /* print PxSERR related error message */ 5628 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5629 5630 /* print task file register value */ 5631 if (intr_status & AHCI_INTR_STATUS_TFES) { 5632 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 5633 "= 0x%x", instance, port, task_file_status); 5634 } 5635 5636 out1: 5637 /* Prepare the argument for the taskq */ 5638 args = ahci_portp->ahciport_event_args; 5639 args->ahciea_ctlp = (void *)ahci_ctlp; 5640 args->ahciea_portp = (void *)ahci_portp; 5641 args->ahciea_event = intr_status; 5642 5643 /* Start the taskq to handle error recovery */ 5644 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 5645 ahci_events_handler, 5646 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5647 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5648 "failed", instance); 5649 } 5650 out0: 5651 mutex_exit(&ahci_portp->ahciport_mutex); 5652 5653 return (AHCI_SUCCESS); 5654 } 5655 5656 /* 5657 * Hot Plug Operation for platforms that support Cold Presence Detect. 5658 * 5659 * When set, a device status has changed as detected by the cold presence 5660 * detect logic. This bit can either be set due to a non-connected port 5661 * receiving a device, or a connected port having its device removed. 5662 * This bit is only valid if the port supports cold presence detect as 5663 * indicated by PxCMD.CPD set to '1'. 5664 * 5665 * At the moment, this interrupt is not needed and disabled and we just 5666 * log the debug message. 5667 */ 5668 static int 5669 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5670 ahci_port_t *ahci_portp, uint8_t port) 5671 { 5672 uint32_t port_cmd_status; 5673 sata_device_t sdevice; 5674 5675 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5676 "ahci_intr_cold_port_detect enter, port %d", port); 5677 5678 mutex_enter(&ahci_portp->ahciport_mutex); 5679 5680 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5681 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5682 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5683 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5684 "port %d does not support cold presence detect, so " 5685 "we just ignore this interrupt", port); 5686 mutex_exit(&ahci_portp->ahciport_mutex); 5687 return (AHCI_SUCCESS); 5688 } 5689 5690 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5691 "port %d device status has changed", port); 5692 5693 bzero((void *)&sdevice, sizeof (sata_device_t)); 5694 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5695 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5696 sdevice.satadev_addr.pmport = 0; 5697 sdevice.satadev_state = SATA_PSTATE_PWRON; 5698 5699 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5700 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5701 "port %d: a device is hot plugged", port); 5702 mutex_exit(&ahci_portp->ahciport_mutex); 5703 sata_hba_event_notify( 5704 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5705 &sdevice, 5706 SATA_EVNT_DEVICE_ATTACHED); 5707 mutex_enter(&ahci_portp->ahciport_mutex); 5708 5709 } else { 5710 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5711 "port %d: a device is hot unplugged", port); 5712 mutex_exit(&ahci_portp->ahciport_mutex); 5713 sata_hba_event_notify( 5714 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5715 &sdevice, 5716 SATA_EVNT_DEVICE_DETACHED); 5717 mutex_enter(&ahci_portp->ahciport_mutex); 5718 } 5719 5720 mutex_exit(&ahci_portp->ahciport_mutex); 5721 5722 return (AHCI_SUCCESS); 5723 } 5724 5725 /* 5726 * Enable the interrupts for a particular port. 5727 * 5728 * WARNING!!! ahciport_mutex should be acquired before the function 5729 * is called. 5730 */ 5731 static void 5732 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5733 { 5734 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5735 "ahci_enable_port_intrs enter, port %d", port); 5736 5737 /* 5738 * Clear port interrupt status before enabling interrupt 5739 */ 5740 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5741 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5742 AHCI_PORT_INTR_MASK); 5743 5744 /* 5745 * Clear the pending bit from IS.IPS 5746 */ 5747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5748 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5749 5750 /* 5751 * Enable the following interrupts: 5752 * Device to Host Register FIS Interrupt (DHRS) 5753 * PIO Setup FIS Interrupt (PSS) 5754 * Set Device Bits Interrupt (SDBS) 5755 * Unknown FIS Interrupt (UFS) 5756 * Port Connect Change Status (PCS) 5757 * PhyRdy Change Status (PRCS) 5758 * Overflow Status (OFS) 5759 * Interface Non-fatal Error Status (INFS) 5760 * Interface Fatal Error Status (IFS) 5761 * Host Bus Data Error Status (HBDS) 5762 * Host Bus Fatal Error Status (HBFS) 5763 * Task File Error Status (TFES) 5764 */ 5765 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5766 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5767 (AHCI_INTR_STATUS_DHRS | 5768 AHCI_INTR_STATUS_PSS | 5769 AHCI_INTR_STATUS_SDBS | 5770 AHCI_INTR_STATUS_UFS | 5771 AHCI_INTR_STATUS_DPS | 5772 AHCI_INTR_STATUS_PCS | 5773 AHCI_INTR_STATUS_PRCS | 5774 AHCI_INTR_STATUS_OFS | 5775 AHCI_INTR_STATUS_INFS | 5776 AHCI_INTR_STATUS_IFS | 5777 AHCI_INTR_STATUS_HBDS | 5778 AHCI_INTR_STATUS_HBFS | 5779 AHCI_INTR_STATUS_TFES)); 5780 } 5781 5782 /* 5783 * Enable interrupts for all the ports. 5784 * 5785 * WARNING!!! ahcictl_mutex should be acquired before the function 5786 * is called. 5787 */ 5788 static void 5789 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5790 { 5791 uint32_t ghc_control; 5792 5793 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 5794 5795 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5796 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5797 5798 ghc_control |= AHCI_HBA_GHC_IE; 5799 5800 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5801 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5802 } 5803 5804 /* 5805 * Disable interrupts for a particular port. 5806 * 5807 * WARNING!!! ahciport_mutex should be acquired before the function 5808 * is called. 5809 */ 5810 static void 5811 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5812 { 5813 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5814 "ahci_disable_port_intrs enter, port %d", port); 5815 5816 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5817 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5818 } 5819 5820 /* 5821 * Disable interrupts for the whole HBA. 5822 * 5823 * The global bit is cleared, then all interrupt sources from all 5824 * ports are disabled. 5825 * 5826 * WARNING!!! ahcictl_mutex should be acquired before the function 5827 * is called. 5828 */ 5829 static void 5830 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5831 { 5832 uint32_t ghc_control; 5833 5834 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 5835 NULL); 5836 5837 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5838 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5839 5840 ghc_control &= ~ AHCI_HBA_GHC_IE; 5841 5842 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5843 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5844 } 5845 5846 /* 5847 * Handle FIXED or MSI interrupts. 5848 */ 5849 /* 5850 * According to AHCI spec, the HBA may support several interrupt modes: 5851 * * pin based interrupts (FIXED) 5852 * * single MSI message interrupts 5853 * * multiple MSI based message interrupts 5854 * 5855 * For pin based interrupts, the software interrupt handler need to check IS 5856 * register to find out which port has pending interrupts. And then check 5857 * PxIS register to find out which interrupt events happened on that port. 5858 * 5859 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 5860 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 5861 * in that software interrupt handler need to check IS register to determine 5862 * which port triggered the interrupts since it uses a single message for all 5863 * port interrupts. 5864 * 5865 * HBA may optionally support multiple MSI message for better performance. In 5866 * this mode, each port may have its own interrupt message, and thus generation 5867 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 5868 * represents a power-of-2 wrapper on the number of implemented ports, and 5869 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 5870 * maximum number of assigned interrupts. When the number of MSI messages 5871 * allocated is less than the number requested, then hardware may have two 5872 * implementation behaviors: 5873 * * assign each ports its own interrupt and then force all additional 5874 * ports to share the last interrupt message, and this condition is 5875 * indicated by clearing GHC.MRSM to '0' 5876 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 5877 * When multiple-message MSI is enabled, hardware will still set IS register 5878 * as single message case. And this IS register may be used by software when 5879 * fewer than the requested number of messages is granted in order to determine 5880 * which port had the interrupt. 5881 * 5882 * Note: The current ahci driver only supports the first two interrupt modes: 5883 * pin based interrupts and single MSI message interrupts, and the reason 5884 * is indicated in below code. 5885 */ 5886 static int 5887 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 5888 { 5889 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5890 int count, avail, actual; 5891 int i, rc; 5892 5893 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 5894 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 5895 5896 /* get number of interrupts. */ 5897 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 5898 if ((rc != DDI_SUCCESS) || (count == 0)) { 5899 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5900 "ddi_intr_get_nintrs() failed, " 5901 "rc %d count %d\n", rc, count); 5902 return (DDI_FAILURE); 5903 } 5904 5905 /* get number of available interrupts. */ 5906 rc = ddi_intr_get_navail(dip, intr_type, &avail); 5907 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5908 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5909 "ddi_intr_get_navail() failed, " 5910 "rc %d avail %d\n", rc, avail); 5911 return (DDI_FAILURE); 5912 } 5913 5914 #if AHCI_DEBUG 5915 if (avail < count) { 5916 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5917 "ddi_intr_get_nintrs returned %d, navail() returned %d", 5918 count, avail); 5919 } 5920 #endif 5921 5922 /* 5923 * Note: So far Solaris restricts the maximum number of messages for 5924 * x86 to 2, that is avail is 2, so here we set the count with 1 to 5925 * force the driver to use single MSI message interrupt. In future if 5926 * Solaris remove the restriction, then we need to delete the below 5927 * code and try to use multiple interrupt routine to gain better 5928 * performance. 5929 */ 5930 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 5931 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 5932 "force to use one interrupt routine though the " 5933 "HBA supports %d interrupt", count); 5934 count = 1; 5935 } 5936 5937 /* Allocate an array of interrupt handles. */ 5938 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5939 ahci_ctlp->ahcictl_intr_htable = 5940 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5941 5942 /* call ddi_intr_alloc(). */ 5943 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5944 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 5945 5946 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5947 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5948 "ddi_intr_alloc() failed, rc %d count %d actual %d " 5949 "avail %d\n", rc, count, actual, avail); 5950 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5951 ahci_ctlp->ahcictl_intr_size); 5952 return (DDI_FAILURE); 5953 } 5954 5955 /* use interrupt count returned */ 5956 #if AHCI_DEBUG 5957 if (actual < count) { 5958 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5959 "Requested: %d, Received: %d", count, actual); 5960 } 5961 #endif 5962 5963 ahci_ctlp->ahcictl_intr_cnt = actual; 5964 5965 /* 5966 * Get priority for first, assume remaining are all the same. 5967 */ 5968 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5969 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5970 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5971 "ddi_intr_get_pri() failed", NULL); 5972 5973 /* Free already allocated intr. */ 5974 for (i = 0; i < actual; i++) { 5975 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5976 } 5977 5978 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5979 ahci_ctlp->ahcictl_intr_size); 5980 return (DDI_FAILURE); 5981 } 5982 5983 /* Test for high level interrupt. */ 5984 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5985 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5986 "ahci_add_intrs: Hi level intr not supported", NULL); 5987 5988 /* Free already allocated intr. */ 5989 for (i = 0; i < actual; i++) { 5990 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5991 } 5992 5993 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5994 sizeof (ddi_intr_handle_t)); 5995 5996 return (DDI_FAILURE); 5997 } 5998 5999 /* Call ddi_intr_add_handler(). */ 6000 for (i = 0; i < actual; i++) { 6001 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 6002 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 6003 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 6004 "ddi_intr_add_handler() failed", NULL); 6005 6006 /* Free already allocated intr. */ 6007 for (i = 0; i < actual; i++) { 6008 (void) ddi_intr_free( 6009 ahci_ctlp->ahcictl_intr_htable[i]); 6010 } 6011 6012 kmem_free(ahci_ctlp->ahcictl_intr_htable, 6013 ahci_ctlp->ahcictl_intr_size); 6014 return (DDI_FAILURE); 6015 } 6016 } 6017 6018 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 6019 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 6020 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 6021 "ddi_intr_get_cap() failed", NULL); 6022 6023 /* Free already allocated intr. */ 6024 for (i = 0; i < actual; i++) { 6025 (void) ddi_intr_free( 6026 ahci_ctlp->ahcictl_intr_htable[i]); 6027 } 6028 6029 kmem_free(ahci_ctlp->ahcictl_intr_htable, 6030 ahci_ctlp->ahcictl_intr_size); 6031 return (DDI_FAILURE); 6032 } 6033 6034 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 6035 /* Call ddi_intr_block_enable() for MSI. */ 6036 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 6037 ahci_ctlp->ahcictl_intr_cnt); 6038 } else { 6039 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 6040 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 6041 (void) ddi_intr_enable( 6042 ahci_ctlp->ahcictl_intr_htable[i]); 6043 } 6044 } 6045 6046 return (DDI_SUCCESS); 6047 } 6048 6049 /* 6050 * Removes the registered interrupts irrespective of whether they 6051 * were legacy or MSI. 6052 * 6053 * WARNING!!! The controller interrupts must be disabled before calling 6054 * this routine. 6055 */ 6056 static void 6057 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 6058 { 6059 int x; 6060 6061 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 6062 6063 /* Disable all interrupts. */ 6064 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 6065 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 6066 /* Call ddi_intr_block_disable(). */ 6067 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 6068 ahci_ctlp->ahcictl_intr_cnt); 6069 } else { 6070 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 6071 (void) ddi_intr_disable( 6072 ahci_ctlp->ahcictl_intr_htable[x]); 6073 } 6074 } 6075 6076 /* Call ddi_intr_remove_handler(). */ 6077 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 6078 (void) ddi_intr_remove_handler( 6079 ahci_ctlp->ahcictl_intr_htable[x]); 6080 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 6081 } 6082 6083 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 6084 } 6085 6086 /* 6087 * This routine tries to put port into P:NotRunning state by clearing 6088 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 6089 * and PxCMD.CR to '0'. 6090 * 6091 * WARNING!!! ahciport_mutex should be acquired before the function 6092 * is called. 6093 */ 6094 static int 6095 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 6096 ahci_port_t *ahci_portp, uint8_t port) 6097 { 6098 uint32_t port_cmd_status; 6099 int loop_count; 6100 6101 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6102 "ahci_put_port_into_notrunning_state enter: port %d", port); 6103 6104 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6105 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6106 6107 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 6108 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6109 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 6110 6111 /* Wait until PxCMD.CR is cleared */ 6112 loop_count = 0; 6113 do { 6114 port_cmd_status = 6115 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6116 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6117 6118 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 6119 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6120 "clearing port %d CMD.CR timeout, " 6121 "port_cmd_status = 0x%x", port, 6122 port_cmd_status); 6123 /* 6124 * We are effectively timing out after 0.5 sec. 6125 * This value is specified in AHCI spec. 6126 */ 6127 break; 6128 } 6129 6130 /* Wait for 10 millisec */ 6131 drv_usecwait(10000); 6132 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 6133 6134 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 6135 6136 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 6137 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 6138 "ahci_put_port_into_notrunning_state: failed to clear " 6139 "PxCMD.CR to '0' after loop count: %d, and " 6140 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6141 return (AHCI_FAILURE); 6142 } else { 6143 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 6144 "ahci_put_port_into_notrunning_state: succeeded to clear " 6145 "PxCMD.CR to '0' after loop count: %d, and " 6146 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 6147 return (AHCI_SUCCESS); 6148 } 6149 } 6150 6151 /* 6152 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 6153 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 6154 * stable state, then set PxCMD.ST to '1' to start the port directly. 6155 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 6156 * COMRESET to the device to put it in an idle state. 6157 * 6158 * The fifth argument returns whether the port reset is involved during 6159 * the process. 6160 * 6161 * The routine will be called under following scenarios: 6162 * + To abort the packet(s) 6163 * + To reset the port 6164 * + To activate the port 6165 * + Fatal error recovery 6166 * + To abort the timeout packet(s) 6167 * 6168 * WARNING!!! ahciport_mutex should be acquired before the function 6169 * is called. And ahciport_mutex will be released before the reset 6170 * event is reported to sata module by calling sata_hba_event_notify, 6171 * and then be acquired again later. 6172 * 6173 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6174 * according to the spec, the clearance of three bits will also clear 6175 * three interrupt status bits. 6176 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6177 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6178 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6179 * 6180 * Among these three interrupt events, the driver needs to take care of 6181 * PxIS.PRCS, which is the hot plug event. When the driver found out 6182 * a device was unplugged, it will call the interrupt handler. 6183 */ 6184 static int 6185 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6186 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6187 { 6188 uint32_t port_sstatus; 6189 uint32_t task_file_status; 6190 sata_device_t sdevice; 6191 int rval; 6192 int dev_exists_begin = 0; 6193 int dev_exists_end = 0; 6194 6195 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6196 "ahci_restart_port_wait_till_ready: port %d enter", port); 6197 6198 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 6199 dev_exists_begin = 1; 6200 6201 /* First clear PxCMD.ST */ 6202 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6203 port); 6204 if (rval != AHCI_SUCCESS) 6205 /* 6206 * If PxCMD.CR does not clear within a reasonable time, it 6207 * may assume the interface is in a hung condition and may 6208 * continue with issuing the port reset. 6209 */ 6210 goto reset; 6211 6212 /* Then clear PxSERR */ 6213 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6214 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6215 AHCI_SERROR_CLEAR_ALL); 6216 6217 /* Then get PxTFD */ 6218 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6219 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6220 6221 /* 6222 * Check whether the device is in a stable status, if yes, 6223 * then start the port directly. However for ahci_tran_dport_reset, 6224 * we may have to perform a port reset. 6225 */ 6226 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6227 !(flag & AHCI_PORT_RESET)) 6228 goto out; 6229 6230 reset: 6231 /* 6232 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6233 * a COMRESET to the device 6234 */ 6235 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6236 6237 if (reset_flag != NULL) 6238 *reset_flag = 1; 6239 6240 /* Indicate to the framework that a reset has happened. */ 6241 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6242 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6243 /* Set the reset in progress flag */ 6244 ahci_portp->ahciport_reset_in_progress = 1; 6245 6246 bzero((void *)&sdevice, sizeof (sata_device_t)); 6247 sdevice.satadev_addr.cport = 6248 ahci_ctlp->ahcictl_port_to_cport[port]; 6249 sdevice.satadev_addr.pmport = 0; 6250 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6251 6252 sdevice.satadev_state = SATA_DSTATE_RESET | 6253 SATA_DSTATE_PWR_ACTIVE; 6254 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6255 mutex_exit(&ahci_portp->ahciport_mutex); 6256 sata_hba_event_notify( 6257 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6258 &sdevice, 6259 SATA_EVNT_DEVICE_RESET); 6260 mutex_enter(&ahci_portp->ahciport_mutex); 6261 } 6262 6263 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 6264 "port %d sending event up: SATA_EVNT_RESET", port); 6265 } else { 6266 ahci_portp->ahciport_reset_in_progress = 0; 6267 } 6268 6269 out: 6270 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6271 6272 /* SStatus tells the presence of device. */ 6273 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6274 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6275 6276 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6277 dev_exists_end = 1; 6278 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6279 } 6280 6281 /* Check whether a hot plug event happened */ 6282 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6283 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6284 "ahci_restart_port_wait_till_ready: port %d " 6285 "device is removed", port); 6286 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6287 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6288 "ahci_restart_port_wait_till_ready: port %d " 6289 "AHCI_PORT_FLAG_NODEV flag is set", port); 6290 mutex_exit(&ahci_portp->ahciport_mutex); 6291 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6292 mutex_enter(&ahci_portp->ahciport_mutex); 6293 } 6294 6295 return (rval); 6296 } 6297 6298 /* 6299 * This routine may be called under four scenarios: 6300 * a) do the recovery from fatal error 6301 * b) or we need to timeout some commands 6302 * c) or we need to abort some commands 6303 * d) or we need reset device/port/controller 6304 * 6305 * In all these scenarios, we need to send any pending unfinished 6306 * commands up to sata framework. 6307 * 6308 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6309 */ 6310 static void 6311 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6312 ahci_port_t *ahci_portp, 6313 uint32_t slot_status, 6314 uint32_t failed_tags, 6315 uint32_t timeout_tags, 6316 uint32_t aborted_tags, 6317 uint32_t reset_tags) 6318 { 6319 uint32_t finished_tags = 0; 6320 uint32_t unfinished_tags = 0; 6321 int tmp_slot; 6322 sata_pkt_t *satapkt; 6323 int ncq_cmd_in_progress = 0; 6324 int err_retri_cmd_in_progress = 0; 6325 6326 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6327 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6328 ahci_portp->ahciport_port_num, slot_status); 6329 6330 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6331 "ahci_mop_commands: failed_tags: 0x%x, " 6332 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6333 "reset_tags: 0x%x", failed_tags, 6334 timeout_tags, aborted_tags, reset_tags); 6335 6336 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6337 finished_tags = ahci_portp->ahciport_pending_tags & 6338 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6339 6340 unfinished_tags = slot_status & 6341 AHCI_SLOT_MASK(ahci_ctlp) & 6342 ~failed_tags & 6343 ~aborted_tags & 6344 ~reset_tags & 6345 ~timeout_tags; 6346 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6347 ncq_cmd_in_progress = 1; 6348 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6349 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6350 6351 unfinished_tags = slot_status & 6352 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6353 ~failed_tags & 6354 ~aborted_tags & 6355 ~reset_tags & 6356 ~timeout_tags; 6357 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6358 6359 /* 6360 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6361 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6362 * complete successfully due to one of the following three 6363 * conditions: 6364 * 6365 * 1. Fatal error - failed_tags includes its slot 6366 * 2. Timed out - timeout_tags includes its slot 6367 * 3. Aborted when hot unplug - aborted_tags includes its 6368 * slot 6369 * 6370 * Please note that the command is always sent down in Slot 0 6371 */ 6372 err_retri_cmd_in_progress = 1; 6373 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6374 "ahci_mop_commands is called for port %d while " 6375 "REQUEST SENSE or READ LOG EXT for error retrieval " 6376 "is being executed slot_status = 0x%x", 6377 ahci_portp->ahciport_port_num, slot_status); 6378 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6379 ASSERT(slot_status == 0x1); 6380 } 6381 6382 /* Send up finished packets with SATA_PKT_COMPLETED */ 6383 while (finished_tags) { 6384 tmp_slot = ddi_ffs(finished_tags) - 1; 6385 if (tmp_slot == -1) { 6386 break; 6387 } 6388 6389 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6390 ASSERT(satapkt != NULL); 6391 6392 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6393 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6394 (void *)satapkt); 6395 6396 /* 6397 * Cannot fetch the return register content since the port 6398 * was restarted, so the corresponding tag will be set to 6399 * aborted tags. 6400 */ 6401 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6402 CLEAR_BIT(finished_tags, tmp_slot); 6403 aborted_tags |= tmp_slot; 6404 continue; 6405 } 6406 6407 if (ncq_cmd_in_progress) 6408 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6409 tmp_slot); 6410 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6411 CLEAR_BIT(finished_tags, tmp_slot); 6412 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6413 6414 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6415 } 6416 6417 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6418 while (failed_tags) { 6419 if (err_retri_cmd_in_progress) { 6420 satapkt = ahci_portp->ahciport_err_retri_pkt; 6421 ASSERT(satapkt != NULL); 6422 ASSERT(failed_tags == 0x1); 6423 6424 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6425 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6426 (void *)satapkt); 6427 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6428 break; 6429 } 6430 6431 tmp_slot = ddi_ffs(failed_tags) - 1; 6432 if (tmp_slot == -1) { 6433 break; 6434 } 6435 6436 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6437 ASSERT(satapkt != NULL); 6438 6439 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6440 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6441 (void *)satapkt); 6442 6443 if (ncq_cmd_in_progress) 6444 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6445 tmp_slot); 6446 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6447 CLEAR_BIT(failed_tags, tmp_slot); 6448 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6449 6450 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6451 } 6452 6453 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6454 while (timeout_tags) { 6455 if (err_retri_cmd_in_progress) { 6456 satapkt = ahci_portp->ahciport_err_retri_pkt; 6457 ASSERT(satapkt != NULL); 6458 ASSERT(timeout_tags == 0x1); 6459 6460 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6461 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6462 (void *)satapkt); 6463 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6464 break; 6465 } 6466 6467 tmp_slot = ddi_ffs(timeout_tags) - 1; 6468 if (tmp_slot == -1) { 6469 break; 6470 } 6471 6472 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6473 ASSERT(satapkt != NULL); 6474 6475 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6476 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6477 (void *)satapkt); 6478 6479 if (ncq_cmd_in_progress) 6480 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6481 tmp_slot); 6482 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6483 CLEAR_BIT(timeout_tags, tmp_slot); 6484 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6485 6486 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6487 } 6488 6489 /* Send up aborted packets with SATA_PKT_ABORTED */ 6490 while (aborted_tags) { 6491 if (err_retri_cmd_in_progress) { 6492 satapkt = ahci_portp->ahciport_err_retri_pkt; 6493 ASSERT(satapkt != NULL); 6494 ASSERT(aborted_tags == 0x1); 6495 6496 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6497 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6498 (void *)satapkt); 6499 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6500 break; 6501 } 6502 6503 tmp_slot = ddi_ffs(aborted_tags) - 1; 6504 if (tmp_slot == -1) { 6505 break; 6506 } 6507 6508 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6509 ASSERT(satapkt != NULL); 6510 6511 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6512 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6513 (void *)satapkt); 6514 6515 if (ncq_cmd_in_progress) 6516 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6517 tmp_slot); 6518 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6519 CLEAR_BIT(aborted_tags, tmp_slot); 6520 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6521 6522 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6523 } 6524 6525 /* Send up reset packets with SATA_PKT_RESET. */ 6526 while (reset_tags) { 6527 tmp_slot = ddi_ffs(reset_tags) - 1; 6528 if (tmp_slot == -1) { 6529 break; 6530 } 6531 6532 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6533 ASSERT(satapkt != NULL); 6534 6535 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6536 "sending up pkt 0x%p with SATA_PKT_RESET", 6537 (void *)satapkt); 6538 6539 if (ncq_cmd_in_progress) 6540 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6541 tmp_slot); 6542 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6543 CLEAR_BIT(reset_tags, tmp_slot); 6544 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6545 6546 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6547 } 6548 6549 /* Send up unfinished packets with SATA_PKT_RESET */ 6550 while (unfinished_tags) { 6551 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6552 if (tmp_slot == -1) { 6553 break; 6554 } 6555 6556 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6557 ASSERT(satapkt != NULL); 6558 6559 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6560 "sending up pkt 0x%p with SATA_PKT_RESET", 6561 (void *)satapkt); 6562 6563 if (ncq_cmd_in_progress) 6564 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6565 tmp_slot); 6566 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6567 CLEAR_BIT(unfinished_tags, tmp_slot); 6568 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6569 6570 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6571 } 6572 6573 ahci_portp->ahciport_mop_in_progress--; 6574 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6575 6576 if (ahci_portp->ahciport_mop_in_progress == 0) 6577 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6578 } 6579 6580 /* 6581 * This routine is going to first request a READ LOG EXT sata pkt from sata 6582 * module, and then deliver it to the HBA to get the ncq failure context. 6583 * The return value is the exactly failed tags. 6584 */ 6585 static uint32_t 6586 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6587 uint8_t port) 6588 { 6589 sata_device_t sdevice; 6590 sata_pkt_t *rdlog_spkt, *spkt; 6591 ddi_dma_handle_t buf_dma_handle; 6592 int loop_count; 6593 int rval; 6594 int failed_slot; 6595 uint32_t failed_tags = 0; 6596 struct sata_ncq_error_recovery_page *ncq_err_page; 6597 6598 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6599 "ahci_get_rdlogext_data enter: port %d", port); 6600 6601 /* Prepare the sdevice data */ 6602 bzero((void *)&sdevice, sizeof (sata_device_t)); 6603 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6604 6605 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6606 sdevice.satadev_addr.pmport = 0; 6607 6608 /* 6609 * Call the sata hba interface to get a rdlog spkt 6610 */ 6611 loop_count = 0; 6612 loop: 6613 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6614 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6615 if (rdlog_spkt == NULL) { 6616 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6617 /* Sleep for a while */ 6618 delay(AHCI_10MS_TICKS); 6619 goto loop; 6620 } 6621 /* Timed out after 1s */ 6622 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6623 "failed to get rdlog spkt for port %d", port); 6624 return (failed_tags); 6625 } 6626 6627 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6628 6629 /* 6630 * This flag is used to handle the specific error recovery when the 6631 * READ LOG EXT command gets a failure (fatal error or time-out). 6632 */ 6633 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6634 6635 /* 6636 * This start is not supposed to fail because after port is restarted, 6637 * the whole command list is empty. 6638 */ 6639 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6640 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6641 ahci_portp->ahciport_err_retri_pkt = NULL; 6642 6643 /* Remove the flag after READ LOG EXT command is completed */ 6644 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6645 6646 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6647 /* Update the request log data */ 6648 buf_dma_handle = *(ddi_dma_handle_t *) 6649 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6650 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6651 DDI_DMA_SYNC_FORKERNEL); 6652 if (rval == DDI_SUCCESS) { 6653 ncq_err_page = 6654 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6655 satapkt_cmd.satacmd_bp->b_un.b_addr; 6656 6657 /* Get the failed tag */ 6658 failed_slot = ncq_err_page->ncq_tag; 6659 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6660 "ahci_get_rdlogext_data: port %d " 6661 "failed slot %d", port, failed_slot); 6662 if (failed_slot & NQ) { 6663 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6664 "the failed slot is not a valid tag", NULL); 6665 goto out; 6666 } 6667 6668 failed_slot &= NCQ_TAG_MASK; 6669 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6670 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6671 "ahci_get_rdlogext_data: failed spkt 0x%p", 6672 (void *)spkt); 6673 if (spkt == NULL) { 6674 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6675 "the failed slot spkt is NULL", NULL); 6676 goto out; 6677 } 6678 6679 failed_tags = 0x1 << failed_slot; 6680 6681 /* Fill out the error context */ 6682 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6683 ncq_err_page); 6684 ahci_update_sata_registers(ahci_ctlp, port, 6685 &spkt->satapkt_device); 6686 } 6687 } 6688 out: 6689 sata_free_error_retrieval_pkt(rdlog_spkt); 6690 6691 return (failed_tags); 6692 } 6693 6694 /* 6695 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6696 * module, and then deliver it to the HBA to get the sense data and copy 6697 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6698 * SENSE sata pkt later. 6699 */ 6700 static void 6701 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6702 uint8_t port, sata_pkt_t *spkt) 6703 { 6704 sata_device_t sdevice; 6705 sata_pkt_t *rs_spkt; 6706 sata_cmd_t *sata_cmd; 6707 ddi_dma_handle_t buf_dma_handle; 6708 int loop_count; 6709 #if AHCI_DEBUG 6710 struct scsi_extended_sense *rqsense; 6711 #endif 6712 6713 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6714 "ahci_get_rqsense_data enter: port %d", port); 6715 6716 /* Prepare the sdevice data */ 6717 bzero((void *)&sdevice, sizeof (sata_device_t)); 6718 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6719 6720 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6721 sdevice.satadev_addr.pmport = 0; 6722 6723 sata_cmd = &spkt->satapkt_cmd; 6724 6725 /* 6726 * Call the sata hba interface to get a rs spkt 6727 */ 6728 loop_count = 0; 6729 loop: 6730 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6731 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6732 if (rs_spkt == NULL) { 6733 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6734 /* Sleep for a while */ 6735 delay(AHCI_10MS_TICKS); 6736 goto loop; 6737 6738 } 6739 /* Timed out after 1s */ 6740 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6741 "failed to get rs spkt for port %d", port); 6742 return; 6743 } 6744 6745 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6746 6747 /* 6748 * This flag is used to handle the specific error recovery when the 6749 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6750 */ 6751 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6752 6753 /* 6754 * This start is not supposed to fail because after port is restarted, 6755 * the whole command list is empty. 6756 */ 6757 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6758 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6759 ahci_portp->ahciport_err_retri_pkt = NULL; 6760 6761 /* Remove the flag after REQUEST SENSE command is completed */ 6762 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6763 6764 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6765 /* Update the request sense data */ 6766 buf_dma_handle = *(ddi_dma_handle_t *) 6767 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6768 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6769 DDI_DMA_SYNC_FORKERNEL); 6770 /* Copy the request sense data */ 6771 bcopy(rs_spkt-> 6772 satapkt_cmd.satacmd_bp->b_un.b_addr, 6773 &sata_cmd->satacmd_rqsense, 6774 SATA_ATAPI_MIN_RQSENSE_LEN); 6775 #if AHCI_DEBUG 6776 rqsense = (struct scsi_extended_sense *) 6777 sata_cmd->satacmd_rqsense; 6778 6779 /* Dump the sense data */ 6780 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 6781 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 6782 "Sense data for satapkt %p ATAPI cmd 0x%x", 6783 spkt, sata_cmd->satacmd_acdb[0]); 6784 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 6785 " es_code 0x%x es_class 0x%x " 6786 "es_key 0x%x es_add_code 0x%x " 6787 "es_qual_code 0x%x", 6788 rqsense->es_code, rqsense->es_class, 6789 rqsense->es_key, rqsense->es_add_code, 6790 rqsense->es_qual_code); 6791 #endif 6792 } 6793 6794 sata_free_error_retrieval_pkt(rs_spkt); 6795 } 6796 6797 /* 6798 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6799 * the port must be restarted. When the HBA detects thus error, it may try 6800 * to abort a transfer. And if the transfer was aborted, the device is 6801 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6802 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6803 * that the device is in a stable status and transfers may be restarted without 6804 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6805 * then the software will send the COMRESET to do the port reset. 6806 * 6807 * Software should perform the appropriate error recovery actions based on 6808 * whether non-queued commands were being issued or natived command queuing 6809 * commands were being issued. 6810 * 6811 * And software will complete the command that had the error with error mark 6812 * to higher level software. 6813 * 6814 * Fatal errors include the following: 6815 * PxIS.IFS - Interface Fatal Error Status 6816 * PxIS.HBDS - Host Bus Data Error Status 6817 * PxIS.HBFS - Host Bus Fatal Error Status 6818 * PxIS.TFES - Task File Error Status 6819 * 6820 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6821 */ 6822 static void 6823 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6824 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6825 { 6826 uint32_t port_cmd_status; 6827 uint32_t slot_status = 0; 6828 uint32_t failed_tags = 0; 6829 int failed_slot; 6830 int reset_flag = 0; 6831 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6832 sata_cmd_t *sata_cmd = NULL; 6833 sata_pkt_t *spkt = NULL; 6834 #if AHCI_DEBUG 6835 ahci_cmd_header_t *cmd_header; 6836 #endif 6837 6838 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6839 "ahci_fatal_error_recovery_handler enter: port %d", port); 6840 6841 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6842 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6843 6844 /* Read PxCI to see which commands are still outstanding */ 6845 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6846 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6847 6848 /* 6849 * Read PxCMD.CCS to determine the slot that the HBA 6850 * was processing when the error occurred. 6851 */ 6852 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6853 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6854 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6855 AHCI_CMD_STATUS_CCS_SHIFT; 6856 6857 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6858 spkt = ahci_portp->ahciport_err_retri_pkt; 6859 ASSERT(spkt != NULL); 6860 } else { 6861 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6862 if (spkt == NULL) { 6863 /* May happen when interface errors occur? */ 6864 goto next; 6865 } 6866 } 6867 6868 #if AHCI_DEBUG 6869 /* 6870 * Debugging purpose... 6871 */ 6872 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6873 cmd_header = 6874 &ahci_portp->ahciport_cmd_list[failed_slot]; 6875 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6876 "ahci_fatal_error_recovery_handler: port %d, " 6877 "PRD Byte Count = 0x%x, " 6878 "ahciport_prd_bytecounts = 0x%x", port, 6879 cmd_header->ahcich_prd_byte_count, 6880 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6881 } 6882 #endif 6883 6884 sata_cmd = &spkt->satapkt_cmd; 6885 6886 /* Fill out the status and error registers for PxIS.TFES */ 6887 if (intr_status & AHCI_INTR_STATUS_TFES) { 6888 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6889 ahcirf_d2h_register_fis); 6890 6891 /* Copy the error context back to the sata_cmd */ 6892 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6893 } 6894 6895 /* The failed command must be one of the outstanding commands */ 6896 failed_tags = 0x1 << failed_slot; 6897 ASSERT(failed_tags & slot_status); 6898 6899 /* Update the sata registers, especially PxSERR register */ 6900 ahci_update_sata_registers(ahci_ctlp, port, 6901 &spkt->satapkt_device); 6902 6903 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6904 /* Read PxSACT to see which commands are still outstanding */ 6905 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6906 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6907 } 6908 next: 6909 6910 #if AHCI_DEBUG 6911 /* 6912 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6913 * set, it means a fatal error happened after REQUEST SENSE command 6914 * or READ LOG EXT command is delivered to the HBA during the error 6915 * recovery process. At this time, the only outstanding command is 6916 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6917 */ 6918 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6919 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6920 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6921 "command or READ LOG EXT command for error data retrieval " 6922 "failed", port); 6923 ASSERT(slot_status == 0x1); 6924 ASSERT(failed_slot == 0); 6925 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6926 SCMD_REQUEST_SENSE || 6927 spkt->satapkt_cmd.satacmd_cmd_reg == 6928 SATAC_READ_LOG_EXT); 6929 } 6930 #endif 6931 6932 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6933 ahci_portp->ahciport_mop_in_progress++; 6934 6935 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6936 port, NULL, &reset_flag); 6937 6938 /* 6939 * Won't retrieve error information: 6940 * 1. Port reset was involved to recover 6941 * 2. Device is gone 6942 * 3. IDENTIFY DEVICE command sent to ATAPI device 6943 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6944 */ 6945 if (reset_flag || 6946 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6947 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6948 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6949 goto out; 6950 6951 /* 6952 * Deliver READ LOG EXT to gather information about the error when 6953 * a COMRESET has not been performed as part of the error recovery 6954 * during NCQ command processing. 6955 */ 6956 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6957 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6958 ahci_portp, port); 6959 goto out; 6960 } 6961 6962 /* 6963 * Deliver REQUEST SENSE for ATAPI command to gather information about 6964 * the error when a COMRESET has not been performed as part of the 6965 * error recovery. 6966 */ 6967 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6968 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6969 out: 6970 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6971 "ahci_fatal_error_recovery_handler: port %d fatal error " 6972 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6973 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6974 port, slot_status, ahci_portp->ahciport_pending_tags, 6975 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6976 6977 ahci_mop_commands(ahci_ctlp, 6978 ahci_portp, 6979 slot_status, 6980 failed_tags, /* failed tags */ 6981 0, /* timeout tags */ 6982 0, /* aborted tags */ 6983 0); /* reset tags */ 6984 } 6985 6986 /* 6987 * Handle events - fatal error recovery 6988 */ 6989 static void 6990 ahci_events_handler(void *args) 6991 { 6992 ahci_event_arg_t *ahci_event_arg; 6993 ahci_ctl_t *ahci_ctlp; 6994 ahci_port_t *ahci_portp; 6995 uint32_t event; 6996 uint8_t port; 6997 6998 ahci_event_arg = (ahci_event_arg_t *)args; 6999 7000 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 7001 ahci_portp = ahci_event_arg->ahciea_portp; 7002 event = ahci_event_arg->ahciea_event; 7003 port = ahci_portp->ahciport_port_num; 7004 7005 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7006 "ahci_events_handler enter: port %d intr_status = 0x%x", 7007 port, event); 7008 7009 mutex_enter(&ahci_portp->ahciport_mutex); 7010 7011 /* 7012 * ahci_intr_phyrdy_change() may have rendered it to 7013 * SATA_DTYPE_NONE. 7014 */ 7015 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7016 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7017 "ahci_events_handler: port %d no device attached, " 7018 "and just return without doing anything", port); 7019 goto out; 7020 } 7021 7022 if (event & (AHCI_INTR_STATUS_IFS | 7023 AHCI_INTR_STATUS_HBDS | 7024 AHCI_INTR_STATUS_HBFS | 7025 AHCI_INTR_STATUS_TFES)) 7026 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 7027 port, event); 7028 7029 out: 7030 mutex_exit(&ahci_portp->ahciport_mutex); 7031 } 7032 7033 /* 7034 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 7035 * detect there are some commands which are timed out. 7036 */ 7037 static void 7038 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7039 uint8_t port, uint32_t tmp_timeout_tags) 7040 { 7041 uint32_t slot_status = 0; 7042 uint32_t finished_tags = 0; 7043 uint32_t timeout_tags = 0; 7044 7045 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 7046 "ahci_timeout_pkts enter: port %d", port); 7047 7048 mutex_enter(&ahci_portp->ahciport_mutex); 7049 7050 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7051 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7052 /* Read PxCI to see which commands are still outstanding */ 7053 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7054 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7055 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7056 /* Read PxSACT to see which commands are still outstanding */ 7057 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7058 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7059 } 7060 7061 #if AHCI_DEBUG 7062 /* 7063 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 7064 * set, it means a fatal error happened after REQUEST SENSE command 7065 * or READ LOG EXT command is delivered to the HBA during the error 7066 * recovery process. At this time, the only outstanding command is 7067 * supposed to be REQUEST SENSE command or READ LOG EXT command. 7068 */ 7069 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7070 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 7071 "ahci_timeout_pkts called while REQUEST SENSE " 7072 "command or READ LOG EXT command for error recovery " 7073 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 7074 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 7075 tmp_timeout_tags, slot_status, 7076 ahci_portp->ahciport_pending_tags, 7077 ahci_portp->ahciport_pending_ncq_tags); 7078 ASSERT(slot_status == 0x1); 7079 } 7080 #endif 7081 7082 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 7083 ahci_portp->ahciport_mop_in_progress++; 7084 7085 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 7086 port, NULL, NULL); 7087 7088 /* 7089 * Re-identify timeout tags because some previously checked commands 7090 * could already complete. 7091 */ 7092 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7093 finished_tags = ahci_portp->ahciport_pending_tags & 7094 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 7095 timeout_tags = tmp_timeout_tags & ~finished_tags; 7096 7097 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 7098 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 7099 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 7100 "pending_tags = 0x%x ", 7101 port, finished_tags, timeout_tags, 7102 slot_status, ahci_portp->ahciport_pending_tags); 7103 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7104 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 7105 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 7106 timeout_tags = tmp_timeout_tags & ~finished_tags; 7107 7108 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 7109 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 7110 "timeout_tags = 0x%x, port_sactive = 0x%x, " 7111 "pending_ncq_tags = 0x%x ", 7112 port, finished_tags, timeout_tags, 7113 slot_status, ahci_portp->ahciport_pending_ncq_tags); 7114 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7115 timeout_tags = tmp_timeout_tags; 7116 } 7117 7118 ahci_mop_commands(ahci_ctlp, 7119 ahci_portp, 7120 slot_status, 7121 0, /* failed tags */ 7122 timeout_tags, /* timeout tags */ 7123 0, /* aborted tags */ 7124 0); /* reset tags */ 7125 7126 mutex_exit(&ahci_portp->ahciport_mutex); 7127 } 7128 7129 /* 7130 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 7131 * for long time. 7132 */ 7133 static void 7134 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 7135 { 7136 ahci_port_t *ahci_portp; 7137 sata_pkt_t *spkt; 7138 uint32_t pending_tags; 7139 uint32_t timeout_tags; 7140 uint32_t port_cmd_status; 7141 uint32_t port_sactive; 7142 uint8_t port; 7143 int tmp_slot; 7144 int current_slot; 7145 uint32_t current_tags; 7146 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7147 /* max number of cycles this packet should survive */ 7148 int max_life_cycles; 7149 7150 /* how many cycles this packet survived so far */ 7151 int watched_cycles; 7152 7153 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7154 7155 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7156 "ahci_watchdog_handler entered", NULL); 7157 7158 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7159 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7160 continue; 7161 } 7162 7163 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7164 7165 mutex_enter(&ahci_portp->ahciport_mutex); 7166 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7167 mutex_exit(&ahci_portp->ahciport_mutex); 7168 continue; 7169 } 7170 7171 /* Skip the check for those ports in error recovery */ 7172 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7173 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7174 mutex_exit(&ahci_portp->ahciport_mutex); 7175 continue; 7176 } 7177 7178 pending_tags = 0; 7179 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7180 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7181 7182 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7183 current_slot = 0; 7184 pending_tags = 0x1; 7185 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7186 current_slot = 7187 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7188 AHCI_CMD_STATUS_CCS_SHIFT; 7189 pending_tags = ahci_portp->ahciport_pending_tags; 7190 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7191 port_sactive = ddi_get32( 7192 ahci_ctlp->ahcictl_ahci_acc_handle, 7193 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7194 current_tags = port_sactive & 7195 ~port_cmd_status & 7196 AHCI_NCQ_SLOT_MASK(ahci_portp); 7197 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7198 } 7199 7200 timeout_tags = 0; 7201 while (pending_tags) { 7202 tmp_slot = ddi_ffs(pending_tags) - 1; 7203 if (tmp_slot == -1) { 7204 break; 7205 } 7206 7207 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7208 spkt = ahci_portp->ahciport_err_retri_pkt; 7209 else 7210 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7211 7212 if ((spkt != NULL) && spkt->satapkt_time && 7213 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7214 /* 7215 * We are overloading satapkt_hba_driver_private 7216 * with watched_cycle count. 7217 * 7218 * If a packet has survived for more than it's 7219 * max life cycles, it is a candidate for time 7220 * out. 7221 */ 7222 watched_cycles = (int)(intptr_t) 7223 spkt->satapkt_hba_driver_private; 7224 watched_cycles++; 7225 max_life_cycles = (spkt->satapkt_time + 7226 ahci_watchdog_timeout - 1) / 7227 ahci_watchdog_timeout; 7228 7229 spkt->satapkt_hba_driver_private = 7230 (void *)(intptr_t)watched_cycles; 7231 7232 if (watched_cycles <= max_life_cycles) 7233 goto next; 7234 7235 #if AHCI_DEBUG 7236 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7237 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7238 ahci_ctlp, "watchdog: the current " 7239 "tags is 0x%x", current_tags); 7240 } else { 7241 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7242 ahci_ctlp, "watchdog: the current " 7243 "slot is %d", current_slot); 7244 } 7245 #endif 7246 7247 /* 7248 * We need to check whether the HBA has 7249 * begun to execute the command, if not, 7250 * then re-set the timer of the command. 7251 */ 7252 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7253 (tmp_slot != current_slot) || 7254 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7255 ((0x1 << tmp_slot) & current_tags)) { 7256 spkt->satapkt_hba_driver_private = 7257 (void *)(intptr_t)0; 7258 } else { 7259 timeout_tags |= (0x1 << tmp_slot); 7260 cmn_err(CE_WARN, "!ahci%d: watchdog " 7261 "port %d satapkt 0x%p timed out\n", 7262 instance, port, (void *)spkt); 7263 } 7264 } 7265 next: 7266 CLEAR_BIT(pending_tags, tmp_slot); 7267 } 7268 7269 if (timeout_tags) { 7270 mutex_exit(&ahci_portp->ahciport_mutex); 7271 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7272 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7273 port, timeout_tags); 7274 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7275 mutex_enter(&ahci_portp->ahciport_mutex); 7276 } 7277 7278 mutex_exit(&ahci_portp->ahciport_mutex); 7279 } 7280 7281 /* Re-install the watchdog timeout handler */ 7282 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7283 ahci_ctlp->ahcictl_timeout_id = 7284 timeout((void (*)(void *))ahci_watchdog_handler, 7285 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7286 } 7287 7288 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7289 } 7290 7291 /* 7292 * Fill the error context into sata_cmd for non-queued command error. 7293 */ 7294 static void 7295 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7296 { 7297 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7298 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7299 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7300 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7301 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7302 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7303 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7304 7305 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7306 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7307 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7308 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7309 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7310 } 7311 } 7312 7313 /* 7314 * Fill the ncq error page into sata_cmd for queued command error. 7315 */ 7316 static void 7317 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7318 struct sata_ncq_error_recovery_page *ncq_err_page) 7319 { 7320 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7321 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7322 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7323 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7324 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7325 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7326 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7327 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7328 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7329 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7330 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7331 } 7332 7333 /* 7334 * Put the respective register value to sata_cmd_t for satacmd_flags. 7335 */ 7336 static void 7337 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7338 { 7339 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7340 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7341 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7342 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7343 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7344 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7345 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7346 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7347 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7348 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7349 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7350 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7351 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7352 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7353 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7354 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7355 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7356 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7357 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7358 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7359 } 7360 7361 static void 7362 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7363 uint32_t intr_status) 7364 { 7365 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7366 7367 if (intr_status & AHCI_INTR_STATUS_IFS) 7368 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 7369 "error", instance, port); 7370 7371 if (intr_status & AHCI_INTR_STATUS_HBDS) 7372 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 7373 instance, port); 7374 7375 if (intr_status & AHCI_INTR_STATUS_HBFS) 7376 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 7377 instance, port); 7378 7379 if (intr_status & AHCI_INTR_STATUS_TFES) 7380 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 7381 instance, port); 7382 7383 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 7384 "recovery", instance, port); 7385 } 7386 7387 /* 7388 * Dump the serror message to the log. 7389 */ 7390 static void 7391 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7392 uint32_t port_serror, int debug_only) 7393 { 7394 static char err_buf[512]; 7395 static char err_msg_header[16]; 7396 char *err_msg = err_buf; 7397 7398 *err_buf = '\0'; 7399 *err_msg_header = '\0'; 7400 7401 if (port_serror & SERROR_DATA_ERR_FIXED) { 7402 err_msg = strcat(err_msg, 7403 "\tRecovered Data Integrity Error (I)\n"); 7404 } 7405 7406 if (port_serror & SERROR_COMM_ERR_FIXED) { 7407 err_msg = strcat(err_msg, 7408 "\tRecovered Communication Error (M)\n"); 7409 } 7410 7411 if (port_serror & SERROR_DATA_ERR) { 7412 err_msg = strcat(err_msg, 7413 "\tTransient Data Integrity Error (T)\n"); 7414 } 7415 7416 if (port_serror & SERROR_PERSISTENT_ERR) { 7417 err_msg = strcat(err_msg, 7418 "\tPersistent Communication or Data Integrity Error (C)\n"); 7419 } 7420 7421 if (port_serror & SERROR_PROTOCOL_ERR) { 7422 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7423 } 7424 7425 if (port_serror & SERROR_INT_ERR) { 7426 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7427 } 7428 7429 if (port_serror & SERROR_PHY_RDY_CHG) { 7430 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7431 } 7432 7433 if (port_serror & SERROR_PHY_INT_ERR) { 7434 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7435 } 7436 7437 if (port_serror & SERROR_COMM_WAKE) { 7438 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7439 } 7440 7441 if (port_serror & SERROR_10B_TO_8B_ERR) { 7442 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7443 } 7444 7445 if (port_serror & SERROR_DISPARITY_ERR) { 7446 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7447 } 7448 7449 if (port_serror & SERROR_CRC_ERR) { 7450 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7451 } 7452 7453 if (port_serror & SERROR_HANDSHAKE_ERR) { 7454 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7455 } 7456 7457 if (port_serror & SERROR_LINK_SEQ_ERR) { 7458 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7459 } 7460 7461 if (port_serror & SERROR_TRANS_ERR) { 7462 err_msg = strcat(err_msg, 7463 "\tTransport state transition error (T)\n"); 7464 } 7465 7466 if (port_serror & SERROR_FIS_TYPE) { 7467 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7468 } 7469 7470 if (port_serror & SERROR_EXCHANGED_ERR) { 7471 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7472 } 7473 7474 if (err_msg == NULL) 7475 return; 7476 7477 if (debug_only) { 7478 (void) sprintf(err_msg_header, "port %d", port); 7479 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 7480 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 7481 } else if (ahci_ctlp) { 7482 cmn_err(CE_WARN, "!ahci%d: %s %s", 7483 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7484 err_msg_header, err_msg); 7485 7486 /* sata trace debug */ 7487 sata_trace_debug(ahci_ctlp->ahcictl_dip, 7488 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 7489 err_msg_header, err_msg); 7490 } else { 7491 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 7492 7493 /* sata trace debug */ 7494 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 7495 } 7496 } 7497 7498 /* 7499 * This routine is to calculate the total number of ports implemented 7500 * by the HBA. 7501 */ 7502 static int 7503 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7504 { 7505 uint8_t i; 7506 int num = 0; 7507 7508 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7509 if (((uint32_t)0x1 << i) & ports_implemented) 7510 num++; 7511 } 7512 7513 return (num); 7514 } 7515 7516 #if AHCI_DEBUG 7517 static void 7518 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7519 { 7520 static char name[16]; 7521 va_list ap; 7522 7523 mutex_enter(&ahci_log_mutex); 7524 7525 va_start(ap, fmt); 7526 if (ahci_ctlp) { 7527 (void) sprintf(name, "ahci%d: ", 7528 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7529 } else { 7530 (void) sprintf(name, "ahci: "); 7531 } 7532 7533 (void) vsprintf(ahci_log_buf, fmt, ap); 7534 va_end(ap); 7535 7536 cmn_err(level, "%s%s", name, ahci_log_buf); 7537 7538 mutex_exit(&ahci_log_mutex); 7539 } 7540 #endif 7541 7542 /* 7543 * quiesce(9E) entry point. 7544 * 7545 * This function is called when the system is single-threaded at high 7546 * PIL with preemption disabled. Therefore, this function must not be 7547 * blocked. 7548 * 7549 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7550 * DDI_FAILURE indicates an error condition and should almost never happen. 7551 */ 7552 static int 7553 ahci_quiesce(dev_info_t *dip) 7554 { 7555 ahci_ctl_t *ahci_ctlp; 7556 ahci_port_t *ahci_portp; 7557 int instance, port; 7558 7559 instance = ddi_get_instance(dip); 7560 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7561 7562 if (ahci_ctlp == NULL) 7563 return (DDI_FAILURE); 7564 7565 #if AHCI_DEBUG 7566 ahci_debug_flags = 0; 7567 #endif 7568 7569 /* disable all the interrupts. */ 7570 ahci_disable_all_intrs(ahci_ctlp); 7571 7572 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7573 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7574 continue; 7575 } 7576 7577 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7578 7579 /* 7580 * Stop the port by clearing PxCMD.ST 7581 * 7582 * Here we must disable the port interrupt because 7583 * ahci_disable_all_intrs only clear GHC.IE, and IS 7584 * register will be still set if PxIE is enabled. 7585 * When ahci shares one IRQ with other drivers, the 7586 * intr handler may claim the intr mistakenly. 7587 */ 7588 ahci_disable_port_intrs(ahci_ctlp, port); 7589 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7590 ahci_portp, port); 7591 } 7592 7593 return (DDI_SUCCESS); 7594 } 7595