1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_drain_ports_taskq(ahci_ctl_t *); 104 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 105 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 107 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 108 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 109 uint8_t, sata_pkt_t *); 110 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 111 uint8_t, sata_pkt_t *); 112 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 113 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 114 static void ahci_copy_ncq_err_page(sata_cmd_t *, 115 struct sata_ncq_error_recovery_page *); 116 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 117 118 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 119 static int ahci_hba_reset(ahci_ctl_t *); 120 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 124 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 125 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 126 uint8_t); 127 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 128 uint8_t, int, int *); 129 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 130 uint32_t, uint32_t, uint32_t, uint32_t); 131 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 132 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 133 uint8_t, sata_pkt_t *); 134 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 135 uint8_t, uint32_t); 136 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 137 uint8_t, uint32_t); 138 static void ahci_events_handler(void *); 139 static void ahci_watchdog_handler(ahci_ctl_t *); 140 141 static uint_t ahci_intr(caddr_t, caddr_t); 142 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 143 static int ahci_add_intrs(ahci_ctl_t *, int); 144 static void ahci_rem_intrs(ahci_ctl_t *); 145 static void ahci_enable_all_intrs(ahci_ctl_t *); 146 static void ahci_disable_all_intrs(ahci_ctl_t *); 147 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 148 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 149 150 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 151 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 154 ahci_port_t *, uint8_t); 155 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, uint32_t); 158 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 159 uint8_t, uint32_t); 160 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 161 162 static int ahci_get_num_implemented_ports(uint32_t); 163 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 164 uint32_t); 165 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 166 #if AHCI_DEBUG 167 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 168 #endif 169 170 171 /* 172 * DMA attributes for the data buffer 173 * 174 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 175 * does not support 64-bit addressing 176 */ 177 static ddi_dma_attr_t buffer_dma_attr = { 178 DMA_ATTR_V0, /* dma_attr_version */ 179 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 180 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 181 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 182 0x2ull, /* dma_attr_align: word aligned */ 183 1, /* dma_attr_burstsizes */ 184 1, /* dma_attr_minxfer */ 185 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 186 0xffffffffull, /* dma_attr_seg */ 187 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 188 512, /* dma_attr_granular */ 189 0, /* dma_attr_flags */ 190 }; 191 192 /* 193 * DMA attributes for the rcvd FIS 194 * 195 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 196 * does not support 64-bit addressing 197 */ 198 static ddi_dma_attr_t rcvd_fis_dma_attr = { 199 DMA_ATTR_V0, /* dma_attr_version */ 200 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 201 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 202 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 203 0x100ull, /* dma_attr_align: 256-byte aligned */ 204 1, /* dma_attr_burstsizes */ 205 1, /* dma_attr_minxfer */ 206 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 207 0xffffffffull, /* dma_attr_seg */ 208 1, /* dma_attr_sgllen */ 209 1, /* dma_attr_granular */ 210 0, /* dma_attr_flags */ 211 }; 212 213 /* 214 * DMA attributes for the command list 215 * 216 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 217 * does not support 64-bit addressing 218 */ 219 static ddi_dma_attr_t cmd_list_dma_attr = { 220 DMA_ATTR_V0, /* dma_attr_version */ 221 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 222 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 223 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 224 0x400ull, /* dma_attr_align: 1K-byte aligned */ 225 1, /* dma_attr_burstsizes */ 226 1, /* dma_attr_minxfer */ 227 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 228 0xffffffffull, /* dma_attr_seg */ 229 1, /* dma_attr_sgllen */ 230 1, /* dma_attr_granular */ 231 0, /* dma_attr_flags */ 232 }; 233 234 /* 235 * DMA attributes for cmd tables 236 * 237 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 238 * does not support 64-bit addressing 239 */ 240 static ddi_dma_attr_t cmd_table_dma_attr = { 241 DMA_ATTR_V0, /* dma_attr_version */ 242 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 243 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 244 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 245 0x80ull, /* dma_attr_align: 128-byte aligned */ 246 1, /* dma_attr_burstsizes */ 247 1, /* dma_attr_minxfer */ 248 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 249 0xffffffffull, /* dma_attr_seg */ 250 1, /* dma_attr_sgllen */ 251 1, /* dma_attr_granular */ 252 0, /* dma_attr_flags */ 253 }; 254 255 256 /* Device access attributes */ 257 static ddi_device_acc_attr_t accattr = { 258 DDI_DEVICE_ATTR_V0, 259 DDI_STRUCTURE_LE_ACC, 260 DDI_STRICTORDER_ACC 261 }; 262 263 264 static struct dev_ops ahcictl_dev_ops = { 265 DEVO_REV, /* devo_rev */ 266 0, /* refcnt */ 267 ahci_getinfo, /* info */ 268 nulldev, /* identify */ 269 nulldev, /* probe */ 270 ahci_attach, /* attach */ 271 ahci_detach, /* detach */ 272 nodev, /* no reset */ 273 (struct cb_ops *)0, /* driver operations */ 274 NULL, /* bus operations */ 275 NULL, /* power */ 276 ahci_quiesce, /* quiesce */ 277 }; 278 279 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 280 SATA_TRAN_HOTPLUG_OPS_REV_1, 281 ahci_tran_hotplug_port_activate, 282 ahci_tran_hotplug_port_deactivate 283 }; 284 285 extern struct mod_ops mod_driverops; 286 287 static struct modldrv modldrv = { 288 &mod_driverops, /* driverops */ 289 ahci_ident, /* short description */ 290 &ahcictl_dev_ops, /* driver ops */ 291 }; 292 293 static struct modlinkage modlinkage = { 294 MODREV_1, 295 &modldrv, 296 NULL 297 }; 298 299 /* The following variables are watchdog handler related */ 300 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 301 static int ahci_watchdog_tick; 302 303 /* 304 * This static variable indicates the size of command table, 305 * and it's changeable with prdt number, which ahci_dma_prdt_number 306 * indicates. 307 */ 308 static size_t ahci_cmd_table_size; 309 310 /* 311 * ahci_dma_prdt_number, ahci_msi_enabled and ahci_64bit_dma_addressed 312 * are global variables, and can be changed via /etc/system file. 313 */ 314 315 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 316 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 317 318 /* AHCI MSI is tunable */ 319 boolean_t ahci_msi_enabled = B_TRUE; 320 321 /* 64-bit dma addressing is tunable */ 322 boolean_t ahci_64bit_dma_addressed = B_TRUE; 323 324 #if AHCI_DEBUG 325 uint32_t ahci_debug_flags = 0; 326 327 /* The following is needed for ahci_log() */ 328 static kmutex_t ahci_log_mutex; 329 static char ahci_log_buf[512]; 330 #endif 331 332 /* Opaque state pointer initialized by ddi_soft_state_init() */ 333 static void *ahci_statep = NULL; 334 335 /* 336 * ahci module initialization. 337 */ 338 int 339 _init(void) 340 { 341 int ret; 342 343 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 344 if (ret != 0) { 345 goto err_out; 346 } 347 348 #if AHCI_DEBUG 349 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 350 #endif 351 352 if ((ret = sata_hba_init(&modlinkage)) != 0) { 353 #if AHCI_DEBUG 354 mutex_destroy(&ahci_log_mutex); 355 #endif 356 ddi_soft_state_fini(&ahci_statep); 357 goto err_out; 358 } 359 360 ret = mod_install(&modlinkage); 361 if (ret != 0) { 362 sata_hba_fini(&modlinkage); 363 #if AHCI_DEBUG 364 mutex_destroy(&ahci_log_mutex); 365 #endif 366 ddi_soft_state_fini(&ahci_statep); 367 goto err_out; 368 } 369 370 /* watchdog tick */ 371 ahci_watchdog_tick = drv_usectohz( 372 (clock_t)ahci_watchdog_timeout * 1000000); 373 return (ret); 374 375 err_out: 376 cmn_err(CE_WARN, "!ahci: Module init failed"); 377 return (ret); 378 } 379 380 /* 381 * ahci module uninitialize. 382 */ 383 int 384 _fini(void) 385 { 386 int ret; 387 388 ret = mod_remove(&modlinkage); 389 if (ret != 0) { 390 return (ret); 391 } 392 393 /* Remove the resources allocated in _init(). */ 394 sata_hba_fini(&modlinkage); 395 #if AHCI_DEBUG 396 mutex_destroy(&ahci_log_mutex); 397 #endif 398 ddi_soft_state_fini(&ahci_statep); 399 400 return (ret); 401 } 402 403 /* 404 * _info entry point 405 */ 406 int 407 _info(struct modinfo *modinfop) 408 { 409 return (mod_info(&modlinkage, modinfop)); 410 } 411 412 /* 413 * The attach entry point for dev_ops. 414 */ 415 static int 416 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 417 { 418 ahci_ctl_t *ahci_ctlp; 419 int instance = ddi_get_instance(dip); 420 int status; 421 int attach_state; 422 uint32_t cap_status, ahci_version; 423 int intr_types; 424 int i; 425 pci_regspec_t *regs; 426 int regs_length; 427 int rnumber; 428 #if AHCI_DEBUG 429 int speed; 430 #endif 431 432 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter"); 433 434 switch (cmd) { 435 case DDI_ATTACH: 436 break; 437 438 case DDI_RESUME: 439 440 /* 441 * During DDI_RESUME, the hardware state of the device 442 * (power may have been removed from the device) must be 443 * restored, allow pending requests to continue, and 444 * service new requests. 445 */ 446 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 447 mutex_enter(&ahci_ctlp->ahcictl_mutex); 448 449 /* Restart watch thread */ 450 if (ahci_ctlp->ahcictl_timeout_id == 0) 451 ahci_ctlp->ahcictl_timeout_id = timeout( 452 (void (*)(void *))ahci_watchdog_handler, 453 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 454 455 mutex_exit(&ahci_ctlp->ahcictl_mutex); 456 457 /* 458 * Re-initialize the controller and enable the interrupts and 459 * restart all the ports. 460 * 461 * Note that so far we don't support hot-plug during 462 * suspend/resume. 463 */ 464 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 465 AHCIDBG0(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 466 "Failed to initialize the controller " 467 "during DDI_RESUME"); 468 return (DDI_FAILURE); 469 } 470 471 mutex_enter(&ahci_ctlp->ahcictl_mutex); 472 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 473 mutex_exit(&ahci_ctlp->ahcictl_mutex); 474 475 return (DDI_SUCCESS); 476 477 default: 478 return (DDI_FAILURE); 479 } 480 481 attach_state = AHCI_ATTACH_STATE_NONE; 482 483 /* Allocate soft state */ 484 status = ddi_soft_state_zalloc(ahci_statep, instance); 485 if (status != DDI_SUCCESS) { 486 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 487 instance); 488 goto err_out; 489 } 490 491 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 492 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 493 ahci_ctlp->ahcictl_dip = dip; 494 495 /* Initialize the cport/port mapping */ 496 for (i = 0; i < AHCI_MAX_PORTS; i++) { 497 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 498 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 499 } 500 501 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 502 503 /* 504 * Now map the AHCI base address; which includes global 505 * registers and port control registers 506 * 507 * According to the spec, the AHCI Base Address is BAR5, 508 * but BAR0-BAR4 are optional, so we need to check which 509 * rnumber is used for BAR5. 510 */ 511 512 /* 513 * search through DDI "reg" property for the AHCI register set 514 */ 515 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 516 DDI_PROP_DONTPASS, "reg", (int **)®s, 517 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 518 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 519 instance); 520 goto err_out; 521 } 522 523 /* AHCI Base Address is located at 0x24 offset */ 524 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 525 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 526 == AHCI_PCI_RNUM) 527 break; 528 } 529 530 ddi_prop_free(regs); 531 532 if (rnumber == regs_length) { 533 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 534 instance); 535 goto err_out; 536 } 537 538 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 539 540 status = ddi_regs_map_setup(dip, 541 rnumber, 542 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 543 0, 544 0, 545 &accattr, 546 &ahci_ctlp->ahcictl_ahci_acc_handle); 547 if (status != DDI_SUCCESS) { 548 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 549 instance); 550 goto err_out; 551 } 552 553 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 554 555 /* Get the AHCI version information */ 556 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 557 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 558 559 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 560 (ahci_version & 0xffff0000) >> 16, 561 ((ahci_version & 0x0000ff00) >> 4 | 562 (ahci_version & 0x000000ff))); 563 564 /* We don't support controllers whose versions are lower than 1.0 */ 565 if (!(ahci_version & 0xffff0000)) { 566 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 567 "than version 1.0", instance); 568 goto err_out; 569 } 570 571 /* Get the HBA capabilities information */ 572 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 573 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 574 575 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 576 cap_status); 577 578 #if AHCI_DEBUG 579 /* Get the interface speed supported by the HBA */ 580 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 581 if (speed == 0x01) { 582 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 583 "hba interface speed support: Gen 1 (1.5Gbps)"); 584 } else if (speed == 0x10) { 585 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 586 "hba interface speed support: Gen 2 (3 Gbps)"); 587 } else if (speed == 0x11) { 588 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 589 "hba interface speed support: Gen 3 (6 Gbps)"); 590 } 591 #endif 592 593 /* Get the number of command slots supported by the HBA */ 594 ahci_ctlp->ahcictl_num_cmd_slots = 595 ((cap_status & AHCI_HBA_CAP_NCS) >> 596 AHCI_HBA_CAP_NCS_SHIFT) + 1; 597 598 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 599 ahci_ctlp->ahcictl_num_cmd_slots); 600 601 /* Get the bit map which indicates ports implemented by the HBA */ 602 ahci_ctlp->ahcictl_ports_implemented = 603 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 604 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 605 606 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 607 ahci_ctlp->ahcictl_ports_implemented); 608 609 /* 610 * According to the AHCI spec, CAP.NP should indicate the maximum 611 * number of ports supported by the HBA silicon, but we found 612 * this value of ICH8 chipset only indicates the number of ports 613 * implemented (exposed) by it. Therefore, the driver should calculate 614 * the potential maximum value by checking PI register, and use 615 * the maximum of this value and CAP.NP. 616 */ 617 ahci_ctlp->ahcictl_num_ports = max( 618 (cap_status & AHCI_HBA_CAP_NP) + 1, 619 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 620 621 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 622 ahci_ctlp->ahcictl_num_ports); 623 624 /* Get the number of implemented ports by the HBA */ 625 ahci_ctlp->ahcictl_num_implemented_ports = 626 ahci_get_num_implemented_ports( 627 ahci_ctlp->ahcictl_ports_implemented); 628 629 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 630 "hba number of implemented ports: %d", 631 ahci_ctlp->ahcictl_num_implemented_ports); 632 633 /* Check whether HBA supports 64bit DMA addressing */ 634 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 635 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 636 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 637 "hba does not support 64-bit addressing"); 638 } 639 640 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 641 != DDI_SUCCESS) { 642 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 643 instance); 644 goto err_out; 645 } 646 647 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 648 649 /* 650 * Check the pci configuration space, and set caps. We also 651 * handle the hardware defect in this function. 652 * 653 * For example, force ATI SB600/SB700/SB750 to use 32-bit dma 654 * addressing since it doesn't support 64-bit dma though their 655 * registers declare they support. 656 */ 657 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 658 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 659 instance); 660 goto err_out; 661 } 662 663 /* 664 * Disable the whole controller interrupts before adding 665 * interrupt handlers(s). 666 */ 667 ahci_disable_all_intrs(ahci_ctlp); 668 669 /* Get supported interrupt types */ 670 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 671 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 672 instance); 673 goto err_out; 674 } 675 676 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 677 "ddi_intr_get_supported_types() returned: 0x%x", 678 intr_types); 679 680 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 681 /* 682 * Try MSI first, but fall back to FIXED if failed 683 */ 684 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 685 DDI_SUCCESS) { 686 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 687 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 688 "Using MSI interrupt type"); 689 goto intr_done; 690 } 691 692 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 693 "MSI registration failed, " 694 "trying FIXED interrupts"); 695 } 696 697 if (intr_types & DDI_INTR_TYPE_FIXED) { 698 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 699 DDI_SUCCESS) { 700 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 701 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 702 "Using FIXED interrupt type"); 703 goto intr_done; 704 } 705 706 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 707 "FIXED interrupt registration failed"); 708 } 709 710 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 711 712 goto err_out; 713 714 intr_done: 715 716 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 717 718 /* Initialize the controller mutex */ 719 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 720 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 721 722 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 723 724 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 725 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 726 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 727 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 728 } 729 730 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 731 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 732 sizeof (ahci_prdt_item_t)); 733 734 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 735 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 736 " ahci_cmd_table_size is 0x%x", 737 ahci_dma_prdt_number, ahci_cmd_table_size); 738 739 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 740 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 741 ahci_dma_prdt_number; 742 743 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 744 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 745 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 746 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 747 748 if ((ahci_64bit_dma_addressed == B_FALSE) || 749 (ahci_ctlp->ahcictl_cap & AHCI_CAP_32BIT_DMA)) { 750 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 751 0xffffffffull; 752 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 753 0xffffffffull; 754 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 755 0xffffffffull; 756 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 757 0xffffffffull; 758 } 759 760 /* Allocate the ports structure */ 761 status = ahci_alloc_ports_state(ahci_ctlp); 762 if (status != AHCI_SUCCESS) { 763 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 764 instance); 765 goto err_out; 766 } 767 768 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 769 770 /* 771 * Initialize the controller and ports. 772 */ 773 status = ahci_initialize_controller(ahci_ctlp); 774 if (status != AHCI_SUCCESS) { 775 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 776 instance); 777 goto err_out; 778 } 779 780 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 781 782 /* Start one thread to check packet timeouts */ 783 ahci_ctlp->ahcictl_timeout_id = timeout( 784 (void (*)(void *))ahci_watchdog_handler, 785 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 786 787 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 788 789 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 790 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 791 instance); 792 goto err_out; 793 } 794 795 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 796 797 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!"); 798 799 return (DDI_SUCCESS); 800 801 err_out: 802 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 803 mutex_enter(&ahci_ctlp->ahcictl_mutex); 804 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 805 ahci_ctlp->ahcictl_timeout_id = 0; 806 mutex_exit(&ahci_ctlp->ahcictl_mutex); 807 } 808 809 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 810 ahci_uninitialize_controller(ahci_ctlp); 811 } 812 813 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 814 ahci_dealloc_ports_state(ahci_ctlp); 815 } 816 817 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 818 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 819 } 820 821 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 822 ahci_rem_intrs(ahci_ctlp); 823 } 824 825 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 826 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 827 } 828 829 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 830 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 831 } 832 833 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 834 ddi_soft_state_free(ahci_statep, instance); 835 } 836 837 return (DDI_FAILURE); 838 } 839 840 /* 841 * The detach entry point for dev_ops. 842 */ 843 static int 844 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 845 { 846 ahci_ctl_t *ahci_ctlp; 847 int instance; 848 int ret; 849 850 instance = ddi_get_instance(dip); 851 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 852 853 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter"); 854 855 switch (cmd) { 856 case DDI_DETACH: 857 858 /* disable the interrupts for an uninterrupted detach */ 859 mutex_enter(&ahci_ctlp->ahcictl_mutex); 860 ahci_disable_all_intrs(ahci_ctlp); 861 mutex_exit(&ahci_ctlp->ahcictl_mutex); 862 863 /* unregister from the sata framework. */ 864 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 865 if (ret != AHCI_SUCCESS) { 866 mutex_enter(&ahci_ctlp->ahcictl_mutex); 867 ahci_enable_all_intrs(ahci_ctlp); 868 mutex_exit(&ahci_ctlp->ahcictl_mutex); 869 return (DDI_FAILURE); 870 } 871 872 mutex_enter(&ahci_ctlp->ahcictl_mutex); 873 874 /* stop the watchdog handler */ 875 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 876 ahci_ctlp->ahcictl_timeout_id = 0; 877 878 mutex_exit(&ahci_ctlp->ahcictl_mutex); 879 880 /* uninitialize the controller */ 881 ahci_uninitialize_controller(ahci_ctlp); 882 883 /* remove the interrupts */ 884 ahci_rem_intrs(ahci_ctlp); 885 886 /* deallocate the ports structures */ 887 ahci_dealloc_ports_state(ahci_ctlp); 888 889 /* destroy mutex */ 890 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 891 892 /* teardown the pci config */ 893 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 894 895 /* remove the reg maps. */ 896 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 897 898 /* free the soft state. */ 899 ddi_soft_state_free(ahci_statep, instance); 900 901 return (DDI_SUCCESS); 902 903 case DDI_SUSPEND: 904 905 /* 906 * The steps associated with suspension must include putting 907 * the underlying device into a quiescent state so that it 908 * will not generate interrupts or modify or access memory. 909 */ 910 mutex_enter(&ahci_ctlp->ahcictl_mutex); 911 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 912 mutex_exit(&ahci_ctlp->ahcictl_mutex); 913 return (DDI_SUCCESS); 914 } 915 916 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 917 918 /* stop the watchdog handler */ 919 if (ahci_ctlp->ahcictl_timeout_id) { 920 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 921 ahci_ctlp->ahcictl_timeout_id = 0; 922 } 923 924 mutex_exit(&ahci_ctlp->ahcictl_mutex); 925 926 /* 927 * drain the taskq 928 */ 929 ahci_drain_ports_taskq(ahci_ctlp); 930 931 /* 932 * Disable the interrupts and stop all the ports. 933 */ 934 ahci_uninitialize_controller(ahci_ctlp); 935 936 return (DDI_SUCCESS); 937 938 default: 939 return (DDI_FAILURE); 940 } 941 } 942 943 /* 944 * The info entry point for dev_ops. 945 * 946 */ 947 static int 948 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 949 void *arg, void **result) 950 { 951 #ifndef __lock_lint 952 _NOTE(ARGUNUSED(dip)) 953 #endif /* __lock_lint */ 954 955 ahci_ctl_t *ahci_ctlp; 956 int instance; 957 dev_t dev; 958 959 dev = (dev_t)arg; 960 instance = getminor(dev); 961 962 switch (infocmd) { 963 case DDI_INFO_DEVT2DEVINFO: 964 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 965 if (ahci_ctlp != NULL) { 966 *result = ahci_ctlp->ahcictl_dip; 967 return (DDI_SUCCESS); 968 } else { 969 *result = NULL; 970 return (DDI_FAILURE); 971 } 972 case DDI_INFO_DEVT2INSTANCE: 973 *(int *)result = instance; 974 break; 975 default: 976 break; 977 } 978 979 return (DDI_SUCCESS); 980 } 981 982 /* 983 * Registers the ahci with sata framework. 984 */ 985 static int 986 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 987 { 988 struct sata_hba_tran *sata_hba_tran; 989 990 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 991 "ahci_register_sata_hba_tran enter"); 992 993 mutex_enter(&ahci_ctlp->ahcictl_mutex); 994 995 /* Allocate memory for the sata_hba_tran */ 996 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 997 998 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 999 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1000 sata_hba_tran->sata_tran_hba_dma_attr = 1001 &ahci_ctlp->ahcictl_buffer_dma_attr; 1002 1003 /* Report the number of implemented ports */ 1004 sata_hba_tran->sata_tran_hba_num_cports = 1005 ahci_ctlp->ahcictl_num_implemented_ports; 1006 1007 /* Support ATAPI device */ 1008 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1009 1010 /* Get the data transfer capability for PIO command by the HBA */ 1011 if (cap_status & AHCI_HBA_CAP_PMD) { 1012 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1013 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1014 "DRQ block data transfer for PIO command protocol"); 1015 } 1016 1017 /* 1018 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1019 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1020 * QUEUED (EXT), and SERVICE commands). Queued operations are 1021 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1022 * QUEUED commands when the HBA and device support native command 1023 * queuing(NCQ). 1024 * 1025 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1026 * CAP register of the HBA indicates NCQ is supported. 1027 * 1028 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1029 * set because the previous register content of PxCI can be re-written 1030 * in the register write. 1031 */ 1032 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1033 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1034 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1035 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1036 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1037 "Command Queuing"); 1038 } 1039 1040 /* Report the number of command slots */ 1041 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1042 1043 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1044 sata_hba_tran->sata_tran_start = ahci_tran_start; 1045 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1046 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1047 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1048 #ifdef __lock_lint 1049 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1050 #endif 1051 /* 1052 * When SATA framework adds support for pwrmgt the 1053 * pwrmgt_ops needs to be updated 1054 */ 1055 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1056 sata_hba_tran->sata_tran_ioctl = NULL; 1057 1058 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1059 1060 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1061 1062 /* Attach it to SATA framework */ 1063 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1064 != DDI_SUCCESS) { 1065 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1066 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1067 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1068 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1069 return (AHCI_FAILURE); 1070 } 1071 1072 return (AHCI_SUCCESS); 1073 } 1074 1075 /* 1076 * Unregisters the ahci with sata framework. 1077 */ 1078 static int 1079 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1080 { 1081 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 1082 "ahci_unregister_sata_hba_tran enter"); 1083 1084 /* Detach from the SATA framework. */ 1085 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1086 DDI_SUCCESS) { 1087 return (AHCI_FAILURE); 1088 } 1089 1090 /* Deallocate sata_hba_tran. */ 1091 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1092 sizeof (sata_hba_tran_t)); 1093 1094 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1095 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1096 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1097 1098 return (AHCI_SUCCESS); 1099 } 1100 1101 /* 1102 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1103 * port status registers and an attached device type via sata_device 1104 * structure. 1105 * 1106 * We return the cached information from a previous hardware probe. The 1107 * actual hardware probing itself was done either from within 1108 * ahci_initialize_controller() during the driver attach or from a phy 1109 * ready change interrupt handler. 1110 */ 1111 static int 1112 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1113 { 1114 ahci_ctl_t *ahci_ctlp; 1115 ahci_port_t *ahci_portp; 1116 uint8_t cport = sd->satadev_addr.cport; 1117 #if AHCI_DEBUG 1118 uint8_t pmport = sd->satadev_addr.pmport; 1119 uint8_t qual = sd->satadev_addr.qual; 1120 #endif 1121 uint8_t device_type; 1122 uint32_t port_state; 1123 uint8_t port; 1124 1125 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1126 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1127 1128 AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp, 1129 "ahci_tran_probe_port enter: cport: %d, " 1130 "pmport: %d, qual: %d", cport, pmport, qual); 1131 1132 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1133 1134 mutex_enter(&ahci_portp->ahciport_mutex); 1135 1136 port_state = ahci_portp->ahciport_port_state; 1137 switch (port_state) { 1138 1139 case SATA_PSTATE_FAILED: 1140 sd->satadev_state = SATA_PSTATE_FAILED; 1141 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1142 "ahci_tran_probe_port: port %d PORT FAILED", port); 1143 goto out; 1144 1145 case SATA_PSTATE_SHUTDOWN: 1146 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1147 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1148 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1149 goto out; 1150 1151 case SATA_PSTATE_PWROFF: 1152 sd->satadev_state = SATA_PSTATE_PWROFF; 1153 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1154 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1155 goto out; 1156 1157 case SATA_PSTATE_PWRON: 1158 sd->satadev_state = SATA_PSTATE_PWRON; 1159 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1160 "ahci_tran_probe_port: port %d PORT PWRON", port); 1161 break; 1162 1163 default: 1164 sd->satadev_state = port_state; 1165 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 1166 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1167 port, port_state); 1168 break; 1169 } 1170 1171 device_type = ahci_portp->ahciport_device_type; 1172 1173 switch (device_type) { 1174 1175 case SATA_DTYPE_ATADISK: 1176 sd->satadev_type = SATA_DTYPE_ATADISK; 1177 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1178 "ahci_tran_probe_port: port %d DISK found", port); 1179 break; 1180 1181 case SATA_DTYPE_ATAPI: 1182 /* 1183 * HBA driver only knows it's an ATAPI device, and don't know 1184 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1185 * type need to be determined by checking IDENTIFY PACKET 1186 * DEVICE data 1187 */ 1188 sd->satadev_type = SATA_DTYPE_ATAPI; 1189 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1190 "ahci_tran_probe_port: port %d ATAPI found", port); 1191 break; 1192 1193 case SATA_DTYPE_PMULT: 1194 sd->satadev_type = SATA_DTYPE_PMULT; 1195 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1196 "ahci_tran_probe_port: port %d Port Multiplier found", 1197 port); 1198 break; 1199 1200 case SATA_DTYPE_UNKNOWN: 1201 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1202 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1203 "ahci_tran_probe_port: port %d Unknown device found", port); 1204 break; 1205 1206 default: 1207 /* we don't support any other device types */ 1208 sd->satadev_type = SATA_DTYPE_NONE; 1209 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1210 "ahci_tran_probe_port: port %d No device found", port); 1211 break; 1212 } 1213 1214 out: 1215 ahci_update_sata_registers(ahci_ctlp, port, sd); 1216 mutex_exit(&ahci_portp->ahciport_mutex); 1217 1218 return (SATA_SUCCESS); 1219 } 1220 1221 /* 1222 * There are four operation modes in sata framework: 1223 * SATA_OPMODE_INTERRUPTS 1224 * SATA_OPMODE_POLLING 1225 * SATA_OPMODE_ASYNCH 1226 * SATA_OPMODE_SYNCH 1227 * 1228 * Their combined meanings as following: 1229 * 1230 * SATA_OPMODE_SYNCH 1231 * The command has to be completed before sata_tran_start functions returns. 1232 * Either interrupts or polling could be used - it's up to the driver. 1233 * Mode used currently for internal, sata-module initiated operations. 1234 * 1235 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1236 * It is the same as the one above. 1237 * 1238 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1239 * The command has to be completed before sata_tran_start function returns. 1240 * No interrupt used, polling only. This should be the mode used for scsi 1241 * packets with FLAG_NOINTR. 1242 * 1243 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1244 * The command may be queued (callback function specified). Interrupts could 1245 * be used. It's normal operation mode. 1246 */ 1247 /* 1248 * Called by sata framework to transport a sata packet down stream. 1249 */ 1250 static int 1251 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1252 { 1253 ahci_ctl_t *ahci_ctlp; 1254 ahci_port_t *ahci_portp; 1255 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1256 uint8_t port; 1257 1258 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1259 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1260 1261 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, 1262 "ahci_tran_start enter: cport %d satapkt 0x%p", 1263 cport, (void *)spkt); 1264 1265 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1266 1267 mutex_enter(&ahci_portp->ahciport_mutex); 1268 1269 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1270 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1271 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1272 /* 1273 * In case the targer driver would send the packet before 1274 * sata framework can have the opportunity to process those 1275 * event reports. 1276 */ 1277 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1278 spkt->satapkt_device.satadev_state = 1279 ahci_portp->ahciport_port_state; 1280 ahci_update_sata_registers(ahci_ctlp, port, 1281 &spkt->satapkt_device); 1282 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1283 "ahci_tran_start returning PORT_ERROR while " 1284 "port in FAILED/SHUTDOWN/PWROFF state: " 1285 "port: %d", port); 1286 mutex_exit(&ahci_portp->ahciport_mutex); 1287 return (SATA_TRAN_PORT_ERROR); 1288 } 1289 1290 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1291 /* 1292 * ahci_intr_phyrdy_change() may have rendered it to 1293 * SATA_DTYPE_NONE. 1294 */ 1295 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1296 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1297 spkt->satapkt_device.satadev_state = 1298 ahci_portp->ahciport_port_state; 1299 ahci_update_sata_registers(ahci_ctlp, port, 1300 &spkt->satapkt_device); 1301 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1302 "ahci_tran_start returning PORT_ERROR while " 1303 "no device attached: port: %d", port); 1304 mutex_exit(&ahci_portp->ahciport_mutex); 1305 return (SATA_TRAN_PORT_ERROR); 1306 } 1307 1308 /* 1309 * SATA HBA driver should remember that a device was reset and it 1310 * is supposed to reject any packets which do not specify either 1311 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1312 * 1313 * This is to prevent a race condition when a device was arbitrarily 1314 * reset by the HBA driver (and lost it's setting) and a target 1315 * driver sending some commands to a device before the sata framework 1316 * has a chance to restore the device setting (such as cache enable/ 1317 * disable or other resettable stuff). 1318 */ 1319 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1320 ahci_portp->ahciport_reset_in_progress = 0; 1321 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1322 "ahci_tran_start clearing the " 1323 "reset_in_progress for port: %d", port); 1324 } 1325 1326 if (ahci_portp->ahciport_reset_in_progress && 1327 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1328 ! ddi_in_panic()) { 1329 spkt->satapkt_reason = SATA_PKT_BUSY; 1330 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1331 "ahci_tran_start returning BUSY while " 1332 "reset in progress: port: %d", port); 1333 mutex_exit(&ahci_portp->ahciport_mutex); 1334 return (SATA_TRAN_BUSY); 1335 } 1336 1337 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1338 spkt->satapkt_reason = SATA_PKT_BUSY; 1339 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1340 "ahci_tran_start returning BUSY while " 1341 "mopping in progress: port: %d", port); 1342 mutex_exit(&ahci_portp->ahciport_mutex); 1343 return (SATA_TRAN_BUSY); 1344 } 1345 1346 if (spkt->satapkt_op_mode & 1347 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1348 /* 1349 * If a SYNC command to be executed in interrupt context, 1350 * bounce it back to sata module. 1351 */ 1352 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1353 servicing_interrupt()) { 1354 spkt->satapkt_reason = SATA_PKT_BUSY; 1355 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1356 "ahci_tran_start returning BUSY while " 1357 "sending SYNC mode under interrupt context: " 1358 "port : %d", port); 1359 mutex_exit(&ahci_portp->ahciport_mutex); 1360 return (SATA_TRAN_BUSY); 1361 } 1362 1363 /* We need to do the sync start now */ 1364 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1365 spkt) == AHCI_FAILURE) { 1366 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1367 "return QUEUE_FULL: port %d", port); 1368 mutex_exit(&ahci_portp->ahciport_mutex); 1369 return (SATA_TRAN_QUEUE_FULL); 1370 } 1371 } else { 1372 /* Async start, using interrupt */ 1373 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1374 == AHCI_FAILURE) { 1375 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1376 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1377 "returning QUEUE_FULL: port %d", port); 1378 mutex_exit(&ahci_portp->ahciport_mutex); 1379 return (SATA_TRAN_QUEUE_FULL); 1380 } 1381 } 1382 1383 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1384 "sata tran accepted: port %d", port); 1385 1386 mutex_exit(&ahci_portp->ahciport_mutex); 1387 return (SATA_TRAN_ACCEPTED); 1388 } 1389 1390 /* 1391 * SATA_OPMODE_SYNCH flag is set 1392 * 1393 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1394 * without interrupt, otherwise we can still use the interrupt. 1395 * 1396 * WARNING!!! ahciport_mutex should be acquired before the function 1397 * is called. 1398 */ 1399 static int 1400 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1401 uint8_t port, sata_pkt_t *spkt) 1402 { 1403 int pkt_timeout_ticks; 1404 uint32_t timeout_tags; 1405 int rval; 1406 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1407 1408 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1409 "port %d spkt 0x%p", port, spkt); 1410 1411 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1412 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1413 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1414 port, spkt)) == AHCI_FAILURE) { 1415 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1416 return (rval); 1417 } 1418 1419 pkt_timeout_ticks = 1420 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1421 1422 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1423 mutex_exit(&ahci_portp->ahciport_mutex); 1424 1425 /* Simulate the interrupt */ 1426 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1427 1428 drv_usecwait(AHCI_1MS_USECS); 1429 1430 mutex_enter(&ahci_portp->ahciport_mutex); 1431 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1432 if (pkt_timeout_ticks < 0) { 1433 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1434 "port %d satapkt 0x%p timed out\n", 1435 instance, port, (void *)spkt); 1436 timeout_tags = (0x1 << rval); 1437 mutex_exit(&ahci_portp->ahciport_mutex); 1438 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1439 port, timeout_tags); 1440 mutex_enter(&ahci_portp->ahciport_mutex); 1441 } 1442 } 1443 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1444 return (AHCI_SUCCESS); 1445 1446 } else { 1447 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1448 port, spkt)) == AHCI_FAILURE) 1449 return (rval); 1450 1451 #if AHCI_DEBUG 1452 /* 1453 * Note that the driver always uses the slot 0 to deliver 1454 * REQUEST SENSE or READ LOG EXT command 1455 */ 1456 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1457 ASSERT(rval == 0); 1458 #endif 1459 1460 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1461 cv_wait(&ahci_portp->ahciport_cv, 1462 &ahci_portp->ahciport_mutex); 1463 1464 return (AHCI_SUCCESS); 1465 } 1466 } 1467 1468 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1469 if (satapkt) { \ 1470 satapkt->satapkt_reason = reason; \ 1471 /* \ 1472 * We set the satapkt_reason in both sync and \ 1473 * non-sync cases. \ 1474 */ \ 1475 } \ 1476 if (satapkt && \ 1477 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1478 satapkt->satapkt_comp) { \ 1479 mutex_exit(&ahci_portp->ahciport_mutex); \ 1480 (*satapkt->satapkt_comp)(satapkt); \ 1481 mutex_enter(&ahci_portp->ahciport_mutex); \ 1482 } else { \ 1483 if (satapkt && \ 1484 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1485 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1486 cv_broadcast(&ahci_portp->ahciport_cv); \ 1487 } 1488 1489 /* 1490 * Searches for and claims a free command slot. 1491 * 1492 * Returns: 1493 * 1494 * AHCI_FAILURE if failed 1495 * 1. if no empty slot left 1496 * 2. non-queued command requested while queued command(s) is outstanding 1497 * 3. queued command requested whild non-queued command(s) is outstanding 1498 * 4. HBA doesn't support multiple-use of command list while already a 1499 * non-queued command is oustanding 1500 * 1501 * claimed slot number if succeeded 1502 * 1503 * NOTE: it will always return slot 0 during error recovery process for 1504 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1505 * 1506 * WARNING!!! ahciport_mutex should be acquired before the function 1507 * is called. 1508 */ 1509 static int 1510 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1511 ahci_port_t *ahci_portp, int command_type) 1512 { 1513 uint32_t free_slots; 1514 int slot; 1515 1516 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1517 "ahciport_pending_tags = 0x%x " 1518 "ahciport_pending_ncq_tags = 0x%x", 1519 ahci_portp->ahciport_pending_tags, 1520 ahci_portp->ahciport_pending_ncq_tags); 1521 1522 /* 1523 * According to the AHCI spec, system software is responsible to 1524 * ensure that queued and non-queued commands are not mixed in 1525 * the command list. 1526 */ 1527 if (command_type == AHCI_NON_NCQ_CMD) { 1528 /* Non-NCQ command request */ 1529 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1530 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1531 "ahci_claim_free_slot: there is still pending " 1532 "queued command(s) in the command list, " 1533 "so no available slot for the non-queued " 1534 "command"); 1535 return (AHCI_FAILURE); 1536 } 1537 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1538 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1539 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 1540 "ahci_claim_free_slot: HBA cannot support multiple-" 1541 "use of the command list for non-queued commands"); 1542 return (AHCI_FAILURE); 1543 } 1544 free_slots = (~ahci_portp->ahciport_pending_tags) & 1545 AHCI_SLOT_MASK(ahci_ctlp); 1546 } else if (command_type == AHCI_NCQ_CMD) { 1547 /* NCQ command request */ 1548 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1549 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1550 "ahci_claim_free_slot: there is still pending " 1551 "non-queued command(s) in the command list, " 1552 "so no available slot for the queued command"); 1553 return (AHCI_FAILURE); 1554 } 1555 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1556 AHCI_NCQ_SLOT_MASK(ahci_portp); 1557 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1558 /* Error retrieval command request */ 1559 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 1560 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1561 "SENSE or READ LOG EXT command"); 1562 slot = 0; 1563 goto out; 1564 } 1565 1566 slot = ddi_ffs(free_slots) - 1; 1567 if (slot == -1) { 1568 AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp, 1569 "ahci_claim_free_slot: no empty slots"); 1570 return (AHCI_FAILURE); 1571 } 1572 1573 /* 1574 * According to the AHCI spec, to allow a simple mechanism for the 1575 * HBA to map command list slots to queue entries, software must 1576 * match the tag number it uses to the slot it is placing the command 1577 * in. For example, if a queued command is placed in slot 5, the tag 1578 * for that command must be 5. 1579 */ 1580 if (command_type == AHCI_NCQ_CMD) { 1581 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1582 } 1583 1584 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1585 1586 out: 1587 AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp, 1588 "ahci_claim_free_slot: found slot: 0x%x", slot); 1589 1590 return (slot); 1591 } 1592 1593 /* 1594 * Builds the Command Table for the sata packet and delivers it to controller. 1595 * 1596 * Returns: 1597 * slot number if we can obtain a slot successfully 1598 * otherwise, return AHCI_FAILURE 1599 * 1600 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1601 */ 1602 static int 1603 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1604 uint8_t port, sata_pkt_t *spkt) 1605 { 1606 int cmd_slot; 1607 sata_cmd_t *scmd; 1608 ahci_fis_h2d_register_t *h2d_register_fisp; 1609 ahci_cmd_table_t *cmd_table; 1610 ahci_cmd_header_t *cmd_header; 1611 int ncookies; 1612 int i; 1613 int command_type = AHCI_NON_NCQ_CMD; 1614 int ncq_qdepth; 1615 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1616 #if AHCI_DEBUG 1617 uint32_t *ptr; 1618 uint8_t *ptr2; 1619 #endif 1620 1621 spkt->satapkt_reason = SATA_PKT_BUSY; 1622 1623 scmd = &spkt->satapkt_cmd; 1624 1625 /* Check if the command is a NCQ command */ 1626 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1627 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1628 command_type = AHCI_NCQ_CMD; 1629 1630 /* 1631 * When NCQ is support, system software must determine the 1632 * maximum tag allowed by the device and the HBA, and it 1633 * must use a value not beyond of the lower bound of the two. 1634 * 1635 * Sata module is going to calculate the qdepth and send 1636 * down to HBA driver via sata_cmd. 1637 */ 1638 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1639 1640 /* 1641 * At the moment, the driver doesn't support the dynamic 1642 * setting of the maximum ncq depth, and the value can be 1643 * set either during the attach or after hot-plug insertion. 1644 */ 1645 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1646 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1647 AHCIDBG2(AHCIDBG_NCQ, ahci_ctlp, 1648 "ahci_deliver_satapkt: port %d the max tags for " 1649 "NCQ command is %d", port, ncq_qdepth); 1650 } else { 1651 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1652 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1653 " port %d the max tag for NCQ command is " 1654 "requested to change from %d to %d, at the" 1655 " moment the driver doesn't support the " 1656 "dynamic change so it's going to " 1657 "still use the previous tag value", 1658 instance, port, 1659 ahci_portp->ahciport_max_ncq_tags, 1660 ncq_qdepth); 1661 } 1662 } 1663 } 1664 1665 /* Check if the command is an error retrieval command */ 1666 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1667 command_type = AHCI_ERR_RETRI_CMD; 1668 1669 /* Check if there is an empty command slot */ 1670 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1671 if (cmd_slot == AHCI_FAILURE) { 1672 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free command slot"); 1673 return (AHCI_FAILURE); 1674 } 1675 1676 AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1677 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1678 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1679 cmd_slot, port, (void *)spkt); 1680 1681 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1682 bzero((void *)cmd_table, ahci_cmd_table_size); 1683 1684 /* For data transfer operations, it is the H2D Register FIS */ 1685 h2d_register_fisp = 1686 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1687 1688 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1689 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1690 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1691 SET_FIS_PMP(h2d_register_fisp, 1692 spkt->satapkt_device.satadev_addr.pmport); 1693 } 1694 1695 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1696 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1697 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1698 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1699 1700 switch (scmd->satacmd_addr_type) { 1701 1702 case 0: 1703 /* 1704 * satacmd_addr_type will be 0 for the commands below: 1705 * ATAPI command 1706 * SATAC_IDLE_IM 1707 * SATAC_STANDBY_IM 1708 * SATAC_DOWNLOAD_MICROCODE 1709 * SATAC_FLUSH_CACHE 1710 * SATAC_SET_FEATURES 1711 * SATAC_SMART 1712 * SATAC_ID_PACKET_DEVICE 1713 * SATAC_ID_DEVICE 1714 */ 1715 /* FALLTHRU */ 1716 1717 case ATA_ADDR_LBA: 1718 /* FALLTHRU */ 1719 1720 case ATA_ADDR_LBA28: 1721 /* LBA[7:0] */ 1722 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1723 1724 /* LBA[15:8] */ 1725 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1726 1727 /* LBA[23:16] */ 1728 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1729 1730 /* LBA [27:24] (also called dev_head) */ 1731 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1732 1733 break; 1734 1735 case ATA_ADDR_LBA48: 1736 /* LBA[7:0] */ 1737 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1738 1739 /* LBA[15:8] */ 1740 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1741 1742 /* LBA[23:16] */ 1743 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1744 1745 /* LBA [31:24] */ 1746 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1747 scmd->satacmd_lba_low_msb); 1748 1749 /* LBA [39:32] */ 1750 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1751 scmd->satacmd_lba_mid_msb); 1752 1753 /* LBA [47:40] */ 1754 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1755 scmd->satacmd_lba_high_msb); 1756 1757 /* Set dev_head */ 1758 SET_FIS_DEV_HEAD(h2d_register_fisp, 1759 scmd->satacmd_device_reg); 1760 1761 /* Set the extended sector count and features */ 1762 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1763 scmd->satacmd_sec_count_msb); 1764 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1765 scmd->satacmd_features_reg_ext); 1766 break; 1767 } 1768 1769 /* 1770 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1771 * filled into features field, and sector count 8:15 is filled into 1772 * features (exp) field. TAG is filled into sector field. 1773 */ 1774 if (command_type == AHCI_NCQ_CMD) { 1775 SET_FIS_FEATURES(h2d_register_fisp, 1776 scmd->satacmd_sec_count_lsb); 1777 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1778 scmd->satacmd_sec_count_msb); 1779 1780 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1781 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1782 } 1783 1784 ncookies = scmd->satacmd_num_dma_cookies; 1785 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1786 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1787 ncookies, ahci_dma_prdt_number); 1788 1789 ASSERT(ncookies <= ahci_dma_prdt_number); 1790 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1791 1792 /* *** now fill the scatter gather list ******* */ 1793 for (i = 0; i < ncookies; i++) { 1794 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1795 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1796 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1797 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1798 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1799 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1800 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1801 scmd->satacmd_dma_cookie_list[i].dmac_size; 1802 } 1803 1804 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1805 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1806 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1807 1808 /* The ACMD field is filled in for ATAPI command */ 1809 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1810 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1811 SATA_ATAPI_MAX_CDB_LEN); 1812 } 1813 1814 /* Set Command Header in Command List */ 1815 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1816 BZERO_DESCR_INFO(cmd_header); 1817 BZERO_PRD_BYTE_COUNT(cmd_header); 1818 1819 /* Set the number of entries in the PRD table */ 1820 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1821 1822 /* Set the length of the command in the CFIS area */ 1823 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1824 1825 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1826 "sata_data_direction = 0x%x", 1827 scmd->satacmd_flags.sata_data_direction); 1828 1829 /* Set A bit if it is an ATAPI command */ 1830 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1831 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1832 1833 /* Set W bit if data is going to the device */ 1834 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1835 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1836 1837 /* 1838 * Set the prefetchable bit - this bit is only valid if the PRDTL 1839 * field is non-zero or the ATAPI 'A' bit is set in the command 1840 * header. This bit cannot be set when using native command 1841 * queuing commands or when using FIS-based switching with a Port 1842 * multiplier. 1843 */ 1844 if (command_type != AHCI_NCQ_CMD) 1845 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1846 1847 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1848 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1849 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1850 1851 /* 1852 * We are overloading satapkt_hba_driver_private with 1853 * watched_cycle count. 1854 */ 1855 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1856 1857 #if AHCI_DEBUG 1858 if (ahci_debug_flags & AHCIDBG_ATACMD && 1859 scmd->satacmd_cmd_reg != SATAC_PACKET || 1860 ahci_debug_flags & AHCIDBG_ATAPICMD && 1861 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1862 1863 /* Dump the command header and table */ 1864 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1865 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1866 "0x%p cmd_reg 0x%x port %d", spkt, 1867 scmd->satacmd_cmd_reg, port); 1868 ptr = (uint32_t *)cmd_header; 1869 ahci_log(ahci_ctlp, CE_WARN, 1870 " Command Header:%8x %8x %8x %8x", 1871 ptr[0], ptr[1], ptr[2], ptr[3]); 1872 1873 /* Dump the H2D register FIS */ 1874 ptr = (uint32_t *)h2d_register_fisp; 1875 ahci_log(ahci_ctlp, CE_WARN, 1876 " Command FIS: %8x %8x %8x %8x", 1877 ptr[0], ptr[1], ptr[2], ptr[3]); 1878 1879 /* Dump the ACMD register FIS */ 1880 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1881 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1882 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1883 ahci_log(ahci_ctlp, CE_WARN, 1884 " ATAPI command: %2x %2x %2x %2x " 1885 "%2x %2x %2x %2x", 1886 ptr2[8 * i], ptr2[8 * i + 1], 1887 ptr2[8 * i + 2], ptr2[8 * i + 3], 1888 ptr2[8 * i + 4], ptr2[8 * i + 5], 1889 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1890 1891 /* Dump the PRDT */ 1892 for (i = 0; i < ncookies; i++) { 1893 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1894 ahci_log(ahci_ctlp, CE_WARN, 1895 " Cookie %d: %8x %8x %8x %8x", 1896 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1897 } 1898 } 1899 #endif 1900 1901 (void) ddi_dma_sync( 1902 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1903 0, 1904 ahci_cmd_table_size, 1905 DDI_DMA_SYNC_FORDEV); 1906 1907 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1908 cmd_slot * sizeof (ahci_cmd_header_t), 1909 sizeof (ahci_cmd_header_t), 1910 DDI_DMA_SYNC_FORDEV); 1911 1912 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1913 if (command_type == AHCI_NCQ_CMD) { 1914 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1915 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1916 (0x1 << cmd_slot)); 1917 } 1918 1919 /* Indicate to the HBA that a command is active. */ 1920 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1921 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1922 (0x1 << cmd_slot)); 1923 1924 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1925 "exit: port %d", port); 1926 1927 return (cmd_slot); 1928 } 1929 1930 /* 1931 * Called by the sata framework to abort the previously sent packet(s). 1932 * 1933 * Reset device to abort commands. 1934 */ 1935 static int 1936 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1937 { 1938 ahci_ctl_t *ahci_ctlp; 1939 ahci_port_t *ahci_portp; 1940 uint32_t slot_status = 0; 1941 uint32_t aborted_tags = 0; 1942 uint32_t finished_tags = 0; 1943 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1944 uint8_t port; 1945 int tmp_slot; 1946 int instance = ddi_get_instance(dip); 1947 1948 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1949 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1950 1951 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 1952 "ahci_tran_abort enter: port %d", port); 1953 1954 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1955 mutex_enter(&ahci_portp->ahciport_mutex); 1956 1957 /* 1958 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 1959 * commands are being mopped, therefore there is nothing else to do 1960 */ 1961 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1962 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1963 "ahci_tran_abort: port %d is in " 1964 "mopping process, so just return directly ", port); 1965 mutex_exit(&ahci_portp->ahciport_mutex); 1966 return (SATA_SUCCESS); 1967 } 1968 1969 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1970 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1971 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1972 /* 1973 * In case the targer driver would send the request before 1974 * sata framework can have the opportunity to process those 1975 * event reports. 1976 */ 1977 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1978 spkt->satapkt_device.satadev_state = 1979 ahci_portp->ahciport_port_state; 1980 ahci_update_sata_registers(ahci_ctlp, port, 1981 &spkt->satapkt_device); 1982 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1983 "ahci_tran_abort returning SATA_FAILURE while " 1984 "port in FAILED/SHUTDOWN/PWROFF state: " 1985 "port: %d", port); 1986 mutex_exit(&ahci_portp->ahciport_mutex); 1987 return (SATA_FAILURE); 1988 } 1989 1990 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1991 /* 1992 * ahci_intr_phyrdy_change() may have rendered it to 1993 * AHCI_PORT_TYPE_NODEV. 1994 */ 1995 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1996 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1997 spkt->satapkt_device.satadev_state = 1998 ahci_portp->ahciport_port_state; 1999 ahci_update_sata_registers(ahci_ctlp, port, 2000 &spkt->satapkt_device); 2001 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2002 "ahci_tran_abort returning SATA_FAILURE while " 2003 "no device attached: port: %d", port); 2004 mutex_exit(&ahci_portp->ahciport_mutex); 2005 return (SATA_FAILURE); 2006 } 2007 2008 if (flag == SATA_ABORT_ALL_PACKETS) { 2009 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2010 aborted_tags = ahci_portp->ahciport_pending_tags; 2011 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2012 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2013 2014 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2015 instance, port); 2016 } else { 2017 aborted_tags = 0xffffffff; 2018 /* 2019 * Aborting one specific packet, first search the 2020 * ahciport_slot_pkts[] list for matching spkt. 2021 */ 2022 for (tmp_slot = 0; 2023 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2024 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2025 aborted_tags = (0x1 << tmp_slot); 2026 break; 2027 } 2028 } 2029 2030 if (aborted_tags == 0xffffffff) { 2031 /* request packet is not on the pending list */ 2032 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 2033 "Cannot find the aborting pkt 0x%p on the " 2034 "pending list", (void *)spkt); 2035 ahci_update_sata_registers(ahci_ctlp, port, 2036 &spkt->satapkt_device); 2037 mutex_exit(&ahci_portp->ahciport_mutex); 2038 return (SATA_FAILURE); 2039 } 2040 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2041 instance, port, (void *)spkt); 2042 } 2043 2044 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2045 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2046 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2047 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2048 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2049 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2050 2051 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2052 ahci_portp->ahciport_mop_in_progress++; 2053 2054 /* 2055 * To abort the packet(s), first we are trying to clear PxCMD.ST 2056 * to stop the port, and if the port can be stopped 2057 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2058 * then we just send back the aborted packet(s) with ABORTED flag 2059 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2060 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2061 * perform a COMRESET. 2062 */ 2063 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2064 ahci_portp, port, NULL, NULL); 2065 2066 /* 2067 * Compute which have finished and which need to be retried. 2068 * 2069 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2070 * minus the slot_status. The aborted_tags has to be deducted by 2071 * finished_tags since we can't possibly abort a tag which had finished 2072 * already. 2073 */ 2074 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2075 finished_tags = ahci_portp->ahciport_pending_tags & 2076 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2077 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2078 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2079 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2080 2081 aborted_tags &= ~finished_tags; 2082 2083 ahci_mop_commands(ahci_ctlp, 2084 ahci_portp, 2085 slot_status, 2086 0, /* failed tags */ 2087 0, /* timeout tags */ 2088 aborted_tags, 2089 0); /* reset tags */ 2090 2091 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2092 mutex_exit(&ahci_portp->ahciport_mutex); 2093 2094 return (SATA_SUCCESS); 2095 } 2096 2097 /* 2098 * Used to do device reset and reject all the pending packets on a device 2099 * during the reset operation. 2100 * 2101 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2102 */ 2103 static int 2104 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2105 ahci_port_t *ahci_portp, uint8_t port) 2106 { 2107 uint32_t slot_status = 0; 2108 uint32_t reset_tags = 0; 2109 uint32_t finished_tags = 0; 2110 sata_device_t sdevice; 2111 int ret; 2112 2113 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2114 "ahci_reset_device_reject_pkts on port: %d", port); 2115 2116 /* 2117 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2118 * commands are being mopped, therefore there is nothing else to do 2119 */ 2120 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2121 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2122 "ahci_reset_device_reject_pkts: port %d is in " 2123 "mopping process, so return directly ", port); 2124 return (SATA_SUCCESS); 2125 } 2126 2127 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2128 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2129 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2130 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2131 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2132 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2133 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2134 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2135 } 2136 2137 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2138 != AHCI_SUCCESS) { 2139 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2140 "Try to do a port reset after software " 2141 "reset failed", port); 2142 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2143 if (ret != AHCI_SUCCESS) { 2144 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2145 "ahci_reset_device_reject_pkts: port %d " 2146 "failed", port); 2147 return (SATA_FAILURE); 2148 } 2149 } 2150 /* Set the reset in progress flag */ 2151 ahci_portp->ahciport_reset_in_progress = 1; 2152 2153 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2154 ahci_portp->ahciport_mop_in_progress++; 2155 2156 /* Indicate to the framework that a reset has happened */ 2157 bzero((void *)&sdevice, sizeof (sata_device_t)); 2158 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2159 sdevice.satadev_addr.pmport = 0; 2160 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2161 2162 sdevice.satadev_state = SATA_DSTATE_RESET | 2163 SATA_DSTATE_PWR_ACTIVE; 2164 mutex_exit(&ahci_portp->ahciport_mutex); 2165 sata_hba_event_notify( 2166 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2167 &sdevice, 2168 SATA_EVNT_DEVICE_RESET); 2169 mutex_enter(&ahci_portp->ahciport_mutex); 2170 2171 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 2172 "port %d sending event up: SATA_EVNT_RESET", port); 2173 2174 /* Next try to mop the pending commands */ 2175 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2176 finished_tags = ahci_portp->ahciport_pending_tags & 2177 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2178 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2179 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2180 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2181 2182 reset_tags &= ~finished_tags; 2183 2184 ahci_mop_commands(ahci_ctlp, 2185 ahci_portp, 2186 slot_status, 2187 0, /* failed tags */ 2188 0, /* timeout tags */ 2189 0, /* aborted tags */ 2190 reset_tags); /* reset tags */ 2191 2192 return (SATA_SUCCESS); 2193 } 2194 2195 /* 2196 * Used to do port reset and reject all the pending packets on a port during 2197 * the reset operation. 2198 * 2199 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2200 */ 2201 static int 2202 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2203 ahci_port_t *ahci_portp, uint8_t port) 2204 { 2205 uint32_t slot_status = 0; 2206 uint32_t reset_tags = 0; 2207 uint32_t finished_tags = 0; 2208 2209 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2210 "ahci_reset_port_reject_pkts on port: %d", port); 2211 2212 /* 2213 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2214 * commands are being mopped, therefore there is nothing else to do 2215 */ 2216 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2217 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2218 "ahci_reset_port_reject_pkts: port %d is in " 2219 "mopping process, so return directly ", port); 2220 return (SATA_SUCCESS); 2221 } 2222 2223 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2224 ahci_portp->ahciport_mop_in_progress++; 2225 2226 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2227 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2228 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2229 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2230 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2231 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2232 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2233 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2234 } 2235 2236 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2237 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2238 NULL) != AHCI_SUCCESS) 2239 return (SATA_FAILURE); 2240 2241 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2242 finished_tags = ahci_portp->ahciport_pending_tags & 2243 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2244 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2245 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2246 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2247 2248 reset_tags &= ~finished_tags; 2249 2250 ahci_mop_commands(ahci_ctlp, 2251 ahci_portp, 2252 slot_status, 2253 0, /* failed tags */ 2254 0, /* timeout tags */ 2255 0, /* aborted tags */ 2256 reset_tags); /* reset tags */ 2257 2258 return (SATA_SUCCESS); 2259 } 2260 2261 /* 2262 * Used to do hba reset and reject all the pending packets on all ports 2263 * during the reset operation. 2264 */ 2265 static int 2266 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2267 { 2268 ahci_port_t *ahci_portp; 2269 uint32_t slot_status[AHCI_MAX_PORTS]; 2270 uint32_t reset_tags[AHCI_MAX_PORTS]; 2271 uint32_t finished_tags[AHCI_MAX_PORTS]; 2272 uint8_t port; 2273 int ret = SATA_SUCCESS; 2274 2275 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 2276 "ahci_reset_hba_reject_pkts enter"); 2277 2278 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2279 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2280 continue; 2281 } 2282 2283 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2284 2285 mutex_enter(&ahci_portp->ahciport_mutex); 2286 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2287 slot_status[port] = ddi_get32( 2288 ahci_ctlp->ahcictl_ahci_acc_handle, 2289 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2290 reset_tags[port] = slot_status[port] & 2291 AHCI_SLOT_MASK(ahci_ctlp); 2292 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2293 slot_status[port] = ddi_get32( 2294 ahci_ctlp->ahcictl_ahci_acc_handle, 2295 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2296 reset_tags[port] = slot_status[port] & 2297 AHCI_NCQ_SLOT_MASK(ahci_portp); 2298 } 2299 mutex_exit(&ahci_portp->ahciport_mutex); 2300 } 2301 2302 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2303 ret = SATA_FAILURE; 2304 goto out; 2305 } 2306 2307 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2308 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2309 continue; 2310 } 2311 2312 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2313 2314 mutex_enter(&ahci_portp->ahciport_mutex); 2315 /* 2316 * To prevent recursive enter to ahci_mop_commands, we need 2317 * check AHCI_PORT_FLAG_MOPPING flag. 2318 */ 2319 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2320 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2321 "ahci_reset_hba_reject_pkts: port %d is in " 2322 "mopping process, so return directly ", port); 2323 mutex_exit(&ahci_portp->ahciport_mutex); 2324 continue; 2325 } 2326 2327 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2328 ahci_portp->ahciport_mop_in_progress++; 2329 2330 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2331 finished_tags[port] = 2332 ahci_portp->ahciport_pending_tags & 2333 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2334 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2335 finished_tags[port] = 2336 ahci_portp->ahciport_pending_ncq_tags & 2337 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2338 2339 reset_tags[port] &= ~finished_tags[port]; 2340 2341 ahci_mop_commands(ahci_ctlp, 2342 ahci_portp, 2343 slot_status[port], 2344 0, /* failed tags */ 2345 0, /* timeout tags */ 2346 0, /* aborted tags */ 2347 reset_tags[port]); /* reset tags */ 2348 mutex_exit(&ahci_portp->ahciport_mutex); 2349 } 2350 out: 2351 return (ret); 2352 } 2353 2354 /* 2355 * Called by sata framework to reset a port(s) or device. 2356 */ 2357 static int 2358 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2359 { 2360 ahci_ctl_t *ahci_ctlp; 2361 ahci_port_t *ahci_portp; 2362 uint8_t cport = sd->satadev_addr.cport; 2363 uint8_t port; 2364 int ret = SATA_SUCCESS; 2365 int instance = ddi_get_instance(dip); 2366 2367 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2368 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2369 2370 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2371 "ahci_tran_reset_port enter: cport: %d", cport); 2372 2373 switch (sd->satadev_addr.qual) { 2374 case SATA_ADDR_CPORT: 2375 /* Port reset */ 2376 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2377 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2378 "reset port", instance, port); 2379 2380 mutex_enter(&ahci_portp->ahciport_mutex); 2381 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2382 mutex_exit(&ahci_portp->ahciport_mutex); 2383 2384 break; 2385 2386 case SATA_ADDR_DCPORT: 2387 /* Device reset */ 2388 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2389 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2390 "reset device", instance, port); 2391 2392 mutex_enter(&ahci_portp->ahciport_mutex); 2393 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2394 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2395 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2396 /* 2397 * In case the targer driver would send the request 2398 * before sata framework can have the opportunity to 2399 * process those event reports. 2400 */ 2401 sd->satadev_state = ahci_portp->ahciport_port_state; 2402 ahci_update_sata_registers(ahci_ctlp, port, sd); 2403 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2404 "ahci_tran_reset_dport returning SATA_FAILURE " 2405 "while port in FAILED/SHUTDOWN/PWROFF state: " 2406 "port: %d", port); 2407 mutex_exit(&ahci_portp->ahciport_mutex); 2408 ret = SATA_FAILURE; 2409 break; 2410 } 2411 2412 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2413 /* 2414 * ahci_intr_phyrdy_change() may have rendered it to 2415 * AHCI_PORT_TYPE_NODEV. 2416 */ 2417 sd->satadev_type = SATA_DTYPE_NONE; 2418 sd->satadev_state = ahci_portp->ahciport_port_state; 2419 ahci_update_sata_registers(ahci_ctlp, port, sd); 2420 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2421 "ahci_tran_reset_dport returning SATA_FAILURE " 2422 "while no device attached: port: %d", port); 2423 mutex_exit(&ahci_portp->ahciport_mutex); 2424 ret = SATA_FAILURE; 2425 break; 2426 } 2427 2428 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2429 ahci_portp, port); 2430 mutex_exit(&ahci_portp->ahciport_mutex); 2431 break; 2432 2433 case SATA_ADDR_CNTRL: 2434 /* Reset the whole controller */ 2435 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2436 "reset the whole hba", instance, port); 2437 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2438 break; 2439 2440 case SATA_ADDR_PMPORT: 2441 case SATA_ADDR_DPMPORT: 2442 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 2443 "ahci_tran_reset_dport: port multiplier will be " 2444 "supported later"); 2445 /* FALLTHRU */ 2446 default: 2447 ret = SATA_FAILURE; 2448 } 2449 2450 return (ret); 2451 } 2452 2453 /* 2454 * Called by sata framework to activate a port as part of hotplug. 2455 * (cfgadm -c connect satax/y) 2456 * Note: Not port-mult aware. 2457 */ 2458 static int 2459 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2460 { 2461 ahci_ctl_t *ahci_ctlp; 2462 ahci_port_t *ahci_portp; 2463 uint8_t cport = satadev->satadev_addr.cport; 2464 uint8_t port; 2465 int instance = ddi_get_instance(dip); 2466 2467 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2468 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2469 2470 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2471 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2472 2473 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2474 2475 mutex_enter(&ahci_portp->ahciport_mutex); 2476 ahci_enable_port_intrs(ahci_ctlp, port); 2477 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2478 2479 /* 2480 * Reset the port so that the PHY communication would be re-established. 2481 * But this reset is an internal operation and the sata module doesn't 2482 * need to know about it. Moreover, the port with a device attached will 2483 * be started too. 2484 */ 2485 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2486 ahci_portp, port, 2487 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2488 NULL); 2489 2490 /* 2491 * Need to check the link status and device status of the port 2492 * and consider raising power if the port was in D3 state 2493 */ 2494 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2495 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2496 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2497 2498 satadev->satadev_state = ahci_portp->ahciport_port_state; 2499 2500 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2501 2502 mutex_exit(&ahci_portp->ahciport_mutex); 2503 return (SATA_SUCCESS); 2504 } 2505 2506 /* 2507 * Called by sata framework to deactivate a port as part of hotplug. 2508 * (cfgadm -c disconnect satax/y) 2509 * Note: Not port-mult aware. 2510 */ 2511 static int 2512 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2513 { 2514 ahci_ctl_t *ahci_ctlp; 2515 ahci_port_t *ahci_portp; 2516 uint8_t cport = satadev->satadev_addr.cport; 2517 uint8_t port; 2518 uint32_t port_scontrol; 2519 int instance = ddi_get_instance(dip); 2520 2521 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2522 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2523 2524 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2525 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2526 2527 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2528 2529 mutex_enter(&ahci_portp->ahciport_mutex); 2530 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2531 instance, port); 2532 2533 /* Disable the interrupts on the port */ 2534 ahci_disable_port_intrs(ahci_ctlp, port); 2535 2536 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2537 goto phy_offline; 2538 } 2539 2540 /* First to abort all the pending commands */ 2541 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2542 2543 /* Then stop the port */ 2544 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2545 ahci_portp, port); 2546 2547 /* Next put the PHY offline */ 2548 2549 phy_offline: 2550 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2551 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2552 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2553 2554 /* Update ahciport_port_state */ 2555 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2556 satadev->satadev_state = ahci_portp->ahciport_port_state; 2557 2558 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2559 2560 mutex_exit(&ahci_portp->ahciport_mutex); 2561 return (SATA_SUCCESS); 2562 } 2563 2564 /* 2565 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2566 * when a device is unplugged or a port is deactivated. 2567 * 2568 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2569 */ 2570 static void 2571 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2572 ahci_port_t *ahci_portp, uint8_t port) 2573 { 2574 uint32_t slot_status = 0; 2575 uint32_t abort_tags = 0; 2576 2577 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2578 "ahci_reject_all_abort_pkts on port: %d", port); 2579 2580 /* 2581 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2582 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2583 * to get the error data, if yes when the device is removed, the 2584 * command needs to be aborted too. 2585 */ 2586 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2587 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2588 slot_status = 0x1; 2589 abort_tags = 0x1; 2590 goto out; 2591 } else { 2592 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2593 "ahci_reject_all_abort_pkts return directly " 2594 "port %d no needs to reject any outstanding " 2595 "commands", port); 2596 return; 2597 } 2598 } 2599 2600 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2601 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2602 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2603 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2604 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2605 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2606 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2607 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2608 } 2609 2610 out: 2611 /* No need to do mop when there is no outstanding commands */ 2612 if (slot_status != 0) { 2613 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2614 ahci_portp->ahciport_mop_in_progress++; 2615 2616 ahci_mop_commands(ahci_ctlp, 2617 ahci_portp, 2618 slot_status, 2619 0, /* failed tags */ 2620 0, /* timeout tags */ 2621 abort_tags, /* aborting tags */ 2622 0); /* reset tags */ 2623 } 2624 } 2625 2626 #if defined(__lock_lint) 2627 static int 2628 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2629 { 2630 return (SATA_SUCCESS); 2631 } 2632 #endif 2633 2634 /* 2635 * Allocate the ports structure, only called by ahci_attach 2636 */ 2637 static int 2638 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2639 { 2640 int port, cport = 0; 2641 2642 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2643 "ahci_alloc_ports_state enter"); 2644 2645 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2646 2647 /* Allocate structures only for the implemented ports */ 2648 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2649 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2650 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2651 "hba port %d not implemented", port); 2652 continue; 2653 } 2654 2655 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2656 ahci_ctlp->ahcictl_port_to_cport[port] = 2657 (uint8_t)cport++; 2658 2659 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2660 goto err_out; 2661 } 2662 } 2663 2664 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2665 return (AHCI_SUCCESS); 2666 2667 err_out: 2668 for (port--; port >= 0; port--) { 2669 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2670 ahci_dealloc_port_state(ahci_ctlp, port); 2671 } 2672 } 2673 2674 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2675 return (AHCI_FAILURE); 2676 } 2677 2678 /* 2679 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2680 */ 2681 static void 2682 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2683 { 2684 int port; 2685 2686 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2687 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2688 /* if this port is implemented by the HBA */ 2689 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2690 ahci_dealloc_port_state(ahci_ctlp, port); 2691 } 2692 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2693 } 2694 2695 /* 2696 * Drain the taskq. 2697 */ 2698 static void 2699 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 2700 { 2701 ahci_port_t *ahci_portp; 2702 int port; 2703 2704 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2705 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2706 continue; 2707 } 2708 2709 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2710 2711 mutex_enter(&ahci_portp->ahciport_mutex); 2712 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 2713 mutex_exit(&ahci_portp->ahciport_mutex); 2714 } 2715 } 2716 2717 /* 2718 * Initialize the controller and all ports. And then try to start the ports 2719 * if there are devices attached. 2720 * 2721 * This routine can be called from three seperate cases: DDI_ATTACH, 2722 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2723 * other two cases; device signature probing are attempted only during 2724 * DDI_ATTACH case. 2725 * 2726 * WARNING!!! Disable the whole controller's interrupts before calling and 2727 * the interrupts will be enabled upon successfully return. 2728 */ 2729 static int 2730 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2731 { 2732 ahci_port_t *ahci_portp; 2733 uint32_t ghc_control; 2734 int port; 2735 2736 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2737 "ahci_initialize_controller enter"); 2738 2739 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2740 2741 /* 2742 * Indicate that system software is AHCI aware by setting 2743 * GHC.AE to 1 2744 */ 2745 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2746 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2747 2748 ghc_control |= AHCI_HBA_GHC_AE; 2749 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2750 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2751 ghc_control); 2752 2753 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2754 2755 /* Initialize the implemented ports and structures */ 2756 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2757 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2758 continue; 2759 } 2760 2761 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2762 mutex_enter(&ahci_portp->ahciport_mutex); 2763 2764 /* 2765 * Ensure that the controller is not in the running state 2766 * by checking every implemented port's PxCMD register 2767 */ 2768 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2769 != AHCI_SUCCESS) { 2770 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2771 "ahci_initialize_controller: failed to " 2772 "initialize port %d", port); 2773 /* 2774 * Set the port state to SATA_PSTATE_FAILED if 2775 * failed to initialize it. 2776 */ 2777 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2778 } 2779 2780 mutex_exit(&ahci_portp->ahciport_mutex); 2781 } 2782 2783 /* Enable the whole controller interrupts */ 2784 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2785 ahci_enable_all_intrs(ahci_ctlp); 2786 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2787 2788 return (AHCI_SUCCESS); 2789 } 2790 2791 /* 2792 * Reverse of ahci_initialize_controller() 2793 * 2794 * We only need to stop the ports and disable the interrupt. 2795 */ 2796 static void 2797 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2798 { 2799 ahci_port_t *ahci_portp; 2800 int port; 2801 2802 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2803 "ahci_uninitialize_controller enter"); 2804 2805 /* disable all the interrupts. */ 2806 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2807 ahci_disable_all_intrs(ahci_ctlp); 2808 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2809 2810 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2811 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2812 continue; 2813 } 2814 2815 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2816 2817 /* Stop the port by clearing PxCMD.ST */ 2818 mutex_enter(&ahci_portp->ahciport_mutex); 2819 2820 /* 2821 * Here we must disable the port interrupt because 2822 * ahci_disable_all_intrs only clear GHC.IE, and IS 2823 * register will be still set if PxIE is enabled. 2824 * When ahci shares one IRQ with other drivers, the 2825 * intr handler may claim the intr mistakenly. 2826 */ 2827 ahci_disable_port_intrs(ahci_ctlp, port); 2828 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2829 ahci_portp, port); 2830 mutex_exit(&ahci_portp->ahciport_mutex); 2831 } 2832 } 2833 2834 /* 2835 * The routine is to initialize the port. First put the port in NOTRunning 2836 * state, then enable port interrupt and clear Serror register. And under 2837 * AHCI_ATTACH case, find device signature and then try to start the port. 2838 * 2839 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2840 */ 2841 static int 2842 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2843 ahci_port_t *ahci_portp, uint8_t port) 2844 { 2845 uint32_t port_cmd_status; 2846 2847 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2848 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2849 2850 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2851 "ahci_initialize_port: port %d " 2852 "port_cmd_status = 0x%x", port, port_cmd_status); 2853 /* 2854 * Check whether the port is in NotRunning state, if not, 2855 * put the port in NotRunning state 2856 */ 2857 if (!(port_cmd_status & 2858 (AHCI_CMD_STATUS_ST | 2859 AHCI_CMD_STATUS_CR | 2860 AHCI_CMD_STATUS_FRE | 2861 AHCI_CMD_STATUS_FR))) { 2862 2863 goto next; 2864 } 2865 2866 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 2867 port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT, NULL) != AHCI_SUCCESS) 2868 return (AHCI_FAILURE); 2869 2870 next: 2871 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2872 "port %d is in NotRunning state now", port); 2873 2874 /* 2875 * At the time being, only probe ports/devices and get the types of 2876 * attached devices during DDI_ATTACH. In fact, the device can be 2877 * changed during power state changes, but at the time being, we 2878 * don't support the situation. 2879 */ 2880 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2881 /* Try to get the device signature */ 2882 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2883 } else { 2884 2885 /* 2886 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2887 * and PxFBU registers in case these registers were cleared 2888 * during the suspend. 2889 */ 2890 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2891 "ahci_initialize_port: port %d " 2892 "reset the port during resume", port); 2893 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2894 2895 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2896 "ahci_initialize_port: port %d " 2897 "set PxCLB, PxCLBU, PxFB and PxFBU " 2898 "during resume", port); 2899 2900 /* Config Port Received FIS Base Address */ 2901 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2902 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2903 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2904 2905 /* Config Port Command List Base Address */ 2906 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2907 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2908 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2909 } 2910 2911 /* Disable the interface power management */ 2912 ahci_disable_interface_pm(ahci_ctlp, port); 2913 2914 /* Return directly if no device connected */ 2915 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2916 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2917 "No device connected to port %d", port); 2918 goto out; 2919 } 2920 2921 /* Try to start the port */ 2922 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 2923 != AHCI_SUCCESS) { 2924 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2925 "failed to start port %d", port); 2926 return (AHCI_FAILURE); 2927 } 2928 out: 2929 /* Enable port interrupts */ 2930 ahci_enable_port_intrs(ahci_ctlp, port); 2931 2932 return (AHCI_SUCCESS); 2933 } 2934 2935 /* 2936 * Check the hardware defects and the power management capability. 2937 */ 2938 static int 2939 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 2940 { 2941 ushort_t venid, devid; 2942 ushort_t caps_ptr, cap_count, cap; 2943 #if AHCI_DEBUG 2944 ushort_t pmcap, pmcsr; 2945 ushort_t msimc; 2946 #endif 2947 uint8_t revision; 2948 2949 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2950 PCI_CONF_VENID); 2951 2952 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2953 PCI_CONF_DEVID); 2954 2955 /* 2956 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 2957 * controllers with 0x00 revision id work on 4-byte aligned buffer, 2958 * which is a bug and was fixed after 0x00 revision id controllers. 2959 * 2960 * Moreover, VT8251 cannot use multiple command slots in the command 2961 * list for non-queued commands because the previous register content 2962 * of PxCI can be re-written in the register write, so a flag will be 2963 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 2964 */ 2965 if (venid == VIA_VENID) { 2966 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 2967 PCI_CONF_REVID); 2968 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2969 "revision id = 0x%x", revision); 2970 if (revision == 0x00) { 2971 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 2972 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2973 "change ddi_attr_align to 0x4"); 2974 } 2975 2976 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 2977 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2978 "VT8251 cannot use multiple command lists for " 2979 "non-queued commands"); 2980 } 2981 2982 /* 2983 * ATI SB600 (1002,4380) and SB700/750 (1002,4391) AHCI chipsets don't 2984 * support 64-bit DMA addressing though they declare the support, 2985 * so we need to set AHCI_CAP_32BIT_DMA flag to force 32-bit DMA. 2986 */ 2987 if (venid == 0x1002 && (devid == 0x4380 || devid == 0x4391)) { 2988 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2989 "ATI SB600/700/750 cannot do 64-bit DMA though CAP " 2990 "indicates support, so force it to use 32-bit DMA"); 2991 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 2992 } 2993 2994 /* ASUS M3N-HT (NVidia 780a) does not support MSI */ 2995 if (venid == 0x10de && devid == 0x0ad4) { 2996 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2997 "ASUS M3N-HT (NVidia 780a) does not support MSI " 2998 "interrupts, so force it to use fixed interrupts."); 2999 ahci_msi_enabled = B_FALSE; 3000 } 3001 3002 /* 3003 * Check if capabilities list is supported and if so, 3004 * get initial capabilities pointer and clear bits 0,1. 3005 */ 3006 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3007 PCI_CONF_STAT) & PCI_STAT_CAP) { 3008 caps_ptr = P2ALIGN(pci_config_get8( 3009 ahci_ctlp->ahcictl_pci_conf_handle, 3010 PCI_CONF_CAP_PTR), 4); 3011 } else { 3012 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3013 } 3014 3015 /* 3016 * Walk capabilities if supported. 3017 */ 3018 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3019 3020 /* 3021 * Check that we haven't exceeded the maximum number of 3022 * capabilities and that the pointer is in a valid range. 3023 */ 3024 if (++cap_count > PCI_CAP_MAX_PTR) { 3025 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3026 "too many device capabilities"); 3027 return (AHCI_FAILURE); 3028 } 3029 if (caps_ptr < PCI_CAP_PTR_OFF) { 3030 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3031 "capabilities pointer 0x%x out of range", 3032 caps_ptr); 3033 return (AHCI_FAILURE); 3034 } 3035 3036 /* 3037 * Get next capability and check that it is valid. 3038 * For now, we only support power management. 3039 */ 3040 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3041 caps_ptr); 3042 switch (cap) { 3043 case PCI_CAP_ID_PM: 3044 3045 /* power management supported */ 3046 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3047 3048 /* Save PMCSR offset */ 3049 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3050 3051 #if AHCI_DEBUG 3052 pmcap = pci_config_get16( 3053 ahci_ctlp->ahcictl_pci_conf_handle, 3054 caps_ptr + PCI_PMCAP); 3055 pmcsr = pci_config_get16( 3056 ahci_ctlp->ahcictl_pci_conf_handle, 3057 ahci_ctlp->ahcictl_pmcsr_offset); 3058 AHCIDBG2(AHCIDBG_PM, ahci_ctlp, 3059 "Power Management capability found PCI_PMCAP " 3060 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3061 if ((pmcap & 0x3) == 0x3) 3062 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3063 "PCI Power Management Interface " 3064 "spec 1.2 compliant"); 3065 #endif 3066 break; 3067 3068 case PCI_CAP_ID_MSI: 3069 #if AHCI_DEBUG 3070 msimc = pci_config_get16( 3071 ahci_ctlp->ahcictl_pci_conf_handle, 3072 caps_ptr + PCI_MSI_CTRL); 3073 AHCIDBG1(AHCIDBG_MSI, ahci_ctlp, 3074 "Message Signaled Interrupt capability found " 3075 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3076 #endif 3077 AHCIDBG0(AHCIDBG_MSI, ahci_ctlp, 3078 "MSI capability found"); 3079 break; 3080 3081 case PCI_CAP_ID_PCIX: 3082 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3083 "PCI-X capability found"); 3084 break; 3085 3086 case PCI_CAP_ID_PCI_E: 3087 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3088 "PCI Express capability found"); 3089 break; 3090 3091 case PCI_CAP_ID_MSI_X: 3092 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3093 "MSI-X capability found"); 3094 break; 3095 3096 case PCI_CAP_ID_SATA: 3097 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3098 "SATA capability found"); 3099 break; 3100 3101 case PCI_CAP_ID_VS: 3102 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3103 "Vendor Specific capability found"); 3104 break; 3105 3106 default: 3107 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 3108 "unrecognized capability 0x%x", cap); 3109 break; 3110 } 3111 3112 /* 3113 * Get next capabilities pointer and clear bits 0,1. 3114 */ 3115 caps_ptr = P2ALIGN(pci_config_get8( 3116 ahci_ctlp->ahcictl_pci_conf_handle, 3117 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3118 } 3119 3120 return (AHCI_SUCCESS); 3121 } 3122 3123 /* 3124 * AHCI device reset ...; a single device on one of the ports is reset, 3125 * but the HBA and physical communication remain intact. This is the 3126 * least intrusive. 3127 * 3128 * When issuing a software reset sequence, there should not be other 3129 * commands in the command list, so we will first clear and then re-set 3130 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3131 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3132 * cleared unless command list override (PxCMD.CLO) is supported. 3133 * 3134 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3135 * set before the function is called. 3136 */ 3137 static int 3138 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3139 uint8_t port) 3140 { 3141 ahci_fis_h2d_register_t *h2d_register_fisp; 3142 ahci_cmd_table_t *cmd_table; 3143 ahci_cmd_header_t *cmd_header; 3144 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3145 int slot, loop_count; 3146 3147 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 3148 "Port %d device resetting", port); 3149 3150 /* First clear PxCMD.ST */ 3151 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3152 ahci_portp, port); 3153 3154 /* Then re-set PxCMD.ST */ 3155 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 3156 3157 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3158 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3159 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3160 3161 if (port_task_file & AHCI_TFD_STS_BSY || 3162 port_task_file & AHCI_TFD_STS_DRQ) { 3163 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3164 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3165 if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) { 3166 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3167 "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, " 3168 "but PxCMD.CLO isn't supported, so a port " 3169 "reset is needed."); 3170 return (AHCI_FAILURE); 3171 } 3172 } 3173 3174 slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, AHCI_NON_NCQ_CMD); 3175 if (slot == AHCI_FAILURE) { 3176 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 3177 "ahci_software_reset: no free slot"); 3178 return (AHCI_FAILURE); 3179 } 3180 3181 /* Now send the first H2D Register FIS with SRST set to 1 */ 3182 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3183 bzero((void *)cmd_table, ahci_cmd_table_size); 3184 3185 h2d_register_fisp = 3186 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3187 3188 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3189 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3190 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3191 3192 /* Set Command Header in Command List */ 3193 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3194 BZERO_DESCR_INFO(cmd_header); 3195 BZERO_PRD_BYTE_COUNT(cmd_header); 3196 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3197 3198 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3199 SET_RESET(cmd_header, 1); 3200 SET_WRITE(cmd_header, 1); 3201 3202 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3203 0, 3204 ahci_cmd_table_size, 3205 DDI_DMA_SYNC_FORDEV); 3206 3207 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3208 slot * sizeof (ahci_cmd_header_t), 3209 sizeof (ahci_cmd_header_t), 3210 DDI_DMA_SYNC_FORDEV); 3211 3212 /* Indicate to the HBA that a command is active. */ 3213 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3214 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3215 (0x1 << slot)); 3216 3217 loop_count = 0; 3218 3219 /* Loop till the first command is finished */ 3220 do { 3221 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3222 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3223 3224 /* We are effectively timing out after 1 sec. */ 3225 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3226 break; 3227 } 3228 /* Wait for 10 millisec */ 3229 delay(AHCI_10MS_TICKS); 3230 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3231 3232 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3233 "ahci_software_reset: 1st loop count: %d, " 3234 "port_cmd_issue = 0x%x, slot = 0x%x", 3235 loop_count, port_cmd_issue, slot); 3236 3237 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3238 3239 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3240 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3241 bzero((void *)cmd_table, ahci_cmd_table_size); 3242 3243 h2d_register_fisp = 3244 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3245 3246 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3247 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3248 3249 /* Set Command Header in Command List */ 3250 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3251 BZERO_DESCR_INFO(cmd_header); 3252 BZERO_PRD_BYTE_COUNT(cmd_header); 3253 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3254 3255 SET_WRITE(cmd_header, 1); 3256 3257 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3258 0, 3259 ahci_cmd_table_size, 3260 DDI_DMA_SYNC_FORDEV); 3261 3262 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3263 slot * sizeof (ahci_cmd_header_t), 3264 sizeof (ahci_cmd_header_t), 3265 DDI_DMA_SYNC_FORDEV); 3266 3267 /* Indicate to the HBA that a command is active. */ 3268 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3269 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3270 (0x1 << slot)); 3271 3272 loop_count = 0; 3273 3274 /* Loop till the second command is finished */ 3275 do { 3276 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3277 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3278 3279 /* We are effectively timing out after 1 sec. */ 3280 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3281 break; 3282 } 3283 /* Wait for 10 millisec */ 3284 delay(AHCI_10MS_TICKS); 3285 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3286 3287 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3288 "ahci_software_reset: 2nd loop count: %d, " 3289 "port_cmd_issue = 0x%x, slot = 0x%x", 3290 loop_count, port_cmd_issue, slot); 3291 3292 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3293 3294 return (AHCI_SUCCESS); 3295 } 3296 3297 /* 3298 * AHCI port reset ...; the physical communication between the HBA and device 3299 * on a port are disabled. This is more intrusive. 3300 * 3301 * When an HBA or port reset occurs, Phy communication is going to 3302 * be re-established with the device through a COMRESET followed by the 3303 * normal out-of-band communication sequence defined in Serial ATA. AT 3304 * the end of reset, the device, if working properly, will send a D2H 3305 * Register FIS, which contains the device signature. When the HBA receives 3306 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3307 * the PxSIG register with the signature. 3308 * 3309 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3310 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3311 * AHCI 1.0 spec. 3312 */ 3313 /* 3314 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3315 * cleared before the function is called. 3316 */ 3317 static int 3318 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3319 { 3320 uint32_t cap_status, port_cmd_status; 3321 uint32_t port_scontrol, port_sstatus; 3322 uint32_t port_intr_status, port_task_file; 3323 #if AHCI_DEBUG 3324 uint32_t port_signature; 3325 #endif 3326 int loop_count; 3327 int rval = AHCI_SUCCESS; 3328 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3329 3330 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3331 "Port %d port resetting...", port); 3332 ahci_portp->ahciport_port_state = 0; 3333 3334 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3335 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3336 3337 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3338 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3339 3340 if (cap_status & AHCI_HBA_CAP_SSS) { 3341 /* 3342 * HBA support staggered spin-up, if the port has 3343 * not spin up yet, then force it to do spin-up 3344 */ 3345 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3346 if (!(ahci_portp->ahciport_flags 3347 & AHCI_PORT_FLAG_SPINUP)) { 3348 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3349 "Port %d PxCMD.SUD is zero, force " 3350 "it to do spin-up", port); 3351 ahci_portp->ahciport_flags |= 3352 AHCI_PORT_FLAG_SPINUP; 3353 } 3354 } 3355 } else { 3356 /* 3357 * HBA doesn't support stagger spin-up, force it 3358 * to do normal COMRESET 3359 */ 3360 if (ahci_portp->ahciport_flags & 3361 AHCI_PORT_FLAG_SPINUP) { 3362 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3363 "HBA does not support staggered spin-up " 3364 "force it to do normal COMRESET"); 3365 ahci_portp->ahciport_flags &= 3366 ~AHCI_PORT_FLAG_SPINUP; 3367 } 3368 } 3369 3370 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3371 /* Do normal COMRESET */ 3372 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3373 "ahci_port_reset: do normal COMRESET", port); 3374 3375 /* 3376 * According to the spec, SUD bit should be set here, 3377 * but JMicron JMB363 doesn't follow it, so remove 3378 * the assertion, and just print a debug message. 3379 */ 3380 #if AHCI_DEBUG 3381 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3382 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3383 "port %d SUD bit not set", port) 3384 #endif 3385 3386 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3387 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3388 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3389 3390 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3391 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3392 port_scontrol); 3393 3394 /* Enable PxCMD.FRE to read device */ 3395 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3396 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3397 port_cmd_status|AHCI_CMD_STATUS_FRE); 3398 3399 /* 3400 * Give time for COMRESET to percolate, according to the AHCI 3401 * spec, software shall wait at least 1 millisecond before 3402 * clearing PxSCTL.DET 3403 */ 3404 delay(AHCI_1MS_TICKS*2); 3405 3406 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3407 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3408 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3409 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3410 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3411 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3412 port_scontrol); 3413 } else { 3414 /* Do staggered spin-up */ 3415 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3416 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3417 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3418 3419 /* PxSCTL.DET must be 0 */ 3420 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3421 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3422 port_scontrol); 3423 3424 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3425 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3426 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3427 port_cmd_status); 3428 3429 /* 0 -> 1 edge */ 3430 delay(AHCI_1MS_TICKS*2); 3431 3432 /* Set PxCMD.SUD to 1 */ 3433 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3434 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3435 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3436 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3437 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3438 port_cmd_status); 3439 3440 /* Enable PxCMD.FRE to read device */ 3441 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3442 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3443 port_cmd_status|AHCI_CMD_STATUS_FRE); 3444 } 3445 3446 /* 3447 * The port enters P:StartComm state, and HBA tells link layer to 3448 * start communication, which involves sending COMRESET to device. 3449 * And the HBA resets PxTFD.STS to 7Fh. 3450 * 3451 * When a COMINIT is received from the device, then the port enters 3452 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3453 * PxSSTS.DET to 1h to indicate a device is detected but communication 3454 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3455 * a COMINIT has been received. 3456 */ 3457 /* 3458 * The DET field is valid only if IPM field indicates 3459 * that the interface is in active state. 3460 */ 3461 loop_count = 0; 3462 do { 3463 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3464 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3465 3466 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3467 /* 3468 * If the interface is not active, the DET field 3469 * is considered not accurate. So we want to 3470 * continue looping. 3471 */ 3472 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3473 } 3474 3475 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3476 /* 3477 * We are effectively timing out after 0.1 sec. 3478 */ 3479 break; 3480 } 3481 3482 /* Wait for 10 millisec */ 3483 delay(AHCI_10MS_TICKS); 3484 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3485 3486 AHCIDBG3(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3487 "ahci_port_reset: 1st loop count: %d, " 3488 "port_sstatus = 0x%x port %d", 3489 loop_count, port_sstatus, port); 3490 3491 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3492 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3493 /* 3494 * Either the port is not active or there 3495 * is no device present. 3496 */ 3497 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3498 goto out; 3499 } 3500 3501 /* Now we can make sure there is a device connected to the port */ 3502 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3503 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3504 3505 /* a COMINIT signal is supposed to be received */ 3506 if (!(port_intr_status & AHCI_INTR_STATUS_PCS)) { 3507 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3508 "COMINIT signal from the device not received", 3509 instance, port); 3510 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3511 rval = AHCI_FAILURE; 3512 goto out; 3513 } 3514 3515 /* 3516 * According to the spec, when PxSCTL.DET is set to 0h, upon 3517 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3518 * shall be set to '1' by the HBA. 3519 * 3520 * However, we found JMicron JMB363 doesn't follow this, so 3521 * remove this check, and just print a debug message. 3522 */ 3523 #if AHCI_DEBUG 3524 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3525 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3526 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3527 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3528 "port %d BSY bit is not set after COMINIT signal " 3529 "is received", port); 3530 } 3531 #endif 3532 3533 /* 3534 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3535 * the D2H FIS received by HBA. 3536 */ 3537 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3538 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3539 SERROR_EXCHANGED_ERR); 3540 3541 /* 3542 * Next check whether COMRESET is completed successfully 3543 */ 3544 loop_count = 0; 3545 do { 3546 port_task_file = 3547 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3548 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3549 3550 /* 3551 * The Error bit '1' means COMRESET is finished successfully 3552 * The device hardware has been initialized and the power-up 3553 * diagnostics successfully completed. 3554 */ 3555 if (((port_task_file & AHCI_TFD_ERR_MASK) 3556 >> AHCI_TFD_ERR_SHIFT) == 0x1) { 3557 #if AHCI_DEBUG 3558 port_signature = ddi_get32( 3559 ahci_ctlp->ahcictl_ahci_acc_handle, 3560 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3561 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 3562 "COMRESET success, D2H register FIS " 3563 "post to received FIS structure " 3564 "port %d signature = 0x%x", 3565 port, port_signature); 3566 #endif 3567 goto out_check; 3568 } 3569 3570 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3571 /* 3572 * We are effectively timing out after 11 sec. 3573 */ 3574 break; 3575 } 3576 3577 /* Wait for 10 millisec */ 3578 delay(AHCI_10MS_TICKS); 3579 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3580 >> AHCI_TFD_ERR_SHIFT) != 0x1); 3581 3582 AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: 2nd loop " 3583 "count: %d, port_task_file = 0x%x port %d", 3584 loop_count, port_task_file, port); 3585 3586 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d the device hardware " 3587 "has been initialized and the power-up diagnostics failed", 3588 instance, port); 3589 3590 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3591 rval = AHCI_FAILURE; 3592 3593 out: 3594 /* Clear port serror register for the port */ 3595 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3596 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3597 AHCI_SERROR_CLEAR_ALL); 3598 3599 return (rval); 3600 3601 out_check: 3602 /* 3603 * Check device status, if keep busy or COMRESET error 3604 * do device reset to patch some SATA disks' issue 3605 * 3606 * For VT8251, sometimes need to do the device reset 3607 */ 3608 if ((port_task_file & AHCI_TFD_STS_BSY) || 3609 (port_task_file & AHCI_TFD_STS_DRQ)) { 3610 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "port %d keep BSY/DRQ set " 3611 "need to do device reset", port); 3612 3613 (void) ahci_software_reset(ahci_ctlp, ahci_portp, port); 3614 3615 port_task_file = 3616 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3617 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3618 3619 if (port_task_file & AHCI_TFD_STS_BSY || 3620 port_task_file & AHCI_TFD_STS_DRQ) { 3621 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset: port %d " 3622 "BSY/DRQ still set after device reset " 3623 "port_task_file = 0x%x", instance, 3624 port, port_task_file); 3625 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3626 rval = AHCI_FAILURE; 3627 } 3628 } 3629 3630 goto out; 3631 } 3632 3633 /* 3634 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3635 * This is the most intrusive. 3636 * 3637 * When an HBA reset occurs, Phy communication will be re-established with 3638 * the device through a COMRESET followed by the normal out-of-band 3639 * communication sequence defined in Serial ATA. AT the end of reset, the 3640 * device, if working properly, will send a D2H Register FIS, which contains 3641 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3642 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3643 * signature. 3644 * 3645 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3646 */ 3647 static int 3648 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3649 { 3650 ahci_port_t *ahci_portp; 3651 uint32_t ghc_control; 3652 uint8_t port; 3653 int loop_count; 3654 int rval = AHCI_SUCCESS; 3655 3656 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting"); 3657 3658 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3659 3660 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3661 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3662 3663 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3664 ghc_control |= AHCI_HBA_GHC_HR; 3665 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3666 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3667 3668 /* 3669 * Wait until HBA Reset complete or timeout 3670 */ 3671 loop_count = 0; 3672 do { 3673 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3674 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3675 3676 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3677 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3678 "ahci hba reset is timing out, " 3679 "ghc_control = 0x%x", ghc_control); 3680 /* We are effectively timing out after 1 sec. */ 3681 break; 3682 } 3683 3684 /* Wait for 10 millisec */ 3685 delay(AHCI_10MS_TICKS); 3686 } while (ghc_control & AHCI_HBA_GHC_HR); 3687 3688 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3689 "ahci_hba_reset: 1st loop count: %d, " 3690 "ghc_control = 0x%x", loop_count, ghc_control); 3691 3692 if (ghc_control & AHCI_HBA_GHC_HR) { 3693 /* The hba is not reset for some reasons */ 3694 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3695 "hba reset failed: HBA in a hung or locked state"); 3696 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3697 return (AHCI_FAILURE); 3698 } 3699 3700 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3701 /* Only check implemented ports */ 3702 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3703 continue; 3704 } 3705 3706 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3707 mutex_enter(&ahci_portp->ahciport_mutex); 3708 3709 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3710 != AHCI_SUCCESS) { 3711 rval = AHCI_FAILURE; 3712 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3713 "ahci_hba_reset: port %d failed", port); 3714 } 3715 3716 mutex_exit(&ahci_portp->ahciport_mutex); 3717 } 3718 3719 /* 3720 * Indicate that system software is AHCI aware by setting 3721 * GHC.AE to 1 3722 */ 3723 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3724 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3725 3726 ghc_control |= AHCI_HBA_GHC_AE; 3727 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3728 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3729 3730 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3731 3732 return (rval); 3733 } 3734 3735 /* 3736 * This routine is only called from AHCI_ATTACH or phyrdy change 3737 * case. It first calls port reset to initialize port, probe port and probe 3738 * device, then try to read PxSIG register to find the type of device 3739 * attached to the port. 3740 * 3741 * WARNING!!! ahciport_mutex should be acquired before the function 3742 * is called. And the port interrupt is disabled. 3743 */ 3744 static void 3745 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3746 ahci_port_t *ahci_portp, uint8_t port) 3747 { 3748 uint32_t signature; 3749 3750 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3751 "ahci_find_dev_signature enter: port %d", port); 3752 3753 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3754 3755 /* Call port reset to check link status and get device signature */ 3756 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 3757 3758 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3759 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3760 "ahci_find_dev_signature: No device is found " 3761 "at port %d", port); 3762 return; 3763 } 3764 3765 /* Check the port state */ 3766 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3767 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 3768 "ahci_find_dev_signature: port %d state 0x%x", 3769 port, ahci_portp->ahciport_port_state); 3770 return; 3771 } 3772 3773 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3774 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3775 3776 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3777 "ahci_find_dev_signature: port %d signature = 0x%x", 3778 port, signature); 3779 3780 switch (signature) { 3781 3782 case AHCI_SIGNATURE_DISK: 3783 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3784 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3785 "Disk is found at port: %d", port); 3786 break; 3787 3788 case AHCI_SIGNATURE_ATAPI: 3789 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3790 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3791 "ATAPI device is found at port: %d", port); 3792 break; 3793 3794 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3795 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3796 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3797 "Port Multiplier is found at port: %d", port); 3798 break; 3799 3800 default: 3801 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3802 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3803 "Unknown device is found at port: %d", port); 3804 } 3805 } 3806 3807 /* 3808 * According to the spec, to reliably detect hot plug removals, software 3809 * must disable interface power management. Software should perform the 3810 * following initialization on a port after a device is attached: 3811 * Set PxSCTL.IPM to 3h to disable interface state transitions 3812 * Set PxCMD.ALPE to '0' to disable aggressive power management 3813 * Disable device initiated interface power management by SET FEATURE 3814 * 3815 * We can ignore the last item because by default the feature is disabled 3816 */ 3817 static void 3818 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3819 { 3820 uint32_t port_scontrol, port_cmd_status; 3821 3822 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3823 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3824 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3825 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3826 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3827 3828 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3829 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3830 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3831 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3832 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3833 } 3834 3835 /* 3836 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3837 * to 1, then set it firstly. 3838 * 3839 * Each port contains two major DMA engines. One DMA engine walks through 3840 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3841 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3842 * 3843 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3844 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 3845 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 3846 * 3847 * Software shall not set PxCMD.ST to '1' unless a functional device is 3848 * present on the port(as determined by PxTFD.STS.BSY = '0', 3849 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 3850 * 3851 * WARNING!!! ahciport_mutex should be acquired before the function 3852 * is called. 3853 */ 3854 static int 3855 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3856 { 3857 uint32_t port_cmd_status; 3858 3859 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 3860 3861 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3862 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3863 "the state for port %d is 0x%x", 3864 port, ahci_portp->ahciport_port_state); 3865 return (AHCI_FAILURE); 3866 } 3867 3868 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3869 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3870 "no device is attached at port %d", port); 3871 return (AHCI_FAILURE); 3872 } 3873 3874 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 3875 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3876 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3877 3878 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 3879 port_cmd_status |= AHCI_CMD_STATUS_FRE; 3880 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3881 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3882 port_cmd_status); 3883 } 3884 3885 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3886 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3887 3888 port_cmd_status |= AHCI_CMD_STATUS_ST; 3889 3890 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3891 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3892 port_cmd_status); 3893 3894 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 3895 3896 return (AHCI_SUCCESS); 3897 } 3898 3899 /* 3900 * Allocate the ahci_port_t including Received FIS and Command List. 3901 * The argument - port is the physical port number, and not logical 3902 * port number seen by the SATA framework. 3903 * 3904 * WARNING!!! ahcictl_mutex should be acquired before the function 3905 * is called. 3906 */ 3907 static int 3908 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3909 { 3910 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 3911 ahci_port_t *ahci_portp; 3912 char taskq_name[64] = "event_handle_taskq"; 3913 3914 ahci_portp = 3915 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 3916 3917 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 3918 ahci_portp->ahciport_port_num = port; 3919 3920 /* Intialize the port condition variable */ 3921 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 3922 3923 /* Initialize the port mutex */ 3924 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 3925 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 3926 3927 mutex_enter(&ahci_portp->ahciport_mutex); 3928 3929 /* 3930 * Allocate memory for received FIS structure and 3931 * command list for this port 3932 */ 3933 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3934 goto err_case1; 3935 } 3936 3937 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3938 goto err_case2; 3939 } 3940 3941 (void) snprintf(taskq_name + strlen(taskq_name), 3942 sizeof (taskq_name) - strlen(taskq_name), 3943 "_port%d", port); 3944 3945 /* Create the taskq for the port */ 3946 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 3947 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 3948 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 3949 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 3950 goto err_case3; 3951 } 3952 3953 /* Allocate the argument for the taskq */ 3954 ahci_portp->ahciport_event_args = 3955 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 3956 3957 if (ahci_portp->ahciport_event_args == NULL) 3958 goto err_case4; 3959 3960 mutex_exit(&ahci_portp->ahciport_mutex); 3961 3962 return (AHCI_SUCCESS); 3963 3964 err_case4: 3965 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 3966 3967 err_case3: 3968 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3969 3970 err_case2: 3971 ahci_dealloc_rcvd_fis(ahci_portp); 3972 3973 err_case1: 3974 mutex_exit(&ahci_portp->ahciport_mutex); 3975 mutex_destroy(&ahci_portp->ahciport_mutex); 3976 cv_destroy(&ahci_portp->ahciport_cv); 3977 3978 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3979 3980 return (AHCI_FAILURE); 3981 } 3982 3983 /* 3984 * Reverse of ahci_dealloc_port_state(). 3985 * 3986 * WARNING!!! ahcictl_mutex should be acquired before the function 3987 * is called. 3988 */ 3989 static void 3990 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3991 { 3992 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3993 3994 ASSERT(ahci_portp != NULL); 3995 3996 mutex_enter(&ahci_portp->ahciport_mutex); 3997 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 3998 ahci_portp->ahciport_event_args = NULL; 3999 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 4000 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 4001 ahci_dealloc_rcvd_fis(ahci_portp); 4002 mutex_exit(&ahci_portp->ahciport_mutex); 4003 4004 mutex_destroy(&ahci_portp->ahciport_mutex); 4005 cv_destroy(&ahci_portp->ahciport_cv); 4006 4007 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4008 4009 ahci_ctlp->ahcictl_ports[port] = NULL; 4010 } 4011 4012 /* 4013 * Allocates memory for the Received FIS Structure 4014 * 4015 * WARNING!!! ahciport_mutex should be acquired before the function 4016 * is called. 4017 */ 4018 static int 4019 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4020 uint8_t port) 4021 { 4022 size_t rcvd_fis_size; 4023 size_t ret_len; 4024 uint_t cookie_count; 4025 4026 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 4027 4028 /* allocate rcvd FIS dma handle. */ 4029 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4030 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 4031 DDI_DMA_SLEEP, 4032 NULL, 4033 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 4034 DDI_SUCCESS) { 4035 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4036 "rcvd FIS dma handle alloc failed"); 4037 4038 return (AHCI_FAILURE); 4039 } 4040 4041 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 4042 rcvd_fis_size, 4043 &accattr, 4044 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4045 DDI_DMA_SLEEP, 4046 NULL, 4047 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4048 &ret_len, 4049 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4050 4051 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4052 "rcvd FIS dma mem alloc fail"); 4053 /* error.. free the dma handle. */ 4054 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4055 return (AHCI_FAILURE); 4056 } 4057 4058 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4059 NULL, 4060 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4061 rcvd_fis_size, 4062 DDI_DMA_CONSISTENT, 4063 DDI_DMA_SLEEP, 4064 NULL, 4065 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4066 &cookie_count) != DDI_DMA_MAPPED) { 4067 4068 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4069 "rcvd FIS dma handle bind fail"); 4070 /* error.. free the dma handle & free the memory. */ 4071 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4072 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4073 return (AHCI_FAILURE); 4074 } 4075 4076 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4077 4078 /* Config Port Received FIS Base Address */ 4079 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4080 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4081 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4082 4083 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4084 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4085 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4086 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4087 4088 return (AHCI_SUCCESS); 4089 } 4090 4091 /* 4092 * Deallocates the Received FIS Structure 4093 * 4094 * WARNING!!! ahciport_mutex should be acquired before the function 4095 * is called. 4096 */ 4097 static void 4098 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4099 { 4100 /* Unbind the cmd list dma handle first. */ 4101 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4102 4103 /* Then free the underlying memory. */ 4104 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4105 4106 /* Now free the handle itself. */ 4107 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4108 } 4109 4110 /* 4111 * Allocates memory for the Command List, which contains up to 32 entries. 4112 * Each entry contains a command header, which is a 32-byte structure that 4113 * includes the pointer to the command table. 4114 * 4115 * WARNING!!! ahciport_mutex should be acquired before the function 4116 * is called. 4117 */ 4118 static int 4119 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4120 uint8_t port) 4121 { 4122 size_t cmd_list_size; 4123 size_t ret_len; 4124 uint_t cookie_count; 4125 4126 cmd_list_size = 4127 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4128 4129 /* allocate cmd list dma handle. */ 4130 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4131 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4132 DDI_DMA_SLEEP, 4133 NULL, 4134 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4135 4136 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4137 "cmd list dma handle alloc failed"); 4138 return (AHCI_FAILURE); 4139 } 4140 4141 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4142 cmd_list_size, 4143 &accattr, 4144 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4145 DDI_DMA_SLEEP, 4146 NULL, 4147 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4148 &ret_len, 4149 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4150 4151 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4152 "cmd list dma mem alloc fail"); 4153 /* error.. free the dma handle. */ 4154 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4155 return (AHCI_FAILURE); 4156 } 4157 4158 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4159 NULL, 4160 (caddr_t)ahci_portp->ahciport_cmd_list, 4161 cmd_list_size, 4162 DDI_DMA_CONSISTENT, 4163 DDI_DMA_SLEEP, 4164 NULL, 4165 &ahci_portp->ahciport_cmd_list_dma_cookie, 4166 &cookie_count) != DDI_DMA_MAPPED) { 4167 4168 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4169 "cmd list dma handle bind fail"); 4170 /* error.. free the dma handle & free the memory. */ 4171 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4172 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4173 return (AHCI_FAILURE); 4174 } 4175 4176 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4177 4178 /* Config Port Command List Base Address */ 4179 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4180 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4181 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4182 4183 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4184 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4185 4186 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4187 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4188 4189 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4190 goto err_out; 4191 } 4192 4193 return (AHCI_SUCCESS); 4194 4195 err_out: 4196 /* Unbind the cmd list dma handle first. */ 4197 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4198 4199 /* Then free the underlying memory. */ 4200 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4201 4202 /* Now free the handle itself. */ 4203 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4204 4205 return (AHCI_FAILURE); 4206 } 4207 4208 /* 4209 * Deallocates the Command List 4210 * 4211 * WARNING!!! ahciport_mutex should be acquired before the function 4212 * is called. 4213 */ 4214 static void 4215 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4216 { 4217 /* First dealloc command table */ 4218 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4219 4220 /* Unbind the cmd list dma handle first. */ 4221 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4222 4223 /* Then free the underlying memory. */ 4224 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4225 4226 /* Now free the handle itself. */ 4227 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4228 } 4229 4230 /* 4231 * Allocates memory for all Command Tables, which contains Command FIS, 4232 * ATAPI Command and Physical Region Descriptor Table. 4233 * 4234 * WARNING!!! ahciport_mutex should be acquired before the function 4235 * is called. 4236 */ 4237 static int 4238 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4239 { 4240 size_t ret_len; 4241 ddi_dma_cookie_t cmd_table_dma_cookie; 4242 uint_t cookie_count; 4243 int slot; 4244 4245 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4246 "ahci_alloc_cmd_tables: port %d enter", 4247 ahci_portp->ahciport_port_num); 4248 4249 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4250 /* Allocate cmd table dma handle. */ 4251 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4252 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4253 DDI_DMA_SLEEP, 4254 NULL, 4255 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4256 DDI_SUCCESS) { 4257 4258 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4259 "cmd table dma handle alloc failed"); 4260 4261 goto err_out; 4262 } 4263 4264 if (ddi_dma_mem_alloc( 4265 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4266 ahci_cmd_table_size, 4267 &accattr, 4268 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4269 DDI_DMA_SLEEP, 4270 NULL, 4271 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4272 &ret_len, 4273 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4274 NULL) { 4275 4276 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4277 "cmd table dma mem alloc fail"); 4278 4279 /* error.. free the dma handle. */ 4280 ddi_dma_free_handle( 4281 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4282 goto err_out; 4283 } 4284 4285 if (ddi_dma_addr_bind_handle( 4286 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4287 NULL, 4288 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4289 ahci_cmd_table_size, 4290 DDI_DMA_CONSISTENT, 4291 DDI_DMA_SLEEP, 4292 NULL, 4293 &cmd_table_dma_cookie, 4294 &cookie_count) != DDI_DMA_MAPPED) { 4295 4296 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4297 "cmd table dma handle bind fail"); 4298 /* error.. free the dma handle & free the memory. */ 4299 ddi_dma_mem_free( 4300 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4301 ddi_dma_free_handle( 4302 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4303 goto err_out; 4304 } 4305 4306 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4307 ahci_cmd_table_size); 4308 4309 /* Config Port Command Table Base Address */ 4310 SET_COMMAND_TABLE_BASE_ADDR( 4311 (&ahci_portp->ahciport_cmd_list[slot]), 4312 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4313 4314 #ifndef __lock_lint 4315 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4316 (&ahci_portp->ahciport_cmd_list[slot]), 4317 cmd_table_dma_cookie.dmac_laddress >> 32); 4318 #endif 4319 } 4320 4321 return (AHCI_SUCCESS); 4322 err_out: 4323 4324 for (slot--; slot >= 0; slot--) { 4325 /* Unbind the cmd table dma handle first */ 4326 (void) ddi_dma_unbind_handle( 4327 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4328 4329 /* Then free the underlying memory */ 4330 ddi_dma_mem_free( 4331 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4332 4333 /* Now free the handle itself */ 4334 ddi_dma_free_handle( 4335 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4336 } 4337 4338 return (AHCI_FAILURE); 4339 } 4340 4341 /* 4342 * Deallocates memory for all Command Tables. 4343 * 4344 * WARNING!!! ahciport_mutex should be acquired before the function 4345 * is called. 4346 */ 4347 static void 4348 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4349 { 4350 int slot; 4351 4352 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 4353 "ahci_dealloc_cmd_tables: %d enter", 4354 ahci_portp->ahciport_port_num); 4355 4356 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4357 /* Unbind the cmd table dma handle first. */ 4358 (void) ddi_dma_unbind_handle( 4359 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4360 4361 /* Then free the underlying memory. */ 4362 ddi_dma_mem_free( 4363 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4364 4365 /* Now free the handle itself. */ 4366 ddi_dma_free_handle( 4367 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4368 } 4369 } 4370 4371 /* 4372 * WARNING!!! ahciport_mutex should be acquired before the function 4373 * is called. 4374 */ 4375 static void 4376 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4377 sata_device_t *sd) 4378 { 4379 sd->satadev_scr.sstatus = 4380 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4381 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4382 sd->satadev_scr.serror = 4383 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4384 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4385 sd->satadev_scr.scontrol = 4386 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4387 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4388 sd->satadev_scr.sactive = 4389 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4390 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4391 } 4392 4393 /* 4394 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4395 */ 4396 static void 4397 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4398 { 4399 uint32_t port_intr_status; 4400 uint32_t port_intr_enable; 4401 4402 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4403 "ahci_port_intr enter: port %d", port); 4404 4405 mutex_enter(&ahci_portp->ahciport_mutex); 4406 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4407 /* For SATA_OPMODE_POLLING commands */ 4408 port_intr_enable = 4409 (AHCI_INTR_STATUS_DHRS | 4410 AHCI_INTR_STATUS_PSS | 4411 AHCI_INTR_STATUS_SDBS | 4412 AHCI_INTR_STATUS_UFS | 4413 AHCI_INTR_STATUS_PCS | 4414 AHCI_INTR_STATUS_PRCS | 4415 AHCI_INTR_STATUS_OFS | 4416 AHCI_INTR_STATUS_INFS | 4417 AHCI_INTR_STATUS_IFS | 4418 AHCI_INTR_STATUS_HBDS | 4419 AHCI_INTR_STATUS_HBFS | 4420 AHCI_INTR_STATUS_TFES); 4421 mutex_exit(&ahci_portp->ahciport_mutex); 4422 goto next; 4423 } 4424 mutex_exit(&ahci_portp->ahciport_mutex); 4425 4426 /* 4427 * port_intr_enable indicates that the corresponding interrrupt 4428 * reporting is enabled. 4429 */ 4430 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4431 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4432 next: 4433 /* 4434 * port_intr_stats indicates that the corresponding interrupt 4435 * condition is active. 4436 */ 4437 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4438 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4439 4440 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4441 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4442 "port_intr_enable = 0x%x", 4443 port, port_intr_status, port_intr_enable); 4444 4445 port_intr_status &= port_intr_enable; 4446 4447 /* First clear the port interrupts status */ 4448 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4449 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4450 port_intr_status); 4451 4452 /* Check the completed non-queued commands */ 4453 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4454 AHCI_INTR_STATUS_PSS)) { 4455 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4456 ahci_portp, port); 4457 } 4458 4459 /* Check the completed queued commands */ 4460 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4461 (void) ahci_intr_set_device_bits(ahci_ctlp, 4462 ahci_portp, port); 4463 } 4464 4465 /* Check the port connect change status interrupt bit */ 4466 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4467 (void) ahci_intr_port_connect_change(ahci_ctlp, 4468 ahci_portp, port); 4469 } 4470 4471 /* Check the device mechanical presence status interrupt bit */ 4472 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4473 (void) ahci_intr_device_mechanical_presence_status( 4474 ahci_ctlp, ahci_portp, port); 4475 } 4476 4477 /* Check the PhyRdy change status interrupt bit */ 4478 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4479 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4480 port); 4481 } 4482 4483 /* 4484 * Check the non-fatal error interrupt bits, there are three 4485 * kinds of non-fatal errors at the time being: 4486 * 4487 * PxIS.UFS - Unknown FIS Error 4488 * PxIS.OFS - Overflow Error 4489 * PxIS.INFS - Interface Non-Fatal Error 4490 * 4491 * For these non-fatal errors, the HBA can continue to operate, 4492 * so the driver just log the error messages. 4493 */ 4494 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4495 AHCI_INTR_STATUS_OFS | 4496 AHCI_INTR_STATUS_INFS)) { 4497 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4498 port, port_intr_status); 4499 } 4500 4501 /* 4502 * Check the fatal error interrupt bits, there are four kinds 4503 * of fatal errors for AHCI controllers: 4504 * 4505 * PxIS.HBFS - Host Bus Fatal Error 4506 * PxIS.HBDS - Host Bus Data Error 4507 * PxIS.IFS - Interface Fatal Error 4508 * PxIS.TFES - Task File Error 4509 * 4510 * The fatal error means the HBA can not recover from it by 4511 * itself, and it will try to abort the transfer, and the software 4512 * must intervene to restart the port. 4513 */ 4514 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4515 AHCI_INTR_STATUS_HBDS | 4516 AHCI_INTR_STATUS_HBFS | 4517 AHCI_INTR_STATUS_TFES)) 4518 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4519 port, port_intr_status); 4520 4521 /* Check the cold port detect interrupt bit */ 4522 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4523 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4524 } 4525 4526 /* Second clear the corresponding bit in IS.IPS */ 4527 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4528 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4529 } 4530 4531 /* 4532 * Interrupt service handler 4533 */ 4534 static uint_t 4535 ahci_intr(caddr_t arg1, caddr_t arg2) 4536 { 4537 #ifndef __lock_lint 4538 _NOTE(ARGUNUSED(arg2)) 4539 #endif 4540 /* LINTED */ 4541 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4542 ahci_port_t *ahci_portp; 4543 int32_t global_intr_status; 4544 uint8_t port; 4545 4546 /* 4547 * global_intr_status indicates that the corresponding port has 4548 * an interrupt pending. 4549 */ 4550 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4551 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4552 4553 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4554 /* The interrupt is not ours */ 4555 return (DDI_INTR_UNCLAIMED); 4556 } 4557 4558 /* Loop for all the ports */ 4559 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4560 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4561 continue; 4562 } 4563 if (!((0x1 << port) & global_intr_status)) { 4564 continue; 4565 } 4566 4567 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4568 4569 /* Call ahci_port_intr */ 4570 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4571 } 4572 4573 return (DDI_INTR_CLAIMED); 4574 } 4575 4576 /* 4577 * For non-queued commands, when the corresponding bit in the PxCI register 4578 * is cleared, it means the command is completed successfully. And according 4579 * to the HBA state machine, there are three conditions which possibly will 4580 * try to clear the PxCI register bit. 4581 * 1. Receive one D2H Register FIS which is with 'I' bit set 4582 * 2. Update PIO Setup FIS 4583 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4584 * 4585 * Process completed non-queued commands when the interrupt status bit - 4586 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4587 * 4588 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4589 * with the 'I' bit set. And the following commands will send thus 4590 * FIS with 'I' bit set upon the successful completion: 4591 * 1. Non-data commands 4592 * 2. DMA data-in command 4593 * 3. DMA data-out command 4594 * 4. PIO data-out command 4595 * 5. PACKET non-data commands 4596 * 6. PACKET PIO data-in command 4597 * 7. PACKET PIO data-out command 4598 * 8. PACKET DMA data-in command 4599 * 9. PACKET DMA data-out command 4600 * 4601 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4602 * with the 'I' bit set. And the following commands will send this 4603 * FIS upon the successful completion: 4604 * 1. PIO data-in command 4605 */ 4606 static int 4607 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4608 ahci_port_t *ahci_portp, uint8_t port) 4609 { 4610 uint32_t port_cmd_issue = 0; 4611 uint32_t finished_tags; 4612 int finished_slot; 4613 sata_pkt_t *satapkt; 4614 ahci_fis_d2h_register_t *rcvd_fisp; 4615 #if AHCI_DEBUG 4616 ahci_cmd_header_t *cmd_header; 4617 uint32_t cmd_dmacount; 4618 #endif 4619 4620 mutex_enter(&ahci_portp->ahciport_mutex); 4621 4622 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4623 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4624 /* 4625 * Spurious interrupt. Nothing to be done. 4626 */ 4627 mutex_exit(&ahci_portp->ahciport_mutex); 4628 return (AHCI_SUCCESS); 4629 } 4630 4631 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4632 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4633 4634 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4635 /* Slot 0 is always used during error recovery */ 4636 finished_tags = 0x1 & ~port_cmd_issue; 4637 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 4638 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4639 "retrieval is finished, and finished_tags = 0x%x", 4640 port, finished_tags); 4641 } else { 4642 finished_tags = ahci_portp->ahciport_pending_tags & 4643 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4644 } 4645 4646 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4647 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4648 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4649 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4650 finished_tags); 4651 4652 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4653 (finished_tags == 0x1)) { 4654 satapkt = ahci_portp->ahciport_err_retri_pkt; 4655 ASSERT(satapkt != NULL); 4656 4657 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4658 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4659 "with SATA_PKT_COMPLETED", (void *)satapkt); 4660 4661 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4662 goto out; 4663 } 4664 4665 while (finished_tags) { 4666 finished_slot = ddi_ffs(finished_tags) - 1; 4667 if (finished_slot == -1) { 4668 goto out; 4669 } 4670 4671 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4672 ASSERT(satapkt != NULL); 4673 #if AHCI_DEBUG 4674 /* 4675 * For non-native queued commands, the PRD byte count field 4676 * shall contain an accurate count of the number of bytes 4677 * transferred for the command before the PxCI bit is cleared 4678 * to '0' for the command. 4679 * 4680 * The purpose of this field is to let software know how many 4681 * bytes transferred for a given operation in order to 4682 * determine if underflow occurred. When issuing native command 4683 * queuing commands, this field should not be used and is not 4684 * required to be valid since in this case underflow is always 4685 * illegal. 4686 * 4687 * For data reads, the HBA will update its PRD byte count with 4688 * the total number of bytes received from the last FIS, and 4689 * may be able to continue normally. For data writes, the 4690 * device will detect an error, and HBA most likely will get 4691 * a fatal error. 4692 * 4693 * Therefore, here just put code to debug part. And please 4694 * refer to the comment above ahci_intr_fatal_error for the 4695 * definition of underflow error. 4696 */ 4697 cmd_dmacount = 4698 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4699 if (cmd_dmacount) { 4700 cmd_header = 4701 &ahci_portp->ahciport_cmd_list[finished_slot]; 4702 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4703 "ahci_intr_cmd_cmplt: port %d, " 4704 "PRD Byte Count = 0x%x, " 4705 "ahciport_prd_bytecounts = 0x%x", port, 4706 cmd_header->ahcich_prd_byte_count, 4707 cmd_dmacount); 4708 4709 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4710 AHCIDBG1(AHCIDBG_UNDERFLOW, ahci_ctlp, 4711 "ahci_intr_cmd_cmplt: port %d, " 4712 "an underflow occurred", port); 4713 } 4714 } 4715 #endif 4716 4717 /* 4718 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4719 * feature, sata_special_regs flag will be set, and the 4720 * driver should copy the status and the other corresponding 4721 * register values in the D2H Register FIS received (It's 4722 * working on Non-data protocol) from the device back to 4723 * the sata_cmd. 4724 * 4725 * For every AHCI port, there is only one Received FIS 4726 * structure, which contains the FISes received from the 4727 * device, So we're trying to copy the content of D2H 4728 * Register FIS in the Received FIS structure back to 4729 * the sata_cmd. 4730 */ 4731 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4732 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4733 ahcirf_d2h_register_fis); 4734 satapkt->satapkt_cmd.satacmd_status_reg = 4735 GET_RFIS_STATUS(rcvd_fisp); 4736 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4737 } 4738 4739 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4740 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4741 "with SATA_PKT_COMPLETED", (void *)satapkt); 4742 4743 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4744 CLEAR_BIT(finished_tags, finished_slot); 4745 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4746 4747 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4748 } 4749 out: 4750 AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp, 4751 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4752 ahci_portp->ahciport_pending_tags); 4753 4754 mutex_exit(&ahci_portp->ahciport_mutex); 4755 4756 return (AHCI_SUCCESS); 4757 } 4758 4759 /* 4760 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4761 * with the 'I' bit set and has been copied into system memory. It will 4762 * be sent under the following situations: 4763 * 4764 * 1. NCQ command is completed 4765 * 2. Asynchronous notification 4766 * 4767 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4768 * via the Set Device Bits FIS. When such event is generated, the software 4769 * needs to read PxSACT register and compares the current value to the 4770 * list of commands previously issue by software. ahciport_pending_ncq_tags 4771 * keeps the tags of previously issued commands. 4772 * 4773 * Asynchronous Notification is a feature in SATA II, which allows an 4774 * ATAPI device to send a signal to the host when media is inserted or 4775 * removed and avoids polling the device for media changes. The signal 4776 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4777 * set to '1'. At the moment, it's not supported yet. 4778 */ 4779 static int 4780 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4781 ahci_port_t *ahci_portp, uint8_t port) 4782 { 4783 uint32_t port_sactive; 4784 uint32_t port_cmd_issue; 4785 uint32_t issued_tags; 4786 int issued_slot; 4787 uint32_t finished_tags; 4788 int finished_slot; 4789 sata_pkt_t *satapkt; 4790 4791 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4792 "ahci_intr_set_device_bits enter: port %d", port); 4793 4794 mutex_enter(&ahci_portp->ahciport_mutex); 4795 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4796 mutex_exit(&ahci_portp->ahciport_mutex); 4797 return (AHCI_SUCCESS); 4798 } 4799 4800 /* 4801 * First the handler got which commands are finished by checking 4802 * PxSACT register 4803 */ 4804 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4805 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4806 4807 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4808 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4809 4810 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4811 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4812 "port_sactive = 0x%x", port, 4813 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4814 4815 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4816 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4817 4818 /* 4819 * For NCQ commands, the software can determine which command has 4820 * already been transmitted to the device by checking PxCI register. 4821 */ 4822 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4823 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4824 4825 issued_tags = ahci_portp->ahciport_pending_tags & 4826 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4827 4828 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4829 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4830 "port_cmd_issue = 0x%x", port, 4831 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4832 4833 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4834 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4835 4836 /* 4837 * Clear ahciport_pending_tags bit when the corresponding command 4838 * is already sent down to the device. 4839 */ 4840 while (issued_tags) { 4841 issued_slot = ddi_ffs(issued_tags) - 1; 4842 if (issued_slot == -1) { 4843 goto next; 4844 } 4845 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 4846 CLEAR_BIT(issued_tags, issued_slot); 4847 } 4848 4849 next: 4850 while (finished_tags) { 4851 finished_slot = ddi_ffs(finished_tags) - 1; 4852 if (finished_slot == -1) { 4853 goto out; 4854 } 4855 4856 /* The command is certainly transmitted to the device */ 4857 ASSERT(!(ahci_portp->ahciport_pending_tags & 4858 (0x1 << finished_slot))); 4859 4860 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4861 ASSERT(satapkt != NULL); 4862 4863 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4864 "ahci_intr_set_device_bits: sending up pkt 0x%p " 4865 "with SATA_PKT_COMPLETED", (void *)satapkt); 4866 4867 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 4868 CLEAR_BIT(finished_tags, finished_slot); 4869 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4870 4871 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4872 } 4873 out: 4874 AHCIDBG3(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 4875 "ahci_intr_set_device_bits: port %d " 4876 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 4877 port, ahci_portp->ahciport_pending_ncq_tags, 4878 ahci_portp->ahciport_pending_tags); 4879 4880 mutex_exit(&ahci_portp->ahciport_mutex); 4881 4882 return (AHCI_SUCCESS); 4883 } 4884 4885 /* 4886 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 4887 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 4888 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 4889 * indicates a COMINIT signal was received. 4890 * 4891 * Hot plug insertion is detected by reception of a COMINIT signal from the 4892 * device. On reception of unsolicited COMINIT, the HBA shall generate a 4893 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 4894 * begin the normal communication negotiation sequence as outlined in the 4895 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 4896 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 4897 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 4898 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 4899 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 4900 * to handle hot plug insertion. In this interrupt handler, just do nothing 4901 * but print some log message and clear the bit. 4902 */ 4903 static int 4904 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 4905 ahci_port_t *ahci_portp, uint8_t port) 4906 { 4907 #if AHCI_DEBUG 4908 uint32_t port_serror; 4909 #endif 4910 4911 mutex_enter(&ahci_portp->ahciport_mutex); 4912 4913 #if AHCI_DEBUG 4914 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4915 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 4916 4917 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4918 "ahci_intr_port_connect_change: port %d, " 4919 "port_serror = 0x%x", port, port_serror); 4920 #endif 4921 4922 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 4923 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4924 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 4925 SERROR_EXCHANGED_ERR); 4926 4927 mutex_exit(&ahci_portp->ahciport_mutex); 4928 4929 return (AHCI_SUCCESS); 4930 } 4931 4932 /* 4933 * Hot Plug Operation for platforms that support Mechanical Presence 4934 * Switches. 4935 * 4936 * When set, it indicates that a mechanical presence switch attached to this 4937 * port has been opened or closed, which may lead to a change in the connection 4938 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 4939 * are set to '1'. 4940 * 4941 * At the moment, this interrupt is not needed and disabled and we just log 4942 * the debug message. 4943 */ 4944 static int 4945 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 4946 ahci_port_t *ahci_portp, uint8_t port) 4947 { 4948 uint32_t cap_status, port_cmd_status; 4949 4950 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4951 "ahci_intr_device_mechanical_presence_status enter, " 4952 "port %d", port); 4953 4954 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4955 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4956 4957 mutex_enter(&ahci_portp->ahciport_mutex); 4958 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4959 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4960 4961 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 4962 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 4963 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4964 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 4965 "the interrupt: cap_status = 0x%x, " 4966 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 4967 mutex_exit(&ahci_portp->ahciport_mutex); 4968 4969 return (AHCI_SUCCESS); 4970 } 4971 4972 #if AHCI_DEBUG 4973 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 4974 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4975 "The mechanical presence switch is open: " 4976 "port %d, port_cmd_status = 0x%x", 4977 port, port_cmd_status); 4978 } else { 4979 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4980 "The mechanical presence switch is close: " 4981 "port %d, port_cmd_status = 0x%x", 4982 port, port_cmd_status); 4983 } 4984 #endif 4985 4986 mutex_exit(&ahci_portp->ahciport_mutex); 4987 4988 return (AHCI_SUCCESS); 4989 } 4990 4991 /* 4992 * Native Hot Plug Support. 4993 * 4994 * When set, it indicates that the internal PHYRDY signal changed state. 4995 * This bit reflects the state of PxSERR.DIAG.N. 4996 * 4997 * There are three kinds of conditions to generate this interrupt event: 4998 * 1. a device is inserted 4999 * 2. a device is disconnected 5000 * 3. when the link enters/exits a Partial or Slumber interface power 5001 * management state 5002 * 5003 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 5004 * bit may be set due to the link entering the Partial or Slumber power 5005 * management state, rather than due to a hot plug insertion or removal 5006 * event. So far, the interface power management is disabled, so the 5007 * driver can reliably get removal detection notification via the 5008 * PxSERR.DIAG.N bit. 5009 */ 5010 static int 5011 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 5012 ahci_port_t *ahci_portp, uint8_t port) 5013 { 5014 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 5015 sata_device_t sdevice; 5016 int dev_exists_now = 0; 5017 int dev_existed_previously = 0; 5018 5019 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5020 "ahci_intr_phyrdy_change enter, port %d", port); 5021 5022 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 5023 mutex_enter(&ahci_portp->ahciport_mutex); 5024 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5025 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5026 SERROR_PHY_RDY_CHG); 5027 mutex_exit(&ahci_portp->ahciport_mutex); 5028 5029 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5030 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 5031 (ahci_portp == NULL)) { 5032 /* The whole controller setup is not yet done. */ 5033 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5034 return (AHCI_SUCCESS); 5035 } 5036 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5037 5038 mutex_enter(&ahci_portp->ahciport_mutex); 5039 5040 /* SStatus tells the presence of device. */ 5041 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5042 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5043 5044 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5045 dev_exists_now = 1; 5046 } 5047 5048 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5049 dev_existed_previously = 1; 5050 } 5051 5052 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5053 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5054 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5055 "ahci_intr_phyrdy_change: port %d " 5056 "AHCI_PORT_FLAG_NODEV is cleared", port); 5057 if (dev_exists_now == 0) 5058 dev_existed_previously = 1; 5059 } 5060 5061 bzero((void *)&sdevice, sizeof (sata_device_t)); 5062 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5063 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5064 sdevice.satadev_addr.pmport = 0; 5065 sdevice.satadev_state = SATA_PSTATE_PWRON; 5066 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5067 5068 if (dev_exists_now) { 5069 if (dev_existed_previously) { 5070 /* Things are fine now. The loss was temporary. */ 5071 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5072 "ahci_intr_phyrdy_change port %d " 5073 "device link lost/established", port); 5074 5075 mutex_exit(&ahci_portp->ahciport_mutex); 5076 sata_hba_event_notify( 5077 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5078 &sdevice, 5079 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5080 mutex_enter(&ahci_portp->ahciport_mutex); 5081 5082 } else { 5083 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5084 "ahci_intr_phyrdy_change: port %d " 5085 "device link established", port); 5086 5087 /* A new device has been detected. */ 5088 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5089 5090 /* Try to start the port */ 5091 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5092 != AHCI_SUCCESS) { 5093 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5094 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5095 "ahci_intr_phyrdy_change: port %d failed " 5096 "at start port", port); 5097 } 5098 5099 /* Clear the max queue depth for inserted device */ 5100 ahci_portp->ahciport_max_ncq_tags = 0; 5101 5102 mutex_exit(&ahci_portp->ahciport_mutex); 5103 sata_hba_event_notify( 5104 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5105 &sdevice, 5106 SATA_EVNT_LINK_ESTABLISHED); 5107 mutex_enter(&ahci_portp->ahciport_mutex); 5108 5109 } 5110 } else { /* No device exists now */ 5111 5112 if (dev_existed_previously) { 5113 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5114 "ahci_intr_phyrdy_change: port %d " 5115 "device link lost", port); 5116 5117 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5118 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5119 ahci_portp, port); 5120 5121 /* An existing device is lost. */ 5122 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5123 5124 mutex_exit(&ahci_portp->ahciport_mutex); 5125 sata_hba_event_notify( 5126 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5127 &sdevice, 5128 SATA_EVNT_LINK_LOST); 5129 mutex_enter(&ahci_portp->ahciport_mutex); 5130 } 5131 } 5132 5133 mutex_exit(&ahci_portp->ahciport_mutex); 5134 5135 return (AHCI_SUCCESS); 5136 } 5137 5138 /* 5139 * PxIS.UFS - Unknown FIS Error 5140 * 5141 * This interrupt event means an unknown FIS was received and has been 5142 * copied into system memory. An unknown FIS is not considered an illegal 5143 * FIS, unless the length received is more than 64 bytes. If an unknown 5144 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5145 * normal operation. If the unknown FIS is more than 64 bytes, then it 5146 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5147 * a fatal error. 5148 * 5149 * PxIS.OFS - Overflow Error 5150 * 5151 * Command list overflow is defined as software building a command table 5152 * that has fewer total bytes than the transaction given to the device. 5153 * On device writes, the HBA will run out of data, and on reads, there 5154 * will be no room to put the data. 5155 * 5156 * For an overflow on data read, either PIO or DMA, the HBA will set 5157 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5158 * non-fatal error when the HBA can continues. Sometimes, it will cause 5159 * a fatal error and need the software to do something. 5160 * 5161 * For an overflow on data write, setting PxIS.OFS is optional for both 5162 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5163 * software to clean up from this serious error. 5164 * 5165 * PxIS.INFS - Interface Non-Fatal Error 5166 * 5167 * This interrupt event indicates that the HBA encountered an error on 5168 * the Serial ATA interface but was able to continue operation. The kind 5169 * of error usually occurred during a non-Data FIS, and under this condition 5170 * the FIS will be re-transmitted by HBA automatically. 5171 * 5172 * When the FMA is implemented, there should be a stat structure to 5173 * record how many every kind of error happens. 5174 */ 5175 static int 5176 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5177 uint8_t port, uint32_t intr_status) 5178 { 5179 uint32_t port_serror; 5180 #if AHCI_DEBUG 5181 uint32_t port_cmd_status; 5182 uint32_t port_cmd_issue; 5183 uint32_t port_sactive; 5184 int current_slot; 5185 uint32_t current_tags; 5186 sata_pkt_t *satapkt; 5187 ahci_cmd_header_t *cmd_header; 5188 uint32_t cmd_dmacount; 5189 #endif 5190 5191 mutex_enter(&ahci_portp->ahciport_mutex); 5192 5193 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5194 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5195 5196 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5197 "ahci_intr_non_fatal_error: port %d, " 5198 "port_serror = 0x%x", port, port_serror); 5199 5200 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5201 5202 if (intr_status & AHCI_INTR_STATUS_UFS) { 5203 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5204 "ahci port %d has unknown FIS error", port); 5205 5206 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5207 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5208 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5209 SERROR_FIS_TYPE); 5210 } 5211 5212 #if AHCI_DEBUG 5213 if (intr_status & AHCI_INTR_STATUS_OFS) { 5214 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5215 "ahci port %d has overflow error", port); 5216 } 5217 5218 if (intr_status & AHCI_INTR_STATUS_INFS) { 5219 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5220 "ahci port %d has interface non fatal error", port); 5221 } 5222 5223 /* 5224 * Record the error occurred command's slot. 5225 */ 5226 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5227 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5228 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5229 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5230 5231 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5232 AHCI_CMD_STATUS_CCS_SHIFT; 5233 5234 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5235 satapkt = ahci_portp->ahciport_err_retri_pkt; 5236 ASSERT(satapkt != NULL); 5237 ASSERT(current_slot == 0); 5238 } else { 5239 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5240 } 5241 5242 if (satapkt != NULL) { 5243 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5244 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5245 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5246 satapkt->satapkt_cmd.satacmd_cmd_reg); 5247 5248 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5249 "ahci_intr_non_fatal_error: port %d, " 5250 "satapkt 0x%p is being processed when error occurs", 5251 port, (void *)satapkt); 5252 5253 /* 5254 * PRD Byte Count field of command header is not 5255 * required to reflect the total number of bytes 5256 * transferred when an overflow occurs, so here 5257 * just log the value. 5258 */ 5259 cmd_dmacount = 5260 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5261 if (cmd_dmacount) { 5262 cmd_header = &ahci_portp-> 5263 ahciport_cmd_list[current_slot]; 5264 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5265 "ahci_intr_non_fatal_error: port %d, " 5266 "PRD Byte Count = 0x%x, " 5267 "ahciport_prd_bytecounts = 0x%x", port, 5268 cmd_header->ahcich_prd_byte_count, 5269 cmd_dmacount); 5270 } 5271 } 5272 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5273 /* 5274 * For queued command, list those command which have already 5275 * been transmitted to the device and still not completed. 5276 */ 5277 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5278 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5279 5280 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5281 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5282 5283 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5284 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5285 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5286 ahci_portp->ahciport_pending_ncq_tags, 5287 port_sactive, port_cmd_issue); 5288 5289 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5290 port_sactive & ~port_cmd_issue & 5291 AHCI_NCQ_SLOT_MASK(ahci_portp); 5292 5293 while (current_tags) { 5294 current_slot = ddi_ffs(current_tags) - 1; 5295 if (current_slot == -1) { 5296 goto out; 5297 } 5298 5299 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5300 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5301 ahci_ctlp, "ahci_intr_non_fatal_error: " 5302 "port %d, satapkt 0x%p is outstanding when " 5303 "error occurs", port, (void *)satapkt); 5304 } 5305 } 5306 out: 5307 #endif 5308 mutex_exit(&ahci_portp->ahciport_mutex); 5309 5310 return (AHCI_SUCCESS); 5311 } 5312 5313 /* 5314 * According to the AHCI spec, the error types include system memory 5315 * errors, interface errors, port multiplier errors, device errors, 5316 * command list overflow, command list underflow, native command 5317 * queuing tag errors and pio data transfer errors. 5318 * 5319 * System memory errors such as target abort, master abort, and parity 5320 * may cause the host to stop, and they are serious errors and needed 5321 * to be recovered with software intervention. When system software 5322 * has given a pointer to the HBA that doesn't exist in physical memory, 5323 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5324 * data error such as CRC or parity occurs, the HBA aborts the transfer 5325 * (if necessary) and PxIS.HBDS will be set. 5326 * 5327 * Interface errors are errors that occur due to electrical issues on 5328 * the interface, or protocol miscommunication between the device and 5329 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5330 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5331 * causes PxIS.IFS/PxIS.INFS to be set are 5332 * 1. in PxSERR.ERR, P bit is set to '1' 5333 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5334 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5335 * If the error occurred during a non-data FIS, the FIS must be 5336 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5337 * the error occurred during a data FIS, the transfer will stop, so 5338 * the error is fatal and PxIS.IFS is set. 5339 * 5340 * When a FIS arrives that updates the taskfile, the HBA checks to see 5341 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5342 * stops processing any more commands. 5343 * 5344 * Command list overflow is defined as software building a command table 5345 * that has fewer total bytes than the transaction given to the device. 5346 * On device writes, the HBA will run out of data, and on reads, there 5347 * will be no room to put the data. For an overflow on data read, either 5348 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5349 * For an overflow on data write, setting PxIS.OFS is optional for both 5350 * DMA and PIO, and a COMRESET is required by software to clean up from 5351 * this serious error. 5352 * 5353 * Command list underflow is defined as software building a command 5354 * table that has more total bytes than the transaction given to the 5355 * device. For data writes, both PIO and DMA, the device will detect 5356 * an error and end the transfer. And these errors are most likely going 5357 * to be fatal errors that will cause the port to be restarted. For 5358 * data reads, the HBA updates its PRD byte count, and may be 5359 * able to continue normally, but is not required to. And The HBA is 5360 * not required to detect underflow conditions for native command 5361 * queuing command. 5362 * 5363 * The HBA does not actively check incoming DMA Setup FISes to ensure 5364 * that the PxSACT register bit for that slot is set. Existing error 5365 * mechanisms, such as host bus failure, or bad protocol, are used to 5366 * recover from this case. 5367 * 5368 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5369 * DATA FIS must be an integral number of Dwords. If the HBA receives 5370 * a request which is not an integral number of Dwords, the HBA 5371 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5372 * software restarts the port. And the HBA ensures that the size 5373 * of the DATA FIS received during a PIO command matches the size in 5374 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5375 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5376 * stop running until software restarts the port. 5377 */ 5378 /* 5379 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5380 * 5381 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5382 * interface which caused the transfer to stop. 5383 * 5384 * PxIS.HBDS indicates that the hba encountered a data error 5385 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5386 * 5387 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5388 * cannot recover from, such as a bad software pointer. 5389 * 5390 * PxIS.TFES is set whenever the status register is updated by the device 5391 * and the error bit (bit 0) is set. 5392 */ 5393 static int 5394 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5395 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5396 { 5397 uint32_t port_cmd_status; 5398 uint32_t port_serror; 5399 uint32_t task_file_status; 5400 int failed_slot; 5401 sata_pkt_t *spkt = NULL; 5402 uint8_t err_byte; 5403 ahci_event_arg_t *args; 5404 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5405 5406 mutex_enter(&ahci_portp->ahciport_mutex); 5407 5408 /* 5409 * ahci_intr_phyrdy_change() may have rendered it to 5410 * SATA_DTYPE_NONE. 5411 */ 5412 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5413 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5414 "ahci_intr_fatal_error: port %d no device attached, " 5415 "and just return without doing anything", port); 5416 goto out0; 5417 } 5418 5419 if (intr_status & AHCI_INTR_STATUS_TFES) { 5420 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5421 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5422 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5423 "ahci_intr_fatal_error: port %d " 5424 "task_file_status = 0x%x", port, task_file_status); 5425 } 5426 5427 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5428 /* 5429 * Read PxCMD.CCS to determine the slot that the HBA 5430 * was processing when the error occurred. 5431 */ 5432 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5433 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5434 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5435 AHCI_CMD_STATUS_CCS_SHIFT; 5436 5437 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5438 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5439 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5440 "fatal error occurred for port %d", spkt, port); 5441 5442 if (intr_status & AHCI_INTR_STATUS_TFES) { 5443 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5444 >> AHCI_TFD_ERR_SHIFT; 5445 5446 /* 5447 * Won't emit the error message if it is an IDENTIFY 5448 * DEVICE command sent to an ATAPI device. 5449 */ 5450 if ((spkt != NULL) && 5451 (spkt->satapkt_cmd.satacmd_cmd_reg == 5452 SATAC_ID_DEVICE) && 5453 (err_byte == SATA_ERROR_ABORT)) 5454 goto out1; 5455 5456 /* 5457 * Won't emit the error message if it is an ATAPI PACKET 5458 * command 5459 */ 5460 if ((spkt != NULL) && 5461 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5462 goto out1; 5463 } 5464 } 5465 5466 /* print the fatal error type */ 5467 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5468 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5469 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5470 5471 /* print PxSERR related error message */ 5472 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5473 5474 /* print task file register value */ 5475 if (intr_status & AHCI_INTR_STATUS_TFES) { 5476 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 5477 "= 0x%x", instance, port, task_file_status); 5478 } 5479 5480 out1: 5481 /* Prepare the argument for the taskq */ 5482 args = ahci_portp->ahciport_event_args; 5483 args->ahciea_ctlp = (void *)ahci_ctlp; 5484 args->ahciea_portp = (void *)ahci_portp; 5485 args->ahciea_event = intr_status; 5486 5487 /* Start the taskq to handle error recovery */ 5488 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 5489 ahci_events_handler, 5490 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5491 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5492 "failed", instance); 5493 } 5494 out0: 5495 mutex_exit(&ahci_portp->ahciport_mutex); 5496 5497 return (AHCI_SUCCESS); 5498 } 5499 5500 /* 5501 * Hot Plug Operation for platforms that support Cold Presence Detect. 5502 * 5503 * When set, a device status has changed as detected by the cold presence 5504 * detect logic. This bit can either be set due to a non-connected port 5505 * receiving a device, or a connected port having its device removed. 5506 * This bit is only valid if the port supports cold presence detect as 5507 * indicated by PxCMD.CPD set to '1'. 5508 * 5509 * At the moment, this interrupt is not needed and disabled and we just 5510 * log the debug message. 5511 */ 5512 static int 5513 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5514 ahci_port_t *ahci_portp, uint8_t port) 5515 { 5516 uint32_t port_cmd_status; 5517 sata_device_t sdevice; 5518 5519 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5520 "ahci_intr_cold_port_detect enter, port %d", port); 5521 5522 mutex_enter(&ahci_portp->ahciport_mutex); 5523 5524 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5525 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5526 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5527 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5528 "port %d does not support cold presence detect, so " 5529 "we just ignore this interrupt", port); 5530 mutex_exit(&ahci_portp->ahciport_mutex); 5531 return (AHCI_SUCCESS); 5532 } 5533 5534 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5535 "port %d device status has changed", port); 5536 5537 bzero((void *)&sdevice, sizeof (sata_device_t)); 5538 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5539 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5540 sdevice.satadev_addr.pmport = 0; 5541 sdevice.satadev_state = SATA_PSTATE_PWRON; 5542 5543 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5544 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5545 "port %d: a device is hot plugged", port); 5546 mutex_exit(&ahci_portp->ahciport_mutex); 5547 sata_hba_event_notify( 5548 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5549 &sdevice, 5550 SATA_EVNT_DEVICE_ATTACHED); 5551 mutex_enter(&ahci_portp->ahciport_mutex); 5552 5553 } else { 5554 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5555 "port %d: a device is hot unplugged", port); 5556 mutex_exit(&ahci_portp->ahciport_mutex); 5557 sata_hba_event_notify( 5558 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5559 &sdevice, 5560 SATA_EVNT_DEVICE_DETACHED); 5561 mutex_enter(&ahci_portp->ahciport_mutex); 5562 } 5563 5564 mutex_exit(&ahci_portp->ahciport_mutex); 5565 5566 return (AHCI_SUCCESS); 5567 } 5568 5569 /* 5570 * Enable the interrupts for a particular port. 5571 * 5572 * WARNING!!! ahciport_mutex should be acquired before the function 5573 * is called. 5574 */ 5575 static void 5576 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5577 { 5578 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5579 "ahci_enable_port_intrs enter, port %d", port); 5580 5581 /* 5582 * Clear port interrupt status before enabling interrupt 5583 */ 5584 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5585 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5586 AHCI_PORT_INTR_MASK); 5587 5588 /* 5589 * Clear the pending bit from IS.IPS 5590 */ 5591 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5592 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5593 5594 /* 5595 * Enable the following interrupts: 5596 * Device to Host Register FIS Interrupt (DHRS) 5597 * PIO Setup FIS Interrupt (PSS) 5598 * Set Device Bits Interrupt (SDBS) 5599 * Unknown FIS Interrupt (UFS) 5600 * Port Connect Change Status (PCS) 5601 * PhyRdy Change Status (PRCS) 5602 * Overflow Status (OFS) 5603 * Interface Non-fatal Error Status (INFS) 5604 * Interface Fatal Error Status (IFS) 5605 * Host Bus Data Error Status (HBDS) 5606 * Host Bus Fatal Error Status (HBFS) 5607 * Task File Error Status (TFES) 5608 */ 5609 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5610 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5611 (AHCI_INTR_STATUS_DHRS | 5612 AHCI_INTR_STATUS_PSS | 5613 AHCI_INTR_STATUS_SDBS | 5614 AHCI_INTR_STATUS_UFS | 5615 AHCI_INTR_STATUS_DPS | 5616 AHCI_INTR_STATUS_PCS | 5617 AHCI_INTR_STATUS_PRCS | 5618 AHCI_INTR_STATUS_OFS | 5619 AHCI_INTR_STATUS_INFS | 5620 AHCI_INTR_STATUS_IFS | 5621 AHCI_INTR_STATUS_HBDS | 5622 AHCI_INTR_STATUS_HBFS | 5623 AHCI_INTR_STATUS_TFES)); 5624 } 5625 5626 /* 5627 * Enable interrupts for all the ports. 5628 * 5629 * WARNING!!! ahcictl_mutex should be acquired before the function 5630 * is called. 5631 */ 5632 static void 5633 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5634 { 5635 uint32_t ghc_control; 5636 5637 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter"); 5638 5639 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5640 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5641 5642 ghc_control |= AHCI_HBA_GHC_IE; 5643 5644 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5645 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5646 } 5647 5648 /* 5649 * Disable interrupts for a particular port. 5650 * 5651 * WARNING!!! ahciport_mutex should be acquired before the function 5652 * is called. 5653 */ 5654 static void 5655 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5656 { 5657 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5658 "ahci_disable_port_intrs enter, port %d", port); 5659 5660 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5661 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5662 } 5663 5664 /* 5665 * Disable interrupts for the whole HBA. 5666 * 5667 * The global bit is cleared, then all interrupt sources from all 5668 * ports are disabled. 5669 * 5670 * WARNING!!! ahcictl_mutex should be acquired before the function 5671 * is called. 5672 */ 5673 static void 5674 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5675 { 5676 uint32_t ghc_control; 5677 5678 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter"); 5679 5680 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5681 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5682 5683 ghc_control &= ~ AHCI_HBA_GHC_IE; 5684 5685 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5686 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5687 } 5688 5689 /* 5690 * Handle FIXED or MSI interrupts. 5691 */ 5692 /* 5693 * According to AHCI spec, the HBA may support several interrupt modes: 5694 * * pin based interrupts (FIXED) 5695 * * single MSI message interrupts 5696 * * multiple MSI based message interrupts 5697 * 5698 * For pin based interrupts, the software interrupt handler need to check IS 5699 * register to find out which port has pending interrupts. And then check 5700 * PxIS register to find out which interrupt events happened on that port. 5701 * 5702 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 5703 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 5704 * in that software interrupt handler need to check IS register to determine 5705 * which port triggered the interrupts since it uses a single message for all 5706 * port interrupts. 5707 * 5708 * HBA may optionally support multiple MSI message for better performance. In 5709 * this mode, each port may have its own interrupt message, and thus generation 5710 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 5711 * represents a power-of-2 wrapper on the number of implemented ports, and 5712 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 5713 * maximum number of assigned interrupts. When the number of MSI messages 5714 * allocated is less than the number requested, then hardware may have two 5715 * implementation behaviors: 5716 * * assign each ports its own interrupt and then force all additional 5717 * ports to share the last interrupt message, and this condition is 5718 * indicated by clearing GHC.MRSM to '0' 5719 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 5720 * When multiple-message MSI is enabled, hardware will still set IS register 5721 * as single message case. And this IS register may be used by software when 5722 * fewer than the requested number of messages is granted in order to determine 5723 * which port had the interrupt. 5724 * 5725 * Note: The current ahci driver only supports the first two interrupt modes: 5726 * pin based interrupts and single MSI message interrupts, and the reason 5727 * is indicated in below code. 5728 */ 5729 static int 5730 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 5731 { 5732 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5733 int count, avail, actual; 5734 int i, rc; 5735 5736 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 5737 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 5738 5739 /* get number of interrupts. */ 5740 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 5741 if ((rc != DDI_SUCCESS) || (count == 0)) { 5742 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5743 "ddi_intr_get_nintrs() failed, " 5744 "rc %d count %d\n", rc, count); 5745 return (DDI_FAILURE); 5746 } 5747 5748 /* get number of available interrupts. */ 5749 rc = ddi_intr_get_navail(dip, intr_type, &avail); 5750 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5751 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5752 "ddi_intr_get_navail() failed, " 5753 "rc %d avail %d\n", rc, avail); 5754 return (DDI_FAILURE); 5755 } 5756 5757 #if AHCI_DEBUG 5758 if (avail < count) { 5759 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5760 "ddi_intr_get_nintrs returned %d, navail() returned %d", 5761 count, avail); 5762 } 5763 #endif 5764 5765 /* 5766 * Note: So far Solaris restricts the maximum number of messages for 5767 * x86 to 2, that is avail is 2, so here we set the count with 1 to 5768 * force the driver to use single MSI message interrupt. In future if 5769 * Solaris remove the restriction, then we need to delete the below 5770 * code and try to use multiple interrupt routine to gain better 5771 * performance. 5772 */ 5773 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 5774 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5775 "force to use one interrupt routine though the " 5776 "HBA supports %d interrupt", count); 5777 count = 1; 5778 } 5779 5780 /* Allocate an array of interrupt handles. */ 5781 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5782 ahci_ctlp->ahcictl_intr_htable = 5783 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5784 5785 /* call ddi_intr_alloc(). */ 5786 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5787 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 5788 5789 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5790 AHCIDBG4(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5791 "ddi_intr_alloc() failed, rc %d count %d actual %d " 5792 "avail %d\n", rc, count, actual, avail); 5793 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5794 ahci_ctlp->ahcictl_intr_size); 5795 return (DDI_FAILURE); 5796 } 5797 5798 /* use interrupt count returned */ 5799 #if AHCI_DEBUG 5800 if (actual < count) { 5801 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5802 "Requested: %d, Received: %d", count, actual); 5803 } 5804 #endif 5805 5806 ahci_ctlp->ahcictl_intr_cnt = actual; 5807 5808 /* 5809 * Get priority for first, assume remaining are all the same. 5810 */ 5811 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5812 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5813 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5814 "ddi_intr_get_pri() failed"); 5815 5816 /* Free already allocated intr. */ 5817 for (i = 0; i < actual; i++) { 5818 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5819 } 5820 5821 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5822 ahci_ctlp->ahcictl_intr_size); 5823 return (DDI_FAILURE); 5824 } 5825 5826 /* Test for high level interrupt. */ 5827 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5828 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5829 "ahci_add_intrs: Hi level intr not supported"); 5830 5831 /* Free already allocated intr. */ 5832 for (i = 0; i < actual; i++) { 5833 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5834 } 5835 5836 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5837 sizeof (ddi_intr_handle_t)); 5838 5839 return (DDI_FAILURE); 5840 } 5841 5842 /* Call ddi_intr_add_handler(). */ 5843 for (i = 0; i < actual; i++) { 5844 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 5845 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5846 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5847 "ddi_intr_add_handler() failed"); 5848 5849 /* Free already allocated intr. */ 5850 for (i = 0; i < actual; i++) { 5851 (void) ddi_intr_free( 5852 ahci_ctlp->ahcictl_intr_htable[i]); 5853 } 5854 5855 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5856 ahci_ctlp->ahcictl_intr_size); 5857 return (DDI_FAILURE); 5858 } 5859 } 5860 5861 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 5862 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 5863 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5864 "ddi_intr_get_cap() failed"); 5865 5866 /* Free already allocated intr. */ 5867 for (i = 0; i < actual; i++) { 5868 (void) ddi_intr_free( 5869 ahci_ctlp->ahcictl_intr_htable[i]); 5870 } 5871 5872 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5873 ahci_ctlp->ahcictl_intr_size); 5874 return (DDI_FAILURE); 5875 } 5876 5877 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 5878 /* Call ddi_intr_block_enable() for MSI. */ 5879 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 5880 ahci_ctlp->ahcictl_intr_cnt); 5881 } else { 5882 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 5883 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 5884 (void) ddi_intr_enable( 5885 ahci_ctlp->ahcictl_intr_htable[i]); 5886 } 5887 } 5888 5889 return (DDI_SUCCESS); 5890 } 5891 5892 /* 5893 * Removes the registered interrupts irrespective of whether they 5894 * were legacy or MSI. 5895 * 5896 * WARNING!!! The controller interrupts must be disabled before calling 5897 * this routine. 5898 */ 5899 static void 5900 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 5901 { 5902 int x; 5903 5904 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered"); 5905 5906 /* Disable all interrupts. */ 5907 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 5908 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 5909 /* Call ddi_intr_block_disable(). */ 5910 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 5911 ahci_ctlp->ahcictl_intr_cnt); 5912 } else { 5913 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5914 (void) ddi_intr_disable( 5915 ahci_ctlp->ahcictl_intr_htable[x]); 5916 } 5917 } 5918 5919 /* Call ddi_intr_remove_handler(). */ 5920 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5921 (void) ddi_intr_remove_handler( 5922 ahci_ctlp->ahcictl_intr_htable[x]); 5923 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5924 } 5925 5926 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 5927 } 5928 5929 /* 5930 * This routine tries to put port into P:NotRunning state by clearing 5931 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 5932 * and PxCMD.CR to '0'. 5933 * 5934 * WARNING!!! ahciport_mutex should be acquired before the function 5935 * is called. 5936 */ 5937 static int 5938 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 5939 ahci_port_t *ahci_portp, uint8_t port) 5940 { 5941 uint32_t port_cmd_status; 5942 int loop_count; 5943 5944 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5945 "ahci_put_port_into_notrunning_state enter: port %d", port); 5946 5947 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5948 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5949 5950 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 5951 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5952 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5953 5954 /* Wait until PxCMD.CR is cleared */ 5955 loop_count = 0; 5956 do { 5957 port_cmd_status = 5958 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5959 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5960 5961 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 5962 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 5963 "clearing port %d CMD.CR timeout, " 5964 "port_cmd_status = 0x%x", port, 5965 port_cmd_status); 5966 /* 5967 * We are effectively timing out after 0.5 sec. 5968 * This value is specified in AHCI spec. 5969 */ 5970 break; 5971 } 5972 5973 /* Wait for 10 millisec */ 5974 drv_usecwait(10000); 5975 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 5976 5977 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 5978 5979 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 5980 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5981 "ahci_put_port_into_notrunning_state: failed to clear " 5982 "PxCMD.CR to '0' after loop count: %d, and " 5983 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5984 return (AHCI_FAILURE); 5985 } else { 5986 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5987 "ahci_put_port_into_notrunning_state: succeeded to clear " 5988 "PxCMD.CR to '0' after loop count: %d, and " 5989 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5990 return (AHCI_SUCCESS); 5991 } 5992 } 5993 5994 /* 5995 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 5996 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 5997 * stable state, then set PxCMD.ST to '1' to start the port directly. 5998 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 5999 * COMRESET to the device to put it in an idle state. 6000 * 6001 * The fifth argument returns whether the port reset is involved during 6002 * the process. 6003 * 6004 * The routine will be called under six scenarios: 6005 * 1. Initialize the port 6006 * 2. To abort the packet(s) 6007 * 3. To reset the port 6008 * 4. To activate the port 6009 * 5. Fatal error recovery 6010 * 6. To abort the timeout packet(s) 6011 * 6012 * WARNING!!! ahciport_mutex should be acquired before the function 6013 * is called. And ahciport_mutex will be released before the reset 6014 * event is reported to sata module by calling sata_hba_event_notify, 6015 * and then be acquired again later. 6016 * 6017 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6018 * according to the spec, the clearance of three bits will also clear 6019 * three interrupt status bits. 6020 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6021 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6022 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6023 * 6024 * Among these three interrupt events, the driver needs to take care of 6025 * PxIS.PRCS, which is the hot plug event. When the driver found out 6026 * a device was unplugged, it will call the interrupt handler. 6027 */ 6028 static int 6029 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6030 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6031 { 6032 uint32_t port_sstatus; 6033 uint32_t task_file_status; 6034 sata_device_t sdevice; 6035 int rval; 6036 int dev_exists_begin = 0; 6037 int dev_exists_end = 0; 6038 6039 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6040 "ahci_restart_port_wait_till_ready: port %d enter", port); 6041 6042 if ((flag != AHCI_PORT_INIT) && 6043 (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)) 6044 dev_exists_begin = 1; 6045 6046 /* First clear PxCMD.ST */ 6047 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6048 port); 6049 if (rval != AHCI_SUCCESS) 6050 /* 6051 * If PxCMD.CR does not clear within a reasonable time, it 6052 * may assume the interface is in a hung condition and may 6053 * continue with issuing the port reset. 6054 */ 6055 goto reset; 6056 6057 /* Then clear PxSERR */ 6058 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6059 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6060 AHCI_SERROR_CLEAR_ALL); 6061 6062 /* Then get PxTFD */ 6063 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6064 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6065 6066 /* 6067 * Check whether the device is in a stable status, if yes, 6068 * then start the port directly. However for ahci_tran_dport_reset, 6069 * we may have to perform a port reset. 6070 */ 6071 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6072 !(flag & AHCI_PORT_RESET)) 6073 goto out; 6074 6075 reset: 6076 /* 6077 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6078 * a COMRESET to the device 6079 */ 6080 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6081 6082 if (reset_flag != NULL) 6083 *reset_flag = 1; 6084 6085 /* Indicate to the framework that a reset has happened. */ 6086 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6087 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6088 /* Set the reset in progress flag */ 6089 ahci_portp->ahciport_reset_in_progress = 1; 6090 6091 bzero((void *)&sdevice, sizeof (sata_device_t)); 6092 sdevice.satadev_addr.cport = 6093 ahci_ctlp->ahcictl_port_to_cport[port]; 6094 sdevice.satadev_addr.pmport = 0; 6095 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6096 6097 sdevice.satadev_state = SATA_DSTATE_RESET | 6098 SATA_DSTATE_PWR_ACTIVE; 6099 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6100 mutex_exit(&ahci_portp->ahciport_mutex); 6101 sata_hba_event_notify( 6102 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6103 &sdevice, 6104 SATA_EVNT_DEVICE_RESET); 6105 mutex_enter(&ahci_portp->ahciport_mutex); 6106 } 6107 6108 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 6109 "port %d sending event up: SATA_EVNT_RESET", port); 6110 } else { 6111 ahci_portp->ahciport_reset_in_progress = 0; 6112 } 6113 6114 out: 6115 /* Start the port if not in initialization phase */ 6116 if (flag & AHCI_PORT_INIT) 6117 return (rval); 6118 6119 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6120 6121 /* SStatus tells the presence of device. */ 6122 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6123 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6124 6125 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6126 dev_exists_end = 1; 6127 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6128 } 6129 6130 /* Check whether a hot plug event happened */ 6131 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6132 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6133 "ahci_restart_port_wait_till_ready: port %d " 6134 "device is removed", port); 6135 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6136 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6137 "ahci_restart_port_wait_till_ready: port %d " 6138 "AHCI_PORT_FLAG_NODEV flag is set", port); 6139 mutex_exit(&ahci_portp->ahciport_mutex); 6140 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6141 mutex_enter(&ahci_portp->ahciport_mutex); 6142 } 6143 6144 return (rval); 6145 } 6146 6147 /* 6148 * This routine may be called under four scenarios: 6149 * a) do the recovery from fatal error 6150 * b) or we need to timeout some commands 6151 * c) or we need to abort some commands 6152 * d) or we need reset device/port/controller 6153 * 6154 * In all these scenarios, we need to send any pending unfinished 6155 * commands up to sata framework. 6156 * 6157 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6158 */ 6159 static void 6160 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6161 ahci_port_t *ahci_portp, 6162 uint32_t slot_status, 6163 uint32_t failed_tags, 6164 uint32_t timeout_tags, 6165 uint32_t aborted_tags, 6166 uint32_t reset_tags) 6167 { 6168 uint32_t finished_tags = 0; 6169 uint32_t unfinished_tags = 0; 6170 int tmp_slot; 6171 sata_pkt_t *satapkt; 6172 int ncq_cmd_in_progress = 0; 6173 int err_retri_cmd_in_progress = 0; 6174 6175 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6176 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6177 ahci_portp->ahciport_port_num, slot_status); 6178 6179 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6180 "ahci_mop_commands: failed_tags: 0x%x, " 6181 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6182 "reset_tags: 0x%x", failed_tags, 6183 timeout_tags, aborted_tags, reset_tags); 6184 6185 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6186 finished_tags = ahci_portp->ahciport_pending_tags & 6187 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6188 6189 unfinished_tags = slot_status & 6190 AHCI_SLOT_MASK(ahci_ctlp) & 6191 ~failed_tags & 6192 ~aborted_tags & 6193 ~reset_tags & 6194 ~timeout_tags; 6195 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6196 ncq_cmd_in_progress = 1; 6197 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6198 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6199 6200 unfinished_tags = slot_status & 6201 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6202 ~failed_tags & 6203 ~aborted_tags & 6204 ~reset_tags & 6205 ~timeout_tags; 6206 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6207 6208 /* 6209 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6210 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6211 * complete successfully due to one of the following three 6212 * conditions: 6213 * 6214 * 1. Fatal error - failed_tags includes its slot 6215 * 2. Timed out - timeout_tags includes its slot 6216 * 3. Aborted when hot unplug - aborted_tags includes its 6217 * slot 6218 * 6219 * Please note that the command is always sent down in Slot 0 6220 */ 6221 err_retri_cmd_in_progress = 1; 6222 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6223 "ahci_mop_commands is called for port %d while " 6224 "REQUEST SENSE or READ LOG EXT for error retrieval " 6225 "is being executed slot_status = 0x%x", 6226 ahci_portp->ahciport_port_num, slot_status); 6227 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6228 ASSERT(slot_status == 0x1); 6229 } 6230 6231 /* Send up finished packets with SATA_PKT_COMPLETED */ 6232 while (finished_tags) { 6233 tmp_slot = ddi_ffs(finished_tags) - 1; 6234 if (tmp_slot == -1) { 6235 break; 6236 } 6237 6238 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6239 ASSERT(satapkt != NULL); 6240 6241 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6242 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6243 (void *)satapkt); 6244 6245 /* 6246 * Cannot fetch the return register content since the port 6247 * was restarted, so the corresponding tag will be set to 6248 * aborted tags. 6249 */ 6250 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6251 CLEAR_BIT(finished_tags, tmp_slot); 6252 aborted_tags |= tmp_slot; 6253 continue; 6254 } 6255 6256 if (ncq_cmd_in_progress) 6257 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6258 tmp_slot); 6259 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6260 CLEAR_BIT(finished_tags, tmp_slot); 6261 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6262 6263 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6264 } 6265 6266 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6267 while (failed_tags) { 6268 if (err_retri_cmd_in_progress) { 6269 satapkt = ahci_portp->ahciport_err_retri_pkt; 6270 ASSERT(satapkt != NULL); 6271 ASSERT(failed_tags == 0x1); 6272 6273 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6274 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6275 (void *)satapkt); 6276 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6277 break; 6278 } 6279 6280 tmp_slot = ddi_ffs(failed_tags) - 1; 6281 if (tmp_slot == -1) { 6282 break; 6283 } 6284 6285 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6286 ASSERT(satapkt != NULL); 6287 6288 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6289 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6290 (void *)satapkt); 6291 6292 if (ncq_cmd_in_progress) 6293 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6294 tmp_slot); 6295 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6296 CLEAR_BIT(failed_tags, tmp_slot); 6297 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6298 6299 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6300 } 6301 6302 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6303 while (timeout_tags) { 6304 if (err_retri_cmd_in_progress) { 6305 satapkt = ahci_portp->ahciport_err_retri_pkt; 6306 ASSERT(satapkt != NULL); 6307 ASSERT(timeout_tags == 0x1); 6308 6309 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6310 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6311 (void *)satapkt); 6312 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6313 break; 6314 } 6315 6316 tmp_slot = ddi_ffs(timeout_tags) - 1; 6317 if (tmp_slot == -1) { 6318 break; 6319 } 6320 6321 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6322 ASSERT(satapkt != NULL); 6323 6324 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6325 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6326 (void *)satapkt); 6327 6328 if (ncq_cmd_in_progress) 6329 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6330 tmp_slot); 6331 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6332 CLEAR_BIT(timeout_tags, tmp_slot); 6333 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6334 6335 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6336 } 6337 6338 /* Send up aborted packets with SATA_PKT_ABORTED */ 6339 while (aborted_tags) { 6340 if (err_retri_cmd_in_progress) { 6341 satapkt = ahci_portp->ahciport_err_retri_pkt; 6342 ASSERT(satapkt != NULL); 6343 ASSERT(aborted_tags == 0x1); 6344 6345 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6346 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6347 (void *)satapkt); 6348 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6349 break; 6350 } 6351 6352 tmp_slot = ddi_ffs(aborted_tags) - 1; 6353 if (tmp_slot == -1) { 6354 break; 6355 } 6356 6357 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6358 ASSERT(satapkt != NULL); 6359 6360 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6361 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6362 (void *)satapkt); 6363 6364 if (ncq_cmd_in_progress) 6365 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6366 tmp_slot); 6367 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6368 CLEAR_BIT(aborted_tags, tmp_slot); 6369 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6370 6371 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6372 } 6373 6374 /* Send up reset packets with SATA_PKT_RESET. */ 6375 while (reset_tags) { 6376 tmp_slot = ddi_ffs(reset_tags) - 1; 6377 if (tmp_slot == -1) { 6378 break; 6379 } 6380 6381 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6382 ASSERT(satapkt != NULL); 6383 6384 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6385 "sending up pkt 0x%p with SATA_PKT_RESET", 6386 (void *)satapkt); 6387 6388 if (ncq_cmd_in_progress) 6389 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6390 tmp_slot); 6391 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6392 CLEAR_BIT(reset_tags, tmp_slot); 6393 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6394 6395 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6396 } 6397 6398 /* Send up unfinished packets with SATA_PKT_RESET */ 6399 while (unfinished_tags) { 6400 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6401 if (tmp_slot == -1) { 6402 break; 6403 } 6404 6405 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6406 ASSERT(satapkt != NULL); 6407 6408 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6409 "sending up pkt 0x%p with SATA_PKT_RESET", 6410 (void *)satapkt); 6411 6412 if (ncq_cmd_in_progress) 6413 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6414 tmp_slot); 6415 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6416 CLEAR_BIT(unfinished_tags, tmp_slot); 6417 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6418 6419 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6420 } 6421 6422 ahci_portp->ahciport_mop_in_progress--; 6423 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6424 6425 if (ahci_portp->ahciport_mop_in_progress == 0) 6426 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6427 } 6428 6429 /* 6430 * This routine is going to first request a READ LOG EXT sata pkt from sata 6431 * module, and then deliver it to the HBA to get the ncq failure context. 6432 * The return value is the exactly failed tags. 6433 */ 6434 static uint32_t 6435 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6436 uint8_t port) 6437 { 6438 sata_device_t sdevice; 6439 sata_pkt_t *rdlog_spkt, *spkt; 6440 ddi_dma_handle_t buf_dma_handle; 6441 int loop_count; 6442 int rval; 6443 int failed_slot; 6444 uint32_t failed_tags = 0; 6445 struct sata_ncq_error_recovery_page *ncq_err_page; 6446 6447 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6448 "ahci_get_rdlogext_data enter: port %d", port); 6449 6450 /* Prepare the sdevice data */ 6451 bzero((void *)&sdevice, sizeof (sata_device_t)); 6452 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6453 6454 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6455 sdevice.satadev_addr.pmport = 0; 6456 6457 /* 6458 * Call the sata hba interface to get a rdlog spkt 6459 */ 6460 loop_count = 0; 6461 loop: 6462 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6463 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6464 if (rdlog_spkt == NULL) { 6465 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6466 /* Sleep for a while */ 6467 delay(AHCI_10MS_TICKS); 6468 goto loop; 6469 } 6470 /* Timed out after 1s */ 6471 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6472 "failed to get rdlog spkt for port %d", port); 6473 return (failed_tags); 6474 } 6475 6476 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6477 6478 /* 6479 * This flag is used to handle the specific error recovery when the 6480 * READ LOG EXT command gets a failure (fatal error or time-out). 6481 */ 6482 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6483 6484 /* 6485 * This start is not supposed to fail because after port is restarted, 6486 * the whole command list is empty. 6487 */ 6488 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6489 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6490 ahci_portp->ahciport_err_retri_pkt = NULL; 6491 6492 /* Remove the flag after READ LOG EXT command is completed */ 6493 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6494 6495 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6496 /* Update the request log data */ 6497 buf_dma_handle = *(ddi_dma_handle_t *) 6498 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6499 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6500 DDI_DMA_SYNC_FORKERNEL); 6501 if (rval == DDI_SUCCESS) { 6502 ncq_err_page = 6503 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6504 satapkt_cmd.satacmd_bp->b_un.b_addr; 6505 6506 /* Get the failed tag */ 6507 failed_slot = ncq_err_page->ncq_tag; 6508 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 6509 "ahci_get_rdlogext_data: port %d " 6510 "failed slot %d", port, failed_slot); 6511 if (failed_slot & NQ) { 6512 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6513 "the failed slot is not a valid tag"); 6514 goto out; 6515 } 6516 6517 failed_slot &= NCQ_TAG_MASK; 6518 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6519 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6520 "ahci_get_rdlogext_data: failed spkt 0x%p", 6521 (void *)spkt); 6522 if (spkt == NULL) { 6523 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6524 "the failed slot spkt is NULL") 6525 goto out; 6526 } 6527 6528 failed_tags = 0x1 << failed_slot; 6529 6530 /* Fill out the error context */ 6531 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6532 ncq_err_page); 6533 ahci_update_sata_registers(ahci_ctlp, port, 6534 &spkt->satapkt_device); 6535 } 6536 } 6537 out: 6538 sata_free_error_retrieval_pkt(rdlog_spkt); 6539 6540 return (failed_tags); 6541 } 6542 6543 /* 6544 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6545 * module, and then deliver it to the HBA to get the sense data and copy 6546 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6547 * SENSE sata pkt later. 6548 */ 6549 static void 6550 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6551 uint8_t port, sata_pkt_t *spkt) 6552 { 6553 sata_device_t sdevice; 6554 sata_pkt_t *rs_spkt; 6555 sata_cmd_t *sata_cmd; 6556 ddi_dma_handle_t buf_dma_handle; 6557 int loop_count; 6558 #if AHCI_DEBUG 6559 struct scsi_extended_sense *rqsense; 6560 #endif 6561 6562 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6563 "ahci_get_rqsense_data enter: port %d", port); 6564 6565 /* Prepare the sdevice data */ 6566 bzero((void *)&sdevice, sizeof (sata_device_t)); 6567 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6568 6569 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6570 sdevice.satadev_addr.pmport = 0; 6571 6572 sata_cmd = &spkt->satapkt_cmd; 6573 6574 /* 6575 * Call the sata hba interface to get a rs spkt 6576 */ 6577 loop_count = 0; 6578 loop: 6579 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6580 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6581 if (rs_spkt == NULL) { 6582 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6583 /* Sleep for a while */ 6584 delay(AHCI_10MS_TICKS); 6585 goto loop; 6586 6587 } 6588 /* Timed out after 1s */ 6589 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6590 "failed to get rs spkt for port %d", port); 6591 return; 6592 } 6593 6594 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6595 6596 /* 6597 * This flag is used to handle the specific error recovery when the 6598 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6599 */ 6600 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6601 6602 /* 6603 * This start is not supposed to fail because after port is restarted, 6604 * the whole command list is empty. 6605 */ 6606 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6607 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6608 ahci_portp->ahciport_err_retri_pkt = NULL; 6609 6610 /* Remove the flag after REQUEST SENSE command is completed */ 6611 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6612 6613 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6614 /* Update the request sense data */ 6615 buf_dma_handle = *(ddi_dma_handle_t *) 6616 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6617 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6618 DDI_DMA_SYNC_FORKERNEL); 6619 /* Copy the request sense data */ 6620 bcopy(rs_spkt-> 6621 satapkt_cmd.satacmd_bp->b_un.b_addr, 6622 &sata_cmd->satacmd_rqsense, 6623 SATA_ATAPI_MIN_RQSENSE_LEN); 6624 #if AHCI_DEBUG 6625 rqsense = (struct scsi_extended_sense *) 6626 sata_cmd->satacmd_rqsense; 6627 6628 /* Dump the sense data */ 6629 AHCIDBG0(AHCIDBG_SENSEDATA, ahci_ctlp, "\n"); 6630 AHCIDBG2(AHCIDBG_SENSEDATA, ahci_ctlp, 6631 "Sense data for satapkt %p ATAPI cmd 0x%x", 6632 spkt, sata_cmd->satacmd_acdb[0]); 6633 AHCIDBG5(AHCIDBG_SENSEDATA, ahci_ctlp, 6634 " es_code 0x%x es_class 0x%x " 6635 "es_key 0x%x es_add_code 0x%x " 6636 "es_qual_code 0x%x", 6637 rqsense->es_code, rqsense->es_class, 6638 rqsense->es_key, rqsense->es_add_code, 6639 rqsense->es_qual_code); 6640 #endif 6641 } 6642 6643 sata_free_error_retrieval_pkt(rs_spkt); 6644 } 6645 6646 /* 6647 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6648 * the port must be restarted. When the HBA detects thus error, it may try 6649 * to abort a transfer. And if the transfer was aborted, the device is 6650 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6651 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6652 * that the device is in a stable status and transfers may be restarted without 6653 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6654 * then the software will send the COMRESET to do the port reset. 6655 * 6656 * Software should perform the appropriate error recovery actions based on 6657 * whether non-queued commands were being issued or natived command queuing 6658 * commands were being issued. 6659 * 6660 * And software will complete the command that had the error with error mark 6661 * to higher level software. 6662 * 6663 * Fatal errors include the following: 6664 * PxIS.IFS - Interface Fatal Error Status 6665 * PxIS.HBDS - Host Bus Data Error Status 6666 * PxIS.HBFS - Host Bus Fatal Error Status 6667 * PxIS.TFES - Task File Error Status 6668 * 6669 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6670 */ 6671 static void 6672 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6673 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6674 { 6675 uint32_t port_cmd_status; 6676 uint32_t slot_status = 0; 6677 uint32_t failed_tags = 0; 6678 int failed_slot; 6679 int reset_flag = 0; 6680 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6681 sata_cmd_t *sata_cmd = NULL; 6682 sata_pkt_t *spkt = NULL; 6683 #if AHCI_DEBUG 6684 ahci_cmd_header_t *cmd_header; 6685 #endif 6686 6687 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6688 "ahci_fatal_error_recovery_handler enter: port %d", port); 6689 6690 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6691 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6692 6693 /* Read PxCI to see which commands are still outstanding */ 6694 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6695 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6696 6697 /* 6698 * Read PxCMD.CCS to determine the slot that the HBA 6699 * was processing when the error occurred. 6700 */ 6701 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6702 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6703 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6704 AHCI_CMD_STATUS_CCS_SHIFT; 6705 6706 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6707 spkt = ahci_portp->ahciport_err_retri_pkt; 6708 ASSERT(spkt != NULL); 6709 } else { 6710 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6711 if (spkt == NULL) { 6712 /* May happen when interface errors occur? */ 6713 goto next; 6714 } 6715 } 6716 6717 #if AHCI_DEBUG 6718 /* 6719 * Debugging purpose... 6720 */ 6721 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6722 cmd_header = 6723 &ahci_portp->ahciport_cmd_list[failed_slot]; 6724 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6725 "ahci_fatal_error_recovery_handler: port %d, " 6726 "PRD Byte Count = 0x%x, " 6727 "ahciport_prd_bytecounts = 0x%x", port, 6728 cmd_header->ahcich_prd_byte_count, 6729 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6730 } 6731 #endif 6732 6733 sata_cmd = &spkt->satapkt_cmd; 6734 6735 /* Fill out the status and error registers for PxIS.TFES */ 6736 if (intr_status & AHCI_INTR_STATUS_TFES) { 6737 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6738 ahcirf_d2h_register_fis); 6739 6740 /* Copy the error context back to the sata_cmd */ 6741 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6742 } 6743 6744 /* The failed command must be one of the outstanding commands */ 6745 failed_tags = 0x1 << failed_slot; 6746 ASSERT(failed_tags & slot_status); 6747 6748 /* Update the sata registers, especially PxSERR register */ 6749 ahci_update_sata_registers(ahci_ctlp, port, 6750 &spkt->satapkt_device); 6751 6752 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6753 /* Read PxSACT to see which commands are still outstanding */ 6754 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6755 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6756 } 6757 next: 6758 6759 #if AHCI_DEBUG 6760 /* 6761 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6762 * set, it means a fatal error happened after REQUEST SENSE command 6763 * or READ LOG EXT command is delivered to the HBA during the error 6764 * recovery process. At this time, the only outstanding command is 6765 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6766 */ 6767 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6768 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6769 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6770 "command or READ LOG EXT command for error data retrieval " 6771 "failed", port); 6772 ASSERT(slot_status == 0x1); 6773 ASSERT(failed_slot == 0); 6774 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6775 SCMD_REQUEST_SENSE || 6776 spkt->satapkt_cmd.satacmd_cmd_reg == 6777 SATAC_READ_LOG_EXT); 6778 } 6779 #endif 6780 6781 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6782 ahci_portp->ahciport_mop_in_progress++; 6783 6784 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6785 port, NULL, &reset_flag); 6786 6787 /* 6788 * Won't retrieve error information: 6789 * 1. Port reset was involved to recover 6790 * 2. Device is gone 6791 * 3. IDENTIFY DEVICE command sent to ATAPI device 6792 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6793 */ 6794 if (reset_flag || 6795 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6796 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6797 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6798 goto out; 6799 6800 /* 6801 * Deliver READ LOG EXT to gather information about the error when 6802 * a COMRESET has not been performed as part of the error recovery 6803 * during NCQ command processing. 6804 */ 6805 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6806 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6807 ahci_portp, port); 6808 goto out; 6809 } 6810 6811 /* 6812 * Deliver REQUEST SENSE for ATAPI command to gather information about 6813 * the error when a COMRESET has not been performed as part of the 6814 * error recovery. 6815 */ 6816 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6817 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6818 out: 6819 AHCIDBG5(AHCIDBG_ERRS, ahci_ctlp, 6820 "ahci_fatal_error_recovery_handler: port %d fatal error " 6821 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6822 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6823 port, slot_status, ahci_portp->ahciport_pending_tags, 6824 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6825 6826 ahci_mop_commands(ahci_ctlp, 6827 ahci_portp, 6828 slot_status, 6829 failed_tags, /* failed tags */ 6830 0, /* timeout tags */ 6831 0, /* aborted tags */ 6832 0); /* reset tags */ 6833 } 6834 6835 /* 6836 * Handle events - fatal error recovery 6837 */ 6838 static void 6839 ahci_events_handler(void *args) 6840 { 6841 ahci_event_arg_t *ahci_event_arg; 6842 ahci_ctl_t *ahci_ctlp; 6843 ahci_port_t *ahci_portp; 6844 uint32_t event; 6845 uint8_t port; 6846 6847 ahci_event_arg = (ahci_event_arg_t *)args; 6848 6849 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 6850 ahci_portp = ahci_event_arg->ahciea_portp; 6851 event = ahci_event_arg->ahciea_event; 6852 port = ahci_portp->ahciport_port_num; 6853 6854 AHCIDBG2(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6855 "ahci_events_handler enter: port %d intr_status = 0x%x", 6856 port, event); 6857 6858 mutex_enter(&ahci_portp->ahciport_mutex); 6859 6860 /* 6861 * ahci_intr_phyrdy_change() may have rendered it to 6862 * SATA_DTYPE_NONE. 6863 */ 6864 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 6865 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6866 "ahci_events_handler: port %d no device attached, " 6867 "and just return without doing anything", port); 6868 goto out; 6869 } 6870 6871 if (event & (AHCI_INTR_STATUS_IFS | 6872 AHCI_INTR_STATUS_HBDS | 6873 AHCI_INTR_STATUS_HBFS | 6874 AHCI_INTR_STATUS_TFES)) 6875 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 6876 port, event); 6877 6878 out: 6879 mutex_exit(&ahci_portp->ahciport_mutex); 6880 } 6881 6882 /* 6883 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 6884 * detect there are some commands which are timed out. 6885 */ 6886 static void 6887 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6888 uint8_t port, uint32_t tmp_timeout_tags) 6889 { 6890 uint32_t slot_status = 0; 6891 uint32_t finished_tags = 0; 6892 uint32_t timeout_tags = 0; 6893 6894 AHCIDBG1(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6895 "ahci_timeout_pkts enter: port %d", port); 6896 6897 mutex_enter(&ahci_portp->ahciport_mutex); 6898 6899 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6900 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6901 /* Read PxCI to see which commands are still outstanding */ 6902 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6903 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6904 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6905 /* Read PxSACT to see which commands are still outstanding */ 6906 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6907 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6908 } 6909 6910 #if AHCI_DEBUG 6911 /* 6912 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6913 * set, it means a fatal error happened after REQUEST SENSE command 6914 * or READ LOG EXT command is delivered to the HBA during the error 6915 * recovery process. At this time, the only outstanding command is 6916 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6917 */ 6918 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6919 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 6920 "ahci_timeout_pkts called while REQUEST SENSE " 6921 "command or READ LOG EXT command for error recovery " 6922 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 6923 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 6924 tmp_timeout_tags, slot_status, 6925 ahci_portp->ahciport_pending_tags, 6926 ahci_portp->ahciport_pending_ncq_tags); 6927 ASSERT(slot_status == 0x1); 6928 } 6929 #endif 6930 6931 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6932 ahci_portp->ahciport_mop_in_progress++; 6933 6934 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6935 port, NULL, NULL); 6936 6937 /* 6938 * Re-identify timeout tags because some previously checked commands 6939 * could already complete. 6940 */ 6941 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6942 finished_tags = ahci_portp->ahciport_pending_tags & 6943 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6944 timeout_tags = tmp_timeout_tags & ~finished_tags; 6945 6946 AHCIDBG5(AHCIDBG_TIMEOUT, ahci_ctlp, 6947 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6948 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 6949 "pending_tags = 0x%x ", 6950 port, finished_tags, timeout_tags, 6951 slot_status, ahci_portp->ahciport_pending_tags); 6952 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6953 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6954 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6955 timeout_tags = tmp_timeout_tags & ~finished_tags; 6956 6957 AHCIDBG5(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 6958 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6959 "timeout_tags = 0x%x, port_sactive = 0x%x, " 6960 "pending_ncq_tags = 0x%x ", 6961 port, finished_tags, timeout_tags, 6962 slot_status, ahci_portp->ahciport_pending_ncq_tags); 6963 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6964 timeout_tags = tmp_timeout_tags; 6965 } 6966 6967 ahci_mop_commands(ahci_ctlp, 6968 ahci_portp, 6969 slot_status, 6970 0, /* failed tags */ 6971 timeout_tags, /* timeout tags */ 6972 0, /* aborted tags */ 6973 0); /* reset tags */ 6974 6975 mutex_exit(&ahci_portp->ahciport_mutex); 6976 } 6977 6978 /* 6979 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 6980 * for long time. 6981 */ 6982 static void 6983 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 6984 { 6985 ahci_port_t *ahci_portp; 6986 sata_pkt_t *spkt; 6987 uint32_t pending_tags; 6988 uint32_t timeout_tags; 6989 uint32_t port_cmd_status; 6990 uint32_t port_sactive; 6991 uint8_t port; 6992 int tmp_slot; 6993 int current_slot; 6994 uint32_t current_tags; 6995 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 6996 /* max number of cycles this packet should survive */ 6997 int max_life_cycles; 6998 6999 /* how many cycles this packet survived so far */ 7000 int watched_cycles; 7001 7002 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7003 7004 AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 7005 "ahci_watchdog_handler entered"); 7006 7007 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7008 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7009 continue; 7010 } 7011 7012 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7013 7014 mutex_enter(&ahci_portp->ahciport_mutex); 7015 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7016 mutex_exit(&ahci_portp->ahciport_mutex); 7017 continue; 7018 } 7019 7020 /* Skip the check for those ports in error recovery */ 7021 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7022 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7023 mutex_exit(&ahci_portp->ahciport_mutex); 7024 continue; 7025 } 7026 7027 pending_tags = 0; 7028 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7029 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7030 7031 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7032 current_slot = 0; 7033 pending_tags = 0x1; 7034 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7035 current_slot = 7036 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7037 AHCI_CMD_STATUS_CCS_SHIFT; 7038 pending_tags = ahci_portp->ahciport_pending_tags; 7039 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7040 port_sactive = ddi_get32( 7041 ahci_ctlp->ahcictl_ahci_acc_handle, 7042 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7043 current_tags = port_sactive & 7044 ~port_cmd_status & 7045 AHCI_NCQ_SLOT_MASK(ahci_portp); 7046 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7047 } 7048 7049 timeout_tags = 0; 7050 while (pending_tags) { 7051 tmp_slot = ddi_ffs(pending_tags) - 1; 7052 if (tmp_slot == -1) { 7053 break; 7054 } 7055 7056 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7057 spkt = ahci_portp->ahciport_err_retri_pkt; 7058 else 7059 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7060 7061 if ((spkt != NULL) && spkt->satapkt_time && 7062 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7063 /* 7064 * We are overloading satapkt_hba_driver_private 7065 * with watched_cycle count. 7066 * 7067 * If a packet has survived for more than it's 7068 * max life cycles, it is a candidate for time 7069 * out. 7070 */ 7071 watched_cycles = (int)(intptr_t) 7072 spkt->satapkt_hba_driver_private; 7073 watched_cycles++; 7074 max_life_cycles = (spkt->satapkt_time + 7075 ahci_watchdog_timeout - 1) / 7076 ahci_watchdog_timeout; 7077 7078 spkt->satapkt_hba_driver_private = 7079 (void *)(intptr_t)watched_cycles; 7080 7081 if (watched_cycles <= max_life_cycles) 7082 goto next; 7083 7084 AHCIDBG1(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7085 ahci_ctlp, "the current slot is %d", 7086 current_slot); 7087 /* 7088 * We need to check whether the HBA has 7089 * begun to execute the command, if not, 7090 * then re-set the timer of the command. 7091 */ 7092 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7093 (tmp_slot != current_slot) || 7094 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7095 ((0x1 << tmp_slot) & current_tags)) { 7096 spkt->satapkt_hba_driver_private = 7097 (void *)(intptr_t)0; 7098 } else { 7099 timeout_tags |= (0x1 << tmp_slot); 7100 cmn_err(CE_WARN, "!ahci%d: watchdog " 7101 "port %d satapkt 0x%p timed out\n", 7102 instance, port, (void *)spkt); 7103 } 7104 } 7105 next: 7106 CLEAR_BIT(pending_tags, tmp_slot); 7107 } 7108 7109 if (timeout_tags) { 7110 mutex_exit(&ahci_portp->ahciport_mutex); 7111 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7112 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7113 port, timeout_tags); 7114 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7115 mutex_enter(&ahci_portp->ahciport_mutex); 7116 } 7117 7118 mutex_exit(&ahci_portp->ahciport_mutex); 7119 } 7120 7121 /* Re-install the watchdog timeout handler */ 7122 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7123 ahci_ctlp->ahcictl_timeout_id = 7124 timeout((void (*)(void *))ahci_watchdog_handler, 7125 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7126 } 7127 7128 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7129 } 7130 7131 /* 7132 * Fill the error context into sata_cmd for non-queued command error. 7133 */ 7134 static void 7135 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7136 { 7137 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7138 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7139 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7140 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7141 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7142 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7143 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7144 7145 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7146 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7147 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7148 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7149 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7150 } 7151 } 7152 7153 /* 7154 * Fill the ncq error page into sata_cmd for queued command error. 7155 */ 7156 static void 7157 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7158 struct sata_ncq_error_recovery_page *ncq_err_page) 7159 { 7160 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7161 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7162 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7163 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7164 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7165 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7166 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7167 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7168 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7169 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7170 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7171 } 7172 7173 /* 7174 * Put the respective register value to sata_cmd_t for satacmd_flags. 7175 */ 7176 static void 7177 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7178 { 7179 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7180 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7181 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7182 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7183 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7184 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7185 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7186 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7187 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7188 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7189 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7190 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7191 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7192 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7193 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7194 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7195 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7196 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7197 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7198 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7199 } 7200 7201 static void 7202 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7203 uint32_t intr_status) 7204 { 7205 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7206 7207 if (intr_status & AHCI_INTR_STATUS_IFS) 7208 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 7209 "error", instance, port); 7210 7211 if (intr_status & AHCI_INTR_STATUS_HBDS) 7212 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 7213 instance, port); 7214 7215 if (intr_status & AHCI_INTR_STATUS_HBFS) 7216 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 7217 instance, port); 7218 7219 if (intr_status & AHCI_INTR_STATUS_TFES) 7220 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 7221 instance, port); 7222 7223 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 7224 "recovery", instance, port); 7225 } 7226 7227 /* 7228 * Dump the serror message to the log. 7229 */ 7230 static void 7231 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7232 uint32_t port_serror, int debug_only) 7233 { 7234 static char err_buf[512]; 7235 static char err_msg_header[16]; 7236 char *err_msg = err_buf; 7237 7238 *err_buf = '\0'; 7239 *err_msg_header = '\0'; 7240 7241 if (port_serror & SERROR_DATA_ERR_FIXED) { 7242 err_msg = strcat(err_msg, 7243 "\tRecovered Data Integrity Error (I)\n"); 7244 } 7245 7246 if (port_serror & SERROR_COMM_ERR_FIXED) { 7247 err_msg = strcat(err_msg, 7248 "\tRecovered Communication Error (M)\n"); 7249 } 7250 7251 if (port_serror & SERROR_DATA_ERR) { 7252 err_msg = strcat(err_msg, 7253 "\tTransient Data Integrity Error (T)\n"); 7254 } 7255 7256 if (port_serror & SERROR_PERSISTENT_ERR) { 7257 err_msg = strcat(err_msg, 7258 "\tPersistent Communication or Data Integrity Error (C)\n"); 7259 } 7260 7261 if (port_serror & SERROR_PROTOCOL_ERR) { 7262 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7263 } 7264 7265 if (port_serror & SERROR_INT_ERR) { 7266 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7267 } 7268 7269 if (port_serror & SERROR_PHY_RDY_CHG) { 7270 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7271 } 7272 7273 if (port_serror & SERROR_PHY_INT_ERR) { 7274 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7275 } 7276 7277 if (port_serror & SERROR_COMM_WAKE) { 7278 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7279 } 7280 7281 if (port_serror & SERROR_10B_TO_8B_ERR) { 7282 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7283 } 7284 7285 if (port_serror & SERROR_DISPARITY_ERR) { 7286 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7287 } 7288 7289 if (port_serror & SERROR_CRC_ERR) { 7290 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7291 } 7292 7293 if (port_serror & SERROR_HANDSHAKE_ERR) { 7294 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7295 } 7296 7297 if (port_serror & SERROR_LINK_SEQ_ERR) { 7298 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7299 } 7300 7301 if (port_serror & SERROR_TRANS_ERR) { 7302 err_msg = strcat(err_msg, 7303 "\tTransport state transition error (T)\n"); 7304 } 7305 7306 if (port_serror & SERROR_FIS_TYPE) { 7307 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7308 } 7309 7310 if (port_serror & SERROR_EXCHANGED_ERR) { 7311 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7312 } 7313 7314 if (err_msg == NULL) 7315 return; 7316 7317 if (debug_only) { 7318 (void) sprintf(err_msg_header, "port %d", port); 7319 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg_header); 7320 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg); 7321 } else if (ahci_ctlp) { 7322 cmn_err(CE_WARN, "!ahci%d: %s %s", 7323 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7324 err_msg_header, err_msg); 7325 } else { 7326 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 7327 } 7328 } 7329 7330 /* 7331 * This routine is to calculate the total number of ports implemented 7332 * by the HBA. 7333 */ 7334 static int 7335 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7336 { 7337 uint8_t i; 7338 int num = 0; 7339 7340 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7341 if (((uint32_t)0x1 << i) & ports_implemented) 7342 num++; 7343 } 7344 7345 return (num); 7346 } 7347 7348 #if AHCI_DEBUG 7349 static void 7350 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7351 { 7352 static char name[16]; 7353 va_list ap; 7354 7355 mutex_enter(&ahci_log_mutex); 7356 7357 va_start(ap, fmt); 7358 if (ahci_ctlp) { 7359 (void) sprintf(name, "ahci%d: ", 7360 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7361 } else { 7362 (void) sprintf(name, "ahci: "); 7363 } 7364 7365 (void) vsprintf(ahci_log_buf, fmt, ap); 7366 va_end(ap); 7367 7368 cmn_err(level, "%s%s", name, ahci_log_buf); 7369 7370 mutex_exit(&ahci_log_mutex); 7371 } 7372 #endif 7373 7374 /* 7375 * quiesce(9E) entry point. 7376 * 7377 * This function is called when the system is single-threaded at high 7378 * PIL with preemption disabled. Therefore, this function must not be 7379 * blocked. 7380 * 7381 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7382 * DDI_FAILURE indicates an error condition and should almost never happen. 7383 */ 7384 static int 7385 ahci_quiesce(dev_info_t *dip) 7386 { 7387 ahci_ctl_t *ahci_ctlp; 7388 ahci_port_t *ahci_portp; 7389 int instance, port; 7390 7391 instance = ddi_get_instance(dip); 7392 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7393 7394 if (ahci_ctlp == NULL) 7395 return (DDI_FAILURE); 7396 7397 #if AHCI_DEBUG 7398 ahci_debug_flags = 0; 7399 #endif 7400 7401 /* disable all the interrupts. */ 7402 ahci_disable_all_intrs(ahci_ctlp); 7403 7404 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7405 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7406 continue; 7407 } 7408 7409 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7410 7411 /* 7412 * Stop the port by clearing PxCMD.ST 7413 * 7414 * Here we must disable the port interrupt because 7415 * ahci_disable_all_intrs only clear GHC.IE, and IS 7416 * register will be still set if PxIE is enabled. 7417 * When ahci shares one IRQ with other drivers, the 7418 * intr handler may claim the intr mistakenly. 7419 */ 7420 ahci_disable_port_intrs(ahci_ctlp, port); 7421 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7422 ahci_portp, port); 7423 } 7424 7425 return (DDI_SUCCESS); 7426 } 7427