1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 104 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 105 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 107 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 108 uint8_t, sata_pkt_t *); 109 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 110 uint8_t, sata_pkt_t *); 111 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 112 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 113 static void ahci_copy_ncq_err_page(sata_cmd_t *, 114 struct sata_ncq_error_recovery_page *); 115 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 116 117 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 118 static int ahci_hba_reset(ahci_ctl_t *); 119 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 120 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 124 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 125 uint8_t); 126 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 127 uint8_t, int, int *); 128 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 129 uint32_t, uint32_t, uint32_t, uint32_t); 130 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 131 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 132 uint8_t, sata_pkt_t *); 133 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 134 uint8_t, uint32_t); 135 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 136 uint8_t, uint32_t); 137 static void ahci_events_handler(void *); 138 static void ahci_watchdog_handler(ahci_ctl_t *); 139 140 static uint_t ahci_intr(caddr_t, caddr_t); 141 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 142 static int ahci_add_legacy_intrs(ahci_ctl_t *); 143 static int ahci_add_msi_intrs(ahci_ctl_t *); 144 static void ahci_rem_intrs(ahci_ctl_t *); 145 static void ahci_enable_all_intrs(ahci_ctl_t *); 146 static void ahci_disable_all_intrs(ahci_ctl_t *); 147 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 148 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 149 150 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 151 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 154 ahci_port_t *, uint8_t); 155 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, uint32_t); 158 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 159 uint8_t, uint32_t); 160 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 161 162 static int ahci_get_num_implemented_ports(uint32_t); 163 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 164 uint32_t); 165 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 166 #if AHCI_DEBUG 167 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 168 #endif 169 170 171 /* 172 * DMA attributes for the data buffer 173 * 174 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 175 * does not support 64-bit addressing 176 */ 177 static ddi_dma_attr_t buffer_dma_attr = { 178 DMA_ATTR_V0, /* dma_attr_version */ 179 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 180 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 181 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 182 0x2ull, /* dma_attr_align: word aligned */ 183 1, /* dma_attr_burstsizes */ 184 1, /* dma_attr_minxfer */ 185 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 186 0xffffffffull, /* dma_attr_seg */ 187 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 188 512, /* dma_attr_granular */ 189 0, /* dma_attr_flags */ 190 }; 191 192 /* 193 * DMA attributes for the rcvd FIS 194 * 195 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 196 * does not support 64-bit addressing 197 */ 198 static ddi_dma_attr_t rcvd_fis_dma_attr = { 199 DMA_ATTR_V0, /* dma_attr_version */ 200 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 201 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 202 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 203 0x100ull, /* dma_attr_align: 256-byte aligned */ 204 1, /* dma_attr_burstsizes */ 205 1, /* dma_attr_minxfer */ 206 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 207 0xffffffffull, /* dma_attr_seg */ 208 1, /* dma_attr_sgllen */ 209 1, /* dma_attr_granular */ 210 0, /* dma_attr_flags */ 211 }; 212 213 /* 214 * DMA attributes for the command list 215 * 216 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 217 * does not support 64-bit addressing 218 */ 219 static ddi_dma_attr_t cmd_list_dma_attr = { 220 DMA_ATTR_V0, /* dma_attr_version */ 221 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 222 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 223 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 224 0x400ull, /* dma_attr_align: 1K-byte aligned */ 225 1, /* dma_attr_burstsizes */ 226 1, /* dma_attr_minxfer */ 227 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 228 0xffffffffull, /* dma_attr_seg */ 229 1, /* dma_attr_sgllen */ 230 1, /* dma_attr_granular */ 231 0, /* dma_attr_flags */ 232 }; 233 234 /* 235 * DMA attributes for cmd tables 236 * 237 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 238 * does not support 64-bit addressing 239 */ 240 static ddi_dma_attr_t cmd_table_dma_attr = { 241 DMA_ATTR_V0, /* dma_attr_version */ 242 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 243 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 244 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 245 0x80ull, /* dma_attr_align: 128-byte aligned */ 246 1, /* dma_attr_burstsizes */ 247 1, /* dma_attr_minxfer */ 248 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 249 0xffffffffull, /* dma_attr_seg */ 250 1, /* dma_attr_sgllen */ 251 1, /* dma_attr_granular */ 252 0, /* dma_attr_flags */ 253 }; 254 255 256 /* Device access attributes */ 257 static ddi_device_acc_attr_t accattr = { 258 DDI_DEVICE_ATTR_V0, 259 DDI_STRUCTURE_LE_ACC, 260 DDI_STRICTORDER_ACC 261 }; 262 263 264 static struct dev_ops ahcictl_dev_ops = { 265 DEVO_REV, /* devo_rev */ 266 0, /* refcnt */ 267 ahci_getinfo, /* info */ 268 nulldev, /* identify */ 269 nulldev, /* probe */ 270 ahci_attach, /* attach */ 271 ahci_detach, /* detach */ 272 nodev, /* no reset */ 273 (struct cb_ops *)0, /* driver operations */ 274 NULL, /* bus operations */ 275 NULL, /* power */ 276 ahci_quiesce, /* quiesce */ 277 }; 278 279 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 280 SATA_TRAN_HOTPLUG_OPS_REV_1, 281 ahci_tran_hotplug_port_activate, 282 ahci_tran_hotplug_port_deactivate 283 }; 284 285 extern struct mod_ops mod_driverops; 286 287 static struct modldrv modldrv = { 288 &mod_driverops, /* driverops */ 289 ahci_ident, /* short description */ 290 &ahcictl_dev_ops, /* driver ops */ 291 }; 292 293 static struct modlinkage modlinkage = { 294 MODREV_1, 295 &modldrv, 296 NULL 297 }; 298 299 /* The following variables are watchdog handler related */ 300 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 301 static int ahci_watchdog_tick; 302 303 /* 304 * This static variable indicates the size of command table, 305 * and it's changeable with prdt number, which ahci_dma_prdt_number 306 * indicates. 307 */ 308 static size_t ahci_cmd_table_size; 309 310 /* 311 * ahci_dma_prdt_number, ahci_msi_enabled and ahci_64bit_dma_addressed 312 * are global variables, and can be changed via /etc/system file. 313 */ 314 315 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 316 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 317 318 /* AHCI MSI is tunable */ 319 boolean_t ahci_msi_enabled = B_FALSE; 320 321 /* 64-bit dma addressing is tunable */ 322 boolean_t ahci_64bit_dma_addressed = B_TRUE; 323 324 #if AHCI_DEBUG 325 uint32_t ahci_debug_flags = 0; 326 327 /* The following is needed for ahci_log() */ 328 static kmutex_t ahci_log_mutex; 329 static char ahci_log_buf[512]; 330 #endif 331 332 /* Opaque state pointer initialized by ddi_soft_state_init() */ 333 static void *ahci_statep = NULL; 334 335 /* 336 * ahci module initialization. 337 */ 338 int 339 _init(void) 340 { 341 int ret; 342 343 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 344 if (ret != 0) { 345 goto err_out; 346 } 347 348 #if AHCI_DEBUG 349 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 350 #endif 351 352 if ((ret = sata_hba_init(&modlinkage)) != 0) { 353 #if AHCI_DEBUG 354 mutex_destroy(&ahci_log_mutex); 355 #endif 356 ddi_soft_state_fini(&ahci_statep); 357 goto err_out; 358 } 359 360 ret = mod_install(&modlinkage); 361 if (ret != 0) { 362 sata_hba_fini(&modlinkage); 363 #if AHCI_DEBUG 364 mutex_destroy(&ahci_log_mutex); 365 #endif 366 ddi_soft_state_fini(&ahci_statep); 367 goto err_out; 368 } 369 370 /* watchdog tick */ 371 ahci_watchdog_tick = drv_usectohz( 372 (clock_t)ahci_watchdog_timeout * 1000000); 373 return (ret); 374 375 err_out: 376 cmn_err(CE_WARN, "!ahci: Module init failed"); 377 return (ret); 378 } 379 380 /* 381 * ahci module uninitialize. 382 */ 383 int 384 _fini(void) 385 { 386 int ret; 387 388 ret = mod_remove(&modlinkage); 389 if (ret != 0) { 390 return (ret); 391 } 392 393 /* Remove the resources allocated in _init(). */ 394 sata_hba_fini(&modlinkage); 395 #if AHCI_DEBUG 396 mutex_destroy(&ahci_log_mutex); 397 #endif 398 ddi_soft_state_fini(&ahci_statep); 399 400 return (ret); 401 } 402 403 /* 404 * _info entry point 405 */ 406 int 407 _info(struct modinfo *modinfop) 408 { 409 return (mod_info(&modlinkage, modinfop)); 410 } 411 412 /* 413 * The attach entry point for dev_ops. 414 */ 415 static int 416 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 417 { 418 ahci_ctl_t *ahci_ctlp; 419 int instance = ddi_get_instance(dip); 420 int status; 421 int attach_state; 422 uint32_t cap_status, ahci_version; 423 int intr_types; 424 int i; 425 pci_regspec_t *regs; 426 int regs_length; 427 int rnumber; 428 #if AHCI_DEBUG 429 int speed; 430 #endif 431 432 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter"); 433 434 switch (cmd) { 435 case DDI_ATTACH: 436 break; 437 438 case DDI_RESUME: 439 440 /* 441 * During DDI_RESUME, the hardware state of the device 442 * (power may have been removed from the device) must be 443 * restored, allow pending requests to continue, and 444 * service new requests. 445 */ 446 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 447 mutex_enter(&ahci_ctlp->ahcictl_mutex); 448 449 /* Restart watch thread */ 450 if (ahci_ctlp->ahcictl_timeout_id == 0) 451 ahci_ctlp->ahcictl_timeout_id = timeout( 452 (void (*)(void *))ahci_watchdog_handler, 453 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 454 455 mutex_exit(&ahci_ctlp->ahcictl_mutex); 456 457 /* 458 * Re-initialize the controller and enable the interrupts and 459 * restart all the ports. 460 * 461 * Note that so far we don't support hot-plug during 462 * suspend/resume. 463 */ 464 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 465 AHCIDBG0(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 466 "Failed to initialize the controller " 467 "during DDI_RESUME"); 468 return (DDI_FAILURE); 469 } 470 471 mutex_enter(&ahci_ctlp->ahcictl_mutex); 472 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 473 mutex_exit(&ahci_ctlp->ahcictl_mutex); 474 475 return (DDI_SUCCESS); 476 477 default: 478 return (DDI_FAILURE); 479 } 480 481 attach_state = AHCI_ATTACH_STATE_NONE; 482 483 /* Allocate soft state */ 484 status = ddi_soft_state_zalloc(ahci_statep, instance); 485 if (status != DDI_SUCCESS) { 486 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 487 instance); 488 goto err_out; 489 } 490 491 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 492 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 493 ahci_ctlp->ahcictl_dip = dip; 494 495 /* Initialize the cport/port mapping */ 496 for (i = 0; i < AHCI_MAX_PORTS; i++) { 497 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 498 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 499 } 500 501 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 502 503 /* 504 * Now map the AHCI base address; which includes global 505 * registers and port control registers 506 * 507 * According to the spec, the AHCI Base Address is BAR5, 508 * but BAR0-BAR4 are optional, so we need to check which 509 * rnumber is used for BAR5. 510 */ 511 512 /* 513 * search through DDI "reg" property for the AHCI register set 514 */ 515 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 516 DDI_PROP_DONTPASS, "reg", (int **)®s, 517 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 518 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 519 instance); 520 goto err_out; 521 } 522 523 /* AHCI Base Address is located at 0x24 offset */ 524 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 525 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 526 == AHCI_PCI_RNUM) 527 break; 528 } 529 530 ddi_prop_free(regs); 531 532 if (rnumber == regs_length) { 533 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 534 instance); 535 goto err_out; 536 } 537 538 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 539 540 status = ddi_regs_map_setup(dip, 541 rnumber, 542 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 543 0, 544 0, 545 &accattr, 546 &ahci_ctlp->ahcictl_ahci_acc_handle); 547 if (status != DDI_SUCCESS) { 548 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 549 instance); 550 goto err_out; 551 } 552 553 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 554 555 /* Get the AHCI version information */ 556 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 557 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 558 559 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 560 (ahci_version & 0xffff0000) >> 16, 561 ((ahci_version & 0x0000ff00) >> 4 | 562 (ahci_version & 0x000000ff))); 563 564 /* We don't support controllers whose versions are lower than 1.0 */ 565 if (!(ahci_version & 0xffff0000)) { 566 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 567 "than version 1.0", instance); 568 goto err_out; 569 } 570 571 /* Get the HBA capabilities information */ 572 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 573 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 574 575 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 576 cap_status); 577 578 #if AHCI_DEBUG 579 /* Get the interface speed supported by the HBA */ 580 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 581 if (speed == 0x01) { 582 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 583 "hba interface speed support: Gen 1 (1.5Gbps)"); 584 } else if (speed == 0x10) { 585 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 586 "hba interface speed support: Gen 2 (3 Gbps)"); 587 } else if (speed == 0x11) { 588 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 589 "hba interface speed support: Gen 3 (6 Gbps)"); 590 } 591 #endif 592 593 /* Get the number of command slots supported by the HBA */ 594 ahci_ctlp->ahcictl_num_cmd_slots = 595 ((cap_status & AHCI_HBA_CAP_NCS) >> 596 AHCI_HBA_CAP_NCS_SHIFT) + 1; 597 598 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 599 ahci_ctlp->ahcictl_num_cmd_slots); 600 601 /* Get the bit map which indicates ports implemented by the HBA */ 602 ahci_ctlp->ahcictl_ports_implemented = 603 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 604 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 605 606 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 607 ahci_ctlp->ahcictl_ports_implemented); 608 609 /* 610 * According to the AHCI spec, CAP.NP should indicate the maximum 611 * number of ports supported by the HBA silicon, but we found 612 * this value of ICH8 chipset only indicates the number of ports 613 * implemented (exposed) by it. Therefore, the driver should calculate 614 * the potential maximum value by checking PI register, and use 615 * the maximum of this value and CAP.NP. 616 */ 617 ahci_ctlp->ahcictl_num_ports = max( 618 (cap_status & AHCI_HBA_CAP_NP) + 1, 619 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 620 621 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 622 ahci_ctlp->ahcictl_num_ports); 623 624 /* Get the number of implemented ports by the HBA */ 625 ahci_ctlp->ahcictl_num_implemented_ports = 626 ahci_get_num_implemented_ports( 627 ahci_ctlp->ahcictl_ports_implemented); 628 629 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 630 "hba number of implemented ports: %d", 631 ahci_ctlp->ahcictl_num_implemented_ports); 632 633 /* Check whether HBA supports 64bit DMA addressing */ 634 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 635 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 636 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 637 "hba does not support 64-bit addressing"); 638 } 639 640 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 641 != DDI_SUCCESS) { 642 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 643 instance); 644 goto err_out; 645 } 646 647 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 648 649 /* 650 * Check the pci configuration space, and set caps. We also 651 * handle the hardware defect in this function. 652 * 653 * For example, force ATI SB600/SB700/SB750 to use 32-bit dma 654 * addressing since it doesn't support 64-bit dma though their 655 * registers declare they support. 656 */ 657 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 658 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 659 instance); 660 goto err_out; 661 } 662 663 /* 664 * Disable the whole controller interrupts before adding 665 * interrupt handlers(s). 666 */ 667 ahci_disable_all_intrs(ahci_ctlp); 668 669 /* Get supported interrupt types */ 670 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 671 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 672 instance); 673 goto err_out; 674 } 675 676 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 677 "ddi_intr_get_supported_types() returned: 0x%x", 678 intr_types); 679 680 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 681 /* 682 * Try MSI first, but fall back to FIXED if failed 683 */ 684 if (ahci_add_msi_intrs(ahci_ctlp) == DDI_SUCCESS) { 685 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 686 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 687 "Using MSI interrupt type"); 688 goto intr_done; 689 } 690 691 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 692 "MSI registration failed, " 693 "trying FIXED interrupts"); 694 } 695 696 if (intr_types & DDI_INTR_TYPE_FIXED) { 697 if (ahci_add_legacy_intrs(ahci_ctlp) == DDI_SUCCESS) { 698 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 699 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 700 "Using FIXED interrupt type"); 701 goto intr_done; 702 } 703 704 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 705 "FIXED interrupt registration failed"); 706 } 707 708 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 709 710 goto err_out; 711 712 intr_done: 713 714 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 715 716 /* Initialize the controller mutex */ 717 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 718 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 719 720 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 721 722 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 723 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 724 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 725 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 726 } 727 728 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 729 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 730 sizeof (ahci_prdt_item_t)); 731 732 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 733 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 734 " ahci_cmd_table_size is 0x%x", 735 ahci_dma_prdt_number, ahci_cmd_table_size); 736 737 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 738 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 739 ahci_dma_prdt_number; 740 741 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 742 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 743 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 744 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 745 746 if ((ahci_64bit_dma_addressed == B_FALSE) || 747 (ahci_ctlp->ahcictl_cap & AHCI_CAP_32BIT_DMA)) { 748 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 749 0xffffffffull; 750 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 751 0xffffffffull; 752 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 753 0xffffffffull; 754 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 755 0xffffffffull; 756 } 757 758 /* Allocate the ports structure */ 759 status = ahci_alloc_ports_state(ahci_ctlp); 760 if (status != AHCI_SUCCESS) { 761 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 762 instance); 763 goto err_out; 764 } 765 766 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 767 768 /* 769 * A taskq is created for dealing with events 770 */ 771 if ((ahci_ctlp->ahcictl_event_taskq = ddi_taskq_create(dip, 772 "ahci_event_handle_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 773 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 774 "handle", instance); 775 goto err_out; 776 } 777 778 attach_state |= AHCI_ATTACH_STATE_ERR_RECV_TASKQ; 779 780 /* 781 * Initialize the controller and ports. 782 */ 783 status = ahci_initialize_controller(ahci_ctlp); 784 if (status != AHCI_SUCCESS) { 785 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 786 instance); 787 goto err_out; 788 } 789 790 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 791 792 /* Start one thread to check packet timeouts */ 793 ahci_ctlp->ahcictl_timeout_id = timeout( 794 (void (*)(void *))ahci_watchdog_handler, 795 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 796 797 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 798 799 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 800 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 801 instance); 802 goto err_out; 803 } 804 805 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 806 807 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!"); 808 809 return (DDI_SUCCESS); 810 811 err_out: 812 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 813 mutex_enter(&ahci_ctlp->ahcictl_mutex); 814 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 815 ahci_ctlp->ahcictl_timeout_id = 0; 816 mutex_exit(&ahci_ctlp->ahcictl_mutex); 817 } 818 819 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 820 ahci_uninitialize_controller(ahci_ctlp); 821 } 822 823 if (attach_state & AHCI_ATTACH_STATE_ERR_RECV_TASKQ) { 824 ddi_taskq_destroy(ahci_ctlp->ahcictl_event_taskq); 825 } 826 827 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 828 ahci_dealloc_ports_state(ahci_ctlp); 829 } 830 831 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 832 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 833 } 834 835 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 836 ahci_rem_intrs(ahci_ctlp); 837 } 838 839 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 840 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 841 } 842 843 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 844 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 845 } 846 847 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 848 ddi_soft_state_free(ahci_statep, instance); 849 } 850 851 return (DDI_FAILURE); 852 } 853 854 /* 855 * The detach entry point for dev_ops. 856 */ 857 static int 858 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 859 { 860 ahci_ctl_t *ahci_ctlp; 861 int instance; 862 int ret; 863 864 instance = ddi_get_instance(dip); 865 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 866 867 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter"); 868 869 switch (cmd) { 870 case DDI_DETACH: 871 872 /* disable the interrupts for an uninterrupted detach */ 873 mutex_enter(&ahci_ctlp->ahcictl_mutex); 874 ahci_disable_all_intrs(ahci_ctlp); 875 mutex_exit(&ahci_ctlp->ahcictl_mutex); 876 877 /* unregister from the sata framework. */ 878 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 879 if (ret != AHCI_SUCCESS) { 880 mutex_enter(&ahci_ctlp->ahcictl_mutex); 881 ahci_enable_all_intrs(ahci_ctlp); 882 mutex_exit(&ahci_ctlp->ahcictl_mutex); 883 return (DDI_FAILURE); 884 } 885 886 mutex_enter(&ahci_ctlp->ahcictl_mutex); 887 888 /* stop the watchdog handler */ 889 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 890 ahci_ctlp->ahcictl_timeout_id = 0; 891 892 mutex_exit(&ahci_ctlp->ahcictl_mutex); 893 894 /* uninitialize the controller */ 895 ahci_uninitialize_controller(ahci_ctlp); 896 897 /* remove the interrupts */ 898 ahci_rem_intrs(ahci_ctlp); 899 900 /* destroy the taskq */ 901 ddi_taskq_destroy(ahci_ctlp->ahcictl_event_taskq); 902 903 /* deallocate the ports structures */ 904 ahci_dealloc_ports_state(ahci_ctlp); 905 906 /* destroy mutex */ 907 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 908 909 /* teardown the pci config */ 910 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 911 912 /* remove the reg maps. */ 913 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 914 915 /* free the soft state. */ 916 ddi_soft_state_free(ahci_statep, instance); 917 918 return (DDI_SUCCESS); 919 920 case DDI_SUSPEND: 921 922 /* 923 * The steps associated with suspension must include putting 924 * the underlying device into a quiescent state so that it 925 * will not generate interrupts or modify or access memory. 926 */ 927 mutex_enter(&ahci_ctlp->ahcictl_mutex); 928 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 929 mutex_exit(&ahci_ctlp->ahcictl_mutex); 930 return (DDI_SUCCESS); 931 } 932 933 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 934 935 /* stop the watchdog handler */ 936 if (ahci_ctlp->ahcictl_timeout_id) { 937 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 938 ahci_ctlp->ahcictl_timeout_id = 0; 939 } 940 941 mutex_exit(&ahci_ctlp->ahcictl_mutex); 942 943 /* 944 * drain the taskq 945 */ 946 ddi_taskq_wait(ahci_ctlp->ahcictl_event_taskq); 947 948 /* 949 * Disable the interrupts and stop all the ports. 950 */ 951 ahci_uninitialize_controller(ahci_ctlp); 952 953 return (DDI_SUCCESS); 954 955 default: 956 return (DDI_FAILURE); 957 } 958 } 959 960 /* 961 * The info entry point for dev_ops. 962 * 963 */ 964 static int 965 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 966 void *arg, void **result) 967 { 968 #ifndef __lock_lint 969 _NOTE(ARGUNUSED(dip)) 970 #endif /* __lock_lint */ 971 972 ahci_ctl_t *ahci_ctlp; 973 int instance; 974 dev_t dev; 975 976 dev = (dev_t)arg; 977 instance = getminor(dev); 978 979 switch (infocmd) { 980 case DDI_INFO_DEVT2DEVINFO: 981 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 982 if (ahci_ctlp != NULL) { 983 *result = ahci_ctlp->ahcictl_dip; 984 return (DDI_SUCCESS); 985 } else { 986 *result = NULL; 987 return (DDI_FAILURE); 988 } 989 case DDI_INFO_DEVT2INSTANCE: 990 *(int *)result = instance; 991 break; 992 default: 993 break; 994 } 995 996 return (DDI_SUCCESS); 997 } 998 999 /* 1000 * Registers the ahci with sata framework. 1001 */ 1002 static int 1003 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1004 { 1005 struct sata_hba_tran *sata_hba_tran; 1006 1007 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1008 "ahci_register_sata_hba_tran enter"); 1009 1010 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1011 1012 /* Allocate memory for the sata_hba_tran */ 1013 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1014 1015 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 1016 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1017 sata_hba_tran->sata_tran_hba_dma_attr = 1018 &ahci_ctlp->ahcictl_buffer_dma_attr; 1019 1020 /* Report the number of implemented ports */ 1021 sata_hba_tran->sata_tran_hba_num_cports = 1022 ahci_ctlp->ahcictl_num_implemented_ports; 1023 1024 /* Support ATAPI device */ 1025 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1026 1027 /* Get the data transfer capability for PIO command by the HBA */ 1028 if (cap_status & AHCI_HBA_CAP_PMD) { 1029 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1030 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1031 "DRQ block data transfer for PIO command protocol"); 1032 } 1033 1034 /* 1035 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1036 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1037 * QUEUED (EXT), and SERVICE commands). Queued operations are 1038 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1039 * QUEUED commands when the HBA and device support native command 1040 * queuing(NCQ). 1041 * 1042 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1043 * CAP register of the HBA indicates NCQ is supported. 1044 * 1045 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1046 * set because the previous register content of PxCI can be re-written 1047 * in the register write. 1048 */ 1049 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1050 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1051 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1052 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1053 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1054 "Command Queuing"); 1055 } 1056 1057 /* Report the number of command slots */ 1058 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1059 1060 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1061 sata_hba_tran->sata_tran_start = ahci_tran_start; 1062 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1063 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1064 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1065 #ifdef __lock_lint 1066 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1067 #endif 1068 /* 1069 * When SATA framework adds support for pwrmgt the 1070 * pwrmgt_ops needs to be updated 1071 */ 1072 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1073 sata_hba_tran->sata_tran_ioctl = NULL; 1074 1075 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1076 1077 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1078 1079 /* Attach it to SATA framework */ 1080 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1081 != DDI_SUCCESS) { 1082 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1083 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1084 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1085 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1086 return (AHCI_FAILURE); 1087 } 1088 1089 return (AHCI_SUCCESS); 1090 } 1091 1092 /* 1093 * Unregisters the ahci with sata framework. 1094 */ 1095 static int 1096 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1097 { 1098 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 1099 "ahci_unregister_sata_hba_tran enter"); 1100 1101 /* Detach from the SATA framework. */ 1102 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1103 DDI_SUCCESS) { 1104 return (AHCI_FAILURE); 1105 } 1106 1107 /* Deallocate sata_hba_tran. */ 1108 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1109 sizeof (sata_hba_tran_t)); 1110 1111 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1112 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1113 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1114 1115 return (AHCI_SUCCESS); 1116 } 1117 1118 /* 1119 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1120 * port status registers and an attached device type via sata_device 1121 * structure. 1122 * 1123 * We return the cached information from a previous hardware probe. The 1124 * actual hardware probing itself was done either from within 1125 * ahci_initialize_controller() during the driver attach or from a phy 1126 * ready change interrupt handler. 1127 */ 1128 static int 1129 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1130 { 1131 ahci_ctl_t *ahci_ctlp; 1132 ahci_port_t *ahci_portp; 1133 uint8_t cport = sd->satadev_addr.cport; 1134 #if AHCI_DEBUG 1135 uint8_t pmport = sd->satadev_addr.pmport; 1136 uint8_t qual = sd->satadev_addr.qual; 1137 #endif 1138 uint8_t device_type; 1139 uint32_t port_state; 1140 uint8_t port; 1141 1142 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1143 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1144 1145 AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp, 1146 "ahci_tran_probe_port enter: cport: %d, " 1147 "pmport: %d, qual: %d", cport, pmport, qual); 1148 1149 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1150 1151 mutex_enter(&ahci_portp->ahciport_mutex); 1152 1153 port_state = ahci_portp->ahciport_port_state; 1154 switch (port_state) { 1155 1156 case SATA_PSTATE_FAILED: 1157 sd->satadev_state = SATA_PSTATE_FAILED; 1158 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1159 "ahci_tran_probe_port: port %d PORT FAILED", port); 1160 goto out; 1161 1162 case SATA_PSTATE_SHUTDOWN: 1163 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1164 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1165 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1166 goto out; 1167 1168 case SATA_PSTATE_PWROFF: 1169 sd->satadev_state = SATA_PSTATE_PWROFF; 1170 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1171 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1172 goto out; 1173 1174 case SATA_PSTATE_PWRON: 1175 sd->satadev_state = SATA_PSTATE_PWRON; 1176 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1177 "ahci_tran_probe_port: port %d PORT PWRON", port); 1178 break; 1179 1180 default: 1181 sd->satadev_state = port_state; 1182 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 1183 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1184 port, port_state); 1185 break; 1186 } 1187 1188 device_type = ahci_portp->ahciport_device_type; 1189 1190 switch (device_type) { 1191 1192 case SATA_DTYPE_ATADISK: 1193 sd->satadev_type = SATA_DTYPE_ATADISK; 1194 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1195 "ahci_tran_probe_port: port %d DISK found", port); 1196 break; 1197 1198 case SATA_DTYPE_ATAPI: 1199 /* 1200 * HBA driver only knows it's an ATAPI device, and don't know 1201 * it's CD/DVD or tape because the ATAPI device type need to 1202 * be determined by checking IDENTIFY PACKET DEVICE data 1203 */ 1204 sd->satadev_type = SATA_DTYPE_ATAPI; 1205 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1206 "ahci_tran_probe_port: port %d ATAPI found", port); 1207 break; 1208 1209 case SATA_DTYPE_PMULT: 1210 sd->satadev_type = SATA_DTYPE_PMULT; 1211 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1212 "ahci_tran_probe_port: port %d Port Multiplier found", 1213 port); 1214 break; 1215 1216 case SATA_DTYPE_UNKNOWN: 1217 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1218 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1219 "ahci_tran_probe_port: port %d Unknown device found", port); 1220 break; 1221 1222 default: 1223 /* we don't support any other device types */ 1224 sd->satadev_type = SATA_DTYPE_NONE; 1225 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1226 "ahci_tran_probe_port: port %d No device found", port); 1227 break; 1228 } 1229 1230 out: 1231 ahci_update_sata_registers(ahci_ctlp, port, sd); 1232 mutex_exit(&ahci_portp->ahciport_mutex); 1233 1234 return (SATA_SUCCESS); 1235 } 1236 1237 /* 1238 * There are four operation modes in sata framework: 1239 * SATA_OPMODE_INTERRUPTS 1240 * SATA_OPMODE_POLLING 1241 * SATA_OPMODE_ASYNCH 1242 * SATA_OPMODE_SYNCH 1243 * 1244 * Their combined meanings as following: 1245 * 1246 * SATA_OPMODE_SYNCH 1247 * The command has to be completed before sata_tran_start functions returns. 1248 * Either interrupts or polling could be used - it's up to the driver. 1249 * Mode used currently for internal, sata-module initiated operations. 1250 * 1251 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1252 * It is the same as the one above. 1253 * 1254 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1255 * The command has to be completed before sata_tran_start function returns. 1256 * No interrupt used, polling only. This should be the mode used for scsi 1257 * packets with FLAG_NOINTR. 1258 * 1259 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1260 * The command may be queued (callback function specified). Interrupts could 1261 * be used. It's normal operation mode. 1262 */ 1263 /* 1264 * Called by sata framework to transport a sata packet down stream. 1265 */ 1266 static int 1267 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1268 { 1269 ahci_ctl_t *ahci_ctlp; 1270 ahci_port_t *ahci_portp; 1271 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1272 uint8_t port; 1273 1274 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1275 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1276 1277 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, 1278 "ahci_tran_start enter: cport %d satapkt 0x%p", 1279 cport, (void *)spkt); 1280 1281 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1282 1283 mutex_enter(&ahci_portp->ahciport_mutex); 1284 1285 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1286 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1287 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1288 /* 1289 * In case the targer driver would send the packet before 1290 * sata framework can have the opportunity to process those 1291 * event reports. 1292 */ 1293 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1294 spkt->satapkt_device.satadev_state = 1295 ahci_portp->ahciport_port_state; 1296 ahci_update_sata_registers(ahci_ctlp, port, 1297 &spkt->satapkt_device); 1298 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1299 "ahci_tran_start returning PORT_ERROR while " 1300 "port in FAILED/SHUTDOWN/PWROFF state: " 1301 "port: %d", port); 1302 mutex_exit(&ahci_portp->ahciport_mutex); 1303 return (SATA_TRAN_PORT_ERROR); 1304 } 1305 1306 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1307 /* 1308 * ahci_intr_phyrdy_change() may have rendered it to 1309 * SATA_DTYPE_NONE. 1310 */ 1311 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1312 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1313 spkt->satapkt_device.satadev_state = 1314 ahci_portp->ahciport_port_state; 1315 ahci_update_sata_registers(ahci_ctlp, port, 1316 &spkt->satapkt_device); 1317 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1318 "ahci_tran_start returning PORT_ERROR while " 1319 "no device attached: port: %d", port); 1320 mutex_exit(&ahci_portp->ahciport_mutex); 1321 return (SATA_TRAN_PORT_ERROR); 1322 } 1323 1324 /* 1325 * SATA HBA driver should remember that a device was reset and it 1326 * is supposed to reject any packets which do not specify either 1327 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1328 * 1329 * This is to prevent a race condition when a device was arbitrarily 1330 * reset by the HBA driver (and lost it's setting) and a target 1331 * driver sending some commands to a device before the sata framework 1332 * has a chance to restore the device setting (such as cache enable/ 1333 * disable or other resettable stuff). 1334 */ 1335 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1336 ahci_portp->ahciport_reset_in_progress = 0; 1337 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1338 "ahci_tran_start clearing the " 1339 "reset_in_progress for port: %d", port); 1340 } 1341 1342 if (ahci_portp->ahciport_reset_in_progress && 1343 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1344 ! ddi_in_panic()) { 1345 spkt->satapkt_reason = SATA_PKT_BUSY; 1346 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1347 "ahci_tran_start returning BUSY while " 1348 "reset in progress: port: %d", port); 1349 mutex_exit(&ahci_portp->ahciport_mutex); 1350 return (SATA_TRAN_BUSY); 1351 } 1352 1353 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1354 spkt->satapkt_reason = SATA_PKT_BUSY; 1355 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1356 "ahci_tran_start returning BUSY while " 1357 "mopping in progress: port: %d", port); 1358 mutex_exit(&ahci_portp->ahciport_mutex); 1359 return (SATA_TRAN_BUSY); 1360 } 1361 1362 if (spkt->satapkt_op_mode & 1363 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1364 /* 1365 * If a SYNC command to be executed in interrupt context, 1366 * bounce it back to sata module. 1367 */ 1368 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1369 servicing_interrupt()) { 1370 spkt->satapkt_reason = SATA_PKT_BUSY; 1371 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1372 "ahci_tran_start returning BUSY while " 1373 "sending SYNC mode under interrupt context: " 1374 "port : %d", port); 1375 mutex_exit(&ahci_portp->ahciport_mutex); 1376 return (SATA_TRAN_BUSY); 1377 } 1378 1379 /* We need to do the sync start now */ 1380 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1381 spkt) == AHCI_FAILURE) { 1382 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1383 "return QUEUE_FULL: port %d", port); 1384 mutex_exit(&ahci_portp->ahciport_mutex); 1385 return (SATA_TRAN_QUEUE_FULL); 1386 } 1387 } else { 1388 /* Async start, using interrupt */ 1389 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1390 == AHCI_FAILURE) { 1391 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1392 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1393 "returning QUEUE_FULL: port %d", port); 1394 mutex_exit(&ahci_portp->ahciport_mutex); 1395 return (SATA_TRAN_QUEUE_FULL); 1396 } 1397 } 1398 1399 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1400 "sata tran accepted: port %d", port); 1401 1402 mutex_exit(&ahci_portp->ahciport_mutex); 1403 return (SATA_TRAN_ACCEPTED); 1404 } 1405 1406 /* 1407 * SATA_OPMODE_SYNCH flag is set 1408 * 1409 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1410 * without interrupt, otherwise we can still use the interrupt. 1411 * 1412 * WARNING!!! ahciport_mutex should be acquired before the function 1413 * is called. 1414 */ 1415 static int 1416 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1417 uint8_t port, sata_pkt_t *spkt) 1418 { 1419 int pkt_timeout_ticks; 1420 uint32_t timeout_tags; 1421 int rval; 1422 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1423 1424 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1425 "port %d spkt 0x%p", port, spkt); 1426 1427 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1428 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1429 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1430 port, spkt)) == AHCI_FAILURE) { 1431 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1432 return (rval); 1433 } 1434 1435 pkt_timeout_ticks = 1436 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1437 1438 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1439 mutex_exit(&ahci_portp->ahciport_mutex); 1440 1441 /* Simulate the interrupt */ 1442 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1443 1444 drv_usecwait(AHCI_1MS_USECS); 1445 1446 mutex_enter(&ahci_portp->ahciport_mutex); 1447 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1448 if (pkt_timeout_ticks < 0) { 1449 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1450 "port %d satapkt 0x%p timed out\n", 1451 instance, port, (void *)spkt); 1452 timeout_tags = (0x1 << rval); 1453 mutex_exit(&ahci_portp->ahciport_mutex); 1454 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1455 port, timeout_tags); 1456 mutex_enter(&ahci_portp->ahciport_mutex); 1457 } 1458 } 1459 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1460 return (AHCI_SUCCESS); 1461 1462 } else { 1463 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1464 port, spkt)) == AHCI_FAILURE) 1465 return (rval); 1466 1467 #if AHCI_DEBUG 1468 /* 1469 * Note that the driver always uses the slot 0 to deliver 1470 * REQUEST SENSE or READ LOG EXT command 1471 */ 1472 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1473 ASSERT(rval == 0); 1474 #endif 1475 1476 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1477 cv_wait(&ahci_portp->ahciport_cv, 1478 &ahci_portp->ahciport_mutex); 1479 1480 return (AHCI_SUCCESS); 1481 } 1482 } 1483 1484 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1485 if (satapkt) { \ 1486 satapkt->satapkt_reason = reason; \ 1487 /* \ 1488 * We set the satapkt_reason in both sync and \ 1489 * non-sync cases. \ 1490 */ \ 1491 } \ 1492 if (satapkt && \ 1493 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1494 satapkt->satapkt_comp) { \ 1495 mutex_exit(&ahci_portp->ahciport_mutex); \ 1496 (*satapkt->satapkt_comp)(satapkt); \ 1497 mutex_enter(&ahci_portp->ahciport_mutex); \ 1498 } else { \ 1499 if (satapkt && \ 1500 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1501 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1502 cv_signal(&ahci_portp->ahciport_cv); \ 1503 } 1504 1505 /* 1506 * Searches for and claims a free command slot. 1507 * 1508 * Returns: 1509 * 1510 * AHCI_FAILURE if failed 1511 * 1. if no empty slot left 1512 * 2. non-queued command requested while queued command(s) is outstanding 1513 * 3. queued command requested whild non-queued command(s) is outstanding 1514 * 4. HBA doesn't support multiple-use of command list while already a 1515 * non-queued command is oustanding 1516 * 1517 * claimed slot number if succeeded 1518 * 1519 * NOTE: it will always return slot 0 during error recovery process for 1520 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1521 * 1522 * WARNING!!! ahciport_mutex should be acquired before the function 1523 * is called. 1524 */ 1525 static int 1526 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1527 ahci_port_t *ahci_portp, int command_type) 1528 { 1529 uint32_t free_slots; 1530 int slot; 1531 1532 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1533 "ahciport_pending_tags = 0x%x " 1534 "ahciport_pending_ncq_tags = 0x%x", 1535 ahci_portp->ahciport_pending_tags, 1536 ahci_portp->ahciport_pending_ncq_tags); 1537 1538 /* 1539 * According to the AHCI spec, system software is responsible to 1540 * ensure that queued and non-queued commands are not mixed in 1541 * the command list. 1542 */ 1543 if (command_type == AHCI_NON_NCQ_CMD) { 1544 /* Non-NCQ command request */ 1545 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1546 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1547 "ahci_claim_free_slot: there is still pending " 1548 "queued command(s) in the command list, " 1549 "so no available slot for the non-queued " 1550 "command"); 1551 return (AHCI_FAILURE); 1552 } 1553 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1554 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1555 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 1556 "ahci_claim_free_slot: HBA cannot support multiple-" 1557 "use of the command list for non-queued commands"); 1558 return (AHCI_FAILURE); 1559 } 1560 free_slots = (~ahci_portp->ahciport_pending_tags) & 1561 AHCI_SLOT_MASK(ahci_ctlp); 1562 } else if (command_type == AHCI_NCQ_CMD) { 1563 /* NCQ command request */ 1564 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1565 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1566 "ahci_claim_free_slot: there is still pending " 1567 "non-queued command(s) in the command list, " 1568 "so no available slot for the queued command"); 1569 return (AHCI_FAILURE); 1570 } 1571 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1572 AHCI_NCQ_SLOT_MASK(ahci_portp); 1573 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1574 /* Error retrieval command request */ 1575 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 1576 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1577 "SENSE or READ LOG EXT command"); 1578 slot = 0; 1579 goto out; 1580 } 1581 1582 slot = ddi_ffs(free_slots) - 1; 1583 if (slot == -1) { 1584 AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp, 1585 "ahci_claim_free_slot: no empty slots"); 1586 return (AHCI_FAILURE); 1587 } 1588 1589 /* 1590 * According to the AHCI spec, to allow a simple mechanism for the 1591 * HBA to map command list slots to queue entries, software must 1592 * match the tag number it uses to the slot it is placing the command 1593 * in. For example, if a queued command is placed in slot 5, the tag 1594 * for that command must be 5. 1595 */ 1596 if (command_type == AHCI_NCQ_CMD) { 1597 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1598 } 1599 1600 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1601 1602 out: 1603 AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp, 1604 "ahci_claim_free_slot: found slot: 0x%x", slot); 1605 1606 return (slot); 1607 } 1608 1609 /* 1610 * Builds the Command Table for the sata packet and delivers it to controller. 1611 * 1612 * Returns: 1613 * slot number if we can obtain a slot successfully 1614 * otherwise, return AHCI_FAILURE 1615 * 1616 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1617 */ 1618 static int 1619 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1620 uint8_t port, sata_pkt_t *spkt) 1621 { 1622 int cmd_slot; 1623 sata_cmd_t *scmd; 1624 ahci_fis_h2d_register_t *h2d_register_fisp; 1625 ahci_cmd_table_t *cmd_table; 1626 ahci_cmd_header_t *cmd_header; 1627 int ncookies; 1628 int i; 1629 int command_type = AHCI_NON_NCQ_CMD; 1630 int ncq_qdepth; 1631 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1632 #if AHCI_DEBUG 1633 uint32_t *ptr; 1634 uint8_t *ptr2; 1635 #endif 1636 1637 spkt->satapkt_reason = SATA_PKT_BUSY; 1638 1639 scmd = &spkt->satapkt_cmd; 1640 1641 /* Check if the command is a NCQ command */ 1642 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1643 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1644 command_type = AHCI_NCQ_CMD; 1645 1646 /* 1647 * When NCQ is support, system software must determine the 1648 * maximum tag allowed by the device and the HBA, and it 1649 * must use a value not beyond of the lower bound of the two. 1650 * 1651 * Sata module is going to calculate the qdepth and send 1652 * down to HBA driver via sata_cmd. 1653 */ 1654 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1655 1656 /* 1657 * At the moment, the driver doesn't support the dynamic 1658 * setting of the maximum ncq depth, and the value can be 1659 * set either during the attach or after hot-plug insertion. 1660 */ 1661 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1662 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1663 AHCIDBG2(AHCIDBG_NCQ, ahci_ctlp, 1664 "ahci_deliver_satapkt: port %d the max tags for " 1665 "NCQ command is %d", port, ncq_qdepth); 1666 } else { 1667 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1668 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1669 " port %d the max tag for NCQ command is " 1670 "requested to change from %d to %d, at the" 1671 " moment the driver doesn't support the " 1672 "dynamic change so it's going to " 1673 "still use the previous tag value", 1674 instance, port, 1675 ahci_portp->ahciport_max_ncq_tags, 1676 ncq_qdepth); 1677 } 1678 } 1679 } 1680 1681 /* Check if the command is an error retrieval command */ 1682 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1683 command_type = AHCI_ERR_RETRI_CMD; 1684 1685 /* Check if there is an empty command slot */ 1686 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1687 if (cmd_slot == AHCI_FAILURE) { 1688 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free command slot"); 1689 return (AHCI_FAILURE); 1690 } 1691 1692 AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1693 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1694 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1695 cmd_slot, port, (void *)spkt); 1696 1697 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1698 bzero((void *)cmd_table, ahci_cmd_table_size); 1699 1700 /* For data transfer operations, it is the H2D Register FIS */ 1701 h2d_register_fisp = 1702 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1703 1704 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1705 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1706 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1707 SET_FIS_PMP(h2d_register_fisp, 1708 spkt->satapkt_device.satadev_addr.pmport); 1709 } 1710 1711 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1712 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1713 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1714 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1715 1716 switch (scmd->satacmd_addr_type) { 1717 1718 case 0: 1719 /* 1720 * satacmd_addr_type will be 0 for the commands below: 1721 * ATAPI command 1722 * SATAC_IDLE_IM 1723 * SATAC_STANDBY_IM 1724 * SATAC_DOWNLOAD_MICROCODE 1725 * SATAC_FLUSH_CACHE 1726 * SATAC_SET_FEATURES 1727 * SATAC_SMART 1728 * SATAC_ID_PACKET_DEVICE 1729 * SATAC_ID_DEVICE 1730 */ 1731 /* FALLTHRU */ 1732 1733 case ATA_ADDR_LBA: 1734 /* FALLTHRU */ 1735 1736 case ATA_ADDR_LBA28: 1737 /* LBA[7:0] */ 1738 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1739 1740 /* LBA[15:8] */ 1741 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1742 1743 /* LBA[23:16] */ 1744 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1745 1746 /* LBA [27:24] (also called dev_head) */ 1747 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1748 1749 break; 1750 1751 case ATA_ADDR_LBA48: 1752 /* LBA[7:0] */ 1753 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1754 1755 /* LBA[15:8] */ 1756 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1757 1758 /* LBA[23:16] */ 1759 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1760 1761 /* LBA [31:24] */ 1762 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1763 scmd->satacmd_lba_low_msb); 1764 1765 /* LBA [39:32] */ 1766 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1767 scmd->satacmd_lba_mid_msb); 1768 1769 /* LBA [47:40] */ 1770 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1771 scmd->satacmd_lba_high_msb); 1772 1773 /* Set dev_head */ 1774 SET_FIS_DEV_HEAD(h2d_register_fisp, 1775 scmd->satacmd_device_reg); 1776 1777 /* Set the extended sector count and features */ 1778 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1779 scmd->satacmd_sec_count_msb); 1780 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1781 scmd->satacmd_features_reg_ext); 1782 break; 1783 } 1784 1785 /* 1786 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1787 * filled into features field, and sector count 8:15 is filled into 1788 * features (exp) field. TAG is filled into sector field. 1789 */ 1790 if (command_type == AHCI_NCQ_CMD) { 1791 SET_FIS_FEATURES(h2d_register_fisp, 1792 scmd->satacmd_sec_count_lsb); 1793 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1794 scmd->satacmd_sec_count_msb); 1795 1796 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1797 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1798 } 1799 1800 ncookies = scmd->satacmd_num_dma_cookies; 1801 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1802 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1803 ncookies, ahci_dma_prdt_number); 1804 1805 ASSERT(ncookies <= ahci_dma_prdt_number); 1806 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1807 1808 /* *** now fill the scatter gather list ******* */ 1809 for (i = 0; i < ncookies; i++) { 1810 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1811 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1812 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1813 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1814 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1815 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1816 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1817 scmd->satacmd_dma_cookie_list[i].dmac_size; 1818 } 1819 1820 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1821 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1822 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1823 1824 /* The ACMD field is filled in for ATAPI command */ 1825 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1826 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1827 SATA_ATAPI_MAX_CDB_LEN); 1828 } 1829 1830 /* Set Command Header in Command List */ 1831 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1832 BZERO_DESCR_INFO(cmd_header); 1833 BZERO_PRD_BYTE_COUNT(cmd_header); 1834 1835 /* Set the number of entries in the PRD table */ 1836 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1837 1838 /* Set the length of the command in the CFIS area */ 1839 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1840 1841 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1842 "sata_data_direction = 0x%x", 1843 scmd->satacmd_flags.sata_data_direction); 1844 1845 /* Set A bit if it is an ATAPI command */ 1846 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1847 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1848 1849 /* Set W bit if data is going to the device */ 1850 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1851 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1852 1853 /* 1854 * Set the prefetchable bit - this bit is only valid if the PRDTL 1855 * field is non-zero or the ATAPI 'A' bit is set in the command 1856 * header. This bit cannot be set when using native command 1857 * queuing commands or when using FIS-based switching with a Port 1858 * multiplier. 1859 */ 1860 if (command_type != AHCI_NCQ_CMD) 1861 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1862 1863 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1864 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1865 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1866 1867 /* 1868 * We are overloading satapkt_hba_driver_private with 1869 * watched_cycle count. 1870 */ 1871 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1872 1873 #if AHCI_DEBUG 1874 if (ahci_debug_flags & AHCIDBG_ATACMD && 1875 scmd->satacmd_cmd_reg != SATAC_PACKET || 1876 ahci_debug_flags & AHCIDBG_ATAPICMD && 1877 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1878 1879 /* Dump the command header and table */ 1880 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1881 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1882 "0x%p cmd_reg 0x%x port %d", spkt, 1883 scmd->satacmd_cmd_reg, port); 1884 ptr = (uint32_t *)cmd_header; 1885 ahci_log(ahci_ctlp, CE_WARN, 1886 " Command Header:%8x %8x %8x %8x", 1887 ptr[0], ptr[1], ptr[2], ptr[3]); 1888 1889 /* Dump the H2D register FIS */ 1890 ptr = (uint32_t *)h2d_register_fisp; 1891 ahci_log(ahci_ctlp, CE_WARN, 1892 " Command FIS: %8x %8x %8x %8x", 1893 ptr[0], ptr[1], ptr[2], ptr[3]); 1894 1895 /* Dump the ACMD register FIS */ 1896 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1897 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1898 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1899 ahci_log(ahci_ctlp, CE_WARN, 1900 " ATAPI command: %2x %2x %2x %2x " 1901 "%2x %2x %2x %2x", 1902 ptr2[8 * i], ptr2[8 * i + 1], 1903 ptr2[8 * i + 2], ptr2[8 * i + 3], 1904 ptr2[8 * i + 4], ptr2[8 * i + 5], 1905 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1906 1907 /* Dump the PRDT */ 1908 for (i = 0; i < ncookies; i++) { 1909 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1910 ahci_log(ahci_ctlp, CE_WARN, 1911 " Cookie %d: %8x %8x %8x %8x", 1912 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1913 } 1914 } 1915 #endif 1916 1917 (void) ddi_dma_sync( 1918 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1919 0, 1920 ahci_cmd_table_size, 1921 DDI_DMA_SYNC_FORDEV); 1922 1923 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1924 cmd_slot * sizeof (ahci_cmd_header_t), 1925 sizeof (ahci_cmd_header_t), 1926 DDI_DMA_SYNC_FORDEV); 1927 1928 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1929 if (command_type == AHCI_NCQ_CMD) { 1930 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1931 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1932 (0x1 << cmd_slot)); 1933 } 1934 1935 /* Indicate to the HBA that a command is active. */ 1936 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1937 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1938 (0x1 << cmd_slot)); 1939 1940 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1941 "exit: port %d", port); 1942 1943 return (cmd_slot); 1944 } 1945 1946 /* 1947 * Called by the sata framework to abort the previously sent packet(s). 1948 * 1949 * Reset device to abort commands. 1950 */ 1951 static int 1952 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1953 { 1954 ahci_ctl_t *ahci_ctlp; 1955 ahci_port_t *ahci_portp; 1956 uint32_t slot_status = 0; 1957 uint32_t aborted_tags = 0; 1958 uint32_t finished_tags = 0; 1959 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1960 uint8_t port; 1961 int tmp_slot; 1962 int instance = ddi_get_instance(dip); 1963 1964 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1965 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1966 1967 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 1968 "ahci_tran_abort enter: port %d", port); 1969 1970 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1971 mutex_enter(&ahci_portp->ahciport_mutex); 1972 1973 /* 1974 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 1975 * commands are being mopped, therefore there is nothing else to do 1976 */ 1977 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1978 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1979 "ahci_tran_abort: port %d is in " 1980 "mopping process, so just return directly ", port); 1981 mutex_exit(&ahci_portp->ahciport_mutex); 1982 return (SATA_SUCCESS); 1983 } 1984 1985 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1986 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1987 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1988 /* 1989 * In case the targer driver would send the request before 1990 * sata framework can have the opportunity to process those 1991 * event reports. 1992 */ 1993 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1994 spkt->satapkt_device.satadev_state = 1995 ahci_portp->ahciport_port_state; 1996 ahci_update_sata_registers(ahci_ctlp, port, 1997 &spkt->satapkt_device); 1998 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1999 "ahci_tran_abort returning SATA_FAILURE while " 2000 "port in FAILED/SHUTDOWN/PWROFF state: " 2001 "port: %d", port); 2002 mutex_exit(&ahci_portp->ahciport_mutex); 2003 return (SATA_FAILURE); 2004 } 2005 2006 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2007 /* 2008 * ahci_intr_phyrdy_change() may have rendered it to 2009 * AHCI_PORT_TYPE_NODEV. 2010 */ 2011 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2012 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2013 spkt->satapkt_device.satadev_state = 2014 ahci_portp->ahciport_port_state; 2015 ahci_update_sata_registers(ahci_ctlp, port, 2016 &spkt->satapkt_device); 2017 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2018 "ahci_tran_abort returning SATA_FAILURE while " 2019 "no device attached: port: %d", port); 2020 mutex_exit(&ahci_portp->ahciport_mutex); 2021 return (SATA_FAILURE); 2022 } 2023 2024 if (flag == SATA_ABORT_ALL_PACKETS) { 2025 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2026 aborted_tags = ahci_portp->ahciport_pending_tags; 2027 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2028 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2029 2030 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2031 instance, port); 2032 } else { 2033 aborted_tags = 0xffffffff; 2034 /* 2035 * Aborting one specific packet, first search the 2036 * ahciport_slot_pkts[] list for matching spkt. 2037 */ 2038 for (tmp_slot = 0; 2039 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2040 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2041 aborted_tags = (0x1 << tmp_slot); 2042 break; 2043 } 2044 } 2045 2046 if (aborted_tags == 0xffffffff) { 2047 /* request packet is not on the pending list */ 2048 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 2049 "Cannot find the aborting pkt 0x%p on the " 2050 "pending list", (void *)spkt); 2051 ahci_update_sata_registers(ahci_ctlp, port, 2052 &spkt->satapkt_device); 2053 mutex_exit(&ahci_portp->ahciport_mutex); 2054 return (SATA_FAILURE); 2055 } 2056 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2057 instance, port, (void *)spkt); 2058 } 2059 2060 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2061 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2062 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2063 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2064 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2065 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2066 2067 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2068 ahci_portp->ahciport_mop_in_progress++; 2069 2070 /* 2071 * To abort the packet(s), first we are trying to clear PxCMD.ST 2072 * to stop the port, and if the port can be stopped 2073 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2074 * then we just send back the aborted packet(s) with ABORTED flag 2075 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2076 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2077 * perform a COMRESET. 2078 */ 2079 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2080 ahci_portp, port, NULL, NULL); 2081 2082 /* 2083 * Compute which have finished and which need to be retried. 2084 * 2085 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2086 * minus the slot_status. The aborted_tags has to be deducted by 2087 * finished_tags since we can't possibly abort a tag which had finished 2088 * already. 2089 */ 2090 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2091 finished_tags = ahci_portp->ahciport_pending_tags & 2092 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2093 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2094 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2095 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2096 2097 aborted_tags &= ~finished_tags; 2098 2099 ahci_mop_commands(ahci_ctlp, 2100 ahci_portp, 2101 slot_status, 2102 0, /* failed tags */ 2103 0, /* timeout tags */ 2104 aborted_tags, 2105 0); /* reset tags */ 2106 2107 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2108 mutex_exit(&ahci_portp->ahciport_mutex); 2109 2110 return (SATA_SUCCESS); 2111 } 2112 2113 /* 2114 * Used to do device reset and reject all the pending packets on a device 2115 * during the reset operation. 2116 * 2117 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2118 */ 2119 static int 2120 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2121 ahci_port_t *ahci_portp, uint8_t port) 2122 { 2123 uint32_t slot_status = 0; 2124 uint32_t reset_tags = 0; 2125 uint32_t finished_tags = 0; 2126 sata_device_t sdevice; 2127 int ret; 2128 2129 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2130 "ahci_reset_device_reject_pkts on port: %d", port); 2131 2132 /* 2133 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2134 * commands are being mopped, therefore there is nothing else to do 2135 */ 2136 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2137 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2138 "ahci_reset_device_reject_pkts: port %d is in " 2139 "mopping process, so return directly ", port); 2140 return (SATA_SUCCESS); 2141 } 2142 2143 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2144 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2145 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2146 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2147 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2148 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2149 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2150 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2151 } 2152 2153 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2154 != AHCI_SUCCESS) { 2155 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2156 "Try to do a port reset after software " 2157 "reset failed", port); 2158 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2159 if (ret != AHCI_SUCCESS) { 2160 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2161 "ahci_reset_device_reject_pkts: port %d " 2162 "failed", port); 2163 return (SATA_FAILURE); 2164 } 2165 } 2166 /* Set the reset in progress flag */ 2167 ahci_portp->ahciport_reset_in_progress = 1; 2168 2169 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2170 ahci_portp->ahciport_mop_in_progress++; 2171 2172 /* Indicate to the framework that a reset has happened */ 2173 bzero((void *)&sdevice, sizeof (sata_device_t)); 2174 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2175 sdevice.satadev_addr.pmport = 0; 2176 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2177 2178 sdevice.satadev_state = SATA_DSTATE_RESET | 2179 SATA_DSTATE_PWR_ACTIVE; 2180 mutex_exit(&ahci_portp->ahciport_mutex); 2181 sata_hba_event_notify( 2182 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2183 &sdevice, 2184 SATA_EVNT_DEVICE_RESET); 2185 mutex_enter(&ahci_portp->ahciport_mutex); 2186 2187 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 2188 "port %d sending event up: SATA_EVNT_RESET", port); 2189 2190 /* Next try to mop the pending commands */ 2191 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2192 finished_tags = ahci_portp->ahciport_pending_tags & 2193 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2194 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2195 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2196 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2197 2198 reset_tags &= ~finished_tags; 2199 2200 ahci_mop_commands(ahci_ctlp, 2201 ahci_portp, 2202 slot_status, 2203 0, /* failed tags */ 2204 0, /* timeout tags */ 2205 0, /* aborted tags */ 2206 reset_tags); /* reset tags */ 2207 2208 return (SATA_SUCCESS); 2209 } 2210 2211 /* 2212 * Used to do port reset and reject all the pending packets on a port during 2213 * the reset operation. 2214 * 2215 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2216 */ 2217 static int 2218 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2219 ahci_port_t *ahci_portp, uint8_t port) 2220 { 2221 uint32_t slot_status = 0; 2222 uint32_t reset_tags = 0; 2223 uint32_t finished_tags = 0; 2224 2225 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2226 "ahci_reset_port_reject_pkts on port: %d", port); 2227 2228 /* 2229 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2230 * commands are being mopped, therefore there is nothing else to do 2231 */ 2232 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2233 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2234 "ahci_reset_port_reject_pkts: port %d is in " 2235 "mopping process, so return directly ", port); 2236 return (SATA_SUCCESS); 2237 } 2238 2239 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2240 ahci_portp->ahciport_mop_in_progress++; 2241 2242 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2243 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2244 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2245 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2246 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2247 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2248 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2249 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2250 } 2251 2252 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2253 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2254 NULL) != AHCI_SUCCESS) 2255 return (SATA_FAILURE); 2256 2257 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2258 finished_tags = ahci_portp->ahciport_pending_tags & 2259 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2260 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2261 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2262 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2263 2264 reset_tags &= ~finished_tags; 2265 2266 ahci_mop_commands(ahci_ctlp, 2267 ahci_portp, 2268 slot_status, 2269 0, /* failed tags */ 2270 0, /* timeout tags */ 2271 0, /* aborted tags */ 2272 reset_tags); /* reset tags */ 2273 2274 return (SATA_SUCCESS); 2275 } 2276 2277 /* 2278 * Used to do hba reset and reject all the pending packets on all ports 2279 * during the reset operation. 2280 */ 2281 static int 2282 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2283 { 2284 ahci_port_t *ahci_portp; 2285 uint32_t slot_status[AHCI_MAX_PORTS]; 2286 uint32_t reset_tags[AHCI_MAX_PORTS]; 2287 uint32_t finished_tags[AHCI_MAX_PORTS]; 2288 uint8_t port; 2289 int ret = SATA_SUCCESS; 2290 2291 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 2292 "ahci_reset_hba_reject_pkts enter"); 2293 2294 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2295 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2296 continue; 2297 } 2298 2299 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2300 2301 mutex_enter(&ahci_portp->ahciport_mutex); 2302 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2303 slot_status[port] = ddi_get32( 2304 ahci_ctlp->ahcictl_ahci_acc_handle, 2305 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2306 reset_tags[port] = slot_status[port] & 2307 AHCI_SLOT_MASK(ahci_ctlp); 2308 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2309 slot_status[port] = ddi_get32( 2310 ahci_ctlp->ahcictl_ahci_acc_handle, 2311 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2312 reset_tags[port] = slot_status[port] & 2313 AHCI_NCQ_SLOT_MASK(ahci_portp); 2314 } 2315 mutex_exit(&ahci_portp->ahciport_mutex); 2316 } 2317 2318 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2319 ret = SATA_FAILURE; 2320 goto out; 2321 } 2322 2323 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2324 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2325 continue; 2326 } 2327 2328 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2329 2330 mutex_enter(&ahci_portp->ahciport_mutex); 2331 /* 2332 * To prevent recursive enter to ahci_mop_commands, we need 2333 * check AHCI_PORT_FLAG_MOPPING flag. 2334 */ 2335 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2336 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2337 "ahci_reset_hba_reject_pkts: port %d is in " 2338 "mopping process, so return directly ", port); 2339 mutex_exit(&ahci_portp->ahciport_mutex); 2340 continue; 2341 } 2342 2343 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2344 ahci_portp->ahciport_mop_in_progress++; 2345 2346 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2347 finished_tags[port] = 2348 ahci_portp->ahciport_pending_tags & 2349 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2350 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2351 finished_tags[port] = 2352 ahci_portp->ahciport_pending_ncq_tags & 2353 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2354 2355 reset_tags[port] &= ~finished_tags[port]; 2356 2357 ahci_mop_commands(ahci_ctlp, 2358 ahci_portp, 2359 slot_status[port], 2360 0, /* failed tags */ 2361 0, /* timeout tags */ 2362 0, /* aborted tags */ 2363 reset_tags[port]); /* reset tags */ 2364 mutex_exit(&ahci_portp->ahciport_mutex); 2365 } 2366 out: 2367 return (ret); 2368 } 2369 2370 /* 2371 * Called by sata framework to reset a port(s) or device. 2372 */ 2373 static int 2374 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2375 { 2376 ahci_ctl_t *ahci_ctlp; 2377 ahci_port_t *ahci_portp; 2378 uint8_t cport = sd->satadev_addr.cport; 2379 uint8_t port; 2380 int ret = SATA_SUCCESS; 2381 int instance = ddi_get_instance(dip); 2382 2383 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2384 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2385 2386 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2387 "ahci_tran_reset_port enter: cport: %d", cport); 2388 2389 switch (sd->satadev_addr.qual) { 2390 case SATA_ADDR_CPORT: 2391 /* Port reset */ 2392 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2393 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2394 "reset port", instance, port); 2395 2396 mutex_enter(&ahci_portp->ahciport_mutex); 2397 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2398 mutex_exit(&ahci_portp->ahciport_mutex); 2399 2400 break; 2401 2402 case SATA_ADDR_DCPORT: 2403 /* Device reset */ 2404 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2405 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2406 "reset device", instance, port); 2407 2408 mutex_enter(&ahci_portp->ahciport_mutex); 2409 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2410 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2411 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2412 /* 2413 * In case the targer driver would send the request 2414 * before sata framework can have the opportunity to 2415 * process those event reports. 2416 */ 2417 sd->satadev_state = ahci_portp->ahciport_port_state; 2418 ahci_update_sata_registers(ahci_ctlp, port, sd); 2419 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2420 "ahci_tran_reset_dport returning SATA_FAILURE " 2421 "while port in FAILED/SHUTDOWN/PWROFF state: " 2422 "port: %d", port); 2423 mutex_exit(&ahci_portp->ahciport_mutex); 2424 ret = SATA_FAILURE; 2425 break; 2426 } 2427 2428 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2429 /* 2430 * ahci_intr_phyrdy_change() may have rendered it to 2431 * AHCI_PORT_TYPE_NODEV. 2432 */ 2433 sd->satadev_type = SATA_DTYPE_NONE; 2434 sd->satadev_state = ahci_portp->ahciport_port_state; 2435 ahci_update_sata_registers(ahci_ctlp, port, sd); 2436 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2437 "ahci_tran_reset_dport returning SATA_FAILURE " 2438 "while no device attached: port: %d", port); 2439 mutex_exit(&ahci_portp->ahciport_mutex); 2440 ret = SATA_FAILURE; 2441 break; 2442 } 2443 2444 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2445 ahci_portp, port); 2446 mutex_exit(&ahci_portp->ahciport_mutex); 2447 break; 2448 2449 case SATA_ADDR_CNTRL: 2450 /* Reset the whole controller */ 2451 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2452 "reset the whole hba", instance, port); 2453 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2454 break; 2455 2456 case SATA_ADDR_PMPORT: 2457 case SATA_ADDR_DPMPORT: 2458 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 2459 "ahci_tran_reset_dport: port multiplier will be " 2460 "supported later"); 2461 /* FALLTHRU */ 2462 default: 2463 ret = SATA_FAILURE; 2464 } 2465 2466 return (ret); 2467 } 2468 2469 /* 2470 * Called by sata framework to activate a port as part of hotplug. 2471 * (cfgadm -c connect satax/y) 2472 * Note: Not port-mult aware. 2473 */ 2474 static int 2475 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2476 { 2477 ahci_ctl_t *ahci_ctlp; 2478 ahci_port_t *ahci_portp; 2479 uint8_t cport = satadev->satadev_addr.cport; 2480 uint8_t port; 2481 int instance = ddi_get_instance(dip); 2482 2483 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2484 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2485 2486 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2487 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2488 2489 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2490 2491 mutex_enter(&ahci_portp->ahciport_mutex); 2492 ahci_enable_port_intrs(ahci_ctlp, port); 2493 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2494 2495 /* 2496 * Reset the port so that the PHY communication would be re-established. 2497 * But this reset is an internal operation and the sata module doesn't 2498 * need to know about it. Moreover, the port with a device attached will 2499 * be started too. 2500 */ 2501 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2502 ahci_portp, port, 2503 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2504 NULL); 2505 2506 /* 2507 * Need to check the link status and device status of the port 2508 * and consider raising power if the port was in D3 state 2509 */ 2510 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2511 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2512 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2513 2514 satadev->satadev_state = ahci_portp->ahciport_port_state; 2515 2516 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2517 2518 mutex_exit(&ahci_portp->ahciport_mutex); 2519 return (SATA_SUCCESS); 2520 } 2521 2522 /* 2523 * Called by sata framework to deactivate a port as part of hotplug. 2524 * (cfgadm -c disconnect satax/y) 2525 * Note: Not port-mult aware. 2526 */ 2527 static int 2528 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2529 { 2530 ahci_ctl_t *ahci_ctlp; 2531 ahci_port_t *ahci_portp; 2532 uint8_t cport = satadev->satadev_addr.cport; 2533 uint8_t port; 2534 uint32_t port_scontrol; 2535 int instance = ddi_get_instance(dip); 2536 2537 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2538 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2539 2540 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2541 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2542 2543 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2544 2545 mutex_enter(&ahci_portp->ahciport_mutex); 2546 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2547 instance, port); 2548 2549 /* Disable the interrupts on the port */ 2550 ahci_disable_port_intrs(ahci_ctlp, port); 2551 2552 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2553 goto phy_offline; 2554 } 2555 2556 /* First to abort all the pending commands */ 2557 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2558 2559 /* Then stop the port */ 2560 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2561 ahci_portp, port); 2562 2563 /* Next put the PHY offline */ 2564 2565 phy_offline: 2566 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2567 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2568 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2569 2570 /* Update ahciport_port_state */ 2571 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2572 satadev->satadev_state = ahci_portp->ahciport_port_state; 2573 2574 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2575 2576 mutex_exit(&ahci_portp->ahciport_mutex); 2577 return (SATA_SUCCESS); 2578 } 2579 2580 /* 2581 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2582 * when a device is unplugged or a port is deactivated. 2583 * 2584 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2585 */ 2586 static void 2587 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2588 ahci_port_t *ahci_portp, uint8_t port) 2589 { 2590 uint32_t slot_status = 0; 2591 uint32_t abort_tags = 0; 2592 2593 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2594 "ahci_reject_all_abort_pkts on port: %d", port); 2595 2596 /* 2597 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2598 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2599 * to get the error data, if yes when the device is removed, the 2600 * command needs to be aborted too. 2601 */ 2602 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2603 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2604 slot_status = 0x1; 2605 abort_tags = 0x1; 2606 goto out; 2607 } else { 2608 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2609 "ahci_reject_all_abort_pkts return directly " 2610 "port %d no needs to reject any outstanding " 2611 "commands", port); 2612 return; 2613 } 2614 } 2615 2616 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2617 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2618 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2619 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2620 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2621 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2622 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2623 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2624 } 2625 2626 out: 2627 /* No need to do mop when there is no outstanding commands */ 2628 if (slot_status != 0) { 2629 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2630 ahci_portp->ahciport_mop_in_progress++; 2631 2632 ahci_mop_commands(ahci_ctlp, 2633 ahci_portp, 2634 slot_status, 2635 0, /* failed tags */ 2636 0, /* timeout tags */ 2637 abort_tags, /* aborting tags */ 2638 0); /* reset tags */ 2639 } 2640 } 2641 2642 #if defined(__lock_lint) 2643 static int 2644 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2645 { 2646 return (SATA_SUCCESS); 2647 } 2648 #endif 2649 2650 /* 2651 * Allocate the ports structure, only called by ahci_attach 2652 */ 2653 static int 2654 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2655 { 2656 int port, cport = 0; 2657 2658 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2659 "ahci_alloc_ports_state enter"); 2660 2661 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2662 2663 /* Allocate structures only for the implemented ports */ 2664 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2665 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2666 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2667 "hba port %d not implemented", port); 2668 continue; 2669 } 2670 2671 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2672 ahci_ctlp->ahcictl_port_to_cport[port] = 2673 (uint8_t)cport++; 2674 2675 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2676 goto err_out; 2677 } 2678 } 2679 2680 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2681 return (AHCI_SUCCESS); 2682 2683 err_out: 2684 for (port--; port >= 0; port--) { 2685 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2686 ahci_dealloc_port_state(ahci_ctlp, port); 2687 } 2688 } 2689 2690 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2691 return (AHCI_FAILURE); 2692 } 2693 2694 /* 2695 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2696 */ 2697 static void 2698 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2699 { 2700 int port; 2701 2702 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2703 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2704 /* if this port is implemented by the HBA */ 2705 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2706 ahci_dealloc_port_state(ahci_ctlp, port); 2707 } 2708 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2709 } 2710 2711 /* 2712 * Initialize the controller and all ports. And then try to start the ports 2713 * if there are devices attached. 2714 * 2715 * This routine can be called from three seperate cases: DDI_ATTACH, 2716 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2717 * other two cases; device signature probing are attempted only during 2718 * DDI_ATTACH case. 2719 * 2720 * WARNING!!! Disable the whole controller's interrupts before calling and 2721 * the interrupts will be enabled upon successfully return. 2722 */ 2723 static int 2724 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2725 { 2726 ahci_port_t *ahci_portp; 2727 uint32_t ghc_control; 2728 int port; 2729 2730 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2731 "ahci_initialize_controller enter"); 2732 2733 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2734 2735 /* 2736 * Indicate that system software is AHCI aware by setting 2737 * GHC.AE to 1 2738 */ 2739 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2740 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2741 2742 ghc_control |= AHCI_HBA_GHC_AE; 2743 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2744 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2745 ghc_control); 2746 2747 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2748 2749 /* Initialize the implemented ports and structures */ 2750 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2751 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2752 continue; 2753 } 2754 2755 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2756 mutex_enter(&ahci_portp->ahciport_mutex); 2757 2758 /* 2759 * Ensure that the controller is not in the running state 2760 * by checking every implemented port's PxCMD register 2761 */ 2762 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2763 != AHCI_SUCCESS) { 2764 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2765 "ahci_initialize_controller: failed to " 2766 "initialize port %d", port); 2767 /* 2768 * Set the port state to SATA_PSTATE_FAILED if 2769 * failed to initialize it. 2770 */ 2771 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2772 } 2773 2774 mutex_exit(&ahci_portp->ahciport_mutex); 2775 } 2776 2777 /* Enable the whole controller interrupts */ 2778 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2779 ahci_enable_all_intrs(ahci_ctlp); 2780 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2781 2782 return (AHCI_SUCCESS); 2783 } 2784 2785 /* 2786 * Reverse of ahci_initialize_controller() 2787 * 2788 * We only need to stop the ports and disable the interrupt. 2789 */ 2790 static void 2791 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2792 { 2793 ahci_port_t *ahci_portp; 2794 int port; 2795 2796 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2797 "ahci_uninitialize_controller enter"); 2798 2799 /* disable all the interrupts. */ 2800 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2801 ahci_disable_all_intrs(ahci_ctlp); 2802 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2803 2804 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2805 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2806 continue; 2807 } 2808 2809 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2810 2811 /* Stop the port by clearing PxCMD.ST */ 2812 mutex_enter(&ahci_portp->ahciport_mutex); 2813 2814 /* 2815 * Here we must disable the port interrupt because 2816 * ahci_disable_all_intrs only clear GHC.IE, and IS 2817 * register will be still set if PxIE is enabled. 2818 * When ahci shares one IRQ with other drivers, the 2819 * intr handler may claim the intr mistakenly. 2820 */ 2821 ahci_disable_port_intrs(ahci_ctlp, port); 2822 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2823 ahci_portp, port); 2824 mutex_exit(&ahci_portp->ahciport_mutex); 2825 } 2826 } 2827 2828 /* 2829 * The routine is to initialize the port. First put the port in NOTRunning 2830 * state, then enable port interrupt and clear Serror register. And under 2831 * AHCI_ATTACH case, find device signature and then try to start the port. 2832 * 2833 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2834 */ 2835 static int 2836 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2837 ahci_port_t *ahci_portp, uint8_t port) 2838 { 2839 uint32_t port_cmd_status; 2840 2841 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2842 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2843 2844 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2845 "ahci_initialize_port: port %d " 2846 "port_cmd_status = 0x%x", port, port_cmd_status); 2847 /* 2848 * Check whether the port is in NotRunning state, if not, 2849 * put the port in NotRunning state 2850 */ 2851 if (!(port_cmd_status & 2852 (AHCI_CMD_STATUS_ST | 2853 AHCI_CMD_STATUS_CR | 2854 AHCI_CMD_STATUS_FRE | 2855 AHCI_CMD_STATUS_FR))) { 2856 2857 goto next; 2858 } 2859 2860 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 2861 port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT, NULL) != AHCI_SUCCESS) 2862 return (AHCI_FAILURE); 2863 2864 next: 2865 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2866 "port %d is in NotRunning state now", port); 2867 2868 /* 2869 * At the time being, only probe ports/devices and get the types of 2870 * attached devices during DDI_ATTACH. In fact, the device can be 2871 * changed during power state changes, but at the time being, we 2872 * don't support the situation. 2873 */ 2874 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2875 /* Try to get the device signature */ 2876 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2877 } else { 2878 2879 /* 2880 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2881 * and PxFBU registers in case these registers were cleared 2882 * during the suspend. 2883 */ 2884 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2885 "ahci_initialize_port: port %d " 2886 "reset the port during resume", port); 2887 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2888 2889 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2890 "ahci_initialize_port: port %d " 2891 "set PxCLB, PxCLBU, PxFB and PxFBU " 2892 "during resume", port); 2893 2894 /* Config Port Received FIS Base Address */ 2895 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2896 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2897 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2898 2899 /* Config Port Command List Base Address */ 2900 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2901 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2902 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2903 } 2904 2905 /* Disable the interface power management */ 2906 ahci_disable_interface_pm(ahci_ctlp, port); 2907 2908 /* Return directly if no device connected */ 2909 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2910 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2911 "No device connected to port %d", port); 2912 goto out; 2913 } 2914 2915 /* Try to start the port */ 2916 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 2917 != AHCI_SUCCESS) { 2918 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2919 "failed to start port %d", port); 2920 return (AHCI_FAILURE); 2921 } 2922 out: 2923 /* Enable port interrupts */ 2924 ahci_enable_port_intrs(ahci_ctlp, port); 2925 2926 return (AHCI_SUCCESS); 2927 } 2928 2929 /* 2930 * Check the hardware defects and the power management capability. 2931 */ 2932 static int 2933 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 2934 { 2935 ushort_t venid, devid; 2936 ushort_t caps_ptr, cap_count, cap; 2937 #if AHCI_DEBUG 2938 ushort_t pmcap, pmcsr; 2939 #endif 2940 uint8_t revision; 2941 2942 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2943 PCI_CONF_VENID); 2944 2945 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2946 PCI_CONF_DEVID); 2947 2948 /* 2949 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 2950 * controllers with 0x00 revision id work on 4-byte aligned buffer, 2951 * which is a bug and was fixed after 0x00 revision id controllers. 2952 * 2953 * Moreover, VT8251 cannot use multiple command slots in the command 2954 * list for non-queued commands because the previous register content 2955 * of PxCI can be re-written in the register write, so a flag will be 2956 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 2957 */ 2958 if (venid == VIA_VENID) { 2959 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 2960 PCI_CONF_REVID); 2961 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2962 "revision id = 0x%x", revision); 2963 if (revision == 0x00) { 2964 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 2965 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2966 "change ddi_attr_align to 0x4"); 2967 } 2968 2969 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 2970 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2971 "VT8251 cannot use multiple command lists for " 2972 "non-queued commands"); 2973 } 2974 2975 /* 2976 * ATI SB600 (1002,4380) and SB700/750 (1002,4391) AHCI chipsets don't 2977 * support 64-bit DMA addressing though they declare the support, 2978 * so we need to set AHCI_CAP_32BIT_DMA flag to force 32-bit DMA. 2979 */ 2980 if (venid == 0x1002 && (devid == 0x4380 || devid == 0x4391)) { 2981 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2982 "ATI SB600/700/750 cannot do 64-bit DMA though CAP " 2983 "indicates support, so force it to use 32-bit DMA"); 2984 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 2985 } 2986 2987 /* 2988 * Check if capabilities list is supported and if so, 2989 * get initial capabilities pointer and clear bits 0,1. 2990 */ 2991 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2992 PCI_CONF_STAT) & PCI_STAT_CAP) { 2993 caps_ptr = P2ALIGN(pci_config_get8( 2994 ahci_ctlp->ahcictl_pci_conf_handle, 2995 PCI_CONF_CAP_PTR), 4); 2996 } else { 2997 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 2998 } 2999 3000 /* 3001 * Walk capabilities if supported. 3002 */ 3003 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3004 3005 /* 3006 * Check that we haven't exceeded the maximum number of 3007 * capabilities and that the pointer is in a valid range. 3008 */ 3009 if (++cap_count > PCI_CAP_MAX_PTR) { 3010 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3011 "too many device capabilities"); 3012 return (AHCI_FAILURE); 3013 } 3014 if (caps_ptr < PCI_CAP_PTR_OFF) { 3015 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3016 "capabilities pointer 0x%x out of range", 3017 caps_ptr); 3018 return (AHCI_FAILURE); 3019 } 3020 3021 /* 3022 * Get next capability and check that it is valid. 3023 * For now, we only support power management. 3024 */ 3025 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3026 caps_ptr); 3027 switch (cap) { 3028 case PCI_CAP_ID_PM: 3029 3030 /* power management supported */ 3031 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3032 3033 /* Save PMCSR offset */ 3034 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3035 3036 #if AHCI_DEBUG 3037 pmcap = pci_config_get16( 3038 ahci_ctlp->ahcictl_pci_conf_handle, 3039 caps_ptr + PCI_PMCAP); 3040 pmcsr = pci_config_get16( 3041 ahci_ctlp->ahcictl_pci_conf_handle, 3042 ahci_ctlp->ahcictl_pmcsr_offset); 3043 AHCIDBG2(AHCIDBG_PM, ahci_ctlp, 3044 "Power Management capability found PCI_PMCAP " 3045 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3046 if ((pmcap & 0x3) == 0x3) 3047 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3048 "PCI Power Management Interface " 3049 "spec 1.2 compliant"); 3050 #endif 3051 break; 3052 3053 case PCI_CAP_ID_MSI: 3054 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, "MSI capability found"); 3055 break; 3056 3057 case PCI_CAP_ID_PCIX: 3058 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3059 "PCI-X capability found"); 3060 break; 3061 3062 case PCI_CAP_ID_PCI_E: 3063 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3064 "PCI Express capability found"); 3065 break; 3066 3067 case PCI_CAP_ID_MSI_X: 3068 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3069 "MSI-X capability found"); 3070 break; 3071 3072 case PCI_CAP_ID_SATA: 3073 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3074 "SATA capability found"); 3075 break; 3076 3077 case PCI_CAP_ID_VS: 3078 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3079 "Vendor Specific capability found"); 3080 break; 3081 3082 default: 3083 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 3084 "unrecognized capability 0x%x", cap); 3085 break; 3086 } 3087 3088 /* 3089 * Get next capabilities pointer and clear bits 0,1. 3090 */ 3091 caps_ptr = P2ALIGN(pci_config_get8( 3092 ahci_ctlp->ahcictl_pci_conf_handle, 3093 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3094 } 3095 3096 return (AHCI_SUCCESS); 3097 } 3098 3099 /* 3100 * AHCI device reset ...; a single device on one of the ports is reset, 3101 * but the HBA and physical communication remain intact. This is the 3102 * least intrusive. 3103 * 3104 * When issuing a software reset sequence, there should not be other 3105 * commands in the command list, so we will first clear and then re-set 3106 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3107 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3108 * cleared unless command list override (PxCMD.CLO) is supported. 3109 * 3110 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3111 * set before the function is called. 3112 */ 3113 static int 3114 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3115 uint8_t port) 3116 { 3117 ahci_fis_h2d_register_t *h2d_register_fisp; 3118 ahci_cmd_table_t *cmd_table; 3119 ahci_cmd_header_t *cmd_header; 3120 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3121 int slot, loop_count; 3122 3123 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 3124 "Port %d device resetting", port); 3125 3126 /* First clear PxCMD.ST */ 3127 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3128 ahci_portp, port); 3129 3130 /* Then re-set PxCMD.ST */ 3131 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 3132 3133 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3134 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3135 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3136 3137 if (port_task_file & AHCI_TFD_STS_BSY || 3138 port_task_file & AHCI_TFD_STS_DRQ) { 3139 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3140 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3141 if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) { 3142 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3143 "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, " 3144 "but PxCMD.CLO isn't supported, so a port " 3145 "reset is needed."); 3146 return (AHCI_FAILURE); 3147 } 3148 } 3149 3150 slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, AHCI_NON_NCQ_CMD); 3151 if (slot == AHCI_FAILURE) { 3152 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 3153 "ahci_software_reset: no free slot"); 3154 return (AHCI_FAILURE); 3155 } 3156 3157 /* Now send the first H2D Register FIS with SRST set to 1 */ 3158 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3159 bzero((void *)cmd_table, ahci_cmd_table_size); 3160 3161 h2d_register_fisp = 3162 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3163 3164 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3165 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3166 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3167 3168 /* Set Command Header in Command List */ 3169 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3170 BZERO_DESCR_INFO(cmd_header); 3171 BZERO_PRD_BYTE_COUNT(cmd_header); 3172 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3173 3174 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3175 SET_RESET(cmd_header, 1); 3176 SET_WRITE(cmd_header, 1); 3177 3178 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3179 0, 3180 ahci_cmd_table_size, 3181 DDI_DMA_SYNC_FORDEV); 3182 3183 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3184 slot * sizeof (ahci_cmd_header_t), 3185 sizeof (ahci_cmd_header_t), 3186 DDI_DMA_SYNC_FORDEV); 3187 3188 /* Indicate to the HBA that a command is active. */ 3189 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3190 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3191 (0x1 << slot)); 3192 3193 loop_count = 0; 3194 3195 /* Loop till the first command is finished */ 3196 do { 3197 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3198 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3199 3200 /* We are effectively timing out after 1 sec. */ 3201 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3202 break; 3203 } 3204 /* Wait for 10 millisec */ 3205 delay(AHCI_10MS_TICKS); 3206 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3207 3208 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3209 "ahci_software_reset: 1st loop count: %d, " 3210 "port_cmd_issue = 0x%x, slot = 0x%x", 3211 loop_count, port_cmd_issue, slot); 3212 3213 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3214 3215 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3216 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3217 bzero((void *)cmd_table, ahci_cmd_table_size); 3218 3219 h2d_register_fisp = 3220 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3221 3222 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3223 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3224 3225 /* Set Command Header in Command List */ 3226 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3227 BZERO_DESCR_INFO(cmd_header); 3228 BZERO_PRD_BYTE_COUNT(cmd_header); 3229 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3230 3231 SET_WRITE(cmd_header, 1); 3232 3233 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3234 0, 3235 ahci_cmd_table_size, 3236 DDI_DMA_SYNC_FORDEV); 3237 3238 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3239 slot * sizeof (ahci_cmd_header_t), 3240 sizeof (ahci_cmd_header_t), 3241 DDI_DMA_SYNC_FORDEV); 3242 3243 /* Indicate to the HBA that a command is active. */ 3244 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3245 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3246 (0x1 << slot)); 3247 3248 loop_count = 0; 3249 3250 /* Loop till the second command is finished */ 3251 do { 3252 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3253 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3254 3255 /* We are effectively timing out after 1 sec. */ 3256 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3257 break; 3258 } 3259 /* Wait for 10 millisec */ 3260 delay(AHCI_10MS_TICKS); 3261 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3262 3263 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3264 "ahci_software_reset: 2nd loop count: %d, " 3265 "port_cmd_issue = 0x%x, slot = 0x%x", 3266 loop_count, port_cmd_issue, slot); 3267 3268 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3269 3270 return (AHCI_SUCCESS); 3271 } 3272 3273 /* 3274 * AHCI port reset ...; the physical communication between the HBA and device 3275 * on a port are disabled. This is more intrusive. 3276 * 3277 * When an HBA or port reset occurs, Phy communication is going to 3278 * be re-established with the device through a COMRESET followed by the 3279 * normal out-of-band communication sequence defined in Serial ATA. AT 3280 * the end of reset, the device, if working properly, will send a D2H 3281 * Register FIS, which contains the device signature. When the HBA receives 3282 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3283 * the PxSIG register with the signature. 3284 * 3285 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3286 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3287 * AHCI 1.0 spec. 3288 */ 3289 /* 3290 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3291 * cleared before the function is called. 3292 */ 3293 static int 3294 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3295 { 3296 uint32_t cap_status, port_cmd_status; 3297 uint32_t port_scontrol, port_sstatus; 3298 uint32_t port_intr_status, port_task_file; 3299 #if AHCI_DEBUG 3300 uint32_t port_signature; 3301 #endif 3302 int loop_count; 3303 int rval = AHCI_SUCCESS; 3304 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3305 3306 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3307 "Port %d port resetting...", port); 3308 ahci_portp->ahciport_port_state = 0; 3309 3310 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3311 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3312 3313 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3314 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3315 3316 if (cap_status & AHCI_HBA_CAP_SSS) { 3317 /* 3318 * HBA support staggered spin-up, if the port has 3319 * not spin up yet, then force it to do spin-up 3320 */ 3321 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3322 if (!(ahci_portp->ahciport_flags 3323 & AHCI_PORT_FLAG_SPINUP)) { 3324 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3325 "Port %d PxCMD.SUD is zero, force " 3326 "it to do spin-up", port); 3327 ahci_portp->ahciport_flags |= 3328 AHCI_PORT_FLAG_SPINUP; 3329 } 3330 } 3331 } else { 3332 /* 3333 * HBA doesn't support stagger spin-up, force it 3334 * to do normal COMRESET 3335 */ 3336 if (ahci_portp->ahciport_flags & 3337 AHCI_PORT_FLAG_SPINUP) { 3338 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3339 "HBA does not support staggered spin-up " 3340 "force it to do normal COMRESET"); 3341 ahci_portp->ahciport_flags &= 3342 ~AHCI_PORT_FLAG_SPINUP; 3343 } 3344 } 3345 3346 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3347 /* Do normal COMRESET */ 3348 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3349 "ahci_port_reset: do normal COMRESET", port); 3350 3351 /* 3352 * According to the spec, SUD bit should be set here, 3353 * but JMicron JMB363 doesn't follow it, so remove 3354 * the assertion, and just print a debug message. 3355 */ 3356 #if AHCI_DEBUG 3357 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3358 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3359 "port %d SUD bit not set", port) 3360 #endif 3361 3362 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3363 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3364 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3365 3366 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3367 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3368 port_scontrol); 3369 3370 /* Enable PxCMD.FRE to read device */ 3371 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3372 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3373 port_cmd_status|AHCI_CMD_STATUS_FRE); 3374 3375 /* 3376 * Give time for COMRESET to percolate, according to the AHCI 3377 * spec, software shall wait at least 1 millisecond before 3378 * clearing PxSCTL.DET 3379 */ 3380 delay(AHCI_1MS_TICKS*2); 3381 3382 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3383 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3384 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3385 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3386 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3387 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3388 port_scontrol); 3389 } else { 3390 /* Do staggered spin-up */ 3391 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3392 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3393 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3394 3395 /* PxSCTL.DET must be 0 */ 3396 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3397 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3398 port_scontrol); 3399 3400 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3401 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3402 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3403 port_cmd_status); 3404 3405 /* 0 -> 1 edge */ 3406 delay(AHCI_1MS_TICKS*2); 3407 3408 /* Set PxCMD.SUD to 1 */ 3409 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3410 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3411 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3412 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3413 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3414 port_cmd_status); 3415 3416 /* Enable PxCMD.FRE to read device */ 3417 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3418 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3419 port_cmd_status|AHCI_CMD_STATUS_FRE); 3420 } 3421 3422 /* 3423 * The port enters P:StartComm state, and HBA tells link layer to 3424 * start communication, which involves sending COMRESET to device. 3425 * And the HBA resets PxTFD.STS to 7Fh. 3426 * 3427 * When a COMINIT is received from the device, then the port enters 3428 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3429 * PxSSTS.DET to 1h to indicate a device is detected but communication 3430 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3431 * a COMINIT has been received. 3432 */ 3433 /* 3434 * The DET field is valid only if IPM field indicates 3435 * that the interface is in active state. 3436 */ 3437 loop_count = 0; 3438 do { 3439 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3440 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3441 3442 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3443 /* 3444 * If the interface is not active, the DET field 3445 * is considered not accurate. So we want to 3446 * continue looping. 3447 */ 3448 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3449 } 3450 3451 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3452 /* 3453 * We are effectively timing out after 0.1 sec. 3454 */ 3455 break; 3456 } 3457 3458 /* Wait for 10 millisec */ 3459 delay(AHCI_10MS_TICKS); 3460 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3461 3462 AHCIDBG3(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3463 "ahci_port_reset: 1st loop count: %d, " 3464 "port_sstatus = 0x%x port %d", 3465 loop_count, port_sstatus, port); 3466 3467 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3468 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3469 /* 3470 * Either the port is not active or there 3471 * is no device present. 3472 */ 3473 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3474 goto out; 3475 } 3476 3477 /* Now we can make sure there is a device connected to the port */ 3478 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3479 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3480 3481 /* a COMINIT signal is supposed to be received */ 3482 if (!(port_intr_status & AHCI_INTR_STATUS_PCS)) { 3483 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3484 "COMINIT signal from the device not received", 3485 instance, port); 3486 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3487 rval = AHCI_FAILURE; 3488 goto out; 3489 } 3490 3491 /* 3492 * According to the spec, when PxSCTL.DET is set to 0h, upon 3493 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3494 * shall be set to '1' by the HBA. 3495 * 3496 * However, we found JMicron JMB363 doesn't follow this, so 3497 * remove this check, and just print a debug message. 3498 */ 3499 #if AHCI_DEBUG 3500 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3501 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3502 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3503 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3504 "port %d BSY bit is not set after COMINIT signal " 3505 "is received", port); 3506 } 3507 #endif 3508 3509 /* 3510 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3511 * the D2H FIS received by HBA. 3512 */ 3513 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3514 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3515 SERROR_EXCHANGED_ERR); 3516 3517 /* 3518 * Next check whether COMRESET is completed successfully 3519 */ 3520 loop_count = 0; 3521 do { 3522 port_task_file = 3523 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3524 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3525 3526 /* 3527 * The Error bit '1' means COMRESET is finished successfully 3528 * The device hardware has been initialized and the power-up 3529 * diagnostics successfully completed. 3530 */ 3531 if (((port_task_file & AHCI_TFD_ERR_MASK) 3532 >> AHCI_TFD_ERR_SHIFT) == 0x1) { 3533 #if AHCI_DEBUG 3534 port_signature = ddi_get32( 3535 ahci_ctlp->ahcictl_ahci_acc_handle, 3536 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3537 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 3538 "COMRESET success, D2H register FIS " 3539 "post to received FIS structure " 3540 "port %d signature = 0x%x", 3541 port, port_signature); 3542 #endif 3543 goto out_check; 3544 } 3545 3546 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3547 /* 3548 * We are effectively timing out after 11 sec. 3549 */ 3550 break; 3551 } 3552 3553 /* Wait for 10 millisec */ 3554 delay(AHCI_10MS_TICKS); 3555 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3556 >> AHCI_TFD_ERR_SHIFT) != 0x1); 3557 3558 AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: 2nd loop " 3559 "count: %d, port_task_file = 0x%x port %d", 3560 loop_count, port_task_file, port); 3561 3562 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d the device hardware " 3563 "has been initialized and the power-up diagnostics failed", 3564 instance, port); 3565 3566 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3567 rval = AHCI_FAILURE; 3568 3569 out: 3570 /* Clear port serror register for the port */ 3571 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3572 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3573 AHCI_SERROR_CLEAR_ALL); 3574 3575 return (rval); 3576 3577 out_check: 3578 /* 3579 * Check device status, if keep busy or COMRESET error 3580 * do device reset to patch some SATA disks' issue 3581 * 3582 * For VT8251, sometimes need to do the device reset 3583 */ 3584 if ((port_task_file & AHCI_TFD_STS_BSY) || 3585 (port_task_file & AHCI_TFD_STS_DRQ)) { 3586 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "port %d keep BSY/DRQ set " 3587 "need to do device reset", port); 3588 3589 (void) ahci_software_reset(ahci_ctlp, ahci_portp, port); 3590 3591 port_task_file = 3592 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3593 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3594 3595 if (port_task_file & AHCI_TFD_STS_BSY || 3596 port_task_file & AHCI_TFD_STS_DRQ) { 3597 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset: port %d " 3598 "BSY/DRQ still set after device reset " 3599 "port_task_file = 0x%x", instance, 3600 port, port_task_file); 3601 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3602 rval = AHCI_FAILURE; 3603 } 3604 } 3605 3606 goto out; 3607 } 3608 3609 /* 3610 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3611 * This is the most intrusive. 3612 * 3613 * When an HBA reset occurs, Phy communication will be re-established with 3614 * the device through a COMRESET followed by the normal out-of-band 3615 * communication sequence defined in Serial ATA. AT the end of reset, the 3616 * device, if working properly, will send a D2H Register FIS, which contains 3617 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3618 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3619 * signature. 3620 * 3621 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3622 */ 3623 static int 3624 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3625 { 3626 ahci_port_t *ahci_portp; 3627 uint32_t ghc_control; 3628 uint8_t port; 3629 int loop_count; 3630 int rval = AHCI_SUCCESS; 3631 3632 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting"); 3633 3634 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3635 3636 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3637 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3638 3639 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3640 ghc_control |= AHCI_HBA_GHC_HR; 3641 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3642 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3643 3644 /* 3645 * Wait until HBA Reset complete or timeout 3646 */ 3647 loop_count = 0; 3648 do { 3649 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3650 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3651 3652 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3653 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3654 "ahci hba reset is timing out, " 3655 "ghc_control = 0x%x", ghc_control); 3656 /* We are effectively timing out after 1 sec. */ 3657 break; 3658 } 3659 3660 /* Wait for 10 millisec */ 3661 delay(AHCI_10MS_TICKS); 3662 } while (ghc_control & AHCI_HBA_GHC_HR); 3663 3664 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3665 "ahci_hba_reset: 1st loop count: %d, " 3666 "ghc_control = 0x%x", loop_count, ghc_control); 3667 3668 if (ghc_control & AHCI_HBA_GHC_HR) { 3669 /* The hba is not reset for some reasons */ 3670 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3671 "hba reset failed: HBA in a hung or locked state"); 3672 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3673 return (AHCI_FAILURE); 3674 } 3675 3676 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3677 /* Only check implemented ports */ 3678 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3679 continue; 3680 } 3681 3682 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3683 mutex_enter(&ahci_portp->ahciport_mutex); 3684 3685 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3686 != AHCI_SUCCESS) { 3687 rval = AHCI_FAILURE; 3688 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3689 "ahci_hba_reset: port %d failed", port); 3690 } 3691 3692 mutex_exit(&ahci_portp->ahciport_mutex); 3693 } 3694 3695 /* 3696 * Indicate that system software is AHCI aware by setting 3697 * GHC.AE to 1 3698 */ 3699 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3700 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3701 3702 ghc_control |= AHCI_HBA_GHC_AE; 3703 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3704 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3705 3706 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3707 3708 return (rval); 3709 } 3710 3711 /* 3712 * This routine is only called from AHCI_ATTACH or phyrdy change 3713 * case. It first calls port reset to initialize port, probe port and probe 3714 * device, then try to read PxSIG register to find the type of device 3715 * attached to the port. 3716 * 3717 * WARNING!!! ahciport_mutex should be acquired before the function 3718 * is called. And the port interrupt is disabled. 3719 */ 3720 static void 3721 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3722 ahci_port_t *ahci_portp, uint8_t port) 3723 { 3724 uint32_t signature; 3725 3726 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3727 "ahci_find_dev_signature enter: port %d", port); 3728 3729 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3730 3731 /* Call port reset to check link status and get device signature */ 3732 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 3733 3734 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3735 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3736 "ahci_find_dev_signature: No device is found " 3737 "at port %d", port); 3738 return; 3739 } 3740 3741 /* Check the port state */ 3742 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3743 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 3744 "ahci_find_dev_signature: port %d state 0x%x", 3745 port, ahci_portp->ahciport_port_state); 3746 return; 3747 } 3748 3749 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3750 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3751 3752 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3753 "ahci_find_dev_signature: port %d signature = 0x%x", 3754 port, signature); 3755 3756 switch (signature) { 3757 3758 case AHCI_SIGNATURE_DISK: 3759 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3760 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3761 "Disk is found at port: %d", port); 3762 break; 3763 3764 case AHCI_SIGNATURE_ATAPI: 3765 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3766 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3767 "ATAPI device is found at port: %d", port); 3768 break; 3769 3770 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3771 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3772 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3773 "Port Multiplier is found at port: %d", port); 3774 break; 3775 3776 default: 3777 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3778 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3779 "Unknown device is found at port: %d", port); 3780 } 3781 } 3782 3783 /* 3784 * According to the spec, to reliably detect hot plug removals, software 3785 * must disable interface power management. Software should perform the 3786 * following initialization on a port after a device is attached: 3787 * Set PxSCTL.IPM to 3h to disable interface state transitions 3788 * Set PxCMD.ALPE to '0' to disable aggressive power management 3789 * Disable device initiated interface power management by SET FEATURE 3790 * 3791 * We can ignore the last item because by default the feature is disabled 3792 */ 3793 static void 3794 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3795 { 3796 uint32_t port_scontrol, port_cmd_status; 3797 3798 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3799 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3800 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3801 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3802 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3803 3804 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3805 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3806 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3807 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3808 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3809 } 3810 3811 /* 3812 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3813 * to 1, then set it firstly. 3814 * 3815 * Each port contains two major DMA engines. One DMA engine walks through 3816 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3817 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3818 * 3819 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3820 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 3821 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 3822 * 3823 * Software shall not set PxCMD.ST to '1' unless a functional device is 3824 * present on the port(as determined by PxTFD.STS.BSY = '0', 3825 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 3826 * 3827 * WARNING!!! ahciport_mutex should be acquired before the function 3828 * is called. 3829 */ 3830 static int 3831 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3832 { 3833 uint32_t port_cmd_status; 3834 3835 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 3836 3837 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3838 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3839 "the state for port %d is 0x%x", 3840 port, ahci_portp->ahciport_port_state); 3841 return (AHCI_FAILURE); 3842 } 3843 3844 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3845 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3846 "no device is attached at port %d", port); 3847 return (AHCI_FAILURE); 3848 } 3849 3850 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 3851 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3852 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3853 3854 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 3855 port_cmd_status |= AHCI_CMD_STATUS_FRE; 3856 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3857 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3858 port_cmd_status); 3859 } 3860 3861 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3862 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3863 3864 port_cmd_status |= AHCI_CMD_STATUS_ST; 3865 3866 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3867 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3868 port_cmd_status); 3869 3870 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 3871 3872 return (AHCI_SUCCESS); 3873 } 3874 3875 /* 3876 * Allocate the ahci_port_t including Received FIS and Command List. 3877 * The argument - port is the physical port number, and not logical 3878 * port number seen by the SATA framework. 3879 * 3880 * WARNING!!! ahcictl_mutex should be acquired before the function 3881 * is called. 3882 */ 3883 static int 3884 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3885 { 3886 ahci_port_t *ahci_portp; 3887 3888 ahci_portp = 3889 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 3890 3891 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 3892 ahci_portp->ahciport_port_num = port; 3893 3894 /* Intialize the port condition variable */ 3895 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 3896 3897 /* Initialize the port mutex */ 3898 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 3899 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 3900 3901 mutex_enter(&ahci_portp->ahciport_mutex); 3902 3903 /* 3904 * Allocate memory for received FIS structure and 3905 * command list for this port 3906 */ 3907 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3908 goto err_case1; 3909 } 3910 3911 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3912 goto err_case2; 3913 } 3914 3915 ahci_portp->ahciport_event_args = 3916 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 3917 3918 if (ahci_portp->ahciport_event_args == NULL) 3919 goto err_case3; 3920 3921 mutex_exit(&ahci_portp->ahciport_mutex); 3922 3923 return (AHCI_SUCCESS); 3924 3925 err_case3: 3926 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3927 3928 err_case2: 3929 ahci_dealloc_rcvd_fis(ahci_portp); 3930 3931 err_case1: 3932 mutex_exit(&ahci_portp->ahciport_mutex); 3933 mutex_destroy(&ahci_portp->ahciport_mutex); 3934 cv_destroy(&ahci_portp->ahciport_cv); 3935 3936 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3937 3938 return (AHCI_FAILURE); 3939 } 3940 3941 /* 3942 * Reverse of ahci_dealloc_port_state(). 3943 * 3944 * WARNING!!! ahcictl_mutex should be acquired before the function 3945 * is called. 3946 */ 3947 static void 3948 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3949 { 3950 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3951 3952 ASSERT(ahci_portp != NULL); 3953 3954 mutex_enter(&ahci_portp->ahciport_mutex); 3955 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 3956 ahci_portp->ahciport_event_args = NULL; 3957 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3958 ahci_dealloc_rcvd_fis(ahci_portp); 3959 mutex_exit(&ahci_portp->ahciport_mutex); 3960 3961 mutex_destroy(&ahci_portp->ahciport_mutex); 3962 cv_destroy(&ahci_portp->ahciport_cv); 3963 3964 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3965 3966 ahci_ctlp->ahcictl_ports[port] = NULL; 3967 } 3968 3969 /* 3970 * Allocates memory for the Received FIS Structure 3971 * 3972 * WARNING!!! ahciport_mutex should be acquired before the function 3973 * is called. 3974 */ 3975 static int 3976 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3977 uint8_t port) 3978 { 3979 size_t rcvd_fis_size; 3980 size_t ret_len; 3981 uint_t cookie_count; 3982 3983 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 3984 3985 /* allocate rcvd FIS dma handle. */ 3986 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 3987 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 3988 DDI_DMA_SLEEP, 3989 NULL, 3990 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 3991 DDI_SUCCESS) { 3992 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3993 "rcvd FIS dma handle alloc failed"); 3994 3995 return (AHCI_FAILURE); 3996 } 3997 3998 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 3999 rcvd_fis_size, 4000 &accattr, 4001 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4002 DDI_DMA_SLEEP, 4003 NULL, 4004 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4005 &ret_len, 4006 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4007 4008 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4009 "rcvd FIS dma mem alloc fail"); 4010 /* error.. free the dma handle. */ 4011 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4012 return (AHCI_FAILURE); 4013 } 4014 4015 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4016 NULL, 4017 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4018 rcvd_fis_size, 4019 DDI_DMA_CONSISTENT, 4020 DDI_DMA_SLEEP, 4021 NULL, 4022 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4023 &cookie_count) != DDI_DMA_MAPPED) { 4024 4025 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4026 "rcvd FIS dma handle bind fail"); 4027 /* error.. free the dma handle & free the memory. */ 4028 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4029 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4030 return (AHCI_FAILURE); 4031 } 4032 4033 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4034 4035 /* Config Port Received FIS Base Address */ 4036 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4037 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4038 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4039 4040 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4041 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4042 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4043 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4044 4045 return (AHCI_SUCCESS); 4046 } 4047 4048 /* 4049 * Deallocates the Received FIS Structure 4050 * 4051 * WARNING!!! ahciport_mutex should be acquired before the function 4052 * is called. 4053 */ 4054 static void 4055 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4056 { 4057 /* Unbind the cmd list dma handle first. */ 4058 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4059 4060 /* Then free the underlying memory. */ 4061 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4062 4063 /* Now free the handle itself. */ 4064 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4065 } 4066 4067 /* 4068 * Allocates memory for the Command List, which contains up to 32 entries. 4069 * Each entry contains a command header, which is a 32-byte structure that 4070 * includes the pointer to the command table. 4071 * 4072 * WARNING!!! ahciport_mutex should be acquired before the function 4073 * is called. 4074 */ 4075 static int 4076 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4077 uint8_t port) 4078 { 4079 size_t cmd_list_size; 4080 size_t ret_len; 4081 uint_t cookie_count; 4082 4083 cmd_list_size = 4084 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4085 4086 /* allocate cmd list dma handle. */ 4087 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4088 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4089 DDI_DMA_SLEEP, 4090 NULL, 4091 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4092 4093 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4094 "cmd list dma handle alloc failed"); 4095 return (AHCI_FAILURE); 4096 } 4097 4098 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4099 cmd_list_size, 4100 &accattr, 4101 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4102 DDI_DMA_SLEEP, 4103 NULL, 4104 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4105 &ret_len, 4106 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4107 4108 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4109 "cmd list dma mem alloc fail"); 4110 /* error.. free the dma handle. */ 4111 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4112 return (AHCI_FAILURE); 4113 } 4114 4115 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4116 NULL, 4117 (caddr_t)ahci_portp->ahciport_cmd_list, 4118 cmd_list_size, 4119 DDI_DMA_CONSISTENT, 4120 DDI_DMA_SLEEP, 4121 NULL, 4122 &ahci_portp->ahciport_cmd_list_dma_cookie, 4123 &cookie_count) != DDI_DMA_MAPPED) { 4124 4125 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4126 "cmd list dma handle bind fail"); 4127 /* error.. free the dma handle & free the memory. */ 4128 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4129 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4130 return (AHCI_FAILURE); 4131 } 4132 4133 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4134 4135 /* Config Port Command List Base Address */ 4136 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4137 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4138 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4139 4140 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4141 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4142 4143 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4144 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4145 4146 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4147 goto err_out; 4148 } 4149 4150 return (AHCI_SUCCESS); 4151 4152 err_out: 4153 /* Unbind the cmd list dma handle first. */ 4154 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4155 4156 /* Then free the underlying memory. */ 4157 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4158 4159 /* Now free the handle itself. */ 4160 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4161 4162 return (AHCI_FAILURE); 4163 } 4164 4165 /* 4166 * Deallocates the Command List 4167 * 4168 * WARNING!!! ahciport_mutex should be acquired before the function 4169 * is called. 4170 */ 4171 static void 4172 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4173 { 4174 /* First dealloc command table */ 4175 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4176 4177 /* Unbind the cmd list dma handle first. */ 4178 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4179 4180 /* Then free the underlying memory. */ 4181 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4182 4183 /* Now free the handle itself. */ 4184 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4185 } 4186 4187 /* 4188 * Allocates memory for all Command Tables, which contains Command FIS, 4189 * ATAPI Command and Physical Region Descriptor Table. 4190 * 4191 * WARNING!!! ahciport_mutex should be acquired before the function 4192 * is called. 4193 */ 4194 static int 4195 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4196 { 4197 size_t ret_len; 4198 ddi_dma_cookie_t cmd_table_dma_cookie; 4199 uint_t cookie_count; 4200 int slot; 4201 4202 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4203 "ahci_alloc_cmd_tables: port %d enter", 4204 ahci_portp->ahciport_port_num); 4205 4206 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4207 /* Allocate cmd table dma handle. */ 4208 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4209 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4210 DDI_DMA_SLEEP, 4211 NULL, 4212 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4213 DDI_SUCCESS) { 4214 4215 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4216 "cmd table dma handle alloc failed"); 4217 4218 goto err_out; 4219 } 4220 4221 if (ddi_dma_mem_alloc( 4222 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4223 ahci_cmd_table_size, 4224 &accattr, 4225 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4226 DDI_DMA_SLEEP, 4227 NULL, 4228 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4229 &ret_len, 4230 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4231 NULL) { 4232 4233 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4234 "cmd table dma mem alloc fail"); 4235 4236 /* error.. free the dma handle. */ 4237 ddi_dma_free_handle( 4238 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4239 goto err_out; 4240 } 4241 4242 if (ddi_dma_addr_bind_handle( 4243 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4244 NULL, 4245 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4246 ahci_cmd_table_size, 4247 DDI_DMA_CONSISTENT, 4248 DDI_DMA_SLEEP, 4249 NULL, 4250 &cmd_table_dma_cookie, 4251 &cookie_count) != DDI_DMA_MAPPED) { 4252 4253 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4254 "cmd table dma handle bind fail"); 4255 /* error.. free the dma handle & free the memory. */ 4256 ddi_dma_mem_free( 4257 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4258 ddi_dma_free_handle( 4259 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4260 goto err_out; 4261 } 4262 4263 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4264 ahci_cmd_table_size); 4265 4266 /* Config Port Command Table Base Address */ 4267 SET_COMMAND_TABLE_BASE_ADDR( 4268 (&ahci_portp->ahciport_cmd_list[slot]), 4269 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4270 4271 #ifndef __lock_lint 4272 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4273 (&ahci_portp->ahciport_cmd_list[slot]), 4274 cmd_table_dma_cookie.dmac_laddress >> 32); 4275 #endif 4276 } 4277 4278 return (AHCI_SUCCESS); 4279 err_out: 4280 4281 for (slot--; slot >= 0; slot--) { 4282 /* Unbind the cmd table dma handle first */ 4283 (void) ddi_dma_unbind_handle( 4284 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4285 4286 /* Then free the underlying memory */ 4287 ddi_dma_mem_free( 4288 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4289 4290 /* Now free the handle itself */ 4291 ddi_dma_free_handle( 4292 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4293 } 4294 4295 return (AHCI_FAILURE); 4296 } 4297 4298 /* 4299 * Deallocates memory for all Command Tables. 4300 * 4301 * WARNING!!! ahciport_mutex should be acquired before the function 4302 * is called. 4303 */ 4304 static void 4305 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4306 { 4307 int slot; 4308 4309 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 4310 "ahci_dealloc_cmd_tables: %d enter", 4311 ahci_portp->ahciport_port_num); 4312 4313 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4314 /* Unbind the cmd table dma handle first. */ 4315 (void) ddi_dma_unbind_handle( 4316 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4317 4318 /* Then free the underlying memory. */ 4319 ddi_dma_mem_free( 4320 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4321 4322 /* Now free the handle itself. */ 4323 ddi_dma_free_handle( 4324 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4325 } 4326 } 4327 4328 /* 4329 * WARNING!!! ahciport_mutex should be acquired before the function 4330 * is called. 4331 */ 4332 static void 4333 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4334 sata_device_t *sd) 4335 { 4336 sd->satadev_scr.sstatus = 4337 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4338 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4339 sd->satadev_scr.serror = 4340 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4341 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4342 sd->satadev_scr.scontrol = 4343 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4344 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4345 sd->satadev_scr.sactive = 4346 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4347 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4348 } 4349 4350 /* 4351 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4352 */ 4353 static void 4354 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4355 { 4356 uint32_t port_intr_status; 4357 uint32_t port_intr_enable; 4358 4359 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4360 "ahci_port_intr enter: port %d", port); 4361 4362 mutex_enter(&ahci_portp->ahciport_mutex); 4363 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4364 /* For SATA_OPMODE_POLLING commands */ 4365 port_intr_enable = 4366 (AHCI_INTR_STATUS_DHRS | 4367 AHCI_INTR_STATUS_PSS | 4368 AHCI_INTR_STATUS_SDBS | 4369 AHCI_INTR_STATUS_UFS | 4370 AHCI_INTR_STATUS_PCS | 4371 AHCI_INTR_STATUS_PRCS | 4372 AHCI_INTR_STATUS_OFS | 4373 AHCI_INTR_STATUS_INFS | 4374 AHCI_INTR_STATUS_IFS | 4375 AHCI_INTR_STATUS_HBDS | 4376 AHCI_INTR_STATUS_HBFS | 4377 AHCI_INTR_STATUS_TFES); 4378 mutex_exit(&ahci_portp->ahciport_mutex); 4379 goto next; 4380 } 4381 mutex_exit(&ahci_portp->ahciport_mutex); 4382 4383 /* 4384 * port_intr_enable indicates that the corresponding interrrupt 4385 * reporting is enabled. 4386 */ 4387 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4388 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4389 next: 4390 /* 4391 * port_intr_stats indicates that the corresponding interrupt 4392 * condition is active. 4393 */ 4394 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4395 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4396 4397 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4398 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4399 "port_intr_enable = 0x%x", 4400 port, port_intr_status, port_intr_enable); 4401 4402 port_intr_status &= port_intr_enable; 4403 4404 /* First clear the port interrupts status */ 4405 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4406 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4407 port_intr_status); 4408 4409 /* Check the completed non-queued commands */ 4410 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4411 AHCI_INTR_STATUS_PSS)) { 4412 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4413 ahci_portp, port); 4414 } 4415 4416 /* Check the completed queued commands */ 4417 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4418 (void) ahci_intr_set_device_bits(ahci_ctlp, 4419 ahci_portp, port); 4420 } 4421 4422 /* Check the port connect change status interrupt bit */ 4423 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4424 (void) ahci_intr_port_connect_change(ahci_ctlp, 4425 ahci_portp, port); 4426 } 4427 4428 /* Check the device mechanical presence status interrupt bit */ 4429 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4430 (void) ahci_intr_device_mechanical_presence_status( 4431 ahci_ctlp, ahci_portp, port); 4432 } 4433 4434 /* Check the PhyRdy change status interrupt bit */ 4435 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4436 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4437 port); 4438 } 4439 4440 /* 4441 * Check the non-fatal error interrupt bits, there are three 4442 * kinds of non-fatal errors at the time being: 4443 * 4444 * PxIS.UFS - Unknown FIS Error 4445 * PxIS.OFS - Overflow Error 4446 * PxIS.INFS - Interface Non-Fatal Error 4447 * 4448 * For these non-fatal errors, the HBA can continue to operate, 4449 * so the driver just log the error messages. 4450 */ 4451 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4452 AHCI_INTR_STATUS_OFS | 4453 AHCI_INTR_STATUS_INFS)) { 4454 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4455 port, port_intr_status); 4456 } 4457 4458 /* 4459 * Check the fatal error interrupt bits, there are four kinds 4460 * of fatal errors for AHCI controllers: 4461 * 4462 * PxIS.HBFS - Host Bus Fatal Error 4463 * PxIS.HBDS - Host Bus Data Error 4464 * PxIS.IFS - Interface Fatal Error 4465 * PxIS.TFES - Task File Error 4466 * 4467 * The fatal error means the HBA can not recover from it by 4468 * itself, and it will try to abort the transfer, and the software 4469 * must intervene to restart the port. 4470 */ 4471 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4472 AHCI_INTR_STATUS_HBDS | 4473 AHCI_INTR_STATUS_HBFS | 4474 AHCI_INTR_STATUS_TFES)) 4475 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4476 port, port_intr_status); 4477 4478 /* Check the cold port detect interrupt bit */ 4479 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4480 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4481 } 4482 4483 /* Second clear the corresponding bit in IS.IPS */ 4484 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4485 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4486 } 4487 4488 /* 4489 * Interrupt service handler 4490 */ 4491 static uint_t 4492 ahci_intr(caddr_t arg1, caddr_t arg2) 4493 { 4494 #ifndef __lock_lint 4495 _NOTE(ARGUNUSED(arg2)) 4496 #endif 4497 /* LINTED */ 4498 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4499 ahci_port_t *ahci_portp; 4500 int32_t global_intr_status; 4501 uint8_t port; 4502 4503 /* 4504 * global_intr_status indicates that the corresponding port has 4505 * an interrupt pending. 4506 */ 4507 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4508 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4509 4510 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4511 /* The interrupt is not ours */ 4512 return (DDI_INTR_UNCLAIMED); 4513 } 4514 4515 /* Loop for all the ports */ 4516 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4517 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4518 continue; 4519 } 4520 if (!((0x1 << port) & global_intr_status)) { 4521 continue; 4522 } 4523 4524 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4525 4526 /* Call ahci_port_intr */ 4527 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4528 } 4529 4530 return (DDI_INTR_CLAIMED); 4531 } 4532 4533 /* 4534 * For non-queued commands, when the corresponding bit in the PxCI register 4535 * is cleared, it means the command is completed successfully. And according 4536 * to the HBA state machine, there are three conditions which possibly will 4537 * try to clear the PxCI register bit. 4538 * 1. Receive one D2H Register FIS which is with 'I' bit set 4539 * 2. Update PIO Setup FIS 4540 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4541 * 4542 * Process completed non-queued commands when the interrupt status bit - 4543 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4544 * 4545 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4546 * with the 'I' bit set. And the following commands will send thus 4547 * FIS with 'I' bit set upon the successful completion: 4548 * 1. Non-data commands 4549 * 2. DMA data-in command 4550 * 3. DMA data-out command 4551 * 4. PIO data-out command 4552 * 5. PACKET non-data commands 4553 * 6. PACKET PIO data-in command 4554 * 7. PACKET PIO data-out command 4555 * 8. PACKET DMA data-in command 4556 * 9. PACKET DMA data-out command 4557 * 4558 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4559 * with the 'I' bit set. And the following commands will send this 4560 * FIS upon the successful completion: 4561 * 1. PIO data-in command 4562 */ 4563 static int 4564 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4565 ahci_port_t *ahci_portp, uint8_t port) 4566 { 4567 uint32_t port_cmd_issue = 0; 4568 uint32_t finished_tags; 4569 int finished_slot; 4570 sata_pkt_t *satapkt; 4571 ahci_fis_d2h_register_t *rcvd_fisp; 4572 #if AHCI_DEBUG 4573 ahci_cmd_header_t *cmd_header; 4574 uint32_t cmd_dmacount; 4575 #endif 4576 4577 mutex_enter(&ahci_portp->ahciport_mutex); 4578 4579 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4580 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4581 /* 4582 * Spurious interrupt. Nothing to be done. 4583 */ 4584 mutex_exit(&ahci_portp->ahciport_mutex); 4585 return (AHCI_SUCCESS); 4586 } 4587 4588 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4589 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4590 4591 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4592 /* Slot 0 is always used during error recovery */ 4593 finished_tags = 0x1 & ~port_cmd_issue; 4594 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 4595 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4596 "retrieval is finished, and finished_tags = 0x%x", 4597 port, finished_tags); 4598 } else { 4599 finished_tags = ahci_portp->ahciport_pending_tags & 4600 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4601 } 4602 4603 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4604 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4605 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4606 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4607 finished_tags); 4608 4609 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4610 (finished_tags == 0x1)) { 4611 satapkt = ahci_portp->ahciport_err_retri_pkt; 4612 ASSERT(satapkt != NULL); 4613 4614 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4615 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4616 "with SATA_PKT_COMPLETED", (void *)satapkt); 4617 4618 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4619 goto out; 4620 } 4621 4622 while (finished_tags) { 4623 finished_slot = ddi_ffs(finished_tags) - 1; 4624 if (finished_slot == -1) { 4625 goto out; 4626 } 4627 4628 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4629 ASSERT(satapkt != NULL); 4630 #if AHCI_DEBUG 4631 /* 4632 * For non-native queued commands, the PRD byte count field 4633 * shall contain an accurate count of the number of bytes 4634 * transferred for the command before the PxCI bit is cleared 4635 * to '0' for the command. 4636 * 4637 * The purpose of this field is to let software know how many 4638 * bytes transferred for a given operation in order to 4639 * determine if underflow occurred. When issuing native command 4640 * queuing commands, this field should not be used and is not 4641 * required to be valid since in this case underflow is always 4642 * illegal. 4643 * 4644 * For data reads, the HBA will update its PRD byte count with 4645 * the total number of bytes received from the last FIS, and 4646 * may be able to continue normally. For data writes, the 4647 * device will detect an error, and HBA most likely will get 4648 * a fatal error. 4649 * 4650 * Therefore, here just put code to debug part. And please 4651 * refer to the comment above ahci_intr_fatal_error for the 4652 * definition of underflow error. 4653 */ 4654 cmd_dmacount = 4655 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4656 if (cmd_dmacount) { 4657 cmd_header = 4658 &ahci_portp->ahciport_cmd_list[finished_slot]; 4659 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4660 "ahci_intr_cmd_cmplt: port %d, " 4661 "PRD Byte Count = 0x%x, " 4662 "ahciport_prd_bytecounts = 0x%x", port, 4663 cmd_header->ahcich_prd_byte_count, 4664 cmd_dmacount); 4665 4666 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4667 AHCIDBG1(AHCIDBG_UNDERFLOW, ahci_ctlp, 4668 "ahci_intr_cmd_cmplt: port %d, " 4669 "an underflow occurred", port); 4670 } 4671 } 4672 #endif 4673 4674 /* 4675 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4676 * feature, sata_special_regs flag will be set, and the 4677 * driver should copy the status and the other corresponding 4678 * register values in the D2H Register FIS received (It's 4679 * working on Non-data protocol) from the device back to 4680 * the sata_cmd. 4681 * 4682 * For every AHCI port, there is only one Received FIS 4683 * structure, which contains the FISes received from the 4684 * device, So we're trying to copy the content of D2H 4685 * Register FIS in the Received FIS structure back to 4686 * the sata_cmd. 4687 */ 4688 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4689 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4690 ahcirf_d2h_register_fis); 4691 satapkt->satapkt_cmd.satacmd_status_reg = 4692 GET_RFIS_STATUS(rcvd_fisp); 4693 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4694 } 4695 4696 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4697 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4698 "with SATA_PKT_COMPLETED", (void *)satapkt); 4699 4700 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4701 CLEAR_BIT(finished_tags, finished_slot); 4702 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4703 4704 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4705 } 4706 out: 4707 AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp, 4708 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4709 ahci_portp->ahciport_pending_tags); 4710 4711 mutex_exit(&ahci_portp->ahciport_mutex); 4712 4713 return (AHCI_SUCCESS); 4714 } 4715 4716 /* 4717 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4718 * with the 'I' bit set and has been copied into system memory. It will 4719 * be sent under the following situations: 4720 * 4721 * 1. NCQ command is completed 4722 * 2. Asynchronous notification 4723 * 4724 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4725 * via the Set Device Bits FIS. When such event is generated, the software 4726 * needs to read PxSACT register and compares the current value to the 4727 * list of commands previously issue by software. ahciport_pending_ncq_tags 4728 * keeps the tags of previously issued commands. 4729 * 4730 * Asynchronous Notification is a feature in SATA II, which allows an 4731 * ATAPI device to send a signal to the host when media is inserted or 4732 * removed and avoids polling the device for media changes. The signal 4733 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4734 * set to '1'. At the moment, it's not supported yet. 4735 */ 4736 static int 4737 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4738 ahci_port_t *ahci_portp, uint8_t port) 4739 { 4740 uint32_t port_sactive; 4741 uint32_t port_cmd_issue; 4742 uint32_t issued_tags; 4743 int issued_slot; 4744 uint32_t finished_tags; 4745 int finished_slot; 4746 sata_pkt_t *satapkt; 4747 4748 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4749 "ahci_intr_set_device_bits enter: port %d", port); 4750 4751 mutex_enter(&ahci_portp->ahciport_mutex); 4752 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4753 mutex_exit(&ahci_portp->ahciport_mutex); 4754 return (AHCI_SUCCESS); 4755 } 4756 4757 /* 4758 * First the handler got which commands are finished by checking 4759 * PxSACT register 4760 */ 4761 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4762 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4763 4764 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4765 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4766 4767 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4768 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4769 "port_sactive = 0x%x", port, 4770 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4771 4772 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4773 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4774 4775 /* 4776 * For NCQ commands, the software can determine which command has 4777 * already been transmitted to the device by checking PxCI register. 4778 */ 4779 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4780 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4781 4782 issued_tags = ahci_portp->ahciport_pending_tags & 4783 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4784 4785 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4786 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4787 "port_cmd_issue = 0x%x", port, 4788 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4789 4790 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4791 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4792 4793 /* 4794 * Clear ahciport_pending_tags bit when the corresponding command 4795 * is already sent down to the device. 4796 */ 4797 while (issued_tags) { 4798 issued_slot = ddi_ffs(issued_tags) - 1; 4799 if (issued_slot == -1) { 4800 goto next; 4801 } 4802 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 4803 CLEAR_BIT(issued_tags, issued_slot); 4804 } 4805 4806 next: 4807 while (finished_tags) { 4808 finished_slot = ddi_ffs(finished_tags) - 1; 4809 if (finished_slot == -1) { 4810 goto out; 4811 } 4812 4813 /* The command is certainly transmitted to the device */ 4814 ASSERT(!(ahci_portp->ahciport_pending_tags & 4815 (0x1 << finished_slot))); 4816 4817 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4818 ASSERT(satapkt != NULL); 4819 4820 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4821 "ahci_intr_set_device_bits: sending up pkt 0x%p " 4822 "with SATA_PKT_COMPLETED", (void *)satapkt); 4823 4824 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 4825 CLEAR_BIT(finished_tags, finished_slot); 4826 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4827 4828 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4829 } 4830 out: 4831 AHCIDBG3(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 4832 "ahci_intr_set_device_bits: port %d " 4833 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 4834 port, ahci_portp->ahciport_pending_ncq_tags, 4835 ahci_portp->ahciport_pending_tags); 4836 4837 mutex_exit(&ahci_portp->ahciport_mutex); 4838 4839 return (AHCI_SUCCESS); 4840 } 4841 4842 /* 4843 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 4844 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 4845 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 4846 * indicates a COMINIT signal was received. 4847 * 4848 * Hot plug insertion is detected by reception of a COMINIT signal from the 4849 * device. On reception of unsolicited COMINIT, the HBA shall generate a 4850 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 4851 * begin the normal communication negotiation sequence as outlined in the 4852 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 4853 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 4854 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 4855 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 4856 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 4857 * to handle hot plug insertion. In this interrupt handler, just do nothing 4858 * but print some log message and clear the bit. 4859 */ 4860 static int 4861 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 4862 ahci_port_t *ahci_portp, uint8_t port) 4863 { 4864 #if AHCI_DEBUG 4865 uint32_t port_serror; 4866 #endif 4867 4868 mutex_enter(&ahci_portp->ahciport_mutex); 4869 4870 #if AHCI_DEBUG 4871 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4872 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 4873 4874 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4875 "ahci_intr_port_connect_change: port %d, " 4876 "port_serror = 0x%x", port, port_serror); 4877 #endif 4878 4879 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 4880 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4881 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 4882 SERROR_EXCHANGED_ERR); 4883 4884 mutex_exit(&ahci_portp->ahciport_mutex); 4885 4886 return (AHCI_SUCCESS); 4887 } 4888 4889 /* 4890 * Hot Plug Operation for platforms that support Mechanical Presence 4891 * Switches. 4892 * 4893 * When set, it indicates that a mechanical presence switch attached to this 4894 * port has been opened or closed, which may lead to a change in the connection 4895 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 4896 * are set to '1'. 4897 * 4898 * At the moment, this interrupt is not needed and disabled and we just log 4899 * the debug message. 4900 */ 4901 static int 4902 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 4903 ahci_port_t *ahci_portp, uint8_t port) 4904 { 4905 uint32_t cap_status, port_cmd_status; 4906 4907 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4908 "ahci_intr_device_mechanical_presence_status enter, " 4909 "port %d", port); 4910 4911 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4912 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4913 4914 mutex_enter(&ahci_portp->ahciport_mutex); 4915 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4916 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4917 4918 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 4919 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 4920 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4921 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 4922 "the interrupt: cap_status = 0x%x, " 4923 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 4924 mutex_exit(&ahci_portp->ahciport_mutex); 4925 4926 return (AHCI_SUCCESS); 4927 } 4928 4929 #if AHCI_DEBUG 4930 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 4931 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4932 "The mechanical presence switch is open: " 4933 "port %d, port_cmd_status = 0x%x", 4934 port, port_cmd_status); 4935 } else { 4936 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4937 "The mechanical presence switch is close: " 4938 "port %d, port_cmd_status = 0x%x", 4939 port, port_cmd_status); 4940 } 4941 #endif 4942 4943 mutex_exit(&ahci_portp->ahciport_mutex); 4944 4945 return (AHCI_SUCCESS); 4946 } 4947 4948 /* 4949 * Native Hot Plug Support. 4950 * 4951 * When set, it indicates that the internal PHYRDY signal changed state. 4952 * This bit reflects the state of PxSERR.DIAG.N. 4953 * 4954 * There are three kinds of conditions to generate this interrupt event: 4955 * 1. a device is inserted 4956 * 2. a device is disconnected 4957 * 3. when the link enters/exits a Partial or Slumber interface power 4958 * management state 4959 * 4960 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 4961 * bit may be set due to the link entering the Partial or Slumber power 4962 * management state, rather than due to a hot plug insertion or removal 4963 * event. So far, the interface power management is disabled, so the 4964 * driver can reliably get removal detection notification via the 4965 * PxSERR.DIAG.N bit. 4966 */ 4967 static int 4968 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 4969 ahci_port_t *ahci_portp, uint8_t port) 4970 { 4971 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 4972 sata_device_t sdevice; 4973 int dev_exists_now = 0; 4974 int dev_existed_previously = 0; 4975 4976 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4977 "ahci_intr_phyrdy_change enter, port %d", port); 4978 4979 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 4980 mutex_enter(&ahci_portp->ahciport_mutex); 4981 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4982 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 4983 SERROR_PHY_RDY_CHG); 4984 mutex_exit(&ahci_portp->ahciport_mutex); 4985 4986 mutex_enter(&ahci_ctlp->ahcictl_mutex); 4987 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 4988 (ahci_portp == NULL)) { 4989 /* The whole controller setup is not yet done. */ 4990 mutex_exit(&ahci_ctlp->ahcictl_mutex); 4991 return (AHCI_SUCCESS); 4992 } 4993 mutex_exit(&ahci_ctlp->ahcictl_mutex); 4994 4995 mutex_enter(&ahci_portp->ahciport_mutex); 4996 4997 /* SStatus tells the presence of device. */ 4998 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4999 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5000 5001 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5002 dev_exists_now = 1; 5003 } 5004 5005 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5006 dev_existed_previously = 1; 5007 } 5008 5009 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5010 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5011 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5012 "ahci_intr_phyrdy_change: port %d " 5013 "AHCI_PORT_FLAG_NODEV is cleared", port); 5014 if (dev_exists_now == 0) 5015 dev_existed_previously = 1; 5016 } 5017 5018 bzero((void *)&sdevice, sizeof (sata_device_t)); 5019 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5020 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5021 sdevice.satadev_addr.pmport = 0; 5022 sdevice.satadev_state = SATA_PSTATE_PWRON; 5023 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5024 5025 if (dev_exists_now) { 5026 if (dev_existed_previously) { 5027 /* Things are fine now. The loss was temporary. */ 5028 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5029 "ahci_intr_phyrdy_change port %d " 5030 "device link lost/established", port); 5031 5032 mutex_exit(&ahci_portp->ahciport_mutex); 5033 sata_hba_event_notify( 5034 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5035 &sdevice, 5036 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5037 mutex_enter(&ahci_portp->ahciport_mutex); 5038 5039 } else { 5040 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5041 "ahci_intr_phyrdy_change: port %d " 5042 "device link established", port); 5043 5044 /* A new device has been detected. */ 5045 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5046 5047 /* Try to start the port */ 5048 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5049 != AHCI_SUCCESS) { 5050 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5051 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5052 "ahci_intr_phyrdy_change: port %d failed " 5053 "at start port", port); 5054 } 5055 5056 /* Clear the max queue depth for inserted device */ 5057 ahci_portp->ahciport_max_ncq_tags = 0; 5058 5059 mutex_exit(&ahci_portp->ahciport_mutex); 5060 sata_hba_event_notify( 5061 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5062 &sdevice, 5063 SATA_EVNT_LINK_ESTABLISHED); 5064 mutex_enter(&ahci_portp->ahciport_mutex); 5065 5066 } 5067 } else { /* No device exists now */ 5068 5069 if (dev_existed_previously) { 5070 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5071 "ahci_intr_phyrdy_change: port %d " 5072 "device link lost", port); 5073 5074 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5075 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5076 ahci_portp, port); 5077 5078 /* An existing device is lost. */ 5079 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5080 5081 mutex_exit(&ahci_portp->ahciport_mutex); 5082 sata_hba_event_notify( 5083 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5084 &sdevice, 5085 SATA_EVNT_LINK_LOST); 5086 mutex_enter(&ahci_portp->ahciport_mutex); 5087 } 5088 } 5089 5090 mutex_exit(&ahci_portp->ahciport_mutex); 5091 5092 return (AHCI_SUCCESS); 5093 } 5094 5095 /* 5096 * PxIS.UFS - Unknown FIS Error 5097 * 5098 * This interrupt event means an unknown FIS was received and has been 5099 * copied into system memory. An unknown FIS is not considered an illegal 5100 * FIS, unless the length received is more than 64 bytes. If an unknown 5101 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5102 * normal operation. If the unknown FIS is more than 64 bytes, then it 5103 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5104 * a fatal error. 5105 * 5106 * PxIS.OFS - Overflow Error 5107 * 5108 * Command list overflow is defined as software building a command table 5109 * that has fewer total bytes than the transaction given to the device. 5110 * On device writes, the HBA will run out of data, and on reads, there 5111 * will be no room to put the data. 5112 * 5113 * For an overflow on data read, either PIO or DMA, the HBA will set 5114 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5115 * non-fatal error when the HBA can continues. Sometimes, it will cause 5116 * a fatal error and need the software to do something. 5117 * 5118 * For an overflow on data write, setting PxIS.OFS is optional for both 5119 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5120 * software to clean up from this serious error. 5121 * 5122 * PxIS.INFS - Interface Non-Fatal Error 5123 * 5124 * This interrupt event indicates that the HBA encountered an error on 5125 * the Serial ATA interface but was able to continue operation. The kind 5126 * of error usually occurred during a non-Data FIS, and under this condition 5127 * the FIS will be re-transmitted by HBA automatically. 5128 * 5129 * When the FMA is implemented, there should be a stat structure to 5130 * record how many every kind of error happens. 5131 */ 5132 static int 5133 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5134 uint8_t port, uint32_t intr_status) 5135 { 5136 uint32_t port_serror; 5137 #if AHCI_DEBUG 5138 uint32_t port_cmd_status; 5139 uint32_t port_cmd_issue; 5140 uint32_t port_sactive; 5141 int current_slot; 5142 uint32_t current_tags; 5143 sata_pkt_t *satapkt; 5144 ahci_cmd_header_t *cmd_header; 5145 uint32_t cmd_dmacount; 5146 #endif 5147 5148 mutex_enter(&ahci_portp->ahciport_mutex); 5149 5150 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5151 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5152 5153 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5154 "ahci_intr_non_fatal_error: port %d, " 5155 "port_serror = 0x%x", port, port_serror); 5156 5157 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5158 5159 if (intr_status & AHCI_INTR_STATUS_UFS) { 5160 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5161 "ahci port %d has unknown FIS error", port); 5162 5163 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5164 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5165 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5166 SERROR_FIS_TYPE); 5167 } 5168 5169 #if AHCI_DEBUG 5170 if (intr_status & AHCI_INTR_STATUS_OFS) { 5171 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5172 "ahci port %d has overflow error", port); 5173 } 5174 5175 if (intr_status & AHCI_INTR_STATUS_INFS) { 5176 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5177 "ahci port %d has interface non fatal error", port); 5178 } 5179 5180 /* 5181 * Record the error occurred command's slot. 5182 */ 5183 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5184 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5185 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5186 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5187 5188 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5189 AHCI_CMD_STATUS_CCS_SHIFT; 5190 5191 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5192 satapkt = ahci_portp->ahciport_err_retri_pkt; 5193 ASSERT(satapkt != NULL); 5194 ASSERT(current_slot == 0); 5195 } else { 5196 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5197 } 5198 5199 if (satapkt != NULL) { 5200 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5201 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5202 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5203 satapkt->satapkt_cmd.satacmd_cmd_reg); 5204 5205 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5206 "ahci_intr_non_fatal_error: port %d, " 5207 "satapkt 0x%p is being processed when error occurs", 5208 port, (void *)satapkt); 5209 5210 /* 5211 * PRD Byte Count field of command header is not 5212 * required to reflect the total number of bytes 5213 * transferred when an overflow occurs, so here 5214 * just log the value. 5215 */ 5216 cmd_dmacount = 5217 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5218 if (cmd_dmacount) { 5219 cmd_header = &ahci_portp-> 5220 ahciport_cmd_list[current_slot]; 5221 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5222 "ahci_intr_non_fatal_error: port %d, " 5223 "PRD Byte Count = 0x%x, " 5224 "ahciport_prd_bytecounts = 0x%x", port, 5225 cmd_header->ahcich_prd_byte_count, 5226 cmd_dmacount); 5227 } 5228 } 5229 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5230 /* 5231 * For queued command, list those command which have already 5232 * been transmitted to the device and still not completed. 5233 */ 5234 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5235 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5236 5237 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5238 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5239 5240 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5241 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5242 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5243 ahci_portp->ahciport_pending_ncq_tags, 5244 port_sactive, port_cmd_issue); 5245 5246 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5247 port_sactive & ~port_cmd_issue & 5248 AHCI_NCQ_SLOT_MASK(ahci_portp); 5249 5250 while (current_tags) { 5251 current_slot = ddi_ffs(current_tags) - 1; 5252 if (current_slot == -1) { 5253 goto out; 5254 } 5255 5256 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5257 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5258 ahci_ctlp, "ahci_intr_non_fatal_error: " 5259 "port %d, satapkt 0x%p is outstanding when " 5260 "error occurs", port, (void *)satapkt); 5261 } 5262 } 5263 out: 5264 #endif 5265 mutex_exit(&ahci_portp->ahciport_mutex); 5266 5267 return (AHCI_SUCCESS); 5268 } 5269 5270 /* 5271 * According to the AHCI spec, the error types include system memory 5272 * errors, interface errors, port multiplier errors, device errors, 5273 * command list overflow, command list underflow, native command 5274 * queuing tag errors and pio data transfer errors. 5275 * 5276 * System memory errors such as target abort, master abort, and parity 5277 * may cause the host to stop, and they are serious errors and needed 5278 * to be recovered with software intervention. When system software 5279 * has given a pointer to the HBA that doesn't exist in physical memory, 5280 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5281 * data error such as CRC or parity occurs, the HBA aborts the transfer 5282 * (if necessary) and PxIS.HBDS will be set. 5283 * 5284 * Interface errors are errors that occur due to electrical issues on 5285 * the interface, or protocol miscommunication between the device and 5286 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5287 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5288 * causes PxIS.IFS/PxIS.INFS to be set are 5289 * 1. in PxSERR.ERR, P bit is set to '1' 5290 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5291 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5292 * If the error occurred during a non-data FIS, the FIS must be 5293 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5294 * the error occurred during a data FIS, the transfer will stop, so 5295 * the error is fatal and PxIS.IFS is set. 5296 * 5297 * When a FIS arrives that updates the taskfile, the HBA checks to see 5298 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5299 * stops processing any more commands. 5300 * 5301 * Command list overflow is defined as software building a command table 5302 * that has fewer total bytes than the transaction given to the device. 5303 * On device writes, the HBA will run out of data, and on reads, there 5304 * will be no room to put the data. For an overflow on data read, either 5305 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5306 * For an overflow on data write, setting PxIS.OFS is optional for both 5307 * DMA and PIO, and a COMRESET is required by software to clean up from 5308 * this serious error. 5309 * 5310 * Command list underflow is defined as software building a command 5311 * table that has more total bytes than the transaction given to the 5312 * device. For data writes, both PIO and DMA, the device will detect 5313 * an error and end the transfer. And these errors are most likely going 5314 * to be fatal errors that will cause the port to be restarted. For 5315 * data reads, the HBA updates its PRD byte count, and may be 5316 * able to continue normally, but is not required to. And The HBA is 5317 * not required to detect underflow conditions for native command 5318 * queuing command. 5319 * 5320 * The HBA does not actively check incoming DMA Setup FISes to ensure 5321 * that the PxSACT register bit for that slot is set. Existing error 5322 * mechanisms, such as host bus failure, or bad protocol, are used to 5323 * recover from this case. 5324 * 5325 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5326 * DATA FIS must be an integral number of Dwords. If the HBA receives 5327 * a request which is not an integral number of Dwords, the HBA 5328 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5329 * software restarts the port. And the HBA ensures that the size 5330 * of the DATA FIS received during a PIO command matches the size in 5331 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5332 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5333 * stop running until software restarts the port. 5334 */ 5335 /* 5336 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5337 * 5338 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5339 * interface which caused the transfer to stop. 5340 * 5341 * PxIS.HBDS indicates that the hba encountered a data error 5342 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5343 * 5344 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5345 * cannot recover from, such as a bad software pointer. 5346 * 5347 * PxIS.TFES is set whenever the status register is updated by the device 5348 * and the error bit (bit 0) is set. 5349 */ 5350 static int 5351 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5352 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5353 { 5354 uint32_t port_cmd_status; 5355 uint32_t port_serror; 5356 uint32_t task_file_status; 5357 int failed_slot; 5358 sata_pkt_t *spkt = NULL; 5359 uint8_t err_byte; 5360 ahci_event_arg_t *args; 5361 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5362 5363 mutex_enter(&ahci_portp->ahciport_mutex); 5364 5365 /* 5366 * ahci_intr_phyrdy_change() may have rendered it to 5367 * SATA_DTYPE_NONE. 5368 */ 5369 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5370 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5371 "ahci_intr_fatal_error: port %d no device attached, " 5372 "and just return without doing anything", port); 5373 goto out0; 5374 } 5375 5376 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5377 /* 5378 * Read PxCMD.CCS to determine the slot that the HBA 5379 * was processing when the error occurred. 5380 */ 5381 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5382 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5383 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5384 AHCI_CMD_STATUS_CCS_SHIFT; 5385 5386 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5387 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5388 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5389 "fatal error occurred for port %d", spkt, port); 5390 5391 if (intr_status & AHCI_INTR_STATUS_TFES) { 5392 task_file_status = ddi_get32( 5393 ahci_ctlp->ahcictl_ahci_acc_handle, 5394 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5395 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5396 "ahci_intr_fatal_error: port %d " 5397 "task_file_status = 0x%x", port, task_file_status); 5398 5399 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5400 >> AHCI_TFD_ERR_SHIFT; 5401 5402 /* 5403 * Won't emit the error message if it is an IDENTIFY 5404 * DEVICE command sent to an ATAPI device. 5405 */ 5406 if ((spkt != NULL) && 5407 (spkt->satapkt_cmd.satacmd_cmd_reg == 5408 SATAC_ID_DEVICE) && 5409 (err_byte == SATA_ERROR_ABORT)) 5410 goto out1; 5411 5412 /* 5413 * Won't emit the error message if it is an ATAPI PACKET 5414 * command 5415 */ 5416 if ((spkt != NULL) && 5417 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5418 goto out1; 5419 } 5420 } 5421 5422 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5423 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5424 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5425 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5426 out1: 5427 /* Prepare the argument for the taskq */ 5428 args = ahci_portp->ahciport_event_args; 5429 args->ahciea_ctlp = (void *)ahci_ctlp; 5430 args->ahciea_portp = (void *)ahci_portp; 5431 args->ahciea_event = intr_status; 5432 5433 /* Start the taskq to handle error recovery */ 5434 if ((ddi_taskq_dispatch(ahci_ctlp->ahcictl_event_taskq, 5435 ahci_events_handler, 5436 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5437 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5438 "failed", instance); 5439 } 5440 out0: 5441 mutex_exit(&ahci_portp->ahciport_mutex); 5442 5443 return (AHCI_SUCCESS); 5444 } 5445 5446 /* 5447 * Hot Plug Operation for platforms that support Cold Presence Detect. 5448 * 5449 * When set, a device status has changed as detected by the cold presence 5450 * detect logic. This bit can either be set due to a non-connected port 5451 * receiving a device, or a connected port having its device removed. 5452 * This bit is only valid if the port supports cold presence detect as 5453 * indicated by PxCMD.CPD set to '1'. 5454 * 5455 * At the moment, this interrupt is not needed and disabled and we just 5456 * log the debug message. 5457 */ 5458 static int 5459 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5460 ahci_port_t *ahci_portp, uint8_t port) 5461 { 5462 uint32_t port_cmd_status; 5463 sata_device_t sdevice; 5464 5465 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5466 "ahci_intr_cold_port_detect enter, port %d", port); 5467 5468 mutex_enter(&ahci_portp->ahciport_mutex); 5469 5470 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5471 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5472 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5473 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5474 "port %d does not support cold presence detect, so " 5475 "we just ignore this interrupt", port); 5476 mutex_exit(&ahci_portp->ahciport_mutex); 5477 return (AHCI_SUCCESS); 5478 } 5479 5480 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5481 "port %d device status has changed", port); 5482 5483 bzero((void *)&sdevice, sizeof (sata_device_t)); 5484 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5485 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5486 sdevice.satadev_addr.pmport = 0; 5487 sdevice.satadev_state = SATA_PSTATE_PWRON; 5488 5489 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5490 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5491 "port %d: a device is hot plugged", port); 5492 mutex_exit(&ahci_portp->ahciport_mutex); 5493 sata_hba_event_notify( 5494 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5495 &sdevice, 5496 SATA_EVNT_DEVICE_ATTACHED); 5497 mutex_enter(&ahci_portp->ahciport_mutex); 5498 5499 } else { 5500 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5501 "port %d: a device is hot unplugged", port); 5502 mutex_exit(&ahci_portp->ahciport_mutex); 5503 sata_hba_event_notify( 5504 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5505 &sdevice, 5506 SATA_EVNT_DEVICE_DETACHED); 5507 mutex_enter(&ahci_portp->ahciport_mutex); 5508 } 5509 5510 mutex_exit(&ahci_portp->ahciport_mutex); 5511 5512 return (AHCI_SUCCESS); 5513 } 5514 5515 /* 5516 * Enable the interrupts for a particular port. 5517 * 5518 * WARNING!!! ahciport_mutex should be acquired before the function 5519 * is called. 5520 */ 5521 static void 5522 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5523 { 5524 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5525 "ahci_enable_port_intrs enter, port %d", port); 5526 5527 /* 5528 * Clear port interrupt status before enabling interrupt 5529 */ 5530 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5531 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5532 AHCI_PORT_INTR_MASK); 5533 5534 /* 5535 * Clear the pending bit from IS.IPS 5536 */ 5537 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5538 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5539 5540 /* 5541 * Enable the following interrupts: 5542 * Device to Host Register FIS Interrupt (DHRS) 5543 * PIO Setup FIS Interrupt (PSS) 5544 * Set Device Bits Interrupt (SDBS) 5545 * Unknown FIS Interrupt (UFS) 5546 * Port Connect Change Status (PCS) 5547 * PhyRdy Change Status (PRCS) 5548 * Overflow Status (OFS) 5549 * Interface Non-fatal Error Status (INFS) 5550 * Interface Fatal Error Status (IFS) 5551 * Host Bus Data Error Status (HBDS) 5552 * Host Bus Fatal Error Status (HBFS) 5553 * Task File Error Status (TFES) 5554 */ 5555 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5556 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5557 (AHCI_INTR_STATUS_DHRS | 5558 AHCI_INTR_STATUS_PSS | 5559 AHCI_INTR_STATUS_SDBS | 5560 AHCI_INTR_STATUS_UFS | 5561 AHCI_INTR_STATUS_DPS | 5562 AHCI_INTR_STATUS_PCS | 5563 AHCI_INTR_STATUS_PRCS | 5564 AHCI_INTR_STATUS_OFS | 5565 AHCI_INTR_STATUS_INFS | 5566 AHCI_INTR_STATUS_IFS | 5567 AHCI_INTR_STATUS_HBDS | 5568 AHCI_INTR_STATUS_HBFS | 5569 AHCI_INTR_STATUS_TFES)); 5570 } 5571 5572 /* 5573 * Enable interrupts for all the ports. 5574 * 5575 * WARNING!!! ahcictl_mutex should be acquired before the function 5576 * is called. 5577 */ 5578 static void 5579 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5580 { 5581 uint32_t ghc_control; 5582 5583 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter"); 5584 5585 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5586 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5587 5588 ghc_control |= AHCI_HBA_GHC_IE; 5589 5590 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5591 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5592 } 5593 5594 /* 5595 * Disable interrupts for a particular port. 5596 * 5597 * WARNING!!! ahciport_mutex should be acquired before the function 5598 * is called. 5599 */ 5600 static void 5601 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5602 { 5603 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5604 "ahci_disable_port_intrs enter, port %d", port); 5605 5606 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5607 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5608 } 5609 5610 /* 5611 * Disable interrupts for the whole HBA. 5612 * 5613 * The global bit is cleared, then all interrupt sources from all 5614 * ports are disabled. 5615 * 5616 * WARNING!!! ahcictl_mutex should be acquired before the function 5617 * is called. 5618 */ 5619 static void 5620 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5621 { 5622 uint32_t ghc_control; 5623 5624 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter"); 5625 5626 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5627 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5628 5629 ghc_control &= ~ AHCI_HBA_GHC_IE; 5630 5631 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5632 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5633 } 5634 5635 /* 5636 * Handle INTx and legacy interrupts. 5637 */ 5638 static int 5639 ahci_add_legacy_intrs(ahci_ctl_t *ahci_ctlp) 5640 { 5641 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5642 int actual, count = 0; 5643 int x, y, rc, inum = 0; 5644 5645 AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp, 5646 "ahci_add_legacy_intrs enter"); 5647 5648 /* get number of interrupts. */ 5649 rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_FIXED, &count); 5650 if ((rc != DDI_SUCCESS) || (count == 0)) { 5651 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5652 "ddi_intr_get_nintrs() failed, " 5653 "rc %d count %d\n", rc, count); 5654 return (DDI_FAILURE); 5655 } 5656 5657 /* Allocate an array of interrupt handles. */ 5658 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5659 ahci_ctlp->ahcictl_intr_htable = 5660 kmem_zalloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5661 5662 /* call ddi_intr_alloc(). */ 5663 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5664 DDI_INTR_TYPE_FIXED, 5665 inum, count, &actual, DDI_INTR_ALLOC_STRICT); 5666 5667 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5668 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5669 "ddi_intr_alloc() failed, rc %d\n", rc); 5670 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5671 ahci_ctlp->ahcictl_intr_size); 5672 return (DDI_FAILURE); 5673 } 5674 5675 if (actual < count) { 5676 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5677 "Requested: %d, Received: %d", count, actual); 5678 5679 for (x = 0; x < actual; x++) { 5680 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5681 } 5682 5683 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5684 ahci_ctlp->ahcictl_intr_size); 5685 return (DDI_FAILURE); 5686 } 5687 5688 ahci_ctlp->ahcictl_intr_cnt = actual; 5689 5690 /* Get intr priority. */ 5691 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5692 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5693 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5694 "ddi_intr_get_pri() failed"); 5695 5696 for (x = 0; x < actual; x++) { 5697 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5698 } 5699 5700 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5701 ahci_ctlp->ahcictl_intr_size); 5702 return (DDI_FAILURE); 5703 } 5704 5705 /* Test for high level interrupt. */ 5706 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5707 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5708 "ahci_add_legacy_intrs: Hi level intr not supported"); 5709 5710 for (x = 0; x < actual; x++) { 5711 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5712 } 5713 5714 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5715 sizeof (ddi_intr_handle_t)); 5716 5717 return (DDI_FAILURE); 5718 } 5719 5720 /* Call ddi_intr_add_handler(). */ 5721 for (x = 0; x < actual; x++) { 5722 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x], 5723 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5724 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5725 "ddi_intr_add_handler() failed"); 5726 5727 for (y = 0; y < actual; y++) { 5728 (void) ddi_intr_free( 5729 ahci_ctlp->ahcictl_intr_htable[y]); 5730 } 5731 5732 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5733 ahci_ctlp->ahcictl_intr_size); 5734 return (DDI_FAILURE); 5735 } 5736 } 5737 5738 /* Call ddi_intr_enable() for legacy interrupts. */ 5739 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5740 (void) ddi_intr_enable(ahci_ctlp->ahcictl_intr_htable[x]); 5741 } 5742 5743 return (DDI_SUCCESS); 5744 } 5745 5746 /* 5747 * Handle MSI interrupts. 5748 */ 5749 static int 5750 ahci_add_msi_intrs(ahci_ctl_t *ahci_ctlp) 5751 { 5752 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5753 int count, avail, actual; 5754 int x, y, rc, inum = 0; 5755 5756 AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp, 5757 "ahci_add_msi_intrs enter"); 5758 5759 /* get number of interrupts. */ 5760 rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &count); 5761 if ((rc != DDI_SUCCESS) || (count == 0)) { 5762 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5763 "ddi_intr_get_nintrs() failed, " 5764 "rc %d count %d\n", rc, count); 5765 return (DDI_FAILURE); 5766 } 5767 5768 /* get number of available interrupts. */ 5769 rc = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail); 5770 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5771 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5772 "ddi_intr_get_navail() failed, " 5773 "rc %d avail %d\n", rc, avail); 5774 return (DDI_FAILURE); 5775 } 5776 5777 #if AHCI_DEBUG 5778 if (avail < count) { 5779 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5780 "ddi_intr_get_nvail returned %d, navail() returned %d", 5781 count, avail); 5782 } 5783 #endif 5784 5785 /* Allocate an array of interrupt handles. */ 5786 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5787 ahci_ctlp->ahcictl_intr_htable = 5788 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5789 5790 /* call ddi_intr_alloc(). */ 5791 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5792 DDI_INTR_TYPE_MSI, inum, count, &actual, DDI_INTR_ALLOC_NORMAL); 5793 5794 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5795 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5796 "ddi_intr_alloc() failed, rc %d\n", rc); 5797 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5798 ahci_ctlp->ahcictl_intr_size); 5799 return (DDI_FAILURE); 5800 } 5801 5802 /* use interrupt count returned */ 5803 #if AHCI_DEBUG 5804 if (actual < count) { 5805 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5806 "Requested: %d, Received: %d", count, actual); 5807 } 5808 #endif 5809 5810 ahci_ctlp->ahcictl_intr_cnt = actual; 5811 5812 /* 5813 * Get priority for first msi, assume remaining are all the same. 5814 */ 5815 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5816 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5817 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5818 "ddi_intr_get_pri() failed"); 5819 5820 /* Free already allocated intr. */ 5821 for (y = 0; y < actual; y++) { 5822 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]); 5823 } 5824 5825 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5826 ahci_ctlp->ahcictl_intr_size); 5827 return (DDI_FAILURE); 5828 } 5829 5830 /* Test for high level interrupt. */ 5831 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5832 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5833 "ahci_add_msi_intrs: Hi level intr not supported"); 5834 5835 /* Free already allocated intr. */ 5836 for (y = 0; y < actual; y++) { 5837 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]); 5838 } 5839 5840 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5841 sizeof (ddi_intr_handle_t)); 5842 5843 return (DDI_FAILURE); 5844 } 5845 5846 /* Call ddi_intr_add_handler(). */ 5847 for (x = 0; x < actual; x++) { 5848 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x], 5849 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5850 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5851 "ddi_intr_add_handler() failed"); 5852 5853 /* Free already allocated intr. */ 5854 for (y = 0; y < actual; y++) { 5855 (void) ddi_intr_free( 5856 ahci_ctlp->ahcictl_intr_htable[y]); 5857 } 5858 5859 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5860 ahci_ctlp->ahcictl_intr_size); 5861 return (DDI_FAILURE); 5862 } 5863 } 5864 5865 5866 (void) ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 5867 &ahci_ctlp->ahcictl_intr_cap); 5868 5869 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 5870 /* Call ddi_intr_block_enable() for MSI. */ 5871 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 5872 ahci_ctlp->ahcictl_intr_cnt); 5873 } else { 5874 /* Call ddi_intr_enable() for MSI non block enable. */ 5875 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5876 (void) ddi_intr_enable( 5877 ahci_ctlp->ahcictl_intr_htable[x]); 5878 } 5879 } 5880 5881 return (DDI_SUCCESS); 5882 } 5883 5884 /* 5885 * Removes the registered interrupts irrespective of whether they 5886 * were legacy or MSI. 5887 * 5888 * WARNING!!! The controller interrupts must be disabled before calling 5889 * this routine. 5890 */ 5891 static void 5892 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 5893 { 5894 int x; 5895 5896 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered"); 5897 5898 /* Disable all interrupts. */ 5899 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 5900 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 5901 /* Call ddi_intr_block_disable(). */ 5902 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 5903 ahci_ctlp->ahcictl_intr_cnt); 5904 } else { 5905 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5906 (void) ddi_intr_disable( 5907 ahci_ctlp->ahcictl_intr_htable[x]); 5908 } 5909 } 5910 5911 /* Call ddi_intr_remove_handler(). */ 5912 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5913 (void) ddi_intr_remove_handler( 5914 ahci_ctlp->ahcictl_intr_htable[x]); 5915 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5916 } 5917 5918 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 5919 } 5920 5921 /* 5922 * This routine tries to put port into P:NotRunning state by clearing 5923 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 5924 * and PxCMD.CR to '0'. 5925 * 5926 * WARNING!!! ahciport_mutex should be acquired before the function 5927 * is called. 5928 */ 5929 static int 5930 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 5931 ahci_port_t *ahci_portp, uint8_t port) 5932 { 5933 uint32_t port_cmd_status; 5934 int loop_count; 5935 5936 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5937 "ahci_put_port_into_notrunning_state enter: port %d", port); 5938 5939 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5940 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5941 5942 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 5943 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5944 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5945 5946 /* Wait until PxCMD.CR is cleared */ 5947 loop_count = 0; 5948 do { 5949 port_cmd_status = 5950 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5951 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5952 5953 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 5954 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 5955 "clearing port %d CMD.CR timeout, " 5956 "port_cmd_status = 0x%x", port, 5957 port_cmd_status); 5958 /* 5959 * We are effectively timing out after 0.5 sec. 5960 * This value is specified in AHCI spec. 5961 */ 5962 break; 5963 } 5964 5965 /* Wait for 10 millisec */ 5966 drv_usecwait(10000); 5967 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 5968 5969 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 5970 5971 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 5972 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5973 "ahci_put_port_into_notrunning_state: failed to clear " 5974 "PxCMD.CR to '0' after loop count: %d, and " 5975 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5976 return (AHCI_FAILURE); 5977 } else { 5978 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5979 "ahci_put_port_into_notrunning_state: succeeded to clear " 5980 "PxCMD.CR to '0' after loop count: %d, and " 5981 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5982 return (AHCI_SUCCESS); 5983 } 5984 } 5985 5986 /* 5987 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 5988 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 5989 * stable state, then set PxCMD.ST to '1' to start the port directly. 5990 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 5991 * COMRESET to the device to put it in an idle state. 5992 * 5993 * The fifth argument returns whether the port reset is involved during 5994 * the process. 5995 * 5996 * The routine will be called under six scenarios: 5997 * 1. Initialize the port 5998 * 2. To abort the packet(s) 5999 * 3. To reset the port 6000 * 4. To activate the port 6001 * 5. Fatal error recovery 6002 * 6. To abort the timeout packet(s) 6003 * 6004 * WARNING!!! ahciport_mutex should be acquired before the function 6005 * is called. And ahciport_mutex will be released before the reset 6006 * event is reported to sata module by calling sata_hba_event_notify, 6007 * and then be acquired again later. 6008 * 6009 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6010 * according to the spec, the clearance of three bits will also clear 6011 * three interrupt status bits. 6012 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6013 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6014 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6015 * 6016 * Among these three interrupt events, the driver needs to take care of 6017 * PxIS.PRCS, which is the hot plug event. When the driver found out 6018 * a device was unplugged, it will call the interrupt handler. 6019 */ 6020 static int 6021 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6022 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6023 { 6024 uint32_t port_sstatus; 6025 uint32_t task_file_status; 6026 sata_device_t sdevice; 6027 int rval; 6028 int dev_exists_begin = 0; 6029 int dev_exists_end = 0; 6030 6031 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6032 "ahci_restart_port_wait_till_ready: port %d enter", port); 6033 6034 if ((flag != AHCI_PORT_INIT) && 6035 (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)) 6036 dev_exists_begin = 1; 6037 6038 /* First clear PxCMD.ST */ 6039 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6040 port); 6041 if (rval != AHCI_SUCCESS) 6042 /* 6043 * If PxCMD.CR does not clear within a reasonable time, it 6044 * may assume the interface is in a hung condition and may 6045 * continue with issuing the port reset. 6046 */ 6047 goto reset; 6048 6049 /* Then clear PxSERR */ 6050 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6051 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6052 AHCI_SERROR_CLEAR_ALL); 6053 6054 /* Then get PxTFD */ 6055 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6056 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6057 6058 /* 6059 * Check whether the device is in a stable status, if yes, 6060 * then start the port directly. However for ahci_tran_dport_reset, 6061 * we may have to perform a port reset. 6062 */ 6063 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6064 !(flag & AHCI_PORT_RESET)) 6065 goto out; 6066 6067 reset: 6068 /* 6069 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6070 * a COMRESET to the device 6071 */ 6072 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6073 6074 if (reset_flag != NULL) 6075 *reset_flag = 1; 6076 6077 /* Indicate to the framework that a reset has happened. */ 6078 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6079 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6080 /* Set the reset in progress flag */ 6081 ahci_portp->ahciport_reset_in_progress = 1; 6082 6083 bzero((void *)&sdevice, sizeof (sata_device_t)); 6084 sdevice.satadev_addr.cport = 6085 ahci_ctlp->ahcictl_port_to_cport[port]; 6086 sdevice.satadev_addr.pmport = 0; 6087 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6088 6089 sdevice.satadev_state = SATA_DSTATE_RESET | 6090 SATA_DSTATE_PWR_ACTIVE; 6091 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6092 mutex_exit(&ahci_portp->ahciport_mutex); 6093 sata_hba_event_notify( 6094 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6095 &sdevice, 6096 SATA_EVNT_DEVICE_RESET); 6097 mutex_enter(&ahci_portp->ahciport_mutex); 6098 } 6099 6100 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 6101 "port %d sending event up: SATA_EVNT_RESET", port); 6102 } else { 6103 ahci_portp->ahciport_reset_in_progress = 0; 6104 } 6105 6106 out: 6107 /* Start the port if not in initialization phase */ 6108 if (flag & AHCI_PORT_INIT) 6109 return (rval); 6110 6111 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6112 6113 /* SStatus tells the presence of device. */ 6114 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6115 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6116 6117 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6118 dev_exists_end = 1; 6119 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6120 } 6121 6122 /* Check whether a hot plug event happened */ 6123 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6124 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6125 "ahci_restart_port_wait_till_ready: port %d " 6126 "device is removed", port); 6127 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6128 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6129 "ahci_restart_port_wait_till_ready: port %d " 6130 "AHCI_PORT_FLAG_NODEV flag is set", port); 6131 mutex_exit(&ahci_portp->ahciport_mutex); 6132 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6133 mutex_enter(&ahci_portp->ahciport_mutex); 6134 } 6135 6136 return (rval); 6137 } 6138 6139 /* 6140 * This routine may be called under four scenarios: 6141 * a) do the recovery from fatal error 6142 * b) or we need to timeout some commands 6143 * c) or we need to abort some commands 6144 * d) or we need reset device/port/controller 6145 * 6146 * In all these scenarios, we need to send any pending unfinished 6147 * commands up to sata framework. 6148 * 6149 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6150 */ 6151 static void 6152 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6153 ahci_port_t *ahci_portp, 6154 uint32_t slot_status, 6155 uint32_t failed_tags, 6156 uint32_t timeout_tags, 6157 uint32_t aborted_tags, 6158 uint32_t reset_tags) 6159 { 6160 uint32_t finished_tags = 0; 6161 uint32_t unfinished_tags = 0; 6162 int tmp_slot; 6163 sata_pkt_t *satapkt; 6164 int ncq_cmd_in_progress = 0; 6165 int err_retri_cmd_in_progress = 0; 6166 6167 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6168 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6169 ahci_portp->ahciport_port_num, slot_status); 6170 6171 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6172 "ahci_mop_commands: failed_tags: 0x%x, " 6173 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6174 "reset_tags: 0x%x", failed_tags, 6175 timeout_tags, aborted_tags, reset_tags); 6176 6177 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6178 finished_tags = ahci_portp->ahciport_pending_tags & 6179 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6180 6181 unfinished_tags = slot_status & 6182 AHCI_SLOT_MASK(ahci_ctlp) & 6183 ~failed_tags & 6184 ~aborted_tags & 6185 ~reset_tags & 6186 ~timeout_tags; 6187 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6188 ncq_cmd_in_progress = 1; 6189 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6190 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6191 6192 unfinished_tags = slot_status & 6193 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6194 ~failed_tags & 6195 ~aborted_tags & 6196 ~reset_tags & 6197 ~timeout_tags; 6198 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6199 6200 /* 6201 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6202 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6203 * complete successfully due to one of the following three 6204 * conditions: 6205 * 6206 * 1. Fatal error - failed_tags includes its slot 6207 * 2. Timed out - timeout_tags includes its slot 6208 * 3. Aborted when hot unplug - aborted_tags includes its 6209 * slot 6210 * 6211 * Please note that the command is always sent down in Slot 0 6212 */ 6213 err_retri_cmd_in_progress = 1; 6214 AHCIDBG1(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6215 "ahci_mop_commands is called for port %d while " 6216 "REQUEST SENSE or READ LOG EXT for error retrieval " 6217 "is being executed", ahci_portp->ahciport_port_num); 6218 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6219 ASSERT(slot_status == 0x1); 6220 } 6221 6222 /* Send up finished packets with SATA_PKT_COMPLETED */ 6223 while (finished_tags) { 6224 tmp_slot = ddi_ffs(finished_tags) - 1; 6225 if (tmp_slot == -1) { 6226 break; 6227 } 6228 6229 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6230 ASSERT(satapkt != NULL); 6231 6232 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6233 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6234 (void *)satapkt); 6235 6236 /* 6237 * Cannot fetch the return register content since the port 6238 * was restarted, so the corresponding tag will be set to 6239 * aborted tags. 6240 */ 6241 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6242 CLEAR_BIT(finished_tags, tmp_slot); 6243 aborted_tags |= tmp_slot; 6244 continue; 6245 } 6246 6247 if (ncq_cmd_in_progress) 6248 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6249 tmp_slot); 6250 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6251 CLEAR_BIT(finished_tags, tmp_slot); 6252 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6253 6254 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6255 } 6256 6257 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6258 while (failed_tags) { 6259 if (err_retri_cmd_in_progress) { 6260 satapkt = ahci_portp->ahciport_err_retri_pkt; 6261 ASSERT(satapkt != NULL); 6262 ASSERT(failed_tags == 0x1); 6263 6264 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6265 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6266 (void *)satapkt); 6267 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6268 break; 6269 } 6270 6271 tmp_slot = ddi_ffs(failed_tags) - 1; 6272 if (tmp_slot == -1) { 6273 break; 6274 } 6275 6276 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6277 ASSERT(satapkt != NULL); 6278 6279 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6280 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6281 (void *)satapkt); 6282 6283 if (ncq_cmd_in_progress) 6284 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6285 tmp_slot); 6286 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6287 CLEAR_BIT(failed_tags, tmp_slot); 6288 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6289 6290 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6291 } 6292 6293 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6294 while (timeout_tags) { 6295 if (err_retri_cmd_in_progress) { 6296 satapkt = ahci_portp->ahciport_err_retri_pkt; 6297 ASSERT(satapkt != NULL); 6298 ASSERT(timeout_tags == 0x1); 6299 6300 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6301 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6302 (void *)satapkt); 6303 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6304 break; 6305 } 6306 6307 tmp_slot = ddi_ffs(timeout_tags) - 1; 6308 if (tmp_slot == -1) { 6309 break; 6310 } 6311 6312 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6313 ASSERT(satapkt != NULL); 6314 6315 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6316 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6317 (void *)satapkt); 6318 6319 if (ncq_cmd_in_progress) 6320 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6321 tmp_slot); 6322 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6323 CLEAR_BIT(timeout_tags, tmp_slot); 6324 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6325 6326 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6327 } 6328 6329 /* Send up aborted packets with SATA_PKT_ABORTED */ 6330 while (aborted_tags) { 6331 if (err_retri_cmd_in_progress) { 6332 satapkt = ahci_portp->ahciport_err_retri_pkt; 6333 ASSERT(satapkt != NULL); 6334 ASSERT(aborted_tags == 0x1); 6335 6336 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6337 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6338 (void *)satapkt); 6339 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6340 break; 6341 } 6342 6343 tmp_slot = ddi_ffs(aborted_tags) - 1; 6344 if (tmp_slot == -1) { 6345 break; 6346 } 6347 6348 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6349 ASSERT(satapkt != NULL); 6350 6351 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6352 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6353 (void *)satapkt); 6354 6355 if (ncq_cmd_in_progress) 6356 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6357 tmp_slot); 6358 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6359 CLEAR_BIT(aborted_tags, tmp_slot); 6360 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6361 6362 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6363 } 6364 6365 /* Send up reset packets with SATA_PKT_RESET. */ 6366 while (reset_tags) { 6367 tmp_slot = ddi_ffs(reset_tags) - 1; 6368 if (tmp_slot == -1) { 6369 break; 6370 } 6371 6372 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6373 ASSERT(satapkt != NULL); 6374 6375 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6376 "sending up pkt 0x%p with SATA_PKT_RESET", 6377 (void *)satapkt); 6378 6379 if (ncq_cmd_in_progress) 6380 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6381 tmp_slot); 6382 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6383 CLEAR_BIT(reset_tags, tmp_slot); 6384 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6385 6386 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6387 } 6388 6389 /* Send up unfinished packets with SATA_PKT_RESET */ 6390 while (unfinished_tags) { 6391 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6392 if (tmp_slot == -1) { 6393 break; 6394 } 6395 6396 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6397 ASSERT(satapkt != NULL); 6398 6399 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6400 "sending up pkt 0x%p with SATA_PKT_RESET", 6401 (void *)satapkt); 6402 6403 if (ncq_cmd_in_progress) 6404 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6405 tmp_slot); 6406 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6407 CLEAR_BIT(unfinished_tags, tmp_slot); 6408 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6409 6410 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6411 } 6412 6413 ahci_portp->ahciport_mop_in_progress--; 6414 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6415 6416 if (ahci_portp->ahciport_mop_in_progress == 0) 6417 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6418 } 6419 6420 /* 6421 * This routine is going to first request a READ LOG EXT sata pkt from sata 6422 * module, and then deliver it to the HBA to get the ncq failure context. 6423 * The return value is the exactly failed tags. 6424 */ 6425 static uint32_t 6426 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6427 uint8_t port) 6428 { 6429 sata_device_t sdevice; 6430 sata_pkt_t *rdlog_spkt, *spkt; 6431 ddi_dma_handle_t buf_dma_handle; 6432 int loop_count; 6433 int rval; 6434 int failed_slot; 6435 uint32_t failed_tags = 0; 6436 struct sata_ncq_error_recovery_page *ncq_err_page; 6437 6438 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6439 "ahci_get_rdlogext_data enter: port %d", port); 6440 6441 /* Prepare the sdevice data */ 6442 bzero((void *)&sdevice, sizeof (sata_device_t)); 6443 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6444 6445 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6446 sdevice.satadev_addr.pmport = 0; 6447 6448 /* 6449 * Call the sata hba interface to get a rdlog spkt 6450 */ 6451 loop_count = 0; 6452 loop: 6453 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6454 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6455 if (rdlog_spkt == NULL) { 6456 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6457 /* Sleep for a while */ 6458 delay(AHCI_10MS_TICKS); 6459 goto loop; 6460 } 6461 /* Timed out after 1s */ 6462 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6463 "failed to get rdlog spkt for port %d", port); 6464 return (failed_tags); 6465 } 6466 6467 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6468 6469 /* 6470 * This flag is used to handle the specific error recovery when the 6471 * READ LOG EXT command gets a failure (fatal error or time-out). 6472 */ 6473 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6474 6475 /* 6476 * This start is not supposed to fail because after port is restarted, 6477 * the whole command list is empty. 6478 */ 6479 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6480 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6481 ahci_portp->ahciport_err_retri_pkt = NULL; 6482 6483 /* Remove the flag after READ LOG EXT command is completed */ 6484 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6485 6486 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6487 /* Update the request log data */ 6488 buf_dma_handle = *(ddi_dma_handle_t *) 6489 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6490 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6491 DDI_DMA_SYNC_FORKERNEL); 6492 if (rval == DDI_SUCCESS) { 6493 ncq_err_page = 6494 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6495 satapkt_cmd.satacmd_bp->b_un.b_addr; 6496 6497 /* Get the failed tag */ 6498 failed_slot = ncq_err_page->ncq_tag; 6499 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 6500 "ahci_get_rdlogext_data: port %d " 6501 "failed slot %d", port, failed_slot); 6502 if (failed_slot & NQ) { 6503 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6504 "the failed slot is not a valid tag"); 6505 goto out; 6506 } 6507 6508 failed_slot &= NCQ_TAG_MASK; 6509 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6510 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6511 "ahci_get_rdlogext_data: failed spkt 0x%p", 6512 (void *)spkt); 6513 if (spkt == NULL) { 6514 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6515 "the failed slot spkt is NULL") 6516 goto out; 6517 } 6518 6519 failed_tags = 0x1 << failed_slot; 6520 6521 /* Fill out the error context */ 6522 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6523 ncq_err_page); 6524 ahci_update_sata_registers(ahci_ctlp, port, 6525 &spkt->satapkt_device); 6526 } 6527 } 6528 out: 6529 sata_free_error_retrieval_pkt(rdlog_spkt); 6530 6531 return (failed_tags); 6532 } 6533 6534 /* 6535 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6536 * module, and then deliver it to the HBA to get the sense data and copy 6537 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6538 * SENSE sata pkt later. 6539 */ 6540 static void 6541 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6542 uint8_t port, sata_pkt_t *spkt) 6543 { 6544 sata_device_t sdevice; 6545 sata_pkt_t *rs_spkt; 6546 sata_cmd_t *sata_cmd; 6547 ddi_dma_handle_t buf_dma_handle; 6548 int loop_count; 6549 #if AHCI_DEBUG 6550 struct scsi_extended_sense *rqsense; 6551 #endif 6552 6553 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6554 "ahci_get_rqsense_data enter: port %d", port); 6555 6556 /* Prepare the sdevice data */ 6557 bzero((void *)&sdevice, sizeof (sata_device_t)); 6558 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6559 6560 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6561 sdevice.satadev_addr.pmport = 0; 6562 6563 sata_cmd = &spkt->satapkt_cmd; 6564 6565 /* 6566 * Call the sata hba interface to get a rs spkt 6567 */ 6568 loop_count = 0; 6569 loop: 6570 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6571 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6572 if (rs_spkt == NULL) { 6573 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6574 /* Sleep for a while */ 6575 delay(AHCI_10MS_TICKS); 6576 goto loop; 6577 6578 } 6579 /* Timed out after 1s */ 6580 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6581 "failed to get rs spkt for port %d", port); 6582 return; 6583 } 6584 6585 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6586 6587 /* 6588 * This flag is used to handle the specific error recovery when the 6589 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6590 */ 6591 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6592 6593 /* 6594 * This start is not supposed to fail because after port is restarted, 6595 * the whole command list is empty. 6596 */ 6597 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6598 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6599 ahci_portp->ahciport_err_retri_pkt = NULL; 6600 6601 /* Remove the flag after REQUEST SENSE command is completed */ 6602 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6603 6604 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6605 /* Update the request sense data */ 6606 buf_dma_handle = *(ddi_dma_handle_t *) 6607 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6608 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6609 DDI_DMA_SYNC_FORKERNEL); 6610 /* Copy the request sense data */ 6611 bcopy(rs_spkt-> 6612 satapkt_cmd.satacmd_bp->b_un.b_addr, 6613 &sata_cmd->satacmd_rqsense, 6614 SATA_ATAPI_MIN_RQSENSE_LEN); 6615 #if AHCI_DEBUG 6616 rqsense = (struct scsi_extended_sense *) 6617 sata_cmd->satacmd_rqsense; 6618 6619 /* Dump the sense data */ 6620 AHCIDBG0(AHCIDBG_SENSEDATA, ahci_ctlp, "\n"); 6621 AHCIDBG2(AHCIDBG_SENSEDATA, ahci_ctlp, 6622 "Sense data for satapkt %p ATAPI cmd 0x%x", 6623 spkt, sata_cmd->satacmd_acdb[0]); 6624 AHCIDBG5(AHCIDBG_SENSEDATA, ahci_ctlp, 6625 " es_code 0x%x es_class 0x%x " 6626 "es_key 0x%x es_add_code 0x%x " 6627 "es_qual_code 0x%x", 6628 rqsense->es_code, rqsense->es_class, 6629 rqsense->es_key, rqsense->es_add_code, 6630 rqsense->es_qual_code); 6631 #endif 6632 } 6633 6634 sata_free_error_retrieval_pkt(rs_spkt); 6635 } 6636 6637 /* 6638 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6639 * the port must be restarted. When the HBA detects thus error, it may try 6640 * to abort a transfer. And if the transfer was aborted, the device is 6641 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6642 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6643 * that the device is in a stable status and transfers may be restarted without 6644 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6645 * then the software will send the COMRESET to do the port reset. 6646 * 6647 * Software should perform the appropriate error recovery actions based on 6648 * whether non-queued commands were being issued or natived command queuing 6649 * commands were being issued. 6650 * 6651 * And software will complete the command that had the error with error mark 6652 * to higher level software. 6653 * 6654 * Fatal errors include the following: 6655 * PxIS.IFS - Interface Fatal Error Status 6656 * PxIS.HBDS - Host Bus Data Error Status 6657 * PxIS.HBFS - Host Bus Fatal Error Status 6658 * PxIS.TFES - Task File Error Status 6659 * 6660 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6661 */ 6662 static void 6663 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6664 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6665 { 6666 uint32_t port_cmd_status; 6667 uint32_t slot_status = 0; 6668 uint32_t failed_tags = 0; 6669 int failed_slot; 6670 int reset_flag = 0; 6671 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6672 sata_cmd_t *sata_cmd = NULL; 6673 sata_pkt_t *spkt = NULL; 6674 #if AHCI_DEBUG 6675 ahci_cmd_header_t *cmd_header; 6676 #endif 6677 6678 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6679 "ahci_fatal_error_recovery_handler enter: port %d", port); 6680 6681 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6682 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6683 6684 /* Read PxCI to see which commands are still outstanding */ 6685 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6686 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6687 6688 /* 6689 * Read PxCMD.CCS to determine the slot that the HBA 6690 * was processing when the error occurred. 6691 */ 6692 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6693 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6694 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6695 AHCI_CMD_STATUS_CCS_SHIFT; 6696 6697 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6698 spkt = ahci_portp->ahciport_err_retri_pkt; 6699 ASSERT(spkt != NULL); 6700 } else { 6701 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6702 if (spkt == NULL) { 6703 /* May happen when interface errors occur? */ 6704 goto next; 6705 } 6706 } 6707 6708 #if AHCI_DEBUG 6709 /* 6710 * Debugging purpose... 6711 */ 6712 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6713 cmd_header = 6714 &ahci_portp->ahciport_cmd_list[failed_slot]; 6715 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6716 "ahci_fatal_error_recovery_handler: port %d, " 6717 "PRD Byte Count = 0x%x, " 6718 "ahciport_prd_bytecounts = 0x%x", port, 6719 cmd_header->ahcich_prd_byte_count, 6720 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6721 } 6722 #endif 6723 6724 sata_cmd = &spkt->satapkt_cmd; 6725 6726 /* Fill out the status and error registers for PxIS.TFES */ 6727 if (intr_status & AHCI_INTR_STATUS_TFES) { 6728 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6729 ahcirf_d2h_register_fis); 6730 6731 /* Copy the error context back to the sata_cmd */ 6732 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6733 } 6734 6735 /* The failed command must be one of the outstanding commands */ 6736 failed_tags = 0x1 << failed_slot; 6737 ASSERT(failed_tags & slot_status); 6738 6739 /* Update the sata registers, especially PxSERR register */ 6740 ahci_update_sata_registers(ahci_ctlp, port, 6741 &spkt->satapkt_device); 6742 6743 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6744 /* Read PxSACT to see which commands are still outstanding */ 6745 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6746 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6747 } 6748 next: 6749 6750 #if AHCI_DEBUG 6751 /* 6752 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6753 * set, it means a fatal error happened after REQUEST SENSE command 6754 * or READ LOG EXT command is delivered to the HBA during the error 6755 * recovery process. At this time, the only outstanding command is 6756 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6757 */ 6758 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6759 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6760 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6761 "command or READ LOG EXT command for error data retrieval " 6762 "failed", port); 6763 ASSERT(slot_status == 0x1); 6764 ASSERT(failed_slot == 0x1); 6765 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6766 SCMD_REQUEST_SENSE || 6767 spkt->satapkt_cmd.satacmd_cmd_reg == 6768 SATAC_READ_LOG_EXT); 6769 } 6770 #endif 6771 6772 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6773 ahci_portp->ahciport_mop_in_progress++; 6774 6775 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6776 port, NULL, &reset_flag); 6777 6778 /* 6779 * Won't retrieve error information: 6780 * 1. Port reset was involved to recover 6781 * 2. Device is gone 6782 * 3. IDENTIFY DEVICE command sent to ATAPI device 6783 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6784 */ 6785 if (reset_flag || 6786 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6787 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6788 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6789 goto out; 6790 6791 /* 6792 * Deliver READ LOG EXT to gather information about the error when 6793 * a COMRESET has not been performed as part of the error recovery 6794 * during NCQ command processing. 6795 */ 6796 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6797 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6798 ahci_portp, port); 6799 goto out; 6800 } 6801 6802 /* 6803 * Deliver REQUEST SENSE for ATAPI command to gather information about 6804 * the error when a COMRESET has not been performed as part of the 6805 * error recovery. 6806 */ 6807 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6808 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6809 out: 6810 AHCIDBG5(AHCIDBG_ERRS, ahci_ctlp, 6811 "ahci_fatal_error_recovery_handler: port %d fatal error " 6812 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6813 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6814 port, slot_status, ahci_portp->ahciport_pending_tags, 6815 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6816 6817 ahci_mop_commands(ahci_ctlp, 6818 ahci_portp, 6819 slot_status, 6820 failed_tags, /* failed tags */ 6821 0, /* timeout tags */ 6822 0, /* aborted tags */ 6823 0); /* reset tags */ 6824 } 6825 6826 /* 6827 * Handle events - fatal error recovery 6828 */ 6829 static void 6830 ahci_events_handler(void *args) 6831 { 6832 ahci_event_arg_t *ahci_event_arg; 6833 ahci_ctl_t *ahci_ctlp; 6834 ahci_port_t *ahci_portp; 6835 uint32_t event; 6836 uint8_t port; 6837 6838 ahci_event_arg = (ahci_event_arg_t *)args; 6839 6840 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 6841 ahci_portp = ahci_event_arg->ahciea_portp; 6842 event = ahci_event_arg->ahciea_event; 6843 port = ahci_portp->ahciport_port_num; 6844 6845 AHCIDBG2(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6846 "ahci_events_handler enter: port %d intr_status = 0x%x", 6847 port, event); 6848 6849 mutex_enter(&ahci_portp->ahciport_mutex); 6850 6851 /* 6852 * ahci_intr_phyrdy_change() may have rendered it to 6853 * SATA_DTYPE_NONE. 6854 */ 6855 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 6856 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6857 "ahci_events_handler: port %d no device attached, " 6858 "and just return without doing anything", port); 6859 goto out; 6860 } 6861 6862 if (event & (AHCI_INTR_STATUS_IFS | 6863 AHCI_INTR_STATUS_HBDS | 6864 AHCI_INTR_STATUS_HBFS | 6865 AHCI_INTR_STATUS_TFES)) 6866 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 6867 port, event); 6868 6869 out: 6870 mutex_exit(&ahci_portp->ahciport_mutex); 6871 } 6872 6873 /* 6874 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 6875 * detect there are some commands which are timed out. 6876 */ 6877 static void 6878 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6879 uint8_t port, uint32_t tmp_timeout_tags) 6880 { 6881 uint32_t slot_status = 0; 6882 uint32_t finished_tags = 0; 6883 uint32_t timeout_tags = 0; 6884 6885 AHCIDBG1(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6886 "ahci_timeout_pkts enter: port %d", port); 6887 6888 mutex_enter(&ahci_portp->ahciport_mutex); 6889 6890 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6891 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6892 /* Read PxCI to see which commands are still outstanding */ 6893 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6894 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6895 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6896 /* Read PxSACT to see which commands are still outstanding */ 6897 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6898 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6899 } 6900 6901 #if AHCI_DEBUG 6902 /* 6903 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6904 * set, it means a fatal error happened after REQUEST SENSE command 6905 * or READ LOG EXT command is delivered to the HBA during the error 6906 * recovery process. At this time, the only outstanding command is 6907 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6908 */ 6909 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6910 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 6911 "ahci_timeout_pkts called while REQUEST SENSE " 6912 "command or READ LOG EXT command for error recovery " 6913 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 6914 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 6915 tmp_timeout_tags, slot_status, 6916 ahci_portp->ahciport_pending_tags, 6917 ahci_portp->ahciport_pending_ncq_tags); 6918 ASSERT(slot_status == 0x1); 6919 } 6920 #endif 6921 6922 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6923 ahci_portp->ahciport_mop_in_progress++; 6924 6925 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6926 port, NULL, NULL); 6927 6928 /* 6929 * Re-identify timeout tags because some previously checked commands 6930 * could already complete. 6931 */ 6932 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6933 finished_tags = ahci_portp->ahciport_pending_tags & 6934 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6935 timeout_tags = tmp_timeout_tags & ~finished_tags; 6936 6937 AHCIDBG5(AHCIDBG_TIMEOUT, ahci_ctlp, 6938 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6939 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 6940 "pending_tags = 0x%x ", 6941 port, finished_tags, timeout_tags, 6942 slot_status, ahci_portp->ahciport_pending_tags); 6943 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6944 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6945 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6946 timeout_tags = tmp_timeout_tags & ~finished_tags; 6947 6948 AHCIDBG5(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 6949 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6950 "timeout_tags = 0x%x, port_sactive = 0x%x, " 6951 "pending_ncq_tags = 0x%x ", 6952 port, finished_tags, timeout_tags, 6953 slot_status, ahci_portp->ahciport_pending_ncq_tags); 6954 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6955 timeout_tags = tmp_timeout_tags; 6956 } 6957 6958 ahci_mop_commands(ahci_ctlp, 6959 ahci_portp, 6960 slot_status, 6961 0, /* failed tags */ 6962 timeout_tags, /* timeout tags */ 6963 0, /* aborted tags */ 6964 0); /* reset tags */ 6965 6966 mutex_exit(&ahci_portp->ahciport_mutex); 6967 } 6968 6969 /* 6970 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 6971 * for long time. 6972 */ 6973 static void 6974 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 6975 { 6976 ahci_port_t *ahci_portp; 6977 sata_pkt_t *spkt; 6978 uint32_t pending_tags; 6979 uint32_t timeout_tags; 6980 uint32_t port_cmd_status; 6981 uint32_t port_sactive; 6982 uint8_t port; 6983 int tmp_slot; 6984 int current_slot; 6985 uint32_t current_tags; 6986 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 6987 /* max number of cycles this packet should survive */ 6988 int max_life_cycles; 6989 6990 /* how many cycles this packet survived so far */ 6991 int watched_cycles; 6992 6993 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6994 6995 AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6996 "ahci_watchdog_handler entered"); 6997 6998 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6999 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7000 continue; 7001 } 7002 7003 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7004 7005 mutex_enter(&ahci_portp->ahciport_mutex); 7006 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7007 mutex_exit(&ahci_portp->ahciport_mutex); 7008 continue; 7009 } 7010 7011 /* Skip the check for those ports in error recovery */ 7012 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7013 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7014 mutex_exit(&ahci_portp->ahciport_mutex); 7015 continue; 7016 } 7017 7018 pending_tags = 0; 7019 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7020 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7021 7022 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7023 current_slot = 0; 7024 pending_tags = 0x1; 7025 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7026 current_slot = 7027 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7028 AHCI_CMD_STATUS_CCS_SHIFT; 7029 pending_tags = ahci_portp->ahciport_pending_tags; 7030 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7031 port_sactive = ddi_get32( 7032 ahci_ctlp->ahcictl_ahci_acc_handle, 7033 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7034 current_tags = port_sactive & 7035 ~port_cmd_status & 7036 AHCI_NCQ_SLOT_MASK(ahci_portp); 7037 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7038 } 7039 7040 timeout_tags = 0; 7041 while (pending_tags) { 7042 tmp_slot = ddi_ffs(pending_tags) - 1; 7043 if (tmp_slot == -1) { 7044 break; 7045 } 7046 7047 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7048 spkt = ahci_portp->ahciport_err_retri_pkt; 7049 else 7050 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7051 7052 if ((spkt != NULL) && spkt->satapkt_time && 7053 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7054 /* 7055 * We are overloading satapkt_hba_driver_private 7056 * with watched_cycle count. 7057 * 7058 * If a packet has survived for more than it's 7059 * max life cycles, it is a candidate for time 7060 * out. 7061 */ 7062 watched_cycles = (int)(intptr_t) 7063 spkt->satapkt_hba_driver_private; 7064 watched_cycles++; 7065 max_life_cycles = (spkt->satapkt_time + 7066 ahci_watchdog_timeout - 1) / 7067 ahci_watchdog_timeout; 7068 7069 spkt->satapkt_hba_driver_private = 7070 (void *)(intptr_t)watched_cycles; 7071 7072 if (watched_cycles <= max_life_cycles) 7073 goto next; 7074 7075 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 7076 "the current slot is %d", current_slot); 7077 /* 7078 * We need to check whether the HBA has 7079 * begun to execute the command, if not, 7080 * then re-set the timer of the command. 7081 */ 7082 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7083 (tmp_slot != current_slot) || 7084 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7085 ((0x1 << tmp_slot) & current_tags)) { 7086 spkt->satapkt_hba_driver_private = 7087 (void *)(intptr_t)0; 7088 } else { 7089 timeout_tags |= (0x1 << tmp_slot); 7090 cmn_err(CE_WARN, "!ahci%d: watchdog " 7091 "port %d satapkt 0x%p timed out\n", 7092 instance, port, (void *)spkt); 7093 } 7094 } 7095 next: 7096 CLEAR_BIT(pending_tags, tmp_slot); 7097 } 7098 7099 if (timeout_tags) { 7100 mutex_exit(&ahci_portp->ahciport_mutex); 7101 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7102 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7103 port, timeout_tags); 7104 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7105 mutex_enter(&ahci_portp->ahciport_mutex); 7106 } 7107 7108 mutex_exit(&ahci_portp->ahciport_mutex); 7109 } 7110 7111 /* Re-install the watchdog timeout handler */ 7112 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7113 ahci_ctlp->ahcictl_timeout_id = 7114 timeout((void (*)(void *))ahci_watchdog_handler, 7115 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7116 } 7117 7118 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7119 } 7120 7121 /* 7122 * Fill the error context into sata_cmd for non-queued command error. 7123 */ 7124 static void 7125 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7126 { 7127 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7128 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7129 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7130 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7131 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7132 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7133 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7134 7135 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7136 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7137 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7138 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7139 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7140 } 7141 } 7142 7143 /* 7144 * Fill the ncq error page into sata_cmd for queued command error. 7145 */ 7146 static void 7147 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7148 struct sata_ncq_error_recovery_page *ncq_err_page) 7149 { 7150 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7151 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7152 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7153 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7154 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7155 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7156 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7157 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7158 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7159 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7160 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7161 } 7162 7163 /* 7164 * Put the respective register value to sata_cmd_t for satacmd_flags. 7165 */ 7166 static void 7167 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7168 { 7169 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7170 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7171 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7172 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7173 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7174 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7175 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7176 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7177 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7178 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7179 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7180 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7181 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7182 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7183 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7184 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7185 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7186 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7187 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7188 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7189 } 7190 7191 static void 7192 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7193 uint32_t intr_status) 7194 { 7195 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7196 7197 if (intr_status & AHCI_INTR_STATUS_IFS) 7198 cmn_err(CE_WARN, "ahci%d: ahci port %d has interface fatal " 7199 "error", instance, port); 7200 7201 if (intr_status & AHCI_INTR_STATUS_HBDS) 7202 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus data error", 7203 instance, port); 7204 7205 if (intr_status & AHCI_INTR_STATUS_HBFS) 7206 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus fatal error", 7207 instance, port); 7208 7209 if (intr_status & AHCI_INTR_STATUS_TFES) 7210 cmn_err(CE_WARN, "ahci%d: ahci port %d has task file error", 7211 instance, port); 7212 7213 cmn_err(CE_WARN, "ahci%d: ahci port %d is trying to do error " 7214 "recovery", instance, port); 7215 } 7216 7217 /* 7218 * Dump the serror message to the log. 7219 */ 7220 static void 7221 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7222 uint32_t port_serror, int debug_only) 7223 { 7224 static char err_buf[512]; 7225 static char err_msg_header[16]; 7226 char *err_msg = err_buf; 7227 7228 *err_buf = '\0'; 7229 *err_msg_header = '\0'; 7230 7231 if (port_serror & SERROR_DATA_ERR_FIXED) { 7232 err_msg = strcat(err_msg, 7233 "\tRecovered Data Integrity Error (I)\n"); 7234 } 7235 7236 if (port_serror & SERROR_COMM_ERR_FIXED) { 7237 err_msg = strcat(err_msg, 7238 "\tRecovered Communication Error (M)\n"); 7239 } 7240 7241 if (port_serror & SERROR_DATA_ERR) { 7242 err_msg = strcat(err_msg, 7243 "\tTransient Data Integrity Error (T)\n"); 7244 } 7245 7246 if (port_serror & SERROR_PERSISTENT_ERR) { 7247 err_msg = strcat(err_msg, 7248 "\tPersistent Communication or Data Integrity Error (C)\n"); 7249 } 7250 7251 if (port_serror & SERROR_PROTOCOL_ERR) { 7252 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7253 } 7254 7255 if (port_serror & SERROR_INT_ERR) { 7256 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7257 } 7258 7259 if (port_serror & SERROR_PHY_RDY_CHG) { 7260 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7261 } 7262 7263 if (port_serror & SERROR_PHY_INT_ERR) { 7264 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7265 } 7266 7267 if (port_serror & SERROR_COMM_WAKE) { 7268 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7269 } 7270 7271 if (port_serror & SERROR_10B_TO_8B_ERR) { 7272 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7273 } 7274 7275 if (port_serror & SERROR_DISPARITY_ERR) { 7276 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7277 } 7278 7279 if (port_serror & SERROR_CRC_ERR) { 7280 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7281 } 7282 7283 if (port_serror & SERROR_HANDSHAKE_ERR) { 7284 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7285 } 7286 7287 if (port_serror & SERROR_LINK_SEQ_ERR) { 7288 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7289 } 7290 7291 if (port_serror & SERROR_TRANS_ERR) { 7292 err_msg = strcat(err_msg, 7293 "\tTransport state transition error (T)\n"); 7294 } 7295 7296 if (port_serror & SERROR_FIS_TYPE) { 7297 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7298 } 7299 7300 if (port_serror & SERROR_EXCHANGED_ERR) { 7301 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7302 } 7303 7304 if (err_msg == NULL) 7305 return; 7306 7307 if (debug_only) { 7308 (void) sprintf(err_msg_header, "port %d", port); 7309 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg_header); 7310 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg); 7311 } else if (ahci_ctlp) { 7312 cmn_err(CE_WARN, "ahci%d: %s %s", 7313 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7314 err_msg_header, err_msg); 7315 } else { 7316 cmn_err(CE_WARN, "ahci: %s %s", err_msg_header, err_msg); 7317 } 7318 } 7319 7320 /* 7321 * This routine is to calculate the total number of ports implemented 7322 * by the HBA. 7323 */ 7324 static int 7325 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7326 { 7327 uint8_t i; 7328 int num = 0; 7329 7330 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7331 if (((uint32_t)0x1 << i) & ports_implemented) 7332 num++; 7333 } 7334 7335 return (num); 7336 } 7337 7338 #if AHCI_DEBUG 7339 static void 7340 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7341 { 7342 static char name[16]; 7343 va_list ap; 7344 7345 mutex_enter(&ahci_log_mutex); 7346 7347 va_start(ap, fmt); 7348 if (ahci_ctlp) { 7349 (void) sprintf(name, "ahci%d: ", 7350 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7351 } else { 7352 (void) sprintf(name, "ahci: "); 7353 } 7354 7355 (void) vsprintf(ahci_log_buf, fmt, ap); 7356 va_end(ap); 7357 7358 cmn_err(level, "%s%s", name, ahci_log_buf); 7359 7360 mutex_exit(&ahci_log_mutex); 7361 } 7362 #endif 7363 7364 /* 7365 * quiesce(9E) entry point. 7366 * 7367 * This function is called when the system is single-threaded at high 7368 * PIL with preemption disabled. Therefore, this function must not be 7369 * blocked. 7370 * 7371 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7372 * DDI_FAILURE indicates an error condition and should almost never happen. 7373 */ 7374 static int 7375 ahci_quiesce(dev_info_t *dip) 7376 { 7377 ahci_ctl_t *ahci_ctlp; 7378 ahci_port_t *ahci_portp; 7379 int instance, port; 7380 7381 instance = ddi_get_instance(dip); 7382 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7383 7384 if (ahci_ctlp == NULL) 7385 return (DDI_FAILURE); 7386 7387 #if AHCI_DEBUG 7388 ahci_debug_flags = 0; 7389 #endif 7390 7391 /* disable all the interrupts. */ 7392 ahci_disable_all_intrs(ahci_ctlp); 7393 7394 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7395 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7396 continue; 7397 } 7398 7399 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7400 7401 /* 7402 * Stop the port by clearing PxCMD.ST 7403 * 7404 * Here we must disable the port interrupt because 7405 * ahci_disable_all_intrs only clear GHC.IE, and IS 7406 * register will be still set if PxIE is enabled. 7407 * When ahci shares one IRQ with other drivers, the 7408 * intr handler may claim the intr mistakenly. 7409 */ 7410 ahci_disable_port_intrs(ahci_ctlp, port); 7411 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7412 ahci_portp, port); 7413 } 7414 7415 return (DDI_SUCCESS); 7416 } 7417