1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2018, Joyent, Inc. 26 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association. 27 */ 28 29 /* 30 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 31 * 32 * Power Management Support 33 * ------------------------ 34 * 35 * At the moment, the ahci driver only implements suspend/resume to 36 * support Suspend to RAM on X86 feature. Device power management isn't 37 * implemented, link power management is disabled, and hot plug isn't 38 * allowed during the period from suspend to resume. 39 * 40 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 41 * and DDI_RESUME entries, and don't need to take care of new requests 42 * sent down after suspend because the target driver (sd) has already 43 * handled these conditions, and blocked these requests. For the detailed 44 * information, please check with sdopen, sdclose and sdioctl routines. 45 * 46 * 47 * Enclosure Management Support 48 * ---------------------------- 49 * 50 * The ahci driver has basic support for AHCI Enclosure Management (EM) 51 * services. The AHCI specification provides an area in the primary ahci BAR for 52 * posting data to send out to the enclosure management and provides a register 53 * that provides both information and control about this. While the 54 * specification allows for multiple forms of enclosure management, the only 55 * supported, and commonly found form, is the AHCI specified LED format. The LED 56 * format is often implemented as a one-way communication mechanism. Software 57 * can write out what it cares about into the aforementioned data buffer and 58 * then we wait for the transmission to be sent. 59 * 60 * This has some drawbacks. It means that we cannot know whether or not it has 61 * succeeded. This means we cannot ask hardware what it thinks the LEDs are 62 * set to. There's also the added unfortunate reality that firmware on the 63 * microcontroller driving this will often not show the LEDs if no drive is 64 * present and that actions taken may potentially cause this to get out of sync 65 * with what we expect it to be. For example, the specification does not 66 * describe what should happen if a drive is removed from the enclosure while 67 * this is set and what should happen when it returns. We can only infer that it 68 * should be the same. 69 * 70 * Because only a single command can be sent at any time and we don't want to 71 * interfere with controller I/O, we create a taskq dedicated to this that has a 72 * single thread. Both resets (which occur on attach and resume) and normal 73 * changes to the LED state will be driven through this taskq. Because the taskq 74 * has a single thread, this guarantees serial processing. 75 * 76 * Each userland-submitted task (basically not resets) has a reference counted 77 * task structure. This allows the thread that called it to be cancelled and 78 * have the system clean itself up. The user thread in ioctl blocks on a CV that 79 * can receive signals as it waits for completion. Note, there is no guarantee 80 * provided by the kernel that the first thread to enter the kernel will be the 81 * first one to change state. 82 */ 83 84 #include <sys/note.h> 85 #include <sys/scsi/scsi.h> 86 #include <sys/pci.h> 87 #include <sys/disp.h> 88 #include <sys/sata/sata_hba.h> 89 #include <sys/sata/adapters/ahci/ahcireg.h> 90 #include <sys/sata/adapters/ahci/ahcivar.h> 91 92 /* 93 * FMA header files 94 */ 95 #include <sys/ddifm.h> 96 #include <sys/fm/protocol.h> 97 #include <sys/fm/util.h> 98 #include <sys/fm/io/ddi.h> 99 100 /* 101 * EM Control header files 102 */ 103 #include <sys/types.h> 104 #include <sys/file.h> 105 #include <sys/errno.h> 106 #include <sys/open.h> 107 #include <sys/cred.h> 108 #include <sys/ddi.h> 109 #include <sys/sunddi.h> 110 111 /* 112 * This is the string displayed by modinfo, etc. 113 */ 114 static char ahci_ident[] = "ahci driver"; 115 116 /* 117 * Function prototypes for driver entry points 118 */ 119 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 120 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 121 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 122 static int ahci_quiesce(dev_info_t *); 123 124 /* 125 * Function prototypes for SATA Framework interfaces 126 */ 127 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 128 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 129 130 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 131 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 132 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 133 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 134 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 135 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 136 #if defined(__lock_lint) 137 static int ahci_selftest(dev_info_t *, sata_device_t *); 138 #endif 139 140 /* 141 * FMA Prototypes 142 */ 143 static void ahci_fm_init(ahci_ctl_t *); 144 static void ahci_fm_fini(ahci_ctl_t *); 145 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*); 146 int ahci_check_acc_handle(ddi_acc_handle_t); 147 int ahci_check_dma_handle(ddi_dma_handle_t); 148 void ahci_fm_ereport(ahci_ctl_t *, char *); 149 static int ahci_check_all_handle(ahci_ctl_t *); 150 static int ahci_check_ctl_handle(ahci_ctl_t *); 151 static int ahci_check_port_handle(ahci_ctl_t *, int); 152 static int ahci_check_slot_handle(ahci_port_t *, int); 153 154 /* 155 * Local function prototypes 156 */ 157 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *); 158 static int ahci_alloc_ports_state(ahci_ctl_t *); 159 static void ahci_dealloc_ports_state(ahci_ctl_t *); 160 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 161 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 162 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *); 163 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 164 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 165 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 166 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 167 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 168 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *); 169 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *); 170 171 static int ahci_initialize_controller(ahci_ctl_t *); 172 static void ahci_uninitialize_controller(ahci_ctl_t *); 173 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 174 static int ahci_config_space_init(ahci_ctl_t *); 175 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t); 176 177 static void ahci_drain_ports_taskq(ahci_ctl_t *); 178 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *, 179 uint8_t); 180 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *); 181 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t); 182 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *, 183 sata_device_t *); 184 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *, 185 sata_pmult_gscr_t *); 186 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 187 sata_device_t *); 188 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 189 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 190 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 191 sata_device_t *); 192 193 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 194 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 195 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, 196 ahci_addr_t *); 197 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 198 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 199 ahci_addr_t *, sata_pkt_t *); 200 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 201 ahci_addr_t *, sata_pkt_t *); 202 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, 203 ahci_addr_t *, int); 204 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 205 static void ahci_copy_ncq_err_page(sata_cmd_t *, 206 struct sata_ncq_error_recovery_page *); 207 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 208 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int); 209 static void ahci_flush_doneq(ahci_port_t *); 210 211 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 212 static int ahci_hba_reset(ahci_ctl_t *); 213 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 214 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 215 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 216 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, 217 ahci_addr_t *); 218 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *, 219 ahci_addr_t *); 220 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, 221 ahci_addr_t *); 222 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 223 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 224 uint8_t); 225 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 226 uint8_t, int, int *); 227 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 228 uint32_t, uint32_t, uint32_t, uint32_t); 229 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 231 uint8_t, sata_pkt_t *); 232 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 233 ahci_addr_t *, uint32_t); 234 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 235 uint8_t, uint32_t); 236 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 237 uint8_t, uint32_t); 238 static void ahci_events_handler(void *); 239 static void ahci_watchdog_handler(ahci_ctl_t *); 240 241 static uint_t ahci_intr(caddr_t, caddr_t); 242 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 243 static int ahci_add_intrs(ahci_ctl_t *, int); 244 static void ahci_rem_intrs(ahci_ctl_t *); 245 static void ahci_enable_all_intrs(ahci_ctl_t *); 246 static void ahci_disable_all_intrs(ahci_ctl_t *); 247 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 248 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 249 250 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 251 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 252 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 253 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t); 254 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 255 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 256 ahci_port_t *, uint8_t); 257 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 258 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 259 uint8_t, uint32_t); 260 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 261 uint8_t, uint32_t); 262 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 263 264 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *); 265 static int ahci_get_num_implemented_ports(uint32_t); 266 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t); 267 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t); 268 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 269 #if AHCI_DEBUG 270 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 271 #endif 272 273 static boolean_t ahci_em_init(ahci_ctl_t *); 274 static void ahci_em_fini(ahci_ctl_t *); 275 static void ahci_em_suspend(ahci_ctl_t *); 276 static void ahci_em_resume(ahci_ctl_t *); 277 static int ahci_em_ioctl(dev_info_t *, int, intptr_t); 278 279 280 /* 281 * DMA attributes for the data buffer 282 * 283 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 284 * does not support 64-bit addressing 285 */ 286 static ddi_dma_attr_t buffer_dma_attr = { 287 DMA_ATTR_V0, /* dma_attr_version */ 288 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 289 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 290 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 291 0x2ull, /* dma_attr_align: word aligned */ 292 1, /* dma_attr_burstsizes */ 293 1, /* dma_attr_minxfer */ 294 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 295 0xffffffffull, /* dma_attr_seg */ 296 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 297 512, /* dma_attr_granular */ 298 0, /* dma_attr_flags */ 299 }; 300 301 /* 302 * DMA attributes for the rcvd FIS 303 * 304 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 305 * does not support 64-bit addressing 306 */ 307 static ddi_dma_attr_t rcvd_fis_dma_attr = { 308 DMA_ATTR_V0, /* dma_attr_version */ 309 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 310 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 311 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 312 0x100ull, /* dma_attr_align: 256-byte aligned */ 313 1, /* dma_attr_burstsizes */ 314 1, /* dma_attr_minxfer */ 315 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 316 0xffffffffull, /* dma_attr_seg */ 317 1, /* dma_attr_sgllen */ 318 1, /* dma_attr_granular */ 319 0, /* dma_attr_flags */ 320 }; 321 322 /* 323 * DMA attributes for the command list 324 * 325 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 326 * does not support 64-bit addressing 327 */ 328 static ddi_dma_attr_t cmd_list_dma_attr = { 329 DMA_ATTR_V0, /* dma_attr_version */ 330 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 331 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 332 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 333 0x400ull, /* dma_attr_align: 1K-byte aligned */ 334 1, /* dma_attr_burstsizes */ 335 1, /* dma_attr_minxfer */ 336 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 337 0xffffffffull, /* dma_attr_seg */ 338 1, /* dma_attr_sgllen */ 339 1, /* dma_attr_granular */ 340 0, /* dma_attr_flags */ 341 }; 342 343 /* 344 * DMA attributes for cmd tables 345 * 346 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 347 * does not support 64-bit addressing 348 */ 349 static ddi_dma_attr_t cmd_table_dma_attr = { 350 DMA_ATTR_V0, /* dma_attr_version */ 351 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 352 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 353 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 354 0x80ull, /* dma_attr_align: 128-byte aligned */ 355 1, /* dma_attr_burstsizes */ 356 1, /* dma_attr_minxfer */ 357 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 358 0xffffffffull, /* dma_attr_seg */ 359 1, /* dma_attr_sgllen */ 360 1, /* dma_attr_granular */ 361 0, /* dma_attr_flags */ 362 }; 363 364 365 /* Device access attributes */ 366 static ddi_device_acc_attr_t accattr = { 367 DDI_DEVICE_ATTR_V1, 368 DDI_STRUCTURE_LE_ACC, 369 DDI_STRICTORDER_ACC, 370 DDI_DEFAULT_ACC 371 }; 372 373 static struct dev_ops ahcictl_dev_ops = { 374 DEVO_REV, /* devo_rev */ 375 0, /* refcnt */ 376 ahci_getinfo, /* info */ 377 nulldev, /* identify */ 378 nulldev, /* probe */ 379 ahci_attach, /* attach */ 380 ahci_detach, /* detach */ 381 nodev, /* no reset */ 382 NULL, /* driver operations */ 383 NULL, /* bus operations */ 384 NULL, /* power */ 385 ahci_quiesce, /* quiesce */ 386 }; 387 388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 389 SATA_TRAN_HOTPLUG_OPS_REV_1, 390 ahci_tran_hotplug_port_activate, 391 ahci_tran_hotplug_port_deactivate 392 }; 393 394 extern struct mod_ops mod_driverops; 395 396 static struct modldrv modldrv = { 397 &mod_driverops, /* driverops */ 398 ahci_ident, /* short description */ 399 &ahcictl_dev_ops, /* driver ops */ 400 }; 401 402 static struct modlinkage modlinkage = { 403 MODREV_1, 404 &modldrv, 405 NULL 406 }; 407 408 /* The following variables are watchdog handler related */ 409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */ 410 static clock_t ahci_watchdog_tick; 411 412 /* 413 * This static variable indicates the size of command table, 414 * and it's changeable with prdt number, which ahci_dma_prdt_number 415 * indicates. 416 */ 417 static size_t ahci_cmd_table_size; 418 419 /* 420 * The below global variables are tunable via /etc/system 421 * 422 * ahci_dma_prdt_number 423 * ahci_msi_enabled 424 * ahci_buf_64bit_dma 425 * ahci_commu_64bit_dma 426 */ 427 428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 430 431 /* AHCI MSI is tunable */ 432 boolean_t ahci_msi_enabled = B_TRUE; 433 434 /* 435 * 64-bit dma addressing for data buffer is tunable 436 * 437 * The variable controls only the below value: 438 * DBAU (upper 32-bits physical address of data block) 439 */ 440 boolean_t ahci_buf_64bit_dma = B_TRUE; 441 442 /* 443 * 64-bit dma addressing for communication system descriptors is tunable 444 * 445 * The variable controls the below three values: 446 * 447 * PxCLBU (upper 32-bits for the command list base physical address) 448 * PxFBU (upper 32-bits for the received FIS base physical address) 449 * CTBAU (upper 32-bits of command table base) 450 */ 451 boolean_t ahci_commu_64bit_dma = B_TRUE; 452 453 /* 454 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600 455 * chipset. If the users want to have a try with 64-bit dma, please change 456 * the below variable value to enable it. 457 */ 458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE; 459 460 /* 461 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI 462 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma, 463 * please change the below value to enable it. 464 */ 465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE; 466 467 /* 468 * These values control the default delay and default number of times to wait 469 * for an enclosure message to complete. 470 */ 471 uint_t ahci_em_reset_delay_ms = 1; 472 uint_t ahci_em_reset_delay_count = 1000; 473 uint_t ahci_em_tx_delay_ms = 1; 474 uint_t ahci_em_tx_delay_count = 1000; 475 476 477 /* 478 * End of global tunable variable definition 479 */ 480 481 #if AHCI_DEBUG 482 uint32_t ahci_debug_flags = 0; 483 #else 484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 485 #endif 486 487 488 #if AHCI_DEBUG 489 /* The following is needed for ahci_log() */ 490 static kmutex_t ahci_log_mutex; 491 static char ahci_log_buf[512]; 492 #endif 493 494 /* Opaque state pointer initialized by ddi_soft_state_init() */ 495 static void *ahci_statep = NULL; 496 497 /* 498 * ahci module initialization. 499 */ 500 int 501 _init(void) 502 { 503 int ret; 504 505 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 506 if (ret != 0) { 507 goto err_out; 508 } 509 510 #if AHCI_DEBUG 511 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 512 #endif 513 514 if ((ret = sata_hba_init(&modlinkage)) != 0) { 515 #if AHCI_DEBUG 516 mutex_destroy(&ahci_log_mutex); 517 #endif 518 ddi_soft_state_fini(&ahci_statep); 519 goto err_out; 520 } 521 522 /* watchdog tick */ 523 ahci_watchdog_tick = drv_usectohz( 524 (clock_t)ahci_watchdog_timeout * 1000000); 525 526 ret = mod_install(&modlinkage); 527 if (ret != 0) { 528 sata_hba_fini(&modlinkage); 529 #if AHCI_DEBUG 530 mutex_destroy(&ahci_log_mutex); 531 #endif 532 ddi_soft_state_fini(&ahci_statep); 533 goto err_out; 534 } 535 536 return (ret); 537 538 err_out: 539 cmn_err(CE_WARN, "!ahci: Module init failed"); 540 return (ret); 541 } 542 543 /* 544 * ahci module uninitialize. 545 */ 546 int 547 _fini(void) 548 { 549 int ret; 550 551 ret = mod_remove(&modlinkage); 552 if (ret != 0) { 553 return (ret); 554 } 555 556 /* Remove the resources allocated in _init(). */ 557 sata_hba_fini(&modlinkage); 558 #if AHCI_DEBUG 559 mutex_destroy(&ahci_log_mutex); 560 #endif 561 ddi_soft_state_fini(&ahci_statep); 562 563 return (ret); 564 } 565 566 /* 567 * _info entry point 568 */ 569 int 570 _info(struct modinfo *modinfop) 571 { 572 return (mod_info(&modlinkage, modinfop)); 573 } 574 575 /* 576 * The attach entry point for dev_ops. 577 */ 578 static int 579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 580 { 581 ahci_ctl_t *ahci_ctlp = NULL; 582 int instance = ddi_get_instance(dip); 583 int status; 584 int attach_state; 585 uint32_t cap_status, ahci_version; 586 uint32_t ghc_control; 587 int intr_types; 588 int i; 589 pci_regspec_t *regs; 590 int regs_length; 591 int rnumber; 592 #if AHCI_DEBUG 593 int speed; 594 #endif 595 596 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 597 NULL); 598 599 switch (cmd) { 600 case DDI_ATTACH: 601 break; 602 603 case DDI_RESUME: 604 605 /* 606 * During DDI_RESUME, the hardware state of the device 607 * (power may have been removed from the device) must be 608 * restored, allow pending requests to continue, and 609 * service new requests. 610 */ 611 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 612 mutex_enter(&ahci_ctlp->ahcictl_mutex); 613 614 /* 615 * GHC.AE must be set to 1 before any other AHCI register 616 * is accessed 617 */ 618 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 619 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 620 ghc_control |= AHCI_HBA_GHC_AE; 621 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 623 624 /* Restart watch thread */ 625 if (ahci_ctlp->ahcictl_timeout_id == 0) 626 ahci_ctlp->ahcictl_timeout_id = timeout( 627 (void (*)(void *))ahci_watchdog_handler, 628 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 629 630 mutex_exit(&ahci_ctlp->ahcictl_mutex); 631 632 /* 633 * Re-initialize the controller and enable the interrupts and 634 * restart all the ports. 635 * 636 * Note that so far we don't support hot-plug during 637 * suspend/resume. 638 */ 639 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 640 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 641 "Failed to initialize the controller " 642 "during DDI_RESUME", NULL); 643 return (DDI_FAILURE); 644 } 645 646 /* 647 * Reset the enclosure services. 648 */ 649 ahci_em_resume(ahci_ctlp); 650 651 mutex_enter(&ahci_ctlp->ahcictl_mutex); 652 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND; 653 mutex_exit(&ahci_ctlp->ahcictl_mutex); 654 655 return (DDI_SUCCESS); 656 657 default: 658 return (DDI_FAILURE); 659 } 660 661 attach_state = AHCI_ATTACH_STATE_NONE; 662 663 /* Allocate soft state */ 664 status = ddi_soft_state_zalloc(ahci_statep, instance); 665 if (status != DDI_SUCCESS) { 666 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 667 instance); 668 goto err_out; 669 } 670 671 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 672 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 673 ahci_ctlp->ahcictl_dip = dip; 674 675 /* Initialize the cport/port mapping */ 676 for (i = 0; i < AHCI_MAX_PORTS; i++) { 677 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 678 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 679 } 680 681 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 682 683 /* Initialize FMA properties */ 684 ahci_fm_init(ahci_ctlp); 685 686 attach_state |= AHCI_ATTACH_STATE_FMA; 687 688 /* 689 * Now map the AHCI base address; which includes global 690 * registers and port control registers 691 * 692 * According to the spec, the AHCI Base Address is BAR5, 693 * but BAR0-BAR4 are optional, so we need to check which 694 * rnumber is used for BAR5. 695 */ 696 697 /* 698 * search through DDI "reg" property for the AHCI register set 699 */ 700 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 701 DDI_PROP_DONTPASS, "reg", (int **)®s, 702 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 703 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 704 instance); 705 goto err_out; 706 } 707 708 /* AHCI Base Address is located at 0x24 offset */ 709 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 710 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 711 == AHCI_PCI_RNUM) 712 break; 713 } 714 715 ddi_prop_free(regs); 716 717 if (rnumber == regs_length) { 718 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 719 instance); 720 goto err_out; 721 } 722 723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 724 725 status = ddi_regs_map_setup(dip, 726 rnumber, 727 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 728 0, 729 0, 730 &accattr, 731 &ahci_ctlp->ahcictl_ahci_acc_handle); 732 if (status != DDI_SUCCESS) { 733 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 734 instance); 735 goto err_out; 736 } 737 738 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 739 740 /* 741 * GHC.AE must be set to 1 before any other AHCI register 742 * is accessed 743 */ 744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 746 ghc_control |= AHCI_HBA_GHC_AE; 747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 749 750 /* Get the AHCI version information */ 751 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 752 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 753 754 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 755 (ahci_version & 0xffff0000) >> 16, 756 ((ahci_version & 0x0000ff00) >> 4 | 757 (ahci_version & 0x000000ff))); 758 759 /* We don't support controllers whose versions are lower than 1.0 */ 760 if (!(ahci_version & 0xffff0000)) { 761 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 762 "than version 1.0", instance); 763 goto err_out; 764 } 765 766 /* Get the HBA capabilities information */ 767 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 768 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 769 770 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x", 771 cap_status); 772 773 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */ 774 if (ahci_version >= 0x00010200) { 775 uint32_t cap2_status; 776 777 /* Get the HBA capabilities extended information */ 778 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 779 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp)); 780 781 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 782 "hba capabilities extended = 0x%x", cap2_status); 783 } 784 785 if (cap_status & AHCI_HBA_CAP_EMS) { 786 ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS; 787 ahci_ctlp->ahcictl_em_loc = 788 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 789 (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp)); 790 ahci_ctlp->ahcictl_em_ctl = 791 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 792 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 793 } 794 795 #if AHCI_DEBUG 796 /* Get the interface speed supported by the HBA */ 797 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 798 if (speed == 0x01) { 799 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 800 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 801 } else if (speed == 0x10) { 802 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 803 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 804 } else if (speed == 0x11) { 805 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 806 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 807 } 808 #endif 809 810 /* Get the number of command slots supported by the HBA */ 811 ahci_ctlp->ahcictl_num_cmd_slots = 812 ((cap_status & AHCI_HBA_CAP_NCS) >> 813 AHCI_HBA_CAP_NCS_SHIFT) + 1; 814 815 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 816 ahci_ctlp->ahcictl_num_cmd_slots); 817 818 /* Get the bit map which indicates ports implemented by the HBA */ 819 ahci_ctlp->ahcictl_ports_implemented = 820 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 821 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 822 823 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 824 ahci_ctlp->ahcictl_ports_implemented); 825 826 /* Max port number implemented */ 827 ahci_ctlp->ahcictl_num_ports = 828 ddi_fls(ahci_ctlp->ahcictl_ports_implemented); 829 830 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 831 (cap_status & AHCI_HBA_CAP_NP) + 1); 832 833 /* Get the number of implemented ports by the HBA */ 834 ahci_ctlp->ahcictl_num_implemented_ports = 835 ahci_get_num_implemented_ports( 836 ahci_ctlp->ahcictl_ports_implemented); 837 838 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 839 "hba number of implemented ports: %d", 840 ahci_ctlp->ahcictl_num_implemented_ports); 841 842 /* Check whether HBA supports 64bit DMA addressing */ 843 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 844 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 845 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 846 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 847 "hba does not support 64-bit addressing", NULL); 848 } 849 850 /* Checking for the support of Port Multiplier */ 851 if (cap_status & AHCI_HBA_CAP_SPM) { 852 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS; 853 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 854 "hba supports port multiplier (CBSS)", NULL); 855 856 /* Support FIS-based switching ? */ 857 if (cap_status & AHCI_HBA_CAP_FBSS) { 858 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS; 859 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 860 "hba supports FIS-based switching (FBSS)", NULL); 861 } 862 } 863 864 /* Checking for Support Command List Override */ 865 if (cap_status & AHCI_HBA_CAP_SCLO) { 866 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 867 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 868 "hba supports command list override.", NULL); 869 } 870 871 /* Checking for Asynchronous Notification */ 872 if (cap_status & AHCI_HBA_CAP_SSNTF) { 873 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF; 874 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 875 "hba supports asynchronous notification.", NULL); 876 } 877 878 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 879 != DDI_SUCCESS) { 880 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 881 instance); 882 goto err_out; 883 } 884 885 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 886 887 /* 888 * Check the pci configuration space, and set caps. We also 889 * handle the hardware defect in this function. 890 * 891 * For example, force ATI SB600 to use 32-bit dma addressing 892 * since it doesn't support 64-bit dma though its CAP register 893 * declares it support. 894 */ 895 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 896 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 897 instance); 898 goto err_out; 899 } 900 901 /* 902 * Disable the whole controller interrupts before adding 903 * interrupt handlers(s). 904 */ 905 ahci_disable_all_intrs(ahci_ctlp); 906 907 /* Get supported interrupt types */ 908 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 909 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 910 instance); 911 goto err_out; 912 } 913 914 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 915 "ddi_intr_get_supported_types() returned: 0x%x", 916 intr_types); 917 918 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 919 /* 920 * Try MSI first, but fall back to FIXED if failed 921 */ 922 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 923 DDI_SUCCESS) { 924 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 925 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 926 "Using MSI interrupt type", NULL); 927 goto intr_done; 928 } 929 930 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 931 "MSI registration failed, " 932 "trying FIXED interrupts", NULL); 933 } 934 935 if (intr_types & DDI_INTR_TYPE_FIXED) { 936 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 937 DDI_SUCCESS) { 938 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 939 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 940 "Using FIXED interrupt type", NULL); 941 goto intr_done; 942 } 943 944 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 945 "FIXED interrupt registration failed", NULL); 946 } 947 948 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 949 950 goto err_out; 951 952 intr_done: 953 954 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 955 956 /* Initialize the controller mutex */ 957 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 958 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 959 960 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 961 962 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 963 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 964 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 965 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 966 } 967 968 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 969 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 970 sizeof (ahci_prdt_item_t)); 971 972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 973 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 974 " ahci_cmd_table_size is 0x%x", 975 ahci_dma_prdt_number, ahci_cmd_table_size); 976 977 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 978 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 979 ahci_dma_prdt_number; 980 981 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 982 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 983 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 984 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 985 986 /* 987 * enable 64bit dma for data buffer for SB600 if 988 * sb600_buf_64bit_dma_disable is B_FALSE 989 */ 990 if ((ahci_buf_64bit_dma == B_FALSE) || 991 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) && 992 !(sb600_buf_64bit_dma_disable == B_FALSE && 993 ahci_ctlp->ahcictl_venid == 0x1002 && 994 ahci_ctlp->ahcictl_devid == 0x4380))) { 995 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 996 0xffffffffull; 997 } 998 999 /* 1000 * enable 64bit dma for command buffer for SB600/700/710/800 1001 * if sbxxx_commu_64bit_dma_disable is B_FALSE 1002 */ 1003 if ((ahci_commu_64bit_dma == B_FALSE) || 1004 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) && 1005 !(sbxxx_commu_64bit_dma_disable == B_FALSE && 1006 ahci_ctlp->ahcictl_venid == 0x1002 && 1007 (ahci_ctlp->ahcictl_devid == 0x4380 || 1008 ahci_ctlp->ahcictl_devid == 0x4391)))) { 1009 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 1010 0xffffffffull; 1011 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 1012 0xffffffffull; 1013 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 1014 0xffffffffull; 1015 } 1016 1017 /* Allocate the ports structure */ 1018 status = ahci_alloc_ports_state(ahci_ctlp); 1019 if (status != AHCI_SUCCESS) { 1020 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 1021 instance); 1022 goto err_out; 1023 } 1024 1025 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 1026 1027 /* 1028 * Initialize the controller and ports. 1029 */ 1030 status = ahci_initialize_controller(ahci_ctlp); 1031 if (status != AHCI_SUCCESS) { 1032 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 1033 instance); 1034 goto err_out; 1035 } 1036 1037 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 1038 1039 /* Start one thread to check packet timeouts */ 1040 ahci_ctlp->ahcictl_timeout_id = timeout( 1041 (void (*)(void *))ahci_watchdog_handler, 1042 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 1043 1044 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 1045 1046 if (!ahci_em_init(ahci_ctlp)) { 1047 cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure " 1048 "services", instance); 1049 goto err_out; 1050 } 1051 attach_state |= AHCI_ATTACH_STATE_ENCLOSURE; 1052 1053 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 1054 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 1055 instance); 1056 goto err_out; 1057 } 1058 1059 /* Check all handles at the end of the attach operation. */ 1060 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) { 1061 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles", 1062 instance); 1063 goto err_out; 1064 } 1065 1066 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 1067 1068 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 1069 1070 return (DDI_SUCCESS); 1071 1072 err_out: 1073 /* FMA message */ 1074 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE); 1075 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST); 1076 1077 if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) { 1078 ahci_em_fini(ahci_ctlp); 1079 } 1080 1081 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 1082 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1083 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1084 ahci_ctlp->ahcictl_timeout_id = 0; 1085 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1086 } 1087 1088 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 1089 ahci_uninitialize_controller(ahci_ctlp); 1090 } 1091 1092 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 1093 ahci_dealloc_ports_state(ahci_ctlp); 1094 } 1095 1096 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 1097 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1098 } 1099 1100 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 1101 ahci_rem_intrs(ahci_ctlp); 1102 } 1103 1104 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 1105 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1106 } 1107 1108 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 1109 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1110 } 1111 1112 if (attach_state & AHCI_ATTACH_STATE_FMA) { 1113 ahci_fm_fini(ahci_ctlp); 1114 } 1115 1116 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 1117 ddi_soft_state_free(ahci_statep, instance); 1118 } 1119 1120 return (DDI_FAILURE); 1121 } 1122 1123 /* 1124 * The detach entry point for dev_ops. 1125 */ 1126 static int 1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 1128 { 1129 ahci_ctl_t *ahci_ctlp; 1130 int instance; 1131 int ret; 1132 1133 instance = ddi_get_instance(dip); 1134 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1135 1136 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 1137 1138 switch (cmd) { 1139 case DDI_DETACH: 1140 1141 /* disable the interrupts for an uninterrupted detach */ 1142 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1143 ahci_disable_all_intrs(ahci_ctlp); 1144 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1145 1146 /* unregister from the sata framework. */ 1147 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 1148 if (ret != AHCI_SUCCESS) { 1149 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1150 ahci_enable_all_intrs(ahci_ctlp); 1151 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1152 return (DDI_FAILURE); 1153 } 1154 1155 ahci_em_fini(ahci_ctlp); 1156 1157 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1158 1159 /* stop the watchdog handler */ 1160 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1161 ahci_ctlp->ahcictl_timeout_id = 0; 1162 1163 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1164 1165 /* uninitialize the controller */ 1166 ahci_uninitialize_controller(ahci_ctlp); 1167 1168 /* remove the interrupts */ 1169 ahci_rem_intrs(ahci_ctlp); 1170 1171 /* deallocate the ports structures */ 1172 ahci_dealloc_ports_state(ahci_ctlp); 1173 1174 /* destroy mutex */ 1175 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1176 1177 /* teardown the pci config */ 1178 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1179 1180 /* remove the reg maps. */ 1181 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1182 1183 /* release fma resource */ 1184 ahci_fm_fini(ahci_ctlp); 1185 1186 /* free the soft state. */ 1187 ddi_soft_state_free(ahci_statep, instance); 1188 1189 return (DDI_SUCCESS); 1190 1191 case DDI_SUSPEND: 1192 1193 /* 1194 * The steps associated with suspension must include putting 1195 * the underlying device into a quiescent state so that it 1196 * will not generate interrupts or modify or access memory. 1197 */ 1198 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1199 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 1200 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1201 return (DDI_SUCCESS); 1202 } 1203 1204 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 1205 1206 /* stop the watchdog handler */ 1207 if (ahci_ctlp->ahcictl_timeout_id) { 1208 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1209 ahci_ctlp->ahcictl_timeout_id = 0; 1210 } 1211 1212 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1213 1214 ahci_em_suspend(ahci_ctlp); 1215 1216 /* 1217 * drain the taskq 1218 */ 1219 ahci_drain_ports_taskq(ahci_ctlp); 1220 1221 /* 1222 * Disable the interrupts and stop all the ports. 1223 */ 1224 ahci_uninitialize_controller(ahci_ctlp); 1225 1226 return (DDI_SUCCESS); 1227 1228 default: 1229 return (DDI_FAILURE); 1230 } 1231 } 1232 1233 /* 1234 * The info entry point for dev_ops. 1235 * 1236 */ 1237 static int 1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 1239 void *arg, void **result) 1240 { 1241 #ifndef __lock_lint 1242 _NOTE(ARGUNUSED(dip)) 1243 #endif /* __lock_lint */ 1244 1245 ahci_ctl_t *ahci_ctlp; 1246 int instance; 1247 dev_t dev; 1248 1249 dev = (dev_t)arg; 1250 instance = getminor(dev); 1251 1252 switch (infocmd) { 1253 case DDI_INFO_DEVT2DEVINFO: 1254 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1255 if (ahci_ctlp != NULL) { 1256 *result = ahci_ctlp->ahcictl_dip; 1257 return (DDI_SUCCESS); 1258 } else { 1259 *result = NULL; 1260 return (DDI_FAILURE); 1261 } 1262 case DDI_INFO_DEVT2INSTANCE: 1263 *(int *)result = instance; 1264 break; 1265 default: 1266 break; 1267 } 1268 1269 return (DDI_SUCCESS); 1270 } 1271 1272 /* 1273 * Registers the ahci with sata framework. 1274 */ 1275 static int 1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1277 { 1278 struct sata_hba_tran *sata_hba_tran; 1279 1280 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1281 "ahci_register_sata_hba_tran enter", NULL); 1282 1283 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1284 1285 /* Allocate memory for the sata_hba_tran */ 1286 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1287 1288 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV; 1289 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1290 sata_hba_tran->sata_tran_hba_dma_attr = 1291 &ahci_ctlp->ahcictl_buffer_dma_attr; 1292 1293 /* Report the number of implemented ports */ 1294 sata_hba_tran->sata_tran_hba_num_cports = 1295 ahci_ctlp->ahcictl_num_implemented_ports; 1296 1297 /* Support ATAPI device */ 1298 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1299 1300 /* Get the data transfer capability for PIO command by the HBA */ 1301 if (cap_status & AHCI_HBA_CAP_PMD) { 1302 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1303 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1304 "DRQ block data transfer for PIO command protocol", NULL); 1305 } 1306 1307 /* 1308 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1309 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1310 * QUEUED (EXT), and SERVICE commands). Queued operations are 1311 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1312 * QUEUED commands when the HBA and device support native command 1313 * queuing(NCQ). 1314 * 1315 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1316 * CAP register of the HBA indicates NCQ is supported. 1317 * 1318 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1319 * set because the previous register content of PxCI can be re-written 1320 * in the register write. 1321 */ 1322 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1323 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1324 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1325 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1326 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1327 "Command Queuing", NULL); 1328 } 1329 1330 /* Support port multiplier? */ 1331 if (cap_status & AHCI_HBA_CAP_SPM) { 1332 sata_hba_tran->sata_tran_hba_features_support |= 1333 SATA_CTLF_PORT_MULTIPLIER; 1334 1335 /* Support FIS-based switching for port multiplier? */ 1336 if (cap_status & AHCI_HBA_CAP_FBSS) { 1337 sata_hba_tran->sata_tran_hba_features_support |= 1338 SATA_CTLF_PMULT_FBS; 1339 } 1340 } 1341 1342 /* Report the number of command slots */ 1343 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1344 1345 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1346 sata_hba_tran->sata_tran_start = ahci_tran_start; 1347 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1348 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1349 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1350 #ifdef __lock_lint 1351 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1352 #endif 1353 /* 1354 * When SATA framework adds support for pwrmgt the 1355 * pwrmgt_ops needs to be updated 1356 */ 1357 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1358 sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl; 1359 1360 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1361 1362 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1363 1364 /* Attach it to SATA framework */ 1365 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1366 != DDI_SUCCESS) { 1367 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1368 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1369 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1370 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1371 return (AHCI_FAILURE); 1372 } 1373 1374 return (AHCI_SUCCESS); 1375 } 1376 1377 /* 1378 * Unregisters the ahci with sata framework. 1379 */ 1380 static int 1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1382 { 1383 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1384 "ahci_unregister_sata_hba_tran enter", NULL); 1385 1386 /* Detach from the SATA framework. */ 1387 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1388 DDI_SUCCESS) { 1389 return (AHCI_FAILURE); 1390 } 1391 1392 /* Deallocate sata_hba_tran. */ 1393 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1394 sizeof (sata_hba_tran_t)); 1395 1396 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1397 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1398 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1399 1400 return (AHCI_SUCCESS); 1401 } 1402 1403 #define SET_PORTSTR(str, addrp) \ 1404 if (AHCI_ADDR_IS_PORT(addrp)) \ 1405 (void) sprintf((str), "%d", (addrp)->aa_port); \ 1406 else if (AHCI_ADDR_IS_PMULT(addrp)) \ 1407 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \ 1408 else \ 1409 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \ 1410 (addrp)->aa_pmport); 1411 1412 /* 1413 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1414 * port status registers and an attached device type via sata_device 1415 * structure. 1416 * 1417 * We return the cached information from a previous hardware probe. The 1418 * actual hardware probing itself was done either from within 1419 * ahci_initialize_controller() during the driver attach or from a phy 1420 * ready change interrupt handler. 1421 */ 1422 static int 1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1424 { 1425 ahci_ctl_t *ahci_ctlp; 1426 ahci_port_t *ahci_portp; 1427 ahci_addr_t addr, pmult_addr; 1428 uint8_t cport = sd->satadev_addr.cport; 1429 char portstr[10]; 1430 uint8_t device_type; 1431 uint32_t port_state; 1432 uint8_t port; 1433 int rval = SATA_SUCCESS, rval_init; 1434 1435 port_state = 0; 1436 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1437 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1438 1439 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1440 1441 mutex_enter(&ahci_portp->ahciport_mutex); 1442 1443 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 1444 ASSERT(AHCI_ADDR_IS_VALID(&addr)); 1445 SET_PORTSTR(portstr, &addr); 1446 1447 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1448 "ahci_tran_probe_port enter: port %s", portstr); 1449 1450 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) && 1451 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1452 ahci_portp->ahciport_pmult_info == NULL)) { 1453 /* port mutliplier is removed. */ 1454 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1455 "ahci_tran_probe_port: " 1456 "pmult is removed from port %s", portstr); 1457 mutex_exit(&ahci_portp->ahciport_mutex); 1458 return (SATA_FAILURE); 1459 } 1460 1461 /* 1462 * The sata_device may refer to 1463 * 1. A controller port. 1464 * A controller port should be ready here. 1465 * 2. A port multiplier. 1466 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize 1467 * it and register the port multiplier to the framework. 1468 * SATA_ADDR_PMULT - check the status of all its device ports. 1469 * 3. A port multiplier port. 1470 * If it has not been initialized, initialized it. 1471 * 1472 * A port multiplier or a port multiplier port may require some 1473 * initialization because we cannot do these time-consuming jobs in an 1474 * interrupt context. 1475 */ 1476 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) { 1477 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 1478 /* Initialize registers on a port multiplier */ 1479 rval_init = ahci_initialize_pmult(ahci_ctlp, 1480 ahci_portp, &pmult_addr, sd); 1481 if (rval_init != AHCI_SUCCESS) { 1482 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1483 "ahci_tran_probe_port: " 1484 "pmult initialization failed.", NULL); 1485 mutex_exit(&ahci_portp->ahciport_mutex); 1486 return (SATA_FAILURE); 1487 } 1488 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) { 1489 /* Check pmports hotplug events */ 1490 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr); 1491 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT | 1492 SATA_ADDR_DPMPORT)) { 1493 if (ahci_probe_pmport(ahci_ctlp, ahci_portp, 1494 &addr, sd) != AHCI_SUCCESS) { 1495 rval = SATA_FAILURE; 1496 goto out; 1497 } 1498 } 1499 1500 /* Update port state and device type */ 1501 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr); 1502 1503 switch (port_state) { 1504 1505 case SATA_PSTATE_FAILED: 1506 sd->satadev_state = SATA_PSTATE_FAILED; 1507 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1508 "ahci_tran_probe_port: port %s PORT FAILED", portstr); 1509 goto out; 1510 1511 case SATA_PSTATE_SHUTDOWN: 1512 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1513 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1514 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr); 1515 goto out; 1516 1517 case SATA_PSTATE_PWROFF: 1518 sd->satadev_state = SATA_PSTATE_PWROFF; 1519 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1520 "ahci_tran_probe_port: port %s PORT PWROFF", portstr); 1521 goto out; 1522 1523 case SATA_PSTATE_PWRON: 1524 sd->satadev_state = SATA_PSTATE_PWRON; 1525 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1526 "ahci_tran_probe_port: port %s PORT PWRON", portstr); 1527 break; 1528 1529 default: 1530 sd->satadev_state = port_state; 1531 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1532 "ahci_tran_probe_port: port %s PORT NORMAL %x", 1533 portstr, port_state); 1534 break; 1535 } 1536 1537 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1538 1539 switch (device_type) { 1540 1541 case SATA_DTYPE_ATADISK: 1542 sd->satadev_type = SATA_DTYPE_ATADISK; 1543 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1544 "ahci_tran_probe_port: port %s DISK found", portstr); 1545 break; 1546 1547 case SATA_DTYPE_ATAPI: 1548 /* 1549 * HBA driver only knows it's an ATAPI device, and don't know 1550 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1551 * type need to be determined by checking IDENTIFY PACKET 1552 * DEVICE data 1553 */ 1554 sd->satadev_type = SATA_DTYPE_ATAPI; 1555 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1556 "ahci_tran_probe_port: port %s ATAPI found", portstr); 1557 break; 1558 1559 case SATA_DTYPE_PMULT: 1560 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)); 1561 sd->satadev_type = SATA_DTYPE_PMULT; 1562 1563 /* Update the number of pmports. */ 1564 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1565 sd->satadev_add_info = ahci_portp-> 1566 ahciport_pmult_info->ahcipmi_num_dev_ports; 1567 1568 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1569 "ahci_tran_probe_port: port %s Port Multiplier found", 1570 portstr); 1571 break; 1572 1573 case SATA_DTYPE_UNKNOWN: 1574 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1575 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1576 "ahci_tran_probe_port: port %s Unknown device found", 1577 portstr); 1578 break; 1579 1580 default: 1581 /* we don't support any other device types */ 1582 sd->satadev_type = SATA_DTYPE_NONE; 1583 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1584 "ahci_tran_probe_port: port %s No device found", portstr); 1585 break; 1586 } 1587 1588 out: 1589 /* Register update only fails while probing a pmult/pmport */ 1590 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) { 1591 ahci_update_sata_registers(ahci_ctlp, port, sd); 1592 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 1593 if (port_state & SATA_STATE_READY) 1594 if (ahci_update_pmult_pscr(ahci_ctlp, 1595 &addr, sd) != AHCI_SUCCESS) 1596 rval = SATA_FAILURE; 1597 } 1598 1599 /* Check handles for the sata registers access */ 1600 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) || 1601 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) { 1602 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 1603 DDI_SERVICE_UNAFFECTED); 1604 rval = SATA_FAILURE; 1605 } 1606 1607 mutex_exit(&ahci_portp->ahciport_mutex); 1608 return (rval); 1609 } 1610 1611 /* 1612 * There are four operation modes in sata framework: 1613 * SATA_OPMODE_INTERRUPTS 1614 * SATA_OPMODE_POLLING 1615 * SATA_OPMODE_ASYNCH 1616 * SATA_OPMODE_SYNCH 1617 * 1618 * Their combined meanings as following: 1619 * 1620 * SATA_OPMODE_SYNCH 1621 * The command has to be completed before sata_tran_start functions returns. 1622 * Either interrupts or polling could be used - it's up to the driver. 1623 * Mode used currently for internal, sata-module initiated operations. 1624 * 1625 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1626 * It is the same as the one above. 1627 * 1628 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1629 * The command has to be completed before sata_tran_start function returns. 1630 * No interrupt used, polling only. This should be the mode used for scsi 1631 * packets with FLAG_NOINTR. 1632 * 1633 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1634 * The command may be queued (callback function specified). Interrupts could 1635 * be used. It's normal operation mode. 1636 */ 1637 /* 1638 * Called by sata framework to transport a sata packet down stream. 1639 */ 1640 static int 1641 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1642 { 1643 ahci_ctl_t *ahci_ctlp; 1644 ahci_port_t *ahci_portp; 1645 ahci_addr_t addr; 1646 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1647 uint8_t port; 1648 char portstr[10]; 1649 1650 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1651 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1652 1653 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1654 "ahci_tran_start enter: cport %d satapkt 0x%p", 1655 cport, (void *)spkt); 1656 1657 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1658 1659 mutex_enter(&ahci_portp->ahciport_mutex); 1660 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr); 1661 SET_PORTSTR(portstr, &addr); 1662 1663 /* Sanity check */ 1664 if (AHCI_ADDR_IS_PMPORT(&addr)) { 1665 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1666 ahci_portp->ahciport_pmult_info == NULL) { 1667 1668 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1669 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1670 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN; 1671 ahci_update_sata_registers(ahci_ctlp, port, 1672 &spkt->satapkt_device); 1673 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1674 "ahci_tran_start returning PORT_ERROR while " 1675 "pmult removed: port: %s", portstr); 1676 mutex_exit(&ahci_portp->ahciport_mutex); 1677 return (SATA_TRAN_PORT_ERROR); 1678 } 1679 1680 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) & 1681 SATA_STATE_READY)) { 1682 if (!ddi_in_panic() || 1683 ahci_initialize_pmport(ahci_ctlp, 1684 ahci_portp, &addr) != AHCI_SUCCESS) { 1685 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1686 spkt->satapkt_device.satadev_type = 1687 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1688 spkt->satapkt_device.satadev_state = 1689 AHCIPORT_GET_STATE(ahci_portp, &addr); 1690 ahci_update_sata_registers(ahci_ctlp, port, 1691 &spkt->satapkt_device); 1692 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1693 "ahci_tran_start returning PORT_ERROR " 1694 "while sub-link is not initialized " 1695 "at port: %s", portstr); 1696 mutex_exit(&ahci_portp->ahciport_mutex); 1697 return (SATA_TRAN_PORT_ERROR); 1698 } 1699 } 1700 } 1701 1702 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED || 1703 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN|| 1704 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) { 1705 /* 1706 * In case the target driver would send the packet before 1707 * sata framework can have the opportunity to process those 1708 * event reports. 1709 */ 1710 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1711 spkt->satapkt_device.satadev_state = 1712 ahci_portp->ahciport_port_state; 1713 ahci_update_sata_registers(ahci_ctlp, port, 1714 &spkt->satapkt_device); 1715 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1716 "ahci_tran_start returning PORT_ERROR while " 1717 "port in FAILED/SHUTDOWN/PWROFF state: " 1718 "port: %s", portstr); 1719 mutex_exit(&ahci_portp->ahciport_mutex); 1720 return (SATA_TRAN_PORT_ERROR); 1721 } 1722 1723 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) { 1724 /* 1725 * ahci_intr_phyrdy_change() may have rendered it to 1726 * SATA_DTYPE_NONE. 1727 */ 1728 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1729 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1730 spkt->satapkt_device.satadev_state = 1731 ahci_portp->ahciport_port_state; 1732 ahci_update_sata_registers(ahci_ctlp, port, 1733 &spkt->satapkt_device); 1734 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1735 "ahci_tran_start returning PORT_ERROR while " 1736 "no device attached: port: %s", portstr); 1737 mutex_exit(&ahci_portp->ahciport_mutex); 1738 return (SATA_TRAN_PORT_ERROR); 1739 } 1740 1741 /* R/W PMULT command will occupy the whole HBA port */ 1742 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1743 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1744 "ahci_tran_start returning BUSY while " 1745 "executing READ/WRITE PORT-MULT command: " 1746 "port: %s", portstr); 1747 spkt->satapkt_reason = SATA_PKT_BUSY; 1748 mutex_exit(&ahci_portp->ahciport_mutex); 1749 return (SATA_TRAN_BUSY); 1750 } 1751 1752 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 1753 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1754 "ahci_tran_start returning BUSY while " 1755 "hot-plug in progress: port: %s", portstr); 1756 spkt->satapkt_reason = SATA_PKT_BUSY; 1757 mutex_exit(&ahci_portp->ahciport_mutex); 1758 return (SATA_TRAN_BUSY); 1759 } 1760 1761 /* 1762 * SATA HBA driver should remember that a device was reset and it 1763 * is supposed to reject any packets which do not specify either 1764 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1765 * 1766 * This is to prevent a race condition when a device was arbitrarily 1767 * reset by the HBA driver (and lost it's setting) and a target 1768 * driver sending some commands to a device before the sata framework 1769 * has a chance to restore the device setting (such as cache enable/ 1770 * disable or other resettable stuff). 1771 */ 1772 /* 1773 * It is unnecessary to use specific flags to indicate 1774 * reset_in_progress for a pmport. While mopping, all command will be 1775 * mopped so that the entire HBA port is being dealt as a single 1776 * object. 1777 */ 1778 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1779 ahci_portp->ahciport_reset_in_progress = 0; 1780 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1781 "ahci_tran_start [CLEAR] the " 1782 "reset_in_progress for port: %d", port); 1783 } 1784 1785 if (ahci_portp->ahciport_reset_in_progress && 1786 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1787 ! ddi_in_panic()) { 1788 spkt->satapkt_reason = SATA_PKT_BUSY; 1789 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1790 "ahci_tran_start returning BUSY while " 1791 "reset in progress: port: %d", port); 1792 mutex_exit(&ahci_portp->ahciport_mutex); 1793 return (SATA_TRAN_BUSY); 1794 } 1795 1796 #ifdef AHCI_DEBUG 1797 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) { 1798 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1799 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d", 1800 spkt, port); 1801 } 1802 #endif 1803 1804 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1805 spkt->satapkt_reason = SATA_PKT_BUSY; 1806 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1807 "ahci_tran_start returning BUSY while " 1808 "mopping in progress: port: %d", port); 1809 mutex_exit(&ahci_portp->ahciport_mutex); 1810 return (SATA_TRAN_BUSY); 1811 } 1812 1813 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 1814 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 1815 DDI_SERVICE_UNAFFECTED); 1816 mutex_exit(&ahci_portp->ahciport_mutex); 1817 return (SATA_TRAN_BUSY); 1818 } 1819 1820 if (spkt->satapkt_op_mode & 1821 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1822 /* 1823 * If a SYNC command to be executed in interrupt context, 1824 * bounce it back to sata module. 1825 */ 1826 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1827 servicing_interrupt()) { 1828 spkt->satapkt_reason = SATA_PKT_BUSY; 1829 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1830 "ahci_tran_start returning BUSY while " 1831 "sending SYNC mode under interrupt context: " 1832 "port : %d", port); 1833 mutex_exit(&ahci_portp->ahciport_mutex); 1834 return (SATA_TRAN_BUSY); 1835 } 1836 1837 /* We need to do the sync start now */ 1838 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, 1839 spkt) == AHCI_FAILURE) { 1840 goto fail_out; 1841 } 1842 } else { 1843 /* Async start, using interrupt */ 1844 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt) 1845 == AHCI_FAILURE) { 1846 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1847 goto fail_out; 1848 } 1849 } 1850 1851 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1852 "sata tran accepted: port %s", portstr); 1853 1854 mutex_exit(&ahci_portp->ahciport_mutex); 1855 return (SATA_TRAN_ACCEPTED); 1856 1857 fail_out: 1858 /* 1859 * Failed to deliver packet to the controller. 1860 * Check if it's caused by invalid handles. 1861 */ 1862 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS || 1863 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) { 1864 spkt->satapkt_device.satadev_type = 1865 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1866 spkt->satapkt_device.satadev_state = 1867 AHCIPORT_GET_STATE(ahci_portp, &addr); 1868 spkt->satapkt_reason = SATA_PKT_DEV_ERROR; 1869 mutex_exit(&ahci_portp->ahciport_mutex); 1870 return (SATA_TRAN_PORT_ERROR); 1871 } 1872 1873 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1874 "return QUEUE_FULL: port %d", port); 1875 mutex_exit(&ahci_portp->ahciport_mutex); 1876 return (SATA_TRAN_QUEUE_FULL); 1877 } 1878 1879 /* 1880 * SATA_OPMODE_SYNCH flag is set 1881 * 1882 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1883 * without interrupt, otherwise we can still use the interrupt. 1884 */ 1885 static int 1886 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1887 ahci_addr_t *addrp, sata_pkt_t *spkt) 1888 { 1889 int pkt_timeout_ticks; 1890 uint32_t timeout_tags; 1891 int rval; 1892 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1893 uint8_t port = addrp->aa_port; 1894 1895 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 1896 1897 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1898 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt); 1899 1900 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1901 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1902 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1903 addrp, spkt)) == AHCI_FAILURE) { 1904 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1905 return (rval); 1906 } 1907 1908 pkt_timeout_ticks = 1909 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1910 1911 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1912 /* Simulate the interrupt */ 1913 mutex_exit(&ahci_portp->ahciport_mutex); 1914 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1915 mutex_enter(&ahci_portp->ahciport_mutex); 1916 1917 if (spkt->satapkt_reason != SATA_PKT_BUSY) 1918 break; 1919 1920 mutex_exit(&ahci_portp->ahciport_mutex); 1921 drv_usecwait(AHCI_1MS_USECS); 1922 mutex_enter(&ahci_portp->ahciport_mutex); 1923 1924 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1925 if (pkt_timeout_ticks < 0) { 1926 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1927 "port %d satapkt 0x%p timed out\n", 1928 instance, port, (void *)spkt); 1929 timeout_tags = (0x1 << rval); 1930 mutex_exit(&ahci_portp->ahciport_mutex); 1931 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1932 port, timeout_tags); 1933 mutex_enter(&ahci_portp->ahciport_mutex); 1934 } 1935 } 1936 1937 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1938 return (AHCI_SUCCESS); 1939 1940 } else { 1941 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1942 addrp, spkt)) == AHCI_FAILURE) 1943 return (rval); 1944 1945 #if AHCI_DEBUG 1946 /* 1947 * Note that the driver always uses the slot 0 to deliver 1948 * REQUEST SENSE or READ LOG EXT command 1949 */ 1950 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1951 ASSERT(rval == 0); 1952 #endif 1953 1954 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1955 cv_wait(&ahci_portp->ahciport_cv, 1956 &ahci_portp->ahciport_mutex); 1957 1958 return (AHCI_SUCCESS); 1959 } 1960 } 1961 1962 /* 1963 * Searches for and claims a free command slot. 1964 * 1965 * Returns value: 1966 * 1967 * AHCI_FAILURE returned only if 1968 * 1. No empty slot left 1969 * 2. Non-queued command requested while queued command(s) is outstanding 1970 * 3. Queued command requested while non-queued command(s) is outstanding 1971 * 4. HBA doesn't support multiple-use of command list while already a 1972 * non-queued command is oustanding 1973 * 5. Queued command requested while some queued command(s) has been 1974 * outstanding on a different port multiplier port. (AHCI spec 1.2, 1975 * 9.1.2) 1976 * 1977 * claimed slot number returned if succeeded 1978 * 1979 * NOTE: it will always return slot 0 for following commands to simplify the 1980 * algorithm. 1981 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process 1982 * 2. READ/WRITE PORTMULT command 1983 */ 1984 static int 1985 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1986 ahci_addr_t *addrp, int command_type) 1987 { 1988 uint32_t port_cmd_issue; 1989 uint32_t free_slots; 1990 int slot; 1991 1992 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 1993 1994 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1995 "ahciport_pending_tags = 0x%x " 1996 "ahciport_pending_ncq_tags = 0x%x", 1997 ahci_portp->ahciport_pending_tags, 1998 ahci_portp->ahciport_pending_ncq_tags); 1999 2000 free_slots = 0; 2001 /* 2002 * According to the AHCI spec, system software is responsible to 2003 * ensure that queued and non-queued commands are not mixed in 2004 * the command list. 2005 */ 2006 if (command_type == AHCI_NON_NCQ_CMD) { 2007 /* Non-NCQ command request */ 2008 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2009 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 2010 "ahci_claim_free_slot: there is still pending " 2011 "queued command(s) in the command list, " 2012 "so no available slot for the non-queued " 2013 "command", NULL); 2014 return (AHCI_FAILURE); 2015 } 2016 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 2017 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 2018 "ahci_claim_free_slot: there is still pending " 2019 "read/write port-mult command(s) in command list, " 2020 "so no available slot for the non-queued command", 2021 NULL); 2022 return (AHCI_FAILURE); 2023 } 2024 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 2025 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2026 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2027 "ahci_claim_free_slot: HBA cannot support multiple-" 2028 "use of the command list for non-queued commands", 2029 NULL); 2030 return (AHCI_FAILURE); 2031 } 2032 free_slots = (~ahci_portp->ahciport_pending_tags) & 2033 AHCI_SLOT_MASK(ahci_ctlp); 2034 } else if (command_type == AHCI_NCQ_CMD) { 2035 /* NCQ command request */ 2036 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2037 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 2038 "ahci_claim_free_slot: there is still pending " 2039 "non-queued command(s) in the command list, " 2040 "so no available slot for the queued command", 2041 NULL); 2042 return (AHCI_FAILURE); 2043 } 2044 2045 /* 2046 * NCQ commands cannot be sent to different port multiplier 2047 * ports in Command-Based Switching mode 2048 */ 2049 /* 2050 * NOTE: In Command-Based Switching mode, AHCI controller 2051 * usually reports a 'Handshake Error' when multiple NCQ 2052 * commands are outstanding simultaneously. 2053 */ 2054 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 2055 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 2056 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) && 2057 NCQ_CMD_IN_PROGRESS(ahci_portp) && 2058 AHCIPORT_NCQ_PMPORT(ahci_portp) != 2059 addrp->aa_pmport) { 2060 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2061 "ahci_claim_free_slot: there is still " 2062 "pending queued command(s) in the " 2063 "command list for another Port Multiplier " 2064 "port, so no available slot.", NULL); 2065 return (AHCI_FAILURE); 2066 } 2067 } 2068 2069 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 2070 AHCI_NCQ_SLOT_MASK(ahci_portp); 2071 } else if (command_type == AHCI_ERR_RETRI_CMD) { 2072 /* Error retrieval command request */ 2073 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2074 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 2075 "SENSE or READ LOG EXT command", NULL); 2076 slot = 0; 2077 goto out; 2078 } else if (command_type == AHCI_RDWR_PMULT_CMD) { 2079 /* 2080 * An extra check on PxCI. Sometimes PxCI bits may not be 2081 * cleared during hot-plug or error recovery process. 2082 */ 2083 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2084 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port)); 2085 2086 if (port_cmd_issue != 0) { 2087 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 2088 "ahci_claim_free_slot: there is still pending " 2089 "command(s) in command list (0x%x/0x%x, PxCI %x)," 2090 "so no available slot for R/W PMULT command.", 2091 NON_NCQ_CMD_IN_PROGRESS(ahci_portp), 2092 NCQ_CMD_IN_PROGRESS(ahci_portp), 2093 port_cmd_issue); 2094 return (AHCI_FAILURE); 2095 } 2096 2097 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2098 "ahci_claim_free_slot: slot 0 is allocated for " 2099 "READ/WRITE PORTMULT command", NULL); 2100 slot = 0; 2101 goto out; 2102 } 2103 2104 slot = ddi_ffs(free_slots) - 1; 2105 if (slot == -1) { 2106 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 2107 "ahci_claim_free_slot: no empty slots", NULL); 2108 return (AHCI_FAILURE); 2109 } 2110 2111 /* 2112 * According to the AHCI spec, to allow a simple mechanism for the 2113 * HBA to map command list slots to queue entries, software must 2114 * match the tag number it uses to the slot it is placing the command 2115 * in. For example, if a queued command is placed in slot 5, the tag 2116 * for that command must be 5. 2117 */ 2118 if (command_type == AHCI_NCQ_CMD) { 2119 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 2120 if (AHCI_ADDR_IS_PMPORT(addrp)) { 2121 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 2122 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport; 2123 } 2124 } 2125 2126 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 2127 2128 out: 2129 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 2130 "ahci_claim_free_slot: found slot: 0x%x", slot); 2131 2132 return (slot); 2133 } 2134 2135 /* 2136 * Builds the Command Table for the sata packet and delivers it to controller. 2137 * 2138 * Returns: 2139 * slot number if we can obtain a slot successfully 2140 * otherwise, return AHCI_FAILURE 2141 */ 2142 static int 2143 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 2144 ahci_addr_t *addrp, sata_pkt_t *spkt) 2145 { 2146 int cmd_slot; 2147 sata_cmd_t *scmd; 2148 ahci_fis_h2d_register_t *h2d_register_fisp; 2149 ahci_cmd_table_t *cmd_table; 2150 ahci_cmd_header_t *cmd_header; 2151 int ncookies; 2152 int i; 2153 int command_type = AHCI_NON_NCQ_CMD; 2154 int ncq_qdepth; 2155 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 2156 uint8_t port, pmport; 2157 #if AHCI_DEBUG 2158 uint32_t *ptr; 2159 uint8_t *ptr2; 2160 #endif 2161 2162 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2163 2164 port = addrp->aa_port; 2165 pmport = addrp->aa_pmport; 2166 2167 spkt->satapkt_reason = SATA_PKT_BUSY; 2168 2169 scmd = &spkt->satapkt_cmd; 2170 2171 /* Check if the command is a NCQ command */ 2172 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 2173 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 2174 command_type = AHCI_NCQ_CMD; 2175 2176 /* 2177 * When NCQ is support, system software must determine the 2178 * maximum tag allowed by the device and the HBA, and it 2179 * must use a value not beyond of the lower bound of the two. 2180 * 2181 * Sata module is going to calculate the qdepth and send 2182 * down to HBA driver via sata_cmd. 2183 */ 2184 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 2185 2186 /* 2187 * At the moment, the driver doesn't support the dynamic 2188 * setting of the maximum ncq depth, and the value can be 2189 * set either during the attach or after hot-plug insertion. 2190 */ 2191 if (ahci_portp->ahciport_max_ncq_tags == 0) { 2192 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 2193 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 2194 "ahci_deliver_satapkt: port %d the max tags for " 2195 "NCQ command is %d", port, ncq_qdepth); 2196 } else { 2197 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 2198 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 2199 " port %d the max tag for NCQ command is " 2200 "requested to change from %d to %d, at the" 2201 " moment the driver doesn't support the " 2202 "dynamic change so it's going to " 2203 "still use the previous tag value", 2204 instance, port, 2205 ahci_portp->ahciport_max_ncq_tags, 2206 ncq_qdepth); 2207 } 2208 } 2209 } 2210 2211 /* Check if the command is an error retrieval command */ 2212 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 2213 command_type = AHCI_ERR_RETRI_CMD; 2214 2215 /* Check if the command is an read/write pmult command */ 2216 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2217 command_type = AHCI_RDWR_PMULT_CMD; 2218 2219 /* Check if there is an empty command slot */ 2220 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, 2221 addrp, command_type); 2222 if (cmd_slot == AHCI_FAILURE) { 2223 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 2224 return (AHCI_FAILURE); 2225 } 2226 2227 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 2228 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 2229 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 2230 cmd_slot, port, (void *)spkt); 2231 2232 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 2233 bzero((void *)cmd_table, ahci_cmd_table_size); 2234 2235 /* For data transfer operations, it is the H2D Register FIS */ 2236 h2d_register_fisp = 2237 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 2238 2239 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 2240 2241 /* 2242 * PMP field only make sense when target is a port multiplier or a 2243 * device behind a port multiplier. Otherwise should set it to 0. 2244 */ 2245 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2246 SET_FIS_PMP(h2d_register_fisp, pmport); 2247 2248 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 2249 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 2250 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 2251 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 2252 2253 switch (scmd->satacmd_addr_type) { 2254 2255 case 0: 2256 /* 2257 * satacmd_addr_type will be 0 for the commands below: 2258 * ATAPI command 2259 * SATAC_IDLE_IM 2260 * SATAC_STANDBY_IM 2261 * SATAC_DOWNLOAD_MICROCODE 2262 * SATAC_FLUSH_CACHE 2263 * SATAC_SET_FEATURES 2264 * SATAC_SMART 2265 * SATAC_ID_PACKET_DEVICE 2266 * SATAC_ID_DEVICE 2267 * SATAC_READ_PORTMULT 2268 * SATAC_WRITE_PORTMULT 2269 */ 2270 /* FALLTHRU */ 2271 2272 case ATA_ADDR_LBA: 2273 /* FALLTHRU */ 2274 2275 case ATA_ADDR_LBA28: 2276 /* LBA[7:0] */ 2277 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2278 2279 /* LBA[15:8] */ 2280 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2281 2282 /* LBA[23:16] */ 2283 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2284 2285 /* LBA [27:24] (also called dev_head) */ 2286 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 2287 2288 break; 2289 2290 case ATA_ADDR_LBA48: 2291 /* LBA[7:0] */ 2292 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2293 2294 /* LBA[15:8] */ 2295 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2296 2297 /* LBA[23:16] */ 2298 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2299 2300 /* LBA [31:24] */ 2301 SET_FIS_SECTOR_EXP(h2d_register_fisp, 2302 scmd->satacmd_lba_low_msb); 2303 2304 /* LBA [39:32] */ 2305 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 2306 scmd->satacmd_lba_mid_msb); 2307 2308 /* LBA [47:40] */ 2309 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 2310 scmd->satacmd_lba_high_msb); 2311 2312 /* Set dev_head */ 2313 SET_FIS_DEV_HEAD(h2d_register_fisp, 2314 scmd->satacmd_device_reg); 2315 2316 /* Set the extended sector count and features */ 2317 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 2318 scmd->satacmd_sec_count_msb); 2319 SET_FIS_FEATURES_EXP(h2d_register_fisp, 2320 scmd->satacmd_features_reg_ext); 2321 break; 2322 } 2323 2324 /* 2325 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 2326 * filled into features field, and sector count 8:15 is filled into 2327 * features (exp) field. The hba driver doesn't need to anything 2328 * special with regard to this, since sata framework has already 2329 * done so. 2330 * 2331 * However the driver needs to make sure TAG is filled into sector 2332 * field. 2333 */ 2334 if (command_type == AHCI_NCQ_CMD) { 2335 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 2336 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 2337 } 2338 2339 ncookies = scmd->satacmd_num_dma_cookies; 2340 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2341 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 2342 ncookies, ahci_dma_prdt_number); 2343 2344 ASSERT(ncookies <= ahci_dma_prdt_number); 2345 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 2346 2347 /* *** now fill the scatter gather list ******* */ 2348 for (i = 0; i < ncookies; i++) { 2349 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 2350 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 2351 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 2352 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 2353 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 2354 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 2355 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 2356 scmd->satacmd_dma_cookie_list[i].dmac_size; 2357 } 2358 2359 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2360 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 2361 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 2362 2363 /* The ACMD field is filled in for ATAPI command */ 2364 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 2365 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 2366 SATA_ATAPI_MAX_CDB_LEN); 2367 } 2368 2369 /* Set Command Header in Command List */ 2370 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 2371 BZERO_DESCR_INFO(cmd_header); 2372 BZERO_PRD_BYTE_COUNT(cmd_header); 2373 2374 /* Set the number of entries in the PRD table */ 2375 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 2376 2377 /* Set the length of the command in the CFIS area */ 2378 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 2379 2380 /* 2381 * PMP field only make sense when target is a port multiplier or a 2382 * device behind a port multiplier. Otherwise should set it to 0. 2383 */ 2384 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2385 SET_PORT_MULTI_PORT(cmd_header, pmport); 2386 2387 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 2388 "sata_data_direction = 0x%x", 2389 scmd->satacmd_flags.sata_data_direction); 2390 2391 /* Set A bit if it is an ATAPI command */ 2392 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 2393 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 2394 2395 /* Set W bit if data is going to the device */ 2396 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 2397 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 2398 2399 /* 2400 * Set the prefetchable bit - this bit is only valid if the PRDTL 2401 * field is non-zero or the ATAPI 'A' bit is set in the command 2402 * header. This bit cannot be set when using native command 2403 * queuing commands or when using FIS-based switching with a Port 2404 * multiplier. 2405 */ 2406 if (command_type != AHCI_NCQ_CMD) 2407 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 2408 2409 /* 2410 * Now remember the sata packet in ahciport_slot_pkts[]. 2411 * Error retrieval command and r/w port multiplier command will 2412 * be stored specifically for each port. 2413 */ 2414 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 2415 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2416 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 2417 2418 /* 2419 * Keep the timeout value 2420 */ 2421 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time; 2422 2423 /* 2424 * If the intial timout is less than 1 tick, then make it longer by 2425 * 1 tick to avoid immediate timeout 2426 */ 2427 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <= 2428 ahci_watchdog_timeout) 2429 ahci_portp->ahciport_slot_timeout[cmd_slot] += 2430 ahci_watchdog_timeout; 2431 2432 #if AHCI_DEBUG 2433 if (ahci_debug_flags & AHCIDBG_ATACMD && 2434 scmd->satacmd_cmd_reg != SATAC_PACKET || 2435 ahci_debug_flags & AHCIDBG_ATAPICMD && 2436 scmd->satacmd_cmd_reg == SATAC_PACKET) { 2437 2438 /* Dump the command header and table */ 2439 ahci_log(ahci_ctlp, CE_WARN, "\n"); 2440 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 2441 "0x%p cmd_reg 0x%x port %d", spkt, 2442 scmd->satacmd_cmd_reg, port); 2443 ptr = (uint32_t *)cmd_header; 2444 ahci_log(ahci_ctlp, CE_WARN, 2445 " Command Header:%8x %8x %8x %8x", 2446 ptr[0], ptr[1], ptr[2], ptr[3]); 2447 2448 /* Dump the H2D register FIS */ 2449 ptr = (uint32_t *)h2d_register_fisp; 2450 ahci_log(ahci_ctlp, CE_WARN, 2451 " Command FIS: %8x %8x %8x %8x", 2452 ptr[0], ptr[1], ptr[2], ptr[3]); 2453 2454 /* Dump the ACMD register FIS */ 2455 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 2456 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 2457 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 2458 ahci_log(ahci_ctlp, CE_WARN, 2459 " ATAPI command: %2x %2x %2x %2x " 2460 "%2x %2x %2x %2x", 2461 ptr2[8 * i], ptr2[8 * i + 1], 2462 ptr2[8 * i + 2], ptr2[8 * i + 3], 2463 ptr2[8 * i + 4], ptr2[8 * i + 5], 2464 ptr2[8 * i + 6], ptr2[8 * i + 7]); 2465 2466 /* Dump the PRDT */ 2467 for (i = 0; i < ncookies; i++) { 2468 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 2469 ahci_log(ahci_ctlp, CE_WARN, 2470 " Cookie %d: %8x %8x %8x %8x", 2471 i, ptr[0], ptr[1], ptr[2], ptr[3]); 2472 } 2473 } 2474 #endif 2475 2476 (void) ddi_dma_sync( 2477 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 2478 0, 2479 ahci_cmd_table_size, 2480 DDI_DMA_SYNC_FORDEV); 2481 2482 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 2483 cmd_slot * sizeof (ahci_cmd_header_t), 2484 sizeof (ahci_cmd_header_t), 2485 DDI_DMA_SYNC_FORDEV); 2486 2487 if ((ahci_check_dma_handle(ahci_portp-> 2488 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) || 2489 ahci_check_dma_handle(ahci_portp-> 2490 ahciport_cmd_list_dma_handle) != DDI_FM_OK) { 2491 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 2492 DDI_SERVICE_UNAFFECTED); 2493 return (AHCI_FAILURE); 2494 } 2495 2496 /* Set the corresponding bit in the PxSACT.DS for queued command */ 2497 if (command_type == AHCI_NCQ_CMD) { 2498 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2499 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 2500 (0x1 << cmd_slot)); 2501 } 2502 2503 /* Indicate to the HBA that a command is active. */ 2504 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2505 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 2506 (0x1 << cmd_slot)); 2507 2508 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 2509 "exit: port %d", port); 2510 2511 /* Make sure the command is started by the PxSACT/PxCI */ 2512 if (ahci_check_acc_handle(ahci_ctlp-> 2513 ahcictl_ahci_acc_handle) != DDI_FM_OK) { 2514 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 2515 DDI_SERVICE_UNAFFECTED); 2516 return (AHCI_FAILURE); 2517 } 2518 2519 return (cmd_slot); 2520 } 2521 2522 /* 2523 * Called by the sata framework to abort the previously sent packet(s). 2524 * 2525 * Reset device to abort commands. 2526 */ 2527 static int 2528 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 2529 { 2530 ahci_ctl_t *ahci_ctlp; 2531 ahci_port_t *ahci_portp; 2532 uint32_t slot_status = 0; 2533 uint32_t aborted_tags = 0; 2534 uint32_t finished_tags = 0; 2535 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 2536 uint8_t port; 2537 int tmp_slot; 2538 int instance = ddi_get_instance(dip); 2539 2540 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2541 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2542 2543 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2544 "ahci_tran_abort enter: port %d", port); 2545 2546 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2547 mutex_enter(&ahci_portp->ahciport_mutex); 2548 2549 /* 2550 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2551 * commands are being mopped, therefore there is nothing else to do 2552 */ 2553 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2554 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2555 "ahci_tran_abort: port %d is in " 2556 "mopping process, so just return directly ", port); 2557 mutex_exit(&ahci_portp->ahciport_mutex); 2558 return (SATA_SUCCESS); 2559 } 2560 2561 /* 2562 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT 2563 * command is being executed so no other commands is outstanding, 2564 * nothing to do. 2565 */ 2566 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) { 2567 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2568 "ahci_tran_abort: port %d is reading/writing " 2569 "port multiplier, so just return directly ", port); 2570 mutex_exit(&ahci_portp->ahciport_mutex); 2571 return (SATA_SUCCESS); 2572 } 2573 2574 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2575 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2576 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2577 /* 2578 * In case the targer driver would send the request before 2579 * sata framework can have the opportunity to process those 2580 * event reports. 2581 */ 2582 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2583 spkt->satapkt_device.satadev_state = 2584 ahci_portp->ahciport_port_state; 2585 ahci_update_sata_registers(ahci_ctlp, port, 2586 &spkt->satapkt_device); 2587 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2588 "ahci_tran_abort returning SATA_FAILURE while " 2589 "port in FAILED/SHUTDOWN/PWROFF state: " 2590 "port: %d", port); 2591 mutex_exit(&ahci_portp->ahciport_mutex); 2592 return (SATA_FAILURE); 2593 } 2594 2595 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2596 /* 2597 * ahci_intr_phyrdy_change() may have rendered it to 2598 * AHCI_PORT_TYPE_NODEV. 2599 */ 2600 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2601 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2602 spkt->satapkt_device.satadev_state = 2603 ahci_portp->ahciport_port_state; 2604 ahci_update_sata_registers(ahci_ctlp, port, 2605 &spkt->satapkt_device); 2606 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2607 "ahci_tran_abort returning SATA_FAILURE while " 2608 "no device attached: port: %d", port); 2609 mutex_exit(&ahci_portp->ahciport_mutex); 2610 return (SATA_FAILURE); 2611 } 2612 2613 if (flag == SATA_ABORT_ALL_PACKETS) { 2614 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2615 aborted_tags = ahci_portp->ahciport_pending_tags; 2616 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2617 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2618 2619 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2620 instance, port); 2621 } else { 2622 aborted_tags = 0xffffffff; 2623 /* 2624 * Aborting one specific packet, first search the 2625 * ahciport_slot_pkts[] list for matching spkt. 2626 */ 2627 for (tmp_slot = 0; 2628 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2629 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2630 aborted_tags = (0x1 << tmp_slot); 2631 break; 2632 } 2633 } 2634 2635 if (aborted_tags == 0xffffffff) { 2636 /* request packet is not on the pending list */ 2637 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2638 "Cannot find the aborting pkt 0x%p on the " 2639 "pending list", (void *)spkt); 2640 ahci_update_sata_registers(ahci_ctlp, port, 2641 &spkt->satapkt_device); 2642 mutex_exit(&ahci_portp->ahciport_mutex); 2643 return (SATA_FAILURE); 2644 } 2645 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2646 instance, port, (void *)spkt); 2647 } 2648 2649 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2650 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2651 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2652 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2653 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2654 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2655 2656 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2657 ahci_portp->ahciport_mop_in_progress++; 2658 2659 /* 2660 * To abort the packet(s), first we are trying to clear PxCMD.ST 2661 * to stop the port, and if the port can be stopped 2662 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2663 * then we just send back the aborted packet(s) with ABORTED flag 2664 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2665 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2666 * perform a COMRESET. 2667 */ 2668 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2669 ahci_portp, port, 0, NULL); 2670 2671 /* 2672 * Compute which have finished and which need to be retried. 2673 * 2674 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2675 * minus the slot_status. The aborted_tags has to be deducted by 2676 * finished_tags since we can't possibly abort a tag which had finished 2677 * already. 2678 */ 2679 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2680 finished_tags = ahci_portp->ahciport_pending_tags & 2681 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2682 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2683 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2684 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2685 2686 aborted_tags &= ~finished_tags; 2687 2688 ahci_mop_commands(ahci_ctlp, 2689 ahci_portp, 2690 slot_status, 2691 0, /* failed tags */ 2692 0, /* timeout tags */ 2693 aborted_tags, 2694 0); /* reset tags */ 2695 2696 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2697 mutex_exit(&ahci_portp->ahciport_mutex); 2698 2699 return (SATA_SUCCESS); 2700 } 2701 2702 /* 2703 * Used to do device reset and reject all the pending packets on a device 2704 * during the reset operation. 2705 * 2706 * NOTE: ONLY called by ahci_tran_reset_dport 2707 */ 2708 static int 2709 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2710 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2711 { 2712 uint32_t slot_status = 0; 2713 uint32_t reset_tags = 0; 2714 uint32_t finished_tags = 0; 2715 uint8_t port = addrp->aa_port; 2716 sata_device_t sdevice; 2717 int ret; 2718 2719 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2720 2721 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2722 "ahci_reset_device_reject_pkts on port: %d", port); 2723 2724 /* 2725 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2726 * commands are being mopped, therefore there is nothing else to do 2727 */ 2728 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2729 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2730 "ahci_reset_device_reject_pkts: port %d is in " 2731 "mopping process, so return directly ", port); 2732 return (SATA_SUCCESS); 2733 } 2734 2735 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2736 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2737 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2738 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2739 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2740 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2741 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2742 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2743 } 2744 2745 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2746 != AHCI_SUCCESS) { 2747 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2748 "Try to do a port reset after software " 2749 "reset failed", port); 2750 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 2751 if (ret != AHCI_SUCCESS) { 2752 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2753 "ahci_reset_device_reject_pkts: port %d " 2754 "failed", port); 2755 return (SATA_FAILURE); 2756 } 2757 } 2758 /* Set the reset in progress flag */ 2759 ahci_portp->ahciport_reset_in_progress = 1; 2760 2761 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2762 ahci_portp->ahciport_mop_in_progress++; 2763 2764 /* Indicate to the framework that a reset has happened */ 2765 bzero((void *)&sdevice, sizeof (sata_device_t)); 2766 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2767 sdevice.satadev_addr.pmport = 0; 2768 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2769 sdevice.satadev_state = SATA_DSTATE_RESET | 2770 SATA_DSTATE_PWR_ACTIVE; 2771 mutex_exit(&ahci_portp->ahciport_mutex); 2772 sata_hba_event_notify( 2773 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2774 &sdevice, 2775 SATA_EVNT_DEVICE_RESET); 2776 mutex_enter(&ahci_portp->ahciport_mutex); 2777 2778 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2779 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 2780 2781 /* Next try to mop the pending commands */ 2782 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2783 finished_tags = ahci_portp->ahciport_pending_tags & 2784 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2785 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2786 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2787 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2788 2789 reset_tags &= ~finished_tags; 2790 2791 ahci_mop_commands(ahci_ctlp, 2792 ahci_portp, 2793 slot_status, 2794 0, /* failed tags */ 2795 0, /* timeout tags */ 2796 0, /* aborted tags */ 2797 reset_tags); /* reset tags */ 2798 2799 return (SATA_SUCCESS); 2800 } 2801 2802 /* 2803 * Used to do device reset and reject all the pending packets on a device 2804 * during the reset operation. 2805 * 2806 * NOTE: ONLY called by ahci_tran_reset_dport 2807 */ 2808 static int 2809 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp, 2810 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2811 { 2812 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 2813 uint8_t port = addrp->aa_port; 2814 uint8_t pmport = addrp->aa_pmport; 2815 sata_device_t sdevice; 2816 2817 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2818 2819 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp, 2820 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport); 2821 2822 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2823 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2824 "ahci_reset_pmdevice_reject_pkts: port %d is in " 2825 "mopping process, so return directly ", port); 2826 return (SATA_SUCCESS); 2827 } 2828 2829 /* Checking for outstanding commands */ 2830 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2831 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2832 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2833 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2834 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2835 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2836 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2837 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2838 } 2839 2840 /* Issue SOFTWARE reset command. */ 2841 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2842 != AHCI_SUCCESS) { 2843 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2844 "Try to do a port reset after software " 2845 "reset failed", port); 2846 return (SATA_FAILURE); 2847 } 2848 2849 /* Set the reset in progress flag */ 2850 ahci_portp->ahciport_reset_in_progress = 1; 2851 2852 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2853 ahci_portp->ahciport_mop_in_progress++; 2854 2855 /* Indicate to the framework that a reset has happened */ 2856 bzero((void *)&sdevice, sizeof (sata_device_t)); 2857 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2858 sdevice.satadev_addr.pmport = pmport; 2859 if (AHCI_ADDR_IS_PMULT(addrp)) 2860 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 2861 else 2862 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT; 2863 sdevice.satadev_state = SATA_DSTATE_RESET | 2864 SATA_DSTATE_PWR_ACTIVE; 2865 mutex_exit(&ahci_portp->ahciport_mutex); 2866 sata_hba_event_notify( 2867 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2868 &sdevice, 2869 SATA_EVNT_DEVICE_RESET); 2870 mutex_enter(&ahci_portp->ahciport_mutex); 2871 2872 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2873 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET", 2874 port, pmport); 2875 2876 /* Next try to mop the pending commands */ 2877 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2878 finished_tags = ahci_portp->ahciport_pending_tags & 2879 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2880 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2881 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2882 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2883 reset_tags &= ~finished_tags; 2884 2885 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 2886 "reset_tags = %x, finished_tags = %x, slot_status = %x", 2887 reset_tags, finished_tags, slot_status); 2888 2889 /* 2890 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we 2891 * try to reset a single device behind a port multiplier will 2892 * terminate all the commands on that HBA port. We need mop these 2893 * commands as well. 2894 */ 2895 ahci_mop_commands(ahci_ctlp, 2896 ahci_portp, 2897 slot_status, 2898 0, /* failed tags */ 2899 0, /* timeout tags */ 2900 0, /* aborted tags */ 2901 reset_tags); /* reset tags */ 2902 2903 return (SATA_SUCCESS); 2904 } 2905 2906 /* 2907 * Used to do port reset and reject all the pending packets on a port during 2908 * the reset operation. 2909 */ 2910 static int 2911 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2912 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2913 { 2914 uint32_t slot_status = 0; 2915 uint32_t reset_tags = 0; 2916 uint32_t finished_tags = 0; 2917 uint8_t port = addrp->aa_port; 2918 2919 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2920 2921 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2922 "ahci_reset_port_reject_pkts at port: %d", port); 2923 2924 /* 2925 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2926 * commands are being mopped, therefore there is nothing else to do 2927 */ 2928 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2929 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2930 "ahci_reset_port_reject_pkts: port %d is in " 2931 "mopping process, so return directly ", port); 2932 return (SATA_SUCCESS); 2933 } 2934 2935 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2936 ahci_portp->ahciport_mop_in_progress++; 2937 2938 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2939 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2940 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2941 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2942 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2943 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2944 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2945 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2946 } 2947 2948 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2949 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2950 NULL) != AHCI_SUCCESS) { 2951 2952 /* Clear mop flag */ 2953 ahci_portp->ahciport_mop_in_progress--; 2954 if (ahci_portp->ahciport_mop_in_progress == 0) 2955 ahci_portp->ahciport_flags &= 2956 ~AHCI_PORT_FLAG_MOPPING; 2957 return (SATA_FAILURE); 2958 } 2959 2960 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2961 finished_tags = ahci_portp->ahciport_pending_tags & 2962 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2963 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2964 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2965 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2966 2967 reset_tags &= ~finished_tags; 2968 2969 ahci_mop_commands(ahci_ctlp, 2970 ahci_portp, 2971 slot_status, 2972 0, /* failed tags */ 2973 0, /* timeout tags */ 2974 0, /* aborted tags */ 2975 reset_tags); /* reset tags */ 2976 2977 return (SATA_SUCCESS); 2978 } 2979 2980 /* 2981 * Used to do hba reset and reject all the pending packets on all ports 2982 * during the reset operation. 2983 */ 2984 static int 2985 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2986 { 2987 ahci_port_t *ahci_portp; 2988 uint32_t slot_status[AHCI_MAX_PORTS]; 2989 uint32_t reset_tags[AHCI_MAX_PORTS]; 2990 uint32_t finished_tags[AHCI_MAX_PORTS]; 2991 int port; 2992 int ret = SATA_SUCCESS; 2993 2994 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2995 "ahci_reset_hba_reject_pkts enter", NULL); 2996 2997 bzero(slot_status, sizeof (slot_status)); 2998 bzero(reset_tags, sizeof (reset_tags)); 2999 bzero(finished_tags, sizeof (finished_tags)); 3000 3001 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3002 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3003 continue; 3004 } 3005 3006 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3007 3008 mutex_enter(&ahci_portp->ahciport_mutex); 3009 ahci_portp->ahciport_reset_in_progress = 1; 3010 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3011 slot_status[port] = ddi_get32( 3012 ahci_ctlp->ahcictl_ahci_acc_handle, 3013 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3014 reset_tags[port] = slot_status[port] & 3015 AHCI_SLOT_MASK(ahci_ctlp); 3016 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3017 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 3018 port, reset_tags[port], 3019 ahci_portp->ahciport_pending_tags); 3020 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3021 slot_status[port] = ddi_get32( 3022 ahci_ctlp->ahcictl_ahci_acc_handle, 3023 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3024 reset_tags[port] = slot_status[port] & 3025 AHCI_NCQ_SLOT_MASK(ahci_portp); 3026 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3027 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 3028 port, reset_tags[port], 3029 ahci_portp->ahciport_pending_tags); 3030 } 3031 mutex_exit(&ahci_portp->ahciport_mutex); 3032 } 3033 3034 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 3035 ret = SATA_FAILURE; 3036 } 3037 3038 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3039 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3040 continue; 3041 } 3042 3043 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3044 3045 mutex_enter(&ahci_portp->ahciport_mutex); 3046 /* 3047 * To prevent recursive enter to ahci_mop_commands, we need 3048 * check AHCI_PORT_FLAG_MOPPING flag. 3049 */ 3050 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3051 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3052 "ahci_reset_hba_reject_pkts: port %d is in " 3053 "mopping process, so return directly ", port); 3054 mutex_exit(&ahci_portp->ahciport_mutex); 3055 continue; 3056 } 3057 3058 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3059 ahci_portp->ahciport_mop_in_progress++; 3060 3061 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 3062 finished_tags[port] = 3063 ahci_portp->ahciport_pending_tags & 3064 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 3065 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 3066 finished_tags[port] = 3067 ahci_portp->ahciport_pending_ncq_tags & 3068 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 3069 3070 reset_tags[port] &= ~finished_tags[port]; 3071 3072 ahci_mop_commands(ahci_ctlp, 3073 ahci_portp, 3074 slot_status[port], 3075 0, /* failed tags */ 3076 0, /* timeout tags */ 3077 0, /* aborted tags */ 3078 reset_tags[port]); /* reset tags */ 3079 mutex_exit(&ahci_portp->ahciport_mutex); 3080 } 3081 out: 3082 return (ret); 3083 } 3084 3085 /* 3086 * Called by sata framework to reset a port(s) or device. 3087 */ 3088 static int 3089 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 3090 { 3091 ahci_ctl_t *ahci_ctlp; 3092 ahci_port_t *ahci_portp; 3093 ahci_addr_t addr; 3094 uint8_t cport = sd->satadev_addr.cport; 3095 uint8_t pmport = sd->satadev_addr.pmport; 3096 uint8_t port; 3097 int ret = SATA_SUCCESS; 3098 int instance = ddi_get_instance(dip); 3099 3100 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3101 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3102 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3103 3104 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 3105 3106 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3107 "ahci_tran_reset_dport enter: cport %d", cport); 3108 3109 switch (sd->satadev_addr.qual) { 3110 case SATA_ADDR_PMPORT: 3111 /* 3112 * If we want to issue a COMRESET on a pmport, we need to 3113 * reject the outstanding commands on that pmport. According 3114 * to AHCI spec, PxCI register could only be cleared by 3115 * clearing PxCMD.ST, which will halt the controller port - as 3116 * well as other pmports. 3117 * 3118 * Therefore we directly reset the controller port for 3119 * simplicity. ahci_tran_probe_port() will handle reset stuff 3120 * like initializing the given pmport. 3121 */ 3122 /* FALLTHRU */ 3123 case SATA_ADDR_CPORT: 3124 /* Port reset */ 3125 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3126 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3127 "port %d reset port", instance, port); 3128 3129 mutex_enter(&ahci_portp->ahciport_mutex); 3130 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr); 3131 mutex_exit(&ahci_portp->ahciport_mutex); 3132 3133 break; 3134 3135 case SATA_ADDR_DPMPORT: 3136 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3137 "port %d:%d reset device", instance, port, pmport); 3138 /* FALLTHRU */ 3139 case SATA_ADDR_DCPORT: 3140 /* Device reset */ 3141 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT) 3142 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3143 "port %d reset device", instance, port); 3144 3145 mutex_enter(&ahci_portp->ahciport_mutex); 3146 /* 3147 * software reset request must be sent to SATA_PMULT_HOSTPORT 3148 * if target is a port multiplier: 3149 */ 3150 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT && 3151 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) 3152 AHCI_ADDR_SET_PMULT(&addr, port); 3153 3154 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 3155 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 3156 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 3157 /* 3158 * In case the targer driver would send the request 3159 * before sata framework can have the opportunity to 3160 * process those event reports. 3161 */ 3162 sd->satadev_state = ahci_portp->ahciport_port_state; 3163 ahci_update_sata_registers(ahci_ctlp, port, sd); 3164 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3165 "ahci_tran_reset_dport returning SATA_FAILURE " 3166 "while port in FAILED/SHUTDOWN/PWROFF state: " 3167 "port: %d", port); 3168 mutex_exit(&ahci_portp->ahciport_mutex); 3169 ret = SATA_FAILURE; 3170 break; 3171 } 3172 3173 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == 3174 SATA_DTYPE_NONE) { 3175 /* 3176 * ahci_intr_phyrdy_change() may have rendered it to 3177 * AHCI_PORT_TYPE_NODEV. 3178 */ 3179 sd->satadev_type = SATA_DTYPE_NONE; 3180 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp, 3181 &addr); 3182 ahci_update_sata_registers(ahci_ctlp, port, sd); 3183 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3184 "ahci_tran_reset_dport returning SATA_FAILURE " 3185 "while no device attached: port: %d", port); 3186 mutex_exit(&ahci_portp->ahciport_mutex); 3187 ret = SATA_FAILURE; 3188 break; 3189 } 3190 3191 if (AHCI_ADDR_IS_PORT(&addr)) { 3192 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 3193 ahci_portp, &addr); 3194 } else { 3195 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp, 3196 ahci_portp, &addr); 3197 } 3198 3199 mutex_exit(&ahci_portp->ahciport_mutex); 3200 break; 3201 3202 case SATA_ADDR_CNTRL: 3203 /* Reset the whole controller */ 3204 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3205 "reset the whole hba", instance); 3206 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 3207 break; 3208 3209 default: 3210 ret = SATA_FAILURE; 3211 } 3212 3213 return (ret); 3214 } 3215 3216 /* 3217 * Called by sata framework to activate a port as part of hotplug. 3218 * (cfgadm -c connect satax/y) 3219 * Support port multiplier. 3220 */ 3221 static int 3222 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 3223 { 3224 ahci_ctl_t *ahci_ctlp; 3225 ahci_port_t *ahci_portp; 3226 ahci_addr_t addr; 3227 uint8_t cport = satadev->satadev_addr.cport; 3228 uint8_t pmport = satadev->satadev_addr.pmport; 3229 uint8_t port; 3230 int instance = ddi_get_instance(dip); 3231 3232 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3233 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3234 3235 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3236 "ahci_tran_hotplug_port_activate enter: cport %d", cport); 3237 3238 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3239 3240 mutex_enter(&ahci_portp->ahciport_mutex); 3241 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3242 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3243 3244 if (AHCI_ADDR_IS_PORT(&addr)) { 3245 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", 3246 instance, port); 3247 3248 /* Enable the interrupts on the port */ 3249 ahci_enable_port_intrs(ahci_ctlp, port); 3250 3251 /* 3252 * Reset the port so that the PHY communication would be 3253 * re-established. But this reset is an internal operation 3254 * and the sata module doesn't need to know about it. 3255 * Moreover, the port with a device attached will be started 3256 * too. 3257 */ 3258 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 3259 ahci_portp, port, 3260 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 3261 NULL); 3262 3263 /* 3264 * Need to check the link status and device status of the port 3265 * and consider raising power if the port was in D3 state 3266 */ 3267 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 3268 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 3269 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 3270 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3271 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated", 3272 instance, port, pmport); 3273 /* AHCI_ADDR_PMPORT */ 3274 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON; 3275 AHCIPORT_PMSTATE(ahci_portp, &addr) &= 3276 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN); 3277 } 3278 3279 satadev->satadev_state = ahci_portp->ahciport_port_state; 3280 3281 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3282 3283 mutex_exit(&ahci_portp->ahciport_mutex); 3284 return (SATA_SUCCESS); 3285 } 3286 3287 /* 3288 * Called by sata framework to deactivate a port as part of hotplug. 3289 * (cfgadm -c disconnect satax/y) 3290 * Support port multiplier. 3291 */ 3292 static int 3293 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 3294 { 3295 ahci_ctl_t *ahci_ctlp; 3296 ahci_port_t *ahci_portp; 3297 ahci_addr_t addr; 3298 uint8_t cport = satadev->satadev_addr.cport; 3299 uint8_t pmport = satadev->satadev_addr.pmport; 3300 uint8_t port; 3301 uint32_t port_scontrol; 3302 int instance = ddi_get_instance(dip); 3303 3304 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3305 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3306 3307 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3308 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport); 3309 3310 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3311 mutex_enter(&ahci_portp->ahciport_mutex); 3312 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3313 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3314 3315 if (AHCI_ADDR_IS_PORT(&addr)) { 3316 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 3317 instance, port); 3318 3319 /* Disable the interrupts on the port */ 3320 ahci_disable_port_intrs(ahci_ctlp, port); 3321 3322 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 3323 3324 /* First to abort all the pending commands */ 3325 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3326 3327 /* Then stop the port */ 3328 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3329 ahci_portp, port); 3330 } 3331 3332 /* Next put the PHY offline */ 3333 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3334 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3335 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 3336 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *) 3337 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3338 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3339 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated", 3340 instance, port, pmport); 3341 3342 ahci_disable_port_intrs(ahci_ctlp, port); 3343 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) 3344 != SATA_DTYPE_NONE) 3345 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3346 3347 /* Re-enable the interrupts for the other pmports */ 3348 ahci_enable_port_intrs(ahci_ctlp, port); 3349 } 3350 3351 /* Update port state */ 3352 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN); 3353 satadev->satadev_state = SATA_PSTATE_SHUTDOWN; 3354 3355 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3356 3357 mutex_exit(&ahci_portp->ahciport_mutex); 3358 return (SATA_SUCCESS); 3359 } 3360 3361 /* 3362 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 3363 * when a device is unplugged or a port is deactivated. 3364 */ 3365 static void 3366 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 3367 ahci_port_t *ahci_portp, uint8_t port) 3368 { 3369 uint32_t slot_status = 0; 3370 uint32_t abort_tags = 0; 3371 3372 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3373 3374 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 3375 "ahci_reject_all_abort_pkts at port: %d", port); 3376 3377 /* Read/write port multiplier command takes highest priority */ 3378 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3379 slot_status = 0x1; 3380 abort_tags = 0x1; 3381 goto out; 3382 } 3383 3384 /* 3385 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 3386 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 3387 * to get the error data, if yes when the device is removed, the 3388 * command needs to be aborted too. 3389 */ 3390 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3391 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 3392 slot_status = 0x1; 3393 abort_tags = 0x1; 3394 goto out; 3395 } else { 3396 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3397 "ahci_reject_all_abort_pkts return directly " 3398 "port %d no needs to reject any outstanding " 3399 "commands", port); 3400 return; 3401 } 3402 } 3403 3404 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3405 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3406 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3407 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 3408 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3409 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3410 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3411 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 3412 } 3413 3414 out: 3415 /* No need to do mop when there is no outstanding commands */ 3416 if (slot_status != 0) { 3417 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3418 ahci_portp->ahciport_mop_in_progress++; 3419 3420 ahci_mop_commands(ahci_ctlp, 3421 ahci_portp, 3422 slot_status, 3423 0, /* failed tags */ 3424 0, /* timeout tags */ 3425 abort_tags, /* aborting tags */ 3426 0); /* reset tags */ 3427 } 3428 } 3429 3430 #if defined(__lock_lint) 3431 static int 3432 ahci_selftest(dev_info_t *dip, sata_device_t *device) 3433 { 3434 return (SATA_SUCCESS); 3435 } 3436 #endif 3437 3438 /* 3439 * Initialize fma capabilities and register with IO fault services. 3440 */ 3441 static void 3442 ahci_fm_init(ahci_ctl_t *ahci_ctlp) 3443 { 3444 /* 3445 * Need to change iblock to priority for new MSI intr 3446 */ 3447 ddi_iblock_cookie_t fm_ibc; 3448 3449 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY, 3450 ahci_ctlp->ahcictl_dip, 3451 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 3452 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 3453 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 3454 3455 /* Only register with IO Fault Services if we have some capability */ 3456 if (ahci_ctlp->ahcictl_fm_cap) { 3457 /* Adjust access and dma attributes for FMA */ 3458 accattr.devacc_attr_access = DDI_FLAGERR_ACC; 3459 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3460 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3461 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3462 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3463 3464 /* 3465 * Register capabilities with IO Fault Services. 3466 * ahcictl_fm_cap will be updated to indicate 3467 * capabilities actually supported (not requested.) 3468 */ 3469 ddi_fm_init(ahci_ctlp->ahcictl_dip, 3470 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc); 3471 3472 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) { 3473 cmn_err(CE_WARN, "!ahci%d: fma init failed.", 3474 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 3475 return; 3476 } 3477 /* 3478 * Initialize pci ereport capabilities if ereport 3479 * capable (should always be.) 3480 */ 3481 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) || 3482 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3483 pci_ereport_setup(ahci_ctlp->ahcictl_dip); 3484 } 3485 3486 /* 3487 * Register error callback if error callback capable. 3488 */ 3489 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3490 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip, 3491 ahci_fm_error_cb, (void *) ahci_ctlp); 3492 } 3493 3494 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3495 "ahci_fm_fini: fma enabled.", NULL); 3496 } 3497 } 3498 3499 /* 3500 * Releases fma capabilities and un-registers with IO fault services. 3501 */ 3502 static void 3503 ahci_fm_fini(ahci_ctl_t *ahci_ctlp) 3504 { 3505 /* Only unregister FMA capabilities if registered */ 3506 if (ahci_ctlp->ahcictl_fm_cap) { 3507 /* 3508 * Un-register error callback if error callback capable. 3509 */ 3510 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3511 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip); 3512 } 3513 3514 /* 3515 * Release any resources allocated by pci_ereport_setup() 3516 */ 3517 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) || 3518 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3519 pci_ereport_teardown(ahci_ctlp->ahcictl_dip); 3520 } 3521 3522 /* Unregister from IO Fault Services */ 3523 ddi_fm_fini(ahci_ctlp->ahcictl_dip); 3524 3525 /* Adjust access and dma attributes for FMA */ 3526 accattr.devacc_attr_access = DDI_DEFAULT_ACC; 3527 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3528 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3529 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3530 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3531 3532 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3533 "ahci_fm_fini: fma disabled.", NULL); 3534 } 3535 } 3536 3537 /*ARGSUSED*/ 3538 static int 3539 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 3540 { 3541 /* 3542 * as the driver can always deal with an error in any dma or 3543 * access handle, we can just return the fme_status value. 3544 */ 3545 pci_ereport_post(dip, err, NULL); 3546 return (err->fme_status); 3547 } 3548 3549 int 3550 ahci_check_acc_handle(ddi_acc_handle_t handle) 3551 { 3552 ddi_fm_error_t de; 3553 3554 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 3555 return (de.fme_status); 3556 } 3557 3558 int 3559 ahci_check_dma_handle(ddi_dma_handle_t handle) 3560 { 3561 ddi_fm_error_t de; 3562 3563 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 3564 return (de.fme_status); 3565 } 3566 3567 /* 3568 * Generate an ereport 3569 */ 3570 void 3571 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail) 3572 { 3573 uint64_t ena; 3574 char buf[FM_MAX_CLASS]; 3575 3576 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 3577 ena = fm_ena_generate(0, FM_ENA_FMT1); 3578 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3579 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena, 3580 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 3581 FM_EREPORT_VERSION, NULL); 3582 } 3583 } 3584 3585 /* 3586 * Check if all handles are correctly allocated. 3587 */ 3588 static int 3589 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp) 3590 { 3591 int port; 3592 3593 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 3594 return (DDI_FAILURE); 3595 } 3596 3597 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3598 ahci_port_t *ahci_portp; 3599 3600 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3601 continue; 3602 3603 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3604 3605 mutex_enter(&ahci_portp->ahciport_mutex); 3606 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) { 3607 mutex_exit(&ahci_portp->ahciport_mutex); 3608 return (DDI_FAILURE); 3609 } 3610 mutex_exit(&ahci_portp->ahciport_mutex); 3611 } 3612 3613 return (DDI_SUCCESS); 3614 } 3615 3616 /* 3617 * Check the access handles for the controller. Note that 3618 * ahcictl_pci_conf_handle is only used in attach process. 3619 */ 3620 static int 3621 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp) 3622 { 3623 if ((ahci_check_acc_handle(ahci_ctlp-> 3624 ahcictl_pci_conf_handle) != DDI_FM_OK) || 3625 (ahci_check_acc_handle(ahci_ctlp-> 3626 ahcictl_ahci_acc_handle) != DDI_FM_OK)) { 3627 return (DDI_FAILURE); 3628 } 3629 return (DDI_SUCCESS); 3630 } 3631 3632 /* 3633 * Check the DMA handles and the access handles of a controller port. 3634 */ 3635 static int 3636 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port) 3637 { 3638 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3639 int slot; 3640 3641 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3642 3643 if ((ahci_check_dma_handle(ahci_portp-> 3644 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) || 3645 (ahci_check_dma_handle(ahci_portp-> 3646 ahciport_cmd_list_dma_handle) != DDI_FM_OK) || 3647 (ahci_check_acc_handle(ahci_portp-> 3648 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) || 3649 (ahci_check_acc_handle(ahci_portp-> 3650 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) { 3651 return (DDI_FAILURE); 3652 } 3653 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 3654 if (ahci_check_slot_handle(ahci_portp, slot) 3655 != DDI_SUCCESS) { 3656 return (DDI_FAILURE); 3657 } 3658 } 3659 return (DDI_SUCCESS); 3660 } 3661 3662 /* 3663 * Check the DMA handles and the access handles of a cmd table slot. 3664 */ 3665 static int 3666 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot) 3667 { 3668 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3669 3670 if ((ahci_check_acc_handle(ahci_portp-> 3671 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) || 3672 (ahci_check_dma_handle(ahci_portp-> 3673 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) { 3674 return (DDI_FAILURE); 3675 } 3676 return (DDI_SUCCESS); 3677 } 3678 3679 /* 3680 * Allocate the ports structure, only called by ahci_attach 3681 */ 3682 static int 3683 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 3684 { 3685 int port, cport = 0; 3686 3687 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3688 "ahci_alloc_ports_state enter", NULL); 3689 3690 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3691 3692 /* Allocate structures only for the implemented ports */ 3693 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3694 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3695 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3696 "hba port %d not implemented", port); 3697 continue; 3698 } 3699 3700 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 3701 ahci_ctlp->ahcictl_port_to_cport[port] = 3702 (uint8_t)cport++; 3703 3704 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 3705 goto err_out; 3706 } 3707 } 3708 3709 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3710 return (AHCI_SUCCESS); 3711 3712 err_out: 3713 for (port--; port >= 0; port--) { 3714 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3715 ahci_dealloc_port_state(ahci_ctlp, port); 3716 } 3717 } 3718 3719 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3720 return (AHCI_FAILURE); 3721 } 3722 3723 /* 3724 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 3725 */ 3726 static void 3727 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 3728 { 3729 int port; 3730 3731 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3732 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3733 /* if this port is implemented by the HBA */ 3734 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3735 ahci_dealloc_port_state(ahci_ctlp, port); 3736 } 3737 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3738 } 3739 3740 /* 3741 * Drain the taskq. 3742 */ 3743 static void 3744 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 3745 { 3746 ahci_port_t *ahci_portp; 3747 int port; 3748 3749 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3750 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3751 continue; 3752 } 3753 3754 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3755 3756 mutex_enter(&ahci_portp->ahciport_mutex); 3757 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 3758 mutex_exit(&ahci_portp->ahciport_mutex); 3759 } 3760 } 3761 3762 /* 3763 * Initialize the controller and all ports. And then try to start the ports 3764 * if there are devices attached. 3765 * 3766 * This routine can be called from three seperate cases: DDI_ATTACH, 3767 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 3768 * other two cases; device signature probing are attempted only during 3769 * DDI_ATTACH case. 3770 */ 3771 static int 3772 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 3773 { 3774 ahci_port_t *ahci_portp; 3775 ahci_addr_t addr; 3776 int port; 3777 3778 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3779 "ahci_initialize_controller enter", NULL); 3780 3781 /* Disable the whole controller interrupts */ 3782 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3783 ahci_disable_all_intrs(ahci_ctlp); 3784 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3785 3786 /* Initialize the implemented ports and structures */ 3787 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3788 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3789 continue; 3790 } 3791 3792 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3793 mutex_enter(&ahci_portp->ahciport_mutex); 3794 3795 /* 3796 * Ensure that the controller is not in the running state 3797 * by checking every implemented port's PxCMD register 3798 */ 3799 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port); 3800 3801 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr) 3802 != AHCI_SUCCESS) { 3803 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3804 "ahci_initialize_controller: failed to " 3805 "initialize port %d", port); 3806 /* 3807 * Set the port state to SATA_PSTATE_FAILED if 3808 * failed to initialize it. 3809 */ 3810 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3811 } 3812 3813 mutex_exit(&ahci_portp->ahciport_mutex); 3814 } 3815 3816 /* Enable the whole controller interrupts */ 3817 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3818 ahci_enable_all_intrs(ahci_ctlp); 3819 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3820 3821 return (AHCI_SUCCESS); 3822 } 3823 3824 /* 3825 * Reverse of ahci_initialize_controller() 3826 * 3827 * We only need to stop the ports and disable the interrupt. 3828 */ 3829 static void 3830 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 3831 { 3832 ahci_port_t *ahci_portp; 3833 int port; 3834 3835 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3836 "ahci_uninitialize_controller enter", NULL); 3837 3838 /* disable all the interrupts. */ 3839 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3840 ahci_disable_all_intrs(ahci_ctlp); 3841 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3842 3843 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3844 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3845 continue; 3846 } 3847 3848 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3849 3850 /* Stop the port by clearing PxCMD.ST */ 3851 mutex_enter(&ahci_portp->ahciport_mutex); 3852 3853 /* 3854 * Here we must disable the port interrupt because 3855 * ahci_disable_all_intrs only clear GHC.IE, and IS 3856 * register will be still set if PxIE is enabled. 3857 * When ahci shares one IRQ with other drivers, the 3858 * intr handler may claim the intr mistakenly. 3859 */ 3860 ahci_disable_port_intrs(ahci_ctlp, port); 3861 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3862 ahci_portp, port); 3863 mutex_exit(&ahci_portp->ahciport_mutex); 3864 } 3865 } 3866 3867 /* 3868 * ahci_alloc_pmult() 3869 * 1. Setting HBA port registers which are necessary for a port multiplier. 3870 * (Set PxCMD.PMA while PxCMD.ST is '0') 3871 * 2. Allocate ahci_pmult_info structure. 3872 * 3873 * NOTE: Must stop port before the function is called. 3874 */ 3875 static void 3876 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3877 { 3878 uint32_t port_cmd_status; 3879 uint8_t port = ahci_portp->ahciport_port_num; 3880 3881 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3882 3883 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3884 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3885 3886 /* The port must have been stopped before. */ 3887 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST)); 3888 3889 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) { 3890 /* set PMA bit */ 3891 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3892 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3893 port_cmd_status|AHCI_CMD_STATUS_PMA); 3894 3895 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3896 "ahci_alloc_pmult: " 3897 "PxCMD.PMA bit set at port %d.", port); 3898 } 3899 3900 /* Allocate port multiplier information structure */ 3901 if (ahci_portp->ahciport_pmult_info == NULL) { 3902 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *) 3903 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP); 3904 } 3905 3906 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 3907 } 3908 3909 /* 3910 * ahci_dealloc_pmult() 3911 * 1. Clearing related registers when a port multiplier is detached. 3912 * (Clear PxCMD.PMA while PxCMD.ST is '0') 3913 * 2. Deallocate ahci_pmult_info structure. 3914 * 3915 * NOTE: Must stop port before the function is called. 3916 */ 3917 static void 3918 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3919 { 3920 uint32_t port_cmd_status; 3921 uint8_t port = ahci_portp->ahciport_port_num; 3922 3923 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3924 3925 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3926 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3927 3928 if (port_cmd_status & AHCI_CMD_STATUS_PMA) { 3929 /* Clear PMA bit */ 3930 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3931 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3932 (port_cmd_status & (~AHCI_CMD_STATUS_PMA))); 3933 3934 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3935 "ahci_dealloc_pmult: " 3936 "PxCMD.PMA bit cleared at port %d.", port); 3937 } 3938 3939 /* Release port multiplier information structure */ 3940 if (ahci_portp->ahciport_pmult_info != NULL) { 3941 kmem_free(ahci_portp->ahciport_pmult_info, 3942 sizeof (ahci_pmult_info_t)); 3943 ahci_portp->ahciport_pmult_info = NULL; 3944 } 3945 } 3946 3947 /* 3948 * Staggered Spin-up. 3949 */ 3950 static void 3951 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port) 3952 { 3953 uint32_t cap_status; 3954 uint32_t port_cmd_status; 3955 3956 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 3957 3958 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3959 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3960 3961 /* Check for staggered spin-up support */ 3962 if (!(cap_status & AHCI_HBA_CAP_SSS)) 3963 return; 3964 3965 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3966 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3967 3968 /* If PxCMD.SUD == 1, no staggered spin-up is needed */ 3969 if (port_cmd_status & AHCI_CMD_STATUS_SUD) 3970 return; 3971 3972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port); 3973 3974 /* Set PxCMD.SUD */ 3975 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3976 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3977 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3978 port_cmd_status); 3979 } 3980 3981 /* 3982 * The routine is to initialize a port. First put the port in NotRunning 3983 * state, then enable port interrupt and clear Serror register. And under 3984 * AHCI_ATTACH case, find device signature and then try to start the port. 3985 * 3986 * Called by 3987 * 1. ahci_initialize_controller 3988 * 2. ahci_intr_phyrdy_change (hotplug) 3989 */ 3990 static int 3991 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3992 ahci_addr_t *addrp) 3993 { 3994 uint32_t port_sstatus, port_task_file, port_cmd_status; 3995 uint8_t port = addrp->aa_port; 3996 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */ 3997 int ret; 3998 3999 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4000 4001 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */ 4002 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 4003 4004 /* 4005 * At the time being, only probe ports/devices and get the types of 4006 * attached devices during DDI_ATTACH. In fact, the device can be 4007 * changed during power state changes, but at the time being, we 4008 * don't support the situation. 4009 */ 4010 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4011 resuming = B_FALSE; 4012 } else { 4013 /* check for DDI_RESUME case */ 4014 mutex_exit(&ahci_portp->ahciport_mutex); 4015 mutex_enter(&ahci_ctlp->ahcictl_mutex); 4016 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) 4017 resuming = B_FALSE; 4018 mutex_exit(&ahci_ctlp->ahcictl_mutex); 4019 mutex_enter(&ahci_portp->ahciport_mutex); 4020 } 4021 4022 if (resuming) { 4023 /* 4024 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 4025 * and PxFBU registers in case these registers were cleared 4026 * during the suspend. 4027 */ 4028 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4029 "ahci_initialize_port: port %d " 4030 "set PxCLB, PxCLBU, PxFB and PxFBU " 4031 "during resume", port); 4032 4033 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) != 4034 AHCI_SUCCESS) 4035 return (AHCI_FAILURE); 4036 } 4037 4038 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4039 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4040 4041 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4042 "ahci_initialize_port: port %d ", port); 4043 4044 /* 4045 * Check whether the port is in NotRunning state, if not, 4046 * put the port in NotRunning state 4047 */ 4048 if (port_cmd_status & 4049 (AHCI_CMD_STATUS_ST | 4050 AHCI_CMD_STATUS_CR | 4051 AHCI_CMD_STATUS_FRE | 4052 AHCI_CMD_STATUS_FR)) { 4053 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 4054 ahci_portp, port); 4055 } 4056 4057 /* Make sure the drive is spun-up */ 4058 ahci_staggered_spin_up(ahci_ctlp, port); 4059 4060 /* Disable interrupt */ 4061 ahci_disable_port_intrs(ahci_ctlp, port); 4062 4063 /* Device is unknown at first */ 4064 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4065 4066 /* Disable the interface power management */ 4067 ahci_disable_interface_pm(ahci_ctlp, port); 4068 4069 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4070 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 4071 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4072 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4073 4074 /* Check physcial link status */ 4075 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 4076 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 4077 4078 /* Check interface status */ 4079 port_task_file & AHCI_TFD_STS_BSY || 4080 port_task_file & AHCI_TFD_STS_DRQ || 4081 4082 /* Check whether port reset must be executed */ 4083 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET || 4084 4085 /* Always reset port on RESUME */ 4086 resuming != B_FALSE) { 4087 4088 /* Something went wrong, we need do some reset things */ 4089 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 4090 4091 /* Does port reset succeed on HBA port? */ 4092 if (ret != AHCI_SUCCESS) { 4093 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4094 "ahci_initialize_port:" 4095 "port reset failed at port %d", port); 4096 return (AHCI_FAILURE); 4097 } 4098 4099 /* Is port failed? */ 4100 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4101 SATA_PSTATE_FAILED) { 4102 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4103 "ahci_initialize_port: port %d state 0x%x", 4104 port, ahci_portp->ahciport_port_state); 4105 return (AHCI_FAILURE); 4106 } 4107 } 4108 4109 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4110 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 4111 4112 /* 4113 * Try to get the device signature if the port is not empty. 4114 */ 4115 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) != 4116 SATA_DTYPE_NONE) 4117 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4118 4119 /* Return directly if no device connected */ 4120 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) { 4121 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4122 "No device connected to port %d", port); 4123 goto out; 4124 } 4125 4126 /* If this is a port multiplier, we need do some initialization */ 4127 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 4128 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4129 "Port multiplier found at port %d", port); 4130 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 4131 } 4132 4133 /* Try to start the port */ 4134 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 4135 != AHCI_SUCCESS) { 4136 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4137 "failed to start port %d", port); 4138 return (AHCI_FAILURE); 4139 } 4140 out: 4141 /* Enable port interrupts */ 4142 ahci_enable_port_intrs(ahci_ctlp, port); 4143 4144 return (AHCI_SUCCESS); 4145 } 4146 4147 /* 4148 * Handle hardware defect, and check the capabilities. For example, 4149 * power management capabilty and MSI capability. 4150 */ 4151 static int 4152 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 4153 { 4154 ushort_t caps_ptr, cap_count, cap; 4155 #if AHCI_DEBUG 4156 ushort_t pmcap, pmcsr; 4157 ushort_t msimc; 4158 #endif 4159 uint8_t revision; 4160 4161 ahci_ctlp->ahcictl_venid = 4162 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4163 PCI_CONF_VENID); 4164 4165 ahci_ctlp->ahcictl_devid = 4166 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4167 PCI_CONF_DEVID); 4168 4169 /* 4170 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 4171 * controllers with 0x00 revision id work on 4-byte aligned buffer, 4172 * which is a bug and was fixed after 0x00 revision id controllers. 4173 * 4174 * Moreover, VT8251 cannot use multiple command slots in the command 4175 * list for non-queued commands because the previous register content 4176 * of PxCI can be re-written in the register write, so a flag will be 4177 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 4178 * 4179 * For VT8251, software reset also has the same defect as the below 4180 * AMD/ATI chipset. That is, software reset will get failed if 0xf 4181 * is filled in pmport field. Therefore, another software reset need 4182 * to be done with 0 filled in pmport field. 4183 */ 4184 if (ahci_ctlp->ahcictl_venid == VIA_VENID) { 4185 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 4186 PCI_CONF_REVID); 4187 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4188 "revision id = 0x%x", revision); 4189 if (revision == 0x00) { 4190 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 4191 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4192 "change ddi_attr_align to 0x4", NULL); 4193 } 4194 4195 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE; 4196 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4197 "VT8251 cannot use multiple command lists for " 4198 "non-queued commands", NULL); 4199 4200 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4201 } 4202 4203 /* 4204 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit 4205 * DMA addressing for communication memory descriptors though S64A bit 4206 * of CAP register declares it supports. Even though 64-bit DMA for 4207 * data buffer works on ASUS M2A-VM with newer BIOS, three other 4208 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and 4209 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller. 4210 * 4211 * Due to certain hardware issue, the chipset must do port reset during 4212 * initialization, otherwise, when retrieving device signature, 4213 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag 4214 * need to set. 4215 * 4216 * For this chipset software reset will get failure if the pmport of 4217 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port 4218 * multiplier is connected to the port. In order to fix the issue, 4219 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software 4220 * reset got failure, the driver will try to do another software reset 4221 * with pmport 0. 4222 */ 4223 if (ahci_ctlp->ahcictl_venid == 0x1002 && 4224 ahci_ctlp->ahcictl_devid == 0x4380) { 4225 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 4226 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 4227 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 4228 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4229 4230 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4231 "ATI SB600 cannot do 64-bit DMA for both data buffer and " 4232 "communication memory descriptors though CAP indicates " 4233 "support, so force it to use 32-bit DMA", NULL); 4234 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4235 "ATI SB600 need to do a port reset during initialization", 4236 NULL); 4237 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4238 "ATI SB600 will get software reset failure if pmport " 4239 "is set 0xf and no port multiplier is attached", NULL); 4240 } 4241 4242 /* 4243 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same 4244 * vendor ID and device ID (0x1002,0x4391). 4245 * 4246 * SB700/750 AHCI chipset on some boards doesn't support 64-bit 4247 * DMA addressing for communication memory descriptors though S64A bit 4248 * of CAP register declares the support. However, it does support 4249 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is 4250 * set for this controller. 4251 * 4252 * SB710 has the same initialization issue as SB600, so it also need 4253 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it. 4254 * 4255 * SB700 also has the same issue about software reset, and thus 4256 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed. 4257 */ 4258 if (ahci_ctlp->ahcictl_venid == 0x1002 && 4259 ahci_ctlp->ahcictl_devid == 0x4391) { 4260 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 4261 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 4262 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4263 4264 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4265 "ATI SB700/750 cannot do 64-bit DMA for communication " 4266 "memory descriptors though CAP indicates support, " 4267 "so force it to use 32-bit DMA", NULL); 4268 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4269 "ATI SB710 need to do a port reset during initialization", 4270 NULL); 4271 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4272 "ATI SB700 will get software reset failure if pmport " 4273 "is set 0xf and no port multiplier is attached", NULL); 4274 } 4275 4276 /* 4277 * Check if capabilities list is supported and if so, 4278 * get initial capabilities pointer and clear bits 0,1. 4279 */ 4280 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4281 PCI_CONF_STAT) & PCI_STAT_CAP) { 4282 caps_ptr = P2ALIGN(pci_config_get8( 4283 ahci_ctlp->ahcictl_pci_conf_handle, 4284 PCI_CONF_CAP_PTR), 4); 4285 } else { 4286 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 4287 } 4288 4289 /* 4290 * Walk capabilities if supported. 4291 */ 4292 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 4293 4294 /* 4295 * Check that we haven't exceeded the maximum number of 4296 * capabilities and that the pointer is in a valid range. 4297 */ 4298 if (++cap_count > PCI_CAP_MAX_PTR) { 4299 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4300 "too many device capabilities", NULL); 4301 return (AHCI_FAILURE); 4302 } 4303 if (caps_ptr < PCI_CAP_PTR_OFF) { 4304 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4305 "capabilities pointer 0x%x out of range", 4306 caps_ptr); 4307 return (AHCI_FAILURE); 4308 } 4309 4310 /* 4311 * Get next capability and check that it is valid. 4312 * For now, we only support power management. 4313 */ 4314 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 4315 caps_ptr); 4316 switch (cap) { 4317 case PCI_CAP_ID_PM: 4318 4319 /* power management supported */ 4320 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 4321 4322 /* Save PMCSR offset */ 4323 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 4324 4325 #if AHCI_DEBUG 4326 pmcap = pci_config_get16( 4327 ahci_ctlp->ahcictl_pci_conf_handle, 4328 caps_ptr + PCI_PMCAP); 4329 pmcsr = pci_config_get16( 4330 ahci_ctlp->ahcictl_pci_conf_handle, 4331 ahci_ctlp->ahcictl_pmcsr_offset); 4332 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4333 "Power Management capability found PCI_PMCAP " 4334 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 4335 if ((pmcap & 0x3) == 0x3) 4336 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4337 "PCI Power Management Interface " 4338 "spec 1.2 compliant", NULL); 4339 #endif 4340 break; 4341 4342 case PCI_CAP_ID_MSI: 4343 #if AHCI_DEBUG 4344 msimc = pci_config_get16( 4345 ahci_ctlp->ahcictl_pci_conf_handle, 4346 caps_ptr + PCI_MSI_CTRL); 4347 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 4348 "Message Signaled Interrupt capability found " 4349 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 4350 #endif 4351 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 4352 "MSI capability found", NULL); 4353 break; 4354 4355 case PCI_CAP_ID_PCIX: 4356 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4357 "PCI-X capability found", NULL); 4358 break; 4359 4360 case PCI_CAP_ID_PCI_E: 4361 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4362 "PCI Express capability found", NULL); 4363 break; 4364 4365 case PCI_CAP_ID_MSI_X: 4366 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4367 "MSI-X capability found", NULL); 4368 break; 4369 4370 case PCI_CAP_ID_SATA: 4371 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4372 "SATA capability found", NULL); 4373 break; 4374 4375 case PCI_CAP_ID_VS: 4376 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4377 "Vendor Specific capability found", NULL); 4378 break; 4379 4380 default: 4381 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4382 "unrecognized capability 0x%x", cap); 4383 break; 4384 } 4385 4386 /* 4387 * Get next capabilities pointer and clear bits 0,1. 4388 */ 4389 caps_ptr = P2ALIGN(pci_config_get8( 4390 ahci_ctlp->ahcictl_pci_conf_handle, 4391 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 4392 } 4393 4394 return (AHCI_SUCCESS); 4395 } 4396 4397 /* 4398 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE 4399 * PORTMULT command. SYNC & POLLING mode is used. 4400 */ 4401 static int 4402 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4403 uint8_t regn, uint32_t *pregv, uint8_t type) 4404 { 4405 ahci_port_t *ahci_portp; 4406 ahci_addr_t pmult_addr; 4407 sata_pkt_t *spkt; 4408 sata_cmd_t *scmd; 4409 sata_device_t sata_device; 4410 uint8_t port = addrp->aa_port; 4411 uint8_t pmport = addrp->aa_pmport; 4412 uint8_t cport; 4413 uint32_t intr_mask; 4414 int rval; 4415 char portstr[10]; 4416 4417 SET_PORTSTR(portstr, addrp); 4418 cport = ahci_ctlp->ahcictl_port_to_cport[port]; 4419 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4420 4421 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp)); 4422 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4423 4424 /* Check the existence of the port multiplier */ 4425 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) 4426 return (AHCI_FAILURE); 4427 4428 /* Request a READ/WRITE PORTMULT sata packet. */ 4429 bzero(&sata_device, sizeof (sata_device_t)); 4430 sata_device.satadev_addr.cport = cport; 4431 sata_device.satadev_addr.pmport = pmport; 4432 sata_device.satadev_addr.qual = SATA_ADDR_PMULT; 4433 sata_device.satadev_rev = SATA_DEVICE_REV; 4434 4435 /* 4436 * Make sure no command is outstanding here. All R/W PMULT requests 4437 * come from 4438 * 4439 * 1. ahci_attach() 4440 * The port should be empty. 4441 * 4442 * 2. ahci_tran_probe_port() 4443 * Any request from SATA framework (via ahci_tran_start) should be 4444 * rejected if R/W PMULT command is outstanding. 4445 * 4446 * If we are doing mopping, do not check those flags because no 4447 * command will be actually outstanding. 4448 * 4449 * If the port has been occupied by any other commands, the probe 4450 * function will return a SATA_RETRY. SATA framework will retry 4451 * later. 4452 */ 4453 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 4454 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4455 "R/W PMULT failed: R/W PMULT in progress at port %d.", 4456 port, ahci_portp->ahciport_flags); 4457 return (AHCI_FAILURE); 4458 } 4459 4460 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && ( 4461 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 4462 NCQ_CMD_IN_PROGRESS(ahci_portp) || 4463 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) { 4464 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4465 "R/W PMULT failed: port %d is occupied (flags 0x%x).", 4466 port, ahci_portp->ahciport_flags); 4467 return (AHCI_FAILURE); 4468 } 4469 4470 /* 4471 * The port multiplier is gone. This may happen when 4472 * 1. Cutting off the power of an enclosure. The device lose the power 4473 * before port multiplier. 4474 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive. 4475 * 4476 * The issued command should be aborted and the following command 4477 * should not be continued. 4478 */ 4479 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) { 4480 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4481 "READ/WRITE PMULT failed: " 4482 "port-mult is removed from port %d", port); 4483 return (AHCI_FAILURE); 4484 } 4485 4486 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT; 4487 4488 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip, 4489 &sata_device, regn, *pregv, type); 4490 4491 /* 4492 * READ/WRITE PORTMULT command is intended to sent to the control port 4493 * of the port multiplier. 4494 */ 4495 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port); 4496 4497 ahci_portp->ahciport_rdwr_pmult_pkt = spkt; 4498 4499 /* No interrupt here. Store the interrupt enable mask. */ 4500 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4501 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4502 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4503 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 4504 4505 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt); 4506 4507 if (rval == AHCI_SUCCESS && 4508 spkt->satapkt_reason == SATA_PKT_COMPLETED) { 4509 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) { 4510 scmd = &spkt->satapkt_cmd; 4511 *pregv = scmd->satacmd_lba_high_lsb << 24 | 4512 scmd->satacmd_lba_mid_lsb << 16 | 4513 scmd->satacmd_lba_low_lsb << 8 | 4514 scmd->satacmd_sec_count_lsb; 4515 } 4516 } else { 4517 /* Failed or not completed. */ 4518 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4519 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s", 4520 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write", 4521 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr); 4522 rval = AHCI_FAILURE; 4523 } 4524 out: 4525 /* Restore the interrupt mask */ 4526 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4527 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask); 4528 4529 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT; 4530 ahci_portp->ahciport_rdwr_pmult_pkt = NULL; 4531 sata_free_rdwr_pmult_pkt(spkt); 4532 return (rval); 4533 } 4534 4535 static int 4536 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4537 uint8_t regn, uint32_t *pregv) 4538 { 4539 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv, 4540 SATA_RDWR_PMULT_PKT_TYPE_READ); 4541 } 4542 4543 static int 4544 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4545 uint8_t regn, uint32_t regv) 4546 { 4547 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v, 4548 SATA_RDWR_PMULT_PKT_TYPE_WRITE); 4549 } 4550 4551 #define READ_PMULT(addrp, r, pv, out) \ 4552 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \ 4553 goto out; 4554 4555 #define WRITE_PMULT(addrp, r, v, out) \ 4556 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \ 4557 goto out; 4558 4559 /* 4560 * Update sata registers on port multiplier, including GSCR/PSCR registers. 4561 * ahci_update_pmult_gscr() 4562 * ahci_update_pmult_pscr() 4563 */ 4564 static int 4565 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4566 sata_pmult_gscr_t *sg) 4567 { 4568 ASSERT(MUTEX_HELD( 4569 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex)); 4570 4571 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err); 4572 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err); 4573 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err); 4574 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err); 4575 4576 return (AHCI_SUCCESS); 4577 4578 err: /* R/W PMULT error */ 4579 return (AHCI_FAILURE); 4580 } 4581 4582 static int 4583 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4584 sata_device_t *sd) 4585 { 4586 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4587 ASSERT(MUTEX_HELD( 4588 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex)); 4589 4590 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err); 4591 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err); 4592 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err); 4593 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err); 4594 4595 return (AHCI_SUCCESS); 4596 4597 err: /* R/W PMULT error */ 4598 return (AHCI_FAILURE); 4599 } 4600 4601 /* 4602 * ahci_initialize_pmult() 4603 * 4604 * Initialize a port multiplier, including 4605 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16) 4606 * 2. Redefine MASK register. (SATA Chap 16.?) 4607 */ 4608 static int 4609 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4610 ahci_addr_t *addrp, sata_device_t *sd) 4611 { 4612 sata_pmult_gscr_t sg; 4613 uint32_t gscr64; 4614 uint8_t port = addrp->aa_port; 4615 4616 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4617 4618 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4619 "[Initialize] Port-multiplier at port %d.", port); 4620 4621 /* 4622 * Enable features of port multiplier. Currently only 4623 * Asynchronous Notification is enabled. 4624 */ 4625 /* Check gscr64 for supported features. */ 4626 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err); 4627 4628 if (gscr64 & SATA_PMULT_CAP_SNOTIF) { 4629 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4630 "port %d: Port Multiplier supports " 4631 "Asynchronous Notification.", port); 4632 4633 /* Write to gscr96 to enabled features */ 4634 WRITE_PMULT(addrp, SATA_PMULT_GSCR96, 4635 SATA_PMULT_CAP_SNOTIF, err); 4636 4637 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4638 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4639 AHCI_SNOTIF_CLEAR_ALL); 4640 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4641 "port %d: PMult PxSNTF cleared.", port); 4642 4643 } 4644 4645 /* 4646 * Now we need to update gscr33 register to enable hot-plug interrupt 4647 * for sub devices behind port multiplier. 4648 */ 4649 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err); 4650 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4651 "port %d: gscr33 mask set to %x.", port, (0x1ffff)); 4652 4653 /* 4654 * Fetch the number of device ports of the port multiplier 4655 */ 4656 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS) 4657 return (AHCI_FAILURE); 4658 4659 /* Register the port multiplier to SATA Framework. */ 4660 mutex_exit(&ahci_portp->ahciport_mutex); 4661 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg); 4662 mutex_enter(&ahci_portp->ahciport_mutex); 4663 4664 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports = 4665 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK; 4666 4667 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4668 "port %d: pmult sub-port number updated to %x.", port, 4669 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports); 4670 4671 /* Till now port-mult is successfully initialized */ 4672 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT; 4673 return (AHCI_SUCCESS); 4674 4675 err: /* R/W PMULT error */ 4676 return (AHCI_FAILURE); 4677 } 4678 4679 /* 4680 * Initialize a port multiplier port. According to spec, firstly we need 4681 * issue a COMRESET, then a software reset to get its signature. 4682 * 4683 * NOTE: This function should only be called in ahci_probe_pmport() 4684 */ 4685 static int 4686 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4687 ahci_addr_t *addrp) 4688 { 4689 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 4690 uint8_t port = addrp->aa_port; 4691 uint8_t pmport = addrp->aa_pmport; 4692 int ret = AHCI_FAILURE; 4693 4694 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4695 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4696 4697 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4698 "ahci_initialize_pmport: port %d:%d", port, pmport); 4699 4700 /* Check HBA port state */ 4701 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4702 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4703 "ahci_initialize_pmport:" 4704 "port %d:%d Port Multiplier is failed.", 4705 port, pmport); 4706 return (AHCI_FAILURE); 4707 } 4708 4709 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4710 return (AHCI_FAILURE); 4711 } 4712 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 4713 4714 /* Checking for outstanding commands */ 4715 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4716 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4717 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4718 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4719 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4720 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4721 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4722 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4723 } 4724 4725 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 4726 ahci_portp->ahciport_mop_in_progress++; 4727 4728 /* Clear status */ 4729 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN); 4730 4731 /* Firstly assume an unknown device */ 4732 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4733 4734 ahci_disable_port_intrs(ahci_ctlp, port); 4735 4736 /* port reset is necessary for port multiplier port */ 4737 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) { 4738 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4739 "ahci_initialize_pmport:" 4740 "port reset failed at port %d:%d", 4741 port, pmport); 4742 goto out; 4743 } 4744 4745 /* Is port failed? */ 4746 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4747 SATA_PSTATE_FAILED) { 4748 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4749 "ahci_initialize_pmport: port %d:%d failed. " 4750 "state = 0x%x", port, pmport, 4751 ahci_portp->ahciport_port_state); 4752 goto out; 4753 } 4754 4755 /* Is there any device attached? */ 4756 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp) 4757 == SATA_DTYPE_NONE) { 4758 /* Do not waste time on an empty port */ 4759 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4760 "ahci_initialize_pmport: No device is found " 4761 "at port %d:%d", port, pmport); 4762 ret = AHCI_SUCCESS; 4763 goto out; 4764 } 4765 4766 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4767 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4768 "port %d:%d is ready now.", port, pmport); 4769 4770 /* 4771 * Till now we can assure a device attached to that HBA port and work 4772 * correctly. Now try to get the device signature. This is an optional 4773 * step. If failed, unknown device is assumed, then SATA module will 4774 * continue to use IDENTIFY DEVICE to get the information of the 4775 * device. 4776 */ 4777 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4778 4779 ret = AHCI_SUCCESS; 4780 4781 out: 4782 /* Next try to mop the pending commands */ 4783 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 4784 finished_tags = ahci_portp->ahciport_pending_tags & 4785 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4786 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 4787 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4788 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4789 reset_tags &= ~finished_tags; 4790 4791 ahci_mop_commands(ahci_ctlp, 4792 ahci_portp, 4793 slot_status, 4794 0, /* failed tags */ 4795 0, /* timeout tags */ 4796 0, /* aborted tags */ 4797 reset_tags); /* reset tags */ 4798 4799 /* Clear PxSNTF register if supported. */ 4800 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 4801 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4802 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4803 AHCI_SNOTIF_CLEAR_ALL); 4804 } 4805 4806 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 4807 ahci_enable_port_intrs(ahci_ctlp, port); 4808 return (ret); 4809 } 4810 4811 /* 4812 * ahci_probe_pmult() 4813 * 4814 * This function will be called to probe a port multiplier, which will 4815 * handle hotplug events on port multiplier ports. 4816 * 4817 * NOTE: Only called from ahci_tran_probe_port() 4818 */ 4819 static int 4820 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4821 ahci_addr_t *addrp) 4822 { 4823 sata_device_t sdevice; 4824 ahci_addr_t pmport_addr; 4825 uint32_t gscr32, port_hotplug_tags; 4826 uint32_t pmport_sstatus; 4827 int dev_exists_now = 0, dev_existed_previously = 0; 4828 uint8_t port = addrp->aa_port; 4829 int npmport; 4830 4831 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4832 4833 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */ 4834 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err); 4835 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp); 4836 4837 do { 4838 npmport = ddi_ffs(port_hotplug_tags) - 1; 4839 if (npmport == -1) 4840 /* no pending hot plug events. */ 4841 return (AHCI_SUCCESS); 4842 4843 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4844 "hot-plug event at port %d:%d", port, npmport); 4845 4846 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport); 4847 4848 /* Check previous device at that port */ 4849 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr) 4850 != SATA_DTYPE_NONE) 4851 dev_existed_previously = 1; 4852 4853 /* PxSStatus tells the presence of device. */ 4854 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS, 4855 &pmport_sstatus, err); 4856 4857 if (SSTATUS_GET_DET(pmport_sstatus) == 4858 SSTATUS_DET_DEVPRE_PHYCOM) 4859 dev_exists_now = 1; 4860 4861 /* 4862 * Clear PxSERR is critical. The transition from 0 to 1 will 4863 * emit a FIS which generates an asynchronous notification 4864 * event at controller. If we fail to clear the PxSERR, the 4865 * Async Notif events will no longer be activated on this 4866 * pmport. 4867 */ 4868 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR, 4869 AHCI_SERROR_CLEAR_ALL, err); 4870 4871 bzero((void *)&sdevice, sizeof (sata_device_t)); 4872 sdevice.satadev_addr.cport = ahci_ctlp-> 4873 ahcictl_port_to_cport[port]; 4874 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT; 4875 sdevice.satadev_addr.pmport = (uint8_t)npmport; 4876 sdevice.satadev_state = SATA_PSTATE_PWRON; 4877 4878 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4879 "[Existence] %d -> %d", dev_existed_previously, 4880 dev_exists_now); 4881 4882 if (dev_exists_now) { 4883 if (dev_existed_previously) { 4884 /* Link (may) not change: Exist -> Exist * */ 4885 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 4886 "ahci_probe_pmult: port %d:%d " 4887 "device link lost/established", 4888 port, npmport); 4889 4890 mutex_exit(&ahci_portp->ahciport_mutex); 4891 sata_hba_event_notify( 4892 ahci_ctlp->ahcictl_sata_hba_tran-> 4893 sata_tran_hba_dip, 4894 &sdevice, 4895 SATA_EVNT_LINK_LOST| 4896 SATA_EVNT_LINK_ESTABLISHED); 4897 mutex_enter(&ahci_portp->ahciport_mutex); 4898 } else { 4899 /* Link change: None -> Exist */ 4900 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4901 "ahci_probe_pmult: port %d:%d " 4902 "device link established", port, npmport); 4903 4904 /* Clear port state */ 4905 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4906 SATA_STATE_UNKNOWN); 4907 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4908 "ahci_probe_pmult: port %d " 4909 "ahciport_port_state [Cleared].", port); 4910 4911 mutex_exit(&ahci_portp->ahciport_mutex); 4912 sata_hba_event_notify( 4913 ahci_ctlp->ahcictl_sata_hba_tran-> 4914 sata_tran_hba_dip, 4915 &sdevice, 4916 SATA_EVNT_LINK_ESTABLISHED); 4917 mutex_enter(&ahci_portp->ahciport_mutex); 4918 } 4919 } else { /* No device exists now */ 4920 if (dev_existed_previously) { 4921 4922 /* Link change: Exist -> None */ 4923 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4924 "ahci_probe_pmult: port %d:%d " 4925 "device link lost", port, npmport); 4926 4927 /* An existing device is lost. */ 4928 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4929 SATA_STATE_UNKNOWN); 4930 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr, 4931 SATA_DTYPE_NONE); 4932 4933 mutex_exit(&ahci_portp->ahciport_mutex); 4934 sata_hba_event_notify( 4935 ahci_ctlp->ahcictl_sata_hba_tran-> 4936 sata_tran_hba_dip, 4937 &sdevice, 4938 SATA_EVNT_LINK_LOST); 4939 mutex_enter(&ahci_portp->ahciport_mutex); 4940 } 4941 } 4942 4943 CLEAR_BIT(port_hotplug_tags, npmport); 4944 } while (port_hotplug_tags != 0); 4945 4946 return (AHCI_SUCCESS); 4947 4948 err: /* R/W PMULT error */ 4949 return (AHCI_FAILURE); 4950 } 4951 4952 /* 4953 * Probe and initialize a port multiplier port. 4954 * A port multiplier port could only be initilaizer here. 4955 */ 4956 static int 4957 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4958 ahci_addr_t *addrp, sata_device_t *sd) 4959 { 4960 uint32_t port_state; 4961 uint8_t port = addrp->aa_port; 4962 ahci_addr_t addr_pmult; 4963 4964 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4965 4966 /* 4967 * Check the parent - port multiplier first. 4968 */ 4969 4970 /* 4971 * Parent port multiplier might have been removed. This event will be 4972 * ignored and failure. 4973 */ 4974 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 4975 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 4976 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4977 "ahci_tran_probe_port: " 4978 "parent device removed, ignore event.", NULL); 4979 4980 return (AHCI_FAILURE); 4981 } 4982 4983 /* The port is ready? */ 4984 port_state = ahci_portp->ahciport_port_state; 4985 if (!(port_state & SATA_STATE_READY)) { 4986 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4987 "ahci_tran_probe_port: " 4988 "parent port-mult is NOT ready.", NULL); 4989 4990 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 4991 ahci_portp, port, AHCI_PORT_RESET, NULL) != 4992 AHCI_SUCCESS) { 4993 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4994 "ahci_tran_probe_port: " 4995 "restart port-mult failed.", NULL); 4996 return (AHCI_FAILURE); 4997 } 4998 } 4999 5000 /* 5001 * If port-mult is restarted due to some reason, we need 5002 * re-initialized the PMult. 5003 */ 5004 if (!(port_state & SATA_DSTATE_PMULT_INIT)) { 5005 /* Initialize registers on a port multiplier */ 5006 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port); 5007 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp, 5008 &addr_pmult, sd) != AHCI_SUCCESS) 5009 return (AHCI_FAILURE); 5010 } 5011 5012 /* 5013 * Then we check the port-mult port 5014 */ 5015 /* Is this pmport initialized? */ 5016 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5017 if (!(port_state & SATA_STATE_READY)) { 5018 5019 /* ahci_initialize_pmport() will set READY state */ 5020 if (ahci_initialize_pmport(ahci_ctlp, 5021 ahci_portp, addrp) != AHCI_SUCCESS) 5022 return (AHCI_FAILURE); 5023 } 5024 5025 return (AHCI_SUCCESS); 5026 } 5027 5028 /* 5029 * AHCI device reset ...; a single device on one of the ports is reset, 5030 * but the HBA and physical communication remain intact. This is the 5031 * least intrusive. 5032 * 5033 * When issuing a software reset sequence, there should not be other 5034 * commands in the command list, so we will first clear and then re-set 5035 * PxCMD.ST to clear PxCI. And before issuing the software reset, 5036 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 5037 * cleared unless command list override (PxCMD.CLO) is supported. 5038 */ 5039 static int 5040 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5041 ahci_addr_t *addrp) 5042 { 5043 ahci_fis_h2d_register_t *h2d_register_fisp; 5044 ahci_cmd_table_t *cmd_table; 5045 ahci_cmd_header_t *cmd_header; 5046 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 5047 int slot, loop_count; 5048 uint8_t port = addrp->aa_port; 5049 uint8_t pmport = addrp->aa_pmport; 5050 int rval = AHCI_FAILURE; 5051 5052 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5053 5054 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5055 "port %d:%d device software resetting (FIS)", port, pmport); 5056 5057 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 5058 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 5059 port) != AHCI_SUCCESS) { 5060 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5061 "ahci_software_reset: cannot stop HBA port %d.", port); 5062 goto out; 5063 } 5064 5065 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 5066 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5067 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5068 5069 if (port_task_file & AHCI_TFD_STS_BSY || 5070 port_task_file & AHCI_TFD_STS_DRQ) { 5071 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 5072 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5073 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 5074 "cannot issue a software reset.", port_task_file); 5075 goto out; 5076 } 5077 5078 /* 5079 * If HBA Support CLO, as Command List Override (CAP.SCLO is 5080 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 5081 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 5082 */ 5083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5084 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 5085 5086 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5087 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5088 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5089 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5090 port_cmd_status|AHCI_CMD_STATUS_CLO); 5091 5092 /* Waiting till PxCMD.SCLO bit is cleared */ 5093 loop_count = 0; 5094 do { 5095 /* Wait for 10 millisec */ 5096 drv_usecwait(AHCI_10MS_USECS); 5097 5098 /* We are effectively timing out after 1 sec. */ 5099 if (loop_count++ > 100) { 5100 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5101 "SCLO time out. port %d is busy.", port); 5102 goto out; 5103 } 5104 5105 port_cmd_status = 5106 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5107 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5108 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 5109 5110 /* Re-check */ 5111 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5112 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5113 if (port_task_file & AHCI_TFD_STS_BSY || 5114 port_task_file & AHCI_TFD_STS_DRQ) { 5115 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5116 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 5117 port_task_file); 5118 goto out; 5119 } 5120 } 5121 5122 /* Then start port */ 5123 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5124 != AHCI_SUCCESS) { 5125 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5126 "ahci_software_reset: cannot start AHCI port %d.", port); 5127 goto out; 5128 } 5129 5130 /* 5131 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 5132 * ahci_port.ahciport_pending_ncq_tags may fail 5133 * ahci_claim_free_slot(). Actually according to spec, by clearing 5134 * PxCMD.ST there is no command outstanding while executing software 5135 * reseting. Hence we directly use slot 0 instead of 5136 * ahci_claim_free_slot(). 5137 */ 5138 slot = 0; 5139 5140 /* Now send the first H2D Register FIS with SRST set to 1 */ 5141 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 5142 bzero((void *)cmd_table, ahci_cmd_table_size); 5143 5144 h2d_register_fisp = 5145 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 5146 5147 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 5148 SET_FIS_PMP(h2d_register_fisp, pmport); 5149 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 5150 5151 /* Set Command Header in Command List */ 5152 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 5153 BZERO_DESCR_INFO(cmd_header); 5154 BZERO_PRD_BYTE_COUNT(cmd_header); 5155 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 5156 SET_PORT_MULTI_PORT(cmd_header, pmport); 5157 5158 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 5159 SET_RESET(cmd_header, 1); 5160 SET_WRITE(cmd_header, 1); 5161 5162 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 5163 0, 5164 ahci_cmd_table_size, 5165 DDI_DMA_SYNC_FORDEV); 5166 5167 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 5168 slot * sizeof (ahci_cmd_header_t), 5169 sizeof (ahci_cmd_header_t), 5170 DDI_DMA_SYNC_FORDEV); 5171 5172 /* Indicate to the HBA that a command is active. */ 5173 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5174 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 5175 (0x1 << slot)); 5176 5177 loop_count = 0; 5178 5179 /* Loop till the first command is finished */ 5180 do { 5181 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5182 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5183 5184 /* We are effectively timing out after 1 sec. */ 5185 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 5186 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5187 "the first SRST FIS is timed out, " 5188 "loop_count = %d", loop_count); 5189 goto out; 5190 } 5191 /* Wait for 10 millisec */ 5192 drv_usecwait(AHCI_10MS_USECS); 5193 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 5194 5195 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5196 "ahci_software_reset: 1st loop count: %d, " 5197 "port_cmd_issue = 0x%x, slot = 0x%x", 5198 loop_count, port_cmd_issue, slot); 5199 5200 /* According to ATA spec, we need wait at least 5 microsecs here. */ 5201 drv_usecwait(AHCI_1MS_USECS); 5202 5203 /* Now send the second H2D Register FIS with SRST cleard to zero */ 5204 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 5205 bzero((void *)cmd_table, ahci_cmd_table_size); 5206 5207 h2d_register_fisp = 5208 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 5209 5210 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 5211 SET_FIS_PMP(h2d_register_fisp, pmport); 5212 5213 /* Set Command Header in Command List */ 5214 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 5215 BZERO_DESCR_INFO(cmd_header); 5216 BZERO_PRD_BYTE_COUNT(cmd_header); 5217 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 5218 SET_PORT_MULTI_PORT(cmd_header, pmport); 5219 5220 SET_WRITE(cmd_header, 1); 5221 5222 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 5223 0, 5224 ahci_cmd_table_size, 5225 DDI_DMA_SYNC_FORDEV); 5226 5227 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 5228 slot * sizeof (ahci_cmd_header_t), 5229 sizeof (ahci_cmd_header_t), 5230 DDI_DMA_SYNC_FORDEV); 5231 5232 /* Indicate to the HBA that a command is active. */ 5233 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5234 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 5235 (0x1 << slot)); 5236 5237 loop_count = 0; 5238 5239 /* Loop till the second command is finished */ 5240 do { 5241 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5242 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5243 5244 /* We are effectively timing out after 1 sec. */ 5245 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 5246 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5247 "the second SRST FIS is timed out, " 5248 "loop_count = %d", loop_count); 5249 goto out; 5250 } 5251 5252 /* Wait for 10 millisec */ 5253 drv_usecwait(AHCI_10MS_USECS); 5254 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 5255 5256 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5257 "ahci_software_reset: 2nd loop count: %d, " 5258 "port_cmd_issue = 0x%x, slot = 0x%x", 5259 loop_count, port_cmd_issue, slot); 5260 5261 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) || 5262 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) { 5263 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 5264 DDI_SERVICE_UNAFFECTED); 5265 goto out; 5266 } 5267 5268 rval = AHCI_SUCCESS; 5269 out: 5270 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5271 "ahci_software_reset: %s at port %d:%d", 5272 rval == AHCI_SUCCESS ? "succeed" : "failed", 5273 port, pmport); 5274 5275 return (rval); 5276 } 5277 5278 /* 5279 * AHCI port reset ...; the physical communication between the HBA and device 5280 * on a port are disabled. This is more intrusive. 5281 * 5282 * When an HBA or port reset occurs, Phy communication is going to 5283 * be re-established with the device through a COMRESET followed by the 5284 * normal out-of-band communication sequence defined in Serial ATA. At 5285 * the end of reset, the device, if working properly, will send a D2H 5286 * Register FIS, which contains the device signature. When the HBA receives 5287 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 5288 * the PxSIG register with the signature. 5289 * 5290 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the 5291 * function is called. If not, it is assumed the interface is in hung 5292 * condition. 5293 */ 5294 static int 5295 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5296 ahci_addr_t *addrp) 5297 { 5298 ahci_addr_t pmult_addr; 5299 uint32_t port_cmd_status; 5300 uint32_t port_scontrol, port_sstatus; 5301 uint32_t port_task_file; 5302 uint32_t port_state; 5303 uint8_t port = addrp->aa_port; 5304 5305 int loop_count; 5306 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5307 5308 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5309 5310 /* Target is a port multiplier port? */ 5311 if (AHCI_ADDR_IS_PMPORT(addrp)) 5312 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp)); 5313 5314 /* Otherwise it must be an HBA port. */ 5315 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5316 5317 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5318 "Port %d port resetting...", port); 5319 ahci_portp->ahciport_port_state = 0; 5320 5321 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5322 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5323 5324 /* 5325 * According to the spec, SUD bit should be set here, 5326 * but JMicron JMB363 doesn't follow it, so print 5327 * a debug message. 5328 */ 5329 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 5330 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5331 "ahci_port_reset: port %d SUD bit not set", port); 5332 5333 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5334 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5335 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5336 5337 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5338 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 5339 port_scontrol); 5340 5341 /* Enable PxCMD.FRE to read device */ 5342 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5343 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5344 port_cmd_status|AHCI_CMD_STATUS_FRE); 5345 5346 /* 5347 * The port enters P:StartComm state, and the HBA tells the link layer 5348 * to start communication, which involves sending COMRESET to the 5349 * device. And the HBA resets PxTFD.STS to 7Fh. 5350 * 5351 * Give time for COMRESET to percolate, according to the AHCI 5352 * spec, software shall wait at least 1 millisecond before 5353 * clearing PxSCTL.DET 5354 */ 5355 drv_usecwait(AHCI_1MS_USECS * 2); 5356 5357 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 5358 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5359 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5360 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5361 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5362 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 5363 port_scontrol); 5364 5365 /* 5366 * When a COMINIT is received from the device, then the port enters 5367 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5368 * PxSSTS.DET to 1h to indicate a device is detected but communication 5369 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5370 * a COMINIT has been received. 5371 */ 5372 /* 5373 * The DET field is valid only if IPM field indicates 5374 * that the interface is in active state. 5375 */ 5376 loop_count = 0; 5377 for (;;) { 5378 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5379 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5380 5381 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5382 /* 5383 * If the interface is not active, the DET field 5384 * is considered not accurate. So we want to 5385 * continue looping. 5386 */ 5387 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5388 } 5389 5390 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) 5391 break; 5392 5393 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5394 /* 5395 * We are effectively timing out after 0.1 sec. 5396 */ 5397 break; 5398 } 5399 5400 /* Wait for 10 millisec */ 5401 drv_usecwait(AHCI_10MS_USECS); 5402 } 5403 5404 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5405 "ahci_port_reset: 1st loop count: %d, " 5406 "port_sstatus = 0x%x port %d", 5407 loop_count, port_sstatus, port); 5408 5409 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) { 5410 /* 5411 * Either the port is not active or there 5412 * is no device present. 5413 */ 5414 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5415 return (AHCI_SUCCESS); 5416 } 5417 5418 /* Clear port serror register for the port */ 5419 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5420 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5421 AHCI_SERROR_CLEAR_ALL); 5422 5423 /* 5424 * Devices should return a FIS contains its signature to HBA after 5425 * COMINIT signal. Check whether a D2H Register FIS is received by 5426 * polling PxTFD.STS. 5427 */ 5428 loop_count = 0; 5429 for (;;) { 5430 port_task_file = 5431 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5432 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5433 5434 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ | 5435 AHCI_TFD_STS_ERR)) == 0) 5436 break; 5437 5438 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5439 /* 5440 * We are effectively timing out after 11 sec. 5441 */ 5442 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5443 "the device hardware has been initialized and " 5444 "the power-up diagnostics failed", 5445 instance, port); 5446 5447 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5448 "port %d: some or all of BSY, DRQ and ERR in " 5449 "PxTFD.STS are not clear. We need another " 5450 "software reset.", port); 5451 5452 /* Clear port serror register for the port */ 5453 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5454 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5455 AHCI_SERROR_CLEAR_ALL); 5456 5457 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 5458 5459 /* Try another software reset. */ 5460 if (ahci_software_reset(ahci_ctlp, ahci_portp, 5461 &pmult_addr) != AHCI_SUCCESS) { 5462 AHCIPORT_SET_STATE(ahci_portp, addrp, 5463 SATA_PSTATE_FAILED); 5464 return (AHCI_FAILURE); 5465 } 5466 break; 5467 } 5468 5469 /* Wait for 10 millisec */ 5470 drv_usecwait(AHCI_10MS_USECS); 5471 } 5472 5473 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5474 "ahci_port_reset: 2nd loop count: %d, " 5475 "port_task_file = 0x%x port %d", 5476 loop_count, port_task_file, port); 5477 5478 /* Clear port serror register for the port */ 5479 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5480 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5481 AHCI_SERROR_CLEAR_ALL); 5482 5483 /* Set port as ready */ 5484 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5485 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5486 5487 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5488 "ahci_port_reset: succeed at port %d.", port); 5489 return (AHCI_SUCCESS); 5490 } 5491 5492 /* 5493 * COMRESET on a port multiplier port. 5494 * 5495 * NOTE: Only called in ahci_port_reset() 5496 */ 5497 static int 5498 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5499 ahci_addr_t *addrp) 5500 { 5501 uint32_t port_scontrol, port_sstatus, port_serror; 5502 uint32_t port_cmd_status, port_intr_status; 5503 uint32_t port_state; 5504 uint8_t port = addrp->aa_port; 5505 uint8_t pmport = addrp->aa_pmport; 5506 int loop_count; 5507 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5508 5509 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5510 "port %d:%d: pmport resetting", port, pmport); 5511 5512 /* Initialize pmport state */ 5513 AHCIPORT_SET_STATE(ahci_portp, addrp, 0); 5514 5515 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5516 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5517 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5518 5519 /* PxCMD.FRE should be set before. */ 5520 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5521 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5522 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE); 5523 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) 5524 return (AHCI_FAILURE); 5525 5526 /* 5527 * Give time for COMRESET to percolate, according to the AHCI 5528 * spec, software shall wait at least 1 millisecond before 5529 * clearing PxSCTL.DET 5530 */ 5531 drv_usecwait(AHCI_1MS_USECS*2); 5532 5533 /* 5534 * Fetch the SCONTROL again and rewrite the DET part with 0 5535 * This will generate an Asychronous Notification events. 5536 */ 5537 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5538 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5539 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5540 5541 /* 5542 * The port enters P:StartComm state, and HBA tells link layer to 5543 * start communication, which involves sending COMRESET to device. 5544 * And the HBA resets PxTFD.STS to 7Fh. 5545 * 5546 * When a COMINIT is received from the device, then the port enters 5547 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5548 * PxSSTS.DET to 1h to indicate a device is detected but communication 5549 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5550 * a COMINIT has been received. 5551 */ 5552 /* 5553 * The DET field is valid only if IPM field indicates 5554 * that the interface is in active state. 5555 */ 5556 loop_count = 0; 5557 do { 5558 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err); 5559 5560 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5561 /* 5562 * If the interface is not active, the DET field 5563 * is considered not accurate. So we want to 5564 * continue looping. 5565 */ 5566 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5567 } 5568 5569 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5570 /* 5571 * We are effectively timing out after 0.1 sec. 5572 */ 5573 break; 5574 } 5575 5576 /* Wait for 10 millisec */ 5577 drv_usecwait(AHCI_10MS_USECS); 5578 5579 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5580 5581 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5582 "ahci_pmport_reset: 1st loop count: %d, " 5583 "port_sstatus = 0x%x port %d:%d", 5584 loop_count, port_sstatus, port, pmport); 5585 5586 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5587 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5588 /* 5589 * Either the port is not active or there 5590 * is no device present. 5591 */ 5592 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5593 "ahci_pmport_reset: " 5594 "no device attached to port %d:%d", 5595 port, pmport); 5596 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5597 return (AHCI_SUCCESS); 5598 } 5599 5600 /* Now we can make sure there is a device connected to the port */ 5601 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */ 5602 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err); 5603 5604 if (!(port_serror & (1 << 26))) { 5605 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: " 5606 "COMINIT signal from the device not received port %d:%d", 5607 instance, port, pmport); 5608 5609 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5610 return (AHCI_FAILURE); 5611 } 5612 5613 /* 5614 * After clear PxSERR register, we will receive a D2H FIS. 5615 * Normally this FIS will cause a IPMS error according to AHCI spec 5616 * v1.2 because there is no command outstanding for it. So we need 5617 * to ignore this error. 5618 */ 5619 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS; 5620 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err); 5621 5622 /* Now we need to check the D2H FIS by checking IPMS error. */ 5623 loop_count = 0; 5624 do { 5625 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5626 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5627 5628 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5629 /* 5630 * No D2H FIS received. This is possible according 5631 * to SATA 2.6 spec. 5632 */ 5633 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d " 5634 "PxIS.IPMS is not set, we need another " 5635 "software reset.", port, pmport); 5636 5637 break; 5638 } 5639 5640 /* Wait for 10 millisec */ 5641 mutex_exit(&ahci_portp->ahciport_mutex); 5642 delay(AHCI_10MS_TICKS); 5643 mutex_enter(&ahci_portp->ahciport_mutex); 5644 5645 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS)); 5646 5647 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5648 "ahci_pmport_reset: 2st loop count: %d, " 5649 "port_sstatus = 0x%x port %d:%d", 5650 loop_count, port_sstatus, port, pmport); 5651 5652 /* Clear IPMS */ 5653 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5654 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5655 AHCI_INTR_STATUS_IPMS); 5656 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5657 5658 /* This pmport is now ready for ahci_tran_start() */ 5659 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5660 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5661 5662 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5663 "ahci_pmport_reset: succeed at port %d:%d", port, pmport); 5664 return (AHCI_SUCCESS); 5665 5666 err: /* R/W PMULT error */ 5667 /* IPMS flags might be set before. */ 5668 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5669 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5670 "ahci_pmport_reset: failed at port %d:%d", port, pmport); 5671 5672 return (AHCI_FAILURE); 5673 } 5674 5675 /* 5676 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 5677 * This is the most intrusive. 5678 * 5679 * When an HBA reset occurs, Phy communication will be re-established with 5680 * the device through a COMRESET followed by the normal out-of-band 5681 * communication sequence defined in Serial ATA. At the end of reset, the 5682 * device, if working properly, will send a D2H Register FIS, which contains 5683 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 5684 * and PxTFD.ERR register fields, and updates the PxSIG register with the 5685 * signature. 5686 * 5687 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 5688 */ 5689 static int 5690 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 5691 { 5692 ahci_port_t *ahci_portp; 5693 uint32_t ghc_control; 5694 uint8_t port; 5695 int loop_count; 5696 int rval = AHCI_SUCCESS; 5697 5698 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 5699 NULL); 5700 5701 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5702 5703 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5704 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5705 5706 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 5707 ghc_control |= AHCI_HBA_GHC_HR; 5708 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5709 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5710 5711 /* 5712 * Wait until HBA Reset complete or timeout 5713 */ 5714 loop_count = 0; 5715 do { 5716 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5717 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5718 5719 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 5720 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5721 "ahci hba reset is timing out, " 5722 "ghc_control = 0x%x", ghc_control); 5723 /* We are effectively timing out after 1 sec. */ 5724 break; 5725 } 5726 5727 /* Wait for 10 millisec */ 5728 drv_usecwait(AHCI_10MS_USECS); 5729 } while (ghc_control & AHCI_HBA_GHC_HR); 5730 5731 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5732 "ahci_hba_reset: 1st loop count: %d, " 5733 "ghc_control = 0x%x", loop_count, ghc_control); 5734 5735 if (ghc_control & AHCI_HBA_GHC_HR) { 5736 /* The hba is not reset for some reasons */ 5737 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5738 "hba reset failed: HBA in a hung or locked state", NULL); 5739 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5740 return (AHCI_FAILURE); 5741 } 5742 5743 /* 5744 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE 5745 */ 5746 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5747 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5748 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE); 5749 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5750 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5751 5752 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5753 5754 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 5755 /* Only check implemented ports */ 5756 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 5757 continue; 5758 } 5759 5760 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5761 mutex_enter(&ahci_portp->ahciport_mutex); 5762 5763 /* Make sure the drive is spun-up */ 5764 ahci_staggered_spin_up(ahci_ctlp, port); 5765 5766 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 5767 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) != 5768 AHCI_SUCCESS) { 5769 rval = AHCI_FAILURE; 5770 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5771 "ahci_hba_reset: port %d failed", port); 5772 /* 5773 * Set the port state to SATA_PSTATE_FAILED if 5774 * failed to initialize it. 5775 */ 5776 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5777 } 5778 5779 mutex_exit(&ahci_portp->ahciport_mutex); 5780 } 5781 5782 return (rval); 5783 } 5784 5785 /* 5786 * This routine is only called from AHCI_ATTACH or phyrdy change 5787 * case. It first calls software reset, then stop the port and try to 5788 * read PxSIG register to find the type of device attached to the port. 5789 * 5790 * The caller should make sure a valid device exists on specified port and 5791 * physical communication has been established so that the signature could 5792 * be retrieved by software reset. 5793 * 5794 * NOTE: The port interrupts should be disabled before the function is called. 5795 */ 5796 static void 5797 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5798 ahci_addr_t *addrp) 5799 { 5800 ahci_addr_t dev_addr; 5801 uint32_t signature; 5802 uint8_t port = addrp->aa_port; 5803 uint8_t pmport = addrp->aa_pmport; 5804 int rval; 5805 5806 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5807 ASSERT(AHCI_ADDR_IS_VALID(addrp)); 5808 5809 /* 5810 * If the HBA doesn't support port multiplier, then the driver 5811 * doesn't need to bother to check port multiplier device. 5812 * 5813 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is 5814 * connected to Silicon Image 4723, to which the two sata drives 5815 * attached can be set with RAID1, RAID0 or Spanning mode. 5816 * 5817 * We found software reset will get failure if port multiplier address 5818 * 0xf is used by software reset, so just ignore the check since 5819 * ICH7 doesn't support port multiplier device at all. 5820 */ 5821 if (AHCI_ADDR_IS_PORT(addrp) && 5822 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) { 5823 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5824 "ahci_find_dev_signature enter: port %d", port); 5825 5826 /* 5827 * NOTE: when the ahci address is a HBA port, we do not know 5828 * it is a device or a port multiplier that attached. we need 5829 * try a software reset at port multiplier address (0xf 5830 * pmport) 5831 */ 5832 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port); 5833 } else { 5834 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5835 "ahci_find_dev_signature enter: port %d:%d", 5836 port, pmport); 5837 dev_addr = *addrp; 5838 } 5839 5840 /* Assume it is unknown. */ 5841 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5842 5843 /* Issue a software reset to get the signature */ 5844 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr); 5845 if (rval != AHCI_SUCCESS) { 5846 5847 /* 5848 * Try to do software reset again with pmport set with 0 if 5849 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and 5850 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf) 5851 */ 5852 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) && 5853 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) { 5854 dev_addr.aa_pmport = 0; 5855 rval = ahci_software_reset(ahci_ctlp, ahci_portp, 5856 &dev_addr); 5857 } 5858 5859 if (rval != AHCI_SUCCESS) { 5860 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5861 "ahci_find_dev_signature: software reset failed " 5862 "at port %d:%d, cannot get signature.", 5863 port, pmport); 5864 5865 AHCIPORT_SET_STATE(ahci_portp, addrp, 5866 SATA_PSTATE_FAILED); 5867 return; 5868 } 5869 } 5870 5871 /* 5872 * ahci_software_reset has started the port, so we need manually stop 5873 * the port again. 5874 */ 5875 if (AHCI_ADDR_IS_PORT(addrp)) { 5876 if (ahci_put_port_into_notrunning_state(ahci_ctlp, 5877 ahci_portp, port) != AHCI_SUCCESS) { 5878 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5879 "ahci_find_dev_signature: cannot stop port %d.", 5880 port); 5881 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5882 return; 5883 } 5884 } 5885 5886 /* Now we can make sure that a valid signature is received. */ 5887 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5888 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 5889 5890 if (AHCI_ADDR_IS_PMPORT(addrp)) { 5891 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 5892 "ahci_find_dev_signature: signature = 0x%x at port %d:%d", 5893 signature, port, pmport); 5894 } else { 5895 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5896 "ahci_find_dev_signature: signature = 0x%x at port %d", 5897 signature, port); 5898 } 5899 5900 /* NOTE: Only support ATAPI device at controller port. */ 5901 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp)) 5902 signature = SATA_DTYPE_UNKNOWN; 5903 5904 switch (signature) { 5905 5906 case AHCI_SIGNATURE_DISK: 5907 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK); 5908 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5909 "Disk is found at port: %d", port); 5910 break; 5911 5912 case AHCI_SIGNATURE_ATAPI: 5913 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI); 5914 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5915 "ATAPI device is found at port: %d", port); 5916 break; 5917 5918 case AHCI_SIGNATURE_PORT_MULTIPLIER: 5919 /* Port Multiplier cannot recursively attached. */ 5920 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5921 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT); 5922 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5923 "Port Multiplier is found at port: %d", port); 5924 break; 5925 5926 default: 5927 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5928 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5929 "Unknown device is found at port: %d", port); 5930 } 5931 } 5932 5933 /* 5934 * According to the spec, to reliably detect hot plug removals, software 5935 * must disable interface power management. Software should perform the 5936 * following initialization on a port after a device is attached: 5937 * Set PxSCTL.IPM to 3h to disable interface state transitions 5938 * Set PxCMD.ALPE to '0' to disable aggressive power management 5939 * Disable device initiated interface power management by SET FEATURE 5940 * 5941 * We can ignore the last item because by default the feature is disabled 5942 */ 5943 static void 5944 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 5945 { 5946 uint32_t port_scontrol, port_cmd_status; 5947 5948 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5949 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5950 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 5951 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5952 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 5953 5954 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5955 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5956 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 5957 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5958 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5959 } 5960 5961 /* 5962 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 5963 * to 1, then set it firstly. 5964 * 5965 * Each port contains two major DMA engines. One DMA engine walks through 5966 * the command list, and is controlled by PxCMD.ST. The second DMA engine 5967 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 5968 * 5969 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 5970 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 5971 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 5972 * 5973 * Software shall not set PxCMD.ST to '1' unless a functional device is 5974 * present on the port(as determined by PxTFD.STS.BSY = '0', 5975 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 5976 */ 5977 static int 5978 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 5979 { 5980 uint32_t port_cmd_status; 5981 5982 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5983 5984 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 5985 5986 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 5987 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5988 "the state for port %d is 0x%x", 5989 port, ahci_portp->ahciport_port_state); 5990 return (AHCI_FAILURE); 5991 } 5992 5993 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5994 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5995 "no device is attached at port %d", port); 5996 return (AHCI_FAILURE); 5997 } 5998 5999 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 6000 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6001 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6002 6003 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 6004 port_cmd_status |= AHCI_CMD_STATUS_FRE; 6005 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6006 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6007 port_cmd_status); 6008 } 6009 6010 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6011 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6012 6013 port_cmd_status |= AHCI_CMD_STATUS_ST; 6014 6015 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6016 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6017 port_cmd_status); 6018 6019 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 6020 6021 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 6022 "PxCMD.ST set to '1' at port %d", port); 6023 6024 return (AHCI_SUCCESS); 6025 } 6026 6027 /* 6028 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need 6029 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared. 6030 * Then set PxCLB, PxCLBU, PxFB, and PxFBU. 6031 */ 6032 static int 6033 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6034 { 6035 uint8_t port = ahci_portp->ahciport_port_num; 6036 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6037 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6038 6039 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6040 6041 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */ 6042 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) { 6043 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6044 port) != AHCI_SUCCESS) 6045 return (AHCI_FAILURE); 6046 6047 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6048 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6049 } 6050 6051 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */ 6052 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) { 6053 int loop_count = 0; 6054 6055 /* Clear PxCMD.FRE */ 6056 port_cmd_status &= ~AHCI_CMD_STATUS_FRE; 6057 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6058 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6059 port_cmd_status); 6060 6061 /* Wait until PxCMD.FR is cleared */ 6062 for (;;) { 6063 port_cmd_status = 6064 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6065 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6066 6067 if (!(port_cmd_status & AHCI_CMD_STATUS_FR)) 6068 break; 6069 6070 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) { 6071 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp, 6072 "ahci_setup_port_base_addresses: cannot " 6073 "clear PxCMD.FR for port %d.", port); 6074 6075 /* 6076 * We are effectively timing out after 0.5 sec. 6077 * This value is specified in AHCI spec. 6078 */ 6079 return (AHCI_FAILURE); 6080 } 6081 6082 /* Wait for 1 millisec */ 6083 drv_usecwait(AHCI_1MS_USECS); 6084 } 6085 } 6086 6087 /* Step 3: Config Port Command List Base Address */ 6088 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6089 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 6090 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 6091 6092 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6093 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port), 6094 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused); 6095 6096 /* Step 4: Config Port Received FIS Base Address */ 6097 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6098 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 6099 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 6100 6101 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6102 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port), 6103 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused); 6104 6105 return (AHCI_SUCCESS); 6106 } 6107 6108 /* 6109 * Allocate the ahci_port_t including Received FIS and Command List. 6110 * The argument - port is the physical port number, and not logical 6111 * port number seen by the SATA framework. 6112 */ 6113 static int 6114 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 6115 { 6116 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 6117 ahci_port_t *ahci_portp; 6118 char taskq_name[64] = "event_handle_taskq"; 6119 6120 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 6121 6122 ahci_portp = 6123 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 6124 6125 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 6126 ahci_portp->ahciport_port_num = port; 6127 6128 /* Initialize the port condition variable */ 6129 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 6130 6131 /* Initialize the port mutex */ 6132 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 6133 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 6134 6135 mutex_enter(&ahci_portp->ahciport_mutex); 6136 6137 /* 6138 * Allocate memory for received FIS structure and 6139 * command list for this port 6140 */ 6141 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6142 goto err_case1; 6143 } 6144 6145 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6146 goto err_case2; 6147 } 6148 6149 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */ 6150 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) != 6151 AHCI_SUCCESS) { 6152 goto err_case3; 6153 } 6154 6155 (void) snprintf(taskq_name + strlen(taskq_name), 6156 sizeof (taskq_name) - strlen(taskq_name), 6157 "_port%d", port); 6158 6159 /* Create the taskq for the port */ 6160 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 6161 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 6162 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 6163 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 6164 goto err_case3; 6165 } 6166 6167 /* Allocate the argument for the taskq */ 6168 ahci_portp->ahciport_event_args = 6169 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 6170 6171 ahci_portp->ahciport_event_args->ahciea_addrp = 6172 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP); 6173 6174 /* Initialize the done queue */ 6175 ahci_portp->ahciport_doneq = NULL; 6176 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq; 6177 ahci_portp->ahciport_doneq_len = 0; 6178 6179 mutex_exit(&ahci_portp->ahciport_mutex); 6180 6181 return (AHCI_SUCCESS); 6182 6183 err_case3: 6184 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 6185 6186 err_case2: 6187 ahci_dealloc_rcvd_fis(ahci_portp); 6188 6189 err_case1: 6190 mutex_exit(&ahci_portp->ahciport_mutex); 6191 mutex_destroy(&ahci_portp->ahciport_mutex); 6192 cv_destroy(&ahci_portp->ahciport_cv); 6193 6194 kmem_free(ahci_portp, sizeof (ahci_port_t)); 6195 6196 return (AHCI_FAILURE); 6197 } 6198 6199 /* 6200 * Reverse of ahci_alloc_port_state(). 6201 */ 6202 static void 6203 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 6204 { 6205 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6206 6207 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 6208 ASSERT(ahci_portp != NULL); 6209 6210 mutex_enter(&ahci_portp->ahciport_mutex); 6211 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp, 6212 sizeof (ahci_addr_t)); 6213 ahci_portp->ahciport_event_args->ahciea_addrp = NULL; 6214 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 6215 ahci_portp->ahciport_event_args = NULL; 6216 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 6217 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 6218 ahci_dealloc_rcvd_fis(ahci_portp); 6219 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 6220 mutex_exit(&ahci_portp->ahciport_mutex); 6221 6222 mutex_destroy(&ahci_portp->ahciport_mutex); 6223 cv_destroy(&ahci_portp->ahciport_cv); 6224 6225 kmem_free(ahci_portp, sizeof (ahci_port_t)); 6226 6227 ahci_ctlp->ahcictl_ports[port] = NULL; 6228 } 6229 6230 /* 6231 * Allocates memory for the Received FIS Structure 6232 */ 6233 static int 6234 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6235 { 6236 size_t rcvd_fis_size; 6237 size_t ret_len; 6238 uint_t cookie_count; 6239 6240 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6241 6242 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 6243 6244 /* allocate rcvd FIS dma handle. */ 6245 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6246 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 6247 DDI_DMA_SLEEP, 6248 NULL, 6249 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 6250 DDI_SUCCESS) { 6251 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6252 "rcvd FIS dma handle alloc failed", NULL); 6253 6254 return (AHCI_FAILURE); 6255 } 6256 6257 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 6258 rcvd_fis_size, 6259 &accattr, 6260 DDI_DMA_CONSISTENT, 6261 DDI_DMA_SLEEP, 6262 NULL, 6263 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 6264 &ret_len, 6265 &ahci_portp->ahciport_rcvd_fis_acc_handle) != 0) { 6266 6267 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6268 "rcvd FIS dma mem alloc fail", NULL); 6269 /* error.. free the dma handle. */ 6270 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6271 return (AHCI_FAILURE); 6272 } 6273 6274 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 6275 NULL, 6276 (caddr_t)ahci_portp->ahciport_rcvd_fis, 6277 rcvd_fis_size, 6278 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6279 DDI_DMA_SLEEP, 6280 NULL, 6281 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 6282 &cookie_count) != DDI_DMA_MAPPED) { 6283 6284 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6285 "rcvd FIS dma handle bind fail", NULL); 6286 /* error.. free the dma handle & free the memory. */ 6287 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 6288 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6289 return (AHCI_FAILURE); 6290 } 6291 6292 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 6293 6294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 6295 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 6296 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 6297 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 6298 6299 return (AHCI_SUCCESS); 6300 } 6301 6302 /* 6303 * Deallocates the Received FIS Structure 6304 */ 6305 static void 6306 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 6307 { 6308 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6309 6310 /* Unbind the cmd list dma handle first. */ 6311 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 6312 6313 /* Then free the underlying memory. */ 6314 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 6315 6316 /* Now free the handle itself. */ 6317 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6318 } 6319 6320 /* 6321 * Allocates memory for the Command List, which contains up to 32 entries. 6322 * Each entry contains a command header, which is a 32-byte structure that 6323 * includes the pointer to the command table. 6324 */ 6325 static int 6326 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6327 { 6328 size_t cmd_list_size; 6329 size_t ret_len; 6330 uint_t cookie_count; 6331 6332 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6333 6334 cmd_list_size = 6335 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 6336 6337 /* allocate cmd list dma handle. */ 6338 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6339 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 6340 DDI_DMA_SLEEP, 6341 NULL, 6342 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 6343 6344 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6345 "cmd list dma handle alloc failed", NULL); 6346 return (AHCI_FAILURE); 6347 } 6348 6349 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 6350 cmd_list_size, 6351 &accattr, 6352 DDI_DMA_CONSISTENT, 6353 DDI_DMA_SLEEP, 6354 NULL, 6355 (caddr_t *)&ahci_portp->ahciport_cmd_list, 6356 &ret_len, 6357 &ahci_portp->ahciport_cmd_list_acc_handle) != 0) { 6358 6359 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6360 "cmd list dma mem alloc fail", NULL); 6361 /* error.. free the dma handle. */ 6362 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6363 return (AHCI_FAILURE); 6364 } 6365 6366 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 6367 NULL, 6368 (caddr_t)ahci_portp->ahciport_cmd_list, 6369 cmd_list_size, 6370 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6371 DDI_DMA_SLEEP, 6372 NULL, 6373 &ahci_portp->ahciport_cmd_list_dma_cookie, 6374 &cookie_count) != DDI_DMA_MAPPED) { 6375 6376 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6377 "cmd list dma handle bind fail", NULL); 6378 /* error.. free the dma handle & free the memory. */ 6379 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6380 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6381 return (AHCI_FAILURE); 6382 } 6383 6384 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 6385 6386 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 6387 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 6388 6389 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 6390 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 6391 6392 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6393 goto err_out; 6394 } 6395 6396 return (AHCI_SUCCESS); 6397 6398 err_out: 6399 /* Unbind the cmd list dma handle first. */ 6400 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 6401 6402 /* Then free the underlying memory. */ 6403 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6404 6405 /* Now free the handle itself. */ 6406 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6407 6408 return (AHCI_FAILURE); 6409 } 6410 6411 /* 6412 * Deallocates the Command List 6413 */ 6414 static void 6415 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6416 { 6417 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6418 6419 /* First dealloc command table */ 6420 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 6421 6422 /* Unbind the cmd list dma handle first. */ 6423 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 6424 6425 /* Then free the underlying memory. */ 6426 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6427 6428 /* Now free the handle itself. */ 6429 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6430 } 6431 6432 /* 6433 * Allocates memory for all Command Tables, which contains Command FIS, 6434 * ATAPI Command and Physical Region Descriptor Table. 6435 */ 6436 static int 6437 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6438 { 6439 size_t ret_len; 6440 ddi_dma_cookie_t cmd_table_dma_cookie; 6441 uint_t cookie_count; 6442 int slot; 6443 6444 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6445 6446 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 6447 "ahci_alloc_cmd_tables: port %d enter", 6448 ahci_portp->ahciport_port_num); 6449 6450 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6451 /* Allocate cmd table dma handle. */ 6452 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6453 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 6454 DDI_DMA_SLEEP, 6455 NULL, 6456 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 6457 DDI_SUCCESS) { 6458 6459 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6460 "cmd table dma handle alloc failed", NULL); 6461 6462 goto err_out; 6463 } 6464 6465 if (ddi_dma_mem_alloc( 6466 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6467 ahci_cmd_table_size, 6468 &accattr, 6469 DDI_DMA_CONSISTENT, 6470 DDI_DMA_SLEEP, 6471 NULL, 6472 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 6473 &ret_len, 6474 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 0) { 6475 6476 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6477 "cmd table dma mem alloc fail", NULL); 6478 6479 /* error.. free the dma handle. */ 6480 ddi_dma_free_handle( 6481 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6482 goto err_out; 6483 } 6484 6485 if (ddi_dma_addr_bind_handle( 6486 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6487 NULL, 6488 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 6489 ahci_cmd_table_size, 6490 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6491 DDI_DMA_SLEEP, 6492 NULL, 6493 &cmd_table_dma_cookie, 6494 &cookie_count) != DDI_DMA_MAPPED) { 6495 6496 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6497 "cmd table dma handle bind fail", NULL); 6498 /* error.. free the dma handle & free the memory. */ 6499 ddi_dma_mem_free( 6500 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6501 ddi_dma_free_handle( 6502 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6503 goto err_out; 6504 } 6505 6506 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 6507 ahci_cmd_table_size); 6508 6509 /* Config Port Command Table Base Address */ 6510 SET_COMMAND_TABLE_BASE_ADDR( 6511 (&ahci_portp->ahciport_cmd_list[slot]), 6512 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 6513 6514 #ifndef __lock_lint 6515 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 6516 (&ahci_portp->ahciport_cmd_list[slot]), 6517 cmd_table_dma_cookie.dmac_laddress >> 32); 6518 #endif 6519 } 6520 6521 return (AHCI_SUCCESS); 6522 err_out: 6523 6524 for (slot--; slot >= 0; slot--) { 6525 /* Unbind the cmd table dma handle first */ 6526 (void) ddi_dma_unbind_handle( 6527 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6528 6529 /* Then free the underlying memory */ 6530 ddi_dma_mem_free( 6531 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6532 6533 /* Now free the handle itself */ 6534 ddi_dma_free_handle( 6535 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6536 } 6537 6538 return (AHCI_FAILURE); 6539 } 6540 6541 /* 6542 * Deallocates memory for all Command Tables. 6543 */ 6544 static void 6545 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6546 { 6547 int slot; 6548 6549 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6550 6551 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6552 "ahci_dealloc_cmd_tables: %d enter", 6553 ahci_portp->ahciport_port_num); 6554 6555 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6556 /* Unbind the cmd table dma handle first. */ 6557 (void) ddi_dma_unbind_handle( 6558 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6559 6560 /* Then free the underlying memory. */ 6561 ddi_dma_mem_free( 6562 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6563 6564 /* Now free the handle itself. */ 6565 ddi_dma_free_handle( 6566 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6567 } 6568 } 6569 6570 /* 6571 * Update SATA registers at controller ports 6572 */ 6573 static void 6574 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 6575 sata_device_t *sd) 6576 { 6577 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 6578 6579 sd->satadev_scr.sstatus = 6580 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6581 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 6582 sd->satadev_scr.serror = 6583 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6584 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 6585 sd->satadev_scr.scontrol = 6586 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6587 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 6588 sd->satadev_scr.sactive = 6589 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6590 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 6591 } 6592 6593 /* 6594 * For poll mode, ahci_port_intr will be called to emulate the interrupt 6595 */ 6596 static void 6597 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 6598 { 6599 uint32_t port_intr_status; 6600 uint32_t port_intr_enable; 6601 6602 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6603 "ahci_port_intr enter: port %d", port); 6604 6605 mutex_enter(&ahci_portp->ahciport_mutex); 6606 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 6607 /* For SATA_OPMODE_POLLING commands */ 6608 port_intr_enable = 6609 (AHCI_INTR_STATUS_DHRS | 6610 AHCI_INTR_STATUS_PSS | 6611 AHCI_INTR_STATUS_SDBS | 6612 AHCI_INTR_STATUS_UFS | 6613 AHCI_INTR_STATUS_PCS | 6614 AHCI_INTR_STATUS_PRCS | 6615 AHCI_INTR_STATUS_OFS | 6616 AHCI_INTR_STATUS_INFS | 6617 AHCI_INTR_STATUS_IFS | 6618 AHCI_INTR_STATUS_HBDS | 6619 AHCI_INTR_STATUS_HBFS | 6620 AHCI_INTR_STATUS_TFES); 6621 } else { 6622 /* 6623 * port_intr_enable indicates that the corresponding interrrupt 6624 * reporting is enabled. 6625 */ 6626 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6627 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 6628 } 6629 6630 /* IPMS error in port reset should be ignored according AHCI spec. */ 6631 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS)) 6632 port_intr_enable |= AHCI_INTR_STATUS_IPMS; 6633 mutex_exit(&ahci_portp->ahciport_mutex); 6634 6635 /* 6636 * port_intr_stats indicates that the corresponding interrupt 6637 * condition is active. 6638 */ 6639 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6640 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 6641 6642 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6643 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 6644 "port_intr_enable = 0x%x", 6645 port, port_intr_status, port_intr_enable); 6646 6647 port_intr_status &= port_intr_enable; 6648 6649 /* 6650 * Pending interrupt events are indicated by the PxIS register. 6651 * Make sure we don't miss any event. 6652 */ 6653 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 6654 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6655 DDI_SERVICE_UNAFFECTED); 6656 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6657 DDI_FME_VERSION); 6658 return; 6659 } 6660 6661 /* First clear the port interrupts status */ 6662 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6663 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 6664 port_intr_status); 6665 6666 /* Check the completed non-queued commands */ 6667 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 6668 AHCI_INTR_STATUS_PSS)) { 6669 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 6670 ahci_portp, port); 6671 } 6672 6673 /* Check the completed queued commands */ 6674 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 6675 (void) ahci_intr_set_device_bits(ahci_ctlp, 6676 ahci_portp, port); 6677 } 6678 6679 /* Check the port connect change status interrupt bit */ 6680 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 6681 (void) ahci_intr_port_connect_change(ahci_ctlp, 6682 ahci_portp, port); 6683 } 6684 6685 /* Check the device mechanical presence status interrupt bit */ 6686 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 6687 (void) ahci_intr_device_mechanical_presence_status( 6688 ahci_ctlp, ahci_portp, port); 6689 } 6690 6691 /* Check the PhyRdy change status interrupt bit */ 6692 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 6693 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 6694 port); 6695 } 6696 6697 /* 6698 * Check the non-fatal error interrupt bits, there are four 6699 * kinds of non-fatal errors at the time being: 6700 * 6701 * PxIS.UFS - Unknown FIS Error 6702 * PxIS.OFS - Overflow Error 6703 * PxIS.INFS - Interface Non-Fatal Error 6704 * PxIS.IPMS - Incorrect Port Multiplier Status Error 6705 * 6706 * For these non-fatal errors, the HBA can continue to operate, 6707 * so the driver just log the error messages. 6708 */ 6709 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 6710 AHCI_INTR_STATUS_OFS | 6711 AHCI_INTR_STATUS_IPMS | 6712 AHCI_INTR_STATUS_INFS)) { 6713 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 6714 port, port_intr_status); 6715 } 6716 6717 /* 6718 * Check the fatal error interrupt bits, there are four kinds 6719 * of fatal errors for AHCI controllers: 6720 * 6721 * PxIS.HBFS - Host Bus Fatal Error 6722 * PxIS.HBDS - Host Bus Data Error 6723 * PxIS.IFS - Interface Fatal Error 6724 * PxIS.TFES - Task File Error 6725 * 6726 * The fatal error means the HBA can not recover from it by 6727 * itself, and it will try to abort the transfer, and the software 6728 * must intervene to restart the port. 6729 */ 6730 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 6731 AHCI_INTR_STATUS_HBDS | 6732 AHCI_INTR_STATUS_HBFS | 6733 AHCI_INTR_STATUS_TFES)) 6734 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 6735 port, port_intr_status); 6736 6737 /* Check the cold port detect interrupt bit */ 6738 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 6739 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 6740 } 6741 6742 /* Second clear the corresponding bit in IS.IPS */ 6743 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6744 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 6745 6746 /* Try to recover at the end of the interrupt handler. */ 6747 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) != 6748 DDI_FM_OK) { 6749 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6750 DDI_SERVICE_UNAFFECTED); 6751 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6752 DDI_FME_VERSION); 6753 } 6754 } 6755 6756 /* 6757 * Interrupt service handler 6758 */ 6759 static uint_t 6760 ahci_intr(caddr_t arg1, caddr_t arg2) 6761 { 6762 #ifndef __lock_lint 6763 _NOTE(ARGUNUSED(arg2)) 6764 #endif 6765 /* LINTED */ 6766 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 6767 ahci_port_t *ahci_portp; 6768 int32_t global_intr_status; 6769 uint8_t port; 6770 6771 /* 6772 * global_intr_status indicates that the corresponding port has 6773 * an interrupt pending. 6774 */ 6775 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6776 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 6777 6778 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 6779 /* The interrupt is not ours */ 6780 return (DDI_INTR_UNCLAIMED); 6781 } 6782 6783 /* 6784 * Check the handle after reading global_intr_status - we don't want 6785 * to miss any port with pending interrupts. 6786 */ 6787 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) != 6788 DDI_FM_OK) { 6789 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6790 DDI_SERVICE_UNAFFECTED); 6791 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6792 DDI_FME_VERSION); 6793 return (DDI_INTR_UNCLAIMED); 6794 } 6795 6796 /* Loop for all the ports */ 6797 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6798 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 6799 continue; 6800 } 6801 if (!((0x1 << port) & global_intr_status)) { 6802 continue; 6803 } 6804 6805 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6806 6807 /* Call ahci_port_intr */ 6808 ahci_port_intr(ahci_ctlp, ahci_portp, port); 6809 } 6810 6811 return (DDI_INTR_CLAIMED); 6812 } 6813 6814 /* 6815 * For non-queued commands, when the corresponding bit in the PxCI register 6816 * is cleared, it means the command is completed successfully. And according 6817 * to the HBA state machine, there are three conditions which possibly will 6818 * try to clear the PxCI register bit. 6819 * 1. Receive one D2H Register FIS which is with 'I' bit set 6820 * 2. Update PIO Setup FIS 6821 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 6822 * 6823 * Process completed non-queued commands when the interrupt status bit - 6824 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 6825 * 6826 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 6827 * with the 'I' bit set. And the following commands will send thus 6828 * FIS with 'I' bit set upon the successful completion: 6829 * 1. Non-data commands 6830 * 2. DMA data-in command 6831 * 3. DMA data-out command 6832 * 4. PIO data-out command 6833 * 5. PACKET non-data commands 6834 * 6. PACKET PIO data-in command 6835 * 7. PACKET PIO data-out command 6836 * 8. PACKET DMA data-in command 6837 * 9. PACKET DMA data-out command 6838 * 6839 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 6840 * with the 'I' bit set. And the following commands will send this 6841 * FIS upon the successful completion: 6842 * 1. PIO data-in command 6843 */ 6844 static int 6845 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 6846 ahci_port_t *ahci_portp, uint8_t port) 6847 { 6848 uint32_t port_cmd_issue = 0; 6849 uint32_t finished_tags; 6850 int finished_slot; 6851 sata_pkt_t *satapkt; 6852 ahci_fis_d2h_register_t *rcvd_fisp; 6853 #if AHCI_DEBUG 6854 ahci_cmd_header_t *cmd_header; 6855 uint32_t cmd_dmacount; 6856 #endif 6857 6858 mutex_enter(&ahci_portp->ahciport_mutex); 6859 6860 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6861 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6862 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6863 /* 6864 * Spurious interrupt. Nothing to be done. 6865 */ 6866 mutex_exit(&ahci_portp->ahciport_mutex); 6867 return (AHCI_SUCCESS); 6868 } 6869 6870 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6871 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6872 6873 /* If the PxCI corrupts, don't complete the commmands. */ 6874 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) 6875 != DDI_FM_OK) { 6876 mutex_exit(&ahci_portp->ahciport_mutex); 6877 return (AHCI_FAILURE); 6878 } 6879 6880 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6881 /* Slot 0 is always used during error recovery */ 6882 finished_tags = 0x1 & ~port_cmd_issue; 6883 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6884 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 6885 "retrieval is finished, and finished_tags = 0x%x", 6886 port, finished_tags); 6887 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6888 finished_tags = 0x1 & ~port_cmd_issue; 6889 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 6890 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w " 6891 "port multiplier is finished, and finished_tags = 0x%x", 6892 port, finished_tags); 6893 6894 } else { 6895 6896 finished_tags = ahci_portp->ahciport_pending_tags & 6897 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6898 } 6899 6900 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6901 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 6902 "port_cmd_issue = 0x%x finished_tags = 0x%x", 6903 ahci_portp->ahciport_pending_tags, port_cmd_issue, 6904 finished_tags); 6905 6906 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6907 (finished_tags == 0x1)) { 6908 satapkt = ahci_portp->ahciport_err_retri_pkt; 6909 ASSERT(satapkt != NULL); 6910 6911 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6912 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6913 "with SATA_PKT_COMPLETED", (void *)satapkt); 6914 6915 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6916 goto out; 6917 } 6918 6919 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6920 (finished_tags == 0x1)) { 6921 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 6922 ASSERT(satapkt != NULL); 6923 6924 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6925 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6926 "with SATA_PKT_COMPLETED", (void *)satapkt); 6927 6928 /* READ PORTMULT need copy out FIS content. */ 6929 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6930 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6931 ahcirf_d2h_register_fis); 6932 satapkt->satapkt_cmd.satacmd_status_reg = 6933 GET_RFIS_STATUS(rcvd_fisp); 6934 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6935 } 6936 6937 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6938 goto out; 6939 } 6940 6941 while (finished_tags) { 6942 finished_slot = ddi_ffs(finished_tags) - 1; 6943 if (finished_slot == -1) { 6944 goto out; 6945 } 6946 6947 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6948 ASSERT(satapkt != NULL); 6949 #if AHCI_DEBUG 6950 /* 6951 * For non-native queued commands, the PRD byte count field 6952 * shall contain an accurate count of the number of bytes 6953 * transferred for the command before the PxCI bit is cleared 6954 * to '0' for the command. 6955 * 6956 * The purpose of this field is to let software know how many 6957 * bytes transferred for a given operation in order to 6958 * determine if underflow occurred. When issuing native command 6959 * queuing commands, this field should not be used and is not 6960 * required to be valid since in this case underflow is always 6961 * illegal. 6962 * 6963 * For data reads, the HBA will update its PRD byte count with 6964 * the total number of bytes received from the last FIS, and 6965 * may be able to continue normally. For data writes, the 6966 * device will detect an error, and HBA most likely will get 6967 * a fatal error. 6968 * 6969 * Therefore, here just put code to debug part. And please 6970 * refer to the comment above ahci_intr_fatal_error for the 6971 * definition of underflow error. 6972 */ 6973 cmd_dmacount = 6974 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 6975 if (cmd_dmacount) { 6976 cmd_header = 6977 &ahci_portp->ahciport_cmd_list[finished_slot]; 6978 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 6979 "ahci_intr_cmd_cmplt: port %d, " 6980 "PRD Byte Count = 0x%x, " 6981 "ahciport_prd_bytecounts = 0x%x", port, 6982 cmd_header->ahcich_prd_byte_count, 6983 cmd_dmacount); 6984 6985 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 6986 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 6987 "ahci_intr_cmd_cmplt: port %d, " 6988 "an underflow occurred", port); 6989 } 6990 } 6991 #endif 6992 6993 /* 6994 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 6995 * feature, sata_special_regs flag will be set, and the 6996 * driver should copy the status and the other corresponding 6997 * register values in the D2H Register FIS received (It's 6998 * working on Non-data protocol) from the device back to 6999 * the sata_cmd. 7000 * 7001 * For every AHCI port, there is only one Received FIS 7002 * structure, which contains the FISes received from the 7003 * device, So we're trying to copy the content of D2H 7004 * Register FIS in the Received FIS structure back to 7005 * the sata_cmd. 7006 */ 7007 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 7008 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 7009 ahcirf_d2h_register_fis); 7010 satapkt->satapkt_cmd.satacmd_status_reg = 7011 GET_RFIS_STATUS(rcvd_fisp); 7012 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 7013 } 7014 7015 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7016 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 7017 "with SATA_PKT_COMPLETED", (void *)satapkt); 7018 7019 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 7020 CLEAR_BIT(finished_tags, finished_slot); 7021 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 7022 7023 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 7024 } 7025 out: 7026 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 7027 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 7028 ahci_portp->ahciport_pending_tags); 7029 7030 ahci_flush_doneq(ahci_portp); 7031 7032 mutex_exit(&ahci_portp->ahciport_mutex); 7033 7034 return (AHCI_SUCCESS); 7035 } 7036 7037 /* 7038 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 7039 * with the 'I' bit set and has been copied into system memory. It will 7040 * be sent under the following situations: 7041 * 7042 * 1. NCQ command is completed 7043 * 7044 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 7045 * via the Set Device Bits FIS. When such event is generated, the software 7046 * needs to read PxSACT register and compares the current value to the 7047 * list of commands previously issue by software. ahciport_pending_ncq_tags 7048 * keeps the tags of previously issued commands. 7049 * 7050 * 2. Asynchronous Notification 7051 * 7052 * Asynchronous Notification is a feature in SATA spec 2.6. 7053 * 7054 * 1) ATAPI device will send a signal to the host when media is inserted or 7055 * removed and avoids polling the device for media changes. The signal 7056 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 7057 * set to '1'. At the moment, it's not supported yet. 7058 * 7059 * 2) Port multiplier will send a signal to the host when a hot plug event 7060 * has occured on a port multiplier port. It is used when command based 7061 * switching is employed. This is handled by ahci_intr_pmult_sntf_events() 7062 */ 7063 static int 7064 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 7065 ahci_port_t *ahci_portp, uint8_t port) 7066 { 7067 ahci_addr_t addr; 7068 7069 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7070 "ahci_intr_set_device_bits enter: port %d", port); 7071 7072 /* Initialize HBA port address */ 7073 AHCI_ADDR_SET_PORT(&addr, port); 7074 7075 /* NCQ plug handler */ 7076 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr); 7077 7078 /* Check port multiplier's asynchronous notification events */ 7079 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 7080 (void) ahci_intr_pmult_sntf_events(ahci_ctlp, 7081 ahci_portp, port); 7082 } 7083 7084 /* ATAPI events is not supported yet */ 7085 7086 return (AHCI_SUCCESS); 7087 } 7088 /* 7089 * NCQ interrupt handler. Called upon a NCQ command is completed. 7090 * Only be called from ahci_intr_set_device_bits(). 7091 */ 7092 static int 7093 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp, 7094 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 7095 { 7096 uint32_t port_sactive; 7097 uint32_t port_cmd_issue; 7098 uint32_t issued_tags; 7099 int issued_slot; 7100 uint32_t finished_tags; 7101 int finished_slot; 7102 uint8_t port = addrp->aa_port; 7103 sata_pkt_t *satapkt; 7104 7105 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7106 "ahci_intr_set_device_bits enter: port %d", port); 7107 7108 mutex_enter(&ahci_portp->ahciport_mutex); 7109 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7110 mutex_exit(&ahci_portp->ahciport_mutex); 7111 return (AHCI_SUCCESS); 7112 } 7113 7114 /* 7115 * First the handler got which commands are finished by checking 7116 * PxSACT register 7117 */ 7118 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7119 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7120 7121 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 7122 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 7123 7124 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7125 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 7126 "port_sactive = 0x%x", port, 7127 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 7128 7129 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7130 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 7131 7132 /* 7133 * For NCQ commands, the software can determine which command has 7134 * already been transmitted to the device by checking PxCI register. 7135 */ 7136 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7137 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7138 7139 issued_tags = ahci_portp->ahciport_pending_tags & 7140 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 7141 7142 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */ 7143 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) 7144 != DDI_FM_OK) { 7145 mutex_exit(&ahci_portp->ahciport_mutex); 7146 return (AHCI_FAILURE); 7147 } 7148 7149 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7150 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 7151 "port_cmd_issue = 0x%x", port, 7152 ahci_portp->ahciport_pending_tags, port_cmd_issue); 7153 7154 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7155 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 7156 7157 /* 7158 * Clear ahciport_pending_tags bit when the corresponding command 7159 * is already sent down to the device. 7160 */ 7161 while (issued_tags) { 7162 issued_slot = ddi_ffs(issued_tags) - 1; 7163 if (issued_slot == -1) { 7164 goto next; 7165 } 7166 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 7167 CLEAR_BIT(issued_tags, issued_slot); 7168 } 7169 7170 next: 7171 while (finished_tags) { 7172 finished_slot = ddi_ffs(finished_tags) - 1; 7173 if (finished_slot == -1) { 7174 goto out; 7175 } 7176 7177 /* The command is certainly transmitted to the device */ 7178 ASSERT(!(ahci_portp->ahciport_pending_tags & 7179 (0x1 << finished_slot))); 7180 7181 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 7182 ASSERT(satapkt != NULL); 7183 7184 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7185 "ahci_intr_set_device_bits: sending up pkt 0x%p " 7186 "with SATA_PKT_COMPLETED", (void *)satapkt); 7187 7188 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 7189 CLEAR_BIT(finished_tags, finished_slot); 7190 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 7191 7192 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 7193 } 7194 out: 7195 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 7196 "ahci_intr_set_device_bits: port %d " 7197 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 7198 port, ahci_portp->ahciport_pending_ncq_tags, 7199 ahci_portp->ahciport_pending_tags); 7200 7201 ahci_flush_doneq(ahci_portp); 7202 7203 mutex_exit(&ahci_portp->ahciport_mutex); 7204 7205 return (AHCI_SUCCESS); 7206 } 7207 7208 /* 7209 * Port multiplier asynchronous notification event handler. Called upon a 7210 * device is hot plugged/pulled. 7211 * 7212 * The async-notification event will only be recorded by ahcipmi_snotif_tags 7213 * here and will be handled by ahci_probe_pmult(). 7214 * 7215 * NOTE: called only from ahci_port_intr(). 7216 */ 7217 static int 7218 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7219 uint8_t port) 7220 { 7221 sata_device_t sdevice; 7222 7223 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7224 "ahci_intr_pmult_sntf_events enter: port %d ", port); 7225 7226 /* no hot-plug while attaching process */ 7227 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7228 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 7229 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7230 return (AHCI_SUCCESS); 7231 } 7232 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7233 7234 mutex_enter(&ahci_portp->ahciport_mutex); 7235 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 7236 mutex_exit(&ahci_portp->ahciport_mutex); 7237 return (AHCI_SUCCESS); 7238 } 7239 7240 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 7241 7242 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags = 7243 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7244 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port)); 7245 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7246 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 7247 AHCI_SNOTIF_CLEAR_ALL); 7248 7249 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) { 7250 mutex_exit(&ahci_portp->ahciport_mutex); 7251 return (AHCI_SUCCESS); 7252 } 7253 7254 /* Port Multiplier sub-device hot-plug handler */ 7255 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 7256 mutex_exit(&ahci_portp->ahciport_mutex); 7257 return (AHCI_SUCCESS); 7258 } 7259 7260 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) { 7261 /* Not allowed to re-enter. */ 7262 mutex_exit(&ahci_portp->ahciport_mutex); 7263 return (AHCI_SUCCESS); 7264 } 7265 7266 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF; 7267 7268 /* 7269 * NOTE: 7270 * Even if Asynchronous Notification is supported (and enabled) by 7271 * both controller and the port multiplier, the content of PxSNTF 7272 * register is always set to 0x8000 by async notification event. We 7273 * need to check GSCR[32] on the port multiplier to find out the 7274 * owner of this event. 7275 * This is not accord with SATA spec 2.6 and needs further 7276 * clarification. 7277 */ 7278 /* hot-plug will not reported while reseting. */ 7279 if (ahci_portp->ahciport_reset_in_progress == 1) { 7280 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 7281 "port %d snotif event ignored", port); 7282 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 7283 mutex_exit(&ahci_portp->ahciport_mutex); 7284 return (AHCI_SUCCESS); 7285 } 7286 7287 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 7288 "PxSNTF is set to 0x%x by port multiplier", 7289 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags); 7290 7291 /* 7292 * Now we need do some necessary operation and inform SATA framework 7293 * that link/device events has happened. 7294 */ 7295 bzero((void *)&sdevice, sizeof (sata_device_t)); 7296 sdevice.satadev_addr.cport = ahci_ctlp-> 7297 ahcictl_port_to_cport[port]; 7298 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 7299 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 7300 sdevice.satadev_state = SATA_PSTATE_PWRON; 7301 7302 /* Just reject packets, do not stop that port. */ 7303 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7304 7305 mutex_exit(&ahci_portp->ahciport_mutex); 7306 sata_hba_event_notify( 7307 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7308 &sdevice, 7309 SATA_EVNT_PMULT_LINK_CHANGED); 7310 mutex_enter(&ahci_portp->ahciport_mutex); 7311 7312 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 7313 mutex_exit(&ahci_portp->ahciport_mutex); 7314 7315 return (AHCI_SUCCESS); 7316 } 7317 7318 /* 7319 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 7320 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 7321 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 7322 * indicates a COMINIT signal was received. 7323 * 7324 * Hot plug insertion is detected by reception of a COMINIT signal from the 7325 * device. On reception of unsolicited COMINIT, the HBA shall generate a 7326 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 7327 * begin the normal communication negotiation sequence as outlined in the 7328 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 7329 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 7330 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 7331 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 7332 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 7333 * to handle hot plug insertion. In this interrupt handler, just do nothing 7334 * but print some log message and clear the bit. 7335 */ 7336 static int 7337 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 7338 ahci_port_t *ahci_portp, uint8_t port) 7339 { 7340 #if AHCI_DEBUG 7341 uint32_t port_serror; 7342 #endif 7343 7344 mutex_enter(&ahci_portp->ahciport_mutex); 7345 7346 #if AHCI_DEBUG 7347 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7348 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7349 7350 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7351 "ahci_intr_port_connect_change: port %d, " 7352 "port_serror = 0x%x", port, port_serror); 7353 #endif 7354 7355 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 7356 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7357 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7358 SERROR_EXCHANGED_ERR); 7359 7360 mutex_exit(&ahci_portp->ahciport_mutex); 7361 7362 return (AHCI_SUCCESS); 7363 } 7364 7365 /* 7366 * Hot Plug Operation for platforms that support Mechanical Presence 7367 * Switches. 7368 * 7369 * When set, it indicates that a mechanical presence switch attached to this 7370 * port has been opened or closed, which may lead to a change in the connection 7371 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 7372 * are set to '1'. 7373 * 7374 * At the moment, this interrupt is not needed and disabled and we just log 7375 * the debug message. 7376 */ 7377 static int 7378 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 7379 ahci_port_t *ahci_portp, uint8_t port) 7380 { 7381 uint32_t cap_status, port_cmd_status; 7382 7383 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7384 "ahci_intr_device_mechanical_presence_status enter, " 7385 "port %d", port); 7386 7387 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7388 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 7389 7390 mutex_enter(&ahci_portp->ahciport_mutex); 7391 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7392 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7393 7394 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 7395 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 7396 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7397 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 7398 "the interrupt: cap_status = 0x%x, " 7399 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 7400 mutex_exit(&ahci_portp->ahciport_mutex); 7401 7402 return (AHCI_SUCCESS); 7403 } 7404 7405 #if AHCI_DEBUG 7406 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 7407 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7408 "The mechanical presence switch is open: " 7409 "port %d, port_cmd_status = 0x%x", 7410 port, port_cmd_status); 7411 } else { 7412 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7413 "The mechanical presence switch is close: " 7414 "port %d, port_cmd_status = 0x%x", 7415 port, port_cmd_status); 7416 } 7417 #endif 7418 7419 mutex_exit(&ahci_portp->ahciport_mutex); 7420 7421 return (AHCI_SUCCESS); 7422 } 7423 7424 /* 7425 * Native Hot Plug Support. 7426 * 7427 * When set, it indicates that the internal PHYRDY signal changed state. 7428 * This bit reflects the state of PxSERR.DIAG.N. 7429 * 7430 * There are three kinds of conditions to generate this interrupt event: 7431 * 1. a device is inserted 7432 * 2. a device is disconnected 7433 * 3. when the link enters/exits a Partial or Slumber interface power 7434 * management state 7435 * 7436 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 7437 * bit may be set due to the link entering the Partial or Slumber power 7438 * management state, rather than due to a hot plug insertion or removal 7439 * event. So far, the interface power management is disabled, so the 7440 * driver can reliably get removal detection notification via the 7441 * PxSERR.DIAG.N bit. 7442 */ 7443 static int 7444 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 7445 ahci_port_t *ahci_portp, uint8_t port) 7446 { 7447 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 7448 sata_device_t sdevice; 7449 int dev_exists_now = 0; 7450 int dev_existed_previously = 0; 7451 ahci_addr_t port_addr; 7452 7453 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7454 "ahci_intr_phyrdy_change enter, port %d", port); 7455 7456 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 7457 mutex_enter(&ahci_portp->ahciport_mutex); 7458 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7459 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7460 SERROR_PHY_RDY_CHG); 7461 mutex_exit(&ahci_portp->ahciport_mutex); 7462 7463 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7464 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 7465 (ahci_portp == NULL)) { 7466 /* The whole controller setup is not yet done. */ 7467 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7468 return (AHCI_SUCCESS); 7469 } 7470 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7471 7472 mutex_enter(&ahci_portp->ahciport_mutex); 7473 7474 /* SStatus tells the presence of device. */ 7475 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7476 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 7477 7478 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 7479 dev_exists_now = 1; 7480 } 7481 7482 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 7483 dev_existed_previously = 1; 7484 } 7485 7486 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 7487 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 7488 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7489 "ahci_intr_phyrdy_change: port %d " 7490 "AHCI_PORT_FLAG_NODEV is cleared", port); 7491 if (dev_exists_now == 0) 7492 dev_existed_previously = 1; 7493 } 7494 7495 bzero((void *)&sdevice, sizeof (sata_device_t)); 7496 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7497 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7498 sdevice.satadev_addr.pmport = 0; 7499 sdevice.satadev_state = SATA_PSTATE_PWRON; 7500 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 7501 7502 AHCI_ADDR_SET_PORT(&port_addr, port); 7503 7504 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 7505 if (dev_exists_now) { 7506 if (dev_existed_previously) { /* 1 -> 1 */ 7507 /* Things are fine now. The loss was temporary. */ 7508 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7509 "ahci_intr_phyrdy_change port %d " 7510 "device link lost/established", port); 7511 7512 mutex_exit(&ahci_portp->ahciport_mutex); 7513 sata_hba_event_notify( 7514 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7515 &sdevice, 7516 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 7517 mutex_enter(&ahci_portp->ahciport_mutex); 7518 7519 } else { /* 0 -> 1 */ 7520 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7521 "ahci_intr_phyrdy_change: port %d " 7522 "device link established", port); 7523 7524 /* 7525 * A new device has been detected. The new device 7526 * might be a port multiplier instead of a drive, so 7527 * we cannot update the signature directly. 7528 */ 7529 (void) ahci_initialize_port(ahci_ctlp, 7530 ahci_portp, &port_addr); 7531 7532 /* Try to start the port */ 7533 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 7534 != AHCI_SUCCESS) { 7535 sdevice.satadev_state |= SATA_PSTATE_FAILED; 7536 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7537 "ahci_intr_phyrdy_change: port %d failed " 7538 "at start port", port); 7539 } 7540 7541 /* Clear the max queue depth for inserted device */ 7542 ahci_portp->ahciport_max_ncq_tags = 0; 7543 7544 mutex_exit(&ahci_portp->ahciport_mutex); 7545 sata_hba_event_notify( 7546 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7547 &sdevice, 7548 SATA_EVNT_LINK_ESTABLISHED); 7549 mutex_enter(&ahci_portp->ahciport_mutex); 7550 7551 } 7552 } else { /* No device exists now */ 7553 7554 if (dev_existed_previously) { /* 1 -> 0 */ 7555 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7556 "ahci_intr_phyrdy_change: port %d " 7557 "device link lost", port); 7558 7559 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7560 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7561 ahci_portp, port); 7562 7563 if (ahci_portp->ahciport_device_type == 7564 SATA_DTYPE_PMULT) { 7565 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 7566 } 7567 7568 /* An existing device is lost. */ 7569 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 7570 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN; 7571 7572 mutex_exit(&ahci_portp->ahciport_mutex); 7573 sata_hba_event_notify( 7574 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7575 &sdevice, 7576 SATA_EVNT_LINK_LOST); 7577 mutex_enter(&ahci_portp->ahciport_mutex); 7578 } 7579 } 7580 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 7581 7582 mutex_exit(&ahci_portp->ahciport_mutex); 7583 7584 return (AHCI_SUCCESS); 7585 } 7586 7587 /* 7588 * PxIS.UFS - Unknown FIS Error 7589 * 7590 * This interrupt event means an unknown FIS was received and has been 7591 * copied into system memory. An unknown FIS is not considered an illegal 7592 * FIS, unless the length received is more than 64 bytes. If an unknown 7593 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 7594 * normal operation. If the unknown FIS is more than 64 bytes, then it 7595 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 7596 * a fatal error. 7597 * 7598 * PxIS.IPMS - Incorrect Port Multiplier Status 7599 * 7600 * IPMS Indicates that the HBA received a FIS from a device that did not 7601 * have a command outstanding. The IPMS bit may be set during enumeration 7602 * of devices on a Port Multiplier due to the normal Port Multiplier 7603 * enumeration process. It is recommended that IPMS only be used after 7604 * enumeration is complete on the Port Multiplier (copied from spec). 7605 * 7606 * PxIS.OFS - Overflow Error 7607 * 7608 * Command list overflow is defined as software building a command table 7609 * that has fewer total bytes than the transaction given to the device. 7610 * On device writes, the HBA will run out of data, and on reads, there 7611 * will be no room to put the data. 7612 * 7613 * For an overflow on data read, either PIO or DMA, the HBA will set 7614 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 7615 * non-fatal error when the HBA can continues. Sometimes, it will cause 7616 * a fatal error and need the software to do something. 7617 * 7618 * For an overflow on data write, setting PxIS.OFS is optional for both 7619 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 7620 * software to clean up from this serious error. 7621 * 7622 * PxIS.INFS - Interface Non-Fatal Error 7623 * 7624 * This interrupt event indicates that the HBA encountered an error on 7625 * the Serial ATA interface but was able to continue operation. The kind 7626 * of error usually occurred during a non-Data FIS, and under this condition 7627 * the FIS will be re-transmitted by HBA automatically. 7628 * 7629 * When the FMA is implemented, there should be a stat structure to 7630 * record how many every kind of error happens. 7631 */ 7632 static int 7633 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7634 uint8_t port, uint32_t intr_status) 7635 { 7636 uint32_t port_serror; 7637 #if AHCI_DEBUG 7638 uint32_t port_cmd_status; 7639 uint32_t port_cmd_issue; 7640 uint32_t port_sactive; 7641 int current_slot; 7642 uint32_t current_tags; 7643 sata_pkt_t *satapkt; 7644 ahci_cmd_header_t *cmd_header; 7645 uint32_t cmd_dmacount; 7646 #endif 7647 7648 mutex_enter(&ahci_portp->ahciport_mutex); 7649 7650 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7651 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7652 7653 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 7654 "ahci_intr_non_fatal_error: port %d, " 7655 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status); 7656 7657 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 7658 7659 if (intr_status & AHCI_INTR_STATUS_UFS) { 7660 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7661 "ahci port %d has unknown FIS error", port); 7662 7663 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 7664 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7665 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7666 SERROR_FIS_TYPE); 7667 } 7668 7669 #if AHCI_DEBUG 7670 if (intr_status & AHCI_INTR_STATUS_IPMS) { 7671 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d " 7672 "has Incorrect Port Multiplier Status error", port); 7673 } 7674 7675 if (intr_status & AHCI_INTR_STATUS_OFS) { 7676 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7677 "ahci port %d has overflow error", port); 7678 } 7679 7680 if (intr_status & AHCI_INTR_STATUS_INFS) { 7681 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7682 "ahci port %d has interface non fatal error", port); 7683 } 7684 7685 /* 7686 * Record the error occurred command's slot. 7687 */ 7688 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7689 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7690 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7691 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7692 7693 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7694 AHCI_CMD_STATUS_CCS_SHIFT; 7695 7696 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7697 satapkt = ahci_portp->ahciport_err_retri_pkt; 7698 ASSERT(satapkt != NULL); 7699 ASSERT(current_slot == 0); 7700 } else { 7701 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7702 } 7703 7704 if (satapkt != NULL) { 7705 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7706 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 7707 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 7708 satapkt->satapkt_cmd.satacmd_cmd_reg); 7709 7710 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7711 "ahci_intr_non_fatal_error: port %d, " 7712 "satapkt 0x%p is being processed when error occurs", 7713 port, (void *)satapkt); 7714 7715 /* 7716 * PRD Byte Count field of command header is not 7717 * required to reflect the total number of bytes 7718 * transferred when an overflow occurs, so here 7719 * just log the value. 7720 */ 7721 cmd_dmacount = 7722 ahci_portp->ahciport_prd_bytecounts[current_slot]; 7723 if (cmd_dmacount) { 7724 cmd_header = &ahci_portp-> 7725 ahciport_cmd_list[current_slot]; 7726 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7727 "ahci_intr_non_fatal_error: port %d, " 7728 "PRD Byte Count = 0x%x, " 7729 "ahciport_prd_bytecounts = 0x%x", port, 7730 cmd_header->ahcich_prd_byte_count, 7731 cmd_dmacount); 7732 } 7733 } 7734 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7735 /* 7736 * For queued command, list those command which have already 7737 * been transmitted to the device and still not completed. 7738 */ 7739 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7740 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7741 7742 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7743 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7744 7745 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 7746 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 7747 "port_sactive = 0x%x port_cmd_issue = 0x%x", 7748 ahci_portp->ahciport_pending_ncq_tags, 7749 port_sactive, port_cmd_issue); 7750 7751 current_tags = ahci_portp->ahciport_pending_ncq_tags & 7752 port_sactive & ~port_cmd_issue & 7753 AHCI_NCQ_SLOT_MASK(ahci_portp); 7754 7755 while (current_tags) { 7756 current_slot = ddi_ffs(current_tags) - 1; 7757 if (current_slot == -1) { 7758 goto out; 7759 } 7760 7761 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7762 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 7763 ahci_ctlp, "ahci_intr_non_fatal_error: " 7764 "port %d, satapkt 0x%p is outstanding when " 7765 "error occurs", port, (void *)satapkt); 7766 7767 CLEAR_BIT(current_tags, current_slot); 7768 } 7769 } 7770 out: 7771 #endif 7772 mutex_exit(&ahci_portp->ahciport_mutex); 7773 7774 return (AHCI_SUCCESS); 7775 } 7776 7777 /* 7778 * According to the AHCI spec, the error types include system memory 7779 * errors, interface errors, port multiplier errors, device errors, 7780 * command list overflow, command list underflow, native command 7781 * queuing tag errors and pio data transfer errors. 7782 * 7783 * System memory errors such as target abort, master abort, and parity 7784 * may cause the host to stop, and they are serious errors and needed 7785 * to be recovered with software intervention. When system software 7786 * has given a pointer to the HBA that doesn't exist in physical memory, 7787 * a master/target abort error occurs, and PxIS.HBFS will be set. A 7788 * data error such as CRC or parity occurs, the HBA aborts the transfer 7789 * (if necessary) and PxIS.HBDS will be set. 7790 * 7791 * Interface errors are errors that occur due to electrical issues on 7792 * the interface, or protocol miscommunication between the device and 7793 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 7794 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 7795 * causes PxIS.IFS/PxIS.INFS to be set are 7796 * 1. in PxSERR.ERR, P bit is set to '1' 7797 * 2. in PxSERR.DIAG, C or H bit is set to '1' 7798 * 3. PhyRdy drop unexpectly, N bit is set to '1' 7799 * If the error occurred during a non-data FIS, the FIS must be 7800 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 7801 * the error occurred during a data FIS, the transfer will stop, so 7802 * the error is fatal and PxIS.IFS is set. 7803 * 7804 * When a FIS arrives that updates the taskfile, the HBA checks to see 7805 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 7806 * stops processing any more commands. 7807 * 7808 * Command list overflow is defined as software building a command table 7809 * that has fewer total bytes than the transaction given to the device. 7810 * On device writes, the HBA will run out of data, and on reads, there 7811 * will be no room to put the data. For an overflow on data read, either 7812 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 7813 * For an overflow on data write, setting PxIS.OFS is optional for both 7814 * DMA and PIO, and a COMRESET is required by software to clean up from 7815 * this serious error. 7816 * 7817 * Command list underflow is defined as software building a command 7818 * table that has more total bytes than the transaction given to the 7819 * device. For data writes, both PIO and DMA, the device will detect 7820 * an error and end the transfer. And these errors are most likely going 7821 * to be fatal errors that will cause the port to be restarted. For 7822 * data reads, the HBA updates its PRD byte count, and may be 7823 * able to continue normally, but is not required to. And The HBA is 7824 * not required to detect underflow conditions for native command 7825 * queuing command. 7826 * 7827 * The HBA does not actively check incoming DMA Setup FISes to ensure 7828 * that the PxSACT register bit for that slot is set. Existing error 7829 * mechanisms, such as host bus failure, or bad protocol, are used to 7830 * recover from this case. 7831 * 7832 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 7833 * DATA FIS must be an integral number of Dwords. If the HBA receives 7834 * a request which is not an integral number of Dwords, the HBA 7835 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 7836 * software restarts the port. And the HBA ensures that the size 7837 * of the DATA FIS received during a PIO command matches the size in 7838 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 7839 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 7840 * stop running until software restarts the port. 7841 */ 7842 /* 7843 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 7844 * 7845 * PxIS.IFS indicates that the hba encountered an error on the serial ata 7846 * interface which caused the transfer to stop. 7847 * 7848 * PxIS.HBDS indicates that the hba encountered a data error 7849 * (uncorrectable ecc/parity) when reading from or writing to system memory. 7850 * 7851 * PxIS.HBFS indicates that the hba encountered a host bus error that it 7852 * cannot recover from, such as a bad software pointer. 7853 * 7854 * PxIS.TFES is set whenever the status register is updated by the device 7855 * and the error bit (bit 0) is set. 7856 */ 7857 static int 7858 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 7859 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 7860 { 7861 uint32_t port_cmd_status; 7862 uint32_t port_serror; 7863 uint32_t task_file_status; 7864 int failed_slot; 7865 sata_pkt_t *spkt = NULL; 7866 uint8_t err_byte; 7867 ahci_event_arg_t *args; 7868 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7869 uint32_t failed_tags = 0; 7870 int task_fail_flag = 0, task_abort_flag = 0; 7871 uint32_t slot_status; 7872 7873 mutex_enter(&ahci_portp->ahciport_mutex); 7874 7875 /* 7876 * ahci_intr_phyrdy_change() may have rendered it to 7877 * SATA_DTYPE_NONE. 7878 */ 7879 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7880 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7881 "ahci_intr_fatal_error: port %d no device attached, " 7882 "and just return without doing anything", port); 7883 goto out0; 7884 } 7885 7886 if (intr_status & AHCI_INTR_STATUS_TFES) { 7887 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7888 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 7889 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7890 "ahci_intr_fatal_error: port %d " 7891 "task_file_status = 0x%x", port, task_file_status); 7892 task_fail_flag = 1; 7893 7894 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 7895 >> AHCI_TFD_ERR_SHIFT; 7896 if (err_byte == SATA_ERROR_ABORT) 7897 task_abort_flag = 1; 7898 } 7899 7900 /* 7901 * Here we just log the fatal error info in interrupt context. 7902 * Misc recovery processing will be handled in task queue. 7903 */ 7904 if (task_fail_flag == 1) { 7905 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7906 /* 7907 * Read PxCMD.CCS to determine the slot that the HBA 7908 * was processing when the error occurred. 7909 */ 7910 port_cmd_status = ddi_get32( 7911 ahci_ctlp->ahcictl_ahci_acc_handle, 7912 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7913 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7914 AHCI_CMD_STATUS_CCS_SHIFT; 7915 failed_tags = 0x1 << failed_slot; 7916 7917 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 7918 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7919 "ahci_intr_fatal_error: spkt 0x%p is being " 7920 "processed when fatal error occurred for port %d", 7921 spkt, port); 7922 7923 /* 7924 * Won't emit the error message if it is an IDENTIFY 7925 * DEVICE command sent to an ATAPI device. 7926 */ 7927 if ((spkt != NULL) && 7928 (spkt->satapkt_cmd.satacmd_cmd_reg == 7929 SATAC_ID_DEVICE) && 7930 (task_abort_flag == 1)) 7931 goto out1; 7932 7933 /* 7934 * Won't emit the error message if it is an ATAPI PACKET 7935 * command 7936 */ 7937 if ((spkt != NULL) && 7938 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 7939 goto out1; 7940 7941 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7942 slot_status = ddi_get32( 7943 ahci_ctlp->ahcictl_ahci_acc_handle, 7944 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7945 failed_tags = slot_status & 7946 AHCI_NCQ_SLOT_MASK(ahci_portp); 7947 } 7948 } 7949 7950 /* print the fatal error type */ 7951 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 7952 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT; 7953 7954 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7955 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7956 7957 /* print PxSERR related error message */ 7958 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 7959 7960 /* print task file register value */ 7961 if (task_fail_flag == 1) { 7962 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 7963 "= 0x%x", instance, port, task_file_status); 7964 if (task_abort_flag == 1) { 7965 cmn_err(CE_WARN, "!ahci%d: the below command (s) on " 7966 "port %d are aborted", instance, port); 7967 ahci_dump_commands(ahci_ctlp, port, failed_tags); 7968 } 7969 } 7970 7971 out1: 7972 /* Prepare the argument for the taskq */ 7973 args = ahci_portp->ahciport_event_args; 7974 args->ahciea_ctlp = (void *)ahci_ctlp; 7975 args->ahciea_portp = (void *)ahci_portp; 7976 args->ahciea_event = intr_status; 7977 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port); 7978 7979 /* Start the taskq to handle error recovery */ 7980 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 7981 ahci_events_handler, 7982 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 7983 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 7984 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery " 7985 "port %d failed", instance, port); 7986 } 7987 out0: 7988 mutex_exit(&ahci_portp->ahciport_mutex); 7989 7990 return (AHCI_SUCCESS); 7991 } 7992 7993 /* 7994 * Hot Plug Operation for platforms that support Cold Presence Detect. 7995 * 7996 * When set, a device status has changed as detected by the cold presence 7997 * detect logic. This bit can either be set due to a non-connected port 7998 * receiving a device, or a connected port having its device removed. 7999 * This bit is only valid if the port supports cold presence detect as 8000 * indicated by PxCMD.CPD set to '1'. 8001 * 8002 * At the moment, this interrupt is not needed and disabled and we just 8003 * log the debug message. 8004 */ 8005 static int 8006 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 8007 ahci_port_t *ahci_portp, uint8_t port) 8008 { 8009 uint32_t port_cmd_status; 8010 sata_device_t sdevice; 8011 8012 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8013 "ahci_intr_cold_port_detect enter, port %d", port); 8014 8015 mutex_enter(&ahci_portp->ahciport_mutex); 8016 8017 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8018 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8019 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 8020 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8021 "port %d does not support cold presence detect, so " 8022 "we just ignore this interrupt", port); 8023 mutex_exit(&ahci_portp->ahciport_mutex); 8024 return (AHCI_SUCCESS); 8025 } 8026 8027 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8028 "port %d device status has changed", port); 8029 8030 bzero((void *)&sdevice, sizeof (sata_device_t)); 8031 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8032 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 8033 sdevice.satadev_addr.pmport = 0; 8034 sdevice.satadev_state = SATA_PSTATE_PWRON; 8035 8036 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 8037 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8038 "port %d: a device is hot plugged", port); 8039 mutex_exit(&ahci_portp->ahciport_mutex); 8040 sata_hba_event_notify( 8041 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8042 &sdevice, 8043 SATA_EVNT_DEVICE_ATTACHED); 8044 mutex_enter(&ahci_portp->ahciport_mutex); 8045 8046 } else { 8047 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8048 "port %d: a device is hot unplugged", port); 8049 mutex_exit(&ahci_portp->ahciport_mutex); 8050 sata_hba_event_notify( 8051 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8052 &sdevice, 8053 SATA_EVNT_DEVICE_DETACHED); 8054 mutex_enter(&ahci_portp->ahciport_mutex); 8055 } 8056 8057 mutex_exit(&ahci_portp->ahciport_mutex); 8058 8059 return (AHCI_SUCCESS); 8060 } 8061 8062 /* 8063 * Enable the interrupts for a particular port. 8064 */ 8065 static void 8066 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 8067 { 8068 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8069 8070 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8071 "ahci_enable_port_intrs enter, port %d", port); 8072 8073 /* 8074 * Clear port interrupt status before enabling interrupt 8075 */ 8076 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8077 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 8078 AHCI_PORT_INTR_MASK); 8079 8080 /* 8081 * Clear the pending bit from IS.IPS 8082 */ 8083 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8084 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 8085 8086 /* 8087 * Enable the following interrupts: 8088 * Device to Host Register FIS Interrupt (DHRS) 8089 * PIO Setup FIS Interrupt (PSS) 8090 * Set Device Bits Interrupt (SDBS) 8091 * Unknown FIS Interrupt (UFS) 8092 * Port Connect Change Status (PCS) 8093 * PhyRdy Change Status (PRCS) 8094 * Overflow Status (OFS) 8095 * Interface Non-fatal Error Status (INFS) 8096 * Interface Fatal Error Status (IFS) 8097 * Host Bus Data Error Status (HBDS) 8098 * Host Bus Fatal Error Status (HBFS) 8099 * Task File Error Status (TFES) 8100 */ 8101 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8102 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 8103 (AHCI_INTR_STATUS_DHRS | 8104 AHCI_INTR_STATUS_PSS | 8105 AHCI_INTR_STATUS_SDBS | 8106 AHCI_INTR_STATUS_UFS | 8107 AHCI_INTR_STATUS_DPS | 8108 AHCI_INTR_STATUS_PCS | 8109 AHCI_INTR_STATUS_PRCS | 8110 AHCI_INTR_STATUS_OFS | 8111 AHCI_INTR_STATUS_INFS | 8112 AHCI_INTR_STATUS_IFS | 8113 AHCI_INTR_STATUS_HBDS | 8114 AHCI_INTR_STATUS_HBFS | 8115 AHCI_INTR_STATUS_TFES)); 8116 } 8117 8118 /* 8119 * Enable interrupts for all the ports. 8120 */ 8121 static void 8122 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 8123 { 8124 uint32_t ghc_control; 8125 8126 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 8127 8128 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 8129 8130 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8131 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 8132 8133 ghc_control |= AHCI_HBA_GHC_IE; 8134 8135 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8136 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 8137 } 8138 8139 /* 8140 * Disable interrupts for a particular port. 8141 */ 8142 static void 8143 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 8144 { 8145 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE || 8146 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8147 8148 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8149 "ahci_disable_port_intrs enter, port %d", port); 8150 8151 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8152 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 8153 } 8154 8155 /* 8156 * Disable interrupts for the whole HBA. 8157 * 8158 * The global bit is cleared, then all interrupt sources from all 8159 * ports are disabled. 8160 */ 8161 static void 8162 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 8163 { 8164 uint32_t ghc_control; 8165 8166 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) || 8167 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 8168 8169 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 8170 NULL); 8171 8172 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8173 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 8174 8175 ghc_control &= ~AHCI_HBA_GHC_IE; 8176 8177 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8178 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 8179 } 8180 8181 /* 8182 * Handle FIXED or MSI interrupts. 8183 */ 8184 /* 8185 * According to AHCI spec, the HBA may support several interrupt modes: 8186 * * pin based interrupts (FIXED) 8187 * * single MSI message interrupts 8188 * * multiple MSI based message interrupts 8189 * 8190 * For pin based interrupts, the software interrupt handler need to check IS 8191 * register to find out which port has pending interrupts. And then check 8192 * PxIS register to find out which interrupt events happened on that port. 8193 * 8194 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 8195 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 8196 * in that software interrupt handler need to check IS register to determine 8197 * which port triggered the interrupts since it uses a single message for all 8198 * port interrupts. 8199 * 8200 * HBA may optionally support multiple MSI message for better performance. In 8201 * this mode, each port may have its own interrupt message, and thus generation 8202 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 8203 * represents a power-of-2 wrapper on the number of implemented ports, and 8204 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 8205 * maximum number of assigned interrupts. When the number of MSI messages 8206 * allocated is less than the number requested, then hardware may have two 8207 * implementation behaviors: 8208 * * assign each ports its own interrupt and then force all additional 8209 * ports to share the last interrupt message, and this condition is 8210 * indicated by clearing GHC.MRSM to '0' 8211 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 8212 * When multiple-message MSI is enabled, hardware will still set IS register 8213 * as single message case. And this IS register may be used by software when 8214 * fewer than the requested number of messages is granted in order to determine 8215 * which port had the interrupt. 8216 * 8217 * Note: The current ahci driver only supports the first two interrupt modes: 8218 * pin based interrupts and single MSI message interrupts, and the reason 8219 * is indicated in below code. 8220 */ 8221 static int 8222 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 8223 { 8224 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 8225 int count, avail, actual; 8226 int i, rc; 8227 8228 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 8229 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 8230 8231 /* get number of interrupts. */ 8232 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 8233 if ((rc != DDI_SUCCESS) || (count == 0)) { 8234 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8235 "ddi_intr_get_nintrs() failed, " 8236 "rc %d count %d\n", rc, count); 8237 return (DDI_FAILURE); 8238 } 8239 8240 /* get number of available interrupts. */ 8241 rc = ddi_intr_get_navail(dip, intr_type, &avail); 8242 if ((rc != DDI_SUCCESS) || (avail == 0)) { 8243 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8244 "ddi_intr_get_navail() failed, " 8245 "rc %d avail %d\n", rc, avail); 8246 return (DDI_FAILURE); 8247 } 8248 8249 #if AHCI_DEBUG 8250 if (avail < count) { 8251 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8252 "ddi_intr_get_nintrs returned %d, navail() returned %d", 8253 count, avail); 8254 } 8255 #endif 8256 8257 /* 8258 * Note: So far Solaris restricts the maximum number of messages for 8259 * x86 to 2, that is avail is 2, so here we set the count with 1 to 8260 * force the driver to use single MSI message interrupt. In future if 8261 * Solaris remove the restriction, then we need to delete the below 8262 * code and try to use multiple interrupt routine to gain better 8263 * performance. 8264 */ 8265 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 8266 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8267 "force to use one interrupt routine though the " 8268 "HBA supports %d interrupt", count); 8269 count = 1; 8270 } 8271 8272 /* Allocate an array of interrupt handles. */ 8273 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 8274 ahci_ctlp->ahcictl_intr_htable = 8275 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 8276 8277 /* call ddi_intr_alloc(). */ 8278 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 8279 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 8280 8281 if ((rc != DDI_SUCCESS) || (actual == 0)) { 8282 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8283 "ddi_intr_alloc() failed, rc %d count %d actual %d " 8284 "avail %d\n", rc, count, actual, avail); 8285 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8286 ahci_ctlp->ahcictl_intr_size); 8287 return (DDI_FAILURE); 8288 } 8289 8290 /* use interrupt count returned */ 8291 #if AHCI_DEBUG 8292 if (actual < count) { 8293 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8294 "Requested: %d, Received: %d", count, actual); 8295 } 8296 #endif 8297 8298 ahci_ctlp->ahcictl_intr_cnt = actual; 8299 8300 /* 8301 * Get priority for first, assume remaining are all the same. 8302 */ 8303 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 8304 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 8305 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8306 "ddi_intr_get_pri() failed", NULL); 8307 8308 /* Free already allocated intr. */ 8309 for (i = 0; i < actual; i++) { 8310 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 8311 } 8312 8313 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8314 ahci_ctlp->ahcictl_intr_size); 8315 return (DDI_FAILURE); 8316 } 8317 8318 /* Test for high level interrupt. */ 8319 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 8320 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8321 "ahci_add_intrs: Hi level intr not supported", NULL); 8322 8323 /* Free already allocated intr. */ 8324 for (i = 0; i < actual; i++) { 8325 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 8326 } 8327 8328 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8329 sizeof (ddi_intr_handle_t)); 8330 8331 return (DDI_FAILURE); 8332 } 8333 8334 /* Call ddi_intr_add_handler(). */ 8335 for (i = 0; i < actual; i++) { 8336 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 8337 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 8338 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8339 "ddi_intr_add_handler() failed", NULL); 8340 8341 /* Free already allocated intr. */ 8342 for (i = 0; i < actual; i++) { 8343 (void) ddi_intr_free( 8344 ahci_ctlp->ahcictl_intr_htable[i]); 8345 } 8346 8347 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8348 ahci_ctlp->ahcictl_intr_size); 8349 return (DDI_FAILURE); 8350 } 8351 } 8352 8353 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 8354 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 8355 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8356 "ddi_intr_get_cap() failed", NULL); 8357 8358 /* Free already allocated intr. */ 8359 for (i = 0; i < actual; i++) { 8360 (void) ddi_intr_free( 8361 ahci_ctlp->ahcictl_intr_htable[i]); 8362 } 8363 8364 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8365 ahci_ctlp->ahcictl_intr_size); 8366 return (DDI_FAILURE); 8367 } 8368 8369 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 8370 /* Call ddi_intr_block_enable() for MSI. */ 8371 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 8372 ahci_ctlp->ahcictl_intr_cnt); 8373 } else { 8374 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 8375 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 8376 (void) ddi_intr_enable( 8377 ahci_ctlp->ahcictl_intr_htable[i]); 8378 } 8379 } 8380 8381 return (DDI_SUCCESS); 8382 } 8383 8384 /* 8385 * Removes the registered interrupts irrespective of whether they 8386 * were legacy or MSI. 8387 * 8388 * NOTE: The controller interrupts must be disabled before calling 8389 * this routine. 8390 */ 8391 static void 8392 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 8393 { 8394 int x; 8395 8396 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 8397 8398 /* Disable all interrupts. */ 8399 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 8400 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 8401 /* Call ddi_intr_block_disable(). */ 8402 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 8403 ahci_ctlp->ahcictl_intr_cnt); 8404 } else { 8405 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 8406 (void) ddi_intr_disable( 8407 ahci_ctlp->ahcictl_intr_htable[x]); 8408 } 8409 } 8410 8411 /* Call ddi_intr_remove_handler(). */ 8412 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 8413 (void) ddi_intr_remove_handler( 8414 ahci_ctlp->ahcictl_intr_htable[x]); 8415 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 8416 } 8417 8418 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 8419 } 8420 8421 /* 8422 * This routine tries to put port into P:NotRunning state by clearing 8423 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 8424 * and PxCMD.CR to '0'. 8425 */ 8426 static int 8427 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 8428 ahci_port_t *ahci_portp, uint8_t port) 8429 { 8430 uint32_t port_cmd_status; 8431 int loop_count; 8432 8433 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE || 8434 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8435 8436 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8437 "ahci_put_port_into_notrunning_state enter: port %d", port); 8438 8439 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8440 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8441 8442 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 8443 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8444 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 8445 8446 /* Wait until PxCMD.CR is cleared */ 8447 loop_count = 0; 8448 do { 8449 port_cmd_status = 8450 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8451 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8452 8453 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 8454 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 8455 "clearing port %d CMD.CR timeout, " 8456 "port_cmd_status = 0x%x", port, 8457 port_cmd_status); 8458 /* 8459 * We are effectively timing out after 0.5 sec. 8460 * This value is specified in AHCI spec. 8461 */ 8462 break; 8463 } 8464 8465 /* Wait for 10 millisec */ 8466 drv_usecwait(AHCI_10MS_USECS); 8467 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 8468 8469 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 8470 8471 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 8472 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 8473 "ahci_put_port_into_notrunning_state: failed to clear " 8474 "PxCMD.CR to '0' after loop count: %d, and " 8475 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 8476 return (AHCI_FAILURE); 8477 } else { 8478 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 8479 "ahci_put_port_into_notrunning_state: succeeded to clear " 8480 "PxCMD.CR to '0' after loop count: %d, and " 8481 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 8482 return (AHCI_SUCCESS); 8483 } 8484 } 8485 8486 /* 8487 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 8488 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 8489 * stable state, then set PxCMD.ST to '1' to start the port directly. 8490 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 8491 * COMRESET to the device to put it in an idle state. 8492 * 8493 * The fifth argument returns whether the port reset is involved during 8494 * the process. 8495 * 8496 * The routine will be called under following scenarios: 8497 * + To reset the HBA 8498 * + To abort the packet(s) 8499 * + To reset the port 8500 * + To activate the port 8501 * + Fatal error recovery 8502 * + To abort the timeout packet(s) 8503 * 8504 * NOTES!!! During this procedure, PxSERR register will be cleared, and 8505 * according to the spec, the clearance of three bits will also clear 8506 * three interrupt status bits. 8507 * 1. PxSERR.DIAG.F will clear PxIS.UFS 8508 * 2. PxSERR.DIAG.X will clear PxIS.PCS 8509 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 8510 * 8511 * Among these three interrupt events, the driver needs to take care of 8512 * PxIS.PRCS, which is the hot plug event. When the driver found out 8513 * a device was unplugged, it will call the interrupt handler. 8514 */ 8515 static int 8516 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 8517 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 8518 { 8519 uint32_t port_sstatus; 8520 uint32_t task_file_status; 8521 sata_device_t sdevice; 8522 int rval; 8523 ahci_addr_t addr_port; 8524 ahci_pmult_info_t *pminfo = NULL; 8525 int dev_exists_begin = 0; 8526 int dev_exists_end = 0; 8527 uint32_t previous_dev_type = ahci_portp->ahciport_device_type; 8528 int npmport = 0; 8529 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8530 8531 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 8532 8533 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8534 "ahci_restart_port_wait_till_ready: port %d enter", port); 8535 8536 AHCI_ADDR_SET_PORT(&addr_port, port); 8537 8538 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 8539 dev_exists_begin = 1; 8540 8541 /* First clear PxCMD.ST */ 8542 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 8543 port); 8544 if (rval != AHCI_SUCCESS) 8545 /* 8546 * If PxCMD.CR does not clear within a reasonable time, it 8547 * may assume the interface is in a hung condition and may 8548 * continue with issuing the port reset. 8549 */ 8550 goto reset; 8551 8552 /* Then clear PxSERR */ 8553 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8554 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 8555 AHCI_SERROR_CLEAR_ALL); 8556 8557 /* Then get PxTFD */ 8558 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8559 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 8560 8561 /* 8562 * Check whether the device is in a stable status, if yes, 8563 * then start the port directly. However for ahci_tran_reset_dport, 8564 * we may have to perform a port reset. 8565 */ 8566 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 8567 !(flag & AHCI_PORT_RESET)) 8568 goto out; 8569 8570 reset: 8571 /* 8572 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 8573 * a COMRESET to the device 8574 */ 8575 ahci_disable_port_intrs(ahci_ctlp, port); 8576 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port); 8577 ahci_enable_port_intrs(ahci_ctlp, port); 8578 8579 #ifdef AHCI_DEBUG 8580 if (rval != AHCI_SUCCESS) 8581 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8582 "ahci_restart_port_wait_till_ready: port %d failed", 8583 port); 8584 #endif 8585 8586 if (reset_flag != NULL) 8587 *reset_flag = 1; 8588 8589 /* Indicate to the framework that a reset has happened. */ 8590 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 8591 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) && 8592 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 8593 /* Set the reset in progress flag */ 8594 ahci_portp->ahciport_reset_in_progress = 1; 8595 8596 bzero((void *)&sdevice, sizeof (sata_device_t)); 8597 sdevice.satadev_addr.cport = 8598 ahci_ctlp->ahcictl_port_to_cport[port]; 8599 sdevice.satadev_addr.pmport = 0; 8600 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8601 8602 sdevice.satadev_state = SATA_DSTATE_RESET | 8603 SATA_DSTATE_PWR_ACTIVE; 8604 if (ahci_ctlp->ahcictl_sata_hba_tran) { 8605 mutex_exit(&ahci_portp->ahciport_mutex); 8606 sata_hba_event_notify( 8607 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8608 &sdevice, 8609 SATA_EVNT_DEVICE_RESET); 8610 mutex_enter(&ahci_portp->ahciport_mutex); 8611 } 8612 8613 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8614 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8615 } else { 8616 ahci_portp->ahciport_reset_in_progress = 0; 8617 } 8618 8619 out: 8620 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8621 8622 /* SStatus tells the presence of device. */ 8623 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8624 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 8625 8626 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 8627 dev_exists_end = 1; 8628 } 8629 8630 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */ 8631 return (rval); 8632 8633 /* Check whether a hot plug event happened */ 8634 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */ 8635 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8636 "ahci_restart_port_wait_till_ready: port %d " 8637 "device is removed", port); 8638 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 8639 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8640 "ahci_restart_port_wait_till_ready: port %d " 8641 "AHCI_PORT_FLAG_NODEV flag is set", port); 8642 mutex_exit(&ahci_portp->ahciport_mutex); 8643 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 8644 mutex_enter(&ahci_portp->ahciport_mutex); 8645 8646 return (rval); 8647 } 8648 8649 8650 /* 0/1 -> 1 : device may change */ 8651 /* 8652 * May be called by ahci_fatal_error_recovery_handler, so 8653 * don't issue software if the previous device is ATAPI. 8654 */ 8655 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 8656 return (rval); 8657 8658 /* 8659 * The COMRESET will make port multiplier enter legacy mode. 8660 * Issue a software reset to make it work again. 8661 */ 8662 ahci_disable_port_intrs(ahci_ctlp, port); 8663 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port); 8664 ahci_enable_port_intrs(ahci_ctlp, port); 8665 8666 /* 8667 * Following codes are specific for the port multiplier 8668 */ 8669 if (previous_dev_type != SATA_DTYPE_PMULT && 8670 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8671 /* in case previous_dev_type is corrupt */ 8672 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8673 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8674 return (rval); 8675 } 8676 8677 /* Device change: PMult -> Non-PMult */ 8678 if (previous_dev_type == SATA_DTYPE_PMULT && 8679 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8680 /* 8681 * This might happen because 8682 * 1. Software reset failed. Port multiplier is not correctly 8683 * enumerated. 8684 * 2. Another non-port-multiplier device is attached. Perhaps 8685 * the port multiplier was replaced by another device by 8686 * whatever reason, but AHCI driver missed hot-plug event. 8687 * 8688 * Now that the port has been initialized, we just need to 8689 * update the port structure according new device, then report 8690 * and wait SATA framework to probe new device. 8691 */ 8692 8693 /* Force to release pmult resource */ 8694 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8695 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8696 8697 bzero((void *)&sdevice, sizeof (sata_device_t)); 8698 sdevice.satadev_addr.cport = 8699 ahci_ctlp->ahcictl_port_to_cport[port]; 8700 sdevice.satadev_addr.pmport = 0; 8701 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8702 8703 sdevice.satadev_state = SATA_DSTATE_RESET | 8704 SATA_DSTATE_PWR_ACTIVE; 8705 8706 mutex_exit(&ahci_portp->ahciport_mutex); 8707 sata_hba_event_notify( 8708 ahci_ctlp->ahcictl_dip, 8709 &sdevice, 8710 SATA_EVNT_DEVICE_RESET); 8711 mutex_enter(&ahci_portp->ahciport_mutex); 8712 8713 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8714 "Port multiplier is [Gone] at port %d ", port); 8715 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8716 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8717 8718 return (AHCI_SUCCESS); 8719 } 8720 8721 /* Device change: Non-PMult -> PMult */ 8722 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8723 8724 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */ 8725 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 8726 8727 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8728 } 8729 pminfo = ahci_portp->ahciport_pmult_info; 8730 ASSERT(pminfo != NULL); 8731 8732 /* Device (may) change: PMult -> PMult */ 8733 /* 8734 * First initialize port multiplier. Set state to READY and wait for 8735 * probe entry point to initialize it 8736 */ 8737 ahci_portp->ahciport_port_state = SATA_STATE_READY; 8738 8739 /* 8740 * It's a little complicated while target is a port multiplier. we 8741 * need to COMRESET all pmports behind that PMult otherwise those 8742 * sub-links between the PMult and the sub-devices will be in an 8743 * inactive state (indicated by PSCR0/PxSSTS) and the following access 8744 * to those sub-devices will be rejected by Link-Fatal-Error. 8745 */ 8746 /* 8747 * The PxSNTF will be set soon after the pmult is plugged. While the 8748 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we 8749 * simply mark every sub-port as 'unknown', then ahci_probe_pmport 8750 * will initialized it. 8751 */ 8752 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++) 8753 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN; 8754 8755 /* Report reset event. */ 8756 ahci_portp->ahciport_reset_in_progress = 1; 8757 8758 bzero((void *)&sdevice, sizeof (sata_device_t)); 8759 sdevice.satadev_addr.cport = cport; 8760 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 8761 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 8762 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE; 8763 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice, 8764 SATA_EVNT_DEVICE_RESET); 8765 8766 return (rval); 8767 } 8768 8769 /* 8770 * This routine may be called under four scenarios: 8771 * a) do the recovery from fatal error 8772 * b) or we need to timeout some commands 8773 * c) or we need to abort some commands 8774 * d) or we need reset device/port/controller 8775 * 8776 * In all these scenarios, we need to send any pending unfinished 8777 * commands up to sata framework. 8778 */ 8779 static void 8780 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 8781 ahci_port_t *ahci_portp, 8782 uint32_t slot_status, 8783 uint32_t failed_tags, 8784 uint32_t timeout_tags, 8785 uint32_t aborted_tags, 8786 uint32_t reset_tags) 8787 { 8788 uint32_t finished_tags = 0; 8789 uint32_t unfinished_tags = 0; 8790 int tmp_slot; 8791 sata_pkt_t *satapkt; 8792 int ncq_cmd_in_progress = 0; 8793 int err_retri_cmd_in_progress = 0; 8794 int rdwr_pmult_cmd_in_progress = 0; 8795 8796 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 8797 8798 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8799 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 8800 ahci_portp->ahciport_port_num, slot_status); 8801 8802 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8803 "ahci_mop_commands: failed_tags: 0x%x, " 8804 "timeout_tags: 0x%x aborted_tags: 0x%x, " 8805 "reset_tags: 0x%x", failed_tags, 8806 timeout_tags, aborted_tags, reset_tags); 8807 8808 #ifdef AHCI_DEBUG 8809 if (ahci_debug_flags & AHCIDBG_ERRS) { 8810 int i; 8811 char msg_buf[200] = {0, }; 8812 for (i = 0x1f; i >= 0; i--) { 8813 if (ahci_portp->ahciport_slot_pkts[i] != NULL) 8814 msg_buf[i] = 'X'; 8815 else 8816 msg_buf[i] = '.'; 8817 } 8818 msg_buf[0x20] = '\0'; 8819 cmn_err(CE_NOTE, "port[%d] slots: %s", 8820 ahci_portp->ahciport_port_num, msg_buf); 8821 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ", 8822 (void *)ahci_portp->ahciport_err_retri_pkt, 8823 (void *)ahci_portp->ahciport_rdwr_pmult_pkt); 8824 } 8825 #endif 8826 8827 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8828 finished_tags = ahci_portp->ahciport_pending_tags & 8829 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 8830 8831 unfinished_tags = slot_status & 8832 AHCI_SLOT_MASK(ahci_ctlp) & 8833 ~failed_tags & 8834 ~aborted_tags & 8835 ~reset_tags & 8836 ~timeout_tags; 8837 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8838 ncq_cmd_in_progress = 1; 8839 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 8840 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 8841 8842 unfinished_tags = slot_status & 8843 AHCI_NCQ_SLOT_MASK(ahci_portp) & 8844 ~failed_tags & 8845 ~aborted_tags & 8846 ~reset_tags & 8847 ~timeout_tags; 8848 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8849 8850 /* 8851 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 8852 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 8853 * complete successfully due to one of the following three 8854 * conditions: 8855 * 8856 * 1. Fatal error - failed_tags includes its slot 8857 * 2. Timed out - timeout_tags includes its slot 8858 * 3. Aborted when hot unplug - aborted_tags includes its 8859 * slot 8860 * 8861 * Please note that the command is always sent down in Slot 0 8862 */ 8863 err_retri_cmd_in_progress = 1; 8864 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 8865 "ahci_mop_commands is called for port %d while " 8866 "REQUEST SENSE or READ LOG EXT for error retrieval " 8867 "is being executed slot_status = 0x%x", 8868 ahci_portp->ahciport_port_num, slot_status); 8869 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 8870 ASSERT(slot_status == 0x1); 8871 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 8872 rdwr_pmult_cmd_in_progress = 1; 8873 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 8874 "ahci_mop_commands is called for port %d while " 8875 "READ/WRITE PORTMULT command is being executed", 8876 ahci_portp->ahciport_port_num); 8877 8878 ASSERT(slot_status == 0x1); 8879 } 8880 8881 #ifdef AHCI_DEBUG 8882 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8883 "ahci_mop_commands: finished_tags: 0x%x, " 8884 "unfinished_tags 0x%x", finished_tags, unfinished_tags); 8885 #endif 8886 8887 /* Send up finished packets with SATA_PKT_COMPLETED */ 8888 while (finished_tags) { 8889 tmp_slot = ddi_ffs(finished_tags) - 1; 8890 if (tmp_slot == -1) { 8891 break; 8892 } 8893 8894 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8895 ASSERT(satapkt != NULL); 8896 8897 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 8898 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 8899 (void *)satapkt); 8900 8901 /* 8902 * Cannot fetch the return register content since the port 8903 * was restarted, so the corresponding tag will be set to 8904 * aborted tags. 8905 */ 8906 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 8907 CLEAR_BIT(finished_tags, tmp_slot); 8908 aborted_tags |= tmp_slot; 8909 continue; 8910 } 8911 8912 if (ncq_cmd_in_progress) 8913 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8914 tmp_slot); 8915 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8916 CLEAR_BIT(finished_tags, tmp_slot); 8917 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8918 8919 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 8920 } 8921 8922 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 8923 while (failed_tags) { 8924 if (err_retri_cmd_in_progress) { 8925 satapkt = ahci_portp->ahciport_err_retri_pkt; 8926 ASSERT(satapkt != NULL); 8927 ASSERT(failed_tags == 0x1); 8928 8929 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8930 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8931 (void *)satapkt); 8932 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8933 break; 8934 } 8935 if (rdwr_pmult_cmd_in_progress) { 8936 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8937 ASSERT(satapkt != NULL); 8938 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8939 "ahci_mop_commands: sending up " 8940 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR", 8941 (void *)satapkt); 8942 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8943 break; 8944 } 8945 8946 tmp_slot = ddi_ffs(failed_tags) - 1; 8947 if (tmp_slot == -1) { 8948 break; 8949 } 8950 8951 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8952 ASSERT(satapkt != NULL); 8953 8954 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8955 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8956 (void *)satapkt); 8957 8958 if (ncq_cmd_in_progress) 8959 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8960 tmp_slot); 8961 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8962 CLEAR_BIT(failed_tags, tmp_slot); 8963 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8964 8965 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8966 } 8967 8968 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 8969 while (timeout_tags) { 8970 if (err_retri_cmd_in_progress) { 8971 satapkt = ahci_portp->ahciport_err_retri_pkt; 8972 ASSERT(satapkt != NULL); 8973 ASSERT(timeout_tags == 0x1); 8974 8975 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8976 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8977 (void *)satapkt); 8978 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8979 break; 8980 } 8981 if (rdwr_pmult_cmd_in_progress) { 8982 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8983 ASSERT(satapkt != NULL); 8984 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8985 "ahci_mop_commands: sending up " 8986 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT", 8987 (void *)satapkt); 8988 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8989 break; 8990 } 8991 8992 tmp_slot = ddi_ffs(timeout_tags) - 1; 8993 if (tmp_slot == -1) { 8994 break; 8995 } 8996 8997 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8998 ASSERT(satapkt != NULL); 8999 9000 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9001 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 9002 (void *)satapkt); 9003 9004 if (ncq_cmd_in_progress) 9005 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9006 tmp_slot); 9007 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9008 CLEAR_BIT(timeout_tags, tmp_slot); 9009 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9010 9011 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 9012 } 9013 9014 /* Send up aborted packets with SATA_PKT_ABORTED */ 9015 while (aborted_tags) { 9016 if (err_retri_cmd_in_progress) { 9017 satapkt = ahci_portp->ahciport_err_retri_pkt; 9018 ASSERT(satapkt != NULL); 9019 ASSERT(aborted_tags == 0x1); 9020 9021 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9022 "sending up pkt 0x%p with SATA_PKT_ABORTED", 9023 (void *)satapkt); 9024 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9025 break; 9026 } 9027 if (rdwr_pmult_cmd_in_progress) { 9028 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9029 ASSERT(satapkt != NULL); 9030 ASSERT(aborted_tags == 0x1); 9031 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9032 "ahci_mop_commands: sending up " 9033 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED", 9034 (void *)satapkt); 9035 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9036 break; 9037 } 9038 9039 tmp_slot = ddi_ffs(aborted_tags) - 1; 9040 if (tmp_slot == -1) { 9041 break; 9042 } 9043 9044 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9045 ASSERT(satapkt != NULL); 9046 9047 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9048 "sending up pkt 0x%p with SATA_PKT_ABORTED", 9049 (void *)satapkt); 9050 9051 if (ncq_cmd_in_progress) 9052 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9053 tmp_slot); 9054 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9055 CLEAR_BIT(aborted_tags, tmp_slot); 9056 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9057 9058 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9059 } 9060 9061 /* Send up reset packets with SATA_PKT_RESET. */ 9062 while (reset_tags) { 9063 if (rdwr_pmult_cmd_in_progress) { 9064 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9065 ASSERT(satapkt != NULL); 9066 ASSERT(aborted_tags == 0x1); 9067 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9068 "ahci_mop_commands: sending up " 9069 "rdwr pmult pkt 0x%p with SATA_PKT_RESET", 9070 (void *)satapkt); 9071 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9072 break; 9073 } 9074 9075 tmp_slot = ddi_ffs(reset_tags) - 1; 9076 if (tmp_slot == -1) { 9077 break; 9078 } 9079 9080 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9081 ASSERT(satapkt != NULL); 9082 9083 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9084 "sending up pkt 0x%p with SATA_PKT_RESET", 9085 (void *)satapkt); 9086 9087 if (ncq_cmd_in_progress) 9088 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9089 tmp_slot); 9090 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9091 CLEAR_BIT(reset_tags, tmp_slot); 9092 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9093 9094 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9095 } 9096 9097 /* Send up unfinished packets with SATA_PKT_RESET */ 9098 while (unfinished_tags) { 9099 tmp_slot = ddi_ffs(unfinished_tags) - 1; 9100 if (tmp_slot == -1) { 9101 break; 9102 } 9103 9104 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9105 ASSERT(satapkt != NULL); 9106 9107 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9108 "sending up pkt 0x%p with SATA_PKT_RESET", 9109 (void *)satapkt); 9110 9111 if (ncq_cmd_in_progress) 9112 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9113 tmp_slot); 9114 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9115 CLEAR_BIT(unfinished_tags, tmp_slot); 9116 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9117 9118 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9119 } 9120 9121 ahci_portp->ahciport_mop_in_progress--; 9122 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 9123 9124 if (ahci_portp->ahciport_mop_in_progress == 0) 9125 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 9126 9127 ahci_flush_doneq(ahci_portp); 9128 } 9129 9130 /* 9131 * This routine is going to first request a READ LOG EXT sata pkt from sata 9132 * module, and then deliver it to the HBA to get the ncq failure context. 9133 * The return value is the exactly failed tags. 9134 */ 9135 static uint32_t 9136 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9137 uint8_t port) 9138 { 9139 sata_device_t sdevice; 9140 sata_pkt_t *rdlog_spkt, *spkt; 9141 ddi_dma_handle_t buf_dma_handle; 9142 ahci_addr_t addr; 9143 int loop_count; 9144 int rval; 9145 int failed_slot; 9146 uint32_t failed_tags = 0; 9147 struct sata_ncq_error_recovery_page *ncq_err_page; 9148 9149 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 9150 "ahci_get_rdlogext_data enter: port %d", port); 9151 9152 /* Prepare the sdevice data */ 9153 bzero((void *)&sdevice, sizeof (sata_device_t)); 9154 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 9155 9156 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 9157 sdevice.satadev_addr.pmport = 0; 9158 9159 /* Translate sata_device.satadev_addr -> ahci_addr */ 9160 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 9161 9162 /* 9163 * Call the sata hba interface to get a rdlog spkt 9164 */ 9165 loop_count = 0; 9166 loop: 9167 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 9168 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 9169 if (rdlog_spkt == NULL) { 9170 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 9171 /* Sleep for a while */ 9172 drv_usecwait(AHCI_10MS_USECS); 9173 goto loop; 9174 } 9175 /* Timed out after 1s */ 9176 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9177 "failed to get rdlog spkt for port %d", port); 9178 return (failed_tags); 9179 } 9180 9181 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 9182 9183 /* 9184 * This flag is used to handle the specific error recovery when the 9185 * READ LOG EXT command gets a failure (fatal error or time-out). 9186 */ 9187 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 9188 9189 /* 9190 * This start is not supposed to fail because after port is restarted, 9191 * the whole command list is empty. 9192 */ 9193 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 9194 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt); 9195 ahci_portp->ahciport_err_retri_pkt = NULL; 9196 9197 /* Remove the flag after READ LOG EXT command is completed */ 9198 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT; 9199 9200 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 9201 /* Update the request log data */ 9202 buf_dma_handle = *(ddi_dma_handle_t *) 9203 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 9204 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 9205 DDI_DMA_SYNC_FORKERNEL); 9206 if (rval == DDI_SUCCESS) { 9207 ncq_err_page = 9208 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 9209 satapkt_cmd.satacmd_bp->b_un.b_addr; 9210 9211 /* Get the failed tag */ 9212 failed_slot = ncq_err_page->ncq_tag; 9213 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9214 "ahci_get_rdlogext_data: port %d " 9215 "failed slot %d", port, failed_slot); 9216 if (failed_slot & NQ) { 9217 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9218 "the failed slot is not a valid tag", NULL); 9219 goto out; 9220 } 9221 9222 failed_slot &= NCQ_TAG_MASK; 9223 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 9224 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9225 "ahci_get_rdlogext_data: failed spkt 0x%p", 9226 (void *)spkt); 9227 if (spkt == NULL) { 9228 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9229 "the failed slot spkt is NULL", NULL); 9230 goto out; 9231 } 9232 9233 failed_tags = 0x1 << failed_slot; 9234 9235 /* Fill out the error context */ 9236 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 9237 ncq_err_page); 9238 ahci_update_sata_registers(ahci_ctlp, port, 9239 &spkt->satapkt_device); 9240 } 9241 } 9242 out: 9243 sata_free_error_retrieval_pkt(rdlog_spkt); 9244 9245 return (failed_tags); 9246 } 9247 9248 /* 9249 * This routine is going to first request a REQUEST SENSE sata pkt from sata 9250 * module, and then deliver it to the HBA to get the sense data and copy 9251 * the sense data back to the orignal failed sata pkt, and free the REQUEST 9252 * SENSE sata pkt later. 9253 */ 9254 static void 9255 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9256 uint8_t port, sata_pkt_t *spkt) 9257 { 9258 sata_device_t sdevice; 9259 sata_pkt_t *rs_spkt; 9260 sata_cmd_t *sata_cmd; 9261 ddi_dma_handle_t buf_dma_handle; 9262 ahci_addr_t addr; 9263 int loop_count; 9264 #if AHCI_DEBUG 9265 struct scsi_extended_sense *rqsense; 9266 #endif 9267 9268 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9269 "ahci_get_rqsense_data enter: port %d", port); 9270 9271 /* Prepare the sdevice data */ 9272 bzero((void *)&sdevice, sizeof (sata_device_t)); 9273 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 9274 9275 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 9276 sdevice.satadev_addr.pmport = 0; 9277 9278 /* Translate sata_device.satadev_addr -> ahci_addr */ 9279 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 9280 9281 sata_cmd = &spkt->satapkt_cmd; 9282 9283 /* 9284 * Call the sata hba interface to get a rs spkt 9285 */ 9286 loop_count = 0; 9287 loop: 9288 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 9289 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 9290 if (rs_spkt == NULL) { 9291 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 9292 /* Sleep for a while */ 9293 drv_usecwait(AHCI_10MS_USECS); 9294 goto loop; 9295 9296 } 9297 /* Timed out after 1s */ 9298 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9299 "failed to get rs spkt for port %d", port); 9300 return; 9301 } 9302 9303 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 9304 9305 /* 9306 * This flag is used to handle the specific error recovery when the 9307 * REQUEST SENSE command gets a faiure (fatal error or time-out). 9308 */ 9309 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 9310 9311 /* 9312 * This start is not supposed to fail because after port is restarted, 9313 * the whole command list is empty. 9314 */ 9315 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 9316 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt); 9317 ahci_portp->ahciport_err_retri_pkt = NULL; 9318 9319 /* Remove the flag after REQUEST SENSE command is completed */ 9320 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE; 9321 9322 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 9323 /* Update the request sense data */ 9324 buf_dma_handle = *(ddi_dma_handle_t *) 9325 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 9326 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 9327 DDI_DMA_SYNC_FORKERNEL); 9328 /* Copy the request sense data */ 9329 bcopy(rs_spkt-> 9330 satapkt_cmd.satacmd_bp->b_un.b_addr, 9331 &sata_cmd->satacmd_rqsense, 9332 SATA_ATAPI_MIN_RQSENSE_LEN); 9333 #if AHCI_DEBUG 9334 rqsense = (struct scsi_extended_sense *) 9335 sata_cmd->satacmd_rqsense; 9336 9337 /* Dump the sense data */ 9338 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 9339 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 9340 "Sense data for satapkt %p ATAPI cmd 0x%x", 9341 spkt, sata_cmd->satacmd_acdb[0]); 9342 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 9343 " es_code 0x%x es_class 0x%x " 9344 "es_key 0x%x es_add_code 0x%x " 9345 "es_qual_code 0x%x", 9346 rqsense->es_code, rqsense->es_class, 9347 rqsense->es_key, rqsense->es_add_code, 9348 rqsense->es_qual_code); 9349 #endif 9350 } 9351 9352 sata_free_error_retrieval_pkt(rs_spkt); 9353 } 9354 9355 /* 9356 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 9357 * the port must be restarted. When the HBA detects thus error, it may try 9358 * to abort a transfer. And if the transfer was aborted, the device is 9359 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 9360 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 9361 * that the device is in a stable status and transfers may be restarted without 9362 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 9363 * then the software will send the COMRESET to do the port reset. 9364 * 9365 * Software should perform the appropriate error recovery actions based on 9366 * whether non-queued commands were being issued or natived command queuing 9367 * commands were being issued. 9368 * 9369 * And software will complete the command that had the error with error mark 9370 * to higher level software. 9371 * 9372 * Fatal errors include the following: 9373 * PxIS.IFS - Interface Fatal Error Status 9374 * PxIS.HBDS - Host Bus Data Error Status 9375 * PxIS.HBFS - Host Bus Fatal Error Status 9376 * PxIS.TFES - Task File Error Status 9377 */ 9378 static void 9379 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9380 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status) 9381 { 9382 uint32_t port_cmd_status; 9383 uint32_t slot_status = 0; 9384 uint32_t failed_tags = 0; 9385 int failed_slot; 9386 int reset_flag = 0, flag = 0; 9387 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 9388 sata_cmd_t *sata_cmd = NULL; 9389 sata_pkt_t *spkt = NULL; 9390 #if AHCI_DEBUG 9391 ahci_cmd_header_t *cmd_header; 9392 #endif 9393 uint8_t port = addrp->aa_port; 9394 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9395 int rval; 9396 9397 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 9398 9399 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9400 "ahci_fatal_error_recovery_handler enter: port %d", port); 9401 9402 /* Port multiplier error */ 9403 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 9404 /* FBS code is neither completed nor tested. */ 9405 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp, 9406 port, intr_status); 9407 9408 /* Force a port reset */ 9409 flag = AHCI_PORT_RESET; 9410 } 9411 9412 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9413 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9414 9415 /* Read PxCI to see which commands are still outstanding */ 9416 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9417 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9418 9419 /* 9420 * Read PxCMD.CCS to determine the slot that the HBA 9421 * was processing when the error occurred. 9422 */ 9423 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9424 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9425 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9426 AHCI_CMD_STATUS_CCS_SHIFT; 9427 9428 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9429 spkt = ahci_portp->ahciport_err_retri_pkt; 9430 ASSERT(spkt != NULL); 9431 } else { 9432 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 9433 if (spkt == NULL) { 9434 /* May happen when interface errors occur? */ 9435 goto next; 9436 } 9437 } 9438 9439 #if AHCI_DEBUG 9440 /* 9441 * Debugging purpose... 9442 */ 9443 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 9444 cmd_header = 9445 &ahci_portp->ahciport_cmd_list[failed_slot]; 9446 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9447 "ahci_fatal_error_recovery_handler: port %d, " 9448 "PRD Byte Count = 0x%x, " 9449 "ahciport_prd_bytecounts = 0x%x", port, 9450 cmd_header->ahcich_prd_byte_count, 9451 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 9452 } 9453 #endif 9454 9455 sata_cmd = &spkt->satapkt_cmd; 9456 9457 /* Fill out the status and error registers for PxIS.TFES */ 9458 if (intr_status & AHCI_INTR_STATUS_TFES) { 9459 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 9460 ahcirf_d2h_register_fis); 9461 9462 /* Copy the error context back to the sata_cmd */ 9463 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 9464 } 9465 9466 /* The failed command must be one of the outstanding commands */ 9467 failed_tags = 0x1 << failed_slot; 9468 ASSERT(failed_tags & slot_status); 9469 9470 /* Update the sata registers, especially PxSERR register */ 9471 ahci_update_sata_registers(ahci_ctlp, port, 9472 &spkt->satapkt_device); 9473 9474 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9475 /* Read PxSACT to see which commands are still outstanding */ 9476 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9477 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9478 } 9479 next: 9480 9481 #if AHCI_DEBUG 9482 /* 9483 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9484 * set, it means a fatal error happened after REQUEST SENSE command 9485 * or READ LOG EXT command is delivered to the HBA during the error 9486 * recovery process. At this time, the only outstanding command is 9487 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9488 */ 9489 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9490 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9491 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 9492 "command or READ LOG EXT command for error data retrieval " 9493 "failed", port); 9494 ASSERT(slot_status == 0x1); 9495 ASSERT(failed_slot == 0); 9496 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 9497 SCMD_REQUEST_SENSE || 9498 spkt->satapkt_cmd.satacmd_cmd_reg == 9499 SATAC_READ_LOG_EXT); 9500 } 9501 #endif 9502 9503 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9504 ahci_portp->ahciport_mop_in_progress++; 9505 9506 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9507 port, flag, &reset_flag); 9508 9509 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) { 9510 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 9511 if (rval == AHCI_SUCCESS) 9512 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9513 "succeed", instance, port); 9514 else 9515 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9516 "failed", instance, port); 9517 } 9518 9519 /* 9520 * Won't retrieve error information: 9521 * 1. Port reset was involved to recover 9522 * 2. Device is gone 9523 * 3. IDENTIFY DEVICE command sent to ATAPI device 9524 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 9525 */ 9526 if (reset_flag || 9527 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 9528 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 9529 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9530 goto out; 9531 9532 /* 9533 * Deliver READ LOG EXT to gather information about the error when 9534 * a COMRESET has not been performed as part of the error recovery 9535 * during NCQ command processing. 9536 */ 9537 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9538 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 9539 ahci_portp, port); 9540 goto out; 9541 } 9542 9543 /* 9544 * Deliver REQUEST SENSE for ATAPI command to gather information about 9545 * the error when a COMRESET has not been performed as part of the 9546 * error recovery. 9547 */ 9548 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 9549 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 9550 out: 9551 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9552 "ahci_fatal_error_recovery_handler: port %d fatal error " 9553 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 9554 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 9555 port, slot_status, ahci_portp->ahciport_pending_tags, 9556 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 9557 9558 ahci_mop_commands(ahci_ctlp, 9559 ahci_portp, 9560 slot_status, 9561 failed_tags, /* failed tags */ 9562 0, /* timeout tags */ 9563 0, /* aborted tags */ 9564 0); /* reset tags */ 9565 } 9566 9567 /* 9568 * Used to recovery a PMULT pmport fatal error under FIS-based switching. 9569 * 1. device specific.PxFBS.SDE=1 9570 * 2. Non Device specific. 9571 * Nothing will be done when Command-based switching is employed. 9572 * 9573 * Currently code is neither completed nor tested. 9574 */ 9575 static void 9576 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9577 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 9578 { 9579 #ifndef __lock_lint 9580 _NOTE(ARGUNUSED(intr_status)) 9581 #endif 9582 uint32_t port_fbs_ctrl; 9583 int loop_count = 0; 9584 ahci_addr_t addr; 9585 9586 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 9587 9588 /* Nothing will be done under Command-based switching. */ 9589 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS)) 9590 return; 9591 9592 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9593 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9594 9595 if (!(port_fbs_ctrl & AHCI_FBS_EN)) 9596 /* FBS is not enabled. */ 9597 return; 9598 9599 /* Problem's getting complicated now. */ 9600 /* 9601 * If FIS-based switching is used, we need to check 9602 * the PxFBS to see the error type. 9603 */ 9604 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9605 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9606 9607 /* Refer to spec(v1.2) 9.3.6.1 */ 9608 if (port_fbs_ctrl & AHCI_FBS_SDE) { 9609 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9610 "A Device Sepcific Error: port %d", port); 9611 /* 9612 * Controller has paused commands for all other 9613 * sub-devices until PxFBS.DEC is set. 9614 */ 9615 ahci_reject_all_abort_pkts(ahci_ctlp, 9616 ahci_portp, 0); 9617 9618 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 9619 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port), 9620 port_fbs_ctrl | AHCI_FBS_DEC); 9621 9622 /* 9623 * Wait controller clear PxFBS.DEC, 9624 * then we can continue. 9625 */ 9626 loop_count = 0; 9627 do { 9628 port_fbs_ctrl = ddi_get32(ahci_ctlp-> 9629 ahcictl_ahci_acc_handle, (uint32_t *) 9630 AHCI_PORT_PxFBS(ahci_ctlp, port)); 9631 9632 if (loop_count++ > 1000) 9633 /* 9634 * Esclate the error. Follow 9635 * non-device specific error 9636 * procedure. 9637 */ 9638 return; 9639 9640 drv_usecwait(AHCI_100US_USECS); 9641 } while (port_fbs_ctrl & AHCI_FBS_DEC); 9642 9643 /* 9644 * Issue a software reset to ensure drive is in 9645 * a known state. 9646 */ 9647 (void) ahci_software_reset(ahci_ctlp, 9648 ahci_portp, &addr); 9649 9650 } else { 9651 9652 /* Process Non-Device Specific Error. */ 9653 /* This will be handled later on. */ 9654 cmn_err(CE_NOTE, "!FBS is not supported now."); 9655 } 9656 } 9657 /* 9658 * Handle events - fatal error recovery 9659 */ 9660 static void 9661 ahci_events_handler(void *args) 9662 { 9663 ahci_event_arg_t *ahci_event_arg; 9664 ahci_ctl_t *ahci_ctlp; 9665 ahci_port_t *ahci_portp; 9666 ahci_addr_t *addrp; 9667 uint32_t event; 9668 int instance; 9669 9670 ahci_event_arg = (ahci_event_arg_t *)args; 9671 9672 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 9673 ahci_portp = ahci_event_arg->ahciea_portp; 9674 addrp = ahci_event_arg->ahciea_addrp; 9675 event = ahci_event_arg->ahciea_event; 9676 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9677 9678 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9679 "ahci_events_handler enter: port %d intr_status = 0x%x", 9680 ahci_portp->ahciport_port_num, event); 9681 9682 mutex_enter(&ahci_portp->ahciport_mutex); 9683 9684 /* 9685 * ahci_intr_phyrdy_change() may have rendered it to 9686 * SATA_DTYPE_NONE. 9687 */ 9688 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9689 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 9690 "ahci_events_handler: port %d no device attached, " 9691 "and just return without doing anything", 9692 ahci_portp->ahciport_port_num); 9693 9694 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) { 9695 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 9696 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9697 "succeed", instance, ahci_portp->ahciport_port_num); 9698 } 9699 9700 goto out; 9701 } 9702 9703 if (event & (AHCI_INTR_STATUS_IFS | 9704 AHCI_INTR_STATUS_HBDS | 9705 AHCI_INTR_STATUS_HBFS | 9706 AHCI_INTR_STATUS_TFES)) 9707 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 9708 addrp, event); 9709 9710 out: 9711 mutex_exit(&ahci_portp->ahciport_mutex); 9712 } 9713 9714 /* 9715 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 9716 * detect there are some commands which are timed out. 9717 */ 9718 static void 9719 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9720 uint8_t port, uint32_t tmp_timeout_tags) 9721 { 9722 uint32_t slot_status = 0; 9723 uint32_t finished_tags = 0; 9724 uint32_t timeout_tags = 0; 9725 9726 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 9727 "ahci_timeout_pkts enter: port %d", port); 9728 9729 mutex_enter(&ahci_portp->ahciport_mutex); 9730 9731 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9732 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) || 9733 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9734 /* Read PxCI to see which commands are still outstanding */ 9735 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9736 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9737 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9738 /* Read PxSACT to see which commands are still outstanding */ 9739 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9740 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9741 } 9742 9743 #if AHCI_DEBUG 9744 /* 9745 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9746 * set, it means a fatal error happened after REQUEST SENSE command 9747 * or READ LOG EXT command is delivered to the HBA during the error 9748 * recovery process. At this time, the only outstanding command is 9749 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9750 */ 9751 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9752 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9753 "ahci_timeout_pkts called while REQUEST SENSE " 9754 "command or READ LOG EXT command for error recovery " 9755 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 9756 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 9757 tmp_timeout_tags, slot_status, 9758 ahci_portp->ahciport_pending_tags, 9759 ahci_portp->ahciport_pending_ncq_tags); 9760 ASSERT(slot_status == 0x1); 9761 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9762 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9763 "ahci_timeout_pkts called while executing R/W PMULT " 9764 "command timeout_tags = 0x%x, slot_status = 0x%x", 9765 tmp_timeout_tags, slot_status); 9766 ASSERT(slot_status == 0x1); 9767 } 9768 #endif 9769 9770 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9771 ahci_portp->ahciport_mop_in_progress++; 9772 9773 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9774 port, AHCI_PORT_RESET, NULL); 9775 9776 /* 9777 * Re-identify timeout tags because some previously checked commands 9778 * could already complete. 9779 */ 9780 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9781 finished_tags = ahci_portp->ahciport_pending_tags & 9782 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 9783 timeout_tags = tmp_timeout_tags & ~finished_tags; 9784 9785 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 9786 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9787 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 9788 "pending_tags = 0x%x ", 9789 port, finished_tags, timeout_tags, 9790 slot_status, ahci_portp->ahciport_pending_tags); 9791 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9792 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 9793 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 9794 timeout_tags = tmp_timeout_tags & ~finished_tags; 9795 9796 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 9797 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9798 "timeout_tags = 0x%x, port_sactive = 0x%x, " 9799 "pending_ncq_tags = 0x%x ", 9800 port, finished_tags, timeout_tags, 9801 slot_status, ahci_portp->ahciport_pending_ncq_tags); 9802 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9803 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9804 timeout_tags = tmp_timeout_tags; 9805 } 9806 9807 ahci_mop_commands(ahci_ctlp, 9808 ahci_portp, 9809 slot_status, 9810 0, /* failed tags */ 9811 timeout_tags, /* timeout tags */ 9812 0, /* aborted tags */ 9813 0); /* reset tags */ 9814 9815 mutex_exit(&ahci_portp->ahciport_mutex); 9816 } 9817 9818 /* 9819 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 9820 * for long time. 9821 */ 9822 static void 9823 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 9824 { 9825 ahci_port_t *ahci_portp; 9826 sata_pkt_t *spkt; 9827 uint32_t pending_tags; 9828 uint32_t timeout_tags; 9829 uint32_t port_cmd_status; 9830 uint32_t port_sactive; 9831 uint8_t port; 9832 int tmp_slot; 9833 int current_slot; 9834 uint32_t current_tags; 9835 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9836 9837 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9838 9839 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9840 "ahci_watchdog_handler entered", NULL); 9841 9842 current_slot = 0; 9843 current_tags = 0; 9844 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9845 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9846 continue; 9847 } 9848 9849 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9850 9851 mutex_enter(&ahci_portp->ahciport_mutex); 9852 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9853 mutex_exit(&ahci_portp->ahciport_mutex); 9854 continue; 9855 } 9856 9857 /* Skip the check for those ports in error recovery */ 9858 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 9859 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 9860 mutex_exit(&ahci_portp->ahciport_mutex); 9861 continue; 9862 } 9863 9864 pending_tags = 0; 9865 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9866 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9867 9868 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9869 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9870 current_slot = 0; 9871 pending_tags = 0x1; 9872 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9873 current_slot = 9874 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9875 AHCI_CMD_STATUS_CCS_SHIFT; 9876 pending_tags = ahci_portp->ahciport_pending_tags; 9877 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9878 port_sactive = ddi_get32( 9879 ahci_ctlp->ahcictl_ahci_acc_handle, 9880 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9881 current_tags = port_sactive & 9882 ~port_cmd_status & 9883 AHCI_NCQ_SLOT_MASK(ahci_portp); 9884 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 9885 } 9886 9887 timeout_tags = 0; 9888 while (pending_tags) { 9889 tmp_slot = ddi_ffs(pending_tags) - 1; 9890 if (tmp_slot == -1) { 9891 break; 9892 } 9893 9894 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9895 spkt = ahci_portp->ahciport_err_retri_pkt; 9896 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 9897 spkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9898 else 9899 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9900 9901 if ((spkt != NULL) && spkt->satapkt_time && 9902 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 9903 /* 9904 * If a packet has survived for more than it's 9905 * max life cycles, it is a candidate for time 9906 * out. 9907 */ 9908 ahci_portp->ahciport_slot_timeout[tmp_slot] -= 9909 ahci_watchdog_timeout; 9910 9911 if (ahci_portp->ahciport_slot_timeout[tmp_slot] 9912 > 0) 9913 goto next; 9914 9915 #if AHCI_DEBUG 9916 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9917 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9918 ahci_ctlp, "watchdog: the current " 9919 "tags is 0x%x", current_tags); 9920 } else { 9921 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9922 ahci_ctlp, "watchdog: the current " 9923 "slot is %d", current_slot); 9924 } 9925 #endif 9926 9927 /* 9928 * We need to check whether the HBA has 9929 * begun to execute the command, if not, 9930 * then re-set the timer of the command. 9931 */ 9932 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 9933 (tmp_slot != current_slot) || 9934 NCQ_CMD_IN_PROGRESS(ahci_portp) && 9935 ((0x1 << tmp_slot) & current_tags)) { 9936 ahci_portp->ahciport_slot_timeout \ 9937 [tmp_slot] = spkt->satapkt_time; 9938 } else { 9939 timeout_tags |= (0x1 << tmp_slot); 9940 cmn_err(CE_WARN, "!ahci%d: watchdog " 9941 "port %d satapkt 0x%p timed out\n", 9942 instance, port, (void *)spkt); 9943 } 9944 } 9945 next: 9946 CLEAR_BIT(pending_tags, tmp_slot); 9947 } 9948 9949 if (timeout_tags) { 9950 mutex_exit(&ahci_portp->ahciport_mutex); 9951 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9952 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 9953 port, timeout_tags); 9954 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9955 mutex_enter(&ahci_portp->ahciport_mutex); 9956 } 9957 9958 mutex_exit(&ahci_portp->ahciport_mutex); 9959 } 9960 9961 /* Re-install the watchdog timeout handler */ 9962 if (ahci_ctlp->ahcictl_timeout_id != 0) { 9963 ahci_ctlp->ahcictl_timeout_id = 9964 timeout((void (*)(void *))ahci_watchdog_handler, 9965 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 9966 } 9967 9968 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9969 } 9970 9971 /* 9972 * Fill the error context into sata_cmd for non-queued command error. 9973 */ 9974 static void 9975 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9976 { 9977 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 9978 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9979 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9980 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9981 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9982 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9983 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9984 9985 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 9986 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9987 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9988 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9989 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9990 } 9991 } 9992 9993 /* 9994 * Fill the ncq error page into sata_cmd for queued command error. 9995 */ 9996 static void 9997 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 9998 struct sata_ncq_error_recovery_page *ncq_err_page) 9999 { 10000 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 10001 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 10002 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 10003 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 10004 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 10005 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 10006 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 10007 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 10008 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 10009 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 10010 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 10011 } 10012 10013 /* 10014 * Put the respective register value to sata_cmd_t for satacmd_flags. 10015 */ 10016 static void 10017 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 10018 { 10019 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 10020 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 10021 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 10022 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 10023 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 10024 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 10025 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 10026 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 10027 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 10028 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 10029 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 10030 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 10031 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 10032 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 10033 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 10034 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 10035 if (scmd->satacmd_flags.sata_copy_out_device_reg) 10036 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 10037 if (scmd->satacmd_flags.sata_copy_out_error_reg) 10038 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 10039 } 10040 10041 static void 10042 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 10043 uint32_t intr_status) 10044 { 10045 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 10046 10047 if (intr_status & AHCI_INTR_STATUS_IFS) 10048 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 10049 "error", instance, port); 10050 10051 if (intr_status & AHCI_INTR_STATUS_HBDS) 10052 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 10053 instance, port); 10054 10055 if (intr_status & AHCI_INTR_STATUS_HBFS) 10056 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 10057 instance, port); 10058 10059 if (intr_status & AHCI_INTR_STATUS_TFES) 10060 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 10061 instance, port); 10062 10063 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 10064 "recovery", instance, port); 10065 } 10066 10067 static void 10068 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port, 10069 uint32_t slot_tags) 10070 { 10071 ahci_port_t *ahci_portp; 10072 int tmp_slot; 10073 sata_pkt_t *spkt; 10074 sata_cmd_t cmd; 10075 10076 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 10077 ASSERT(ahci_portp != NULL); 10078 10079 while (slot_tags) { 10080 tmp_slot = ddi_ffs(slot_tags) - 1; 10081 if (tmp_slot == -1) { 10082 break; 10083 } 10084 10085 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 10086 if (spkt != NULL) { 10087 cmd = spkt->satapkt_cmd; 10088 10089 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x " 10090 "features_reg = 0x%x sec_count_msb = 0x%x " 10091 "lba_low_msb = 0x%x lba_mid_msb = 0x%x " 10092 "lba_high_msb = 0x%x sec_count_lsb = 0x%x " 10093 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x " 10094 "lba_high_lsb = 0x%x device_reg = 0x%x " 10095 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt, 10096 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg, 10097 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb, 10098 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb, 10099 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb, 10100 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb, 10101 cmd.satacmd_device_reg, cmd.satacmd_addr_type, 10102 *((uint32_t *)&(cmd.satacmd_flags))); 10103 } 10104 10105 CLEAR_BIT(slot_tags, tmp_slot); 10106 } 10107 } 10108 10109 /* 10110 * Dump the serror message to the log. 10111 */ 10112 static void 10113 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 10114 uint32_t port_serror, int debug_only) 10115 { 10116 static char err_buf[512]; 10117 static char err_msg_header[16]; 10118 char *err_msg = err_buf; 10119 10120 *err_buf = '\0'; 10121 *err_msg_header = '\0'; 10122 10123 if (port_serror & SERROR_DATA_ERR_FIXED) { 10124 err_msg = strcat(err_msg, 10125 "\tRecovered Data Integrity Error (I)\n"); 10126 } 10127 10128 if (port_serror & SERROR_COMM_ERR_FIXED) { 10129 err_msg = strcat(err_msg, 10130 "\tRecovered Communication Error (M)\n"); 10131 } 10132 10133 if (port_serror & SERROR_DATA_ERR) { 10134 err_msg = strcat(err_msg, 10135 "\tTransient Data Integrity Error (T)\n"); 10136 } 10137 10138 if (port_serror & SERROR_PERSISTENT_ERR) { 10139 err_msg = strcat(err_msg, 10140 "\tPersistent Communication or Data Integrity Error (C)\n"); 10141 } 10142 10143 if (port_serror & SERROR_PROTOCOL_ERR) { 10144 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 10145 } 10146 10147 if (port_serror & SERROR_INT_ERR) { 10148 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 10149 } 10150 10151 if (port_serror & SERROR_PHY_RDY_CHG) { 10152 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 10153 } 10154 10155 if (port_serror & SERROR_PHY_INT_ERR) { 10156 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 10157 } 10158 10159 if (port_serror & SERROR_COMM_WAKE) { 10160 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 10161 } 10162 10163 if (port_serror & SERROR_10B_TO_8B_ERR) { 10164 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 10165 } 10166 10167 if (port_serror & SERROR_DISPARITY_ERR) { 10168 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 10169 } 10170 10171 if (port_serror & SERROR_CRC_ERR) { 10172 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 10173 } 10174 10175 if (port_serror & SERROR_HANDSHAKE_ERR) { 10176 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 10177 } 10178 10179 if (port_serror & SERROR_LINK_SEQ_ERR) { 10180 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 10181 } 10182 10183 if (port_serror & SERROR_TRANS_ERR) { 10184 err_msg = strcat(err_msg, 10185 "\tTransport state transition error (T)\n"); 10186 } 10187 10188 if (port_serror & SERROR_FIS_TYPE) { 10189 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 10190 } 10191 10192 if (port_serror & SERROR_EXCHANGED_ERR) { 10193 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 10194 } 10195 10196 if (*err_msg == '\0') 10197 return; 10198 10199 if (debug_only) { 10200 (void) sprintf(err_msg_header, "port %d", port); 10201 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 10202 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 10203 } else if (ahci_ctlp) { 10204 cmn_err(CE_WARN, "!ahci%d: %s %s", 10205 ddi_get_instance(ahci_ctlp->ahcictl_dip), 10206 err_msg_header, err_msg); 10207 10208 /* sata trace debug */ 10209 sata_trace_debug(ahci_ctlp->ahcictl_dip, 10210 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 10211 err_msg_header, err_msg); 10212 } else { 10213 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 10214 10215 /* sata trace debug */ 10216 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 10217 } 10218 } 10219 10220 /* 10221 * Translate the sata_address_t type into the ahci_addr_t type. 10222 * sata_device.satadev_addr structure is used as source. 10223 */ 10224 static void 10225 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd, 10226 ahci_addr_t *ahci_addrp) 10227 { 10228 sata_address_t *sata_addrp = &sd->satadev_addr; 10229 ahci_addrp->aa_port = 10230 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport]; 10231 ahci_addrp->aa_pmport = sata_addrp->pmport; 10232 10233 switch (sata_addrp->qual) { 10234 case SATA_ADDR_DCPORT: 10235 case SATA_ADDR_CPORT: 10236 ahci_addrp->aa_qual = AHCI_ADDR_PORT; 10237 break; 10238 case SATA_ADDR_PMULT: 10239 case SATA_ADDR_PMULT_SPEC: 10240 ahci_addrp->aa_qual = AHCI_ADDR_PMULT; 10241 break; 10242 case SATA_ADDR_DPMPORT: 10243 case SATA_ADDR_PMPORT: 10244 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT; 10245 break; 10246 case SATA_ADDR_NULL: 10247 default: 10248 /* something went wrong */ 10249 ahci_addrp->aa_qual = AHCI_ADDR_NULL; 10250 break; 10251 } 10252 } 10253 10254 /* 10255 * This routine is to calculate the total number of ports implemented 10256 * by the HBA. 10257 */ 10258 static int 10259 ahci_get_num_implemented_ports(uint32_t ports_implemented) 10260 { 10261 uint8_t i; 10262 int num = 0; 10263 10264 for (i = 0; i < AHCI_MAX_PORTS; i++) { 10265 if (((uint32_t)0x1 << i) & ports_implemented) 10266 num++; 10267 } 10268 10269 return (num); 10270 } 10271 10272 #if AHCI_DEBUG 10273 static void 10274 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 10275 { 10276 static char name[16]; 10277 va_list ap; 10278 10279 mutex_enter(&ahci_log_mutex); 10280 10281 va_start(ap, fmt); 10282 if (ahci_ctlp) { 10283 (void) sprintf(name, "ahci%d: ", 10284 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10285 } else { 10286 (void) sprintf(name, "ahci: "); 10287 } 10288 10289 (void) vsprintf(ahci_log_buf, fmt, ap); 10290 va_end(ap); 10291 10292 cmn_err(level, "%s%s", name, ahci_log_buf); 10293 10294 mutex_exit(&ahci_log_mutex); 10295 } 10296 #endif 10297 10298 /* 10299 * quiesce(9E) entry point. 10300 * 10301 * This function is called when the system is single-threaded at high 10302 * PIL with preemption disabled. Therefore, this function must not be 10303 * blocked. Because no taskqs are running, there is no need for us to 10304 * take any action for enclosure services which are running in the 10305 * taskq context, especially as no interrupts are generated by it nor 10306 * are any messages expected to come in. 10307 * 10308 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 10309 * DDI_FAILURE indicates an error condition and should almost never happen. 10310 */ 10311 static int 10312 ahci_quiesce(dev_info_t *dip) 10313 { 10314 ahci_ctl_t *ahci_ctlp; 10315 ahci_port_t *ahci_portp; 10316 int instance, port; 10317 10318 instance = ddi_get_instance(dip); 10319 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 10320 10321 if (ahci_ctlp == NULL) 10322 return (DDI_FAILURE); 10323 10324 #if AHCI_DEBUG 10325 ahci_debug_flags = 0; 10326 #endif 10327 10328 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE; 10329 10330 /* disable all the interrupts. */ 10331 ahci_disable_all_intrs(ahci_ctlp); 10332 10333 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 10334 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 10335 continue; 10336 } 10337 10338 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 10339 10340 /* 10341 * Stop the port by clearing PxCMD.ST 10342 * 10343 * Here we must disable the port interrupt because 10344 * ahci_disable_all_intrs only clear GHC.IE, and IS 10345 * register will be still set if PxIE is enabled. 10346 * When ahci shares one IRQ with other drivers, the 10347 * intr handler may claim the intr mistakenly. 10348 */ 10349 ahci_disable_port_intrs(ahci_ctlp, port); 10350 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 10351 ahci_portp, port); 10352 } 10353 10354 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE; 10355 10356 return (DDI_SUCCESS); 10357 } 10358 10359 /* 10360 * The function will add a sata packet to the done queue. 10361 */ 10362 static void 10363 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason) 10364 { 10365 ASSERT(satapkt != NULL); 10366 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 10367 10368 /* set the reason for all packets */ 10369 satapkt->satapkt_reason = reason; 10370 satapkt->satapkt_hba_driver_private = NULL; 10371 10372 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && 10373 satapkt->satapkt_comp) { 10374 /* 10375 * only add to queue when mode is not synch and there is 10376 * completion callback 10377 */ 10378 *ahci_portp->ahciport_doneqtail = satapkt; 10379 ahci_portp->ahciport_doneqtail = 10380 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private); 10381 ahci_portp->ahciport_doneq_len++; 10382 10383 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && 10384 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) 10385 /* 10386 * for sync/non-poll mode, just call cv_broadcast 10387 */ 10388 cv_broadcast(&ahci_portp->ahciport_cv); 10389 } 10390 10391 /* 10392 * The function will call completion callback of sata packet on the 10393 * completed queue 10394 */ 10395 static void 10396 ahci_flush_doneq(ahci_port_t *ahci_portp) 10397 { 10398 sata_pkt_t *satapkt, *next; 10399 10400 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 10401 10402 if (ahci_portp->ahciport_doneq) { 10403 satapkt = ahci_portp->ahciport_doneq; 10404 10405 ahci_portp->ahciport_doneq = NULL; 10406 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq; 10407 ahci_portp->ahciport_doneq_len = 0; 10408 10409 mutex_exit(&ahci_portp->ahciport_mutex); 10410 10411 while (satapkt != NULL) { 10412 next = satapkt->satapkt_hba_driver_private; 10413 satapkt->satapkt_hba_driver_private = NULL; 10414 10415 /* Call the callback */ 10416 (*satapkt->satapkt_comp)(satapkt); 10417 10418 satapkt = next; 10419 } 10420 10421 mutex_enter(&ahci_portp->ahciport_mutex); 10422 } 10423 } 10424 10425 /* 10426 * Sets the state for the specified port on the controller to desired state. 10427 * This must be run in the context of the enclosure taskq which ensures that 10428 * only one event is outstanding at any time. 10429 */ 10430 static boolean_t 10431 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire) 10432 { 10433 ahci_em_led_msg_t msg; 10434 ahci_em_msg_hdr_t hdr; 10435 uint32_t msgval, hdrval; 10436 uint_t i, max_delay = ahci_em_tx_delay_count; 10437 10438 msg.alm_hba = port; 10439 msg.alm_pminfo = 0; 10440 msg.alm_value = 0; 10441 10442 if (desire & AHCI_EM_LED_IDENT_ENABLE) { 10443 msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF; 10444 } 10445 10446 if (desire & AHCI_EM_LED_FAULT_ENABLE) { 10447 msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF; 10448 } 10449 10450 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 && 10451 (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) { 10452 msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF; 10453 } 10454 10455 hdr.aemh_rsvd = 0; 10456 hdr.aemh_mlen = sizeof (ahci_em_led_msg_t); 10457 hdr.aemh_dlen = 0; 10458 hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED; 10459 10460 bcopy(&msg, &msgval, sizeof (msgval)); 10461 bcopy(&hdr, &hdrval, sizeof (hdrval)); 10462 10463 /* 10464 * First, make sure we can transmit. We should not have been placed in a 10465 * situation where an outstanding transmission is going on. 10466 */ 10467 for (i = 0; i < max_delay; i++) { 10468 uint32_t val; 10469 10470 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10471 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10472 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0) 10473 break; 10474 10475 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000)); 10476 } 10477 10478 if (i == max_delay) 10479 return (B_FALSE); 10480 10481 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10482 (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval); 10483 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10484 (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval); 10485 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10486 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM); 10487 10488 for (i = 0; i < max_delay; i++) { 10489 uint32_t val; 10490 10491 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10492 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10493 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0) 10494 break; 10495 10496 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000)); 10497 } 10498 10499 if (i == max_delay) 10500 return (B_FALSE); 10501 10502 return (B_TRUE); 10503 } 10504 10505 typedef struct ahci_em_led_task_arg { 10506 ahci_ctl_t *aelta_ctl; 10507 uint8_t aelta_port; 10508 uint_t aelta_op; 10509 ahci_em_led_state_t aelta_state; 10510 uint_t aelta_ret; 10511 kcondvar_t aelta_cv; 10512 uint_t aelta_ref; 10513 } ahci_em_led_task_arg_t; 10514 10515 static void 10516 ahci_em_led_task_free(ahci_em_led_task_arg_t *task) 10517 { 10518 ASSERT3U(task->aelta_ref, ==, 0); 10519 cv_destroy(&task->aelta_cv); 10520 kmem_free(task, sizeof (*task)); 10521 } 10522 10523 static void 10524 ahci_em_led_task(void *arg) 10525 { 10526 boolean_t ret, cleanup = B_FALSE; 10527 ahci_em_led_task_arg_t *led = arg; 10528 ahci_em_led_state_t state; 10529 10530 mutex_enter(&led->aelta_ctl->ahcictl_mutex); 10531 if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) { 10532 led->aelta_ret = EIO; 10533 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10534 return; 10535 } 10536 10537 state = led->aelta_ctl->ahcictl_em_state[led->aelta_port]; 10538 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10539 10540 switch (led->aelta_op) { 10541 case AHCI_EM_IOC_SET_OP_ADD: 10542 state |= led->aelta_state; 10543 break; 10544 case AHCI_EM_IOC_SET_OP_REM: 10545 state &= ~led->aelta_state; 10546 break; 10547 case AHCI_EM_IOC_SET_OP_SET: 10548 state = led->aelta_state; 10549 break; 10550 default: 10551 led->aelta_ret = ENOTSUP; 10552 return; 10553 } 10554 10555 ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state); 10556 10557 mutex_enter(&led->aelta_ctl->ahcictl_mutex); 10558 if (ret) { 10559 led->aelta_ctl->ahcictl_em_state[led->aelta_port] = state; 10560 led->aelta_ret = 0; 10561 } else { 10562 led->aelta_ret = EIO; 10563 led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10564 } 10565 led->aelta_ref--; 10566 if (led->aelta_ref > 0) { 10567 cv_signal(&led->aelta_cv); 10568 } else { 10569 cleanup = B_TRUE; 10570 } 10571 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10572 10573 if (cleanup) { 10574 ahci_em_led_task_free(led); 10575 } 10576 } 10577 10578 static void 10579 ahci_em_reset(void *arg) 10580 { 10581 uint_t i, max_delay = ahci_em_reset_delay_count; 10582 ahci_ctl_t *ahci_ctlp = arg; 10583 10584 /* 10585 * We've been asked to reset the device. The caller should have set the 10586 * resetting flag. Make sure that we don't have a request to quiesce. 10587 */ 10588 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10589 ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING); 10590 if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) { 10591 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10592 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10593 return; 10594 } 10595 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10596 10597 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10598 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST); 10599 for (i = 0; i < max_delay; i++) { 10600 uint32_t val; 10601 10602 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10603 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10604 if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0) 10605 break; 10606 10607 delay(drv_usectohz(ahci_em_reset_delay_ms * 1000)); 10608 } 10609 10610 if (i == max_delay) { 10611 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10612 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10613 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10614 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10615 cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting", 10616 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10617 return; 10618 } 10619 10620 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) { 10621 10622 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) 10623 continue; 10624 10625 /* 10626 * Try to flush all the LEDs as part of reset. If it fails, 10627 * drive on. 10628 */ 10629 if (!ahci_em_set_led(ahci_ctlp, i, 10630 ahci_ctlp->ahcictl_em_state[i])) { 10631 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10632 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10633 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10634 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10635 cmn_err(CE_WARN, "!ahci%d: enclosure timed out " 10636 "setting port %u", 10637 ddi_get_instance(ahci_ctlp->ahcictl_dip), i); 10638 return; 10639 } 10640 } 10641 10642 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10643 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10644 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY; 10645 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10646 } 10647 10648 static boolean_t 10649 ahci_em_init(ahci_ctl_t *ahci_ctlp) 10650 { 10651 char name[128]; 10652 10653 /* 10654 * First make sure we actually have enclosure services and if so, that 10655 * we have the hardware support that we care about for this. 10656 */ 10657 if (ahci_ctlp->ahcictl_em_loc == 0 || 10658 (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0) 10659 return (B_TRUE); 10660 10661 /* 10662 * Next, make sure that the buffer is large enough for us. We need two 10663 * dwords or 8 bytes. The location register is stored in dwords. 10664 */ 10665 if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) < 10666 AHCI_EM_BUFFER_MIN) { 10667 return (B_TRUE); 10668 } 10669 10670 ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc & 10671 AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4; 10672 ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr; 10673 10674 bzero(ahci_ctlp->ahcictl_em_state, 10675 sizeof (ahci_ctlp->ahcictl_em_state)); 10676 10677 (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d", 10678 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10679 if ((ahci_ctlp->ahcictl_em_taskq = 10680 ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1, 10681 TASKQ_DEFAULTPRI, 0)) == NULL) { 10682 cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em " 10683 "services", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10684 return (B_FALSE); 10685 } 10686 10687 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10688 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT | AHCI_EM_RESETTING; 10689 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10690 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset, 10691 ahci_ctlp, DDI_SLEEP); 10692 10693 return (B_TRUE); 10694 } 10695 10696 static int 10697 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg) 10698 { 10699 int i; 10700 ahci_ioc_em_get_t get; 10701 10702 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10703 return (ENOTSUP); 10704 } 10705 10706 bzero(&get, sizeof (get)); 10707 get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented; 10708 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) { 10709 get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY; 10710 } 10711 10712 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10713 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) { 10714 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) { 10715 continue; 10716 } 10717 get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i]; 10718 } 10719 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10720 10721 if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0) 10722 return (EFAULT); 10723 10724 return (0); 10725 } 10726 10727 static int 10728 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg) 10729 { 10730 int ret; 10731 ahci_ioc_em_set_t set; 10732 ahci_em_led_task_arg_t *task; 10733 boolean_t signal, cleanup; 10734 10735 if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0) 10736 return (EFAULT); 10737 10738 if (set.aiems_port > ahci_ctlp->ahcictl_num_ports) 10739 return (EINVAL); 10740 10741 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) { 10742 return (EINVAL); 10743 } 10744 10745 if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE | 10746 AHCI_EM_LED_FAULT_ENABLE | 10747 AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) { 10748 return (EINVAL); 10749 } 10750 10751 switch (set.aiems_op) { 10752 case AHCI_EM_IOC_SET_OP_ADD: 10753 case AHCI_EM_IOC_SET_OP_REM: 10754 case AHCI_EM_IOC_SET_OP_SET: 10755 break; 10756 default: 10757 return (EINVAL); 10758 } 10759 10760 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10761 return (ENOTSUP); 10762 } 10763 10764 if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 && 10765 ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) { 10766 return (ENOTSUP); 10767 } 10768 10769 task = kmem_alloc(sizeof (*task), KM_NOSLEEP | KM_NORMALPRI); 10770 if (task == NULL) { 10771 return (ENOMEM); 10772 } 10773 10774 task->aelta_ctl = ahci_ctlp; 10775 task->aelta_port = (uint8_t)set.aiems_port; 10776 task->aelta_op = set.aiems_op; 10777 task->aelta_state = set.aiems_leds; 10778 10779 cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL); 10780 10781 /* 10782 * Initialize the reference count to two. One for us and one for the 10783 * taskq. This will be used in case we get canceled. 10784 */ 10785 task->aelta_ref = 2; 10786 10787 /* 10788 * Once dispatched, the task state is protected by our global mutex. 10789 */ 10790 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, 10791 ahci_em_led_task, task, DDI_SLEEP); 10792 10793 signal = B_FALSE; 10794 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10795 while (task->aelta_ref > 1) { 10796 if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) == 10797 0) { 10798 signal = B_TRUE; 10799 break; 10800 } 10801 } 10802 10803 /* 10804 * Remove our reference count. If we were woken up because of a signal 10805 * then the taskq may still be dispatched. In which case we shouldn't 10806 * free this memory until it is done. In that case, the taskq will take 10807 * care of it. 10808 */ 10809 task->aelta_ref--; 10810 cleanup = (task->aelta_ref == 0); 10811 if (signal) { 10812 ret = EINTR; 10813 } else { 10814 ret = task->aelta_ret; 10815 } 10816 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10817 10818 if (cleanup) { 10819 ahci_em_led_task_free(task); 10820 } 10821 10822 return (ret); 10823 } 10824 10825 static int 10826 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg) 10827 { 10828 int inst; 10829 ahci_ctl_t *ahci_ctlp; 10830 10831 inst = ddi_get_instance(dip); 10832 if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) { 10833 return (ENXIO); 10834 } 10835 10836 switch (cmd) { 10837 case AHCI_EM_IOC_GET: 10838 return (ahci_em_ioctl_get(ahci_ctlp, arg)); 10839 case AHCI_EM_IOC_SET: 10840 return (ahci_em_ioctl_set(ahci_ctlp, arg)); 10841 default: 10842 return (ENOTTY); 10843 } 10844 10845 } 10846 10847 static void 10848 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp) 10849 { 10850 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10851 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10852 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10853 return; 10854 } 10855 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE; 10856 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10857 10858 ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq); 10859 } 10860 10861 static void 10862 ahci_em_suspend(ahci_ctl_t *ahci_ctlp) 10863 { 10864 ahci_em_quiesce(ahci_ctlp); 10865 10866 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10867 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY; 10868 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10869 } 10870 10871 static void 10872 ahci_em_resume(ahci_ctl_t *ahci_ctlp) 10873 { 10874 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10875 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10876 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10877 return; 10878 } 10879 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING; 10880 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10881 10882 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset, 10883 ahci_ctlp, DDI_SLEEP); 10884 } 10885 10886 static void 10887 ahci_em_fini(ahci_ctl_t *ahci_ctlp) 10888 { 10889 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10890 return; 10891 } 10892 10893 ahci_em_quiesce(ahci_ctlp); 10894 ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq); 10895 ahci_ctlp->ahcictl_em_taskq = NULL; 10896 } 10897