1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2018 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2018, Joyent, Inc. 26 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association. 27 */ 28 29 /* 30 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 31 * 32 * Power Management Support 33 * ------------------------ 34 * 35 * At the moment, the ahci driver only implements suspend/resume to 36 * support Suspend to RAM on X86 feature. Device power management isn't 37 * implemented, link power management is disabled, and hot plug isn't 38 * allowed during the period from suspend to resume. 39 * 40 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 41 * and DDI_RESUME entries, and don't need to take care of new requests 42 * sent down after suspend because the target driver (sd) has already 43 * handled these conditions, and blocked these requests. For the detailed 44 * information, please check with sdopen, sdclose and sdioctl routines. 45 * 46 * 47 * Enclosure Management Support 48 * ---------------------------- 49 * 50 * The ahci driver has basic support for AHCI Enclosure Management (EM) 51 * services. The AHCI specification provides an area in the primary ahci BAR for 52 * posting data to send out to the enclosure management and provides a register 53 * that provides both information and control about this. While the 54 * specification allows for multiple forms of enclosure management, the only 55 * supported, and commonly found form, is the AHCI specified LED format. The LED 56 * format is often implemented as a one-way communication mechanism. Software 57 * can write out what it cares about into the aforementioned data buffer and 58 * then we wait for the transmission to be sent. 59 * 60 * This has some drawbacks. It means that we cannot know whether or not it has 61 * succeeded. This means we cannot ask hardware what it thinks the LEDs are 62 * set to. There's also the added unfortunate reality that firmware on the 63 * microcontroller driving this will often not show the LEDs if no drive is 64 * present and that actions taken may potentially cause this to get out of sync 65 * with what we expect it to be. For example, the specification does not 66 * describe what should happen if a drive is removed from the enclosure while 67 * this is set and what should happen when it returns. We can only infer that it 68 * should be the same. 69 * 70 * Because only a single command can be sent at any time and we don't want to 71 * interfere with controller I/O, we create a taskq dedicated to this that has a 72 * single thread. Both resets (which occur on attach and resume) and normal 73 * changes to the LED state will be driven through this taskq. Because the taskq 74 * has a single thread, this guarantees serial processing. 75 * 76 * Each userland-submitted task (basically not resets) has a reference counted 77 * task structure. This allows the thread that called it to be cancelled and 78 * have the system clean itself up. The user thread in ioctl blocks on a CV that 79 * can receive signals as it waits for completion. Note, there is no guarantee 80 * provided by the kernel that the first thread to enter the kernel will be the 81 * first one to change state. 82 */ 83 84 #include <sys/note.h> 85 #include <sys/scsi/scsi.h> 86 #include <sys/pci.h> 87 #include <sys/disp.h> 88 #include <sys/sata/sata_hba.h> 89 #include <sys/sata/adapters/ahci/ahcireg.h> 90 #include <sys/sata/adapters/ahci/ahcivar.h> 91 92 /* 93 * FMA header files 94 */ 95 #include <sys/ddifm.h> 96 #include <sys/fm/protocol.h> 97 #include <sys/fm/util.h> 98 #include <sys/fm/io/ddi.h> 99 100 /* 101 * EM Control header files 102 */ 103 #include <sys/types.h> 104 #include <sys/file.h> 105 #include <sys/errno.h> 106 #include <sys/open.h> 107 #include <sys/cred.h> 108 #include <sys/ddi.h> 109 #include <sys/sunddi.h> 110 111 /* 112 * This is the string displayed by modinfo, etc. 113 */ 114 static char ahci_ident[] = "ahci driver"; 115 116 /* 117 * Function prototypes for driver entry points 118 */ 119 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 120 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 121 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 122 static int ahci_quiesce(dev_info_t *); 123 124 /* 125 * Function prototypes for SATA Framework interfaces 126 */ 127 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 128 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 129 130 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 131 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 132 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 133 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 134 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 135 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 136 #if defined(__lock_lint) 137 static int ahci_selftest(dev_info_t *, sata_device_t *); 138 #endif 139 140 /* 141 * FMA Prototypes 142 */ 143 static void ahci_fm_init(ahci_ctl_t *); 144 static void ahci_fm_fini(ahci_ctl_t *); 145 static int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*); 146 int ahci_check_acc_handle(ddi_acc_handle_t); 147 int ahci_check_dma_handle(ddi_dma_handle_t); 148 void ahci_fm_ereport(ahci_ctl_t *, char *); 149 static int ahci_check_all_handle(ahci_ctl_t *); 150 static int ahci_check_ctl_handle(ahci_ctl_t *); 151 static int ahci_check_port_handle(ahci_ctl_t *, int); 152 static int ahci_check_slot_handle(ahci_port_t *, int); 153 154 /* 155 * Local function prototypes 156 */ 157 static int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *); 158 static int ahci_alloc_ports_state(ahci_ctl_t *); 159 static void ahci_dealloc_ports_state(ahci_ctl_t *); 160 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 161 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 162 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *); 163 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 164 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 165 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 166 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 167 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 168 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *); 169 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *); 170 171 static int ahci_initialize_controller(ahci_ctl_t *); 172 static void ahci_uninitialize_controller(ahci_ctl_t *); 173 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 174 static int ahci_config_space_init(ahci_ctl_t *); 175 static void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t); 176 177 static void ahci_drain_ports_taskq(ahci_ctl_t *); 178 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *, 179 uint8_t); 180 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *); 181 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t); 182 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *, 183 sata_device_t *); 184 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *, 185 sata_pmult_gscr_t *); 186 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 187 sata_device_t *); 188 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 189 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 190 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 191 sata_device_t *); 192 193 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 194 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 195 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, 196 ahci_addr_t *); 197 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 198 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 199 ahci_addr_t *, sata_pkt_t *); 200 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 201 ahci_addr_t *, sata_pkt_t *); 202 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, 203 ahci_addr_t *, int); 204 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 205 static void ahci_copy_ncq_err_page(sata_cmd_t *, 206 struct sata_ncq_error_recovery_page *); 207 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 208 static void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int); 209 static void ahci_flush_doneq(ahci_port_t *); 210 211 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 212 static int ahci_hba_reset(ahci_ctl_t *); 213 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 214 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 215 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 216 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, 217 ahci_addr_t *); 218 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *, 219 ahci_addr_t *); 220 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, 221 ahci_addr_t *); 222 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 223 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 224 uint8_t); 225 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 226 uint8_t, int, int *); 227 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 228 uint32_t, uint32_t, uint32_t, uint32_t); 229 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 231 uint8_t, sata_pkt_t *); 232 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 233 ahci_addr_t *, uint32_t); 234 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 235 uint8_t, uint32_t); 236 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 237 uint8_t, uint32_t); 238 static void ahci_events_handler(void *); 239 static void ahci_watchdog_handler(ahci_ctl_t *); 240 241 static uint_t ahci_intr(caddr_t, caddr_t); 242 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 243 static int ahci_add_intrs(ahci_ctl_t *, int); 244 static void ahci_rem_intrs(ahci_ctl_t *); 245 static void ahci_enable_all_intrs(ahci_ctl_t *); 246 static void ahci_disable_all_intrs(ahci_ctl_t *); 247 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 248 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 249 250 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 251 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 252 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 253 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t); 254 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 255 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 256 ahci_port_t *, uint8_t); 257 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 258 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 259 uint8_t, uint32_t); 260 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 261 uint8_t, uint32_t); 262 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 263 264 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *); 265 static int ahci_get_num_implemented_ports(uint32_t); 266 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t); 267 static void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t); 268 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 269 #if AHCI_DEBUG 270 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 271 #endif 272 273 static boolean_t ahci_em_init(ahci_ctl_t *); 274 static void ahci_em_fini(ahci_ctl_t *); 275 static void ahci_em_suspend(ahci_ctl_t *); 276 static void ahci_em_resume(ahci_ctl_t *); 277 static int ahci_em_ioctl(dev_info_t *, int, intptr_t); 278 279 280 /* 281 * DMA attributes for the data buffer 282 * 283 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 284 * does not support 64-bit addressing 285 */ 286 static ddi_dma_attr_t buffer_dma_attr = { 287 DMA_ATTR_V0, /* dma_attr_version */ 288 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 289 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 290 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 291 0x2ull, /* dma_attr_align: word aligned */ 292 1, /* dma_attr_burstsizes */ 293 1, /* dma_attr_minxfer */ 294 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 295 0xffffffffull, /* dma_attr_seg */ 296 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 297 512, /* dma_attr_granular */ 298 0, /* dma_attr_flags */ 299 }; 300 301 /* 302 * DMA attributes for the rcvd FIS 303 * 304 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 305 * does not support 64-bit addressing 306 */ 307 static ddi_dma_attr_t rcvd_fis_dma_attr = { 308 DMA_ATTR_V0, /* dma_attr_version */ 309 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 310 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 311 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 312 0x100ull, /* dma_attr_align: 256-byte aligned */ 313 1, /* dma_attr_burstsizes */ 314 1, /* dma_attr_minxfer */ 315 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 316 0xffffffffull, /* dma_attr_seg */ 317 1, /* dma_attr_sgllen */ 318 1, /* dma_attr_granular */ 319 0, /* dma_attr_flags */ 320 }; 321 322 /* 323 * DMA attributes for the command list 324 * 325 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 326 * does not support 64-bit addressing 327 */ 328 static ddi_dma_attr_t cmd_list_dma_attr = { 329 DMA_ATTR_V0, /* dma_attr_version */ 330 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 331 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 332 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 333 0x400ull, /* dma_attr_align: 1K-byte aligned */ 334 1, /* dma_attr_burstsizes */ 335 1, /* dma_attr_minxfer */ 336 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 337 0xffffffffull, /* dma_attr_seg */ 338 1, /* dma_attr_sgllen */ 339 1, /* dma_attr_granular */ 340 0, /* dma_attr_flags */ 341 }; 342 343 /* 344 * DMA attributes for cmd tables 345 * 346 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 347 * does not support 64-bit addressing 348 */ 349 static ddi_dma_attr_t cmd_table_dma_attr = { 350 DMA_ATTR_V0, /* dma_attr_version */ 351 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 352 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 353 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 354 0x80ull, /* dma_attr_align: 128-byte aligned */ 355 1, /* dma_attr_burstsizes */ 356 1, /* dma_attr_minxfer */ 357 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 358 0xffffffffull, /* dma_attr_seg */ 359 1, /* dma_attr_sgllen */ 360 1, /* dma_attr_granular */ 361 0, /* dma_attr_flags */ 362 }; 363 364 365 /* Device access attributes */ 366 static ddi_device_acc_attr_t accattr = { 367 DDI_DEVICE_ATTR_V1, 368 DDI_STRUCTURE_LE_ACC, 369 DDI_STRICTORDER_ACC, 370 DDI_DEFAULT_ACC 371 }; 372 373 static struct dev_ops ahcictl_dev_ops = { 374 DEVO_REV, /* devo_rev */ 375 0, /* refcnt */ 376 ahci_getinfo, /* info */ 377 nulldev, /* identify */ 378 nulldev, /* probe */ 379 ahci_attach, /* attach */ 380 ahci_detach, /* detach */ 381 nodev, /* no reset */ 382 NULL, /* driver operations */ 383 NULL, /* bus operations */ 384 NULL, /* power */ 385 ahci_quiesce, /* quiesce */ 386 }; 387 388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 389 SATA_TRAN_HOTPLUG_OPS_REV_1, 390 ahci_tran_hotplug_port_activate, 391 ahci_tran_hotplug_port_deactivate 392 }; 393 394 extern struct mod_ops mod_driverops; 395 396 static struct modldrv modldrv = { 397 &mod_driverops, /* driverops */ 398 ahci_ident, /* short description */ 399 &ahcictl_dev_ops, /* driver ops */ 400 }; 401 402 static struct modlinkage modlinkage = { 403 MODREV_1, 404 &modldrv, 405 NULL 406 }; 407 408 /* The following variables are watchdog handler related */ 409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */ 410 static clock_t ahci_watchdog_tick; 411 412 /* 413 * This static variable indicates the size of command table, 414 * and it's changeable with prdt number, which ahci_dma_prdt_number 415 * indicates. 416 */ 417 static size_t ahci_cmd_table_size; 418 419 /* 420 * The below global variables are tunable via /etc/system 421 * 422 * ahci_dma_prdt_number 423 * ahci_msi_enabled 424 * ahci_buf_64bit_dma 425 * ahci_commu_64bit_dma 426 */ 427 428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 430 431 /* AHCI MSI is tunable */ 432 boolean_t ahci_msi_enabled = B_TRUE; 433 434 /* 435 * 64-bit dma addressing for data buffer is tunable 436 * 437 * The variable controls only the below value: 438 * DBAU (upper 32-bits physical address of data block) 439 */ 440 boolean_t ahci_buf_64bit_dma = B_TRUE; 441 442 /* 443 * 64-bit dma addressing for communication system descriptors is tunable 444 * 445 * The variable controls the below three values: 446 * 447 * PxCLBU (upper 32-bits for the command list base physical address) 448 * PxFBU (upper 32-bits for the received FIS base physical address) 449 * CTBAU (upper 32-bits of command table base) 450 */ 451 boolean_t ahci_commu_64bit_dma = B_TRUE; 452 453 /* 454 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600 455 * chipset. If the users want to have a try with 64-bit dma, please change 456 * the below variable value to enable it. 457 */ 458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE; 459 460 /* 461 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI 462 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma, 463 * please change the below value to enable it. 464 */ 465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE; 466 467 /* 468 * These values control the default delay and default number of times to wait 469 * for an enclosure message to complete. 470 */ 471 uint_t ahci_em_reset_delay_ms = 1; 472 uint_t ahci_em_reset_delay_count = 1000; 473 uint_t ahci_em_tx_delay_ms = 1; 474 uint_t ahci_em_tx_delay_count = 1000; 475 476 477 /* 478 * End of global tunable variable definition 479 */ 480 481 #if AHCI_DEBUG 482 uint32_t ahci_debug_flags = 0; 483 #else 484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 485 #endif 486 487 488 #if AHCI_DEBUG 489 /* The following is needed for ahci_log() */ 490 static kmutex_t ahci_log_mutex; 491 static char ahci_log_buf[512]; 492 #endif 493 494 /* Opaque state pointer initialized by ddi_soft_state_init() */ 495 static void *ahci_statep = NULL; 496 497 /* 498 * ahci module initialization. 499 */ 500 int 501 _init(void) 502 { 503 int ret; 504 505 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 506 if (ret != 0) { 507 goto err_out; 508 } 509 510 #if AHCI_DEBUG 511 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 512 #endif 513 514 if ((ret = sata_hba_init(&modlinkage)) != 0) { 515 #if AHCI_DEBUG 516 mutex_destroy(&ahci_log_mutex); 517 #endif 518 ddi_soft_state_fini(&ahci_statep); 519 goto err_out; 520 } 521 522 /* watchdog tick */ 523 ahci_watchdog_tick = drv_usectohz( 524 (clock_t)ahci_watchdog_timeout * 1000000); 525 526 ret = mod_install(&modlinkage); 527 if (ret != 0) { 528 sata_hba_fini(&modlinkage); 529 #if AHCI_DEBUG 530 mutex_destroy(&ahci_log_mutex); 531 #endif 532 ddi_soft_state_fini(&ahci_statep); 533 goto err_out; 534 } 535 536 return (ret); 537 538 err_out: 539 cmn_err(CE_WARN, "!ahci: Module init failed"); 540 return (ret); 541 } 542 543 /* 544 * ahci module uninitialize. 545 */ 546 int 547 _fini(void) 548 { 549 int ret; 550 551 ret = mod_remove(&modlinkage); 552 if (ret != 0) { 553 return (ret); 554 } 555 556 /* Remove the resources allocated in _init(). */ 557 sata_hba_fini(&modlinkage); 558 #if AHCI_DEBUG 559 mutex_destroy(&ahci_log_mutex); 560 #endif 561 ddi_soft_state_fini(&ahci_statep); 562 563 return (ret); 564 } 565 566 /* 567 * _info entry point 568 */ 569 int 570 _info(struct modinfo *modinfop) 571 { 572 return (mod_info(&modlinkage, modinfop)); 573 } 574 575 /* 576 * The attach entry point for dev_ops. 577 */ 578 static int 579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 580 { 581 ahci_ctl_t *ahci_ctlp = NULL; 582 int instance = ddi_get_instance(dip); 583 int status; 584 int attach_state; 585 uint32_t cap_status, ahci_version; 586 uint32_t ghc_control; 587 int intr_types; 588 int i; 589 pci_regspec_t *regs; 590 int regs_length; 591 int rnumber; 592 #if AHCI_DEBUG 593 int speed; 594 #endif 595 596 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 597 NULL); 598 599 switch (cmd) { 600 case DDI_ATTACH: 601 break; 602 603 case DDI_RESUME: 604 605 /* 606 * During DDI_RESUME, the hardware state of the device 607 * (power may have been removed from the device) must be 608 * restored, allow pending requests to continue, and 609 * service new requests. 610 */ 611 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 612 mutex_enter(&ahci_ctlp->ahcictl_mutex); 613 614 /* 615 * GHC.AE must be set to 1 before any other AHCI register 616 * is accessed 617 */ 618 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 619 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 620 ghc_control |= AHCI_HBA_GHC_AE; 621 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 623 624 /* Restart watch thread */ 625 if (ahci_ctlp->ahcictl_timeout_id == 0) 626 ahci_ctlp->ahcictl_timeout_id = timeout( 627 (void (*)(void *))ahci_watchdog_handler, 628 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 629 630 mutex_exit(&ahci_ctlp->ahcictl_mutex); 631 632 /* 633 * Re-initialize the controller and enable the interrupts and 634 * restart all the ports. 635 * 636 * Note that so far we don't support hot-plug during 637 * suspend/resume. 638 */ 639 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 640 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 641 "Failed to initialize the controller " 642 "during DDI_RESUME", NULL); 643 return (DDI_FAILURE); 644 } 645 646 /* 647 * Reset the enclosure services. 648 */ 649 ahci_em_resume(ahci_ctlp); 650 651 mutex_enter(&ahci_ctlp->ahcictl_mutex); 652 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND; 653 mutex_exit(&ahci_ctlp->ahcictl_mutex); 654 655 return (DDI_SUCCESS); 656 657 default: 658 return (DDI_FAILURE); 659 } 660 661 attach_state = AHCI_ATTACH_STATE_NONE; 662 663 /* Allocate soft state */ 664 status = ddi_soft_state_zalloc(ahci_statep, instance); 665 if (status != DDI_SUCCESS) { 666 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 667 instance); 668 goto err_out; 669 } 670 671 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 672 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 673 ahci_ctlp->ahcictl_dip = dip; 674 675 /* Initialize the cport/port mapping */ 676 for (i = 0; i < AHCI_MAX_PORTS; i++) { 677 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 678 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 679 } 680 681 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 682 683 /* Initialize FMA properties */ 684 ahci_fm_init(ahci_ctlp); 685 686 attach_state |= AHCI_ATTACH_STATE_FMA; 687 688 /* 689 * Now map the AHCI base address; which includes global 690 * registers and port control registers 691 * 692 * According to the spec, the AHCI Base Address is BAR5, 693 * but BAR0-BAR4 are optional, so we need to check which 694 * rnumber is used for BAR5. 695 */ 696 697 /* 698 * search through DDI "reg" property for the AHCI register set 699 */ 700 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 701 DDI_PROP_DONTPASS, "reg", (int **)®s, 702 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 703 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 704 instance); 705 goto err_out; 706 } 707 708 /* AHCI Base Address is located at 0x24 offset */ 709 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 710 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 711 == AHCI_PCI_RNUM) 712 break; 713 } 714 715 ddi_prop_free(regs); 716 717 if (rnumber == regs_length) { 718 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 719 instance); 720 goto err_out; 721 } 722 723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 724 725 status = ddi_regs_map_setup(dip, 726 rnumber, 727 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 728 0, 729 0, 730 &accattr, 731 &ahci_ctlp->ahcictl_ahci_acc_handle); 732 if (status != DDI_SUCCESS) { 733 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 734 instance); 735 goto err_out; 736 } 737 738 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 739 740 /* 741 * GHC.AE must be set to 1 before any other AHCI register 742 * is accessed 743 */ 744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 746 ghc_control |= AHCI_HBA_GHC_AE; 747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 749 750 /* Get the AHCI version information */ 751 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 752 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 753 754 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 755 (ahci_version & 0xffff0000) >> 16, 756 ((ahci_version & 0x0000ff00) >> 4 | 757 (ahci_version & 0x000000ff))); 758 759 /* We don't support controllers whose versions are lower than 1.0 */ 760 if (!(ahci_version & 0xffff0000)) { 761 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 762 "than version 1.0", instance); 763 goto err_out; 764 } 765 766 /* Get the HBA capabilities information */ 767 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 768 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 769 770 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x", 771 cap_status); 772 773 /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */ 774 if (ahci_version >= 0x00010200) { 775 uint32_t cap2_status; 776 777 /* Get the HBA capabilities extended information */ 778 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 779 (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp)); 780 781 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 782 "hba capabilities extended = 0x%x", cap2_status); 783 } 784 785 if (cap_status & AHCI_HBA_CAP_EMS) { 786 ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS; 787 ahci_ctlp->ahcictl_em_loc = 788 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 789 (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp)); 790 ahci_ctlp->ahcictl_em_ctl = 791 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 792 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 793 } 794 795 #if AHCI_DEBUG 796 /* Get the interface speed supported by the HBA */ 797 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 798 if (speed == 0x01) { 799 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 800 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 801 } else if (speed == 0x10) { 802 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 803 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 804 } else if (speed == 0x11) { 805 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 806 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 807 } 808 #endif 809 810 /* Get the number of command slots supported by the HBA */ 811 ahci_ctlp->ahcictl_num_cmd_slots = 812 ((cap_status & AHCI_HBA_CAP_NCS) >> 813 AHCI_HBA_CAP_NCS_SHIFT) + 1; 814 815 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 816 ahci_ctlp->ahcictl_num_cmd_slots); 817 818 /* Get the bit map which indicates ports implemented by the HBA */ 819 ahci_ctlp->ahcictl_ports_implemented = 820 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 821 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 822 823 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 824 ahci_ctlp->ahcictl_ports_implemented); 825 826 /* Max port number implemented */ 827 ahci_ctlp->ahcictl_num_ports = 828 ddi_fls(ahci_ctlp->ahcictl_ports_implemented); 829 830 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 831 (cap_status & AHCI_HBA_CAP_NP) + 1); 832 833 /* Get the number of implemented ports by the HBA */ 834 ahci_ctlp->ahcictl_num_implemented_ports = 835 ahci_get_num_implemented_ports( 836 ahci_ctlp->ahcictl_ports_implemented); 837 838 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 839 "hba number of implemented ports: %d", 840 ahci_ctlp->ahcictl_num_implemented_ports); 841 842 /* Check whether HBA supports 64bit DMA addressing */ 843 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 844 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 845 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 846 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 847 "hba does not support 64-bit addressing", NULL); 848 } 849 850 /* Checking for the support of Port Multiplier */ 851 if (cap_status & AHCI_HBA_CAP_SPM) { 852 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS; 853 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 854 "hba supports port multiplier (CBSS)", NULL); 855 856 /* Support FIS-based switching ? */ 857 if (cap_status & AHCI_HBA_CAP_FBSS) { 858 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS; 859 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 860 "hba supports FIS-based switching (FBSS)", NULL); 861 } 862 } 863 864 /* Checking for Support Command List Override */ 865 if (cap_status & AHCI_HBA_CAP_SCLO) { 866 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 867 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 868 "hba supports command list override.", NULL); 869 } 870 871 /* Checking for Asynchronous Notification */ 872 if (cap_status & AHCI_HBA_CAP_SSNTF) { 873 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF; 874 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 875 "hba supports asynchronous notification.", NULL); 876 } 877 878 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 879 != DDI_SUCCESS) { 880 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 881 instance); 882 goto err_out; 883 } 884 885 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 886 887 /* 888 * Check the pci configuration space, and set caps. We also 889 * handle the hardware defect in this function. 890 * 891 * For example, force ATI SB600 to use 32-bit dma addressing 892 * since it doesn't support 64-bit dma though its CAP register 893 * declares it support. 894 */ 895 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 896 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 897 instance); 898 goto err_out; 899 } 900 901 /* 902 * Disable the whole controller interrupts before adding 903 * interrupt handlers(s). 904 */ 905 ahci_disable_all_intrs(ahci_ctlp); 906 907 /* Get supported interrupt types */ 908 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 909 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 910 instance); 911 goto err_out; 912 } 913 914 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 915 "ddi_intr_get_supported_types() returned: 0x%x", 916 intr_types); 917 918 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 919 /* 920 * Try MSI first, but fall back to FIXED if failed 921 */ 922 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 923 DDI_SUCCESS) { 924 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 925 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 926 "Using MSI interrupt type", NULL); 927 goto intr_done; 928 } 929 930 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 931 "MSI registration failed, " 932 "trying FIXED interrupts", NULL); 933 } 934 935 if (intr_types & DDI_INTR_TYPE_FIXED) { 936 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 937 DDI_SUCCESS) { 938 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 939 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 940 "Using FIXED interrupt type", NULL); 941 goto intr_done; 942 } 943 944 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 945 "FIXED interrupt registration failed", NULL); 946 } 947 948 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 949 950 goto err_out; 951 952 intr_done: 953 954 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 955 956 /* Initialize the controller mutex */ 957 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 958 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 959 960 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 961 962 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 963 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 964 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 965 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 966 } 967 968 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 969 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 970 sizeof (ahci_prdt_item_t)); 971 972 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 973 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 974 " ahci_cmd_table_size is 0x%x", 975 ahci_dma_prdt_number, ahci_cmd_table_size); 976 977 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 978 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 979 ahci_dma_prdt_number; 980 981 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 982 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 983 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 984 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 985 986 /* 987 * enable 64bit dma for data buffer for SB600 if 988 * sb600_buf_64bit_dma_disable is B_FALSE 989 */ 990 if ((ahci_buf_64bit_dma == B_FALSE) || 991 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) && 992 !(sb600_buf_64bit_dma_disable == B_FALSE && 993 ahci_ctlp->ahcictl_venid == 0x1002 && 994 ahci_ctlp->ahcictl_devid == 0x4380))) { 995 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 996 0xffffffffull; 997 } 998 999 /* 1000 * enable 64bit dma for command buffer for SB600/700/710/800 1001 * if sbxxx_commu_64bit_dma_disable is B_FALSE 1002 */ 1003 if ((ahci_commu_64bit_dma == B_FALSE) || 1004 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) && 1005 !(sbxxx_commu_64bit_dma_disable == B_FALSE && 1006 ahci_ctlp->ahcictl_venid == 0x1002 && 1007 (ahci_ctlp->ahcictl_devid == 0x4380 || 1008 ahci_ctlp->ahcictl_devid == 0x4391)))) { 1009 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 1010 0xffffffffull; 1011 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 1012 0xffffffffull; 1013 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 1014 0xffffffffull; 1015 } 1016 1017 /* Allocate the ports structure */ 1018 status = ahci_alloc_ports_state(ahci_ctlp); 1019 if (status != AHCI_SUCCESS) { 1020 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 1021 instance); 1022 goto err_out; 1023 } 1024 1025 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 1026 1027 /* 1028 * Initialize the controller and ports. 1029 */ 1030 status = ahci_initialize_controller(ahci_ctlp); 1031 if (status != AHCI_SUCCESS) { 1032 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 1033 instance); 1034 goto err_out; 1035 } 1036 1037 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 1038 1039 /* Start one thread to check packet timeouts */ 1040 ahci_ctlp->ahcictl_timeout_id = timeout( 1041 (void (*)(void *))ahci_watchdog_handler, 1042 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 1043 1044 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 1045 1046 if (!ahci_em_init(ahci_ctlp)) { 1047 cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure " 1048 "services", instance); 1049 goto err_out; 1050 } 1051 attach_state |= AHCI_ATTACH_STATE_ENCLOSURE; 1052 1053 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 1054 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 1055 instance); 1056 goto err_out; 1057 } 1058 1059 /* Check all handles at the end of the attach operation. */ 1060 if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) { 1061 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles", 1062 instance); 1063 goto err_out; 1064 } 1065 1066 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 1067 1068 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 1069 1070 return (DDI_SUCCESS); 1071 1072 err_out: 1073 /* FMA message */ 1074 ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE); 1075 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST); 1076 1077 if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) { 1078 ahci_em_fini(ahci_ctlp); 1079 } 1080 1081 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 1082 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1083 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1084 ahci_ctlp->ahcictl_timeout_id = 0; 1085 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1086 } 1087 1088 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 1089 ahci_uninitialize_controller(ahci_ctlp); 1090 } 1091 1092 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 1093 ahci_dealloc_ports_state(ahci_ctlp); 1094 } 1095 1096 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 1097 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1098 } 1099 1100 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 1101 ahci_rem_intrs(ahci_ctlp); 1102 } 1103 1104 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 1105 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1106 } 1107 1108 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 1109 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1110 } 1111 1112 if (attach_state & AHCI_ATTACH_STATE_FMA) { 1113 ahci_fm_fini(ahci_ctlp); 1114 } 1115 1116 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 1117 ddi_soft_state_free(ahci_statep, instance); 1118 } 1119 1120 return (DDI_FAILURE); 1121 } 1122 1123 /* 1124 * The detach entry point for dev_ops. 1125 */ 1126 static int 1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 1128 { 1129 ahci_ctl_t *ahci_ctlp; 1130 int instance; 1131 int ret; 1132 1133 instance = ddi_get_instance(dip); 1134 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1135 1136 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 1137 1138 switch (cmd) { 1139 case DDI_DETACH: 1140 1141 /* disable the interrupts for an uninterrupted detach */ 1142 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1143 ahci_disable_all_intrs(ahci_ctlp); 1144 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1145 1146 /* unregister from the sata framework. */ 1147 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 1148 if (ret != AHCI_SUCCESS) { 1149 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1150 ahci_enable_all_intrs(ahci_ctlp); 1151 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1152 return (DDI_FAILURE); 1153 } 1154 1155 ahci_em_fini(ahci_ctlp); 1156 1157 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1158 1159 /* stop the watchdog handler */ 1160 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1161 ahci_ctlp->ahcictl_timeout_id = 0; 1162 1163 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1164 1165 /* uninitialize the controller */ 1166 ahci_uninitialize_controller(ahci_ctlp); 1167 1168 /* remove the interrupts */ 1169 ahci_rem_intrs(ahci_ctlp); 1170 1171 /* deallocate the ports structures */ 1172 ahci_dealloc_ports_state(ahci_ctlp); 1173 1174 /* destroy mutex */ 1175 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1176 1177 /* teardown the pci config */ 1178 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1179 1180 /* remove the reg maps. */ 1181 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1182 1183 /* release fma resource */ 1184 ahci_fm_fini(ahci_ctlp); 1185 1186 /* free the soft state. */ 1187 ddi_soft_state_free(ahci_statep, instance); 1188 1189 return (DDI_SUCCESS); 1190 1191 case DDI_SUSPEND: 1192 1193 /* 1194 * The steps associated with suspension must include putting 1195 * the underlying device into a quiescent state so that it 1196 * will not generate interrupts or modify or access memory. 1197 */ 1198 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1199 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 1200 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1201 return (DDI_SUCCESS); 1202 } 1203 1204 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 1205 1206 /* stop the watchdog handler */ 1207 if (ahci_ctlp->ahcictl_timeout_id) { 1208 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1209 ahci_ctlp->ahcictl_timeout_id = 0; 1210 } 1211 1212 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1213 1214 ahci_em_suspend(ahci_ctlp); 1215 1216 /* 1217 * drain the taskq 1218 */ 1219 ahci_drain_ports_taskq(ahci_ctlp); 1220 1221 /* 1222 * Disable the interrupts and stop all the ports. 1223 */ 1224 ahci_uninitialize_controller(ahci_ctlp); 1225 1226 return (DDI_SUCCESS); 1227 1228 default: 1229 return (DDI_FAILURE); 1230 } 1231 } 1232 1233 /* 1234 * The info entry point for dev_ops. 1235 * 1236 */ 1237 static int 1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 1239 void *arg, void **result) 1240 { 1241 #ifndef __lock_lint 1242 _NOTE(ARGUNUSED(dip)) 1243 #endif /* __lock_lint */ 1244 1245 ahci_ctl_t *ahci_ctlp; 1246 int instance; 1247 dev_t dev; 1248 1249 dev = (dev_t)arg; 1250 instance = getminor(dev); 1251 1252 switch (infocmd) { 1253 case DDI_INFO_DEVT2DEVINFO: 1254 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1255 if (ahci_ctlp != NULL) { 1256 *result = ahci_ctlp->ahcictl_dip; 1257 return (DDI_SUCCESS); 1258 } else { 1259 *result = NULL; 1260 return (DDI_FAILURE); 1261 } 1262 case DDI_INFO_DEVT2INSTANCE: 1263 *(int *)result = instance; 1264 break; 1265 default: 1266 break; 1267 } 1268 1269 return (DDI_SUCCESS); 1270 } 1271 1272 /* 1273 * Registers the ahci with sata framework. 1274 */ 1275 static int 1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1277 { 1278 struct sata_hba_tran *sata_hba_tran; 1279 1280 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1281 "ahci_register_sata_hba_tran enter", NULL); 1282 1283 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1284 1285 /* Allocate memory for the sata_hba_tran */ 1286 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1287 1288 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV; 1289 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1290 sata_hba_tran->sata_tran_hba_dma_attr = 1291 &ahci_ctlp->ahcictl_buffer_dma_attr; 1292 1293 /* Report the number of implemented ports */ 1294 sata_hba_tran->sata_tran_hba_num_cports = 1295 ahci_ctlp->ahcictl_num_implemented_ports; 1296 1297 /* Support ATAPI device */ 1298 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1299 1300 /* Get the data transfer capability for PIO command by the HBA */ 1301 if (cap_status & AHCI_HBA_CAP_PMD) { 1302 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1303 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1304 "DRQ block data transfer for PIO command protocol", NULL); 1305 } 1306 1307 /* 1308 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1309 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1310 * QUEUED (EXT), and SERVICE commands). Queued operations are 1311 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1312 * QUEUED commands when the HBA and device support native command 1313 * queuing(NCQ). 1314 * 1315 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1316 * CAP register of the HBA indicates NCQ is supported. 1317 * 1318 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1319 * set because the previous register content of PxCI can be re-written 1320 * in the register write. 1321 */ 1322 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1323 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1324 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1325 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1326 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1327 "Command Queuing", NULL); 1328 } 1329 1330 /* Support port multiplier? */ 1331 if (cap_status & AHCI_HBA_CAP_SPM) { 1332 sata_hba_tran->sata_tran_hba_features_support |= 1333 SATA_CTLF_PORT_MULTIPLIER; 1334 1335 /* Support FIS-based switching for port multiplier? */ 1336 if (cap_status & AHCI_HBA_CAP_FBSS) { 1337 sata_hba_tran->sata_tran_hba_features_support |= 1338 SATA_CTLF_PMULT_FBS; 1339 } 1340 } 1341 1342 /* Report the number of command slots */ 1343 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1344 1345 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1346 sata_hba_tran->sata_tran_start = ahci_tran_start; 1347 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1348 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1349 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1350 #ifdef __lock_lint 1351 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1352 #endif 1353 /* 1354 * When SATA framework adds support for pwrmgt the 1355 * pwrmgt_ops needs to be updated 1356 */ 1357 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1358 sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl; 1359 1360 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1361 1362 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1363 1364 /* Attach it to SATA framework */ 1365 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1366 != DDI_SUCCESS) { 1367 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1368 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1369 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1370 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1371 return (AHCI_FAILURE); 1372 } 1373 1374 return (AHCI_SUCCESS); 1375 } 1376 1377 /* 1378 * Unregisters the ahci with sata framework. 1379 */ 1380 static int 1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1382 { 1383 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1384 "ahci_unregister_sata_hba_tran enter", NULL); 1385 1386 /* Detach from the SATA framework. */ 1387 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1388 DDI_SUCCESS) { 1389 return (AHCI_FAILURE); 1390 } 1391 1392 /* Deallocate sata_hba_tran. */ 1393 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1394 sizeof (sata_hba_tran_t)); 1395 1396 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1397 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1398 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1399 1400 return (AHCI_SUCCESS); 1401 } 1402 1403 #define SET_PORTSTR(str, addrp) \ 1404 if (AHCI_ADDR_IS_PORT(addrp)) \ 1405 (void) sprintf((str), "%d", (addrp)->aa_port); \ 1406 else if (AHCI_ADDR_IS_PMULT(addrp)) \ 1407 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \ 1408 else \ 1409 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \ 1410 (addrp)->aa_pmport); 1411 1412 /* 1413 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1414 * port status registers and an attached device type via sata_device 1415 * structure. 1416 * 1417 * We return the cached information from a previous hardware probe. The 1418 * actual hardware probing itself was done either from within 1419 * ahci_initialize_controller() during the driver attach or from a phy 1420 * ready change interrupt handler. 1421 */ 1422 static int 1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1424 { 1425 ahci_ctl_t *ahci_ctlp; 1426 ahci_port_t *ahci_portp; 1427 ahci_addr_t addr, pmult_addr; 1428 uint8_t cport = sd->satadev_addr.cport; 1429 char portstr[10]; 1430 uint8_t device_type; 1431 uint32_t port_state; 1432 uint8_t port; 1433 int rval = SATA_SUCCESS, rval_init; 1434 1435 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1436 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1437 1438 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1439 1440 mutex_enter(&ahci_portp->ahciport_mutex); 1441 1442 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 1443 ASSERT(AHCI_ADDR_IS_VALID(&addr)); 1444 SET_PORTSTR(portstr, &addr); 1445 1446 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1447 "ahci_tran_probe_port enter: port %s", portstr); 1448 1449 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) && 1450 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1451 ahci_portp->ahciport_pmult_info == NULL)) { 1452 /* port mutliplier is removed. */ 1453 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1454 "ahci_tran_probe_port: " 1455 "pmult is removed from port %s", portstr); 1456 mutex_exit(&ahci_portp->ahciport_mutex); 1457 return (SATA_FAILURE); 1458 } 1459 1460 /* 1461 * The sata_device may refer to 1462 * 1. A controller port. 1463 * A controller port should be ready here. 1464 * 2. A port multiplier. 1465 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize 1466 * it and register the port multiplier to the framework. 1467 * SATA_ADDR_PMULT - check the status of all its device ports. 1468 * 3. A port multiplier port. 1469 * If it has not been initialized, initialized it. 1470 * 1471 * A port multiplier or a port multiplier port may require some 1472 * initialization because we cannot do these time-consuming jobs in an 1473 * interrupt context. 1474 */ 1475 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) { 1476 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 1477 /* Initialize registers on a port multiplier */ 1478 rval_init = ahci_initialize_pmult(ahci_ctlp, 1479 ahci_portp, &pmult_addr, sd); 1480 if (rval_init != AHCI_SUCCESS) { 1481 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1482 "ahci_tran_probe_port: " 1483 "pmult initialization failed.", NULL); 1484 mutex_exit(&ahci_portp->ahciport_mutex); 1485 return (SATA_FAILURE); 1486 } 1487 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) { 1488 /* Check pmports hotplug events */ 1489 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr); 1490 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT | 1491 SATA_ADDR_DPMPORT)) { 1492 if (ahci_probe_pmport(ahci_ctlp, ahci_portp, 1493 &addr, sd) != AHCI_SUCCESS) { 1494 rval = SATA_FAILURE; 1495 goto out; 1496 } 1497 } 1498 1499 /* Update port state and device type */ 1500 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr); 1501 1502 switch (port_state) { 1503 1504 case SATA_PSTATE_FAILED: 1505 sd->satadev_state = SATA_PSTATE_FAILED; 1506 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1507 "ahci_tran_probe_port: port %s PORT FAILED", portstr); 1508 goto out; 1509 1510 case SATA_PSTATE_SHUTDOWN: 1511 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1512 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1513 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr); 1514 goto out; 1515 1516 case SATA_PSTATE_PWROFF: 1517 sd->satadev_state = SATA_PSTATE_PWROFF; 1518 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1519 "ahci_tran_probe_port: port %s PORT PWROFF", portstr); 1520 goto out; 1521 1522 case SATA_PSTATE_PWRON: 1523 sd->satadev_state = SATA_PSTATE_PWRON; 1524 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1525 "ahci_tran_probe_port: port %s PORT PWRON", portstr); 1526 break; 1527 1528 default: 1529 sd->satadev_state = port_state; 1530 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1531 "ahci_tran_probe_port: port %s PORT NORMAL %x", 1532 portstr, port_state); 1533 break; 1534 } 1535 1536 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1537 1538 switch (device_type) { 1539 1540 case SATA_DTYPE_ATADISK: 1541 sd->satadev_type = SATA_DTYPE_ATADISK; 1542 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1543 "ahci_tran_probe_port: port %s DISK found", portstr); 1544 break; 1545 1546 case SATA_DTYPE_ATAPI: 1547 /* 1548 * HBA driver only knows it's an ATAPI device, and don't know 1549 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1550 * type need to be determined by checking IDENTIFY PACKET 1551 * DEVICE data 1552 */ 1553 sd->satadev_type = SATA_DTYPE_ATAPI; 1554 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1555 "ahci_tran_probe_port: port %s ATAPI found", portstr); 1556 break; 1557 1558 case SATA_DTYPE_PMULT: 1559 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)); 1560 sd->satadev_type = SATA_DTYPE_PMULT; 1561 1562 /* Update the number of pmports. */ 1563 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1564 sd->satadev_add_info = ahci_portp-> 1565 ahciport_pmult_info->ahcipmi_num_dev_ports; 1566 1567 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1568 "ahci_tran_probe_port: port %s Port Multiplier found", 1569 portstr); 1570 break; 1571 1572 case SATA_DTYPE_UNKNOWN: 1573 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1574 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1575 "ahci_tran_probe_port: port %s Unknown device found", 1576 portstr); 1577 break; 1578 1579 default: 1580 /* we don't support any other device types */ 1581 sd->satadev_type = SATA_DTYPE_NONE; 1582 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1583 "ahci_tran_probe_port: port %s No device found", portstr); 1584 break; 1585 } 1586 1587 out: 1588 /* Register update only fails while probing a pmult/pmport */ 1589 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) { 1590 ahci_update_sata_registers(ahci_ctlp, port, sd); 1591 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 1592 if (port_state & SATA_STATE_READY) 1593 if (ahci_update_pmult_pscr(ahci_ctlp, 1594 &addr, sd) != AHCI_SUCCESS) 1595 rval = SATA_FAILURE; 1596 } 1597 1598 /* Check handles for the sata registers access */ 1599 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) || 1600 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) { 1601 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 1602 DDI_SERVICE_UNAFFECTED); 1603 rval = SATA_FAILURE; 1604 } 1605 1606 mutex_exit(&ahci_portp->ahciport_mutex); 1607 return (rval); 1608 } 1609 1610 /* 1611 * There are four operation modes in sata framework: 1612 * SATA_OPMODE_INTERRUPTS 1613 * SATA_OPMODE_POLLING 1614 * SATA_OPMODE_ASYNCH 1615 * SATA_OPMODE_SYNCH 1616 * 1617 * Their combined meanings as following: 1618 * 1619 * SATA_OPMODE_SYNCH 1620 * The command has to be completed before sata_tran_start functions returns. 1621 * Either interrupts or polling could be used - it's up to the driver. 1622 * Mode used currently for internal, sata-module initiated operations. 1623 * 1624 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1625 * It is the same as the one above. 1626 * 1627 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1628 * The command has to be completed before sata_tran_start function returns. 1629 * No interrupt used, polling only. This should be the mode used for scsi 1630 * packets with FLAG_NOINTR. 1631 * 1632 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1633 * The command may be queued (callback function specified). Interrupts could 1634 * be used. It's normal operation mode. 1635 */ 1636 /* 1637 * Called by sata framework to transport a sata packet down stream. 1638 */ 1639 static int 1640 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1641 { 1642 ahci_ctl_t *ahci_ctlp; 1643 ahci_port_t *ahci_portp; 1644 ahci_addr_t addr; 1645 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1646 uint8_t port; 1647 char portstr[10]; 1648 1649 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1650 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1651 1652 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1653 "ahci_tran_start enter: cport %d satapkt 0x%p", 1654 cport, (void *)spkt); 1655 1656 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1657 1658 mutex_enter(&ahci_portp->ahciport_mutex); 1659 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr); 1660 SET_PORTSTR(portstr, &addr); 1661 1662 /* Sanity check */ 1663 if (AHCI_ADDR_IS_PMPORT(&addr)) { 1664 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1665 ahci_portp->ahciport_pmult_info == NULL) { 1666 1667 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1668 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1669 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN; 1670 ahci_update_sata_registers(ahci_ctlp, port, 1671 &spkt->satapkt_device); 1672 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1673 "ahci_tran_start returning PORT_ERROR while " 1674 "pmult removed: port: %s", portstr); 1675 mutex_exit(&ahci_portp->ahciport_mutex); 1676 return (SATA_TRAN_PORT_ERROR); 1677 } 1678 1679 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) & 1680 SATA_STATE_READY)) { 1681 if (!ddi_in_panic() || 1682 ahci_initialize_pmport(ahci_ctlp, 1683 ahci_portp, &addr) != AHCI_SUCCESS) { 1684 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1685 spkt->satapkt_device.satadev_type = 1686 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1687 spkt->satapkt_device.satadev_state = 1688 AHCIPORT_GET_STATE(ahci_portp, &addr); 1689 ahci_update_sata_registers(ahci_ctlp, port, 1690 &spkt->satapkt_device); 1691 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1692 "ahci_tran_start returning PORT_ERROR " 1693 "while sub-link is not initialized " 1694 "at port: %s", portstr); 1695 mutex_exit(&ahci_portp->ahciport_mutex); 1696 return (SATA_TRAN_PORT_ERROR); 1697 } 1698 } 1699 } 1700 1701 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED || 1702 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN|| 1703 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) { 1704 /* 1705 * In case the target driver would send the packet before 1706 * sata framework can have the opportunity to process those 1707 * event reports. 1708 */ 1709 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1710 spkt->satapkt_device.satadev_state = 1711 ahci_portp->ahciport_port_state; 1712 ahci_update_sata_registers(ahci_ctlp, port, 1713 &spkt->satapkt_device); 1714 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1715 "ahci_tran_start returning PORT_ERROR while " 1716 "port in FAILED/SHUTDOWN/PWROFF state: " 1717 "port: %s", portstr); 1718 mutex_exit(&ahci_portp->ahciport_mutex); 1719 return (SATA_TRAN_PORT_ERROR); 1720 } 1721 1722 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) { 1723 /* 1724 * ahci_intr_phyrdy_change() may have rendered it to 1725 * SATA_DTYPE_NONE. 1726 */ 1727 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1728 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1729 spkt->satapkt_device.satadev_state = 1730 ahci_portp->ahciport_port_state; 1731 ahci_update_sata_registers(ahci_ctlp, port, 1732 &spkt->satapkt_device); 1733 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1734 "ahci_tran_start returning PORT_ERROR while " 1735 "no device attached: port: %s", portstr); 1736 mutex_exit(&ahci_portp->ahciport_mutex); 1737 return (SATA_TRAN_PORT_ERROR); 1738 } 1739 1740 /* R/W PMULT command will occupy the whole HBA port */ 1741 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1742 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1743 "ahci_tran_start returning BUSY while " 1744 "executing READ/WRITE PORT-MULT command: " 1745 "port: %s", portstr); 1746 spkt->satapkt_reason = SATA_PKT_BUSY; 1747 mutex_exit(&ahci_portp->ahciport_mutex); 1748 return (SATA_TRAN_BUSY); 1749 } 1750 1751 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 1752 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1753 "ahci_tran_start returning BUSY while " 1754 "hot-plug in progress: port: %s", portstr); 1755 spkt->satapkt_reason = SATA_PKT_BUSY; 1756 mutex_exit(&ahci_portp->ahciport_mutex); 1757 return (SATA_TRAN_BUSY); 1758 } 1759 1760 /* 1761 * SATA HBA driver should remember that a device was reset and it 1762 * is supposed to reject any packets which do not specify either 1763 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1764 * 1765 * This is to prevent a race condition when a device was arbitrarily 1766 * reset by the HBA driver (and lost it's setting) and a target 1767 * driver sending some commands to a device before the sata framework 1768 * has a chance to restore the device setting (such as cache enable/ 1769 * disable or other resettable stuff). 1770 */ 1771 /* 1772 * It is unnecessary to use specific flags to indicate 1773 * reset_in_progress for a pmport. While mopping, all command will be 1774 * mopped so that the entire HBA port is being dealt as a single 1775 * object. 1776 */ 1777 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1778 ahci_portp->ahciport_reset_in_progress = 0; 1779 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1780 "ahci_tran_start [CLEAR] the " 1781 "reset_in_progress for port: %d", port); 1782 } 1783 1784 if (ahci_portp->ahciport_reset_in_progress && 1785 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1786 ! ddi_in_panic()) { 1787 spkt->satapkt_reason = SATA_PKT_BUSY; 1788 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1789 "ahci_tran_start returning BUSY while " 1790 "reset in progress: port: %d", port); 1791 mutex_exit(&ahci_portp->ahciport_mutex); 1792 return (SATA_TRAN_BUSY); 1793 } 1794 1795 #ifdef AHCI_DEBUG 1796 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) { 1797 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1798 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d", 1799 spkt, port); 1800 } 1801 #endif 1802 1803 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1804 spkt->satapkt_reason = SATA_PKT_BUSY; 1805 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1806 "ahci_tran_start returning BUSY while " 1807 "mopping in progress: port: %d", port); 1808 mutex_exit(&ahci_portp->ahciport_mutex); 1809 return (SATA_TRAN_BUSY); 1810 } 1811 1812 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 1813 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 1814 DDI_SERVICE_UNAFFECTED); 1815 mutex_exit(&ahci_portp->ahciport_mutex); 1816 return (SATA_TRAN_BUSY); 1817 } 1818 1819 if (spkt->satapkt_op_mode & 1820 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1821 /* 1822 * If a SYNC command to be executed in interrupt context, 1823 * bounce it back to sata module. 1824 */ 1825 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1826 servicing_interrupt()) { 1827 spkt->satapkt_reason = SATA_PKT_BUSY; 1828 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1829 "ahci_tran_start returning BUSY while " 1830 "sending SYNC mode under interrupt context: " 1831 "port : %d", port); 1832 mutex_exit(&ahci_portp->ahciport_mutex); 1833 return (SATA_TRAN_BUSY); 1834 } 1835 1836 /* We need to do the sync start now */ 1837 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, 1838 spkt) == AHCI_FAILURE) { 1839 goto fail_out; 1840 } 1841 } else { 1842 /* Async start, using interrupt */ 1843 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt) 1844 == AHCI_FAILURE) { 1845 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1846 goto fail_out; 1847 } 1848 } 1849 1850 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1851 "sata tran accepted: port %s", portstr); 1852 1853 mutex_exit(&ahci_portp->ahciport_mutex); 1854 return (SATA_TRAN_ACCEPTED); 1855 1856 fail_out: 1857 /* 1858 * Failed to deliver packet to the controller. 1859 * Check if it's caused by invalid handles. 1860 */ 1861 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS || 1862 ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) { 1863 spkt->satapkt_device.satadev_type = 1864 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1865 spkt->satapkt_device.satadev_state = 1866 AHCIPORT_GET_STATE(ahci_portp, &addr); 1867 spkt->satapkt_reason = SATA_PKT_DEV_ERROR; 1868 mutex_exit(&ahci_portp->ahciport_mutex); 1869 return (SATA_TRAN_PORT_ERROR); 1870 } 1871 1872 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1873 "return QUEUE_FULL: port %d", port); 1874 mutex_exit(&ahci_portp->ahciport_mutex); 1875 return (SATA_TRAN_QUEUE_FULL); 1876 } 1877 1878 /* 1879 * SATA_OPMODE_SYNCH flag is set 1880 * 1881 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1882 * without interrupt, otherwise we can still use the interrupt. 1883 */ 1884 static int 1885 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1886 ahci_addr_t *addrp, sata_pkt_t *spkt) 1887 { 1888 int pkt_timeout_ticks; 1889 uint32_t timeout_tags; 1890 int rval; 1891 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1892 uint8_t port = addrp->aa_port; 1893 1894 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 1895 1896 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1897 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt); 1898 1899 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1900 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1901 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1902 addrp, spkt)) == AHCI_FAILURE) { 1903 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1904 return (rval); 1905 } 1906 1907 pkt_timeout_ticks = 1908 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1909 1910 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1911 /* Simulate the interrupt */ 1912 mutex_exit(&ahci_portp->ahciport_mutex); 1913 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1914 mutex_enter(&ahci_portp->ahciport_mutex); 1915 1916 if (spkt->satapkt_reason != SATA_PKT_BUSY) 1917 break; 1918 1919 mutex_exit(&ahci_portp->ahciport_mutex); 1920 drv_usecwait(AHCI_1MS_USECS); 1921 mutex_enter(&ahci_portp->ahciport_mutex); 1922 1923 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1924 if (pkt_timeout_ticks < 0) { 1925 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1926 "port %d satapkt 0x%p timed out\n", 1927 instance, port, (void *)spkt); 1928 timeout_tags = (0x1 << rval); 1929 mutex_exit(&ahci_portp->ahciport_mutex); 1930 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1931 port, timeout_tags); 1932 mutex_enter(&ahci_portp->ahciport_mutex); 1933 } 1934 } 1935 1936 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1937 return (AHCI_SUCCESS); 1938 1939 } else { 1940 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1941 addrp, spkt)) == AHCI_FAILURE) 1942 return (rval); 1943 1944 #if AHCI_DEBUG 1945 /* 1946 * Note that the driver always uses the slot 0 to deliver 1947 * REQUEST SENSE or READ LOG EXT command 1948 */ 1949 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1950 ASSERT(rval == 0); 1951 #endif 1952 1953 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1954 cv_wait(&ahci_portp->ahciport_cv, 1955 &ahci_portp->ahciport_mutex); 1956 1957 return (AHCI_SUCCESS); 1958 } 1959 } 1960 1961 /* 1962 * Searches for and claims a free command slot. 1963 * 1964 * Returns value: 1965 * 1966 * AHCI_FAILURE returned only if 1967 * 1. No empty slot left 1968 * 2. Non-queued command requested while queued command(s) is outstanding 1969 * 3. Queued command requested while non-queued command(s) is outstanding 1970 * 4. HBA doesn't support multiple-use of command list while already a 1971 * non-queued command is oustanding 1972 * 5. Queued command requested while some queued command(s) has been 1973 * outstanding on a different port multiplier port. (AHCI spec 1.2, 1974 * 9.1.2) 1975 * 1976 * claimed slot number returned if succeeded 1977 * 1978 * NOTE: it will always return slot 0 for following commands to simplify the 1979 * algorithm. 1980 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process 1981 * 2. READ/WRITE PORTMULT command 1982 */ 1983 static int 1984 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1985 ahci_addr_t *addrp, int command_type) 1986 { 1987 uint32_t port_cmd_issue; 1988 uint32_t free_slots; 1989 int slot; 1990 1991 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 1992 1993 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1994 "ahciport_pending_tags = 0x%x " 1995 "ahciport_pending_ncq_tags = 0x%x", 1996 ahci_portp->ahciport_pending_tags, 1997 ahci_portp->ahciport_pending_ncq_tags); 1998 1999 /* 2000 * According to the AHCI spec, system software is responsible to 2001 * ensure that queued and non-queued commands are not mixed in 2002 * the command list. 2003 */ 2004 if (command_type == AHCI_NON_NCQ_CMD) { 2005 /* Non-NCQ command request */ 2006 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2007 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 2008 "ahci_claim_free_slot: there is still pending " 2009 "queued command(s) in the command list, " 2010 "so no available slot for the non-queued " 2011 "command", NULL); 2012 return (AHCI_FAILURE); 2013 } 2014 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 2015 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 2016 "ahci_claim_free_slot: there is still pending " 2017 "read/write port-mult command(s) in command list, " 2018 "so no available slot for the non-queued command", 2019 NULL); 2020 return (AHCI_FAILURE); 2021 } 2022 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 2023 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2024 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2025 "ahci_claim_free_slot: HBA cannot support multiple-" 2026 "use of the command list for non-queued commands", 2027 NULL); 2028 return (AHCI_FAILURE); 2029 } 2030 free_slots = (~ahci_portp->ahciport_pending_tags) & 2031 AHCI_SLOT_MASK(ahci_ctlp); 2032 } else if (command_type == AHCI_NCQ_CMD) { 2033 /* NCQ command request */ 2034 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2035 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 2036 "ahci_claim_free_slot: there is still pending " 2037 "non-queued command(s) in the command list, " 2038 "so no available slot for the queued command", 2039 NULL); 2040 return (AHCI_FAILURE); 2041 } 2042 2043 /* 2044 * NCQ commands cannot be sent to different port multiplier 2045 * ports in Command-Based Switching mode 2046 */ 2047 /* 2048 * NOTE: In Command-Based Switching mode, AHCI controller 2049 * usually reports a 'Handshake Error' when multiple NCQ 2050 * commands are outstanding simultaneously. 2051 */ 2052 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 2053 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 2054 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) && 2055 NCQ_CMD_IN_PROGRESS(ahci_portp) && 2056 AHCIPORT_NCQ_PMPORT(ahci_portp) != 2057 addrp->aa_pmport) { 2058 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2059 "ahci_claim_free_slot: there is still " 2060 "pending queued command(s) in the " 2061 "command list for another Port Multiplier " 2062 "port, so no available slot.", NULL); 2063 return (AHCI_FAILURE); 2064 } 2065 } 2066 2067 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 2068 AHCI_NCQ_SLOT_MASK(ahci_portp); 2069 } else if (command_type == AHCI_ERR_RETRI_CMD) { 2070 /* Error retrieval command request */ 2071 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2072 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 2073 "SENSE or READ LOG EXT command", NULL); 2074 slot = 0; 2075 goto out; 2076 } else if (command_type == AHCI_RDWR_PMULT_CMD) { 2077 /* 2078 * An extra check on PxCI. Sometimes PxCI bits may not be 2079 * cleared during hot-plug or error recovery process. 2080 */ 2081 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2082 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port)); 2083 2084 if (port_cmd_issue != 0) { 2085 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 2086 "ahci_claim_free_slot: there is still pending " 2087 "command(s) in command list (0x%x/0x%x, PxCI %x)," 2088 "so no available slot for R/W PMULT command.", 2089 NON_NCQ_CMD_IN_PROGRESS(ahci_portp), 2090 NCQ_CMD_IN_PROGRESS(ahci_portp), 2091 port_cmd_issue); 2092 return (AHCI_FAILURE); 2093 } 2094 2095 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2096 "ahci_claim_free_slot: slot 0 is allocated for " 2097 "READ/WRITE PORTMULT command", NULL); 2098 slot = 0; 2099 goto out; 2100 } 2101 2102 slot = ddi_ffs(free_slots) - 1; 2103 if (slot == -1) { 2104 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 2105 "ahci_claim_free_slot: no empty slots", NULL); 2106 return (AHCI_FAILURE); 2107 } 2108 2109 /* 2110 * According to the AHCI spec, to allow a simple mechanism for the 2111 * HBA to map command list slots to queue entries, software must 2112 * match the tag number it uses to the slot it is placing the command 2113 * in. For example, if a queued command is placed in slot 5, the tag 2114 * for that command must be 5. 2115 */ 2116 if (command_type == AHCI_NCQ_CMD) { 2117 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 2118 if (AHCI_ADDR_IS_PMPORT(addrp)) { 2119 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 2120 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport; 2121 } 2122 } 2123 2124 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 2125 2126 out: 2127 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 2128 "ahci_claim_free_slot: found slot: 0x%x", slot); 2129 2130 return (slot); 2131 } 2132 2133 /* 2134 * Builds the Command Table for the sata packet and delivers it to controller. 2135 * 2136 * Returns: 2137 * slot number if we can obtain a slot successfully 2138 * otherwise, return AHCI_FAILURE 2139 */ 2140 static int 2141 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 2142 ahci_addr_t *addrp, sata_pkt_t *spkt) 2143 { 2144 int cmd_slot; 2145 sata_cmd_t *scmd; 2146 ahci_fis_h2d_register_t *h2d_register_fisp; 2147 ahci_cmd_table_t *cmd_table; 2148 ahci_cmd_header_t *cmd_header; 2149 int ncookies; 2150 int i; 2151 int command_type = AHCI_NON_NCQ_CMD; 2152 int ncq_qdepth; 2153 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 2154 uint8_t port, pmport; 2155 #if AHCI_DEBUG 2156 uint32_t *ptr; 2157 uint8_t *ptr2; 2158 #endif 2159 2160 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2161 2162 port = addrp->aa_port; 2163 pmport = addrp->aa_pmport; 2164 2165 spkt->satapkt_reason = SATA_PKT_BUSY; 2166 2167 scmd = &spkt->satapkt_cmd; 2168 2169 /* Check if the command is a NCQ command */ 2170 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 2171 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 2172 command_type = AHCI_NCQ_CMD; 2173 2174 /* 2175 * When NCQ is support, system software must determine the 2176 * maximum tag allowed by the device and the HBA, and it 2177 * must use a value not beyond of the lower bound of the two. 2178 * 2179 * Sata module is going to calculate the qdepth and send 2180 * down to HBA driver via sata_cmd. 2181 */ 2182 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 2183 2184 /* 2185 * At the moment, the driver doesn't support the dynamic 2186 * setting of the maximum ncq depth, and the value can be 2187 * set either during the attach or after hot-plug insertion. 2188 */ 2189 if (ahci_portp->ahciport_max_ncq_tags == 0) { 2190 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 2191 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 2192 "ahci_deliver_satapkt: port %d the max tags for " 2193 "NCQ command is %d", port, ncq_qdepth); 2194 } else { 2195 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 2196 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 2197 " port %d the max tag for NCQ command is " 2198 "requested to change from %d to %d, at the" 2199 " moment the driver doesn't support the " 2200 "dynamic change so it's going to " 2201 "still use the previous tag value", 2202 instance, port, 2203 ahci_portp->ahciport_max_ncq_tags, 2204 ncq_qdepth); 2205 } 2206 } 2207 } 2208 2209 /* Check if the command is an error retrieval command */ 2210 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 2211 command_type = AHCI_ERR_RETRI_CMD; 2212 2213 /* Check if the command is an read/write pmult command */ 2214 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2215 command_type = AHCI_RDWR_PMULT_CMD; 2216 2217 /* Check if there is an empty command slot */ 2218 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, 2219 addrp, command_type); 2220 if (cmd_slot == AHCI_FAILURE) { 2221 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 2222 return (AHCI_FAILURE); 2223 } 2224 2225 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 2226 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 2227 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 2228 cmd_slot, port, (void *)spkt); 2229 2230 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 2231 bzero((void *)cmd_table, ahci_cmd_table_size); 2232 2233 /* For data transfer operations, it is the H2D Register FIS */ 2234 h2d_register_fisp = 2235 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 2236 2237 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 2238 2239 /* 2240 * PMP field only make sense when target is a port multiplier or a 2241 * device behind a port multiplier. Otherwise should set it to 0. 2242 */ 2243 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2244 SET_FIS_PMP(h2d_register_fisp, pmport); 2245 2246 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 2247 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 2248 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 2249 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 2250 2251 switch (scmd->satacmd_addr_type) { 2252 2253 case 0: 2254 /* 2255 * satacmd_addr_type will be 0 for the commands below: 2256 * ATAPI command 2257 * SATAC_IDLE_IM 2258 * SATAC_STANDBY_IM 2259 * SATAC_DOWNLOAD_MICROCODE 2260 * SATAC_FLUSH_CACHE 2261 * SATAC_SET_FEATURES 2262 * SATAC_SMART 2263 * SATAC_ID_PACKET_DEVICE 2264 * SATAC_ID_DEVICE 2265 * SATAC_READ_PORTMULT 2266 * SATAC_WRITE_PORTMULT 2267 */ 2268 /* FALLTHRU */ 2269 2270 case ATA_ADDR_LBA: 2271 /* FALLTHRU */ 2272 2273 case ATA_ADDR_LBA28: 2274 /* LBA[7:0] */ 2275 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2276 2277 /* LBA[15:8] */ 2278 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2279 2280 /* LBA[23:16] */ 2281 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2282 2283 /* LBA [27:24] (also called dev_head) */ 2284 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 2285 2286 break; 2287 2288 case ATA_ADDR_LBA48: 2289 /* LBA[7:0] */ 2290 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2291 2292 /* LBA[15:8] */ 2293 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2294 2295 /* LBA[23:16] */ 2296 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2297 2298 /* LBA [31:24] */ 2299 SET_FIS_SECTOR_EXP(h2d_register_fisp, 2300 scmd->satacmd_lba_low_msb); 2301 2302 /* LBA [39:32] */ 2303 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 2304 scmd->satacmd_lba_mid_msb); 2305 2306 /* LBA [47:40] */ 2307 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 2308 scmd->satacmd_lba_high_msb); 2309 2310 /* Set dev_head */ 2311 SET_FIS_DEV_HEAD(h2d_register_fisp, 2312 scmd->satacmd_device_reg); 2313 2314 /* Set the extended sector count and features */ 2315 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 2316 scmd->satacmd_sec_count_msb); 2317 SET_FIS_FEATURES_EXP(h2d_register_fisp, 2318 scmd->satacmd_features_reg_ext); 2319 break; 2320 } 2321 2322 /* 2323 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 2324 * filled into features field, and sector count 8:15 is filled into 2325 * features (exp) field. The hba driver doesn't need to anything 2326 * special with regard to this, since sata framework has already 2327 * done so. 2328 * 2329 * However the driver needs to make sure TAG is filled into sector 2330 * field. 2331 */ 2332 if (command_type == AHCI_NCQ_CMD) { 2333 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 2334 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 2335 } 2336 2337 ncookies = scmd->satacmd_num_dma_cookies; 2338 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2339 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 2340 ncookies, ahci_dma_prdt_number); 2341 2342 ASSERT(ncookies <= ahci_dma_prdt_number); 2343 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 2344 2345 /* *** now fill the scatter gather list ******* */ 2346 for (i = 0; i < ncookies; i++) { 2347 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 2348 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 2349 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 2350 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 2351 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 2352 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 2353 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 2354 scmd->satacmd_dma_cookie_list[i].dmac_size; 2355 } 2356 2357 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2358 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 2359 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 2360 2361 /* The ACMD field is filled in for ATAPI command */ 2362 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 2363 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 2364 SATA_ATAPI_MAX_CDB_LEN); 2365 } 2366 2367 /* Set Command Header in Command List */ 2368 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 2369 BZERO_DESCR_INFO(cmd_header); 2370 BZERO_PRD_BYTE_COUNT(cmd_header); 2371 2372 /* Set the number of entries in the PRD table */ 2373 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 2374 2375 /* Set the length of the command in the CFIS area */ 2376 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 2377 2378 /* 2379 * PMP field only make sense when target is a port multiplier or a 2380 * device behind a port multiplier. Otherwise should set it to 0. 2381 */ 2382 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2383 SET_PORT_MULTI_PORT(cmd_header, pmport); 2384 2385 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 2386 "sata_data_direction = 0x%x", 2387 scmd->satacmd_flags.sata_data_direction); 2388 2389 /* Set A bit if it is an ATAPI command */ 2390 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 2391 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 2392 2393 /* Set W bit if data is going to the device */ 2394 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 2395 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 2396 2397 /* 2398 * Set the prefetchable bit - this bit is only valid if the PRDTL 2399 * field is non-zero or the ATAPI 'A' bit is set in the command 2400 * header. This bit cannot be set when using native command 2401 * queuing commands or when using FIS-based switching with a Port 2402 * multiplier. 2403 */ 2404 if (command_type != AHCI_NCQ_CMD) 2405 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 2406 2407 /* 2408 * Now remember the sata packet in ahciport_slot_pkts[]. 2409 * Error retrieval command and r/w port multiplier command will 2410 * be stored specifically for each port. 2411 */ 2412 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 2413 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2414 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 2415 2416 /* 2417 * Keep the timeout value 2418 */ 2419 ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time; 2420 2421 /* 2422 * If the intial timout is less than 1 tick, then make it longer by 2423 * 1 tick to avoid immediate timeout 2424 */ 2425 if (ahci_portp->ahciport_slot_timeout[cmd_slot] <= 2426 ahci_watchdog_timeout) 2427 ahci_portp->ahciport_slot_timeout[cmd_slot] += 2428 ahci_watchdog_timeout; 2429 2430 #if AHCI_DEBUG 2431 if (ahci_debug_flags & AHCIDBG_ATACMD && 2432 scmd->satacmd_cmd_reg != SATAC_PACKET || 2433 ahci_debug_flags & AHCIDBG_ATAPICMD && 2434 scmd->satacmd_cmd_reg == SATAC_PACKET) { 2435 2436 /* Dump the command header and table */ 2437 ahci_log(ahci_ctlp, CE_WARN, "\n"); 2438 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 2439 "0x%p cmd_reg 0x%x port %d", spkt, 2440 scmd->satacmd_cmd_reg, port); 2441 ptr = (uint32_t *)cmd_header; 2442 ahci_log(ahci_ctlp, CE_WARN, 2443 " Command Header:%8x %8x %8x %8x", 2444 ptr[0], ptr[1], ptr[2], ptr[3]); 2445 2446 /* Dump the H2D register FIS */ 2447 ptr = (uint32_t *)h2d_register_fisp; 2448 ahci_log(ahci_ctlp, CE_WARN, 2449 " Command FIS: %8x %8x %8x %8x", 2450 ptr[0], ptr[1], ptr[2], ptr[3]); 2451 2452 /* Dump the ACMD register FIS */ 2453 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 2454 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 2455 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 2456 ahci_log(ahci_ctlp, CE_WARN, 2457 " ATAPI command: %2x %2x %2x %2x " 2458 "%2x %2x %2x %2x", 2459 ptr2[8 * i], ptr2[8 * i + 1], 2460 ptr2[8 * i + 2], ptr2[8 * i + 3], 2461 ptr2[8 * i + 4], ptr2[8 * i + 5], 2462 ptr2[8 * i + 6], ptr2[8 * i + 7]); 2463 2464 /* Dump the PRDT */ 2465 for (i = 0; i < ncookies; i++) { 2466 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 2467 ahci_log(ahci_ctlp, CE_WARN, 2468 " Cookie %d: %8x %8x %8x %8x", 2469 i, ptr[0], ptr[1], ptr[2], ptr[3]); 2470 } 2471 } 2472 #endif 2473 2474 (void) ddi_dma_sync( 2475 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 2476 0, 2477 ahci_cmd_table_size, 2478 DDI_DMA_SYNC_FORDEV); 2479 2480 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 2481 cmd_slot * sizeof (ahci_cmd_header_t), 2482 sizeof (ahci_cmd_header_t), 2483 DDI_DMA_SYNC_FORDEV); 2484 2485 if ((ahci_check_dma_handle(ahci_portp-> 2486 ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) || 2487 ahci_check_dma_handle(ahci_portp-> 2488 ahciport_cmd_list_dma_handle) != DDI_FM_OK) { 2489 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 2490 DDI_SERVICE_UNAFFECTED); 2491 return (AHCI_FAILURE); 2492 } 2493 2494 /* Set the corresponding bit in the PxSACT.DS for queued command */ 2495 if (command_type == AHCI_NCQ_CMD) { 2496 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2497 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 2498 (0x1 << cmd_slot)); 2499 } 2500 2501 /* Indicate to the HBA that a command is active. */ 2502 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2503 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 2504 (0x1 << cmd_slot)); 2505 2506 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 2507 "exit: port %d", port); 2508 2509 /* Make sure the command is started by the PxSACT/PxCI */ 2510 if (ahci_check_acc_handle(ahci_ctlp-> 2511 ahcictl_ahci_acc_handle) != DDI_FM_OK) { 2512 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 2513 DDI_SERVICE_UNAFFECTED); 2514 return (AHCI_FAILURE); 2515 } 2516 2517 return (cmd_slot); 2518 } 2519 2520 /* 2521 * Called by the sata framework to abort the previously sent packet(s). 2522 * 2523 * Reset device to abort commands. 2524 */ 2525 static int 2526 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 2527 { 2528 ahci_ctl_t *ahci_ctlp; 2529 ahci_port_t *ahci_portp; 2530 uint32_t slot_status = 0; 2531 uint32_t aborted_tags = 0; 2532 uint32_t finished_tags = 0; 2533 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 2534 uint8_t port; 2535 int tmp_slot; 2536 int instance = ddi_get_instance(dip); 2537 2538 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2539 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2540 2541 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2542 "ahci_tran_abort enter: port %d", port); 2543 2544 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2545 mutex_enter(&ahci_portp->ahciport_mutex); 2546 2547 /* 2548 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2549 * commands are being mopped, therefore there is nothing else to do 2550 */ 2551 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2552 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2553 "ahci_tran_abort: port %d is in " 2554 "mopping process, so just return directly ", port); 2555 mutex_exit(&ahci_portp->ahciport_mutex); 2556 return (SATA_SUCCESS); 2557 } 2558 2559 /* 2560 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT 2561 * command is being executed so no other commands is outstanding, 2562 * nothing to do. 2563 */ 2564 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) { 2565 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2566 "ahci_tran_abort: port %d is reading/writing " 2567 "port multiplier, so just return directly ", port); 2568 mutex_exit(&ahci_portp->ahciport_mutex); 2569 return (SATA_SUCCESS); 2570 } 2571 2572 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2573 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2574 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2575 /* 2576 * In case the targer driver would send the request before 2577 * sata framework can have the opportunity to process those 2578 * event reports. 2579 */ 2580 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2581 spkt->satapkt_device.satadev_state = 2582 ahci_portp->ahciport_port_state; 2583 ahci_update_sata_registers(ahci_ctlp, port, 2584 &spkt->satapkt_device); 2585 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2586 "ahci_tran_abort returning SATA_FAILURE while " 2587 "port in FAILED/SHUTDOWN/PWROFF state: " 2588 "port: %d", port); 2589 mutex_exit(&ahci_portp->ahciport_mutex); 2590 return (SATA_FAILURE); 2591 } 2592 2593 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2594 /* 2595 * ahci_intr_phyrdy_change() may have rendered it to 2596 * AHCI_PORT_TYPE_NODEV. 2597 */ 2598 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2599 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2600 spkt->satapkt_device.satadev_state = 2601 ahci_portp->ahciport_port_state; 2602 ahci_update_sata_registers(ahci_ctlp, port, 2603 &spkt->satapkt_device); 2604 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2605 "ahci_tran_abort returning SATA_FAILURE while " 2606 "no device attached: port: %d", port); 2607 mutex_exit(&ahci_portp->ahciport_mutex); 2608 return (SATA_FAILURE); 2609 } 2610 2611 if (flag == SATA_ABORT_ALL_PACKETS) { 2612 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2613 aborted_tags = ahci_portp->ahciport_pending_tags; 2614 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2615 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2616 2617 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2618 instance, port); 2619 } else { 2620 aborted_tags = 0xffffffff; 2621 /* 2622 * Aborting one specific packet, first search the 2623 * ahciport_slot_pkts[] list for matching spkt. 2624 */ 2625 for (tmp_slot = 0; 2626 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2627 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2628 aborted_tags = (0x1 << tmp_slot); 2629 break; 2630 } 2631 } 2632 2633 if (aborted_tags == 0xffffffff) { 2634 /* request packet is not on the pending list */ 2635 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2636 "Cannot find the aborting pkt 0x%p on the " 2637 "pending list", (void *)spkt); 2638 ahci_update_sata_registers(ahci_ctlp, port, 2639 &spkt->satapkt_device); 2640 mutex_exit(&ahci_portp->ahciport_mutex); 2641 return (SATA_FAILURE); 2642 } 2643 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2644 instance, port, (void *)spkt); 2645 } 2646 2647 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2648 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2649 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2650 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2651 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2652 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2653 2654 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2655 ahci_portp->ahciport_mop_in_progress++; 2656 2657 /* 2658 * To abort the packet(s), first we are trying to clear PxCMD.ST 2659 * to stop the port, and if the port can be stopped 2660 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2661 * then we just send back the aborted packet(s) with ABORTED flag 2662 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2663 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2664 * perform a COMRESET. 2665 */ 2666 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2667 ahci_portp, port, 0, NULL); 2668 2669 /* 2670 * Compute which have finished and which need to be retried. 2671 * 2672 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2673 * minus the slot_status. The aborted_tags has to be deducted by 2674 * finished_tags since we can't possibly abort a tag which had finished 2675 * already. 2676 */ 2677 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2678 finished_tags = ahci_portp->ahciport_pending_tags & 2679 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2680 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2681 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2682 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2683 2684 aborted_tags &= ~finished_tags; 2685 2686 ahci_mop_commands(ahci_ctlp, 2687 ahci_portp, 2688 slot_status, 2689 0, /* failed tags */ 2690 0, /* timeout tags */ 2691 aborted_tags, 2692 0); /* reset tags */ 2693 2694 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2695 mutex_exit(&ahci_portp->ahciport_mutex); 2696 2697 return (SATA_SUCCESS); 2698 } 2699 2700 /* 2701 * Used to do device reset and reject all the pending packets on a device 2702 * during the reset operation. 2703 * 2704 * NOTE: ONLY called by ahci_tran_reset_dport 2705 */ 2706 static int 2707 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2708 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2709 { 2710 uint32_t slot_status = 0; 2711 uint32_t reset_tags = 0; 2712 uint32_t finished_tags = 0; 2713 uint8_t port = addrp->aa_port; 2714 sata_device_t sdevice; 2715 int ret; 2716 2717 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2718 2719 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2720 "ahci_reset_device_reject_pkts on port: %d", port); 2721 2722 /* 2723 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2724 * commands are being mopped, therefore there is nothing else to do 2725 */ 2726 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2727 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2728 "ahci_reset_device_reject_pkts: port %d is in " 2729 "mopping process, so return directly ", port); 2730 return (SATA_SUCCESS); 2731 } 2732 2733 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2735 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2736 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2737 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2738 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2739 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2740 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2741 } 2742 2743 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2744 != AHCI_SUCCESS) { 2745 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2746 "Try to do a port reset after software " 2747 "reset failed", port); 2748 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 2749 if (ret != AHCI_SUCCESS) { 2750 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2751 "ahci_reset_device_reject_pkts: port %d " 2752 "failed", port); 2753 return (SATA_FAILURE); 2754 } 2755 } 2756 /* Set the reset in progress flag */ 2757 ahci_portp->ahciport_reset_in_progress = 1; 2758 2759 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2760 ahci_portp->ahciport_mop_in_progress++; 2761 2762 /* Indicate to the framework that a reset has happened */ 2763 bzero((void *)&sdevice, sizeof (sata_device_t)); 2764 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2765 sdevice.satadev_addr.pmport = 0; 2766 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2767 sdevice.satadev_state = SATA_DSTATE_RESET | 2768 SATA_DSTATE_PWR_ACTIVE; 2769 mutex_exit(&ahci_portp->ahciport_mutex); 2770 sata_hba_event_notify( 2771 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2772 &sdevice, 2773 SATA_EVNT_DEVICE_RESET); 2774 mutex_enter(&ahci_portp->ahciport_mutex); 2775 2776 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2777 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 2778 2779 /* Next try to mop the pending commands */ 2780 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2781 finished_tags = ahci_portp->ahciport_pending_tags & 2782 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2783 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2784 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2785 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2786 2787 reset_tags &= ~finished_tags; 2788 2789 ahci_mop_commands(ahci_ctlp, 2790 ahci_portp, 2791 slot_status, 2792 0, /* failed tags */ 2793 0, /* timeout tags */ 2794 0, /* aborted tags */ 2795 reset_tags); /* reset tags */ 2796 2797 return (SATA_SUCCESS); 2798 } 2799 2800 /* 2801 * Used to do device reset and reject all the pending packets on a device 2802 * during the reset operation. 2803 * 2804 * NOTE: ONLY called by ahci_tran_reset_dport 2805 */ 2806 static int 2807 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp, 2808 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2809 { 2810 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 2811 uint8_t port = addrp->aa_port; 2812 uint8_t pmport = addrp->aa_pmport; 2813 sata_device_t sdevice; 2814 2815 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2816 2817 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp, 2818 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport); 2819 2820 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2821 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2822 "ahci_reset_pmdevice_reject_pkts: port %d is in " 2823 "mopping process, so return directly ", port); 2824 return (SATA_SUCCESS); 2825 } 2826 2827 /* Checking for outstanding commands */ 2828 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2829 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2830 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2831 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2832 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2833 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2834 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2835 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2836 } 2837 2838 /* Issue SOFTWARE reset command. */ 2839 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2840 != AHCI_SUCCESS) { 2841 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2842 "Try to do a port reset after software " 2843 "reset failed", port); 2844 return (SATA_FAILURE); 2845 } 2846 2847 /* Set the reset in progress flag */ 2848 ahci_portp->ahciport_reset_in_progress = 1; 2849 2850 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2851 ahci_portp->ahciport_mop_in_progress++; 2852 2853 /* Indicate to the framework that a reset has happened */ 2854 bzero((void *)&sdevice, sizeof (sata_device_t)); 2855 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2856 sdevice.satadev_addr.pmport = pmport; 2857 if (AHCI_ADDR_IS_PMULT(addrp)) 2858 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 2859 else 2860 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT; 2861 sdevice.satadev_state = SATA_DSTATE_RESET | 2862 SATA_DSTATE_PWR_ACTIVE; 2863 mutex_exit(&ahci_portp->ahciport_mutex); 2864 sata_hba_event_notify( 2865 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2866 &sdevice, 2867 SATA_EVNT_DEVICE_RESET); 2868 mutex_enter(&ahci_portp->ahciport_mutex); 2869 2870 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2871 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET", 2872 port, pmport); 2873 2874 /* Next try to mop the pending commands */ 2875 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2876 finished_tags = ahci_portp->ahciport_pending_tags & 2877 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2878 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2879 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2880 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2881 reset_tags &= ~finished_tags; 2882 2883 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 2884 "reset_tags = %x, finished_tags = %x, slot_status = %x", 2885 reset_tags, finished_tags, slot_status); 2886 2887 /* 2888 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we 2889 * try to reset a single device behind a port multiplier will 2890 * terminate all the commands on that HBA port. We need mop these 2891 * commands as well. 2892 */ 2893 ahci_mop_commands(ahci_ctlp, 2894 ahci_portp, 2895 slot_status, 2896 0, /* failed tags */ 2897 0, /* timeout tags */ 2898 0, /* aborted tags */ 2899 reset_tags); /* reset tags */ 2900 2901 return (SATA_SUCCESS); 2902 } 2903 2904 /* 2905 * Used to do port reset and reject all the pending packets on a port during 2906 * the reset operation. 2907 */ 2908 static int 2909 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2910 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2911 { 2912 uint32_t slot_status = 0; 2913 uint32_t reset_tags = 0; 2914 uint32_t finished_tags = 0; 2915 uint8_t port = addrp->aa_port; 2916 2917 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 2918 2919 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2920 "ahci_reset_port_reject_pkts at port: %d", port); 2921 2922 /* 2923 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2924 * commands are being mopped, therefore there is nothing else to do 2925 */ 2926 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2927 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2928 "ahci_reset_port_reject_pkts: port %d is in " 2929 "mopping process, so return directly ", port); 2930 return (SATA_SUCCESS); 2931 } 2932 2933 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2934 ahci_portp->ahciport_mop_in_progress++; 2935 2936 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2937 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2938 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2939 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2940 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2941 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2942 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2943 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2944 } 2945 2946 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2947 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2948 NULL) != AHCI_SUCCESS) { 2949 2950 /* Clear mop flag */ 2951 ahci_portp->ahciport_mop_in_progress--; 2952 if (ahci_portp->ahciport_mop_in_progress == 0) 2953 ahci_portp->ahciport_flags &= 2954 ~AHCI_PORT_FLAG_MOPPING; 2955 return (SATA_FAILURE); 2956 } 2957 2958 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2959 finished_tags = ahci_portp->ahciport_pending_tags & 2960 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2961 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2962 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2963 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2964 2965 reset_tags &= ~finished_tags; 2966 2967 ahci_mop_commands(ahci_ctlp, 2968 ahci_portp, 2969 slot_status, 2970 0, /* failed tags */ 2971 0, /* timeout tags */ 2972 0, /* aborted tags */ 2973 reset_tags); /* reset tags */ 2974 2975 return (SATA_SUCCESS); 2976 } 2977 2978 /* 2979 * Used to do hba reset and reject all the pending packets on all ports 2980 * during the reset operation. 2981 */ 2982 static int 2983 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2984 { 2985 ahci_port_t *ahci_portp; 2986 uint32_t slot_status[AHCI_MAX_PORTS]; 2987 uint32_t reset_tags[AHCI_MAX_PORTS]; 2988 uint32_t finished_tags[AHCI_MAX_PORTS]; 2989 int port; 2990 int ret = SATA_SUCCESS; 2991 2992 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2993 "ahci_reset_hba_reject_pkts enter", NULL); 2994 2995 bzero(slot_status, sizeof (slot_status)); 2996 bzero(reset_tags, sizeof (reset_tags)); 2997 bzero(finished_tags, sizeof (finished_tags)); 2998 2999 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3000 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3001 continue; 3002 } 3003 3004 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3005 3006 mutex_enter(&ahci_portp->ahciport_mutex); 3007 ahci_portp->ahciport_reset_in_progress = 1; 3008 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3009 slot_status[port] = ddi_get32( 3010 ahci_ctlp->ahcictl_ahci_acc_handle, 3011 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3012 reset_tags[port] = slot_status[port] & 3013 AHCI_SLOT_MASK(ahci_ctlp); 3014 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3015 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 3016 port, reset_tags[port], 3017 ahci_portp->ahciport_pending_tags); 3018 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3019 slot_status[port] = ddi_get32( 3020 ahci_ctlp->ahcictl_ahci_acc_handle, 3021 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3022 reset_tags[port] = slot_status[port] & 3023 AHCI_NCQ_SLOT_MASK(ahci_portp); 3024 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3025 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 3026 port, reset_tags[port], 3027 ahci_portp->ahciport_pending_tags); 3028 } 3029 mutex_exit(&ahci_portp->ahciport_mutex); 3030 } 3031 3032 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 3033 ret = SATA_FAILURE; 3034 } 3035 3036 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3037 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3038 continue; 3039 } 3040 3041 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3042 3043 mutex_enter(&ahci_portp->ahciport_mutex); 3044 /* 3045 * To prevent recursive enter to ahci_mop_commands, we need 3046 * check AHCI_PORT_FLAG_MOPPING flag. 3047 */ 3048 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3049 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3050 "ahci_reset_hba_reject_pkts: port %d is in " 3051 "mopping process, so return directly ", port); 3052 mutex_exit(&ahci_portp->ahciport_mutex); 3053 continue; 3054 } 3055 3056 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3057 ahci_portp->ahciport_mop_in_progress++; 3058 3059 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 3060 finished_tags[port] = 3061 ahci_portp->ahciport_pending_tags & 3062 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 3063 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 3064 finished_tags[port] = 3065 ahci_portp->ahciport_pending_ncq_tags & 3066 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 3067 3068 reset_tags[port] &= ~finished_tags[port]; 3069 3070 ahci_mop_commands(ahci_ctlp, 3071 ahci_portp, 3072 slot_status[port], 3073 0, /* failed tags */ 3074 0, /* timeout tags */ 3075 0, /* aborted tags */ 3076 reset_tags[port]); /* reset tags */ 3077 mutex_exit(&ahci_portp->ahciport_mutex); 3078 } 3079 out: 3080 return (ret); 3081 } 3082 3083 /* 3084 * Called by sata framework to reset a port(s) or device. 3085 */ 3086 static int 3087 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 3088 { 3089 ahci_ctl_t *ahci_ctlp; 3090 ahci_port_t *ahci_portp; 3091 ahci_addr_t addr; 3092 uint8_t cport = sd->satadev_addr.cport; 3093 uint8_t pmport = sd->satadev_addr.pmport; 3094 uint8_t port; 3095 int ret = SATA_SUCCESS; 3096 int instance = ddi_get_instance(dip); 3097 3098 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3099 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3100 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3101 3102 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 3103 3104 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3105 "ahci_tran_reset_dport enter: cport %d", cport); 3106 3107 switch (sd->satadev_addr.qual) { 3108 case SATA_ADDR_PMPORT: 3109 /* 3110 * If we want to issue a COMRESET on a pmport, we need to 3111 * reject the outstanding commands on that pmport. According 3112 * to AHCI spec, PxCI register could only be cleared by 3113 * clearing PxCMD.ST, which will halt the controller port - as 3114 * well as other pmports. 3115 * 3116 * Therefore we directly reset the controller port for 3117 * simplicity. ahci_tran_probe_port() will handle reset stuff 3118 * like initializing the given pmport. 3119 */ 3120 /* FALLTHRU */ 3121 case SATA_ADDR_CPORT: 3122 /* Port reset */ 3123 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3124 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3125 "port %d reset port", instance, port); 3126 3127 mutex_enter(&ahci_portp->ahciport_mutex); 3128 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr); 3129 mutex_exit(&ahci_portp->ahciport_mutex); 3130 3131 break; 3132 3133 case SATA_ADDR_DPMPORT: 3134 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3135 "port %d:%d reset device", instance, port, pmport); 3136 /* FALLTHRU */ 3137 case SATA_ADDR_DCPORT: 3138 /* Device reset */ 3139 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT) 3140 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3141 "port %d reset device", instance, port); 3142 3143 mutex_enter(&ahci_portp->ahciport_mutex); 3144 /* 3145 * software reset request must be sent to SATA_PMULT_HOSTPORT 3146 * if target is a port multiplier: 3147 */ 3148 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT && 3149 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) 3150 AHCI_ADDR_SET_PMULT(&addr, port); 3151 3152 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 3153 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 3154 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 3155 /* 3156 * In case the targer driver would send the request 3157 * before sata framework can have the opportunity to 3158 * process those event reports. 3159 */ 3160 sd->satadev_state = ahci_portp->ahciport_port_state; 3161 ahci_update_sata_registers(ahci_ctlp, port, sd); 3162 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3163 "ahci_tran_reset_dport returning SATA_FAILURE " 3164 "while port in FAILED/SHUTDOWN/PWROFF state: " 3165 "port: %d", port); 3166 mutex_exit(&ahci_portp->ahciport_mutex); 3167 ret = SATA_FAILURE; 3168 break; 3169 } 3170 3171 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == 3172 SATA_DTYPE_NONE) { 3173 /* 3174 * ahci_intr_phyrdy_change() may have rendered it to 3175 * AHCI_PORT_TYPE_NODEV. 3176 */ 3177 sd->satadev_type = SATA_DTYPE_NONE; 3178 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp, 3179 &addr); 3180 ahci_update_sata_registers(ahci_ctlp, port, sd); 3181 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3182 "ahci_tran_reset_dport returning SATA_FAILURE " 3183 "while no device attached: port: %d", port); 3184 mutex_exit(&ahci_portp->ahciport_mutex); 3185 ret = SATA_FAILURE; 3186 break; 3187 } 3188 3189 if (AHCI_ADDR_IS_PORT(&addr)) { 3190 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 3191 ahci_portp, &addr); 3192 } else { 3193 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp, 3194 ahci_portp, &addr); 3195 } 3196 3197 mutex_exit(&ahci_portp->ahciport_mutex); 3198 break; 3199 3200 case SATA_ADDR_CNTRL: 3201 /* Reset the whole controller */ 3202 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 3203 "reset the whole hba", instance); 3204 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 3205 break; 3206 3207 default: 3208 ret = SATA_FAILURE; 3209 } 3210 3211 return (ret); 3212 } 3213 3214 /* 3215 * Called by sata framework to activate a port as part of hotplug. 3216 * (cfgadm -c connect satax/y) 3217 * Support port multiplier. 3218 */ 3219 static int 3220 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 3221 { 3222 ahci_ctl_t *ahci_ctlp; 3223 ahci_port_t *ahci_portp; 3224 ahci_addr_t addr; 3225 uint8_t cport = satadev->satadev_addr.cport; 3226 uint8_t pmport = satadev->satadev_addr.pmport; 3227 uint8_t port; 3228 int instance = ddi_get_instance(dip); 3229 3230 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3231 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3232 3233 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3234 "ahci_tran_hotplug_port_activate enter: cport %d", cport); 3235 3236 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3237 3238 mutex_enter(&ahci_portp->ahciport_mutex); 3239 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3240 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3241 3242 if (AHCI_ADDR_IS_PORT(&addr)) { 3243 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", 3244 instance, port); 3245 3246 /* Enable the interrupts on the port */ 3247 ahci_enable_port_intrs(ahci_ctlp, port); 3248 3249 /* 3250 * Reset the port so that the PHY communication would be 3251 * re-established. But this reset is an internal operation 3252 * and the sata module doesn't need to know about it. 3253 * Moreover, the port with a device attached will be started 3254 * too. 3255 */ 3256 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 3257 ahci_portp, port, 3258 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 3259 NULL); 3260 3261 /* 3262 * Need to check the link status and device status of the port 3263 * and consider raising power if the port was in D3 state 3264 */ 3265 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 3266 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 3267 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 3268 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3269 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated", 3270 instance, port, pmport); 3271 /* AHCI_ADDR_PMPORT */ 3272 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON; 3273 AHCIPORT_PMSTATE(ahci_portp, &addr) &= 3274 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN); 3275 } 3276 3277 satadev->satadev_state = ahci_portp->ahciport_port_state; 3278 3279 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3280 3281 mutex_exit(&ahci_portp->ahciport_mutex); 3282 return (SATA_SUCCESS); 3283 } 3284 3285 /* 3286 * Called by sata framework to deactivate a port as part of hotplug. 3287 * (cfgadm -c disconnect satax/y) 3288 * Support port multiplier. 3289 */ 3290 static int 3291 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 3292 { 3293 ahci_ctl_t *ahci_ctlp; 3294 ahci_port_t *ahci_portp; 3295 ahci_addr_t addr; 3296 uint8_t cport = satadev->satadev_addr.cport; 3297 uint8_t pmport = satadev->satadev_addr.pmport; 3298 uint8_t port; 3299 uint32_t port_scontrol; 3300 int instance = ddi_get_instance(dip); 3301 3302 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3303 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3304 3305 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3306 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport); 3307 3308 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3309 mutex_enter(&ahci_portp->ahciport_mutex); 3310 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3311 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3312 3313 if (AHCI_ADDR_IS_PORT(&addr)) { 3314 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 3315 instance, port); 3316 3317 /* Disable the interrupts on the port */ 3318 ahci_disable_port_intrs(ahci_ctlp, port); 3319 3320 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 3321 3322 /* First to abort all the pending commands */ 3323 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3324 3325 /* Then stop the port */ 3326 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3327 ahci_portp, port); 3328 } 3329 3330 /* Next put the PHY offline */ 3331 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3332 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3333 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 3334 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *) 3335 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3336 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3337 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated", 3338 instance, port, pmport); 3339 3340 ahci_disable_port_intrs(ahci_ctlp, port); 3341 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) 3342 != SATA_DTYPE_NONE) 3343 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3344 3345 /* Re-enable the interrupts for the other pmports */ 3346 ahci_enable_port_intrs(ahci_ctlp, port); 3347 } 3348 3349 /* Update port state */ 3350 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN); 3351 satadev->satadev_state = SATA_PSTATE_SHUTDOWN; 3352 3353 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3354 3355 mutex_exit(&ahci_portp->ahciport_mutex); 3356 return (SATA_SUCCESS); 3357 } 3358 3359 /* 3360 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 3361 * when a device is unplugged or a port is deactivated. 3362 */ 3363 static void 3364 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 3365 ahci_port_t *ahci_portp, uint8_t port) 3366 { 3367 uint32_t slot_status = 0; 3368 uint32_t abort_tags = 0; 3369 3370 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3371 3372 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 3373 "ahci_reject_all_abort_pkts at port: %d", port); 3374 3375 /* Read/write port multiplier command takes highest priority */ 3376 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3377 slot_status = 0x1; 3378 abort_tags = 0x1; 3379 goto out; 3380 } 3381 3382 /* 3383 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 3384 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 3385 * to get the error data, if yes when the device is removed, the 3386 * command needs to be aborted too. 3387 */ 3388 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3389 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 3390 slot_status = 0x1; 3391 abort_tags = 0x1; 3392 goto out; 3393 } else { 3394 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3395 "ahci_reject_all_abort_pkts return directly " 3396 "port %d no needs to reject any outstanding " 3397 "commands", port); 3398 return; 3399 } 3400 } 3401 3402 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3403 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3404 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3405 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 3406 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3407 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3408 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3409 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 3410 } 3411 3412 out: 3413 /* No need to do mop when there is no outstanding commands */ 3414 if (slot_status != 0) { 3415 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3416 ahci_portp->ahciport_mop_in_progress++; 3417 3418 ahci_mop_commands(ahci_ctlp, 3419 ahci_portp, 3420 slot_status, 3421 0, /* failed tags */ 3422 0, /* timeout tags */ 3423 abort_tags, /* aborting tags */ 3424 0); /* reset tags */ 3425 } 3426 } 3427 3428 #if defined(__lock_lint) 3429 static int 3430 ahci_selftest(dev_info_t *dip, sata_device_t *device) 3431 { 3432 return (SATA_SUCCESS); 3433 } 3434 #endif 3435 3436 /* 3437 * Initialize fma capabilities and register with IO fault services. 3438 */ 3439 static void 3440 ahci_fm_init(ahci_ctl_t *ahci_ctlp) 3441 { 3442 /* 3443 * Need to change iblock to priority for new MSI intr 3444 */ 3445 ddi_iblock_cookie_t fm_ibc; 3446 3447 ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY, 3448 ahci_ctlp->ahcictl_dip, 3449 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 3450 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 3451 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 3452 3453 /* Only register with IO Fault Services if we have some capability */ 3454 if (ahci_ctlp->ahcictl_fm_cap) { 3455 /* Adjust access and dma attributes for FMA */ 3456 accattr.devacc_attr_access = DDI_FLAGERR_ACC; 3457 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3458 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3459 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3460 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 3461 3462 /* 3463 * Register capabilities with IO Fault Services. 3464 * ahcictl_fm_cap will be updated to indicate 3465 * capabilities actually supported (not requested.) 3466 */ 3467 ddi_fm_init(ahci_ctlp->ahcictl_dip, 3468 &ahci_ctlp->ahcictl_fm_cap, &fm_ibc); 3469 3470 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) { 3471 cmn_err(CE_WARN, "!ahci%d: fma init failed.", 3472 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 3473 return; 3474 } 3475 /* 3476 * Initialize pci ereport capabilities if ereport 3477 * capable (should always be.) 3478 */ 3479 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) || 3480 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3481 pci_ereport_setup(ahci_ctlp->ahcictl_dip); 3482 } 3483 3484 /* 3485 * Register error callback if error callback capable. 3486 */ 3487 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3488 ddi_fm_handler_register(ahci_ctlp->ahcictl_dip, 3489 ahci_fm_error_cb, (void *) ahci_ctlp); 3490 } 3491 3492 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3493 "ahci_fm_fini: fma enabled.", NULL); 3494 } 3495 } 3496 3497 /* 3498 * Releases fma capabilities and un-registers with IO fault services. 3499 */ 3500 static void 3501 ahci_fm_fini(ahci_ctl_t *ahci_ctlp) 3502 { 3503 /* Only unregister FMA capabilities if registered */ 3504 if (ahci_ctlp->ahcictl_fm_cap) { 3505 /* 3506 * Un-register error callback if error callback capable. 3507 */ 3508 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3509 ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip); 3510 } 3511 3512 /* 3513 * Release any resources allocated by pci_ereport_setup() 3514 */ 3515 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) || 3516 DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3517 pci_ereport_teardown(ahci_ctlp->ahcictl_dip); 3518 } 3519 3520 /* Unregister from IO Fault Services */ 3521 ddi_fm_fini(ahci_ctlp->ahcictl_dip); 3522 3523 /* Adjust access and dma attributes for FMA */ 3524 accattr.devacc_attr_access = DDI_DEFAULT_ACC; 3525 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3526 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3527 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3528 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 3529 3530 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3531 "ahci_fm_fini: fma disabled.", NULL); 3532 } 3533 } 3534 3535 /*ARGSUSED*/ 3536 static int 3537 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 3538 { 3539 /* 3540 * as the driver can always deal with an error in any dma or 3541 * access handle, we can just return the fme_status value. 3542 */ 3543 pci_ereport_post(dip, err, NULL); 3544 return (err->fme_status); 3545 } 3546 3547 int 3548 ahci_check_acc_handle(ddi_acc_handle_t handle) 3549 { 3550 ddi_fm_error_t de; 3551 3552 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 3553 return (de.fme_status); 3554 } 3555 3556 int 3557 ahci_check_dma_handle(ddi_dma_handle_t handle) 3558 { 3559 ddi_fm_error_t de; 3560 3561 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 3562 return (de.fme_status); 3563 } 3564 3565 /* 3566 * Generate an ereport 3567 */ 3568 void 3569 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail) 3570 { 3571 uint64_t ena; 3572 char buf[FM_MAX_CLASS]; 3573 3574 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 3575 ena = fm_ena_generate(0, FM_ENA_FMT1); 3576 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) { 3577 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena, 3578 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 3579 FM_EREPORT_VERSION, NULL); 3580 } 3581 } 3582 3583 /* 3584 * Check if all handles are correctly allocated. 3585 */ 3586 static int 3587 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp) 3588 { 3589 int port; 3590 3591 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 3592 return (DDI_FAILURE); 3593 } 3594 3595 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3596 ahci_port_t *ahci_portp; 3597 3598 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3599 continue; 3600 3601 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3602 3603 mutex_enter(&ahci_portp->ahciport_mutex); 3604 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) { 3605 mutex_exit(&ahci_portp->ahciport_mutex); 3606 return (DDI_FAILURE); 3607 } 3608 mutex_exit(&ahci_portp->ahciport_mutex); 3609 } 3610 3611 return (DDI_SUCCESS); 3612 } 3613 3614 /* 3615 * Check the access handles for the controller. Note that 3616 * ahcictl_pci_conf_handle is only used in attach process. 3617 */ 3618 static int 3619 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp) 3620 { 3621 if ((ahci_check_acc_handle(ahci_ctlp-> 3622 ahcictl_pci_conf_handle) != DDI_FM_OK) || 3623 (ahci_check_acc_handle(ahci_ctlp-> 3624 ahcictl_ahci_acc_handle) != DDI_FM_OK)) { 3625 return (DDI_FAILURE); 3626 } 3627 return (DDI_SUCCESS); 3628 } 3629 3630 /* 3631 * Check the DMA handles and the access handles of a controller port. 3632 */ 3633 static int 3634 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port) 3635 { 3636 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3637 int slot; 3638 3639 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3640 3641 if ((ahci_check_dma_handle(ahci_portp-> 3642 ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) || 3643 (ahci_check_dma_handle(ahci_portp-> 3644 ahciport_cmd_list_dma_handle) != DDI_FM_OK) || 3645 (ahci_check_acc_handle(ahci_portp-> 3646 ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) || 3647 (ahci_check_acc_handle(ahci_portp-> 3648 ahciport_cmd_list_acc_handle) != DDI_FM_OK)) { 3649 return (DDI_FAILURE); 3650 } 3651 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 3652 if (ahci_check_slot_handle(ahci_portp, slot) 3653 != DDI_SUCCESS) { 3654 return (DDI_FAILURE); 3655 } 3656 } 3657 return (DDI_SUCCESS); 3658 } 3659 3660 /* 3661 * Check the DMA handles and the access handles of a cmd table slot. 3662 */ 3663 static int 3664 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot) 3665 { 3666 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3667 3668 if ((ahci_check_acc_handle(ahci_portp-> 3669 ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) || 3670 (ahci_check_dma_handle(ahci_portp-> 3671 ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) { 3672 return (DDI_FAILURE); 3673 } 3674 return (DDI_SUCCESS); 3675 } 3676 3677 /* 3678 * Allocate the ports structure, only called by ahci_attach 3679 */ 3680 static int 3681 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 3682 { 3683 int port, cport = 0; 3684 3685 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3686 "ahci_alloc_ports_state enter", NULL); 3687 3688 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3689 3690 /* Allocate structures only for the implemented ports */ 3691 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3692 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3693 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3694 "hba port %d not implemented", port); 3695 continue; 3696 } 3697 3698 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 3699 ahci_ctlp->ahcictl_port_to_cport[port] = 3700 (uint8_t)cport++; 3701 3702 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 3703 goto err_out; 3704 } 3705 } 3706 3707 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3708 return (AHCI_SUCCESS); 3709 3710 err_out: 3711 for (port--; port >= 0; port--) { 3712 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3713 ahci_dealloc_port_state(ahci_ctlp, port); 3714 } 3715 } 3716 3717 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3718 return (AHCI_FAILURE); 3719 } 3720 3721 /* 3722 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 3723 */ 3724 static void 3725 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 3726 { 3727 int port; 3728 3729 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3730 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3731 /* if this port is implemented by the HBA */ 3732 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3733 ahci_dealloc_port_state(ahci_ctlp, port); 3734 } 3735 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3736 } 3737 3738 /* 3739 * Drain the taskq. 3740 */ 3741 static void 3742 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 3743 { 3744 ahci_port_t *ahci_portp; 3745 int port; 3746 3747 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3748 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3749 continue; 3750 } 3751 3752 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3753 3754 mutex_enter(&ahci_portp->ahciport_mutex); 3755 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 3756 mutex_exit(&ahci_portp->ahciport_mutex); 3757 } 3758 } 3759 3760 /* 3761 * Initialize the controller and all ports. And then try to start the ports 3762 * if there are devices attached. 3763 * 3764 * This routine can be called from three seperate cases: DDI_ATTACH, 3765 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 3766 * other two cases; device signature probing are attempted only during 3767 * DDI_ATTACH case. 3768 */ 3769 static int 3770 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 3771 { 3772 ahci_port_t *ahci_portp; 3773 ahci_addr_t addr; 3774 int port; 3775 3776 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3777 "ahci_initialize_controller enter", NULL); 3778 3779 /* Disable the whole controller interrupts */ 3780 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3781 ahci_disable_all_intrs(ahci_ctlp); 3782 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3783 3784 /* Initialize the implemented ports and structures */ 3785 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3786 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3787 continue; 3788 } 3789 3790 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3791 mutex_enter(&ahci_portp->ahciport_mutex); 3792 3793 /* 3794 * Ensure that the controller is not in the running state 3795 * by checking every implemented port's PxCMD register 3796 */ 3797 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port); 3798 3799 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr) 3800 != AHCI_SUCCESS) { 3801 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3802 "ahci_initialize_controller: failed to " 3803 "initialize port %d", port); 3804 /* 3805 * Set the port state to SATA_PSTATE_FAILED if 3806 * failed to initialize it. 3807 */ 3808 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3809 } 3810 3811 mutex_exit(&ahci_portp->ahciport_mutex); 3812 } 3813 3814 /* Enable the whole controller interrupts */ 3815 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3816 ahci_enable_all_intrs(ahci_ctlp); 3817 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3818 3819 return (AHCI_SUCCESS); 3820 } 3821 3822 /* 3823 * Reverse of ahci_initialize_controller() 3824 * 3825 * We only need to stop the ports and disable the interrupt. 3826 */ 3827 static void 3828 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 3829 { 3830 ahci_port_t *ahci_portp; 3831 int port; 3832 3833 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3834 "ahci_uninitialize_controller enter", NULL); 3835 3836 /* disable all the interrupts. */ 3837 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3838 ahci_disable_all_intrs(ahci_ctlp); 3839 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3840 3841 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3842 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3843 continue; 3844 } 3845 3846 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3847 3848 /* Stop the port by clearing PxCMD.ST */ 3849 mutex_enter(&ahci_portp->ahciport_mutex); 3850 3851 /* 3852 * Here we must disable the port interrupt because 3853 * ahci_disable_all_intrs only clear GHC.IE, and IS 3854 * register will be still set if PxIE is enabled. 3855 * When ahci shares one IRQ with other drivers, the 3856 * intr handler may claim the intr mistakenly. 3857 */ 3858 ahci_disable_port_intrs(ahci_ctlp, port); 3859 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3860 ahci_portp, port); 3861 mutex_exit(&ahci_portp->ahciport_mutex); 3862 } 3863 } 3864 3865 /* 3866 * ahci_alloc_pmult() 3867 * 1. Setting HBA port registers which are necessary for a port multiplier. 3868 * (Set PxCMD.PMA while PxCMD.ST is '0') 3869 * 2. Allocate ahci_pmult_info structure. 3870 * 3871 * NOTE: Must stop port before the function is called. 3872 */ 3873 static void 3874 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3875 { 3876 uint32_t port_cmd_status; 3877 uint8_t port = ahci_portp->ahciport_port_num; 3878 3879 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3880 3881 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3882 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3883 3884 /* The port must have been stopped before. */ 3885 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST)); 3886 3887 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) { 3888 /* set PMA bit */ 3889 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3890 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3891 port_cmd_status|AHCI_CMD_STATUS_PMA); 3892 3893 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3894 "ahci_alloc_pmult: " 3895 "PxCMD.PMA bit set at port %d.", port); 3896 } 3897 3898 /* Allocate port multiplier information structure */ 3899 if (ahci_portp->ahciport_pmult_info == NULL) { 3900 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *) 3901 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP); 3902 } 3903 3904 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 3905 } 3906 3907 /* 3908 * ahci_dealloc_pmult() 3909 * 1. Clearing related registers when a port multiplier is detached. 3910 * (Clear PxCMD.PMA while PxCMD.ST is '0') 3911 * 2. Deallocate ahci_pmult_info structure. 3912 * 3913 * NOTE: Must stop port before the function is called. 3914 */ 3915 static void 3916 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3917 { 3918 uint32_t port_cmd_status; 3919 uint8_t port = ahci_portp->ahciport_port_num; 3920 3921 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3922 3923 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3924 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3925 3926 if (port_cmd_status & AHCI_CMD_STATUS_PMA) { 3927 /* Clear PMA bit */ 3928 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3929 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3930 (port_cmd_status & (~AHCI_CMD_STATUS_PMA))); 3931 3932 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3933 "ahci_dealloc_pmult: " 3934 "PxCMD.PMA bit cleared at port %d.", port); 3935 } 3936 3937 /* Release port multiplier information structure */ 3938 if (ahci_portp->ahciport_pmult_info != NULL) { 3939 kmem_free(ahci_portp->ahciport_pmult_info, 3940 sizeof (ahci_pmult_info_t)); 3941 ahci_portp->ahciport_pmult_info = NULL; 3942 } 3943 } 3944 3945 /* 3946 * Staggered Spin-up. 3947 */ 3948 static void 3949 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port) 3950 { 3951 uint32_t cap_status; 3952 uint32_t port_cmd_status; 3953 3954 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 3955 3956 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3957 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3958 3959 /* Check for staggered spin-up support */ 3960 if (!(cap_status & AHCI_HBA_CAP_SSS)) 3961 return; 3962 3963 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3964 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3965 3966 /* If PxCMD.SUD == 1, no staggered spin-up is needed */ 3967 if (port_cmd_status & AHCI_CMD_STATUS_SUD) 3968 return; 3969 3970 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port); 3971 3972 /* Set PxCMD.SUD */ 3973 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3974 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3975 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3976 port_cmd_status); 3977 } 3978 3979 /* 3980 * The routine is to initialize a port. First put the port in NotRunning 3981 * state, then enable port interrupt and clear Serror register. And under 3982 * AHCI_ATTACH case, find device signature and then try to start the port. 3983 * 3984 * Called by 3985 * 1. ahci_initialize_controller 3986 * 2. ahci_intr_phyrdy_change (hotplug) 3987 */ 3988 static int 3989 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3990 ahci_addr_t *addrp) 3991 { 3992 uint32_t port_sstatus, port_task_file, port_cmd_status; 3993 uint8_t port = addrp->aa_port; 3994 boolean_t resuming = B_TRUE; /* processing DDI_RESUME */ 3995 int ret; 3996 3997 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 3998 3999 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */ 4000 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 4001 4002 /* 4003 * At the time being, only probe ports/devices and get the types of 4004 * attached devices during DDI_ATTACH. In fact, the device can be 4005 * changed during power state changes, but at the time being, we 4006 * don't support the situation. 4007 */ 4008 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4009 resuming = B_FALSE; 4010 } else { 4011 /* check for DDI_RESUME case */ 4012 mutex_exit(&ahci_portp->ahciport_mutex); 4013 mutex_enter(&ahci_ctlp->ahcictl_mutex); 4014 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) 4015 resuming = B_FALSE; 4016 mutex_exit(&ahci_ctlp->ahcictl_mutex); 4017 mutex_enter(&ahci_portp->ahciport_mutex); 4018 } 4019 4020 if (resuming) { 4021 /* 4022 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 4023 * and PxFBU registers in case these registers were cleared 4024 * during the suspend. 4025 */ 4026 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4027 "ahci_initialize_port: port %d " 4028 "set PxCLB, PxCLBU, PxFB and PxFBU " 4029 "during resume", port); 4030 4031 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) != 4032 AHCI_SUCCESS) 4033 return (AHCI_FAILURE); 4034 } 4035 4036 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4037 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4038 4039 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4040 "ahci_initialize_port: port %d ", port); 4041 4042 /* 4043 * Check whether the port is in NotRunning state, if not, 4044 * put the port in NotRunning state 4045 */ 4046 if (port_cmd_status & 4047 (AHCI_CMD_STATUS_ST | 4048 AHCI_CMD_STATUS_CR | 4049 AHCI_CMD_STATUS_FRE | 4050 AHCI_CMD_STATUS_FR)) { 4051 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 4052 ahci_portp, port); 4053 } 4054 4055 /* Make sure the drive is spun-up */ 4056 ahci_staggered_spin_up(ahci_ctlp, port); 4057 4058 /* Disable interrupt */ 4059 ahci_disable_port_intrs(ahci_ctlp, port); 4060 4061 /* Device is unknown at first */ 4062 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4063 4064 /* Disable the interface power management */ 4065 ahci_disable_interface_pm(ahci_ctlp, port); 4066 4067 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4068 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 4069 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4070 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4071 4072 /* Check physcial link status */ 4073 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 4074 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 4075 4076 /* Check interface status */ 4077 port_task_file & AHCI_TFD_STS_BSY || 4078 port_task_file & AHCI_TFD_STS_DRQ || 4079 4080 /* Check whether port reset must be executed */ 4081 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET || 4082 4083 /* Always reset port on RESUME */ 4084 resuming != B_FALSE) { 4085 4086 /* Something went wrong, we need do some reset things */ 4087 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 4088 4089 /* Does port reset succeed on HBA port? */ 4090 if (ret != AHCI_SUCCESS) { 4091 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4092 "ahci_initialize_port:" 4093 "port reset failed at port %d", port); 4094 return (AHCI_FAILURE); 4095 } 4096 4097 /* Is port failed? */ 4098 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4099 SATA_PSTATE_FAILED) { 4100 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4101 "ahci_initialize_port: port %d state 0x%x", 4102 port, ahci_portp->ahciport_port_state); 4103 return (AHCI_FAILURE); 4104 } 4105 } 4106 4107 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4108 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 4109 4110 /* 4111 * Try to get the device signature if the port is not empty. 4112 */ 4113 if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) != 4114 SATA_DTYPE_NONE) 4115 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4116 4117 /* Return directly if no device connected */ 4118 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) { 4119 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4120 "No device connected to port %d", port); 4121 goto out; 4122 } 4123 4124 /* If this is a port multiplier, we need do some initialization */ 4125 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 4126 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4127 "Port multiplier found at port %d", port); 4128 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 4129 } 4130 4131 /* Try to start the port */ 4132 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 4133 != AHCI_SUCCESS) { 4134 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4135 "failed to start port %d", port); 4136 return (AHCI_FAILURE); 4137 } 4138 out: 4139 /* Enable port interrupts */ 4140 ahci_enable_port_intrs(ahci_ctlp, port); 4141 4142 return (AHCI_SUCCESS); 4143 } 4144 4145 /* 4146 * Handle hardware defect, and check the capabilities. For example, 4147 * power management capabilty and MSI capability. 4148 */ 4149 static int 4150 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 4151 { 4152 ushort_t caps_ptr, cap_count, cap; 4153 #if AHCI_DEBUG 4154 ushort_t pmcap, pmcsr; 4155 ushort_t msimc; 4156 #endif 4157 uint8_t revision; 4158 4159 ahci_ctlp->ahcictl_venid = 4160 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4161 PCI_CONF_VENID); 4162 4163 ahci_ctlp->ahcictl_devid = 4164 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4165 PCI_CONF_DEVID); 4166 4167 /* 4168 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 4169 * controllers with 0x00 revision id work on 4-byte aligned buffer, 4170 * which is a bug and was fixed after 0x00 revision id controllers. 4171 * 4172 * Moreover, VT8251 cannot use multiple command slots in the command 4173 * list for non-queued commands because the previous register content 4174 * of PxCI can be re-written in the register write, so a flag will be 4175 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 4176 * 4177 * For VT8251, software reset also has the same defect as the below 4178 * AMD/ATI chipset. That is, software reset will get failed if 0xf 4179 * is filled in pmport field. Therefore, another software reset need 4180 * to be done with 0 filled in pmport field. 4181 */ 4182 if (ahci_ctlp->ahcictl_venid == VIA_VENID) { 4183 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 4184 PCI_CONF_REVID); 4185 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4186 "revision id = 0x%x", revision); 4187 if (revision == 0x00) { 4188 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 4189 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4190 "change ddi_attr_align to 0x4", NULL); 4191 } 4192 4193 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE; 4194 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4195 "VT8251 cannot use multiple command lists for " 4196 "non-queued commands", NULL); 4197 4198 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4199 } 4200 4201 /* 4202 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit 4203 * DMA addressing for communication memory descriptors though S64A bit 4204 * of CAP register declares it supports. Even though 64-bit DMA for 4205 * data buffer works on ASUS M2A-VM with newer BIOS, three other 4206 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and 4207 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller. 4208 * 4209 * Due to certain hardware issue, the chipset must do port reset during 4210 * initialization, otherwise, when retrieving device signature, 4211 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag 4212 * need to set. 4213 * 4214 * For this chipset software reset will get failure if the pmport of 4215 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port 4216 * multiplier is connected to the port. In order to fix the issue, 4217 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software 4218 * reset got failure, the driver will try to do another software reset 4219 * with pmport 0. 4220 */ 4221 if (ahci_ctlp->ahcictl_venid == 0x1002 && 4222 ahci_ctlp->ahcictl_devid == 0x4380) { 4223 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 4224 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 4225 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 4226 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4227 4228 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4229 "ATI SB600 cannot do 64-bit DMA for both data buffer and " 4230 "communication memory descriptors though CAP indicates " 4231 "support, so force it to use 32-bit DMA", NULL); 4232 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4233 "ATI SB600 need to do a port reset during initialization", 4234 NULL); 4235 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4236 "ATI SB600 will get software reset failure if pmport " 4237 "is set 0xf and no port multiplier is attached", NULL); 4238 } 4239 4240 /* 4241 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same 4242 * vendor ID and device ID (0x1002,0x4391). 4243 * 4244 * SB700/750 AHCI chipset on some boards doesn't support 64-bit 4245 * DMA addressing for communication memory descriptors though S64A bit 4246 * of CAP register declares the support. However, it does support 4247 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is 4248 * set for this controller. 4249 * 4250 * SB710 has the same initialization issue as SB600, so it also need 4251 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it. 4252 * 4253 * SB700 also has the same issue about software reset, and thus 4254 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed. 4255 */ 4256 if (ahci_ctlp->ahcictl_venid == 0x1002 && 4257 ahci_ctlp->ahcictl_devid == 0x4391) { 4258 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 4259 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 4260 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 4261 4262 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4263 "ATI SB700/750 cannot do 64-bit DMA for communication " 4264 "memory descriptors though CAP indicates support, " 4265 "so force it to use 32-bit DMA", NULL); 4266 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4267 "ATI SB710 need to do a port reset during initialization", 4268 NULL); 4269 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4270 "ATI SB700 will get software reset failure if pmport " 4271 "is set 0xf and no port multiplier is attached", NULL); 4272 } 4273 4274 /* 4275 * Check if capabilities list is supported and if so, 4276 * get initial capabilities pointer and clear bits 0,1. 4277 */ 4278 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 4279 PCI_CONF_STAT) & PCI_STAT_CAP) { 4280 caps_ptr = P2ALIGN(pci_config_get8( 4281 ahci_ctlp->ahcictl_pci_conf_handle, 4282 PCI_CONF_CAP_PTR), 4); 4283 } else { 4284 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 4285 } 4286 4287 /* 4288 * Walk capabilities if supported. 4289 */ 4290 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 4291 4292 /* 4293 * Check that we haven't exceeded the maximum number of 4294 * capabilities and that the pointer is in a valid range. 4295 */ 4296 if (++cap_count > PCI_CAP_MAX_PTR) { 4297 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4298 "too many device capabilities", NULL); 4299 return (AHCI_FAILURE); 4300 } 4301 if (caps_ptr < PCI_CAP_PTR_OFF) { 4302 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4303 "capabilities pointer 0x%x out of range", 4304 caps_ptr); 4305 return (AHCI_FAILURE); 4306 } 4307 4308 /* 4309 * Get next capability and check that it is valid. 4310 * For now, we only support power management. 4311 */ 4312 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 4313 caps_ptr); 4314 switch (cap) { 4315 case PCI_CAP_ID_PM: 4316 4317 /* power management supported */ 4318 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 4319 4320 /* Save PMCSR offset */ 4321 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 4322 4323 #if AHCI_DEBUG 4324 pmcap = pci_config_get16( 4325 ahci_ctlp->ahcictl_pci_conf_handle, 4326 caps_ptr + PCI_PMCAP); 4327 pmcsr = pci_config_get16( 4328 ahci_ctlp->ahcictl_pci_conf_handle, 4329 ahci_ctlp->ahcictl_pmcsr_offset); 4330 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4331 "Power Management capability found PCI_PMCAP " 4332 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 4333 if ((pmcap & 0x3) == 0x3) 4334 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4335 "PCI Power Management Interface " 4336 "spec 1.2 compliant", NULL); 4337 #endif 4338 break; 4339 4340 case PCI_CAP_ID_MSI: 4341 #if AHCI_DEBUG 4342 msimc = pci_config_get16( 4343 ahci_ctlp->ahcictl_pci_conf_handle, 4344 caps_ptr + PCI_MSI_CTRL); 4345 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 4346 "Message Signaled Interrupt capability found " 4347 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 4348 #endif 4349 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 4350 "MSI capability found", NULL); 4351 break; 4352 4353 case PCI_CAP_ID_PCIX: 4354 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4355 "PCI-X capability found", NULL); 4356 break; 4357 4358 case PCI_CAP_ID_PCI_E: 4359 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4360 "PCI Express capability found", NULL); 4361 break; 4362 4363 case PCI_CAP_ID_MSI_X: 4364 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4365 "MSI-X capability found", NULL); 4366 break; 4367 4368 case PCI_CAP_ID_SATA: 4369 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4370 "SATA capability found", NULL); 4371 break; 4372 4373 case PCI_CAP_ID_VS: 4374 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4375 "Vendor Specific capability found", NULL); 4376 break; 4377 4378 default: 4379 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 4380 "unrecognized capability 0x%x", cap); 4381 break; 4382 } 4383 4384 /* 4385 * Get next capabilities pointer and clear bits 0,1. 4386 */ 4387 caps_ptr = P2ALIGN(pci_config_get8( 4388 ahci_ctlp->ahcictl_pci_conf_handle, 4389 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 4390 } 4391 4392 return (AHCI_SUCCESS); 4393 } 4394 4395 /* 4396 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE 4397 * PORTMULT command. SYNC & POLLING mode is used. 4398 */ 4399 static int 4400 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4401 uint8_t regn, uint32_t *pregv, uint8_t type) 4402 { 4403 ahci_port_t *ahci_portp; 4404 ahci_addr_t pmult_addr; 4405 sata_pkt_t *spkt; 4406 sata_cmd_t *scmd; 4407 sata_device_t sata_device; 4408 uint8_t port = addrp->aa_port; 4409 uint8_t pmport = addrp->aa_pmport; 4410 uint8_t cport; 4411 uint32_t intr_mask; 4412 int rval; 4413 char portstr[10]; 4414 4415 SET_PORTSTR(portstr, addrp); 4416 cport = ahci_ctlp->ahcictl_port_to_cport[port]; 4417 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4418 4419 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp)); 4420 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4421 4422 /* Check the existence of the port multiplier */ 4423 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) 4424 return (AHCI_FAILURE); 4425 4426 /* Request a READ/WRITE PORTMULT sata packet. */ 4427 bzero(&sata_device, sizeof (sata_device_t)); 4428 sata_device.satadev_addr.cport = cport; 4429 sata_device.satadev_addr.pmport = pmport; 4430 sata_device.satadev_addr.qual = SATA_ADDR_PMULT; 4431 sata_device.satadev_rev = SATA_DEVICE_REV; 4432 4433 /* 4434 * Make sure no command is outstanding here. All R/W PMULT requests 4435 * come from 4436 * 4437 * 1. ahci_attach() 4438 * The port should be empty. 4439 * 4440 * 2. ahci_tran_probe_port() 4441 * Any request from SATA framework (via ahci_tran_start) should be 4442 * rejected if R/W PMULT command is outstanding. 4443 * 4444 * If we are doing mopping, do not check those flags because no 4445 * command will be actually outstanding. 4446 * 4447 * If the port has been occupied by any other commands, the probe 4448 * function will return a SATA_RETRY. SATA framework will retry 4449 * later. 4450 */ 4451 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 4452 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4453 "R/W PMULT failed: R/W PMULT in progress at port %d.", 4454 port, ahci_portp->ahciport_flags); 4455 return (AHCI_FAILURE); 4456 } 4457 4458 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && ( 4459 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 4460 NCQ_CMD_IN_PROGRESS(ahci_portp) || 4461 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) { 4462 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4463 "R/W PMULT failed: port %d is occupied (flags 0x%x).", 4464 port, ahci_portp->ahciport_flags); 4465 return (AHCI_FAILURE); 4466 } 4467 4468 /* 4469 * The port multiplier is gone. This may happen when 4470 * 1. Cutting off the power of an enclosure. The device lose the power 4471 * before port multiplier. 4472 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive. 4473 * 4474 * The issued command should be aborted and the following command 4475 * should not be continued. 4476 */ 4477 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) { 4478 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4479 "READ/WRITE PMULT failed: " 4480 "port-mult is removed from port %d", port); 4481 return (AHCI_FAILURE); 4482 } 4483 4484 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT; 4485 4486 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip, 4487 &sata_device, regn, *pregv, type); 4488 4489 /* 4490 * READ/WRITE PORTMULT command is intended to sent to the control port 4491 * of the port multiplier. 4492 */ 4493 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port); 4494 4495 ahci_portp->ahciport_rdwr_pmult_pkt = spkt; 4496 4497 /* No interrupt here. Store the interrupt enable mask. */ 4498 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4499 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4500 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4501 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 4502 4503 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt); 4504 4505 if (rval == AHCI_SUCCESS && 4506 spkt->satapkt_reason == SATA_PKT_COMPLETED) { 4507 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) { 4508 scmd = &spkt->satapkt_cmd; 4509 *pregv = scmd->satacmd_lba_high_lsb << 24 | 4510 scmd->satacmd_lba_mid_lsb << 16 | 4511 scmd->satacmd_lba_low_lsb << 8 | 4512 scmd->satacmd_sec_count_lsb; 4513 } 4514 } else { 4515 /* Failed or not completed. */ 4516 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4517 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s", 4518 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write", 4519 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr); 4520 rval = AHCI_FAILURE; 4521 } 4522 out: 4523 /* Restore the interrupt mask */ 4524 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4525 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask); 4526 4527 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT; 4528 ahci_portp->ahciport_rdwr_pmult_pkt = NULL; 4529 sata_free_rdwr_pmult_pkt(spkt); 4530 return (rval); 4531 } 4532 4533 static int 4534 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4535 uint8_t regn, uint32_t *pregv) 4536 { 4537 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv, 4538 SATA_RDWR_PMULT_PKT_TYPE_READ); 4539 } 4540 4541 static int 4542 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4543 uint8_t regn, uint32_t regv) 4544 { 4545 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v, 4546 SATA_RDWR_PMULT_PKT_TYPE_WRITE); 4547 } 4548 4549 #define READ_PMULT(addrp, r, pv, out) \ 4550 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \ 4551 goto out; 4552 4553 #define WRITE_PMULT(addrp, r, v, out) \ 4554 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \ 4555 goto out; 4556 4557 /* 4558 * Update sata registers on port multiplier, including GSCR/PSCR registers. 4559 * ahci_update_pmult_gscr() 4560 * ahci_update_pmult_pscr() 4561 */ 4562 static int 4563 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4564 sata_pmult_gscr_t *sg) 4565 { 4566 ASSERT(MUTEX_HELD( 4567 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex)); 4568 4569 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err); 4570 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err); 4571 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err); 4572 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err); 4573 4574 return (AHCI_SUCCESS); 4575 4576 err: /* R/W PMULT error */ 4577 return (AHCI_FAILURE); 4578 } 4579 4580 static int 4581 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4582 sata_device_t *sd) 4583 { 4584 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4585 ASSERT(MUTEX_HELD( 4586 &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex)); 4587 4588 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err); 4589 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err); 4590 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err); 4591 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err); 4592 4593 return (AHCI_SUCCESS); 4594 4595 err: /* R/W PMULT error */ 4596 return (AHCI_FAILURE); 4597 } 4598 4599 /* 4600 * ahci_initialize_pmult() 4601 * 4602 * Initialize a port multiplier, including 4603 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16) 4604 * 2. Redefine MASK register. (SATA Chap 16.?) 4605 */ 4606 static int 4607 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4608 ahci_addr_t *addrp, sata_device_t *sd) 4609 { 4610 sata_pmult_gscr_t sg; 4611 uint32_t gscr64; 4612 uint8_t port = addrp->aa_port; 4613 4614 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4615 4616 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4617 "[Initialize] Port-multiplier at port %d.", port); 4618 4619 /* 4620 * Enable features of port multiplier. Currently only 4621 * Asynchronous Notification is enabled. 4622 */ 4623 /* Check gscr64 for supported features. */ 4624 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err); 4625 4626 if (gscr64 & SATA_PMULT_CAP_SNOTIF) { 4627 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4628 "port %d: Port Multiplier supports " 4629 "Asynchronous Notification.", port); 4630 4631 /* Write to gscr96 to enabled features */ 4632 WRITE_PMULT(addrp, SATA_PMULT_GSCR96, 4633 SATA_PMULT_CAP_SNOTIF, err); 4634 4635 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4636 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4637 AHCI_SNOTIF_CLEAR_ALL); 4638 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4639 "port %d: PMult PxSNTF cleared.", port); 4640 4641 } 4642 4643 /* 4644 * Now we need to update gscr33 register to enable hot-plug interrupt 4645 * for sub devices behind port multiplier. 4646 */ 4647 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err); 4648 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4649 "port %d: gscr33 mask set to %x.", port, (0x1ffff)); 4650 4651 /* 4652 * Fetch the number of device ports of the port multiplier 4653 */ 4654 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS) 4655 return (AHCI_FAILURE); 4656 4657 /* Register the port multiplier to SATA Framework. */ 4658 mutex_exit(&ahci_portp->ahciport_mutex); 4659 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg); 4660 mutex_enter(&ahci_portp->ahciport_mutex); 4661 4662 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports = 4663 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK; 4664 4665 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4666 "port %d: pmult sub-port number updated to %x.", port, 4667 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports); 4668 4669 /* Till now port-mult is successfully initialized */ 4670 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT; 4671 return (AHCI_SUCCESS); 4672 4673 err: /* R/W PMULT error */ 4674 return (AHCI_FAILURE); 4675 } 4676 4677 /* 4678 * Initialize a port multiplier port. According to spec, firstly we need 4679 * issue a COMRESET, then a software reset to get its signature. 4680 * 4681 * NOTE: This function should only be called in ahci_probe_pmport() 4682 */ 4683 static int 4684 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4685 ahci_addr_t *addrp) 4686 { 4687 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 4688 uint8_t port = addrp->aa_port; 4689 uint8_t pmport = addrp->aa_pmport; 4690 int ret = AHCI_FAILURE; 4691 4692 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4693 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4694 4695 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4696 "ahci_initialize_pmport: port %d:%d", port, pmport); 4697 4698 /* Check HBA port state */ 4699 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4700 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4701 "ahci_initialize_pmport:" 4702 "port %d:%d Port Multiplier is failed.", 4703 port, pmport); 4704 return (AHCI_FAILURE); 4705 } 4706 4707 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4708 return (AHCI_FAILURE); 4709 } 4710 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 4711 4712 /* Checking for outstanding commands */ 4713 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4714 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4715 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4716 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4717 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4718 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4719 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4720 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4721 } 4722 4723 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 4724 ahci_portp->ahciport_mop_in_progress++; 4725 4726 /* Clear status */ 4727 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN); 4728 4729 /* Firstly assume an unknown device */ 4730 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4731 4732 ahci_disable_port_intrs(ahci_ctlp, port); 4733 4734 /* port reset is necessary for port multiplier port */ 4735 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) { 4736 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4737 "ahci_initialize_pmport:" 4738 "port reset failed at port %d:%d", 4739 port, pmport); 4740 goto out; 4741 } 4742 4743 /* Is port failed? */ 4744 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4745 SATA_PSTATE_FAILED) { 4746 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4747 "ahci_initialize_pmport: port %d:%d failed. " 4748 "state = 0x%x", port, pmport, 4749 ahci_portp->ahciport_port_state); 4750 goto out; 4751 } 4752 4753 /* Is there any device attached? */ 4754 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp) 4755 == SATA_DTYPE_NONE) { 4756 /* Do not waste time on an empty port */ 4757 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4758 "ahci_initialize_pmport: No device is found " 4759 "at port %d:%d", port, pmport); 4760 ret = AHCI_SUCCESS; 4761 goto out; 4762 } 4763 4764 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4765 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4766 "port %d:%d is ready now.", port, pmport); 4767 4768 /* 4769 * Till now we can assure a device attached to that HBA port and work 4770 * correctly. Now try to get the device signature. This is an optional 4771 * step. If failed, unknown device is assumed, then SATA module will 4772 * continue to use IDENTIFY DEVICE to get the information of the 4773 * device. 4774 */ 4775 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4776 4777 ret = AHCI_SUCCESS; 4778 4779 out: 4780 /* Next try to mop the pending commands */ 4781 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 4782 finished_tags = ahci_portp->ahciport_pending_tags & 4783 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4784 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 4785 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4786 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4787 reset_tags &= ~finished_tags; 4788 4789 ahci_mop_commands(ahci_ctlp, 4790 ahci_portp, 4791 slot_status, 4792 0, /* failed tags */ 4793 0, /* timeout tags */ 4794 0, /* aborted tags */ 4795 reset_tags); /* reset tags */ 4796 4797 /* Clear PxSNTF register if supported. */ 4798 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 4799 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4800 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4801 AHCI_SNOTIF_CLEAR_ALL); 4802 } 4803 4804 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 4805 ahci_enable_port_intrs(ahci_ctlp, port); 4806 return (ret); 4807 } 4808 4809 /* 4810 * ahci_probe_pmult() 4811 * 4812 * This function will be called to probe a port multiplier, which will 4813 * handle hotplug events on port multiplier ports. 4814 * 4815 * NOTE: Only called from ahci_tran_probe_port() 4816 */ 4817 static int 4818 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4819 ahci_addr_t *addrp) 4820 { 4821 sata_device_t sdevice; 4822 ahci_addr_t pmport_addr; 4823 uint32_t gscr32, port_hotplug_tags; 4824 uint32_t pmport_sstatus; 4825 int dev_exists_now = 0, dev_existed_previously = 0; 4826 uint8_t port = addrp->aa_port; 4827 int npmport; 4828 4829 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4830 4831 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */ 4832 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err); 4833 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp); 4834 4835 do { 4836 npmport = ddi_ffs(port_hotplug_tags) - 1; 4837 if (npmport == -1) 4838 /* no pending hot plug events. */ 4839 return (AHCI_SUCCESS); 4840 4841 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4842 "hot-plug event at port %d:%d", port, npmport); 4843 4844 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport); 4845 4846 /* Check previous device at that port */ 4847 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr) 4848 != SATA_DTYPE_NONE) 4849 dev_existed_previously = 1; 4850 4851 /* PxSStatus tells the presence of device. */ 4852 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS, 4853 &pmport_sstatus, err); 4854 4855 if (SSTATUS_GET_DET(pmport_sstatus) == 4856 SSTATUS_DET_DEVPRE_PHYCOM) 4857 dev_exists_now = 1; 4858 4859 /* 4860 * Clear PxSERR is critical. The transition from 0 to 1 will 4861 * emit a FIS which generates an asynchronous notification 4862 * event at controller. If we fail to clear the PxSERR, the 4863 * Async Notif events will no longer be activated on this 4864 * pmport. 4865 */ 4866 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR, 4867 AHCI_SERROR_CLEAR_ALL, err); 4868 4869 bzero((void *)&sdevice, sizeof (sata_device_t)); 4870 sdevice.satadev_addr.cport = ahci_ctlp-> 4871 ahcictl_port_to_cport[port]; 4872 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT; 4873 sdevice.satadev_addr.pmport = (uint8_t)npmport; 4874 sdevice.satadev_state = SATA_PSTATE_PWRON; 4875 4876 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4877 "[Existence] %d -> %d", dev_existed_previously, 4878 dev_exists_now); 4879 4880 if (dev_exists_now) { 4881 if (dev_existed_previously) { 4882 /* Link (may) not change: Exist -> Exist * */ 4883 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 4884 "ahci_probe_pmult: port %d:%d " 4885 "device link lost/established", 4886 port, npmport); 4887 4888 mutex_exit(&ahci_portp->ahciport_mutex); 4889 sata_hba_event_notify( 4890 ahci_ctlp->ahcictl_sata_hba_tran-> 4891 sata_tran_hba_dip, 4892 &sdevice, 4893 SATA_EVNT_LINK_LOST| 4894 SATA_EVNT_LINK_ESTABLISHED); 4895 mutex_enter(&ahci_portp->ahciport_mutex); 4896 } else { 4897 /* Link change: None -> Exist */ 4898 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4899 "ahci_probe_pmult: port %d:%d " 4900 "device link established", port, npmport); 4901 4902 /* Clear port state */ 4903 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4904 SATA_STATE_UNKNOWN); 4905 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4906 "ahci_probe_pmult: port %d " 4907 "ahciport_port_state [Cleared].", port); 4908 4909 mutex_exit(&ahci_portp->ahciport_mutex); 4910 sata_hba_event_notify( 4911 ahci_ctlp->ahcictl_sata_hba_tran-> 4912 sata_tran_hba_dip, 4913 &sdevice, 4914 SATA_EVNT_LINK_ESTABLISHED); 4915 mutex_enter(&ahci_portp->ahciport_mutex); 4916 } 4917 } else { /* No device exists now */ 4918 if (dev_existed_previously) { 4919 4920 /* Link change: Exist -> None */ 4921 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4922 "ahci_probe_pmult: port %d:%d " 4923 "device link lost", port, npmport); 4924 4925 /* An existing device is lost. */ 4926 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4927 SATA_STATE_UNKNOWN); 4928 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr, 4929 SATA_DTYPE_NONE); 4930 4931 mutex_exit(&ahci_portp->ahciport_mutex); 4932 sata_hba_event_notify( 4933 ahci_ctlp->ahcictl_sata_hba_tran-> 4934 sata_tran_hba_dip, 4935 &sdevice, 4936 SATA_EVNT_LINK_LOST); 4937 mutex_enter(&ahci_portp->ahciport_mutex); 4938 } 4939 } 4940 4941 CLEAR_BIT(port_hotplug_tags, npmport); 4942 } while (port_hotplug_tags != 0); 4943 4944 return (AHCI_SUCCESS); 4945 4946 err: /* R/W PMULT error */ 4947 return (AHCI_FAILURE); 4948 } 4949 4950 /* 4951 * Probe and initialize a port multiplier port. 4952 * A port multiplier port could only be initilaizer here. 4953 */ 4954 static int 4955 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4956 ahci_addr_t *addrp, sata_device_t *sd) 4957 { 4958 uint32_t port_state; 4959 uint8_t port = addrp->aa_port; 4960 ahci_addr_t addr_pmult; 4961 4962 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 4963 4964 /* 4965 * Check the parent - port multiplier first. 4966 */ 4967 4968 /* 4969 * Parent port multiplier might have been removed. This event will be 4970 * ignored and failure. 4971 */ 4972 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 4973 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 4974 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4975 "ahci_tran_probe_port: " 4976 "parent device removed, ignore event.", NULL); 4977 4978 return (AHCI_FAILURE); 4979 } 4980 4981 /* The port is ready? */ 4982 port_state = ahci_portp->ahciport_port_state; 4983 if (!(port_state & SATA_STATE_READY)) { 4984 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4985 "ahci_tran_probe_port: " 4986 "parent port-mult is NOT ready.", NULL); 4987 4988 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 4989 ahci_portp, port, AHCI_PORT_RESET, NULL) != 4990 AHCI_SUCCESS) { 4991 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4992 "ahci_tran_probe_port: " 4993 "restart port-mult failed.", NULL); 4994 return (AHCI_FAILURE); 4995 } 4996 } 4997 4998 /* 4999 * If port-mult is restarted due to some reason, we need 5000 * re-initialized the PMult. 5001 */ 5002 if (!(port_state & SATA_DSTATE_PMULT_INIT)) { 5003 /* Initialize registers on a port multiplier */ 5004 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port); 5005 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp, 5006 &addr_pmult, sd) != AHCI_SUCCESS) 5007 return (AHCI_FAILURE); 5008 } 5009 5010 /* 5011 * Then we check the port-mult port 5012 */ 5013 /* Is this pmport initialized? */ 5014 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5015 if (!(port_state & SATA_STATE_READY)) { 5016 5017 /* ahci_initialize_pmport() will set READY state */ 5018 if (ahci_initialize_pmport(ahci_ctlp, 5019 ahci_portp, addrp) != AHCI_SUCCESS) 5020 return (AHCI_FAILURE); 5021 } 5022 5023 return (AHCI_SUCCESS); 5024 } 5025 5026 /* 5027 * AHCI device reset ...; a single device on one of the ports is reset, 5028 * but the HBA and physical communication remain intact. This is the 5029 * least intrusive. 5030 * 5031 * When issuing a software reset sequence, there should not be other 5032 * commands in the command list, so we will first clear and then re-set 5033 * PxCMD.ST to clear PxCI. And before issuing the software reset, 5034 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 5035 * cleared unless command list override (PxCMD.CLO) is supported. 5036 */ 5037 static int 5038 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5039 ahci_addr_t *addrp) 5040 { 5041 ahci_fis_h2d_register_t *h2d_register_fisp; 5042 ahci_cmd_table_t *cmd_table; 5043 ahci_cmd_header_t *cmd_header; 5044 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 5045 int slot, loop_count; 5046 uint8_t port = addrp->aa_port; 5047 uint8_t pmport = addrp->aa_pmport; 5048 int rval = AHCI_FAILURE; 5049 5050 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5051 5052 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5053 "port %d:%d device software resetting (FIS)", port, pmport); 5054 5055 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 5056 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 5057 port) != AHCI_SUCCESS) { 5058 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5059 "ahci_software_reset: cannot stop HBA port %d.", port); 5060 goto out; 5061 } 5062 5063 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 5064 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5065 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5066 5067 if (port_task_file & AHCI_TFD_STS_BSY || 5068 port_task_file & AHCI_TFD_STS_DRQ) { 5069 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 5070 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5071 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 5072 "cannot issue a software reset.", port_task_file); 5073 goto out; 5074 } 5075 5076 /* 5077 * If HBA Support CLO, as Command List Override (CAP.SCLO is 5078 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 5079 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 5080 */ 5081 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5082 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 5083 5084 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5085 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5086 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5087 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5088 port_cmd_status|AHCI_CMD_STATUS_CLO); 5089 5090 /* Waiting till PxCMD.SCLO bit is cleared */ 5091 loop_count = 0; 5092 do { 5093 /* Wait for 10 millisec */ 5094 drv_usecwait(AHCI_10MS_USECS); 5095 5096 /* We are effectively timing out after 1 sec. */ 5097 if (loop_count++ > 100) { 5098 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5099 "SCLO time out. port %d is busy.", port); 5100 goto out; 5101 } 5102 5103 port_cmd_status = 5104 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5105 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5106 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 5107 5108 /* Re-check */ 5109 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5110 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5111 if (port_task_file & AHCI_TFD_STS_BSY || 5112 port_task_file & AHCI_TFD_STS_DRQ) { 5113 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5114 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 5115 port_task_file); 5116 goto out; 5117 } 5118 } 5119 5120 /* Then start port */ 5121 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5122 != AHCI_SUCCESS) { 5123 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5124 "ahci_software_reset: cannot start AHCI port %d.", port); 5125 goto out; 5126 } 5127 5128 /* 5129 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 5130 * ahci_port.ahciport_pending_ncq_tags may fail 5131 * ahci_claim_free_slot(). Actually according to spec, by clearing 5132 * PxCMD.ST there is no command outstanding while executing software 5133 * reseting. Hence we directly use slot 0 instead of 5134 * ahci_claim_free_slot(). 5135 */ 5136 slot = 0; 5137 5138 /* Now send the first H2D Register FIS with SRST set to 1 */ 5139 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 5140 bzero((void *)cmd_table, ahci_cmd_table_size); 5141 5142 h2d_register_fisp = 5143 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 5144 5145 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 5146 SET_FIS_PMP(h2d_register_fisp, pmport); 5147 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 5148 5149 /* Set Command Header in Command List */ 5150 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 5151 BZERO_DESCR_INFO(cmd_header); 5152 BZERO_PRD_BYTE_COUNT(cmd_header); 5153 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 5154 SET_PORT_MULTI_PORT(cmd_header, pmport); 5155 5156 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 5157 SET_RESET(cmd_header, 1); 5158 SET_WRITE(cmd_header, 1); 5159 5160 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 5161 0, 5162 ahci_cmd_table_size, 5163 DDI_DMA_SYNC_FORDEV); 5164 5165 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 5166 slot * sizeof (ahci_cmd_header_t), 5167 sizeof (ahci_cmd_header_t), 5168 DDI_DMA_SYNC_FORDEV); 5169 5170 /* Indicate to the HBA that a command is active. */ 5171 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5172 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 5173 (0x1 << slot)); 5174 5175 loop_count = 0; 5176 5177 /* Loop till the first command is finished */ 5178 do { 5179 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5180 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5181 5182 /* We are effectively timing out after 1 sec. */ 5183 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 5184 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5185 "the first SRST FIS is timed out, " 5186 "loop_count = %d", loop_count); 5187 goto out; 5188 } 5189 /* Wait for 10 millisec */ 5190 drv_usecwait(AHCI_10MS_USECS); 5191 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 5192 5193 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5194 "ahci_software_reset: 1st loop count: %d, " 5195 "port_cmd_issue = 0x%x, slot = 0x%x", 5196 loop_count, port_cmd_issue, slot); 5197 5198 /* According to ATA spec, we need wait at least 5 microsecs here. */ 5199 drv_usecwait(AHCI_1MS_USECS); 5200 5201 /* Now send the second H2D Register FIS with SRST cleard to zero */ 5202 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 5203 bzero((void *)cmd_table, ahci_cmd_table_size); 5204 5205 h2d_register_fisp = 5206 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 5207 5208 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 5209 SET_FIS_PMP(h2d_register_fisp, pmport); 5210 5211 /* Set Command Header in Command List */ 5212 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 5213 BZERO_DESCR_INFO(cmd_header); 5214 BZERO_PRD_BYTE_COUNT(cmd_header); 5215 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 5216 SET_PORT_MULTI_PORT(cmd_header, pmport); 5217 5218 SET_WRITE(cmd_header, 1); 5219 5220 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 5221 0, 5222 ahci_cmd_table_size, 5223 DDI_DMA_SYNC_FORDEV); 5224 5225 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 5226 slot * sizeof (ahci_cmd_header_t), 5227 sizeof (ahci_cmd_header_t), 5228 DDI_DMA_SYNC_FORDEV); 5229 5230 /* Indicate to the HBA that a command is active. */ 5231 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5232 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 5233 (0x1 << slot)); 5234 5235 loop_count = 0; 5236 5237 /* Loop till the second command is finished */ 5238 do { 5239 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5240 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5241 5242 /* We are effectively timing out after 1 sec. */ 5243 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 5244 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5245 "the second SRST FIS is timed out, " 5246 "loop_count = %d", loop_count); 5247 goto out; 5248 } 5249 5250 /* Wait for 10 millisec */ 5251 drv_usecwait(AHCI_10MS_USECS); 5252 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 5253 5254 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5255 "ahci_software_reset: 2nd loop count: %d, " 5256 "port_cmd_issue = 0x%x, slot = 0x%x", 5257 loop_count, port_cmd_issue, slot); 5258 5259 if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) || 5260 (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) { 5261 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 5262 DDI_SERVICE_UNAFFECTED); 5263 goto out; 5264 } 5265 5266 rval = AHCI_SUCCESS; 5267 out: 5268 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5269 "ahci_software_reset: %s at port %d:%d", 5270 rval == AHCI_SUCCESS ? "succeed" : "failed", 5271 port, pmport); 5272 5273 return (rval); 5274 } 5275 5276 /* 5277 * AHCI port reset ...; the physical communication between the HBA and device 5278 * on a port are disabled. This is more intrusive. 5279 * 5280 * When an HBA or port reset occurs, Phy communication is going to 5281 * be re-established with the device through a COMRESET followed by the 5282 * normal out-of-band communication sequence defined in Serial ATA. At 5283 * the end of reset, the device, if working properly, will send a D2H 5284 * Register FIS, which contains the device signature. When the HBA receives 5285 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 5286 * the PxSIG register with the signature. 5287 * 5288 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the 5289 * function is called. If not, it is assumed the interface is in hung 5290 * condition. 5291 */ 5292 static int 5293 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5294 ahci_addr_t *addrp) 5295 { 5296 ahci_addr_t pmult_addr; 5297 uint32_t port_cmd_status; 5298 uint32_t port_scontrol, port_sstatus; 5299 uint32_t port_task_file; 5300 uint32_t port_state; 5301 uint8_t port = addrp->aa_port; 5302 5303 int loop_count; 5304 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5305 5306 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5307 5308 /* Target is a port multiplier port? */ 5309 if (AHCI_ADDR_IS_PMPORT(addrp)) 5310 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp)); 5311 5312 /* Otherwise it must be an HBA port. */ 5313 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5314 5315 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5316 "Port %d port resetting...", port); 5317 ahci_portp->ahciport_port_state = 0; 5318 5319 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5320 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5321 5322 /* 5323 * According to the spec, SUD bit should be set here, 5324 * but JMicron JMB363 doesn't follow it, so print 5325 * a debug message. 5326 */ 5327 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 5328 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5329 "ahci_port_reset: port %d SUD bit not set", port); 5330 5331 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5332 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5333 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5334 5335 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5336 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 5337 port_scontrol); 5338 5339 /* Enable PxCMD.FRE to read device */ 5340 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5341 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5342 port_cmd_status|AHCI_CMD_STATUS_FRE); 5343 5344 /* 5345 * The port enters P:StartComm state, and the HBA tells the link layer 5346 * to start communication, which involves sending COMRESET to the 5347 * device. And the HBA resets PxTFD.STS to 7Fh. 5348 * 5349 * Give time for COMRESET to percolate, according to the AHCI 5350 * spec, software shall wait at least 1 millisecond before 5351 * clearing PxSCTL.DET 5352 */ 5353 drv_usecwait(AHCI_1MS_USECS * 2); 5354 5355 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 5356 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5357 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5358 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5359 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5360 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 5361 port_scontrol); 5362 5363 /* 5364 * When a COMINIT is received from the device, then the port enters 5365 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5366 * PxSSTS.DET to 1h to indicate a device is detected but communication 5367 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5368 * a COMINIT has been received. 5369 */ 5370 /* 5371 * The DET field is valid only if IPM field indicates 5372 * that the interface is in active state. 5373 */ 5374 loop_count = 0; 5375 for (;;) { 5376 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5377 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5378 5379 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5380 /* 5381 * If the interface is not active, the DET field 5382 * is considered not accurate. So we want to 5383 * continue looping. 5384 */ 5385 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5386 } 5387 5388 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) 5389 break; 5390 5391 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5392 /* 5393 * We are effectively timing out after 0.1 sec. 5394 */ 5395 break; 5396 } 5397 5398 /* Wait for 10 millisec */ 5399 drv_usecwait(AHCI_10MS_USECS); 5400 } 5401 5402 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5403 "ahci_port_reset: 1st loop count: %d, " 5404 "port_sstatus = 0x%x port %d", 5405 loop_count, port_sstatus, port); 5406 5407 if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) { 5408 /* 5409 * Either the port is not active or there 5410 * is no device present. 5411 */ 5412 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5413 return (AHCI_SUCCESS); 5414 } 5415 5416 /* Clear port serror register for the port */ 5417 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5418 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5419 AHCI_SERROR_CLEAR_ALL); 5420 5421 /* 5422 * Devices should return a FIS contains its signature to HBA after 5423 * COMINIT signal. Check whether a D2H Register FIS is received by 5424 * polling PxTFD.STS. 5425 */ 5426 loop_count = 0; 5427 for (;;) { 5428 port_task_file = 5429 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5430 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5431 5432 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ | 5433 AHCI_TFD_STS_ERR)) == 0) 5434 break; 5435 5436 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5437 /* 5438 * We are effectively timing out after 11 sec. 5439 */ 5440 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5441 "the device hardware has been initialized and " 5442 "the power-up diagnostics failed", 5443 instance, port); 5444 5445 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5446 "port %d: some or all of BSY, DRQ and ERR in " 5447 "PxTFD.STS are not clear. We need another " 5448 "software reset.", port); 5449 5450 /* Clear port serror register for the port */ 5451 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5452 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5453 AHCI_SERROR_CLEAR_ALL); 5454 5455 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 5456 5457 /* Try another software reset. */ 5458 if (ahci_software_reset(ahci_ctlp, ahci_portp, 5459 &pmult_addr) != AHCI_SUCCESS) { 5460 AHCIPORT_SET_STATE(ahci_portp, addrp, 5461 SATA_PSTATE_FAILED); 5462 return (AHCI_FAILURE); 5463 } 5464 break; 5465 } 5466 5467 /* Wait for 10 millisec */ 5468 drv_usecwait(AHCI_10MS_USECS); 5469 } 5470 5471 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5472 "ahci_port_reset: 2nd loop count: %d, " 5473 "port_task_file = 0x%x port %d", 5474 loop_count, port_task_file, port); 5475 5476 /* Clear port serror register for the port */ 5477 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5478 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5479 AHCI_SERROR_CLEAR_ALL); 5480 5481 /* Set port as ready */ 5482 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5483 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5484 5485 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5486 "ahci_port_reset: succeed at port %d.", port); 5487 return (AHCI_SUCCESS); 5488 } 5489 5490 /* 5491 * COMRESET on a port multiplier port. 5492 * 5493 * NOTE: Only called in ahci_port_reset() 5494 */ 5495 static int 5496 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5497 ahci_addr_t *addrp) 5498 { 5499 uint32_t port_scontrol, port_sstatus, port_serror; 5500 uint32_t port_cmd_status, port_intr_status; 5501 uint32_t port_state; 5502 uint8_t port = addrp->aa_port; 5503 uint8_t pmport = addrp->aa_pmport; 5504 int loop_count; 5505 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5506 5507 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5508 "port %d:%d: pmport resetting", port, pmport); 5509 5510 /* Initialize pmport state */ 5511 AHCIPORT_SET_STATE(ahci_portp, addrp, 0); 5512 5513 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5514 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5515 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5516 5517 /* PxCMD.FRE should be set before. */ 5518 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5519 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5520 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE); 5521 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) 5522 return (AHCI_FAILURE); 5523 5524 /* 5525 * Give time for COMRESET to percolate, according to the AHCI 5526 * spec, software shall wait at least 1 millisecond before 5527 * clearing PxSCTL.DET 5528 */ 5529 drv_usecwait(AHCI_1MS_USECS*2); 5530 5531 /* 5532 * Fetch the SCONTROL again and rewrite the DET part with 0 5533 * This will generate an Asychronous Notification events. 5534 */ 5535 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5536 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5537 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5538 5539 /* 5540 * The port enters P:StartComm state, and HBA tells link layer to 5541 * start communication, which involves sending COMRESET to device. 5542 * And the HBA resets PxTFD.STS to 7Fh. 5543 * 5544 * When a COMINIT is received from the device, then the port enters 5545 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5546 * PxSSTS.DET to 1h to indicate a device is detected but communication 5547 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5548 * a COMINIT has been received. 5549 */ 5550 /* 5551 * The DET field is valid only if IPM field indicates 5552 * that the interface is in active state. 5553 */ 5554 loop_count = 0; 5555 do { 5556 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err); 5557 5558 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5559 /* 5560 * If the interface is not active, the DET field 5561 * is considered not accurate. So we want to 5562 * continue looping. 5563 */ 5564 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5565 } 5566 5567 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5568 /* 5569 * We are effectively timing out after 0.1 sec. 5570 */ 5571 break; 5572 } 5573 5574 /* Wait for 10 millisec */ 5575 drv_usecwait(AHCI_10MS_USECS); 5576 5577 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5578 5579 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5580 "ahci_pmport_reset: 1st loop count: %d, " 5581 "port_sstatus = 0x%x port %d:%d", 5582 loop_count, port_sstatus, port, pmport); 5583 5584 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5585 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5586 /* 5587 * Either the port is not active or there 5588 * is no device present. 5589 */ 5590 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5591 "ahci_pmport_reset: " 5592 "no device attached to port %d:%d", 5593 port, pmport); 5594 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5595 return (AHCI_SUCCESS); 5596 } 5597 5598 /* Now we can make sure there is a device connected to the port */ 5599 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */ 5600 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err); 5601 5602 if (!(port_serror & (1 << 26))) { 5603 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: " 5604 "COMINIT signal from the device not received port %d:%d", 5605 instance, port, pmport); 5606 5607 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5608 return (AHCI_FAILURE); 5609 } 5610 5611 /* 5612 * After clear PxSERR register, we will receive a D2H FIS. 5613 * Normally this FIS will cause a IPMS error according to AHCI spec 5614 * v1.2 because there is no command outstanding for it. So we need 5615 * to ignore this error. 5616 */ 5617 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS; 5618 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err); 5619 5620 /* Now we need to check the D2H FIS by checking IPMS error. */ 5621 loop_count = 0; 5622 do { 5623 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5624 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5625 5626 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5627 /* 5628 * No D2H FIS received. This is possible according 5629 * to SATA 2.6 spec. 5630 */ 5631 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d " 5632 "PxIS.IPMS is not set, we need another " 5633 "software reset.", port, pmport); 5634 5635 break; 5636 } 5637 5638 /* Wait for 10 millisec */ 5639 mutex_exit(&ahci_portp->ahciport_mutex); 5640 delay(AHCI_10MS_TICKS); 5641 mutex_enter(&ahci_portp->ahciport_mutex); 5642 5643 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS)); 5644 5645 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5646 "ahci_pmport_reset: 2st loop count: %d, " 5647 "port_sstatus = 0x%x port %d:%d", 5648 loop_count, port_sstatus, port, pmport); 5649 5650 /* Clear IPMS */ 5651 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5652 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5653 AHCI_INTR_STATUS_IPMS); 5654 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5655 5656 /* This pmport is now ready for ahci_tran_start() */ 5657 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5658 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5659 5660 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5661 "ahci_pmport_reset: succeed at port %d:%d", port, pmport); 5662 return (AHCI_SUCCESS); 5663 5664 err: /* R/W PMULT error */ 5665 /* IPMS flags might be set before. */ 5666 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5667 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5668 "ahci_pmport_reset: failed at port %d:%d", port, pmport); 5669 5670 return (AHCI_FAILURE); 5671 } 5672 5673 /* 5674 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 5675 * This is the most intrusive. 5676 * 5677 * When an HBA reset occurs, Phy communication will be re-established with 5678 * the device through a COMRESET followed by the normal out-of-band 5679 * communication sequence defined in Serial ATA. At the end of reset, the 5680 * device, if working properly, will send a D2H Register FIS, which contains 5681 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 5682 * and PxTFD.ERR register fields, and updates the PxSIG register with the 5683 * signature. 5684 * 5685 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 5686 */ 5687 static int 5688 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 5689 { 5690 ahci_port_t *ahci_portp; 5691 uint32_t ghc_control; 5692 uint8_t port; 5693 int loop_count; 5694 int rval = AHCI_SUCCESS; 5695 5696 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 5697 NULL); 5698 5699 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5700 5701 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5702 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5703 5704 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 5705 ghc_control |= AHCI_HBA_GHC_HR; 5706 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5707 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5708 5709 /* 5710 * Wait until HBA Reset complete or timeout 5711 */ 5712 loop_count = 0; 5713 do { 5714 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5715 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5716 5717 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 5718 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5719 "ahci hba reset is timing out, " 5720 "ghc_control = 0x%x", ghc_control); 5721 /* We are effectively timing out after 1 sec. */ 5722 break; 5723 } 5724 5725 /* Wait for 10 millisec */ 5726 drv_usecwait(AHCI_10MS_USECS); 5727 } while (ghc_control & AHCI_HBA_GHC_HR); 5728 5729 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5730 "ahci_hba_reset: 1st loop count: %d, " 5731 "ghc_control = 0x%x", loop_count, ghc_control); 5732 5733 if (ghc_control & AHCI_HBA_GHC_HR) { 5734 /* The hba is not reset for some reasons */ 5735 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5736 "hba reset failed: HBA in a hung or locked state", NULL); 5737 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5738 return (AHCI_FAILURE); 5739 } 5740 5741 /* 5742 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE 5743 */ 5744 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5745 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5746 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE); 5747 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5748 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5749 5750 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5751 5752 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 5753 /* Only check implemented ports */ 5754 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 5755 continue; 5756 } 5757 5758 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5759 mutex_enter(&ahci_portp->ahciport_mutex); 5760 5761 /* Make sure the drive is spun-up */ 5762 ahci_staggered_spin_up(ahci_ctlp, port); 5763 5764 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 5765 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) != 5766 AHCI_SUCCESS) { 5767 rval = AHCI_FAILURE; 5768 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5769 "ahci_hba_reset: port %d failed", port); 5770 /* 5771 * Set the port state to SATA_PSTATE_FAILED if 5772 * failed to initialize it. 5773 */ 5774 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5775 } 5776 5777 mutex_exit(&ahci_portp->ahciport_mutex); 5778 } 5779 5780 return (rval); 5781 } 5782 5783 /* 5784 * This routine is only called from AHCI_ATTACH or phyrdy change 5785 * case. It first calls software reset, then stop the port and try to 5786 * read PxSIG register to find the type of device attached to the port. 5787 * 5788 * The caller should make sure a valid device exists on specified port and 5789 * physical communication has been established so that the signature could 5790 * be retrieved by software reset. 5791 * 5792 * NOTE: The port interrupts should be disabled before the function is called. 5793 */ 5794 static void 5795 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5796 ahci_addr_t *addrp) 5797 { 5798 ahci_addr_t dev_addr; 5799 uint32_t signature; 5800 uint8_t port = addrp->aa_port; 5801 uint8_t pmport = addrp->aa_pmport; 5802 int rval; 5803 5804 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5805 ASSERT(AHCI_ADDR_IS_VALID(addrp)); 5806 5807 /* 5808 * If the HBA doesn't support port multiplier, then the driver 5809 * doesn't need to bother to check port multiplier device. 5810 * 5811 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is 5812 * connected to Silicon Image 4723, to which the two sata drives 5813 * attached can be set with RAID1, RAID0 or Spanning mode. 5814 * 5815 * We found software reset will get failure if port multiplier address 5816 * 0xf is used by software reset, so just ignore the check since 5817 * ICH7 doesn't support port multiplier device at all. 5818 */ 5819 if (AHCI_ADDR_IS_PORT(addrp) && 5820 (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) { 5821 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5822 "ahci_find_dev_signature enter: port %d", port); 5823 5824 /* 5825 * NOTE: when the ahci address is a HBA port, we do not know 5826 * it is a device or a port multiplier that attached. we need 5827 * try a software reset at port multiplier address (0xf 5828 * pmport) 5829 */ 5830 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port); 5831 } else { 5832 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5833 "ahci_find_dev_signature enter: port %d:%d", 5834 port, pmport); 5835 dev_addr = *addrp; 5836 } 5837 5838 /* Assume it is unknown. */ 5839 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5840 5841 /* Issue a software reset to get the signature */ 5842 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr); 5843 if (rval != AHCI_SUCCESS) { 5844 5845 /* 5846 * Try to do software reset again with pmport set with 0 if 5847 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and 5848 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf) 5849 */ 5850 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) && 5851 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) { 5852 dev_addr.aa_pmport = 0; 5853 rval = ahci_software_reset(ahci_ctlp, ahci_portp, 5854 &dev_addr); 5855 } 5856 5857 if (rval != AHCI_SUCCESS) { 5858 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5859 "ahci_find_dev_signature: software reset failed " 5860 "at port %d:%d, cannot get signature.", 5861 port, pmport); 5862 5863 AHCIPORT_SET_STATE(ahci_portp, addrp, 5864 SATA_PSTATE_FAILED); 5865 return; 5866 } 5867 } 5868 5869 /* 5870 * ahci_software_reset has started the port, so we need manually stop 5871 * the port again. 5872 */ 5873 if (AHCI_ADDR_IS_PORT(addrp)) { 5874 if (ahci_put_port_into_notrunning_state(ahci_ctlp, 5875 ahci_portp, port) != AHCI_SUCCESS) { 5876 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5877 "ahci_find_dev_signature: cannot stop port %d.", 5878 port); 5879 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5880 return; 5881 } 5882 } 5883 5884 /* Now we can make sure that a valid signature is received. */ 5885 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5886 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 5887 5888 if (AHCI_ADDR_IS_PMPORT(addrp)) { 5889 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 5890 "ahci_find_dev_signature: signature = 0x%x at port %d:%d", 5891 signature, port, pmport); 5892 } else { 5893 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5894 "ahci_find_dev_signature: signature = 0x%x at port %d", 5895 signature, port); 5896 } 5897 5898 /* NOTE: Only support ATAPI device at controller port. */ 5899 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp)) 5900 signature = SATA_DTYPE_UNKNOWN; 5901 5902 switch (signature) { 5903 5904 case AHCI_SIGNATURE_DISK: 5905 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK); 5906 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5907 "Disk is found at port: %d", port); 5908 break; 5909 5910 case AHCI_SIGNATURE_ATAPI: 5911 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI); 5912 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5913 "ATAPI device is found at port: %d", port); 5914 break; 5915 5916 case AHCI_SIGNATURE_PORT_MULTIPLIER: 5917 /* Port Multiplier cannot recursively attached. */ 5918 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5919 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT); 5920 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5921 "Port Multiplier is found at port: %d", port); 5922 break; 5923 5924 default: 5925 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5926 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5927 "Unknown device is found at port: %d", port); 5928 } 5929 } 5930 5931 /* 5932 * According to the spec, to reliably detect hot plug removals, software 5933 * must disable interface power management. Software should perform the 5934 * following initialization on a port after a device is attached: 5935 * Set PxSCTL.IPM to 3h to disable interface state transitions 5936 * Set PxCMD.ALPE to '0' to disable aggressive power management 5937 * Disable device initiated interface power management by SET FEATURE 5938 * 5939 * We can ignore the last item because by default the feature is disabled 5940 */ 5941 static void 5942 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 5943 { 5944 uint32_t port_scontrol, port_cmd_status; 5945 5946 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5947 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5948 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 5949 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5950 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 5951 5952 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5953 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5954 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 5955 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5956 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5957 } 5958 5959 /* 5960 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 5961 * to 1, then set it firstly. 5962 * 5963 * Each port contains two major DMA engines. One DMA engine walks through 5964 * the command list, and is controlled by PxCMD.ST. The second DMA engine 5965 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 5966 * 5967 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 5968 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 5969 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 5970 * 5971 * Software shall not set PxCMD.ST to '1' unless a functional device is 5972 * present on the port(as determined by PxTFD.STS.BSY = '0', 5973 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 5974 */ 5975 static int 5976 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 5977 { 5978 uint32_t port_cmd_status; 5979 5980 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 5981 5982 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 5983 5984 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 5985 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5986 "the state for port %d is 0x%x", 5987 port, ahci_portp->ahciport_port_state); 5988 return (AHCI_FAILURE); 5989 } 5990 5991 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5992 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5993 "no device is attached at port %d", port); 5994 return (AHCI_FAILURE); 5995 } 5996 5997 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 5998 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5999 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6000 6001 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 6002 port_cmd_status |= AHCI_CMD_STATUS_FRE; 6003 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6004 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6005 port_cmd_status); 6006 } 6007 6008 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6009 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6010 6011 port_cmd_status |= AHCI_CMD_STATUS_ST; 6012 6013 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6014 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6015 port_cmd_status); 6016 6017 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 6018 6019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 6020 "PxCMD.ST set to '1' at port %d", port); 6021 6022 return (AHCI_SUCCESS); 6023 } 6024 6025 /* 6026 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need 6027 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared. 6028 * Then set PxCLB, PxCLBU, PxFB, and PxFBU. 6029 */ 6030 static int 6031 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6032 { 6033 uint8_t port = ahci_portp->ahciport_port_num; 6034 uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6035 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6036 6037 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6038 6039 /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */ 6040 if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) { 6041 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6042 port) != AHCI_SUCCESS) 6043 return (AHCI_FAILURE); 6044 6045 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6046 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6047 } 6048 6049 /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */ 6050 if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) { 6051 int loop_count = 0; 6052 6053 /* Clear PxCMD.FRE */ 6054 port_cmd_status &= ~AHCI_CMD_STATUS_FRE; 6055 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6056 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 6057 port_cmd_status); 6058 6059 /* Wait until PxCMD.FR is cleared */ 6060 for (;;) { 6061 port_cmd_status = 6062 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6063 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6064 6065 if (!(port_cmd_status & AHCI_CMD_STATUS_FR)) 6066 break; 6067 6068 if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) { 6069 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp, 6070 "ahci_setup_port_base_addresses: cannot " 6071 "clear PxCMD.FR for port %d.", port); 6072 6073 /* 6074 * We are effectively timing out after 0.5 sec. 6075 * This value is specified in AHCI spec. 6076 */ 6077 return (AHCI_FAILURE); 6078 } 6079 6080 /* Wait for 1 millisec */ 6081 drv_usecwait(AHCI_1MS_USECS); 6082 } 6083 } 6084 6085 /* Step 3: Config Port Command List Base Address */ 6086 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6087 (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 6088 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 6089 6090 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6091 (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port), 6092 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused); 6093 6094 /* Step 4: Config Port Received FIS Base Address */ 6095 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6096 (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 6097 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 6098 6099 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6100 (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port), 6101 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused); 6102 6103 return (AHCI_SUCCESS); 6104 } 6105 6106 /* 6107 * Allocate the ahci_port_t including Received FIS and Command List. 6108 * The argument - port is the physical port number, and not logical 6109 * port number seen by the SATA framework. 6110 */ 6111 static int 6112 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 6113 { 6114 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 6115 ahci_port_t *ahci_portp; 6116 char taskq_name[64] = "event_handle_taskq"; 6117 6118 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 6119 6120 ahci_portp = 6121 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 6122 6123 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 6124 ahci_portp->ahciport_port_num = port; 6125 6126 /* Initialize the port condition variable */ 6127 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 6128 6129 /* Initialize the port mutex */ 6130 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 6131 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 6132 6133 mutex_enter(&ahci_portp->ahciport_mutex); 6134 6135 /* 6136 * Allocate memory for received FIS structure and 6137 * command list for this port 6138 */ 6139 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6140 goto err_case1; 6141 } 6142 6143 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6144 goto err_case2; 6145 } 6146 6147 /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */ 6148 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) != 6149 AHCI_SUCCESS) { 6150 goto err_case3; 6151 } 6152 6153 (void) snprintf(taskq_name + strlen(taskq_name), 6154 sizeof (taskq_name) - strlen(taskq_name), 6155 "_port%d", port); 6156 6157 /* Create the taskq for the port */ 6158 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 6159 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 6160 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 6161 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 6162 goto err_case3; 6163 } 6164 6165 /* Allocate the argument for the taskq */ 6166 ahci_portp->ahciport_event_args = 6167 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 6168 6169 ahci_portp->ahciport_event_args->ahciea_addrp = 6170 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP); 6171 6172 /* Initialize the done queue */ 6173 ahci_portp->ahciport_doneq = NULL; 6174 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq; 6175 ahci_portp->ahciport_doneq_len = 0; 6176 6177 mutex_exit(&ahci_portp->ahciport_mutex); 6178 6179 return (AHCI_SUCCESS); 6180 6181 err_case3: 6182 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 6183 6184 err_case2: 6185 ahci_dealloc_rcvd_fis(ahci_portp); 6186 6187 err_case1: 6188 mutex_exit(&ahci_portp->ahciport_mutex); 6189 mutex_destroy(&ahci_portp->ahciport_mutex); 6190 cv_destroy(&ahci_portp->ahciport_cv); 6191 6192 kmem_free(ahci_portp, sizeof (ahci_port_t)); 6193 6194 return (AHCI_FAILURE); 6195 } 6196 6197 /* 6198 * Reverse of ahci_alloc_port_state(). 6199 */ 6200 static void 6201 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 6202 { 6203 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6204 6205 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 6206 ASSERT(ahci_portp != NULL); 6207 6208 mutex_enter(&ahci_portp->ahciport_mutex); 6209 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp, 6210 sizeof (ahci_addr_t)); 6211 ahci_portp->ahciport_event_args->ahciea_addrp = NULL; 6212 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 6213 ahci_portp->ahciport_event_args = NULL; 6214 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 6215 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 6216 ahci_dealloc_rcvd_fis(ahci_portp); 6217 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 6218 mutex_exit(&ahci_portp->ahciport_mutex); 6219 6220 mutex_destroy(&ahci_portp->ahciport_mutex); 6221 cv_destroy(&ahci_portp->ahciport_cv); 6222 6223 kmem_free(ahci_portp, sizeof (ahci_port_t)); 6224 6225 ahci_ctlp->ahcictl_ports[port] = NULL; 6226 } 6227 6228 /* 6229 * Allocates memory for the Received FIS Structure 6230 */ 6231 static int 6232 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6233 { 6234 size_t rcvd_fis_size; 6235 size_t ret_len; 6236 uint_t cookie_count; 6237 6238 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6239 6240 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 6241 6242 /* allocate rcvd FIS dma handle. */ 6243 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6244 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 6245 DDI_DMA_SLEEP, 6246 NULL, 6247 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 6248 DDI_SUCCESS) { 6249 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6250 "rcvd FIS dma handle alloc failed", NULL); 6251 6252 return (AHCI_FAILURE); 6253 } 6254 6255 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 6256 rcvd_fis_size, 6257 &accattr, 6258 DDI_DMA_CONSISTENT, 6259 DDI_DMA_SLEEP, 6260 NULL, 6261 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 6262 &ret_len, 6263 &ahci_portp->ahciport_rcvd_fis_acc_handle) != 0) { 6264 6265 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6266 "rcvd FIS dma mem alloc fail", NULL); 6267 /* error.. free the dma handle. */ 6268 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6269 return (AHCI_FAILURE); 6270 } 6271 6272 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 6273 NULL, 6274 (caddr_t)ahci_portp->ahciport_rcvd_fis, 6275 rcvd_fis_size, 6276 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6277 DDI_DMA_SLEEP, 6278 NULL, 6279 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 6280 &cookie_count) != DDI_DMA_MAPPED) { 6281 6282 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6283 "rcvd FIS dma handle bind fail", NULL); 6284 /* error.. free the dma handle & free the memory. */ 6285 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 6286 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6287 return (AHCI_FAILURE); 6288 } 6289 6290 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 6291 6292 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 6293 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 6294 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 6295 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 6296 6297 return (AHCI_SUCCESS); 6298 } 6299 6300 /* 6301 * Deallocates the Received FIS Structure 6302 */ 6303 static void 6304 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 6305 { 6306 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6307 6308 /* Unbind the cmd list dma handle first. */ 6309 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 6310 6311 /* Then free the underlying memory. */ 6312 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 6313 6314 /* Now free the handle itself. */ 6315 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 6316 } 6317 6318 /* 6319 * Allocates memory for the Command List, which contains up to 32 entries. 6320 * Each entry contains a command header, which is a 32-byte structure that 6321 * includes the pointer to the command table. 6322 */ 6323 static int 6324 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6325 { 6326 size_t cmd_list_size; 6327 size_t ret_len; 6328 uint_t cookie_count; 6329 6330 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6331 6332 cmd_list_size = 6333 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 6334 6335 /* allocate cmd list dma handle. */ 6336 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6337 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 6338 DDI_DMA_SLEEP, 6339 NULL, 6340 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 6341 6342 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6343 "cmd list dma handle alloc failed", NULL); 6344 return (AHCI_FAILURE); 6345 } 6346 6347 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 6348 cmd_list_size, 6349 &accattr, 6350 DDI_DMA_CONSISTENT, 6351 DDI_DMA_SLEEP, 6352 NULL, 6353 (caddr_t *)&ahci_portp->ahciport_cmd_list, 6354 &ret_len, 6355 &ahci_portp->ahciport_cmd_list_acc_handle) != 0) { 6356 6357 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6358 "cmd list dma mem alloc fail", NULL); 6359 /* error.. free the dma handle. */ 6360 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6361 return (AHCI_FAILURE); 6362 } 6363 6364 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 6365 NULL, 6366 (caddr_t)ahci_portp->ahciport_cmd_list, 6367 cmd_list_size, 6368 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6369 DDI_DMA_SLEEP, 6370 NULL, 6371 &ahci_portp->ahciport_cmd_list_dma_cookie, 6372 &cookie_count) != DDI_DMA_MAPPED) { 6373 6374 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6375 "cmd list dma handle bind fail", NULL); 6376 /* error.. free the dma handle & free the memory. */ 6377 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6378 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6379 return (AHCI_FAILURE); 6380 } 6381 6382 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 6383 6384 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 6385 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 6386 6387 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 6388 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 6389 6390 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 6391 goto err_out; 6392 } 6393 6394 return (AHCI_SUCCESS); 6395 6396 err_out: 6397 /* Unbind the cmd list dma handle first. */ 6398 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 6399 6400 /* Then free the underlying memory. */ 6401 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6402 6403 /* Now free the handle itself. */ 6404 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6405 6406 return (AHCI_FAILURE); 6407 } 6408 6409 /* 6410 * Deallocates the Command List 6411 */ 6412 static void 6413 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6414 { 6415 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6416 6417 /* First dealloc command table */ 6418 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 6419 6420 /* Unbind the cmd list dma handle first. */ 6421 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 6422 6423 /* Then free the underlying memory. */ 6424 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6425 6426 /* Now free the handle itself. */ 6427 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6428 } 6429 6430 /* 6431 * Allocates memory for all Command Tables, which contains Command FIS, 6432 * ATAPI Command and Physical Region Descriptor Table. 6433 */ 6434 static int 6435 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6436 { 6437 size_t ret_len; 6438 ddi_dma_cookie_t cmd_table_dma_cookie; 6439 uint_t cookie_count; 6440 int slot; 6441 6442 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6443 6444 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 6445 "ahci_alloc_cmd_tables: port %d enter", 6446 ahci_portp->ahciport_port_num); 6447 6448 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6449 /* Allocate cmd table dma handle. */ 6450 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6451 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 6452 DDI_DMA_SLEEP, 6453 NULL, 6454 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 6455 DDI_SUCCESS) { 6456 6457 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6458 "cmd table dma handle alloc failed", NULL); 6459 6460 goto err_out; 6461 } 6462 6463 if (ddi_dma_mem_alloc( 6464 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6465 ahci_cmd_table_size, 6466 &accattr, 6467 DDI_DMA_CONSISTENT, 6468 DDI_DMA_SLEEP, 6469 NULL, 6470 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 6471 &ret_len, 6472 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 0) { 6473 6474 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6475 "cmd table dma mem alloc fail", NULL); 6476 6477 /* error.. free the dma handle. */ 6478 ddi_dma_free_handle( 6479 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6480 goto err_out; 6481 } 6482 6483 if (ddi_dma_addr_bind_handle( 6484 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6485 NULL, 6486 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 6487 ahci_cmd_table_size, 6488 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6489 DDI_DMA_SLEEP, 6490 NULL, 6491 &cmd_table_dma_cookie, 6492 &cookie_count) != DDI_DMA_MAPPED) { 6493 6494 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6495 "cmd table dma handle bind fail", NULL); 6496 /* error.. free the dma handle & free the memory. */ 6497 ddi_dma_mem_free( 6498 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6499 ddi_dma_free_handle( 6500 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6501 goto err_out; 6502 } 6503 6504 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 6505 ahci_cmd_table_size); 6506 6507 /* Config Port Command Table Base Address */ 6508 SET_COMMAND_TABLE_BASE_ADDR( 6509 (&ahci_portp->ahciport_cmd_list[slot]), 6510 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 6511 6512 #ifndef __lock_lint 6513 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 6514 (&ahci_portp->ahciport_cmd_list[slot]), 6515 cmd_table_dma_cookie.dmac_laddress >> 32); 6516 #endif 6517 } 6518 6519 return (AHCI_SUCCESS); 6520 err_out: 6521 6522 for (slot--; slot >= 0; slot--) { 6523 /* Unbind the cmd table dma handle first */ 6524 (void) ddi_dma_unbind_handle( 6525 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6526 6527 /* Then free the underlying memory */ 6528 ddi_dma_mem_free( 6529 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6530 6531 /* Now free the handle itself */ 6532 ddi_dma_free_handle( 6533 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6534 } 6535 6536 return (AHCI_FAILURE); 6537 } 6538 6539 /* 6540 * Deallocates memory for all Command Tables. 6541 */ 6542 static void 6543 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6544 { 6545 int slot; 6546 6547 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 6548 6549 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6550 "ahci_dealloc_cmd_tables: %d enter", 6551 ahci_portp->ahciport_port_num); 6552 6553 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6554 /* Unbind the cmd table dma handle first. */ 6555 (void) ddi_dma_unbind_handle( 6556 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6557 6558 /* Then free the underlying memory. */ 6559 ddi_dma_mem_free( 6560 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6561 6562 /* Now free the handle itself. */ 6563 ddi_dma_free_handle( 6564 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6565 } 6566 } 6567 6568 /* 6569 * Update SATA registers at controller ports 6570 */ 6571 static void 6572 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 6573 sata_device_t *sd) 6574 { 6575 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 6576 6577 sd->satadev_scr.sstatus = 6578 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6579 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 6580 sd->satadev_scr.serror = 6581 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6582 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 6583 sd->satadev_scr.scontrol = 6584 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6585 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 6586 sd->satadev_scr.sactive = 6587 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6588 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 6589 } 6590 6591 /* 6592 * For poll mode, ahci_port_intr will be called to emulate the interrupt 6593 */ 6594 static void 6595 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 6596 { 6597 uint32_t port_intr_status; 6598 uint32_t port_intr_enable; 6599 6600 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6601 "ahci_port_intr enter: port %d", port); 6602 6603 mutex_enter(&ahci_portp->ahciport_mutex); 6604 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 6605 /* For SATA_OPMODE_POLLING commands */ 6606 port_intr_enable = 6607 (AHCI_INTR_STATUS_DHRS | 6608 AHCI_INTR_STATUS_PSS | 6609 AHCI_INTR_STATUS_SDBS | 6610 AHCI_INTR_STATUS_UFS | 6611 AHCI_INTR_STATUS_PCS | 6612 AHCI_INTR_STATUS_PRCS | 6613 AHCI_INTR_STATUS_OFS | 6614 AHCI_INTR_STATUS_INFS | 6615 AHCI_INTR_STATUS_IFS | 6616 AHCI_INTR_STATUS_HBDS | 6617 AHCI_INTR_STATUS_HBFS | 6618 AHCI_INTR_STATUS_TFES); 6619 } else { 6620 /* 6621 * port_intr_enable indicates that the corresponding interrrupt 6622 * reporting is enabled. 6623 */ 6624 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6625 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 6626 } 6627 6628 /* IPMS error in port reset should be ignored according AHCI spec. */ 6629 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS)) 6630 port_intr_enable |= AHCI_INTR_STATUS_IPMS; 6631 mutex_exit(&ahci_portp->ahciport_mutex); 6632 6633 /* 6634 * port_intr_stats indicates that the corresponding interrupt 6635 * condition is active. 6636 */ 6637 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6638 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 6639 6640 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6641 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 6642 "port_intr_enable = 0x%x", 6643 port, port_intr_status, port_intr_enable); 6644 6645 port_intr_status &= port_intr_enable; 6646 6647 /* 6648 * Pending interrupt events are indicated by the PxIS register. 6649 * Make sure we don't miss any event. 6650 */ 6651 if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) { 6652 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6653 DDI_SERVICE_UNAFFECTED); 6654 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6655 DDI_FME_VERSION); 6656 return; 6657 } 6658 6659 /* First clear the port interrupts status */ 6660 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6661 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 6662 port_intr_status); 6663 6664 /* Check the completed non-queued commands */ 6665 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 6666 AHCI_INTR_STATUS_PSS)) { 6667 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 6668 ahci_portp, port); 6669 } 6670 6671 /* Check the completed queued commands */ 6672 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 6673 (void) ahci_intr_set_device_bits(ahci_ctlp, 6674 ahci_portp, port); 6675 } 6676 6677 /* Check the port connect change status interrupt bit */ 6678 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 6679 (void) ahci_intr_port_connect_change(ahci_ctlp, 6680 ahci_portp, port); 6681 } 6682 6683 /* Check the device mechanical presence status interrupt bit */ 6684 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 6685 (void) ahci_intr_device_mechanical_presence_status( 6686 ahci_ctlp, ahci_portp, port); 6687 } 6688 6689 /* Check the PhyRdy change status interrupt bit */ 6690 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 6691 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 6692 port); 6693 } 6694 6695 /* 6696 * Check the non-fatal error interrupt bits, there are four 6697 * kinds of non-fatal errors at the time being: 6698 * 6699 * PxIS.UFS - Unknown FIS Error 6700 * PxIS.OFS - Overflow Error 6701 * PxIS.INFS - Interface Non-Fatal Error 6702 * PxIS.IPMS - Incorrect Port Multiplier Status Error 6703 * 6704 * For these non-fatal errors, the HBA can continue to operate, 6705 * so the driver just log the error messages. 6706 */ 6707 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 6708 AHCI_INTR_STATUS_OFS | 6709 AHCI_INTR_STATUS_IPMS | 6710 AHCI_INTR_STATUS_INFS)) { 6711 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 6712 port, port_intr_status); 6713 } 6714 6715 /* 6716 * Check the fatal error interrupt bits, there are four kinds 6717 * of fatal errors for AHCI controllers: 6718 * 6719 * PxIS.HBFS - Host Bus Fatal Error 6720 * PxIS.HBDS - Host Bus Data Error 6721 * PxIS.IFS - Interface Fatal Error 6722 * PxIS.TFES - Task File Error 6723 * 6724 * The fatal error means the HBA can not recover from it by 6725 * itself, and it will try to abort the transfer, and the software 6726 * must intervene to restart the port. 6727 */ 6728 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 6729 AHCI_INTR_STATUS_HBDS | 6730 AHCI_INTR_STATUS_HBFS | 6731 AHCI_INTR_STATUS_TFES)) 6732 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 6733 port, port_intr_status); 6734 6735 /* Check the cold port detect interrupt bit */ 6736 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 6737 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 6738 } 6739 6740 /* Second clear the corresponding bit in IS.IPS */ 6741 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6742 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 6743 6744 /* Try to recover at the end of the interrupt handler. */ 6745 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) != 6746 DDI_FM_OK) { 6747 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6748 DDI_SERVICE_UNAFFECTED); 6749 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6750 DDI_FME_VERSION); 6751 } 6752 } 6753 6754 /* 6755 * Interrupt service handler 6756 */ 6757 static uint_t 6758 ahci_intr(caddr_t arg1, caddr_t arg2) 6759 { 6760 #ifndef __lock_lint 6761 _NOTE(ARGUNUSED(arg2)) 6762 #endif 6763 /* LINTED */ 6764 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 6765 ahci_port_t *ahci_portp; 6766 int32_t global_intr_status; 6767 uint8_t port; 6768 6769 /* 6770 * global_intr_status indicates that the corresponding port has 6771 * an interrupt pending. 6772 */ 6773 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6774 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 6775 6776 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 6777 /* The interrupt is not ours */ 6778 return (DDI_INTR_UNCLAIMED); 6779 } 6780 6781 /* 6782 * Check the handle after reading global_intr_status - we don't want 6783 * to miss any port with pending interrupts. 6784 */ 6785 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) != 6786 DDI_FM_OK) { 6787 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, 6788 DDI_SERVICE_UNAFFECTED); 6789 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle, 6790 DDI_FME_VERSION); 6791 return (DDI_INTR_UNCLAIMED); 6792 } 6793 6794 /* Loop for all the ports */ 6795 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6796 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 6797 continue; 6798 } 6799 if (!((0x1 << port) & global_intr_status)) { 6800 continue; 6801 } 6802 6803 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6804 6805 /* Call ahci_port_intr */ 6806 ahci_port_intr(ahci_ctlp, ahci_portp, port); 6807 } 6808 6809 return (DDI_INTR_CLAIMED); 6810 } 6811 6812 /* 6813 * For non-queued commands, when the corresponding bit in the PxCI register 6814 * is cleared, it means the command is completed successfully. And according 6815 * to the HBA state machine, there are three conditions which possibly will 6816 * try to clear the PxCI register bit. 6817 * 1. Receive one D2H Register FIS which is with 'I' bit set 6818 * 2. Update PIO Setup FIS 6819 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 6820 * 6821 * Process completed non-queued commands when the interrupt status bit - 6822 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 6823 * 6824 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 6825 * with the 'I' bit set. And the following commands will send thus 6826 * FIS with 'I' bit set upon the successful completion: 6827 * 1. Non-data commands 6828 * 2. DMA data-in command 6829 * 3. DMA data-out command 6830 * 4. PIO data-out command 6831 * 5. PACKET non-data commands 6832 * 6. PACKET PIO data-in command 6833 * 7. PACKET PIO data-out command 6834 * 8. PACKET DMA data-in command 6835 * 9. PACKET DMA data-out command 6836 * 6837 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 6838 * with the 'I' bit set. And the following commands will send this 6839 * FIS upon the successful completion: 6840 * 1. PIO data-in command 6841 */ 6842 static int 6843 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 6844 ahci_port_t *ahci_portp, uint8_t port) 6845 { 6846 uint32_t port_cmd_issue = 0; 6847 uint32_t finished_tags; 6848 int finished_slot; 6849 sata_pkt_t *satapkt; 6850 ahci_fis_d2h_register_t *rcvd_fisp; 6851 #if AHCI_DEBUG 6852 ahci_cmd_header_t *cmd_header; 6853 uint32_t cmd_dmacount; 6854 #endif 6855 6856 mutex_enter(&ahci_portp->ahciport_mutex); 6857 6858 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6859 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6860 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6861 /* 6862 * Spurious interrupt. Nothing to be done. 6863 */ 6864 mutex_exit(&ahci_portp->ahciport_mutex); 6865 return (AHCI_SUCCESS); 6866 } 6867 6868 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6869 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6870 6871 /* If the PxCI corrupts, don't complete the commmands. */ 6872 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) 6873 != DDI_FM_OK) { 6874 mutex_exit(&ahci_portp->ahciport_mutex); 6875 return (AHCI_FAILURE); 6876 } 6877 6878 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6879 /* Slot 0 is always used during error recovery */ 6880 finished_tags = 0x1 & ~port_cmd_issue; 6881 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6882 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 6883 "retrieval is finished, and finished_tags = 0x%x", 6884 port, finished_tags); 6885 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6886 finished_tags = 0x1 & ~port_cmd_issue; 6887 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 6888 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w " 6889 "port multiplier is finished, and finished_tags = 0x%x", 6890 port, finished_tags); 6891 6892 } else { 6893 6894 finished_tags = ahci_portp->ahciport_pending_tags & 6895 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6896 } 6897 6898 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6899 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 6900 "port_cmd_issue = 0x%x finished_tags = 0x%x", 6901 ahci_portp->ahciport_pending_tags, port_cmd_issue, 6902 finished_tags); 6903 6904 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6905 (finished_tags == 0x1)) { 6906 satapkt = ahci_portp->ahciport_err_retri_pkt; 6907 ASSERT(satapkt != NULL); 6908 6909 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6910 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6911 "with SATA_PKT_COMPLETED", (void *)satapkt); 6912 6913 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6914 goto out; 6915 } 6916 6917 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6918 (finished_tags == 0x1)) { 6919 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 6920 ASSERT(satapkt != NULL); 6921 6922 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6923 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6924 "with SATA_PKT_COMPLETED", (void *)satapkt); 6925 6926 /* READ PORTMULT need copy out FIS content. */ 6927 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6928 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6929 ahcirf_d2h_register_fis); 6930 satapkt->satapkt_cmd.satacmd_status_reg = 6931 GET_RFIS_STATUS(rcvd_fisp); 6932 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6933 } 6934 6935 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6936 goto out; 6937 } 6938 6939 while (finished_tags) { 6940 finished_slot = ddi_ffs(finished_tags) - 1; 6941 if (finished_slot == -1) { 6942 goto out; 6943 } 6944 6945 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6946 ASSERT(satapkt != NULL); 6947 #if AHCI_DEBUG 6948 /* 6949 * For non-native queued commands, the PRD byte count field 6950 * shall contain an accurate count of the number of bytes 6951 * transferred for the command before the PxCI bit is cleared 6952 * to '0' for the command. 6953 * 6954 * The purpose of this field is to let software know how many 6955 * bytes transferred for a given operation in order to 6956 * determine if underflow occurred. When issuing native command 6957 * queuing commands, this field should not be used and is not 6958 * required to be valid since in this case underflow is always 6959 * illegal. 6960 * 6961 * For data reads, the HBA will update its PRD byte count with 6962 * the total number of bytes received from the last FIS, and 6963 * may be able to continue normally. For data writes, the 6964 * device will detect an error, and HBA most likely will get 6965 * a fatal error. 6966 * 6967 * Therefore, here just put code to debug part. And please 6968 * refer to the comment above ahci_intr_fatal_error for the 6969 * definition of underflow error. 6970 */ 6971 cmd_dmacount = 6972 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 6973 if (cmd_dmacount) { 6974 cmd_header = 6975 &ahci_portp->ahciport_cmd_list[finished_slot]; 6976 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 6977 "ahci_intr_cmd_cmplt: port %d, " 6978 "PRD Byte Count = 0x%x, " 6979 "ahciport_prd_bytecounts = 0x%x", port, 6980 cmd_header->ahcich_prd_byte_count, 6981 cmd_dmacount); 6982 6983 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 6984 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 6985 "ahci_intr_cmd_cmplt: port %d, " 6986 "an underflow occurred", port); 6987 } 6988 } 6989 #endif 6990 6991 /* 6992 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 6993 * feature, sata_special_regs flag will be set, and the 6994 * driver should copy the status and the other corresponding 6995 * register values in the D2H Register FIS received (It's 6996 * working on Non-data protocol) from the device back to 6997 * the sata_cmd. 6998 * 6999 * For every AHCI port, there is only one Received FIS 7000 * structure, which contains the FISes received from the 7001 * device, So we're trying to copy the content of D2H 7002 * Register FIS in the Received FIS structure back to 7003 * the sata_cmd. 7004 */ 7005 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 7006 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 7007 ahcirf_d2h_register_fis); 7008 satapkt->satapkt_cmd.satacmd_status_reg = 7009 GET_RFIS_STATUS(rcvd_fisp); 7010 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 7011 } 7012 7013 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7014 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 7015 "with SATA_PKT_COMPLETED", (void *)satapkt); 7016 7017 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 7018 CLEAR_BIT(finished_tags, finished_slot); 7019 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 7020 7021 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 7022 } 7023 out: 7024 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 7025 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 7026 ahci_portp->ahciport_pending_tags); 7027 7028 ahci_flush_doneq(ahci_portp); 7029 7030 mutex_exit(&ahci_portp->ahciport_mutex); 7031 7032 return (AHCI_SUCCESS); 7033 } 7034 7035 /* 7036 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 7037 * with the 'I' bit set and has been copied into system memory. It will 7038 * be sent under the following situations: 7039 * 7040 * 1. NCQ command is completed 7041 * 7042 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 7043 * via the Set Device Bits FIS. When such event is generated, the software 7044 * needs to read PxSACT register and compares the current value to the 7045 * list of commands previously issue by software. ahciport_pending_ncq_tags 7046 * keeps the tags of previously issued commands. 7047 * 7048 * 2. Asynchronous Notification 7049 * 7050 * Asynchronous Notification is a feature in SATA spec 2.6. 7051 * 7052 * 1) ATAPI device will send a signal to the host when media is inserted or 7053 * removed and avoids polling the device for media changes. The signal 7054 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 7055 * set to '1'. At the moment, it's not supported yet. 7056 * 7057 * 2) Port multiplier will send a signal to the host when a hot plug event 7058 * has occured on a port multiplier port. It is used when command based 7059 * switching is employed. This is handled by ahci_intr_pmult_sntf_events() 7060 */ 7061 static int 7062 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 7063 ahci_port_t *ahci_portp, uint8_t port) 7064 { 7065 ahci_addr_t addr; 7066 7067 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7068 "ahci_intr_set_device_bits enter: port %d", port); 7069 7070 /* Initialize HBA port address */ 7071 AHCI_ADDR_SET_PORT(&addr, port); 7072 7073 /* NCQ plug handler */ 7074 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr); 7075 7076 /* Check port multiplier's asynchronous notification events */ 7077 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 7078 (void) ahci_intr_pmult_sntf_events(ahci_ctlp, 7079 ahci_portp, port); 7080 } 7081 7082 /* ATAPI events is not supported yet */ 7083 7084 return (AHCI_SUCCESS); 7085 } 7086 /* 7087 * NCQ interrupt handler. Called upon a NCQ command is completed. 7088 * Only be called from ahci_intr_set_device_bits(). 7089 */ 7090 static int 7091 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp, 7092 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 7093 { 7094 uint32_t port_sactive; 7095 uint32_t port_cmd_issue; 7096 uint32_t issued_tags; 7097 int issued_slot; 7098 uint32_t finished_tags; 7099 int finished_slot; 7100 uint8_t port = addrp->aa_port; 7101 sata_pkt_t *satapkt; 7102 7103 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7104 "ahci_intr_set_device_bits enter: port %d", port); 7105 7106 mutex_enter(&ahci_portp->ahciport_mutex); 7107 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7108 mutex_exit(&ahci_portp->ahciport_mutex); 7109 return (AHCI_SUCCESS); 7110 } 7111 7112 /* 7113 * First the handler got which commands are finished by checking 7114 * PxSACT register 7115 */ 7116 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7117 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7118 7119 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 7120 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 7121 7122 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7123 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 7124 "port_sactive = 0x%x", port, 7125 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 7126 7127 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7128 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 7129 7130 /* 7131 * For NCQ commands, the software can determine which command has 7132 * already been transmitted to the device by checking PxCI register. 7133 */ 7134 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7135 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7136 7137 issued_tags = ahci_portp->ahciport_pending_tags & 7138 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 7139 7140 /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */ 7141 if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) 7142 != DDI_FM_OK) { 7143 mutex_exit(&ahci_portp->ahciport_mutex); 7144 return (AHCI_FAILURE); 7145 } 7146 7147 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7148 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 7149 "port_cmd_issue = 0x%x", port, 7150 ahci_portp->ahciport_pending_tags, port_cmd_issue); 7151 7152 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7153 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 7154 7155 /* 7156 * Clear ahciport_pending_tags bit when the corresponding command 7157 * is already sent down to the device. 7158 */ 7159 while (issued_tags) { 7160 issued_slot = ddi_ffs(issued_tags) - 1; 7161 if (issued_slot == -1) { 7162 goto next; 7163 } 7164 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 7165 CLEAR_BIT(issued_tags, issued_slot); 7166 } 7167 7168 next: 7169 while (finished_tags) { 7170 finished_slot = ddi_ffs(finished_tags) - 1; 7171 if (finished_slot == -1) { 7172 goto out; 7173 } 7174 7175 /* The command is certainly transmitted to the device */ 7176 ASSERT(!(ahci_portp->ahciport_pending_tags & 7177 (0x1 << finished_slot))); 7178 7179 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 7180 ASSERT(satapkt != NULL); 7181 7182 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 7183 "ahci_intr_set_device_bits: sending up pkt 0x%p " 7184 "with SATA_PKT_COMPLETED", (void *)satapkt); 7185 7186 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 7187 CLEAR_BIT(finished_tags, finished_slot); 7188 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 7189 7190 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 7191 } 7192 out: 7193 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 7194 "ahci_intr_set_device_bits: port %d " 7195 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 7196 port, ahci_portp->ahciport_pending_ncq_tags, 7197 ahci_portp->ahciport_pending_tags); 7198 7199 ahci_flush_doneq(ahci_portp); 7200 7201 mutex_exit(&ahci_portp->ahciport_mutex); 7202 7203 return (AHCI_SUCCESS); 7204 } 7205 7206 /* 7207 * Port multiplier asynchronous notification event handler. Called upon a 7208 * device is hot plugged/pulled. 7209 * 7210 * The async-notification event will only be recorded by ahcipmi_snotif_tags 7211 * here and will be handled by ahci_probe_pmult(). 7212 * 7213 * NOTE: called only from ahci_port_intr(). 7214 */ 7215 static int 7216 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7217 uint8_t port) 7218 { 7219 sata_device_t sdevice; 7220 7221 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7222 "ahci_intr_pmult_sntf_events enter: port %d ", port); 7223 7224 /* no hot-plug while attaching process */ 7225 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7226 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 7227 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7228 return (AHCI_SUCCESS); 7229 } 7230 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7231 7232 mutex_enter(&ahci_portp->ahciport_mutex); 7233 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 7234 mutex_exit(&ahci_portp->ahciport_mutex); 7235 return (AHCI_SUCCESS); 7236 } 7237 7238 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 7239 7240 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags = 7241 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7242 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port)); 7243 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7244 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 7245 AHCI_SNOTIF_CLEAR_ALL); 7246 7247 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) { 7248 mutex_exit(&ahci_portp->ahciport_mutex); 7249 return (AHCI_SUCCESS); 7250 } 7251 7252 /* Port Multiplier sub-device hot-plug handler */ 7253 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 7254 mutex_exit(&ahci_portp->ahciport_mutex); 7255 return (AHCI_SUCCESS); 7256 } 7257 7258 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) { 7259 /* Not allowed to re-enter. */ 7260 mutex_exit(&ahci_portp->ahciport_mutex); 7261 return (AHCI_SUCCESS); 7262 } 7263 7264 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF; 7265 7266 /* 7267 * NOTE: 7268 * Even if Asynchronous Notification is supported (and enabled) by 7269 * both controller and the port multiplier, the content of PxSNTF 7270 * register is always set to 0x8000 by async notification event. We 7271 * need to check GSCR[32] on the port multiplier to find out the 7272 * owner of this event. 7273 * This is not accord with SATA spec 2.6 and needs further 7274 * clarification. 7275 */ 7276 /* hot-plug will not reported while reseting. */ 7277 if (ahci_portp->ahciport_reset_in_progress == 1) { 7278 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 7279 "port %d snotif event ignored", port); 7280 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 7281 mutex_exit(&ahci_portp->ahciport_mutex); 7282 return (AHCI_SUCCESS); 7283 } 7284 7285 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 7286 "PxSNTF is set to 0x%x by port multiplier", 7287 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags); 7288 7289 /* 7290 * Now we need do some necessary operation and inform SATA framework 7291 * that link/device events has happened. 7292 */ 7293 bzero((void *)&sdevice, sizeof (sata_device_t)); 7294 sdevice.satadev_addr.cport = ahci_ctlp-> 7295 ahcictl_port_to_cport[port]; 7296 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 7297 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 7298 sdevice.satadev_state = SATA_PSTATE_PWRON; 7299 7300 /* Just reject packets, do not stop that port. */ 7301 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7302 7303 mutex_exit(&ahci_portp->ahciport_mutex); 7304 sata_hba_event_notify( 7305 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7306 &sdevice, 7307 SATA_EVNT_PMULT_LINK_CHANGED); 7308 mutex_enter(&ahci_portp->ahciport_mutex); 7309 7310 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 7311 mutex_exit(&ahci_portp->ahciport_mutex); 7312 7313 return (AHCI_SUCCESS); 7314 } 7315 7316 /* 7317 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 7318 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 7319 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 7320 * indicates a COMINIT signal was received. 7321 * 7322 * Hot plug insertion is detected by reception of a COMINIT signal from the 7323 * device. On reception of unsolicited COMINIT, the HBA shall generate a 7324 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 7325 * begin the normal communication negotiation sequence as outlined in the 7326 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 7327 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 7328 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 7329 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 7330 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 7331 * to handle hot plug insertion. In this interrupt handler, just do nothing 7332 * but print some log message and clear the bit. 7333 */ 7334 static int 7335 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 7336 ahci_port_t *ahci_portp, uint8_t port) 7337 { 7338 #if AHCI_DEBUG 7339 uint32_t port_serror; 7340 #endif 7341 7342 mutex_enter(&ahci_portp->ahciport_mutex); 7343 7344 #if AHCI_DEBUG 7345 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7346 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7347 7348 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7349 "ahci_intr_port_connect_change: port %d, " 7350 "port_serror = 0x%x", port, port_serror); 7351 #endif 7352 7353 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 7354 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7355 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7356 SERROR_EXCHANGED_ERR); 7357 7358 mutex_exit(&ahci_portp->ahciport_mutex); 7359 7360 return (AHCI_SUCCESS); 7361 } 7362 7363 /* 7364 * Hot Plug Operation for platforms that support Mechanical Presence 7365 * Switches. 7366 * 7367 * When set, it indicates that a mechanical presence switch attached to this 7368 * port has been opened or closed, which may lead to a change in the connection 7369 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 7370 * are set to '1'. 7371 * 7372 * At the moment, this interrupt is not needed and disabled and we just log 7373 * the debug message. 7374 */ 7375 static int 7376 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 7377 ahci_port_t *ahci_portp, uint8_t port) 7378 { 7379 uint32_t cap_status, port_cmd_status; 7380 7381 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7382 "ahci_intr_device_mechanical_presence_status enter, " 7383 "port %d", port); 7384 7385 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7386 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 7387 7388 mutex_enter(&ahci_portp->ahciport_mutex); 7389 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7390 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7391 7392 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 7393 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 7394 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7395 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 7396 "the interrupt: cap_status = 0x%x, " 7397 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 7398 mutex_exit(&ahci_portp->ahciport_mutex); 7399 7400 return (AHCI_SUCCESS); 7401 } 7402 7403 #if AHCI_DEBUG 7404 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 7405 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7406 "The mechanical presence switch is open: " 7407 "port %d, port_cmd_status = 0x%x", 7408 port, port_cmd_status); 7409 } else { 7410 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7411 "The mechanical presence switch is close: " 7412 "port %d, port_cmd_status = 0x%x", 7413 port, port_cmd_status); 7414 } 7415 #endif 7416 7417 mutex_exit(&ahci_portp->ahciport_mutex); 7418 7419 return (AHCI_SUCCESS); 7420 } 7421 7422 /* 7423 * Native Hot Plug Support. 7424 * 7425 * When set, it indicates that the internal PHYRDY signal changed state. 7426 * This bit reflects the state of PxSERR.DIAG.N. 7427 * 7428 * There are three kinds of conditions to generate this interrupt event: 7429 * 1. a device is inserted 7430 * 2. a device is disconnected 7431 * 3. when the link enters/exits a Partial or Slumber interface power 7432 * management state 7433 * 7434 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 7435 * bit may be set due to the link entering the Partial or Slumber power 7436 * management state, rather than due to a hot plug insertion or removal 7437 * event. So far, the interface power management is disabled, so the 7438 * driver can reliably get removal detection notification via the 7439 * PxSERR.DIAG.N bit. 7440 */ 7441 static int 7442 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 7443 ahci_port_t *ahci_portp, uint8_t port) 7444 { 7445 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 7446 sata_device_t sdevice; 7447 int dev_exists_now = 0; 7448 int dev_existed_previously = 0; 7449 ahci_addr_t port_addr; 7450 7451 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 7452 "ahci_intr_phyrdy_change enter, port %d", port); 7453 7454 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 7455 mutex_enter(&ahci_portp->ahciport_mutex); 7456 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7457 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7458 SERROR_PHY_RDY_CHG); 7459 mutex_exit(&ahci_portp->ahciport_mutex); 7460 7461 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7462 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 7463 (ahci_portp == NULL)) { 7464 /* The whole controller setup is not yet done. */ 7465 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7466 return (AHCI_SUCCESS); 7467 } 7468 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7469 7470 mutex_enter(&ahci_portp->ahciport_mutex); 7471 7472 /* SStatus tells the presence of device. */ 7473 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7474 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 7475 7476 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 7477 dev_exists_now = 1; 7478 } 7479 7480 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 7481 dev_existed_previously = 1; 7482 } 7483 7484 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 7485 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 7486 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7487 "ahci_intr_phyrdy_change: port %d " 7488 "AHCI_PORT_FLAG_NODEV is cleared", port); 7489 if (dev_exists_now == 0) 7490 dev_existed_previously = 1; 7491 } 7492 7493 bzero((void *)&sdevice, sizeof (sata_device_t)); 7494 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7495 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7496 sdevice.satadev_addr.pmport = 0; 7497 sdevice.satadev_state = SATA_PSTATE_PWRON; 7498 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 7499 7500 AHCI_ADDR_SET_PORT(&port_addr, port); 7501 7502 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 7503 if (dev_exists_now) { 7504 if (dev_existed_previously) { /* 1 -> 1 */ 7505 /* Things are fine now. The loss was temporary. */ 7506 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7507 "ahci_intr_phyrdy_change port %d " 7508 "device link lost/established", port); 7509 7510 mutex_exit(&ahci_portp->ahciport_mutex); 7511 sata_hba_event_notify( 7512 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7513 &sdevice, 7514 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 7515 mutex_enter(&ahci_portp->ahciport_mutex); 7516 7517 } else { /* 0 -> 1 */ 7518 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7519 "ahci_intr_phyrdy_change: port %d " 7520 "device link established", port); 7521 7522 /* 7523 * A new device has been detected. The new device 7524 * might be a port multiplier instead of a drive, so 7525 * we cannot update the signature directly. 7526 */ 7527 (void) ahci_initialize_port(ahci_ctlp, 7528 ahci_portp, &port_addr); 7529 7530 /* Try to start the port */ 7531 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 7532 != AHCI_SUCCESS) { 7533 sdevice.satadev_state |= SATA_PSTATE_FAILED; 7534 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7535 "ahci_intr_phyrdy_change: port %d failed " 7536 "at start port", port); 7537 } 7538 7539 /* Clear the max queue depth for inserted device */ 7540 ahci_portp->ahciport_max_ncq_tags = 0; 7541 7542 mutex_exit(&ahci_portp->ahciport_mutex); 7543 sata_hba_event_notify( 7544 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7545 &sdevice, 7546 SATA_EVNT_LINK_ESTABLISHED); 7547 mutex_enter(&ahci_portp->ahciport_mutex); 7548 7549 } 7550 } else { /* No device exists now */ 7551 7552 if (dev_existed_previously) { /* 1 -> 0 */ 7553 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7554 "ahci_intr_phyrdy_change: port %d " 7555 "device link lost", port); 7556 7557 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7558 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7559 ahci_portp, port); 7560 7561 if (ahci_portp->ahciport_device_type == 7562 SATA_DTYPE_PMULT) { 7563 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 7564 } 7565 7566 /* An existing device is lost. */ 7567 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 7568 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN; 7569 7570 mutex_exit(&ahci_portp->ahciport_mutex); 7571 sata_hba_event_notify( 7572 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7573 &sdevice, 7574 SATA_EVNT_LINK_LOST); 7575 mutex_enter(&ahci_portp->ahciport_mutex); 7576 } 7577 } 7578 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 7579 7580 mutex_exit(&ahci_portp->ahciport_mutex); 7581 7582 return (AHCI_SUCCESS); 7583 } 7584 7585 /* 7586 * PxIS.UFS - Unknown FIS Error 7587 * 7588 * This interrupt event means an unknown FIS was received and has been 7589 * copied into system memory. An unknown FIS is not considered an illegal 7590 * FIS, unless the length received is more than 64 bytes. If an unknown 7591 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 7592 * normal operation. If the unknown FIS is more than 64 bytes, then it 7593 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 7594 * a fatal error. 7595 * 7596 * PxIS.IPMS - Incorrect Port Multiplier Status 7597 * 7598 * IPMS Indicates that the HBA received a FIS from a device that did not 7599 * have a command outstanding. The IPMS bit may be set during enumeration 7600 * of devices on a Port Multiplier due to the normal Port Multiplier 7601 * enumeration process. It is recommended that IPMS only be used after 7602 * enumeration is complete on the Port Multiplier (copied from spec). 7603 * 7604 * PxIS.OFS - Overflow Error 7605 * 7606 * Command list overflow is defined as software building a command table 7607 * that has fewer total bytes than the transaction given to the device. 7608 * On device writes, the HBA will run out of data, and on reads, there 7609 * will be no room to put the data. 7610 * 7611 * For an overflow on data read, either PIO or DMA, the HBA will set 7612 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 7613 * non-fatal error when the HBA can continues. Sometimes, it will cause 7614 * a fatal error and need the software to do something. 7615 * 7616 * For an overflow on data write, setting PxIS.OFS is optional for both 7617 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 7618 * software to clean up from this serious error. 7619 * 7620 * PxIS.INFS - Interface Non-Fatal Error 7621 * 7622 * This interrupt event indicates that the HBA encountered an error on 7623 * the Serial ATA interface but was able to continue operation. The kind 7624 * of error usually occurred during a non-Data FIS, and under this condition 7625 * the FIS will be re-transmitted by HBA automatically. 7626 * 7627 * When the FMA is implemented, there should be a stat structure to 7628 * record how many every kind of error happens. 7629 */ 7630 static int 7631 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7632 uint8_t port, uint32_t intr_status) 7633 { 7634 uint32_t port_serror; 7635 #if AHCI_DEBUG 7636 uint32_t port_cmd_status; 7637 uint32_t port_cmd_issue; 7638 uint32_t port_sactive; 7639 int current_slot; 7640 uint32_t current_tags; 7641 sata_pkt_t *satapkt; 7642 ahci_cmd_header_t *cmd_header; 7643 uint32_t cmd_dmacount; 7644 #endif 7645 7646 mutex_enter(&ahci_portp->ahciport_mutex); 7647 7648 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7649 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7650 7651 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 7652 "ahci_intr_non_fatal_error: port %d, " 7653 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status); 7654 7655 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 7656 7657 if (intr_status & AHCI_INTR_STATUS_UFS) { 7658 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7659 "ahci port %d has unknown FIS error", port); 7660 7661 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 7662 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7663 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7664 SERROR_FIS_TYPE); 7665 } 7666 7667 #if AHCI_DEBUG 7668 if (intr_status & AHCI_INTR_STATUS_IPMS) { 7669 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d " 7670 "has Incorrect Port Multiplier Status error", port); 7671 } 7672 7673 if (intr_status & AHCI_INTR_STATUS_OFS) { 7674 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7675 "ahci port %d has overflow error", port); 7676 } 7677 7678 if (intr_status & AHCI_INTR_STATUS_INFS) { 7679 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7680 "ahci port %d has interface non fatal error", port); 7681 } 7682 7683 /* 7684 * Record the error occurred command's slot. 7685 */ 7686 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7687 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7688 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7689 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7690 7691 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7692 AHCI_CMD_STATUS_CCS_SHIFT; 7693 7694 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7695 satapkt = ahci_portp->ahciport_err_retri_pkt; 7696 ASSERT(satapkt != NULL); 7697 ASSERT(current_slot == 0); 7698 } else { 7699 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7700 } 7701 7702 if (satapkt != NULL) { 7703 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7704 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 7705 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 7706 satapkt->satapkt_cmd.satacmd_cmd_reg); 7707 7708 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7709 "ahci_intr_non_fatal_error: port %d, " 7710 "satapkt 0x%p is being processed when error occurs", 7711 port, (void *)satapkt); 7712 7713 /* 7714 * PRD Byte Count field of command header is not 7715 * required to reflect the total number of bytes 7716 * transferred when an overflow occurs, so here 7717 * just log the value. 7718 */ 7719 cmd_dmacount = 7720 ahci_portp->ahciport_prd_bytecounts[current_slot]; 7721 if (cmd_dmacount) { 7722 cmd_header = &ahci_portp-> 7723 ahciport_cmd_list[current_slot]; 7724 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7725 "ahci_intr_non_fatal_error: port %d, " 7726 "PRD Byte Count = 0x%x, " 7727 "ahciport_prd_bytecounts = 0x%x", port, 7728 cmd_header->ahcich_prd_byte_count, 7729 cmd_dmacount); 7730 } 7731 } 7732 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7733 /* 7734 * For queued command, list those command which have already 7735 * been transmitted to the device and still not completed. 7736 */ 7737 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7738 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7739 7740 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7741 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7742 7743 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 7744 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 7745 "port_sactive = 0x%x port_cmd_issue = 0x%x", 7746 ahci_portp->ahciport_pending_ncq_tags, 7747 port_sactive, port_cmd_issue); 7748 7749 current_tags = ahci_portp->ahciport_pending_ncq_tags & 7750 port_sactive & ~port_cmd_issue & 7751 AHCI_NCQ_SLOT_MASK(ahci_portp); 7752 7753 while (current_tags) { 7754 current_slot = ddi_ffs(current_tags) - 1; 7755 if (current_slot == -1) { 7756 goto out; 7757 } 7758 7759 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7760 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 7761 ahci_ctlp, "ahci_intr_non_fatal_error: " 7762 "port %d, satapkt 0x%p is outstanding when " 7763 "error occurs", port, (void *)satapkt); 7764 7765 CLEAR_BIT(current_tags, current_slot); 7766 } 7767 } 7768 out: 7769 #endif 7770 mutex_exit(&ahci_portp->ahciport_mutex); 7771 7772 return (AHCI_SUCCESS); 7773 } 7774 7775 /* 7776 * According to the AHCI spec, the error types include system memory 7777 * errors, interface errors, port multiplier errors, device errors, 7778 * command list overflow, command list underflow, native command 7779 * queuing tag errors and pio data transfer errors. 7780 * 7781 * System memory errors such as target abort, master abort, and parity 7782 * may cause the host to stop, and they are serious errors and needed 7783 * to be recovered with software intervention. When system software 7784 * has given a pointer to the HBA that doesn't exist in physical memory, 7785 * a master/target abort error occurs, and PxIS.HBFS will be set. A 7786 * data error such as CRC or parity occurs, the HBA aborts the transfer 7787 * (if necessary) and PxIS.HBDS will be set. 7788 * 7789 * Interface errors are errors that occur due to electrical issues on 7790 * the interface, or protocol miscommunication between the device and 7791 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 7792 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 7793 * causes PxIS.IFS/PxIS.INFS to be set are 7794 * 1. in PxSERR.ERR, P bit is set to '1' 7795 * 2. in PxSERR.DIAG, C or H bit is set to '1' 7796 * 3. PhyRdy drop unexpectly, N bit is set to '1' 7797 * If the error occurred during a non-data FIS, the FIS must be 7798 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 7799 * the error occurred during a data FIS, the transfer will stop, so 7800 * the error is fatal and PxIS.IFS is set. 7801 * 7802 * When a FIS arrives that updates the taskfile, the HBA checks to see 7803 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 7804 * stops processing any more commands. 7805 * 7806 * Command list overflow is defined as software building a command table 7807 * that has fewer total bytes than the transaction given to the device. 7808 * On device writes, the HBA will run out of data, and on reads, there 7809 * will be no room to put the data. For an overflow on data read, either 7810 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 7811 * For an overflow on data write, setting PxIS.OFS is optional for both 7812 * DMA and PIO, and a COMRESET is required by software to clean up from 7813 * this serious error. 7814 * 7815 * Command list underflow is defined as software building a command 7816 * table that has more total bytes than the transaction given to the 7817 * device. For data writes, both PIO and DMA, the device will detect 7818 * an error and end the transfer. And these errors are most likely going 7819 * to be fatal errors that will cause the port to be restarted. For 7820 * data reads, the HBA updates its PRD byte count, and may be 7821 * able to continue normally, but is not required to. And The HBA is 7822 * not required to detect underflow conditions for native command 7823 * queuing command. 7824 * 7825 * The HBA does not actively check incoming DMA Setup FISes to ensure 7826 * that the PxSACT register bit for that slot is set. Existing error 7827 * mechanisms, such as host bus failure, or bad protocol, are used to 7828 * recover from this case. 7829 * 7830 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 7831 * DATA FIS must be an integral number of Dwords. If the HBA receives 7832 * a request which is not an integral number of Dwords, the HBA 7833 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 7834 * software restarts the port. And the HBA ensures that the size 7835 * of the DATA FIS received during a PIO command matches the size in 7836 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 7837 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 7838 * stop running until software restarts the port. 7839 */ 7840 /* 7841 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 7842 * 7843 * PxIS.IFS indicates that the hba encountered an error on the serial ata 7844 * interface which caused the transfer to stop. 7845 * 7846 * PxIS.HBDS indicates that the hba encountered a data error 7847 * (uncorrectable ecc/parity) when reading from or writing to system memory. 7848 * 7849 * PxIS.HBFS indicates that the hba encountered a host bus error that it 7850 * cannot recover from, such as a bad software pointer. 7851 * 7852 * PxIS.TFES is set whenever the status register is updated by the device 7853 * and the error bit (bit 0) is set. 7854 */ 7855 static int 7856 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 7857 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 7858 { 7859 uint32_t port_cmd_status; 7860 uint32_t port_serror; 7861 uint32_t task_file_status; 7862 int failed_slot; 7863 sata_pkt_t *spkt = NULL; 7864 uint8_t err_byte; 7865 ahci_event_arg_t *args; 7866 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7867 uint32_t failed_tags = 0; 7868 int task_fail_flag = 0, task_abort_flag = 0; 7869 uint32_t slot_status; 7870 7871 mutex_enter(&ahci_portp->ahciport_mutex); 7872 7873 /* 7874 * ahci_intr_phyrdy_change() may have rendered it to 7875 * SATA_DTYPE_NONE. 7876 */ 7877 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7878 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7879 "ahci_intr_fatal_error: port %d no device attached, " 7880 "and just return without doing anything", port); 7881 goto out0; 7882 } 7883 7884 if (intr_status & AHCI_INTR_STATUS_TFES) { 7885 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7886 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 7887 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7888 "ahci_intr_fatal_error: port %d " 7889 "task_file_status = 0x%x", port, task_file_status); 7890 task_fail_flag = 1; 7891 7892 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 7893 >> AHCI_TFD_ERR_SHIFT; 7894 if (err_byte == SATA_ERROR_ABORT) 7895 task_abort_flag = 1; 7896 } 7897 7898 /* 7899 * Here we just log the fatal error info in interrupt context. 7900 * Misc recovery processing will be handled in task queue. 7901 */ 7902 if (task_fail_flag == 1) { 7903 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7904 /* 7905 * Read PxCMD.CCS to determine the slot that the HBA 7906 * was processing when the error occurred. 7907 */ 7908 port_cmd_status = ddi_get32( 7909 ahci_ctlp->ahcictl_ahci_acc_handle, 7910 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7911 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7912 AHCI_CMD_STATUS_CCS_SHIFT; 7913 failed_tags = 0x1 << failed_slot; 7914 7915 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 7916 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7917 "ahci_intr_fatal_error: spkt 0x%p is being " 7918 "processed when fatal error occurred for port %d", 7919 spkt, port); 7920 7921 /* 7922 * Won't emit the error message if it is an IDENTIFY 7923 * DEVICE command sent to an ATAPI device. 7924 */ 7925 if ((spkt != NULL) && 7926 (spkt->satapkt_cmd.satacmd_cmd_reg == 7927 SATAC_ID_DEVICE) && 7928 (task_abort_flag == 1)) 7929 goto out1; 7930 7931 /* 7932 * Won't emit the error message if it is an ATAPI PACKET 7933 * command 7934 */ 7935 if ((spkt != NULL) && 7936 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 7937 goto out1; 7938 7939 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7940 slot_status = ddi_get32( 7941 ahci_ctlp->ahcictl_ahci_acc_handle, 7942 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7943 failed_tags = slot_status & 7944 AHCI_NCQ_SLOT_MASK(ahci_portp); 7945 } 7946 } 7947 7948 /* print the fatal error type */ 7949 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 7950 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT; 7951 7952 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7953 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7954 7955 /* print PxSERR related error message */ 7956 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 7957 7958 /* print task file register value */ 7959 if (task_fail_flag == 1) { 7960 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 7961 "= 0x%x", instance, port, task_file_status); 7962 if (task_abort_flag == 1) { 7963 cmn_err(CE_WARN, "!ahci%d: the below command (s) on " 7964 "port %d are aborted", instance, port); 7965 ahci_dump_commands(ahci_ctlp, port, failed_tags); 7966 } 7967 } 7968 7969 out1: 7970 /* Prepare the argument for the taskq */ 7971 args = ahci_portp->ahciport_event_args; 7972 args->ahciea_ctlp = (void *)ahci_ctlp; 7973 args->ahciea_portp = (void *)ahci_portp; 7974 args->ahciea_event = intr_status; 7975 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port); 7976 7977 /* Start the taskq to handle error recovery */ 7978 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 7979 ahci_events_handler, 7980 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 7981 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 7982 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery " 7983 "port %d failed", instance, port); 7984 } 7985 out0: 7986 mutex_exit(&ahci_portp->ahciport_mutex); 7987 7988 return (AHCI_SUCCESS); 7989 } 7990 7991 /* 7992 * Hot Plug Operation for platforms that support Cold Presence Detect. 7993 * 7994 * When set, a device status has changed as detected by the cold presence 7995 * detect logic. This bit can either be set due to a non-connected port 7996 * receiving a device, or a connected port having its device removed. 7997 * This bit is only valid if the port supports cold presence detect as 7998 * indicated by PxCMD.CPD set to '1'. 7999 * 8000 * At the moment, this interrupt is not needed and disabled and we just 8001 * log the debug message. 8002 */ 8003 static int 8004 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 8005 ahci_port_t *ahci_portp, uint8_t port) 8006 { 8007 uint32_t port_cmd_status; 8008 sata_device_t sdevice; 8009 8010 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8011 "ahci_intr_cold_port_detect enter, port %d", port); 8012 8013 mutex_enter(&ahci_portp->ahciport_mutex); 8014 8015 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8016 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8017 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 8018 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8019 "port %d does not support cold presence detect, so " 8020 "we just ignore this interrupt", port); 8021 mutex_exit(&ahci_portp->ahciport_mutex); 8022 return (AHCI_SUCCESS); 8023 } 8024 8025 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8026 "port %d device status has changed", port); 8027 8028 bzero((void *)&sdevice, sizeof (sata_device_t)); 8029 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8030 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 8031 sdevice.satadev_addr.pmport = 0; 8032 sdevice.satadev_state = SATA_PSTATE_PWRON; 8033 8034 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 8035 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8036 "port %d: a device is hot plugged", port); 8037 mutex_exit(&ahci_portp->ahciport_mutex); 8038 sata_hba_event_notify( 8039 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8040 &sdevice, 8041 SATA_EVNT_DEVICE_ATTACHED); 8042 mutex_enter(&ahci_portp->ahciport_mutex); 8043 8044 } else { 8045 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8046 "port %d: a device is hot unplugged", port); 8047 mutex_exit(&ahci_portp->ahciport_mutex); 8048 sata_hba_event_notify( 8049 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8050 &sdevice, 8051 SATA_EVNT_DEVICE_DETACHED); 8052 mutex_enter(&ahci_portp->ahciport_mutex); 8053 } 8054 8055 mutex_exit(&ahci_portp->ahciport_mutex); 8056 8057 return (AHCI_SUCCESS); 8058 } 8059 8060 /* 8061 * Enable the interrupts for a particular port. 8062 */ 8063 static void 8064 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 8065 { 8066 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8067 8068 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8069 "ahci_enable_port_intrs enter, port %d", port); 8070 8071 /* 8072 * Clear port interrupt status before enabling interrupt 8073 */ 8074 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8075 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 8076 AHCI_PORT_INTR_MASK); 8077 8078 /* 8079 * Clear the pending bit from IS.IPS 8080 */ 8081 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8082 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 8083 8084 /* 8085 * Enable the following interrupts: 8086 * Device to Host Register FIS Interrupt (DHRS) 8087 * PIO Setup FIS Interrupt (PSS) 8088 * Set Device Bits Interrupt (SDBS) 8089 * Unknown FIS Interrupt (UFS) 8090 * Port Connect Change Status (PCS) 8091 * PhyRdy Change Status (PRCS) 8092 * Overflow Status (OFS) 8093 * Interface Non-fatal Error Status (INFS) 8094 * Interface Fatal Error Status (IFS) 8095 * Host Bus Data Error Status (HBDS) 8096 * Host Bus Fatal Error Status (HBFS) 8097 * Task File Error Status (TFES) 8098 */ 8099 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8100 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 8101 (AHCI_INTR_STATUS_DHRS | 8102 AHCI_INTR_STATUS_PSS | 8103 AHCI_INTR_STATUS_SDBS | 8104 AHCI_INTR_STATUS_UFS | 8105 AHCI_INTR_STATUS_DPS | 8106 AHCI_INTR_STATUS_PCS | 8107 AHCI_INTR_STATUS_PRCS | 8108 AHCI_INTR_STATUS_OFS | 8109 AHCI_INTR_STATUS_INFS | 8110 AHCI_INTR_STATUS_IFS | 8111 AHCI_INTR_STATUS_HBDS | 8112 AHCI_INTR_STATUS_HBFS | 8113 AHCI_INTR_STATUS_TFES)); 8114 } 8115 8116 /* 8117 * Enable interrupts for all the ports. 8118 */ 8119 static void 8120 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 8121 { 8122 uint32_t ghc_control; 8123 8124 ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 8125 8126 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 8127 8128 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8129 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 8130 8131 ghc_control |= AHCI_HBA_GHC_IE; 8132 8133 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8134 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 8135 } 8136 8137 /* 8138 * Disable interrupts for a particular port. 8139 */ 8140 static void 8141 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 8142 { 8143 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE || 8144 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8145 8146 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8147 "ahci_disable_port_intrs enter, port %d", port); 8148 8149 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8150 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 8151 } 8152 8153 /* 8154 * Disable interrupts for the whole HBA. 8155 * 8156 * The global bit is cleared, then all interrupt sources from all 8157 * ports are disabled. 8158 */ 8159 static void 8160 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 8161 { 8162 uint32_t ghc_control; 8163 8164 ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) || 8165 MUTEX_HELD(&ahci_ctlp->ahcictl_mutex)); 8166 8167 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 8168 NULL); 8169 8170 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8171 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 8172 8173 ghc_control &= ~AHCI_HBA_GHC_IE; 8174 8175 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8176 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 8177 } 8178 8179 /* 8180 * Handle FIXED or MSI interrupts. 8181 */ 8182 /* 8183 * According to AHCI spec, the HBA may support several interrupt modes: 8184 * * pin based interrupts (FIXED) 8185 * * single MSI message interrupts 8186 * * multiple MSI based message interrupts 8187 * 8188 * For pin based interrupts, the software interrupt handler need to check IS 8189 * register to find out which port has pending interrupts. And then check 8190 * PxIS register to find out which interrupt events happened on that port. 8191 * 8192 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 8193 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 8194 * in that software interrupt handler need to check IS register to determine 8195 * which port triggered the interrupts since it uses a single message for all 8196 * port interrupts. 8197 * 8198 * HBA may optionally support multiple MSI message for better performance. In 8199 * this mode, each port may have its own interrupt message, and thus generation 8200 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 8201 * represents a power-of-2 wrapper on the number of implemented ports, and 8202 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 8203 * maximum number of assigned interrupts. When the number of MSI messages 8204 * allocated is less than the number requested, then hardware may have two 8205 * implementation behaviors: 8206 * * assign each ports its own interrupt and then force all additional 8207 * ports to share the last interrupt message, and this condition is 8208 * indicated by clearing GHC.MRSM to '0' 8209 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 8210 * When multiple-message MSI is enabled, hardware will still set IS register 8211 * as single message case. And this IS register may be used by software when 8212 * fewer than the requested number of messages is granted in order to determine 8213 * which port had the interrupt. 8214 * 8215 * Note: The current ahci driver only supports the first two interrupt modes: 8216 * pin based interrupts and single MSI message interrupts, and the reason 8217 * is indicated in below code. 8218 */ 8219 static int 8220 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 8221 { 8222 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 8223 int count, avail, actual; 8224 int i, rc; 8225 8226 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 8227 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 8228 8229 /* get number of interrupts. */ 8230 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 8231 if ((rc != DDI_SUCCESS) || (count == 0)) { 8232 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8233 "ddi_intr_get_nintrs() failed, " 8234 "rc %d count %d\n", rc, count); 8235 return (DDI_FAILURE); 8236 } 8237 8238 /* get number of available interrupts. */ 8239 rc = ddi_intr_get_navail(dip, intr_type, &avail); 8240 if ((rc != DDI_SUCCESS) || (avail == 0)) { 8241 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8242 "ddi_intr_get_navail() failed, " 8243 "rc %d avail %d\n", rc, avail); 8244 return (DDI_FAILURE); 8245 } 8246 8247 #if AHCI_DEBUG 8248 if (avail < count) { 8249 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8250 "ddi_intr_get_nintrs returned %d, navail() returned %d", 8251 count, avail); 8252 } 8253 #endif 8254 8255 /* 8256 * Note: So far Solaris restricts the maximum number of messages for 8257 * x86 to 2, that is avail is 2, so here we set the count with 1 to 8258 * force the driver to use single MSI message interrupt. In future if 8259 * Solaris remove the restriction, then we need to delete the below 8260 * code and try to use multiple interrupt routine to gain better 8261 * performance. 8262 */ 8263 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 8264 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 8265 "force to use one interrupt routine though the " 8266 "HBA supports %d interrupt", count); 8267 count = 1; 8268 } 8269 8270 /* Allocate an array of interrupt handles. */ 8271 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 8272 ahci_ctlp->ahcictl_intr_htable = 8273 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 8274 8275 /* call ddi_intr_alloc(). */ 8276 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 8277 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 8278 8279 if ((rc != DDI_SUCCESS) || (actual == 0)) { 8280 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8281 "ddi_intr_alloc() failed, rc %d count %d actual %d " 8282 "avail %d\n", rc, count, actual, avail); 8283 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8284 ahci_ctlp->ahcictl_intr_size); 8285 return (DDI_FAILURE); 8286 } 8287 8288 /* use interrupt count returned */ 8289 #if AHCI_DEBUG 8290 if (actual < count) { 8291 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8292 "Requested: %d, Received: %d", count, actual); 8293 } 8294 #endif 8295 8296 ahci_ctlp->ahcictl_intr_cnt = actual; 8297 8298 /* 8299 * Get priority for first, assume remaining are all the same. 8300 */ 8301 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 8302 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 8303 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8304 "ddi_intr_get_pri() failed", NULL); 8305 8306 /* Free already allocated intr. */ 8307 for (i = 0; i < actual; i++) { 8308 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 8309 } 8310 8311 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8312 ahci_ctlp->ahcictl_intr_size); 8313 return (DDI_FAILURE); 8314 } 8315 8316 /* Test for high level interrupt. */ 8317 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 8318 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8319 "ahci_add_intrs: Hi level intr not supported", NULL); 8320 8321 /* Free already allocated intr. */ 8322 for (i = 0; i < actual; i++) { 8323 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 8324 } 8325 8326 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8327 sizeof (ddi_intr_handle_t)); 8328 8329 return (DDI_FAILURE); 8330 } 8331 8332 /* Call ddi_intr_add_handler(). */ 8333 for (i = 0; i < actual; i++) { 8334 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 8335 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 8336 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8337 "ddi_intr_add_handler() failed", NULL); 8338 8339 /* Free already allocated intr. */ 8340 for (i = 0; i < actual; i++) { 8341 (void) ddi_intr_free( 8342 ahci_ctlp->ahcictl_intr_htable[i]); 8343 } 8344 8345 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8346 ahci_ctlp->ahcictl_intr_size); 8347 return (DDI_FAILURE); 8348 } 8349 } 8350 8351 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 8352 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 8353 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 8354 "ddi_intr_get_cap() failed", NULL); 8355 8356 /* Free already allocated intr. */ 8357 for (i = 0; i < actual; i++) { 8358 (void) ddi_intr_free( 8359 ahci_ctlp->ahcictl_intr_htable[i]); 8360 } 8361 8362 kmem_free(ahci_ctlp->ahcictl_intr_htable, 8363 ahci_ctlp->ahcictl_intr_size); 8364 return (DDI_FAILURE); 8365 } 8366 8367 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 8368 /* Call ddi_intr_block_enable() for MSI. */ 8369 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 8370 ahci_ctlp->ahcictl_intr_cnt); 8371 } else { 8372 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 8373 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 8374 (void) ddi_intr_enable( 8375 ahci_ctlp->ahcictl_intr_htable[i]); 8376 } 8377 } 8378 8379 return (DDI_SUCCESS); 8380 } 8381 8382 /* 8383 * Removes the registered interrupts irrespective of whether they 8384 * were legacy or MSI. 8385 * 8386 * NOTE: The controller interrupts must be disabled before calling 8387 * this routine. 8388 */ 8389 static void 8390 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 8391 { 8392 int x; 8393 8394 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 8395 8396 /* Disable all interrupts. */ 8397 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 8398 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 8399 /* Call ddi_intr_block_disable(). */ 8400 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 8401 ahci_ctlp->ahcictl_intr_cnt); 8402 } else { 8403 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 8404 (void) ddi_intr_disable( 8405 ahci_ctlp->ahcictl_intr_htable[x]); 8406 } 8407 } 8408 8409 /* Call ddi_intr_remove_handler(). */ 8410 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 8411 (void) ddi_intr_remove_handler( 8412 ahci_ctlp->ahcictl_intr_htable[x]); 8413 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 8414 } 8415 8416 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 8417 } 8418 8419 /* 8420 * This routine tries to put port into P:NotRunning state by clearing 8421 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 8422 * and PxCMD.CR to '0'. 8423 */ 8424 static int 8425 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 8426 ahci_port_t *ahci_portp, uint8_t port) 8427 { 8428 uint32_t port_cmd_status; 8429 int loop_count; 8430 8431 ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE || 8432 MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex)); 8433 8434 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8435 "ahci_put_port_into_notrunning_state enter: port %d", port); 8436 8437 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8438 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8439 8440 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 8441 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8442 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 8443 8444 /* Wait until PxCMD.CR is cleared */ 8445 loop_count = 0; 8446 do { 8447 port_cmd_status = 8448 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8449 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8450 8451 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 8452 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 8453 "clearing port %d CMD.CR timeout, " 8454 "port_cmd_status = 0x%x", port, 8455 port_cmd_status); 8456 /* 8457 * We are effectively timing out after 0.5 sec. 8458 * This value is specified in AHCI spec. 8459 */ 8460 break; 8461 } 8462 8463 /* Wait for 10 millisec */ 8464 drv_usecwait(AHCI_10MS_USECS); 8465 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 8466 8467 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 8468 8469 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 8470 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 8471 "ahci_put_port_into_notrunning_state: failed to clear " 8472 "PxCMD.CR to '0' after loop count: %d, and " 8473 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 8474 return (AHCI_FAILURE); 8475 } else { 8476 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 8477 "ahci_put_port_into_notrunning_state: succeeded to clear " 8478 "PxCMD.CR to '0' after loop count: %d, and " 8479 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 8480 return (AHCI_SUCCESS); 8481 } 8482 } 8483 8484 /* 8485 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 8486 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 8487 * stable state, then set PxCMD.ST to '1' to start the port directly. 8488 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 8489 * COMRESET to the device to put it in an idle state. 8490 * 8491 * The fifth argument returns whether the port reset is involved during 8492 * the process. 8493 * 8494 * The routine will be called under following scenarios: 8495 * + To reset the HBA 8496 * + To abort the packet(s) 8497 * + To reset the port 8498 * + To activate the port 8499 * + Fatal error recovery 8500 * + To abort the timeout packet(s) 8501 * 8502 * NOTES!!! During this procedure, PxSERR register will be cleared, and 8503 * according to the spec, the clearance of three bits will also clear 8504 * three interrupt status bits. 8505 * 1. PxSERR.DIAG.F will clear PxIS.UFS 8506 * 2. PxSERR.DIAG.X will clear PxIS.PCS 8507 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 8508 * 8509 * Among these three interrupt events, the driver needs to take care of 8510 * PxIS.PRCS, which is the hot plug event. When the driver found out 8511 * a device was unplugged, it will call the interrupt handler. 8512 */ 8513 static int 8514 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 8515 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 8516 { 8517 uint32_t port_sstatus; 8518 uint32_t task_file_status; 8519 sata_device_t sdevice; 8520 int rval; 8521 ahci_addr_t addr_port; 8522 ahci_pmult_info_t *pminfo = NULL; 8523 int dev_exists_begin = 0; 8524 int dev_exists_end = 0; 8525 uint32_t previous_dev_type = ahci_portp->ahciport_device_type; 8526 int npmport = 0; 8527 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8528 8529 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 8530 8531 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8532 "ahci_restart_port_wait_till_ready: port %d enter", port); 8533 8534 AHCI_ADDR_SET_PORT(&addr_port, port); 8535 8536 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 8537 dev_exists_begin = 1; 8538 8539 /* First clear PxCMD.ST */ 8540 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 8541 port); 8542 if (rval != AHCI_SUCCESS) 8543 /* 8544 * If PxCMD.CR does not clear within a reasonable time, it 8545 * may assume the interface is in a hung condition and may 8546 * continue with issuing the port reset. 8547 */ 8548 goto reset; 8549 8550 /* Then clear PxSERR */ 8551 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8552 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 8553 AHCI_SERROR_CLEAR_ALL); 8554 8555 /* Then get PxTFD */ 8556 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8557 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 8558 8559 /* 8560 * Check whether the device is in a stable status, if yes, 8561 * then start the port directly. However for ahci_tran_reset_dport, 8562 * we may have to perform a port reset. 8563 */ 8564 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 8565 !(flag & AHCI_PORT_RESET)) 8566 goto out; 8567 8568 reset: 8569 /* 8570 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 8571 * a COMRESET to the device 8572 */ 8573 ahci_disable_port_intrs(ahci_ctlp, port); 8574 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port); 8575 ahci_enable_port_intrs(ahci_ctlp, port); 8576 8577 #ifdef AHCI_DEBUG 8578 if (rval != AHCI_SUCCESS) 8579 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8580 "ahci_restart_port_wait_till_ready: port %d failed", 8581 port); 8582 #endif 8583 8584 if (reset_flag != NULL) 8585 *reset_flag = 1; 8586 8587 /* Indicate to the framework that a reset has happened. */ 8588 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 8589 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) && 8590 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 8591 /* Set the reset in progress flag */ 8592 ahci_portp->ahciport_reset_in_progress = 1; 8593 8594 bzero((void *)&sdevice, sizeof (sata_device_t)); 8595 sdevice.satadev_addr.cport = 8596 ahci_ctlp->ahcictl_port_to_cport[port]; 8597 sdevice.satadev_addr.pmport = 0; 8598 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8599 8600 sdevice.satadev_state = SATA_DSTATE_RESET | 8601 SATA_DSTATE_PWR_ACTIVE; 8602 if (ahci_ctlp->ahcictl_sata_hba_tran) { 8603 mutex_exit(&ahci_portp->ahciport_mutex); 8604 sata_hba_event_notify( 8605 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8606 &sdevice, 8607 SATA_EVNT_DEVICE_RESET); 8608 mutex_enter(&ahci_portp->ahciport_mutex); 8609 } 8610 8611 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8612 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8613 } else { 8614 ahci_portp->ahciport_reset_in_progress = 0; 8615 } 8616 8617 out: 8618 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8619 8620 /* SStatus tells the presence of device. */ 8621 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8622 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 8623 8624 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 8625 dev_exists_end = 1; 8626 } 8627 8628 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */ 8629 return (rval); 8630 8631 /* Check whether a hot plug event happened */ 8632 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */ 8633 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8634 "ahci_restart_port_wait_till_ready: port %d " 8635 "device is removed", port); 8636 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 8637 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8638 "ahci_restart_port_wait_till_ready: port %d " 8639 "AHCI_PORT_FLAG_NODEV flag is set", port); 8640 mutex_exit(&ahci_portp->ahciport_mutex); 8641 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 8642 mutex_enter(&ahci_portp->ahciport_mutex); 8643 8644 return (rval); 8645 } 8646 8647 8648 /* 0/1 -> 1 : device may change */ 8649 /* 8650 * May be called by ahci_fatal_error_recovery_handler, so 8651 * don't issue software if the previous device is ATAPI. 8652 */ 8653 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 8654 return (rval); 8655 8656 /* 8657 * The COMRESET will make port multiplier enter legacy mode. 8658 * Issue a software reset to make it work again. 8659 */ 8660 ahci_disable_port_intrs(ahci_ctlp, port); 8661 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port); 8662 ahci_enable_port_intrs(ahci_ctlp, port); 8663 8664 /* 8665 * Following codes are specific for the port multiplier 8666 */ 8667 if (previous_dev_type != SATA_DTYPE_PMULT && 8668 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8669 /* in case previous_dev_type is corrupt */ 8670 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8671 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8672 return (rval); 8673 } 8674 8675 /* Device change: PMult -> Non-PMult */ 8676 if (previous_dev_type == SATA_DTYPE_PMULT && 8677 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8678 /* 8679 * This might happen because 8680 * 1. Software reset failed. Port multiplier is not correctly 8681 * enumerated. 8682 * 2. Another non-port-multiplier device is attached. Perhaps 8683 * the port multiplier was replaced by another device by 8684 * whatever reason, but AHCI driver missed hot-plug event. 8685 * 8686 * Now that the port has been initialized, we just need to 8687 * update the port structure according new device, then report 8688 * and wait SATA framework to probe new device. 8689 */ 8690 8691 /* Force to release pmult resource */ 8692 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8693 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8694 8695 bzero((void *)&sdevice, sizeof (sata_device_t)); 8696 sdevice.satadev_addr.cport = 8697 ahci_ctlp->ahcictl_port_to_cport[port]; 8698 sdevice.satadev_addr.pmport = 0; 8699 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8700 8701 sdevice.satadev_state = SATA_DSTATE_RESET | 8702 SATA_DSTATE_PWR_ACTIVE; 8703 8704 mutex_exit(&ahci_portp->ahciport_mutex); 8705 sata_hba_event_notify( 8706 ahci_ctlp->ahcictl_dip, 8707 &sdevice, 8708 SATA_EVNT_DEVICE_RESET); 8709 mutex_enter(&ahci_portp->ahciport_mutex); 8710 8711 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8712 "Port multiplier is [Gone] at port %d ", port); 8713 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8714 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8715 8716 return (AHCI_SUCCESS); 8717 } 8718 8719 /* Device change: Non-PMult -> PMult */ 8720 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8721 8722 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */ 8723 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 8724 8725 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8726 } 8727 pminfo = ahci_portp->ahciport_pmult_info; 8728 ASSERT(pminfo != NULL); 8729 8730 /* Device (may) change: PMult -> PMult */ 8731 /* 8732 * First initialize port multiplier. Set state to READY and wait for 8733 * probe entry point to initialize it 8734 */ 8735 ahci_portp->ahciport_port_state = SATA_STATE_READY; 8736 8737 /* 8738 * It's a little complicated while target is a port multiplier. we 8739 * need to COMRESET all pmports behind that PMult otherwise those 8740 * sub-links between the PMult and the sub-devices will be in an 8741 * inactive state (indicated by PSCR0/PxSSTS) and the following access 8742 * to those sub-devices will be rejected by Link-Fatal-Error. 8743 */ 8744 /* 8745 * The PxSNTF will be set soon after the pmult is plugged. While the 8746 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we 8747 * simply mark every sub-port as 'unknown', then ahci_probe_pmport 8748 * will initialized it. 8749 */ 8750 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++) 8751 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN; 8752 8753 /* Report reset event. */ 8754 ahci_portp->ahciport_reset_in_progress = 1; 8755 8756 bzero((void *)&sdevice, sizeof (sata_device_t)); 8757 sdevice.satadev_addr.cport = cport; 8758 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 8759 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 8760 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE; 8761 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice, 8762 SATA_EVNT_DEVICE_RESET); 8763 8764 return (rval); 8765 } 8766 8767 /* 8768 * This routine may be called under four scenarios: 8769 * a) do the recovery from fatal error 8770 * b) or we need to timeout some commands 8771 * c) or we need to abort some commands 8772 * d) or we need reset device/port/controller 8773 * 8774 * In all these scenarios, we need to send any pending unfinished 8775 * commands up to sata framework. 8776 */ 8777 static void 8778 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 8779 ahci_port_t *ahci_portp, 8780 uint32_t slot_status, 8781 uint32_t failed_tags, 8782 uint32_t timeout_tags, 8783 uint32_t aborted_tags, 8784 uint32_t reset_tags) 8785 { 8786 uint32_t finished_tags = 0; 8787 uint32_t unfinished_tags = 0; 8788 int tmp_slot; 8789 sata_pkt_t *satapkt; 8790 int ncq_cmd_in_progress = 0; 8791 int err_retri_cmd_in_progress = 0; 8792 int rdwr_pmult_cmd_in_progress = 0; 8793 8794 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 8795 8796 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8797 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 8798 ahci_portp->ahciport_port_num, slot_status); 8799 8800 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8801 "ahci_mop_commands: failed_tags: 0x%x, " 8802 "timeout_tags: 0x%x aborted_tags: 0x%x, " 8803 "reset_tags: 0x%x", failed_tags, 8804 timeout_tags, aborted_tags, reset_tags); 8805 8806 #ifdef AHCI_DEBUG 8807 if (ahci_debug_flags & AHCIDBG_ERRS) { 8808 int i; 8809 char msg_buf[200] = {0, }; 8810 for (i = 0x1f; i >= 0; i--) { 8811 if (ahci_portp->ahciport_slot_pkts[i] != NULL) 8812 msg_buf[i] = 'X'; 8813 else 8814 msg_buf[i] = '.'; 8815 } 8816 msg_buf[0x20] = '\0'; 8817 cmn_err(CE_NOTE, "port[%d] slots: %s", 8818 ahci_portp->ahciport_port_num, msg_buf); 8819 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ", 8820 (void *)ahci_portp->ahciport_err_retri_pkt, 8821 (void *)ahci_portp->ahciport_rdwr_pmult_pkt); 8822 } 8823 #endif 8824 8825 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8826 finished_tags = ahci_portp->ahciport_pending_tags & 8827 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 8828 8829 unfinished_tags = slot_status & 8830 AHCI_SLOT_MASK(ahci_ctlp) & 8831 ~failed_tags & 8832 ~aborted_tags & 8833 ~reset_tags & 8834 ~timeout_tags; 8835 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8836 ncq_cmd_in_progress = 1; 8837 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 8838 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 8839 8840 unfinished_tags = slot_status & 8841 AHCI_NCQ_SLOT_MASK(ahci_portp) & 8842 ~failed_tags & 8843 ~aborted_tags & 8844 ~reset_tags & 8845 ~timeout_tags; 8846 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8847 8848 /* 8849 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 8850 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 8851 * complete successfully due to one of the following three 8852 * conditions: 8853 * 8854 * 1. Fatal error - failed_tags includes its slot 8855 * 2. Timed out - timeout_tags includes its slot 8856 * 3. Aborted when hot unplug - aborted_tags includes its 8857 * slot 8858 * 8859 * Please note that the command is always sent down in Slot 0 8860 */ 8861 err_retri_cmd_in_progress = 1; 8862 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 8863 "ahci_mop_commands is called for port %d while " 8864 "REQUEST SENSE or READ LOG EXT for error retrieval " 8865 "is being executed slot_status = 0x%x", 8866 ahci_portp->ahciport_port_num, slot_status); 8867 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 8868 ASSERT(slot_status == 0x1); 8869 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 8870 rdwr_pmult_cmd_in_progress = 1; 8871 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 8872 "ahci_mop_commands is called for port %d while " 8873 "READ/WRITE PORTMULT command is being executed", 8874 ahci_portp->ahciport_port_num); 8875 8876 ASSERT(slot_status == 0x1); 8877 } 8878 8879 #ifdef AHCI_DEBUG 8880 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8881 "ahci_mop_commands: finished_tags: 0x%x, " 8882 "unfinished_tags 0x%x", finished_tags, unfinished_tags); 8883 #endif 8884 8885 /* Send up finished packets with SATA_PKT_COMPLETED */ 8886 while (finished_tags) { 8887 tmp_slot = ddi_ffs(finished_tags) - 1; 8888 if (tmp_slot == -1) { 8889 break; 8890 } 8891 8892 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8893 ASSERT(satapkt != NULL); 8894 8895 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 8896 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 8897 (void *)satapkt); 8898 8899 /* 8900 * Cannot fetch the return register content since the port 8901 * was restarted, so the corresponding tag will be set to 8902 * aborted tags. 8903 */ 8904 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 8905 CLEAR_BIT(finished_tags, tmp_slot); 8906 aborted_tags |= tmp_slot; 8907 continue; 8908 } 8909 8910 if (ncq_cmd_in_progress) 8911 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8912 tmp_slot); 8913 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8914 CLEAR_BIT(finished_tags, tmp_slot); 8915 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8916 8917 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED); 8918 } 8919 8920 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 8921 while (failed_tags) { 8922 if (err_retri_cmd_in_progress) { 8923 satapkt = ahci_portp->ahciport_err_retri_pkt; 8924 ASSERT(satapkt != NULL); 8925 ASSERT(failed_tags == 0x1); 8926 8927 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8928 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8929 (void *)satapkt); 8930 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8931 break; 8932 } 8933 if (rdwr_pmult_cmd_in_progress) { 8934 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8935 ASSERT(satapkt != NULL); 8936 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8937 "ahci_mop_commands: sending up " 8938 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR", 8939 (void *)satapkt); 8940 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8941 break; 8942 } 8943 8944 tmp_slot = ddi_ffs(failed_tags) - 1; 8945 if (tmp_slot == -1) { 8946 break; 8947 } 8948 8949 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8950 ASSERT(satapkt != NULL); 8951 8952 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8953 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8954 (void *)satapkt); 8955 8956 if (ncq_cmd_in_progress) 8957 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8958 tmp_slot); 8959 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8960 CLEAR_BIT(failed_tags, tmp_slot); 8961 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8962 8963 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8964 } 8965 8966 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 8967 while (timeout_tags) { 8968 if (err_retri_cmd_in_progress) { 8969 satapkt = ahci_portp->ahciport_err_retri_pkt; 8970 ASSERT(satapkt != NULL); 8971 ASSERT(timeout_tags == 0x1); 8972 8973 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8974 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8975 (void *)satapkt); 8976 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8977 break; 8978 } 8979 if (rdwr_pmult_cmd_in_progress) { 8980 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8981 ASSERT(satapkt != NULL); 8982 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8983 "ahci_mop_commands: sending up " 8984 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT", 8985 (void *)satapkt); 8986 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8987 break; 8988 } 8989 8990 tmp_slot = ddi_ffs(timeout_tags) - 1; 8991 if (tmp_slot == -1) { 8992 break; 8993 } 8994 8995 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8996 ASSERT(satapkt != NULL); 8997 8998 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8999 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 9000 (void *)satapkt); 9001 9002 if (ncq_cmd_in_progress) 9003 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9004 tmp_slot); 9005 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9006 CLEAR_BIT(timeout_tags, tmp_slot); 9007 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9008 9009 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 9010 } 9011 9012 /* Send up aborted packets with SATA_PKT_ABORTED */ 9013 while (aborted_tags) { 9014 if (err_retri_cmd_in_progress) { 9015 satapkt = ahci_portp->ahciport_err_retri_pkt; 9016 ASSERT(satapkt != NULL); 9017 ASSERT(aborted_tags == 0x1); 9018 9019 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9020 "sending up pkt 0x%p with SATA_PKT_ABORTED", 9021 (void *)satapkt); 9022 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9023 break; 9024 } 9025 if (rdwr_pmult_cmd_in_progress) { 9026 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9027 ASSERT(satapkt != NULL); 9028 ASSERT(aborted_tags == 0x1); 9029 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9030 "ahci_mop_commands: sending up " 9031 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED", 9032 (void *)satapkt); 9033 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9034 break; 9035 } 9036 9037 tmp_slot = ddi_ffs(aborted_tags) - 1; 9038 if (tmp_slot == -1) { 9039 break; 9040 } 9041 9042 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9043 ASSERT(satapkt != NULL); 9044 9045 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9046 "sending up pkt 0x%p with SATA_PKT_ABORTED", 9047 (void *)satapkt); 9048 9049 if (ncq_cmd_in_progress) 9050 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9051 tmp_slot); 9052 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9053 CLEAR_BIT(aborted_tags, tmp_slot); 9054 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9055 9056 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED); 9057 } 9058 9059 /* Send up reset packets with SATA_PKT_RESET. */ 9060 while (reset_tags) { 9061 if (rdwr_pmult_cmd_in_progress) { 9062 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9063 ASSERT(satapkt != NULL); 9064 ASSERT(aborted_tags == 0x1); 9065 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9066 "ahci_mop_commands: sending up " 9067 "rdwr pmult pkt 0x%p with SATA_PKT_RESET", 9068 (void *)satapkt); 9069 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9070 break; 9071 } 9072 9073 tmp_slot = ddi_ffs(reset_tags) - 1; 9074 if (tmp_slot == -1) { 9075 break; 9076 } 9077 9078 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9079 ASSERT(satapkt != NULL); 9080 9081 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9082 "sending up pkt 0x%p with SATA_PKT_RESET", 9083 (void *)satapkt); 9084 9085 if (ncq_cmd_in_progress) 9086 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9087 tmp_slot); 9088 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9089 CLEAR_BIT(reset_tags, tmp_slot); 9090 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9091 9092 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9093 } 9094 9095 /* Send up unfinished packets with SATA_PKT_RESET */ 9096 while (unfinished_tags) { 9097 tmp_slot = ddi_ffs(unfinished_tags) - 1; 9098 if (tmp_slot == -1) { 9099 break; 9100 } 9101 9102 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9103 ASSERT(satapkt != NULL); 9104 9105 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 9106 "sending up pkt 0x%p with SATA_PKT_RESET", 9107 (void *)satapkt); 9108 9109 if (ncq_cmd_in_progress) 9110 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 9111 tmp_slot); 9112 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 9113 CLEAR_BIT(unfinished_tags, tmp_slot); 9114 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 9115 9116 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET); 9117 } 9118 9119 ahci_portp->ahciport_mop_in_progress--; 9120 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 9121 9122 if (ahci_portp->ahciport_mop_in_progress == 0) 9123 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 9124 9125 ahci_flush_doneq(ahci_portp); 9126 } 9127 9128 /* 9129 * This routine is going to first request a READ LOG EXT sata pkt from sata 9130 * module, and then deliver it to the HBA to get the ncq failure context. 9131 * The return value is the exactly failed tags. 9132 */ 9133 static uint32_t 9134 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9135 uint8_t port) 9136 { 9137 sata_device_t sdevice; 9138 sata_pkt_t *rdlog_spkt, *spkt; 9139 ddi_dma_handle_t buf_dma_handle; 9140 ahci_addr_t addr; 9141 int loop_count; 9142 int rval; 9143 int failed_slot; 9144 uint32_t failed_tags = 0; 9145 struct sata_ncq_error_recovery_page *ncq_err_page; 9146 9147 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 9148 "ahci_get_rdlogext_data enter: port %d", port); 9149 9150 /* Prepare the sdevice data */ 9151 bzero((void *)&sdevice, sizeof (sata_device_t)); 9152 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 9153 9154 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 9155 sdevice.satadev_addr.pmport = 0; 9156 9157 /* Translate sata_device.satadev_addr -> ahci_addr */ 9158 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 9159 9160 /* 9161 * Call the sata hba interface to get a rdlog spkt 9162 */ 9163 loop_count = 0; 9164 loop: 9165 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 9166 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 9167 if (rdlog_spkt == NULL) { 9168 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 9169 /* Sleep for a while */ 9170 drv_usecwait(AHCI_10MS_USECS); 9171 goto loop; 9172 } 9173 /* Timed out after 1s */ 9174 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9175 "failed to get rdlog spkt for port %d", port); 9176 return (failed_tags); 9177 } 9178 9179 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 9180 9181 /* 9182 * This flag is used to handle the specific error recovery when the 9183 * READ LOG EXT command gets a failure (fatal error or time-out). 9184 */ 9185 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 9186 9187 /* 9188 * This start is not supposed to fail because after port is restarted, 9189 * the whole command list is empty. 9190 */ 9191 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 9192 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt); 9193 ahci_portp->ahciport_err_retri_pkt = NULL; 9194 9195 /* Remove the flag after READ LOG EXT command is completed */ 9196 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT; 9197 9198 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 9199 /* Update the request log data */ 9200 buf_dma_handle = *(ddi_dma_handle_t *) 9201 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 9202 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 9203 DDI_DMA_SYNC_FORKERNEL); 9204 if (rval == DDI_SUCCESS) { 9205 ncq_err_page = 9206 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 9207 satapkt_cmd.satacmd_bp->b_un.b_addr; 9208 9209 /* Get the failed tag */ 9210 failed_slot = ncq_err_page->ncq_tag; 9211 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9212 "ahci_get_rdlogext_data: port %d " 9213 "failed slot %d", port, failed_slot); 9214 if (failed_slot & NQ) { 9215 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9216 "the failed slot is not a valid tag", NULL); 9217 goto out; 9218 } 9219 9220 failed_slot &= NCQ_TAG_MASK; 9221 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 9222 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9223 "ahci_get_rdlogext_data: failed spkt 0x%p", 9224 (void *)spkt); 9225 if (spkt == NULL) { 9226 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9227 "the failed slot spkt is NULL", NULL); 9228 goto out; 9229 } 9230 9231 failed_tags = 0x1 << failed_slot; 9232 9233 /* Fill out the error context */ 9234 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 9235 ncq_err_page); 9236 ahci_update_sata_registers(ahci_ctlp, port, 9237 &spkt->satapkt_device); 9238 } 9239 } 9240 out: 9241 sata_free_error_retrieval_pkt(rdlog_spkt); 9242 9243 return (failed_tags); 9244 } 9245 9246 /* 9247 * This routine is going to first request a REQUEST SENSE sata pkt from sata 9248 * module, and then deliver it to the HBA to get the sense data and copy 9249 * the sense data back to the orignal failed sata pkt, and free the REQUEST 9250 * SENSE sata pkt later. 9251 */ 9252 static void 9253 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9254 uint8_t port, sata_pkt_t *spkt) 9255 { 9256 sata_device_t sdevice; 9257 sata_pkt_t *rs_spkt; 9258 sata_cmd_t *sata_cmd; 9259 ddi_dma_handle_t buf_dma_handle; 9260 ahci_addr_t addr; 9261 int loop_count; 9262 #if AHCI_DEBUG 9263 struct scsi_extended_sense *rqsense; 9264 #endif 9265 9266 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9267 "ahci_get_rqsense_data enter: port %d", port); 9268 9269 /* Prepare the sdevice data */ 9270 bzero((void *)&sdevice, sizeof (sata_device_t)); 9271 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 9272 9273 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 9274 sdevice.satadev_addr.pmport = 0; 9275 9276 /* Translate sata_device.satadev_addr -> ahci_addr */ 9277 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 9278 9279 sata_cmd = &spkt->satapkt_cmd; 9280 9281 /* 9282 * Call the sata hba interface to get a rs spkt 9283 */ 9284 loop_count = 0; 9285 loop: 9286 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 9287 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 9288 if (rs_spkt == NULL) { 9289 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 9290 /* Sleep for a while */ 9291 drv_usecwait(AHCI_10MS_USECS); 9292 goto loop; 9293 9294 } 9295 /* Timed out after 1s */ 9296 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9297 "failed to get rs spkt for port %d", port); 9298 return; 9299 } 9300 9301 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 9302 9303 /* 9304 * This flag is used to handle the specific error recovery when the 9305 * REQUEST SENSE command gets a faiure (fatal error or time-out). 9306 */ 9307 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 9308 9309 /* 9310 * This start is not supposed to fail because after port is restarted, 9311 * the whole command list is empty. 9312 */ 9313 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 9314 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt); 9315 ahci_portp->ahciport_err_retri_pkt = NULL; 9316 9317 /* Remove the flag after REQUEST SENSE command is completed */ 9318 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE; 9319 9320 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 9321 /* Update the request sense data */ 9322 buf_dma_handle = *(ddi_dma_handle_t *) 9323 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 9324 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 9325 DDI_DMA_SYNC_FORKERNEL); 9326 /* Copy the request sense data */ 9327 bcopy(rs_spkt-> 9328 satapkt_cmd.satacmd_bp->b_un.b_addr, 9329 &sata_cmd->satacmd_rqsense, 9330 SATA_ATAPI_MIN_RQSENSE_LEN); 9331 #if AHCI_DEBUG 9332 rqsense = (struct scsi_extended_sense *) 9333 sata_cmd->satacmd_rqsense; 9334 9335 /* Dump the sense data */ 9336 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 9337 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 9338 "Sense data for satapkt %p ATAPI cmd 0x%x", 9339 spkt, sata_cmd->satacmd_acdb[0]); 9340 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 9341 " es_code 0x%x es_class 0x%x " 9342 "es_key 0x%x es_add_code 0x%x " 9343 "es_qual_code 0x%x", 9344 rqsense->es_code, rqsense->es_class, 9345 rqsense->es_key, rqsense->es_add_code, 9346 rqsense->es_qual_code); 9347 #endif 9348 } 9349 9350 sata_free_error_retrieval_pkt(rs_spkt); 9351 } 9352 9353 /* 9354 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 9355 * the port must be restarted. When the HBA detects thus error, it may try 9356 * to abort a transfer. And if the transfer was aborted, the device is 9357 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 9358 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 9359 * that the device is in a stable status and transfers may be restarted without 9360 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 9361 * then the software will send the COMRESET to do the port reset. 9362 * 9363 * Software should perform the appropriate error recovery actions based on 9364 * whether non-queued commands were being issued or natived command queuing 9365 * commands were being issued. 9366 * 9367 * And software will complete the command that had the error with error mark 9368 * to higher level software. 9369 * 9370 * Fatal errors include the following: 9371 * PxIS.IFS - Interface Fatal Error Status 9372 * PxIS.HBDS - Host Bus Data Error Status 9373 * PxIS.HBFS - Host Bus Fatal Error Status 9374 * PxIS.TFES - Task File Error Status 9375 */ 9376 static void 9377 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9378 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status) 9379 { 9380 uint32_t port_cmd_status; 9381 uint32_t slot_status = 0; 9382 uint32_t failed_tags = 0; 9383 int failed_slot; 9384 int reset_flag = 0, flag = 0; 9385 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 9386 sata_cmd_t *sata_cmd = NULL; 9387 sata_pkt_t *spkt = NULL; 9388 #if AHCI_DEBUG 9389 ahci_cmd_header_t *cmd_header; 9390 #endif 9391 uint8_t port = addrp->aa_port; 9392 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9393 int rval; 9394 9395 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 9396 9397 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9398 "ahci_fatal_error_recovery_handler enter: port %d", port); 9399 9400 /* Port multiplier error */ 9401 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 9402 /* FBS code is neither completed nor tested. */ 9403 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp, 9404 port, intr_status); 9405 9406 /* Force a port reset */ 9407 flag = AHCI_PORT_RESET; 9408 } 9409 9410 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9411 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9412 9413 /* Read PxCI to see which commands are still outstanding */ 9414 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9415 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9416 9417 /* 9418 * Read PxCMD.CCS to determine the slot that the HBA 9419 * was processing when the error occurred. 9420 */ 9421 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9422 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9423 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9424 AHCI_CMD_STATUS_CCS_SHIFT; 9425 9426 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9427 spkt = ahci_portp->ahciport_err_retri_pkt; 9428 ASSERT(spkt != NULL); 9429 } else { 9430 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 9431 if (spkt == NULL) { 9432 /* May happen when interface errors occur? */ 9433 goto next; 9434 } 9435 } 9436 9437 #if AHCI_DEBUG 9438 /* 9439 * Debugging purpose... 9440 */ 9441 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 9442 cmd_header = 9443 &ahci_portp->ahciport_cmd_list[failed_slot]; 9444 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9445 "ahci_fatal_error_recovery_handler: port %d, " 9446 "PRD Byte Count = 0x%x, " 9447 "ahciport_prd_bytecounts = 0x%x", port, 9448 cmd_header->ahcich_prd_byte_count, 9449 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 9450 } 9451 #endif 9452 9453 sata_cmd = &spkt->satapkt_cmd; 9454 9455 /* Fill out the status and error registers for PxIS.TFES */ 9456 if (intr_status & AHCI_INTR_STATUS_TFES) { 9457 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 9458 ahcirf_d2h_register_fis); 9459 9460 /* Copy the error context back to the sata_cmd */ 9461 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 9462 } 9463 9464 /* The failed command must be one of the outstanding commands */ 9465 failed_tags = 0x1 << failed_slot; 9466 ASSERT(failed_tags & slot_status); 9467 9468 /* Update the sata registers, especially PxSERR register */ 9469 ahci_update_sata_registers(ahci_ctlp, port, 9470 &spkt->satapkt_device); 9471 9472 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9473 /* Read PxSACT to see which commands are still outstanding */ 9474 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9475 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9476 } 9477 next: 9478 9479 #if AHCI_DEBUG 9480 /* 9481 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9482 * set, it means a fatal error happened after REQUEST SENSE command 9483 * or READ LOG EXT command is delivered to the HBA during the error 9484 * recovery process. At this time, the only outstanding command is 9485 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9486 */ 9487 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9488 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9489 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 9490 "command or READ LOG EXT command for error data retrieval " 9491 "failed", port); 9492 ASSERT(slot_status == 0x1); 9493 ASSERT(failed_slot == 0); 9494 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 9495 SCMD_REQUEST_SENSE || 9496 spkt->satapkt_cmd.satacmd_cmd_reg == 9497 SATAC_READ_LOG_EXT); 9498 } 9499 #endif 9500 9501 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9502 ahci_portp->ahciport_mop_in_progress++; 9503 9504 rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9505 port, flag, &reset_flag); 9506 9507 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) { 9508 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 9509 if (rval == AHCI_SUCCESS) 9510 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9511 "succeed", instance, port); 9512 else 9513 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9514 "failed", instance, port); 9515 } 9516 9517 /* 9518 * Won't retrieve error information: 9519 * 1. Port reset was involved to recover 9520 * 2. Device is gone 9521 * 3. IDENTIFY DEVICE command sent to ATAPI device 9522 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 9523 */ 9524 if (reset_flag || 9525 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 9526 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 9527 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9528 goto out; 9529 9530 /* 9531 * Deliver READ LOG EXT to gather information about the error when 9532 * a COMRESET has not been performed as part of the error recovery 9533 * during NCQ command processing. 9534 */ 9535 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9536 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 9537 ahci_portp, port); 9538 goto out; 9539 } 9540 9541 /* 9542 * Deliver REQUEST SENSE for ATAPI command to gather information about 9543 * the error when a COMRESET has not been performed as part of the 9544 * error recovery. 9545 */ 9546 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 9547 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 9548 out: 9549 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9550 "ahci_fatal_error_recovery_handler: port %d fatal error " 9551 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 9552 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 9553 port, slot_status, ahci_portp->ahciport_pending_tags, 9554 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 9555 9556 ahci_mop_commands(ahci_ctlp, 9557 ahci_portp, 9558 slot_status, 9559 failed_tags, /* failed tags */ 9560 0, /* timeout tags */ 9561 0, /* aborted tags */ 9562 0); /* reset tags */ 9563 } 9564 9565 /* 9566 * Used to recovery a PMULT pmport fatal error under FIS-based switching. 9567 * 1. device specific.PxFBS.SDE=1 9568 * 2. Non Device specific. 9569 * Nothing will be done when Command-based switching is employed. 9570 * 9571 * Currently code is neither completed nor tested. 9572 */ 9573 static void 9574 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9575 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 9576 { 9577 #ifndef __lock_lint 9578 _NOTE(ARGUNUSED(intr_status)) 9579 #endif 9580 uint32_t port_fbs_ctrl; 9581 int loop_count = 0; 9582 ahci_addr_t addr; 9583 9584 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 9585 9586 /* Nothing will be done under Command-based switching. */ 9587 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS)) 9588 return; 9589 9590 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9591 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9592 9593 if (!(port_fbs_ctrl & AHCI_FBS_EN)) 9594 /* FBS is not enabled. */ 9595 return; 9596 9597 /* Problem's getting complicated now. */ 9598 /* 9599 * If FIS-based switching is used, we need to check 9600 * the PxFBS to see the error type. 9601 */ 9602 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9603 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9604 9605 /* Refer to spec(v1.2) 9.3.6.1 */ 9606 if (port_fbs_ctrl & AHCI_FBS_SDE) { 9607 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9608 "A Device Sepcific Error: port %d", port); 9609 /* 9610 * Controller has paused commands for all other 9611 * sub-devices until PxFBS.DEC is set. 9612 */ 9613 ahci_reject_all_abort_pkts(ahci_ctlp, 9614 ahci_portp, 0); 9615 9616 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 9617 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port), 9618 port_fbs_ctrl | AHCI_FBS_DEC); 9619 9620 /* 9621 * Wait controller clear PxFBS.DEC, 9622 * then we can continue. 9623 */ 9624 loop_count = 0; 9625 do { 9626 port_fbs_ctrl = ddi_get32(ahci_ctlp-> 9627 ahcictl_ahci_acc_handle, (uint32_t *) 9628 AHCI_PORT_PxFBS(ahci_ctlp, port)); 9629 9630 if (loop_count++ > 1000) 9631 /* 9632 * Esclate the error. Follow 9633 * non-device specific error 9634 * procedure. 9635 */ 9636 return; 9637 9638 drv_usecwait(AHCI_100US_USECS); 9639 } while (port_fbs_ctrl & AHCI_FBS_DEC); 9640 9641 /* 9642 * Issue a software reset to ensure drive is in 9643 * a known state. 9644 */ 9645 (void) ahci_software_reset(ahci_ctlp, 9646 ahci_portp, &addr); 9647 9648 } else { 9649 9650 /* Process Non-Device Specific Error. */ 9651 /* This will be handled later on. */ 9652 cmn_err(CE_NOTE, "!FBS is not supported now."); 9653 } 9654 } 9655 /* 9656 * Handle events - fatal error recovery 9657 */ 9658 static void 9659 ahci_events_handler(void *args) 9660 { 9661 ahci_event_arg_t *ahci_event_arg; 9662 ahci_ctl_t *ahci_ctlp; 9663 ahci_port_t *ahci_portp; 9664 ahci_addr_t *addrp; 9665 uint32_t event; 9666 int instance; 9667 9668 ahci_event_arg = (ahci_event_arg_t *)args; 9669 9670 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 9671 ahci_portp = ahci_event_arg->ahciea_portp; 9672 addrp = ahci_event_arg->ahciea_addrp; 9673 event = ahci_event_arg->ahciea_event; 9674 instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9675 9676 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9677 "ahci_events_handler enter: port %d intr_status = 0x%x", 9678 ahci_portp->ahciport_port_num, event); 9679 9680 mutex_enter(&ahci_portp->ahciport_mutex); 9681 9682 /* 9683 * ahci_intr_phyrdy_change() may have rendered it to 9684 * SATA_DTYPE_NONE. 9685 */ 9686 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9687 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 9688 "ahci_events_handler: port %d no device attached, " 9689 "and just return without doing anything", 9690 ahci_portp->ahciport_port_num); 9691 9692 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) { 9693 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT; 9694 cmn_err(CE_WARN, "!ahci%d: error recovery for port %d " 9695 "succeed", instance, ahci_portp->ahciport_port_num); 9696 } 9697 9698 goto out; 9699 } 9700 9701 if (event & (AHCI_INTR_STATUS_IFS | 9702 AHCI_INTR_STATUS_HBDS | 9703 AHCI_INTR_STATUS_HBFS | 9704 AHCI_INTR_STATUS_TFES)) 9705 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 9706 addrp, event); 9707 9708 out: 9709 mutex_exit(&ahci_portp->ahciport_mutex); 9710 } 9711 9712 /* 9713 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 9714 * detect there are some commands which are timed out. 9715 */ 9716 static void 9717 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9718 uint8_t port, uint32_t tmp_timeout_tags) 9719 { 9720 uint32_t slot_status = 0; 9721 uint32_t finished_tags = 0; 9722 uint32_t timeout_tags = 0; 9723 9724 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 9725 "ahci_timeout_pkts enter: port %d", port); 9726 9727 mutex_enter(&ahci_portp->ahciport_mutex); 9728 9729 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9730 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) || 9731 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9732 /* Read PxCI to see which commands are still outstanding */ 9733 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9734 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9735 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9736 /* Read PxSACT to see which commands are still outstanding */ 9737 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9738 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9739 } 9740 9741 #if AHCI_DEBUG 9742 /* 9743 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9744 * set, it means a fatal error happened after REQUEST SENSE command 9745 * or READ LOG EXT command is delivered to the HBA during the error 9746 * recovery process. At this time, the only outstanding command is 9747 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9748 */ 9749 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9750 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9751 "ahci_timeout_pkts called while REQUEST SENSE " 9752 "command or READ LOG EXT command for error recovery " 9753 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 9754 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 9755 tmp_timeout_tags, slot_status, 9756 ahci_portp->ahciport_pending_tags, 9757 ahci_portp->ahciport_pending_ncq_tags); 9758 ASSERT(slot_status == 0x1); 9759 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9760 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9761 "ahci_timeout_pkts called while executing R/W PMULT " 9762 "command timeout_tags = 0x%x, slot_status = 0x%x", 9763 tmp_timeout_tags, slot_status); 9764 ASSERT(slot_status == 0x1); 9765 } 9766 #endif 9767 9768 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9769 ahci_portp->ahciport_mop_in_progress++; 9770 9771 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9772 port, AHCI_PORT_RESET, NULL); 9773 9774 /* 9775 * Re-identify timeout tags because some previously checked commands 9776 * could already complete. 9777 */ 9778 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9779 finished_tags = ahci_portp->ahciport_pending_tags & 9780 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 9781 timeout_tags = tmp_timeout_tags & ~finished_tags; 9782 9783 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 9784 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9785 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 9786 "pending_tags = 0x%x ", 9787 port, finished_tags, timeout_tags, 9788 slot_status, ahci_portp->ahciport_pending_tags); 9789 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9790 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 9791 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 9792 timeout_tags = tmp_timeout_tags & ~finished_tags; 9793 9794 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 9795 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9796 "timeout_tags = 0x%x, port_sactive = 0x%x, " 9797 "pending_ncq_tags = 0x%x ", 9798 port, finished_tags, timeout_tags, 9799 slot_status, ahci_portp->ahciport_pending_ncq_tags); 9800 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9801 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9802 timeout_tags = tmp_timeout_tags; 9803 } 9804 9805 ahci_mop_commands(ahci_ctlp, 9806 ahci_portp, 9807 slot_status, 9808 0, /* failed tags */ 9809 timeout_tags, /* timeout tags */ 9810 0, /* aborted tags */ 9811 0); /* reset tags */ 9812 9813 mutex_exit(&ahci_portp->ahciport_mutex); 9814 } 9815 9816 /* 9817 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 9818 * for long time. 9819 */ 9820 static void 9821 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 9822 { 9823 ahci_port_t *ahci_portp; 9824 sata_pkt_t *spkt; 9825 uint32_t pending_tags; 9826 uint32_t timeout_tags; 9827 uint32_t port_cmd_status; 9828 uint32_t port_sactive; 9829 uint8_t port; 9830 int tmp_slot; 9831 int current_slot; 9832 uint32_t current_tags; 9833 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9834 9835 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9836 9837 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9838 "ahci_watchdog_handler entered", NULL); 9839 9840 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9841 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9842 continue; 9843 } 9844 9845 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9846 9847 mutex_enter(&ahci_portp->ahciport_mutex); 9848 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9849 mutex_exit(&ahci_portp->ahciport_mutex); 9850 continue; 9851 } 9852 9853 /* Skip the check for those ports in error recovery */ 9854 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 9855 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 9856 mutex_exit(&ahci_portp->ahciport_mutex); 9857 continue; 9858 } 9859 9860 pending_tags = 0; 9861 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9862 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9863 9864 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9865 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9866 current_slot = 0; 9867 pending_tags = 0x1; 9868 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9869 current_slot = 9870 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9871 AHCI_CMD_STATUS_CCS_SHIFT; 9872 pending_tags = ahci_portp->ahciport_pending_tags; 9873 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9874 port_sactive = ddi_get32( 9875 ahci_ctlp->ahcictl_ahci_acc_handle, 9876 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9877 current_tags = port_sactive & 9878 ~port_cmd_status & 9879 AHCI_NCQ_SLOT_MASK(ahci_portp); 9880 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 9881 } 9882 9883 timeout_tags = 0; 9884 while (pending_tags) { 9885 tmp_slot = ddi_ffs(pending_tags) - 1; 9886 if (tmp_slot == -1) { 9887 break; 9888 } 9889 9890 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9891 spkt = ahci_portp->ahciport_err_retri_pkt; 9892 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 9893 spkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9894 else 9895 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9896 9897 if ((spkt != NULL) && spkt->satapkt_time && 9898 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 9899 /* 9900 * If a packet has survived for more than it's 9901 * max life cycles, it is a candidate for time 9902 * out. 9903 */ 9904 ahci_portp->ahciport_slot_timeout[tmp_slot] -= 9905 ahci_watchdog_timeout; 9906 9907 if (ahci_portp->ahciport_slot_timeout[tmp_slot] 9908 > 0) 9909 goto next; 9910 9911 #if AHCI_DEBUG 9912 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9913 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9914 ahci_ctlp, "watchdog: the current " 9915 "tags is 0x%x", current_tags); 9916 } else { 9917 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9918 ahci_ctlp, "watchdog: the current " 9919 "slot is %d", current_slot); 9920 } 9921 #endif 9922 9923 /* 9924 * We need to check whether the HBA has 9925 * begun to execute the command, if not, 9926 * then re-set the timer of the command. 9927 */ 9928 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 9929 (tmp_slot != current_slot) || 9930 NCQ_CMD_IN_PROGRESS(ahci_portp) && 9931 ((0x1 << tmp_slot) & current_tags)) { 9932 ahci_portp->ahciport_slot_timeout \ 9933 [tmp_slot] = spkt->satapkt_time; 9934 } else { 9935 timeout_tags |= (0x1 << tmp_slot); 9936 cmn_err(CE_WARN, "!ahci%d: watchdog " 9937 "port %d satapkt 0x%p timed out\n", 9938 instance, port, (void *)spkt); 9939 } 9940 } 9941 next: 9942 CLEAR_BIT(pending_tags, tmp_slot); 9943 } 9944 9945 if (timeout_tags) { 9946 mutex_exit(&ahci_portp->ahciport_mutex); 9947 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9948 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 9949 port, timeout_tags); 9950 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9951 mutex_enter(&ahci_portp->ahciport_mutex); 9952 } 9953 9954 mutex_exit(&ahci_portp->ahciport_mutex); 9955 } 9956 9957 /* Re-install the watchdog timeout handler */ 9958 if (ahci_ctlp->ahcictl_timeout_id != 0) { 9959 ahci_ctlp->ahcictl_timeout_id = 9960 timeout((void (*)(void *))ahci_watchdog_handler, 9961 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 9962 } 9963 9964 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9965 } 9966 9967 /* 9968 * Fill the error context into sata_cmd for non-queued command error. 9969 */ 9970 static void 9971 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9972 { 9973 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 9974 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9975 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9976 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9977 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9978 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9979 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9980 9981 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 9982 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9983 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9984 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9985 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9986 } 9987 } 9988 9989 /* 9990 * Fill the ncq error page into sata_cmd for queued command error. 9991 */ 9992 static void 9993 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 9994 struct sata_ncq_error_recovery_page *ncq_err_page) 9995 { 9996 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 9997 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 9998 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 9999 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 10000 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 10001 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 10002 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 10003 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 10004 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 10005 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 10006 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 10007 } 10008 10009 /* 10010 * Put the respective register value to sata_cmd_t for satacmd_flags. 10011 */ 10012 static void 10013 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 10014 { 10015 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 10016 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 10017 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 10018 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 10019 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 10020 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 10021 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 10022 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 10023 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 10024 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 10025 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 10026 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 10027 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 10028 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 10029 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 10030 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 10031 if (scmd->satacmd_flags.sata_copy_out_device_reg) 10032 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 10033 if (scmd->satacmd_flags.sata_copy_out_error_reg) 10034 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 10035 } 10036 10037 static void 10038 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 10039 uint32_t intr_status) 10040 { 10041 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 10042 10043 if (intr_status & AHCI_INTR_STATUS_IFS) 10044 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 10045 "error", instance, port); 10046 10047 if (intr_status & AHCI_INTR_STATUS_HBDS) 10048 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 10049 instance, port); 10050 10051 if (intr_status & AHCI_INTR_STATUS_HBFS) 10052 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 10053 instance, port); 10054 10055 if (intr_status & AHCI_INTR_STATUS_TFES) 10056 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 10057 instance, port); 10058 10059 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 10060 "recovery", instance, port); 10061 } 10062 10063 static void 10064 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port, 10065 uint32_t slot_tags) 10066 { 10067 ahci_port_t *ahci_portp; 10068 int tmp_slot; 10069 sata_pkt_t *spkt; 10070 sata_cmd_t cmd; 10071 10072 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 10073 ASSERT(ahci_portp != NULL); 10074 10075 while (slot_tags) { 10076 tmp_slot = ddi_ffs(slot_tags) - 1; 10077 if (tmp_slot == -1) { 10078 break; 10079 } 10080 10081 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 10082 if (spkt != NULL) { 10083 cmd = spkt->satapkt_cmd; 10084 10085 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x " 10086 "features_reg = 0x%x sec_count_msb = 0x%x " 10087 "lba_low_msb = 0x%x lba_mid_msb = 0x%x " 10088 "lba_high_msb = 0x%x sec_count_lsb = 0x%x " 10089 "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x " 10090 "lba_high_lsb = 0x%x device_reg = 0x%x " 10091 "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt, 10092 cmd.satacmd_cmd_reg, cmd.satacmd_features_reg, 10093 cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb, 10094 cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb, 10095 cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb, 10096 cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb, 10097 cmd.satacmd_device_reg, cmd.satacmd_addr_type, 10098 *((uint32_t *)&(cmd.satacmd_flags))); 10099 } 10100 10101 CLEAR_BIT(slot_tags, tmp_slot); 10102 } 10103 } 10104 10105 /* 10106 * Dump the serror message to the log. 10107 */ 10108 static void 10109 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 10110 uint32_t port_serror, int debug_only) 10111 { 10112 static char err_buf[512]; 10113 static char err_msg_header[16]; 10114 char *err_msg = err_buf; 10115 10116 *err_buf = '\0'; 10117 *err_msg_header = '\0'; 10118 10119 if (port_serror & SERROR_DATA_ERR_FIXED) { 10120 err_msg = strcat(err_msg, 10121 "\tRecovered Data Integrity Error (I)\n"); 10122 } 10123 10124 if (port_serror & SERROR_COMM_ERR_FIXED) { 10125 err_msg = strcat(err_msg, 10126 "\tRecovered Communication Error (M)\n"); 10127 } 10128 10129 if (port_serror & SERROR_DATA_ERR) { 10130 err_msg = strcat(err_msg, 10131 "\tTransient Data Integrity Error (T)\n"); 10132 } 10133 10134 if (port_serror & SERROR_PERSISTENT_ERR) { 10135 err_msg = strcat(err_msg, 10136 "\tPersistent Communication or Data Integrity Error (C)\n"); 10137 } 10138 10139 if (port_serror & SERROR_PROTOCOL_ERR) { 10140 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 10141 } 10142 10143 if (port_serror & SERROR_INT_ERR) { 10144 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 10145 } 10146 10147 if (port_serror & SERROR_PHY_RDY_CHG) { 10148 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 10149 } 10150 10151 if (port_serror & SERROR_PHY_INT_ERR) { 10152 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 10153 } 10154 10155 if (port_serror & SERROR_COMM_WAKE) { 10156 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 10157 } 10158 10159 if (port_serror & SERROR_10B_TO_8B_ERR) { 10160 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 10161 } 10162 10163 if (port_serror & SERROR_DISPARITY_ERR) { 10164 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 10165 } 10166 10167 if (port_serror & SERROR_CRC_ERR) { 10168 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 10169 } 10170 10171 if (port_serror & SERROR_HANDSHAKE_ERR) { 10172 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 10173 } 10174 10175 if (port_serror & SERROR_LINK_SEQ_ERR) { 10176 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 10177 } 10178 10179 if (port_serror & SERROR_TRANS_ERR) { 10180 err_msg = strcat(err_msg, 10181 "\tTransport state transition error (T)\n"); 10182 } 10183 10184 if (port_serror & SERROR_FIS_TYPE) { 10185 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 10186 } 10187 10188 if (port_serror & SERROR_EXCHANGED_ERR) { 10189 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 10190 } 10191 10192 if (*err_msg == '\0') 10193 return; 10194 10195 if (debug_only) { 10196 (void) sprintf(err_msg_header, "port %d", port); 10197 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 10198 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 10199 } else if (ahci_ctlp) { 10200 cmn_err(CE_WARN, "!ahci%d: %s %s", 10201 ddi_get_instance(ahci_ctlp->ahcictl_dip), 10202 err_msg_header, err_msg); 10203 10204 /* sata trace debug */ 10205 sata_trace_debug(ahci_ctlp->ahcictl_dip, 10206 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 10207 err_msg_header, err_msg); 10208 } else { 10209 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 10210 10211 /* sata trace debug */ 10212 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 10213 } 10214 } 10215 10216 /* 10217 * Translate the sata_address_t type into the ahci_addr_t type. 10218 * sata_device.satadev_addr structure is used as source. 10219 */ 10220 static void 10221 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd, 10222 ahci_addr_t *ahci_addrp) 10223 { 10224 sata_address_t *sata_addrp = &sd->satadev_addr; 10225 ahci_addrp->aa_port = 10226 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport]; 10227 ahci_addrp->aa_pmport = sata_addrp->pmport; 10228 10229 switch (sata_addrp->qual) { 10230 case SATA_ADDR_DCPORT: 10231 case SATA_ADDR_CPORT: 10232 ahci_addrp->aa_qual = AHCI_ADDR_PORT; 10233 break; 10234 case SATA_ADDR_PMULT: 10235 case SATA_ADDR_PMULT_SPEC: 10236 ahci_addrp->aa_qual = AHCI_ADDR_PMULT; 10237 break; 10238 case SATA_ADDR_DPMPORT: 10239 case SATA_ADDR_PMPORT: 10240 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT; 10241 break; 10242 case SATA_ADDR_NULL: 10243 default: 10244 /* something went wrong */ 10245 ahci_addrp->aa_qual = AHCI_ADDR_NULL; 10246 break; 10247 } 10248 } 10249 10250 /* 10251 * This routine is to calculate the total number of ports implemented 10252 * by the HBA. 10253 */ 10254 static int 10255 ahci_get_num_implemented_ports(uint32_t ports_implemented) 10256 { 10257 uint8_t i; 10258 int num = 0; 10259 10260 for (i = 0; i < AHCI_MAX_PORTS; i++) { 10261 if (((uint32_t)0x1 << i) & ports_implemented) 10262 num++; 10263 } 10264 10265 return (num); 10266 } 10267 10268 #if AHCI_DEBUG 10269 static void 10270 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 10271 { 10272 static char name[16]; 10273 va_list ap; 10274 10275 mutex_enter(&ahci_log_mutex); 10276 10277 va_start(ap, fmt); 10278 if (ahci_ctlp) { 10279 (void) sprintf(name, "ahci%d: ", 10280 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10281 } else { 10282 (void) sprintf(name, "ahci: "); 10283 } 10284 10285 (void) vsprintf(ahci_log_buf, fmt, ap); 10286 va_end(ap); 10287 10288 cmn_err(level, "%s%s", name, ahci_log_buf); 10289 10290 mutex_exit(&ahci_log_mutex); 10291 } 10292 #endif 10293 10294 /* 10295 * quiesce(9E) entry point. 10296 * 10297 * This function is called when the system is single-threaded at high 10298 * PIL with preemption disabled. Therefore, this function must not be 10299 * blocked. Because no taskqs are running, there is no need for us to 10300 * take any action for enclosure services which are running in the 10301 * taskq context, especially as no interrupts are generated by it nor 10302 * are any messages expected to come in. 10303 * 10304 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 10305 * DDI_FAILURE indicates an error condition and should almost never happen. 10306 */ 10307 static int 10308 ahci_quiesce(dev_info_t *dip) 10309 { 10310 ahci_ctl_t *ahci_ctlp; 10311 ahci_port_t *ahci_portp; 10312 int instance, port; 10313 10314 instance = ddi_get_instance(dip); 10315 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 10316 10317 if (ahci_ctlp == NULL) 10318 return (DDI_FAILURE); 10319 10320 #if AHCI_DEBUG 10321 ahci_debug_flags = 0; 10322 #endif 10323 10324 ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE; 10325 10326 /* disable all the interrupts. */ 10327 ahci_disable_all_intrs(ahci_ctlp); 10328 10329 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 10330 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 10331 continue; 10332 } 10333 10334 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 10335 10336 /* 10337 * Stop the port by clearing PxCMD.ST 10338 * 10339 * Here we must disable the port interrupt because 10340 * ahci_disable_all_intrs only clear GHC.IE, and IS 10341 * register will be still set if PxIE is enabled. 10342 * When ahci shares one IRQ with other drivers, the 10343 * intr handler may claim the intr mistakenly. 10344 */ 10345 ahci_disable_port_intrs(ahci_ctlp, port); 10346 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 10347 ahci_portp, port); 10348 } 10349 10350 ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE; 10351 10352 return (DDI_SUCCESS); 10353 } 10354 10355 /* 10356 * The function will add a sata packet to the done queue. 10357 */ 10358 static void 10359 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason) 10360 { 10361 ASSERT(satapkt != NULL); 10362 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 10363 10364 /* set the reason for all packets */ 10365 satapkt->satapkt_reason = reason; 10366 satapkt->satapkt_hba_driver_private = NULL; 10367 10368 if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && 10369 satapkt->satapkt_comp) { 10370 /* 10371 * only add to queue when mode is not synch and there is 10372 * completion callback 10373 */ 10374 *ahci_portp->ahciport_doneqtail = satapkt; 10375 ahci_portp->ahciport_doneqtail = 10376 (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private); 10377 ahci_portp->ahciport_doneq_len++; 10378 10379 } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && 10380 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) 10381 /* 10382 * for sync/non-poll mode, just call cv_broadcast 10383 */ 10384 cv_broadcast(&ahci_portp->ahciport_cv); 10385 } 10386 10387 /* 10388 * The function will call completion callback of sata packet on the 10389 * completed queue 10390 */ 10391 static void 10392 ahci_flush_doneq(ahci_port_t *ahci_portp) 10393 { 10394 sata_pkt_t *satapkt, *next; 10395 10396 ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex)); 10397 10398 if (ahci_portp->ahciport_doneq) { 10399 satapkt = ahci_portp->ahciport_doneq; 10400 10401 ahci_portp->ahciport_doneq = NULL; 10402 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq; 10403 ahci_portp->ahciport_doneq_len = 0; 10404 10405 mutex_exit(&ahci_portp->ahciport_mutex); 10406 10407 while (satapkt != NULL) { 10408 next = satapkt->satapkt_hba_driver_private; 10409 satapkt->satapkt_hba_driver_private = NULL; 10410 10411 /* Call the callback */ 10412 (*satapkt->satapkt_comp)(satapkt); 10413 10414 satapkt = next; 10415 } 10416 10417 mutex_enter(&ahci_portp->ahciport_mutex); 10418 } 10419 } 10420 10421 /* 10422 * Sets the state for the specified port on the controller to desired state. 10423 * This must be run in the context of the enclosure taskq which ensures that 10424 * only one event is outstanding at any time. 10425 */ 10426 static boolean_t 10427 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire) 10428 { 10429 ahci_em_led_msg_t msg; 10430 ahci_em_msg_hdr_t hdr; 10431 uint32_t msgval, hdrval; 10432 uint_t i, max_delay = ahci_em_tx_delay_count; 10433 10434 msg.alm_hba = port; 10435 msg.alm_pminfo = 0; 10436 msg.alm_value = 0; 10437 10438 if (desire & AHCI_EM_LED_IDENT_ENABLE) { 10439 msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF; 10440 } 10441 10442 if (desire & AHCI_EM_LED_FAULT_ENABLE) { 10443 msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF; 10444 } 10445 10446 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 && 10447 (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) { 10448 msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF; 10449 } 10450 10451 hdr.aemh_rsvd = 0; 10452 hdr.aemh_mlen = sizeof (ahci_em_led_msg_t); 10453 hdr.aemh_dlen = 0; 10454 hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED; 10455 10456 bcopy(&msg, &msgval, sizeof (msgval)); 10457 bcopy(&hdr, &hdrval, sizeof (hdrval)); 10458 10459 /* 10460 * First, make sure we can transmit. We should not have been placed in a 10461 * situation where an outstanding transmission is going on. 10462 */ 10463 for (i = 0; i < max_delay; i++) { 10464 uint32_t val; 10465 10466 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10467 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10468 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0) 10469 break; 10470 10471 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000)); 10472 } 10473 10474 if (i == max_delay) 10475 return (B_FALSE); 10476 10477 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10478 (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval); 10479 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10480 (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval); 10481 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10482 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM); 10483 10484 for (i = 0; i < max_delay; i++) { 10485 uint32_t val; 10486 10487 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10488 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10489 if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0) 10490 break; 10491 10492 delay(drv_usectohz(ahci_em_tx_delay_ms * 1000)); 10493 } 10494 10495 if (i == max_delay) 10496 return (B_FALSE); 10497 10498 return (B_TRUE); 10499 } 10500 10501 typedef struct ahci_em_led_task_arg { 10502 ahci_ctl_t *aelta_ctl; 10503 uint8_t aelta_port; 10504 uint_t aelta_op; 10505 ahci_em_led_state_t aelta_state; 10506 uint_t aelta_ret; 10507 kcondvar_t aelta_cv; 10508 uint_t aelta_ref; 10509 } ahci_em_led_task_arg_t; 10510 10511 static void 10512 ahci_em_led_task_free(ahci_em_led_task_arg_t *task) 10513 { 10514 ASSERT3U(task->aelta_ref, ==, 0); 10515 cv_destroy(&task->aelta_cv); 10516 kmem_free(task, sizeof (*task)); 10517 } 10518 10519 static void 10520 ahci_em_led_task(void *arg) 10521 { 10522 boolean_t ret, cleanup = B_FALSE; 10523 ahci_em_led_task_arg_t *led = arg; 10524 ahci_em_led_state_t state; 10525 10526 mutex_enter(&led->aelta_ctl->ahcictl_mutex); 10527 if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) { 10528 led->aelta_ret = EIO; 10529 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10530 return; 10531 } 10532 10533 state = led->aelta_ctl->ahcictl_em_state[led->aelta_port]; 10534 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10535 10536 switch (led->aelta_op) { 10537 case AHCI_EM_IOC_SET_OP_ADD: 10538 state |= led->aelta_state; 10539 break; 10540 case AHCI_EM_IOC_SET_OP_REM: 10541 state &= ~led->aelta_state; 10542 break; 10543 case AHCI_EM_IOC_SET_OP_SET: 10544 state = led->aelta_state; 10545 break; 10546 default: 10547 led->aelta_ret = ENOTSUP; 10548 return; 10549 } 10550 10551 ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state); 10552 10553 mutex_enter(&led->aelta_ctl->ahcictl_mutex); 10554 if (ret) { 10555 led->aelta_ctl->ahcictl_em_state[led->aelta_port] = state; 10556 led->aelta_ret = 0; 10557 } else { 10558 led->aelta_ret = EIO; 10559 led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10560 } 10561 led->aelta_ref--; 10562 if (led->aelta_ref > 0) { 10563 cv_signal(&led->aelta_cv); 10564 } else { 10565 cleanup = B_TRUE; 10566 } 10567 mutex_exit(&led->aelta_ctl->ahcictl_mutex); 10568 10569 if (cleanup) { 10570 ahci_em_led_task_free(led); 10571 } 10572 } 10573 10574 static void 10575 ahci_em_reset(void *arg) 10576 { 10577 uint_t i, max_delay = ahci_em_reset_delay_count; 10578 ahci_ctl_t *ahci_ctlp = arg; 10579 10580 /* 10581 * We've been asked to reset the device. The caller should have set the 10582 * resetting flag. Make sure that we don't have a request to quiesce. 10583 */ 10584 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10585 ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING); 10586 if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) { 10587 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10588 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10589 return; 10590 } 10591 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10592 10593 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 10594 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST); 10595 for (i = 0; i < max_delay; i++) { 10596 uint32_t val; 10597 10598 val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 10599 (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp)); 10600 if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0) 10601 break; 10602 10603 delay(drv_usectohz(ahci_em_reset_delay_ms * 1000)); 10604 } 10605 10606 if (i == max_delay) { 10607 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10608 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10609 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10610 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10611 cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting", 10612 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10613 return; 10614 } 10615 10616 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) { 10617 10618 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) 10619 continue; 10620 10621 /* 10622 * Try to flush all the LEDs as part of reset. If it fails, 10623 * drive on. 10624 */ 10625 if (!ahci_em_set_led(ahci_ctlp, i, 10626 ahci_ctlp->ahcictl_em_state[i])) { 10627 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10628 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10629 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT; 10630 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10631 cmn_err(CE_WARN, "!ahci%d: enclosure timed out " 10632 "setting port %u", 10633 ddi_get_instance(ahci_ctlp->ahcictl_dip), i); 10634 return; 10635 } 10636 } 10637 10638 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10639 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING; 10640 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY; 10641 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10642 } 10643 10644 static boolean_t 10645 ahci_em_init(ahci_ctl_t *ahci_ctlp) 10646 { 10647 char name[128]; 10648 10649 /* 10650 * First make sure we actually have enclosure services and if so, that 10651 * we have the hardware support that we care about for this. 10652 */ 10653 if (ahci_ctlp->ahcictl_em_loc == 0 || 10654 (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0) 10655 return (B_TRUE); 10656 10657 /* 10658 * Next, make sure that the buffer is large enough for us. We need two 10659 * dwords or 8 bytes. The location register is stored in dwords. 10660 */ 10661 if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) < 10662 AHCI_EM_BUFFER_MIN) { 10663 return (B_TRUE); 10664 } 10665 10666 ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc & 10667 AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4; 10668 ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr; 10669 10670 bzero(ahci_ctlp->ahcictl_em_state, 10671 sizeof (ahci_ctlp->ahcictl_em_state)); 10672 10673 (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d", 10674 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10675 if ((ahci_ctlp->ahcictl_em_taskq = 10676 ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1, 10677 TASKQ_DEFAULTPRI, 0)) == NULL) { 10678 cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em " 10679 "services", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 10680 return (B_FALSE); 10681 } 10682 10683 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10684 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT | AHCI_EM_RESETTING; 10685 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10686 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset, 10687 ahci_ctlp, DDI_SLEEP); 10688 10689 return (B_TRUE); 10690 } 10691 10692 static int 10693 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg) 10694 { 10695 int i; 10696 ahci_ioc_em_get_t get; 10697 10698 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10699 return (ENOTSUP); 10700 } 10701 10702 bzero(&get, sizeof (get)); 10703 get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented; 10704 if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) { 10705 get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY; 10706 } 10707 10708 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10709 for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) { 10710 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) { 10711 continue; 10712 } 10713 get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i]; 10714 } 10715 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10716 10717 if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0) 10718 return (EFAULT); 10719 10720 return (0); 10721 } 10722 10723 static int 10724 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg) 10725 { 10726 int ret; 10727 ahci_ioc_em_set_t set; 10728 ahci_em_led_task_arg_t *task; 10729 boolean_t signal, cleanup; 10730 10731 if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0) 10732 return (EFAULT); 10733 10734 if (set.aiems_port > ahci_ctlp->ahcictl_num_ports) 10735 return (EINVAL); 10736 10737 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) { 10738 return (EINVAL); 10739 } 10740 10741 if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE | 10742 AHCI_EM_LED_FAULT_ENABLE | 10743 AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) { 10744 return (EINVAL); 10745 } 10746 10747 switch (set.aiems_op) { 10748 case AHCI_EM_IOC_SET_OP_ADD: 10749 case AHCI_EM_IOC_SET_OP_REM: 10750 case AHCI_EM_IOC_SET_OP_SET: 10751 break; 10752 default: 10753 return (EINVAL); 10754 } 10755 10756 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10757 return (ENOTSUP); 10758 } 10759 10760 if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 && 10761 ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) { 10762 return (ENOTSUP); 10763 } 10764 10765 task = kmem_alloc(sizeof (*task), KM_NOSLEEP | KM_NORMALPRI); 10766 if (task == NULL) { 10767 return (ENOMEM); 10768 } 10769 10770 task->aelta_ctl = ahci_ctlp; 10771 task->aelta_port = (uint8_t)set.aiems_port; 10772 task->aelta_op = set.aiems_op; 10773 task->aelta_state = set.aiems_leds; 10774 10775 cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL); 10776 10777 /* 10778 * Initialize the reference count to two. One for us and one for the 10779 * taskq. This will be used in case we get canceled. 10780 */ 10781 task->aelta_ref = 2; 10782 10783 /* 10784 * Once dispatched, the task state is protected by our global mutex. 10785 */ 10786 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, 10787 ahci_em_led_task, task, DDI_SLEEP); 10788 10789 signal = B_FALSE; 10790 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10791 while (task->aelta_ref > 1) { 10792 if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) == 10793 0) { 10794 signal = B_TRUE; 10795 break; 10796 } 10797 } 10798 10799 /* 10800 * Remove our reference count. If we were woken up because of a signal 10801 * then the taskq may still be dispatched. In which case we shouldn't 10802 * free this memory until it is done. In that case, the taskq will take 10803 * care of it. 10804 */ 10805 task->aelta_ref--; 10806 cleanup = (task->aelta_ref == 0); 10807 if (signal) { 10808 ret = EINTR; 10809 } else { 10810 ret = task->aelta_ret; 10811 } 10812 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10813 10814 if (cleanup) { 10815 ahci_em_led_task_free(task); 10816 } 10817 10818 return (ret); 10819 } 10820 10821 static int 10822 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg) 10823 { 10824 int inst; 10825 ahci_ctl_t *ahci_ctlp; 10826 10827 inst = ddi_get_instance(dip); 10828 if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) { 10829 return (ENXIO); 10830 } 10831 10832 switch (cmd) { 10833 case AHCI_EM_IOC_GET: 10834 return (ahci_em_ioctl_get(ahci_ctlp, arg)); 10835 case AHCI_EM_IOC_SET: 10836 return (ahci_em_ioctl_set(ahci_ctlp, arg)); 10837 default: 10838 return (ENOTTY); 10839 } 10840 10841 } 10842 10843 static void 10844 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp) 10845 { 10846 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10847 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10848 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10849 return; 10850 } 10851 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE; 10852 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10853 10854 ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq); 10855 } 10856 10857 static void 10858 ahci_em_suspend(ahci_ctl_t *ahci_ctlp) 10859 { 10860 ahci_em_quiesce(ahci_ctlp); 10861 10862 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10863 ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY; 10864 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10865 } 10866 10867 static void 10868 ahci_em_resume(ahci_ctl_t *ahci_ctlp) 10869 { 10870 mutex_enter(&ahci_ctlp->ahcictl_mutex); 10871 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10872 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10873 return; 10874 } 10875 ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING; 10876 mutex_exit(&ahci_ctlp->ahcictl_mutex); 10877 10878 (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset, 10879 ahci_ctlp, DDI_SLEEP); 10880 } 10881 10882 static void 10883 ahci_em_fini(ahci_ctl_t *ahci_ctlp) 10884 { 10885 if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) { 10886 return; 10887 } 10888 10889 ahci_em_quiesce(ahci_ctlp); 10890 ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq); 10891 ahci_ctlp->ahcictl_em_taskq = NULL; 10892 } 10893