xref: /illumos-gate/usr/src/uts/common/io/sata/adapters/ahci/ahci.c (revision 33b27906b01ade9752c1935377f3fefdf49b8f95)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
25  * Copyright (c) 2018, Joyent, Inc.
26  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
27  * Copyright 2021 RackTop Systems, Inc.
28  */
29 
30 /*
31  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
32  *
33  * Power Management Support
34  * ------------------------
35  *
36  * At the moment, the ahci driver only implements suspend/resume to
37  * support Suspend to RAM on X86 feature. Device power management isn't
38  * implemented, link power management is disabled, and hot plug isn't
39  * allowed during the period from suspend to resume.
40  *
41  * For s/r support, the ahci driver only need to implement DDI_SUSPEND
42  * and DDI_RESUME entries, and don't need to take care of new requests
43  * sent down after suspend because the target driver (sd) has already
44  * handled these conditions, and blocked these requests. For the detailed
45  * information, please check with sdopen, sdclose and sdioctl routines.
46  *
47  *
48  * Enclosure Management Support
49  * ----------------------------
50  *
51  * The ahci driver has basic support for AHCI Enclosure Management (EM)
52  * services. The AHCI specification provides an area in the primary ahci BAR for
53  * posting data to send out to the enclosure management and provides a register
54  * that provides both information and control about this. While the
55  * specification allows for multiple forms of enclosure management, the only
56  * supported, and commonly found form, is the AHCI specified LED format. The LED
57  * format is often implemented as a one-way communication mechanism. Software
58  * can write out what it cares about into the aforementioned data buffer and
59  * then we wait for the transmission to be sent.
60  *
61  * This has some drawbacks. It means that we cannot know whether or not it has
62  * succeeded. This means we cannot ask hardware what it thinks the LEDs are
63  * set to. There's also the added unfortunate reality that firmware on the
64  * microcontroller driving this will often not show the LEDs if no drive is
65  * present and that actions taken may potentially cause this to get out of sync
66  * with what we expect it to be. For example, the specification does not
67  * describe what should happen if a drive is removed from the enclosure while
68  * this is set and what should happen when it returns. We can only infer that it
69  * should be the same.
70  *
71  * Because only a single command can be sent at any time and we don't want to
72  * interfere with controller I/O, we create a taskq dedicated to this that has a
73  * single thread. Both resets (which occur on attach and resume) and normal
74  * changes to the LED state will be driven through this taskq. Because the taskq
75  * has a single thread, this guarantees serial processing.
76  *
77  * Each userland-submitted task (basically not resets) has a reference counted
78  * task structure. This allows the thread that called it to be cancelled and
79  * have the system clean itself up. The user thread in ioctl blocks on a CV that
80  * can receive signals as it waits for completion.  Note, there is no guarantee
81  * provided by the kernel that the first thread to enter the kernel will be the
82  * first one to change state.
83  */
84 
85 #include <sys/note.h>
86 #include <sys/scsi/scsi.h>
87 #include <sys/pci.h>
88 #include <sys/disp.h>
89 #include <sys/sata/sata_hba.h>
90 #include <sys/sata/adapters/ahci/ahcireg.h>
91 #include <sys/sata/adapters/ahci/ahcivar.h>
92 
93 /*
94  * FMA header files
95  */
96 #include <sys/ddifm.h>
97 #include <sys/fm/protocol.h>
98 #include <sys/fm/util.h>
99 #include <sys/fm/io/ddi.h>
100 
101 /*
102  * EM Control header files
103  */
104 #include <sys/types.h>
105 #include <sys/file.h>
106 #include <sys/errno.h>
107 #include <sys/open.h>
108 #include <sys/cred.h>
109 #include <sys/ddi.h>
110 #include <sys/sunddi.h>
111 
112 /*
113  * This is the string displayed by modinfo, etc.
114  */
115 static char ahci_ident[] = "ahci driver";
116 
117 /*
118  * Function prototypes for driver entry points
119  */
120 static	int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
121 static	int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
122 static	int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
123 static	int ahci_quiesce(dev_info_t *);
124 
125 /*
126  * Function prototypes for SATA Framework interfaces
127  */
128 static	int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
129 static	int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
130 
131 static	int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
132 static	int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
133 static	int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
134 static	int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
135 static	int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
136 static	int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
137 #if defined(__lock_lint)
138 static	int ahci_selftest(dev_info_t *, sata_device_t *);
139 #endif
140 
141 /*
142  * FMA Prototypes
143  */
144 static	void ahci_fm_init(ahci_ctl_t *);
145 static	void ahci_fm_fini(ahci_ctl_t *);
146 static	int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
147 int	ahci_check_acc_handle(ddi_acc_handle_t);
148 int	ahci_check_dma_handle(ddi_dma_handle_t);
149 void	ahci_fm_ereport(ahci_ctl_t *, char *);
150 static	int ahci_check_all_handle(ahci_ctl_t *);
151 static	int ahci_check_ctl_handle(ahci_ctl_t *);
152 static	int ahci_check_port_handle(ahci_ctl_t *, int);
153 static	int ahci_check_slot_handle(ahci_port_t *, int);
154 
155 /*
156  * Local function prototypes
157  */
158 static	int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
159 static	int ahci_alloc_ports_state(ahci_ctl_t *);
160 static	void ahci_dealloc_ports_state(ahci_ctl_t *);
161 static	int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
162 static	void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
163 static	int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
164 static	void ahci_dealloc_rcvd_fis(ahci_port_t *);
165 static	int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
166 static	void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
167 static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
168 static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
169 static	void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
170 static	void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
171 
172 static	int ahci_initialize_controller(ahci_ctl_t *);
173 static	void ahci_uninitialize_controller(ahci_ctl_t *);
174 static	int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
175 static	int ahci_config_space_init(ahci_ctl_t *);
176 static	void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
177 
178 static	void ahci_drain_ports_taskq(ahci_ctl_t *);
179 static	int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
180     uint8_t);
181 static	int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
182 static	int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
183 static	int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
184     sata_device_t *);
185 static	int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
186     sata_pmult_gscr_t *);
187 static	int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
188     sata_device_t *);
189 static	int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
190 static	int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
191 static	int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
192     sata_device_t *);
193 
194 static	void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
195 static	int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
196 static	void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
197     ahci_addr_t *);
198 static	void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
199 static	int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
200     ahci_addr_t *, sata_pkt_t *);
201 static	int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
202     ahci_addr_t *, sata_pkt_t *);
203 static	int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
204     ahci_addr_t *, int);
205 static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
206 static	void ahci_copy_ncq_err_page(sata_cmd_t *,
207     struct sata_ncq_error_recovery_page *);
208 static	void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
209 static	void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
210 static	void ahci_flush_doneq(ahci_port_t *);
211 
212 static	int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
213 static	int ahci_hba_reset(ahci_ctl_t *);
214 static	int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
215 static	int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
216 static	void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
217 static	int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
218     ahci_addr_t *);
219 static	int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
220     ahci_addr_t *);
221 static	int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
222     ahci_addr_t *);
223 static	int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
224 static	int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
225     uint8_t);
226 static	int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
227     uint8_t, int, int *);
228 static	void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
229     uint32_t, uint32_t, uint32_t, uint32_t);
230 static	uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
231 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
232     uint8_t, sata_pkt_t *);
233 static	void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
234     ahci_addr_t *, uint32_t);
235 static	void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
236     uint8_t, uint32_t);
237 static	void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
238     uint8_t, uint32_t);
239 static	void ahci_events_handler(void *);
240 static	void ahci_watchdog_handler(ahci_ctl_t *);
241 
242 static	uint_t ahci_intr(caddr_t, caddr_t);
243 static	void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
244 static	int ahci_add_intrs(ahci_ctl_t *, int);
245 static	void ahci_rem_intrs(ahci_ctl_t *);
246 static	void ahci_enable_all_intrs(ahci_ctl_t *);
247 static	void ahci_disable_all_intrs(ahci_ctl_t *);
248 static	void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
249 static	void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
250 
251 static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
252 static	int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
253 static	int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
254 static	int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
255 static	int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
256 static	int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
257     ahci_port_t *, uint8_t);
258 static	int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
259 static	int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
260     uint8_t, uint32_t);
261 static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
262     uint8_t, uint32_t);
263 static	int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
264 
265 static	void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
266 static	int ahci_get_num_implemented_ports(uint32_t);
267 static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
268 static	void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
269 static	void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
270 #if AHCI_DEBUG
271 static	void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
272 #endif
273 
274 static	boolean_t ahci_em_init(ahci_ctl_t *);
275 static	void ahci_em_fini(ahci_ctl_t *);
276 static	void ahci_em_suspend(ahci_ctl_t *);
277 static	void ahci_em_resume(ahci_ctl_t *);
278 static	int ahci_em_ioctl(dev_info_t *, int, intptr_t);
279 
280 
281 /*
282  * DMA attributes for the data buffer
283  *
284  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
285  * does not support 64-bit addressing
286  */
287 static ddi_dma_attr_t buffer_dma_attr = {
288 	DMA_ATTR_V0,		/* dma_attr_version */
289 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
290 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
291 	0x3fffffull,		/* dma_attr_count_max i.e. for one cookie */
292 	0x2ull,			/* dma_attr_align: word aligned */
293 	1,			/* dma_attr_burstsizes */
294 	1,			/* dma_attr_minxfer */
295 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
296 	0xffffffffull,		/* dma_attr_seg */
297 	AHCI_PRDT_NUMBER,	/* dma_attr_sgllen */
298 	512,			/* dma_attr_granular */
299 	0,			/* dma_attr_flags */
300 };
301 
302 /*
303  * DMA attributes for the rcvd FIS
304  *
305  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
306  * does not support 64-bit addressing
307  */
308 static ddi_dma_attr_t rcvd_fis_dma_attr = {
309 	DMA_ATTR_V0,		/* dma_attr_version */
310 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
311 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
312 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
313 	0x100ull,		/* dma_attr_align: 256-byte aligned */
314 	1,			/* dma_attr_burstsizes */
315 	1,			/* dma_attr_minxfer */
316 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
317 	0xffffffffull,		/* dma_attr_seg */
318 	1,			/* dma_attr_sgllen */
319 	1,			/* dma_attr_granular */
320 	0,			/* dma_attr_flags */
321 };
322 
323 /*
324  * DMA attributes for the command list
325  *
326  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
327  * does not support 64-bit addressing
328  */
329 static ddi_dma_attr_t cmd_list_dma_attr = {
330 	DMA_ATTR_V0,		/* dma_attr_version */
331 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
332 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
333 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
334 	0x400ull,		/* dma_attr_align: 1K-byte aligned */
335 	1,			/* dma_attr_burstsizes */
336 	1,			/* dma_attr_minxfer */
337 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
338 	0xffffffffull,		/* dma_attr_seg */
339 	1,			/* dma_attr_sgllen */
340 	1,			/* dma_attr_granular */
341 	0,			/* dma_attr_flags */
342 };
343 
344 /*
345  * DMA attributes for cmd tables
346  *
347  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
348  * does not support 64-bit addressing
349  */
350 static ddi_dma_attr_t cmd_table_dma_attr = {
351 	DMA_ATTR_V0,		/* dma_attr_version */
352 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
353 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
354 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
355 	0x80ull,		/* dma_attr_align: 128-byte aligned */
356 	1,			/* dma_attr_burstsizes */
357 	1,			/* dma_attr_minxfer */
358 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
359 	0xffffffffull,		/* dma_attr_seg */
360 	1,			/* dma_attr_sgllen */
361 	1,			/* dma_attr_granular */
362 	0,			/* dma_attr_flags */
363 };
364 
365 
366 /* Device access attributes */
367 static ddi_device_acc_attr_t accattr = {
368 	DDI_DEVICE_ATTR_V1,
369 	DDI_STRUCTURE_LE_ACC,
370 	DDI_STRICTORDER_ACC,
371 	DDI_DEFAULT_ACC
372 };
373 
374 static struct dev_ops ahcictl_dev_ops = {
375 	DEVO_REV,		/* devo_rev */
376 	0,			/* refcnt  */
377 	ahci_getinfo,		/* info */
378 	nulldev,		/* identify */
379 	nulldev,		/* probe */
380 	ahci_attach,		/* attach */
381 	ahci_detach,		/* detach */
382 	nodev,			/* no reset */
383 	NULL,			/* driver operations */
384 	NULL,			/* bus operations */
385 	NULL,			/* power */
386 	ahci_quiesce,		/* quiesce */
387 };
388 
389 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
390 	SATA_TRAN_HOTPLUG_OPS_REV_1,
391 	ahci_tran_hotplug_port_activate,
392 	ahci_tran_hotplug_port_deactivate
393 };
394 
395 extern struct mod_ops mod_driverops;
396 
397 static  struct modldrv modldrv = {
398 	&mod_driverops,		/* driverops */
399 	ahci_ident,		/* short description */
400 	&ahcictl_dev_ops,	/* driver ops */
401 };
402 
403 static  struct modlinkage modlinkage = {
404 	MODREV_1,
405 	&modldrv,
406 	NULL
407 };
408 
409 /* The following variables are watchdog handler related */
410 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
411 static clock_t ahci_watchdog_tick;
412 
413 /*
414  * This static variable indicates the size of command table,
415  * and it's changeable with prdt number, which ahci_dma_prdt_number
416  * indicates.
417  */
418 static size_t ahci_cmd_table_size;
419 
420 /*
421  * The below global variables are tunable via /etc/system
422  *
423  *	ahci_dma_prdt_number
424  *	ahci_msi_enabled
425  *	ahci_buf_64bit_dma
426  *	ahci_commu_64bit_dma
427  */
428 
429 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
430 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
431 
432 /* AHCI MSI is tunable */
433 boolean_t ahci_msi_enabled = B_TRUE;
434 
435 /*
436  * 64-bit dma addressing for data buffer is tunable
437  *
438  * The variable controls only the below value:
439  *	DBAU (upper 32-bits physical address of data block)
440  */
441 boolean_t ahci_buf_64bit_dma = B_TRUE;
442 
443 /*
444  * 64-bit dma addressing for communication system descriptors is tunable
445  *
446  * The variable controls the below three values:
447  *
448  *	PxCLBU (upper 32-bits for the command list base physical address)
449  *	PxFBU (upper 32-bits for the received FIS base physical address)
450  *	CTBAU (upper 32-bits of command table base)
451  */
452 boolean_t ahci_commu_64bit_dma = B_TRUE;
453 
454 /*
455  * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
456  * chipset. If the users want to have a try with 64-bit dma, please change
457  * the below variable value to enable it.
458  */
459 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
460 
461 /*
462  * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
463  * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
464  * please change the below value to enable it.
465  */
466 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
467 
468 /*
469  * These values control the default delay and default number of times to wait
470  * for an enclosure message to complete.
471  */
472 uint_t	ahci_em_reset_delay_ms = 1;
473 uint_t	ahci_em_reset_delay_count = 1000;
474 uint_t	ahci_em_tx_delay_ms = 1;
475 uint_t	ahci_em_tx_delay_count = 1000;
476 
477 
478 /*
479  * End of global tunable variable definition
480  */
481 
482 #if AHCI_DEBUG
483 uint32_t ahci_debug_flags = 0;
484 #else
485 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
486 #endif
487 
488 
489 #if AHCI_DEBUG
490 /* The following is needed for ahci_log() */
491 static kmutex_t ahci_log_mutex;
492 static char ahci_log_buf[512];
493 #endif
494 
495 /* Opaque state pointer initialized by ddi_soft_state_init() */
496 static void *ahci_statep = NULL;
497 
498 /*
499  *  ahci module initialization.
500  */
501 int
_init(void)502 _init(void)
503 {
504 	int	ret;
505 
506 	ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
507 	if (ret != 0) {
508 		goto err_out;
509 	}
510 
511 #if AHCI_DEBUG
512 	mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
513 #endif
514 
515 	if ((ret = sata_hba_init(&modlinkage)) != 0) {
516 #if AHCI_DEBUG
517 		mutex_destroy(&ahci_log_mutex);
518 #endif
519 		ddi_soft_state_fini(&ahci_statep);
520 		goto err_out;
521 	}
522 
523 	/* watchdog tick */
524 	ahci_watchdog_tick = drv_usectohz(
525 	    (clock_t)ahci_watchdog_timeout * 1000000);
526 
527 	ret = mod_install(&modlinkage);
528 	if (ret != 0) {
529 		sata_hba_fini(&modlinkage);
530 #if AHCI_DEBUG
531 		mutex_destroy(&ahci_log_mutex);
532 #endif
533 		ddi_soft_state_fini(&ahci_statep);
534 		goto err_out;
535 	}
536 
537 	return (ret);
538 
539 err_out:
540 	cmn_err(CE_WARN, "!ahci: Module init failed");
541 	return (ret);
542 }
543 
544 /*
545  * ahci module uninitialize.
546  */
547 int
_fini(void)548 _fini(void)
549 {
550 	int	ret;
551 
552 	ret = mod_remove(&modlinkage);
553 	if (ret != 0) {
554 		return (ret);
555 	}
556 
557 	/* Remove the resources allocated in _init(). */
558 	sata_hba_fini(&modlinkage);
559 #if AHCI_DEBUG
560 	mutex_destroy(&ahci_log_mutex);
561 #endif
562 	ddi_soft_state_fini(&ahci_statep);
563 
564 	return (ret);
565 }
566 
567 /*
568  * _info entry point
569  */
570 int
_info(struct modinfo * modinfop)571 _info(struct modinfo *modinfop)
572 {
573 	return (mod_info(&modlinkage, modinfop));
574 }
575 
576 /*
577  * The attach entry point for dev_ops.
578  */
579 static int
ahci_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)580 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
581 {
582 	ahci_ctl_t *ahci_ctlp = NULL;
583 	int instance = ddi_get_instance(dip);
584 	int status;
585 	int attach_state;
586 	uint32_t cap_status, ahci_version;
587 	uint32_t ghc_control;
588 	int intr_types;
589 	int i;
590 	pci_regspec_t *regs;
591 	int regs_length;
592 	int rnumber;
593 #if AHCI_DEBUG
594 	int speed;
595 #endif
596 
597 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
598 	    NULL);
599 
600 	switch (cmd) {
601 	case DDI_ATTACH:
602 		break;
603 
604 	case DDI_RESUME:
605 
606 		/*
607 		 * During DDI_RESUME, the hardware state of the device
608 		 * (power may have been removed from the device) must be
609 		 * restored, allow pending requests to continue, and
610 		 * service new requests.
611 		 */
612 		ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
613 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
614 
615 		/*
616 		 * GHC.AE must be set to 1 before any other AHCI register
617 		 * is accessed
618 		 */
619 		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
620 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
621 		ghc_control |= AHCI_HBA_GHC_AE;
622 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
623 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
624 
625 		/* Restart watch thread */
626 		if (ahci_ctlp->ahcictl_timeout_id == 0)
627 			ahci_ctlp->ahcictl_timeout_id = timeout(
628 			    (void (*)(void *))ahci_watchdog_handler,
629 			    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
630 
631 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
632 
633 		/*
634 		 * Re-initialize the controller and enable the interrupts and
635 		 * restart all the ports.
636 		 *
637 		 * Note that so far we don't support hot-plug during
638 		 * suspend/resume.
639 		 */
640 		if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
641 			AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
642 			    "Failed to initialize the controller "
643 			    "during DDI_RESUME", NULL);
644 			return (DDI_FAILURE);
645 		}
646 
647 		/*
648 		 * Reset the enclosure services.
649 		 */
650 		ahci_em_resume(ahci_ctlp);
651 
652 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
653 		ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
654 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
655 
656 		return (DDI_SUCCESS);
657 
658 	default:
659 		return (DDI_FAILURE);
660 	}
661 
662 	attach_state = AHCI_ATTACH_STATE_NONE;
663 
664 	/* Allocate soft state */
665 	status = ddi_soft_state_zalloc(ahci_statep, instance);
666 	if (status != DDI_SUCCESS) {
667 		cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
668 		    instance);
669 		goto err_out;
670 	}
671 
672 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
673 	ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
674 	ahci_ctlp->ahcictl_dip = dip;
675 
676 	/* Initialize the cport/port mapping */
677 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
678 		ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
679 		ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
680 	}
681 
682 	attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
683 
684 	/* Initialize FMA properties */
685 	ahci_fm_init(ahci_ctlp);
686 
687 	attach_state |= AHCI_ATTACH_STATE_FMA;
688 
689 	/*
690 	 * Now map the AHCI base address; which includes global
691 	 * registers and port control registers
692 	 *
693 	 * According to the spec, the AHCI Base Address is BAR5,
694 	 * but BAR0-BAR4 are optional, so we need to check which
695 	 * rnumber is used for BAR5.
696 	 */
697 
698 	/*
699 	 * search through DDI "reg" property for the AHCI register set
700 	 */
701 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
702 	    DDI_PROP_DONTPASS, "reg", (int **)&regs,
703 	    (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
704 		cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
705 		    instance);
706 		goto err_out;
707 	}
708 
709 	/* AHCI Base Address is located at 0x24 offset */
710 	for (rnumber = 0; rnumber < regs_length; ++rnumber) {
711 		if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
712 		    == AHCI_PCI_RNUM)
713 			break;
714 	}
715 
716 	ddi_prop_free(regs);
717 
718 	if (rnumber == regs_length) {
719 		cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
720 		    instance);
721 		goto err_out;
722 	}
723 
724 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
725 
726 	status = ddi_regs_map_setup(dip,
727 	    rnumber,
728 	    (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
729 	    0,
730 	    0,
731 	    &accattr,
732 	    &ahci_ctlp->ahcictl_ahci_acc_handle);
733 	if (status != DDI_SUCCESS) {
734 		cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
735 		    instance);
736 		goto err_out;
737 	}
738 
739 	attach_state |= AHCI_ATTACH_STATE_REG_MAP;
740 
741 	/*
742 	 * GHC.AE must be set to 1 before any other AHCI register
743 	 * is accessed
744 	 */
745 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
746 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
747 	ghc_control |= AHCI_HBA_GHC_AE;
748 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
749 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
750 
751 	/* Get the AHCI version information */
752 	ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
753 	    (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
754 
755 	cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
756 	    (ahci_version & 0xffff0000) >> 16,
757 	    ((ahci_version & 0x0000ff00) >> 4 |
758 	    (ahci_version & 0x000000ff)));
759 
760 	/* We don't support controllers whose versions are lower than 1.0 */
761 	if (!(ahci_version & 0xffff0000)) {
762 		cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
763 		    "than version 1.0", instance);
764 		goto err_out;
765 	}
766 
767 	/* Get the HBA capabilities information */
768 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
769 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
770 
771 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
772 	    cap_status);
773 
774 	/* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
775 	if (ahci_version >= 0x00010200) {
776 		uint32_t cap2_status;
777 
778 		/* Get the HBA capabilities extended information */
779 		cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
780 		    (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
781 
782 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
783 		    "hba capabilities extended = 0x%x", cap2_status);
784 
785 		if (cap2_status & AHCI_HBA_CAP2_SDS) {
786 			ahci_ctlp->ahcictl_cap |= AHCI_CAP_SDS;
787 		}
788 	}
789 
790 	if (cap_status & AHCI_HBA_CAP_EMS) {
791 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
792 		ahci_ctlp->ahcictl_em_loc =
793 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
794 		    (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
795 		ahci_ctlp->ahcictl_em_ctl =
796 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
797 		    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
798 	}
799 
800 #if AHCI_DEBUG
801 	/* Get the interface speed supported by the HBA */
802 	speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
803 	if (speed == 0x01) {
804 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
805 		    "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
806 	} else if (speed == 0x10) {
807 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
808 		    "hba interface speed support: Gen 2 (3 Gbps)", NULL);
809 	} else if (speed == 0x11) {
810 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
811 		    "hba interface speed support: Gen 3 (6 Gbps)", NULL);
812 	}
813 #endif
814 
815 	/* Get the number of command slots supported by the HBA */
816 	ahci_ctlp->ahcictl_num_cmd_slots =
817 	    ((cap_status & AHCI_HBA_CAP_NCS) >>
818 	    AHCI_HBA_CAP_NCS_SHIFT) + 1;
819 
820 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
821 	    ahci_ctlp->ahcictl_num_cmd_slots);
822 
823 	/* Get the bit map which indicates ports implemented by the HBA */
824 	ahci_ctlp->ahcictl_ports_implemented =
825 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
826 	    (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
827 
828 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
829 	    ahci_ctlp->ahcictl_ports_implemented);
830 
831 	/* Max port number implemented */
832 	ahci_ctlp->ahcictl_num_ports =
833 	    ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
834 
835 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
836 	    (cap_status & AHCI_HBA_CAP_NP) + 1);
837 
838 	/* Get the number of implemented ports by the HBA */
839 	ahci_ctlp->ahcictl_num_implemented_ports =
840 	    ahci_get_num_implemented_ports(
841 	    ahci_ctlp->ahcictl_ports_implemented);
842 
843 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
844 	    "hba number of implemented ports: %d",
845 	    ahci_ctlp->ahcictl_num_implemented_ports);
846 
847 	/* Check whether HBA supports 64bit DMA addressing */
848 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
849 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
850 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
851 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
852 		    "hba does not support 64-bit addressing", NULL);
853 	}
854 
855 	/* Checking for the support of Port Multiplier */
856 	if (cap_status & AHCI_HBA_CAP_SPM) {
857 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
858 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
859 		    "hba supports port multiplier (CBSS)", NULL);
860 
861 		/* Support FIS-based switching ? */
862 		if (cap_status & AHCI_HBA_CAP_FBSS) {
863 			ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
864 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
865 			    "hba supports FIS-based switching (FBSS)", NULL);
866 		}
867 	}
868 
869 	/* Checking for Support Command List Override */
870 	if (cap_status & AHCI_HBA_CAP_SCLO) {
871 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
872 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
873 		    "hba supports command list override.", NULL);
874 	}
875 
876 	/* Checking for Asynchronous Notification */
877 	if (cap_status & AHCI_HBA_CAP_SSNTF) {
878 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
879 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
880 		    "hba supports asynchronous notification.", NULL);
881 	}
882 
883 	if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
884 	    != DDI_SUCCESS) {
885 		cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
886 		    instance);
887 		goto err_out;
888 	}
889 
890 	attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
891 
892 	/*
893 	 * Check the pci configuration space, and set caps. We also
894 	 * handle the hardware defect in this function.
895 	 *
896 	 * For example, force ATI SB600 to use 32-bit dma addressing
897 	 * since it doesn't support 64-bit dma though its CAP register
898 	 * declares it support.
899 	 */
900 	if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
901 		cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
902 		    instance);
903 		goto err_out;
904 	}
905 
906 	/*
907 	 * Disable the whole controller interrupts before adding
908 	 * interrupt handlers(s).
909 	 */
910 	ahci_disable_all_intrs(ahci_ctlp);
911 
912 	/* Get supported interrupt types */
913 	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
914 		cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
915 		    instance);
916 		goto err_out;
917 	}
918 
919 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
920 	    "ddi_intr_get_supported_types() returned: 0x%x",
921 	    intr_types);
922 
923 	if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
924 		/*
925 		 * Try MSI first, but fall back to FIXED if failed
926 		 */
927 		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
928 		    DDI_SUCCESS) {
929 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
930 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
931 			    "Using MSI interrupt type", NULL);
932 			goto intr_done;
933 		}
934 
935 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
936 		    "MSI registration failed, "
937 		    "trying FIXED interrupts", NULL);
938 	}
939 
940 	if (intr_types & DDI_INTR_TYPE_FIXED) {
941 		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
942 		    DDI_SUCCESS) {
943 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
944 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
945 			    "Using FIXED interrupt type", NULL);
946 			goto intr_done;
947 		}
948 
949 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
950 		    "FIXED interrupt registration failed", NULL);
951 	}
952 
953 	cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
954 
955 	goto err_out;
956 
957 intr_done:
958 
959 	attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
960 
961 	/* Initialize the controller mutex */
962 	mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
963 	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
964 
965 	attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
966 
967 	if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
968 		ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
969 	} else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
970 		ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
971 	}
972 
973 	ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
974 	    (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
975 	    sizeof (ahci_prdt_item_t));
976 
977 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
978 	    "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
979 	    " ahci_cmd_table_size is 0x%x",
980 	    ahci_dma_prdt_number, ahci_cmd_table_size);
981 
982 	if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
983 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
984 		    ahci_dma_prdt_number;
985 
986 	ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
987 	ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
988 	ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
989 	ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
990 
991 	/*
992 	 * enable 64bit dma for data buffer for SB600 if
993 	 * sb600_buf_64bit_dma_disable is B_FALSE
994 	 */
995 	if ((ahci_buf_64bit_dma == B_FALSE) ||
996 	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
997 	    !(sb600_buf_64bit_dma_disable == B_FALSE &&
998 	    ahci_ctlp->ahcictl_venid == 0x1002 &&
999 	    ahci_ctlp->ahcictl_devid == 0x4380))) {
1000 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
1001 		    0xffffffffull;
1002 	}
1003 
1004 	/*
1005 	 * enable 64bit dma for command buffer for SB600/700/710/800
1006 	 * if sbxxx_commu_64bit_dma_disable is B_FALSE
1007 	 */
1008 	if ((ahci_commu_64bit_dma == B_FALSE) ||
1009 	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
1010 	    !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
1011 	    ahci_ctlp->ahcictl_venid == 0x1002 &&
1012 	    (ahci_ctlp->ahcictl_devid == 0x4380 ||
1013 	    ahci_ctlp->ahcictl_devid == 0x4391)))) {
1014 		ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
1015 		    0xffffffffull;
1016 		ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
1017 		    0xffffffffull;
1018 		ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
1019 		    0xffffffffull;
1020 	}
1021 
1022 	/* Allocate the ports structure */
1023 	status = ahci_alloc_ports_state(ahci_ctlp);
1024 	if (status != AHCI_SUCCESS) {
1025 		cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
1026 		    instance);
1027 		goto err_out;
1028 	}
1029 
1030 	attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
1031 
1032 	/*
1033 	 * Initialize the controller and ports.
1034 	 */
1035 	status = ahci_initialize_controller(ahci_ctlp);
1036 	if (status != AHCI_SUCCESS) {
1037 		cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
1038 		    instance);
1039 		goto err_out;
1040 	}
1041 
1042 	attach_state |= AHCI_ATTACH_STATE_HW_INIT;
1043 
1044 	/* Start one thread to check packet timeouts */
1045 	ahci_ctlp->ahcictl_timeout_id = timeout(
1046 	    (void (*)(void *))ahci_watchdog_handler,
1047 	    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
1048 
1049 	attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
1050 
1051 	if (!ahci_em_init(ahci_ctlp)) {
1052 		cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
1053 		    "services", instance);
1054 		goto err_out;
1055 	}
1056 	attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;
1057 
1058 	if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
1059 		cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
1060 		    instance);
1061 		goto err_out;
1062 	}
1063 
1064 	/* Check all handles at the end of the attach operation. */
1065 	if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
1066 		cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
1067 		    instance);
1068 		goto err_out;
1069 	}
1070 
1071 	ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
1072 
1073 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
1074 
1075 	return (DDI_SUCCESS);
1076 
1077 err_out:
1078 	/* FMA message */
1079 	ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
1080 	ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
1081 
1082 	if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
1083 		ahci_em_fini(ahci_ctlp);
1084 	}
1085 
1086 	if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
1087 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1088 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1089 		ahci_ctlp->ahcictl_timeout_id = 0;
1090 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1091 	}
1092 
1093 	if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1094 		ahci_uninitialize_controller(ahci_ctlp);
1095 	}
1096 
1097 	if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1098 		ahci_dealloc_ports_state(ahci_ctlp);
1099 	}
1100 
1101 	if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1102 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1103 	}
1104 
1105 	if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1106 		ahci_rem_intrs(ahci_ctlp);
1107 	}
1108 
1109 	if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1110 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1111 	}
1112 
1113 	if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1114 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1115 	}
1116 
1117 	if (attach_state & AHCI_ATTACH_STATE_FMA) {
1118 		ahci_fm_fini(ahci_ctlp);
1119 	}
1120 
1121 	if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1122 		ddi_soft_state_free(ahci_statep, instance);
1123 	}
1124 
1125 	return (DDI_FAILURE);
1126 }
1127 
1128 /*
1129  * The detach entry point for dev_ops.
1130  */
1131 static int
ahci_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)1132 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1133 {
1134 	ahci_ctl_t *ahci_ctlp;
1135 	int instance;
1136 	int ret;
1137 
1138 	instance = ddi_get_instance(dip);
1139 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1140 
1141 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1142 
1143 	switch (cmd) {
1144 	case DDI_DETACH:
1145 
1146 		/* disable the interrupts for an uninterrupted detach */
1147 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1148 		ahci_disable_all_intrs(ahci_ctlp);
1149 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1150 
1151 		/* unregister from the sata framework. */
1152 		ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1153 		if (ret != AHCI_SUCCESS) {
1154 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
1155 			ahci_enable_all_intrs(ahci_ctlp);
1156 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
1157 			return (DDI_FAILURE);
1158 		}
1159 
1160 		ahci_em_fini(ahci_ctlp);
1161 
1162 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1163 
1164 		/* stop the watchdog handler */
1165 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1166 		ahci_ctlp->ahcictl_timeout_id = 0;
1167 
1168 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1169 
1170 		/* uninitialize the controller */
1171 		ahci_uninitialize_controller(ahci_ctlp);
1172 
1173 		/* remove the interrupts */
1174 		ahci_rem_intrs(ahci_ctlp);
1175 
1176 		/* deallocate the ports structures */
1177 		ahci_dealloc_ports_state(ahci_ctlp);
1178 
1179 		/* destroy mutex */
1180 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1181 
1182 		/* teardown the pci config */
1183 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1184 
1185 		/* remove the reg maps. */
1186 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1187 
1188 		/* release fma resource */
1189 		ahci_fm_fini(ahci_ctlp);
1190 
1191 		/* free the soft state. */
1192 		ddi_soft_state_free(ahci_statep, instance);
1193 
1194 		return (DDI_SUCCESS);
1195 
1196 	case DDI_SUSPEND:
1197 
1198 		/*
1199 		 * The steps associated with suspension must include putting
1200 		 * the underlying device into a quiescent state so that it
1201 		 * will not generate interrupts or modify or access memory.
1202 		 */
1203 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1204 		if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1205 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
1206 			return (DDI_SUCCESS);
1207 		}
1208 
1209 		ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1210 
1211 		/* stop the watchdog handler */
1212 		if (ahci_ctlp->ahcictl_timeout_id) {
1213 			(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1214 			ahci_ctlp->ahcictl_timeout_id = 0;
1215 		}
1216 
1217 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1218 
1219 		ahci_em_suspend(ahci_ctlp);
1220 
1221 		/*
1222 		 * drain the taskq
1223 		 */
1224 		ahci_drain_ports_taskq(ahci_ctlp);
1225 
1226 		/*
1227 		 * Disable the interrupts and stop all the ports.
1228 		 */
1229 		ahci_uninitialize_controller(ahci_ctlp);
1230 
1231 		return (DDI_SUCCESS);
1232 
1233 	default:
1234 		return (DDI_FAILURE);
1235 	}
1236 }
1237 
1238 /*
1239  * The info entry point for dev_ops.
1240  *
1241  */
1242 static int
ahci_getinfo(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)1243 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1244     void *arg, void **result)
1245 {
1246 #ifndef __lock_lint
1247 	_NOTE(ARGUNUSED(dip))
1248 #endif /* __lock_lint */
1249 
1250 	ahci_ctl_t *ahci_ctlp;
1251 	int instance;
1252 	dev_t dev;
1253 
1254 	dev = (dev_t)arg;
1255 	instance = getminor(dev);
1256 
1257 	switch (infocmd) {
1258 		case DDI_INFO_DEVT2DEVINFO:
1259 			ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
1260 			if (ahci_ctlp != NULL) {
1261 				*result = ahci_ctlp->ahcictl_dip;
1262 				return (DDI_SUCCESS);
1263 			} else {
1264 				*result = NULL;
1265 				return (DDI_FAILURE);
1266 			}
1267 		case DDI_INFO_DEVT2INSTANCE:
1268 			*(int *)result = instance;
1269 			break;
1270 		default:
1271 			break;
1272 	}
1273 
1274 	return (DDI_SUCCESS);
1275 }
1276 
1277 /*
1278  * Registers the ahci with sata framework.
1279  */
1280 static int
ahci_register_sata_hba_tran(ahci_ctl_t * ahci_ctlp,uint32_t cap_status)1281 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1282 {
1283 	struct	sata_hba_tran	*sata_hba_tran;
1284 
1285 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1286 	    "ahci_register_sata_hba_tran enter", NULL);
1287 
1288 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
1289 
1290 	/* Allocate memory for the sata_hba_tran  */
1291 	sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1292 
1293 	sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1294 	sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1295 	sata_hba_tran->sata_tran_hba_dma_attr =
1296 	    &ahci_ctlp->ahcictl_buffer_dma_attr;
1297 
1298 	/* Report the number of implemented ports */
1299 	sata_hba_tran->sata_tran_hba_num_cports =
1300 	    ahci_ctlp->ahcictl_num_implemented_ports;
1301 
1302 	/* Support ATAPI device */
1303 	sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1304 
1305 	/* Get the data transfer capability for PIO command by the HBA */
1306 	if (cap_status & AHCI_HBA_CAP_PMD) {
1307 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1308 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1309 		    "DRQ block data transfer for PIO command protocol", NULL);
1310 	}
1311 
1312 	/*
1313 	 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1314 	 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1315 	 * QUEUED (EXT), and SERVICE commands). Queued operations are
1316 	 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1317 	 * QUEUED commands when the HBA and device support native command
1318 	 * queuing(NCQ).
1319 	 *
1320 	 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1321 	 * CAP register of the HBA indicates NCQ is supported.
1322 	 *
1323 	 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1324 	 * set because the previous register content of PxCI can be re-written
1325 	 * in the register write.
1326 	 */
1327 	if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1328 	    !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1329 		sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1330 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1331 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1332 		    "Command Queuing", NULL);
1333 	}
1334 
1335 	/* Support port multiplier? */
1336 	if (cap_status & AHCI_HBA_CAP_SPM) {
1337 		sata_hba_tran->sata_tran_hba_features_support |=
1338 		    SATA_CTLF_PORT_MULTIPLIER;
1339 
1340 		/* Support FIS-based switching for port multiplier? */
1341 		if (cap_status & AHCI_HBA_CAP_FBSS) {
1342 			sata_hba_tran->sata_tran_hba_features_support |=
1343 			    SATA_CTLF_PMULT_FBS;
1344 		}
1345 	}
1346 
1347 	/* Report the number of command slots */
1348 	sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1349 
1350 	sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1351 	sata_hba_tran->sata_tran_start = ahci_tran_start;
1352 	sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1353 	sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1354 	sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1355 #ifdef __lock_lint
1356 	sata_hba_tran->sata_tran_selftest = ahci_selftest;
1357 #endif
1358 	/*
1359 	 * When SATA framework adds support for pwrmgt the
1360 	 * pwrmgt_ops needs to be updated
1361 	 */
1362 	sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1363 	sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;
1364 
1365 	ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1366 
1367 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
1368 
1369 	/* Attach it to SATA framework */
1370 	if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1371 	    != DDI_SUCCESS) {
1372 		kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1373 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1374 		ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1375 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1376 		return (AHCI_FAILURE);
1377 	}
1378 
1379 	return (AHCI_SUCCESS);
1380 }
1381 
1382 /*
1383  * Unregisters the ahci with sata framework.
1384  */
1385 static int
ahci_unregister_sata_hba_tran(ahci_ctl_t * ahci_ctlp)1386 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1387 {
1388 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1389 	    "ahci_unregister_sata_hba_tran enter", NULL);
1390 
1391 	/* Detach from the SATA framework. */
1392 	if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1393 	    DDI_SUCCESS) {
1394 		return (AHCI_FAILURE);
1395 	}
1396 
1397 	/* Deallocate sata_hba_tran. */
1398 	kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1399 	    sizeof (sata_hba_tran_t));
1400 
1401 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
1402 	ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1403 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
1404 
1405 	return (AHCI_SUCCESS);
1406 }
1407 
1408 #define	SET_PORTSTR(str, addrp)						\
1409 	if (AHCI_ADDR_IS_PORT(addrp))					\
1410 		(void) sprintf((str), "%d", (addrp)->aa_port);		\
1411 	else if (AHCI_ADDR_IS_PMULT(addrp))				\
1412 		(void) sprintf((str), "%d (pmult)", (addrp)->aa_port);	\
1413 	else								\
1414 		(void) sprintf((str), "%d:%d", (addrp)->aa_port,	\
1415 		    (addrp)->aa_pmport);
1416 
1417 /*
1418  * ahci_tran_probe_port is called by SATA framework. It returns port state,
1419  * port status registers and an attached device type via sata_device
1420  * structure.
1421  *
1422  * We return the cached information from a previous hardware probe. The
1423  * actual hardware probing itself was done either from within
1424  * ahci_initialize_controller() during the driver attach or from a phy
1425  * ready change interrupt handler.
1426  */
1427 static int
ahci_tran_probe_port(dev_info_t * dip,sata_device_t * sd)1428 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1429 {
1430 	ahci_ctl_t *ahci_ctlp;
1431 	ahci_port_t *ahci_portp;
1432 	ahci_addr_t addr, pmult_addr;
1433 	uint8_t cport = sd->satadev_addr.cport;
1434 	char portstr[10];
1435 	uint8_t device_type;
1436 	uint32_t port_state;
1437 	uint8_t port;
1438 	int rval = SATA_SUCCESS, rval_init;
1439 
1440 	port_state = 0;
1441 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1442 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1443 
1444 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1445 
1446 	mutex_enter(&ahci_portp->ahciport_mutex);
1447 
1448 	ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1449 	ASSERT(AHCI_ADDR_IS_VALID(&addr));
1450 	SET_PORTSTR(portstr, &addr);
1451 
1452 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1453 	    "ahci_tran_probe_port enter: port %s", portstr);
1454 
1455 	if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1456 	    (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1457 	    ahci_portp->ahciport_pmult_info == NULL)) {
1458 		/* port mutliplier is removed. */
1459 		AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1460 		    "ahci_tran_probe_port: "
1461 		    "pmult is removed from port %s", portstr);
1462 		mutex_exit(&ahci_portp->ahciport_mutex);
1463 		return (SATA_FAILURE);
1464 	}
1465 
1466 	/*
1467 	 * The sata_device may refer to
1468 	 * 1. A controller port.
1469 	 *    A controller port should be ready here.
1470 	 * 2. A port multiplier.
1471 	 *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1472 	 *    it and register the port multiplier to the framework.
1473 	 *    SATA_ADDR_PMULT - check the status of all its device ports.
1474 	 * 3. A port multiplier port.
1475 	 *    If it has not been initialized, initialized it.
1476 	 *
1477 	 * A port multiplier or a port multiplier port may require some
1478 	 * initialization because we cannot do these time-consuming jobs in an
1479 	 * interrupt context.
1480 	 */
1481 	if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1482 		AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1483 		/* Initialize registers on a port multiplier */
1484 		rval_init = ahci_initialize_pmult(ahci_ctlp,
1485 		    ahci_portp, &pmult_addr, sd);
1486 		if (rval_init != AHCI_SUCCESS) {
1487 			AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1488 			    "ahci_tran_probe_port: "
1489 			    "pmult initialization failed.", NULL);
1490 			mutex_exit(&ahci_portp->ahciport_mutex);
1491 			return (SATA_FAILURE);
1492 		}
1493 	} else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1494 		/* Check pmports hotplug events */
1495 		(void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1496 	} else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1497 	    SATA_ADDR_DPMPORT)) {
1498 		if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1499 		    &addr, sd) != AHCI_SUCCESS) {
1500 			rval = SATA_FAILURE;
1501 			goto out;
1502 		}
1503 	}
1504 
1505 	/* Update port state and device type */
1506 	port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1507 
1508 	switch (port_state) {
1509 
1510 	case SATA_PSTATE_FAILED:
1511 		sd->satadev_state = SATA_PSTATE_FAILED;
1512 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1513 		    "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1514 		goto out;
1515 
1516 	case SATA_PSTATE_SHUTDOWN:
1517 		sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1518 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1519 		    "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1520 		goto out;
1521 
1522 	case SATA_PSTATE_PWROFF:
1523 		sd->satadev_state = SATA_PSTATE_PWROFF;
1524 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1525 		    "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1526 		goto out;
1527 
1528 	case SATA_PSTATE_PWRON:
1529 		sd->satadev_state = SATA_PSTATE_PWRON;
1530 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1531 		    "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1532 		break;
1533 
1534 	default:
1535 		sd->satadev_state = port_state;
1536 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1537 		    "ahci_tran_probe_port: port %s PORT NORMAL %x",
1538 		    portstr, port_state);
1539 		break;
1540 	}
1541 
1542 	device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1543 
1544 	switch (device_type) {
1545 
1546 	case SATA_DTYPE_ATADISK:
1547 		sd->satadev_type = SATA_DTYPE_ATADISK;
1548 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1549 		    "ahci_tran_probe_port: port %s DISK found", portstr);
1550 		break;
1551 
1552 	case SATA_DTYPE_ATAPI:
1553 		/*
1554 		 * HBA driver only knows it's an ATAPI device, and don't know
1555 		 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1556 		 * type need to be determined by checking IDENTIFY PACKET
1557 		 * DEVICE data
1558 		 */
1559 		sd->satadev_type = SATA_DTYPE_ATAPI;
1560 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1561 		    "ahci_tran_probe_port: port %s ATAPI found", portstr);
1562 		break;
1563 
1564 	case SATA_DTYPE_PMULT:
1565 		ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1566 		sd->satadev_type = SATA_DTYPE_PMULT;
1567 
1568 		/* Update the number of pmports. */
1569 		ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1570 		sd->satadev_add_info = ahci_portp->
1571 		    ahciport_pmult_info->ahcipmi_num_dev_ports;
1572 
1573 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1574 		    "ahci_tran_probe_port: port %s Port Multiplier found",
1575 		    portstr);
1576 		break;
1577 
1578 	case SATA_DTYPE_UNKNOWN:
1579 		sd->satadev_type = SATA_DTYPE_UNKNOWN;
1580 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1581 		    "ahci_tran_probe_port: port %s Unknown device found",
1582 		    portstr);
1583 		break;
1584 
1585 	default:
1586 		/* we don't support any other device types */
1587 		sd->satadev_type = SATA_DTYPE_NONE;
1588 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1589 		    "ahci_tran_probe_port: port %s No device found", portstr);
1590 		break;
1591 	}
1592 
1593 out:
1594 	/* Register update only fails while probing a pmult/pmport */
1595 	if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1596 		ahci_update_sata_registers(ahci_ctlp, port, sd);
1597 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1598 		if (port_state & SATA_STATE_READY)
1599 			if (ahci_update_pmult_pscr(ahci_ctlp,
1600 			    &addr, sd) != AHCI_SUCCESS)
1601 				rval = SATA_FAILURE;
1602 	}
1603 
1604 	/* Check handles for the sata registers access */
1605 	if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1606 	    (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1607 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1608 		    DDI_SERVICE_UNAFFECTED);
1609 		rval = SATA_FAILURE;
1610 	}
1611 
1612 	mutex_exit(&ahci_portp->ahciport_mutex);
1613 	return (rval);
1614 }
1615 
1616 /*
1617  * There are four operation modes in sata framework:
1618  * SATA_OPMODE_INTERRUPTS
1619  * SATA_OPMODE_POLLING
1620  * SATA_OPMODE_ASYNCH
1621  * SATA_OPMODE_SYNCH
1622  *
1623  * Their combined meanings as following:
1624  *
1625  * SATA_OPMODE_SYNCH
1626  * The command has to be completed before sata_tran_start functions returns.
1627  * Either interrupts or polling could be used - it's up to the driver.
1628  * Mode used currently for internal, sata-module initiated operations.
1629  *
1630  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1631  * It is the same as the one above.
1632  *
1633  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1634  * The command has to be completed before sata_tran_start function returns.
1635  * No interrupt used, polling only. This should be the mode used for scsi
1636  * packets with FLAG_NOINTR.
1637  *
1638  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1639  * The command may be queued (callback function specified). Interrupts could
1640  * be used. It's normal operation mode.
1641  */
1642 /*
1643  * Called by sata framework to transport a sata packet down stream.
1644  */
1645 static int
ahci_tran_start(dev_info_t * dip,sata_pkt_t * spkt)1646 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1647 {
1648 	ahci_ctl_t *ahci_ctlp;
1649 	ahci_port_t *ahci_portp;
1650 	ahci_addr_t addr;
1651 	uint8_t	cport = spkt->satapkt_device.satadev_addr.cport;
1652 	uint8_t port;
1653 	char portstr[10];
1654 
1655 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1656 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1657 
1658 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1659 	    "ahci_tran_start enter: cport %d satapkt 0x%p",
1660 	    cport, (void *)spkt);
1661 
1662 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1663 
1664 	mutex_enter(&ahci_portp->ahciport_mutex);
1665 	ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1666 	SET_PORTSTR(portstr, &addr);
1667 
1668 	/* Sanity check */
1669 	if (AHCI_ADDR_IS_PMPORT(&addr)) {
1670 		if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1671 		    ahci_portp->ahciport_pmult_info == NULL) {
1672 
1673 			spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1674 			spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1675 			spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1676 			ahci_update_sata_registers(ahci_ctlp, port,
1677 			    &spkt->satapkt_device);
1678 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1679 			    "ahci_tran_start returning PORT_ERROR while "
1680 			    "pmult removed: port: %s", portstr);
1681 			mutex_exit(&ahci_portp->ahciport_mutex);
1682 			return (SATA_TRAN_PORT_ERROR);
1683 		}
1684 
1685 		if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1686 		    SATA_STATE_READY)) {
1687 			if (!ddi_in_panic() ||
1688 			    ahci_initialize_pmport(ahci_ctlp,
1689 			    ahci_portp, &addr) != AHCI_SUCCESS) {
1690 				spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1691 				spkt->satapkt_device.satadev_type =
1692 				    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1693 				spkt->satapkt_device.satadev_state =
1694 				    AHCIPORT_GET_STATE(ahci_portp, &addr);
1695 				ahci_update_sata_registers(ahci_ctlp, port,
1696 				    &spkt->satapkt_device);
1697 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1698 				    "ahci_tran_start returning PORT_ERROR "
1699 				    "while sub-link is not initialized "
1700 				    "at port: %s", portstr);
1701 				mutex_exit(&ahci_portp->ahciport_mutex);
1702 				return (SATA_TRAN_PORT_ERROR);
1703 			}
1704 		}
1705 	}
1706 
1707 	if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1708 	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1709 	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1710 		/*
1711 		 * In case the target driver would send the packet before
1712 		 * sata framework can have the opportunity to process those
1713 		 * event reports.
1714 		 */
1715 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1716 		spkt->satapkt_device.satadev_state =
1717 		    ahci_portp->ahciport_port_state;
1718 		ahci_update_sata_registers(ahci_ctlp, port,
1719 		    &spkt->satapkt_device);
1720 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1721 		    "ahci_tran_start returning PORT_ERROR while "
1722 		    "port in FAILED/SHUTDOWN/PWROFF state: "
1723 		    "port: %s", portstr);
1724 		mutex_exit(&ahci_portp->ahciport_mutex);
1725 		return (SATA_TRAN_PORT_ERROR);
1726 	}
1727 
1728 	if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1729 		/*
1730 		 * ahci_intr_phyrdy_change() may have rendered it to
1731 		 * SATA_DTYPE_NONE.
1732 		 */
1733 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1734 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1735 		spkt->satapkt_device.satadev_state =
1736 		    ahci_portp->ahciport_port_state;
1737 		ahci_update_sata_registers(ahci_ctlp, port,
1738 		    &spkt->satapkt_device);
1739 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1740 		    "ahci_tran_start returning PORT_ERROR while "
1741 		    "no device attached: port: %s", portstr);
1742 		mutex_exit(&ahci_portp->ahciport_mutex);
1743 		return (SATA_TRAN_PORT_ERROR);
1744 	}
1745 
1746 	/* R/W PMULT command will occupy the whole HBA port */
1747 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1748 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1749 		    "ahci_tran_start returning BUSY while "
1750 		    "executing READ/WRITE PORT-MULT command: "
1751 		    "port: %s", portstr);
1752 		spkt->satapkt_reason = SATA_PKT_BUSY;
1753 		mutex_exit(&ahci_portp->ahciport_mutex);
1754 		return (SATA_TRAN_BUSY);
1755 	}
1756 
1757 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1758 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1759 		    "ahci_tran_start returning BUSY while "
1760 		    "hot-plug in progress: port: %s", portstr);
1761 		spkt->satapkt_reason = SATA_PKT_BUSY;
1762 		mutex_exit(&ahci_portp->ahciport_mutex);
1763 		return (SATA_TRAN_BUSY);
1764 	}
1765 
1766 	/*
1767 	 * SATA HBA driver should remember that a device was reset and it
1768 	 * is supposed to reject any packets which do not specify either
1769 	 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1770 	 *
1771 	 * This is to prevent a race condition when a device was arbitrarily
1772 	 * reset by the HBA driver (and lost it's setting) and a target
1773 	 * driver sending some commands to a device before the sata framework
1774 	 * has a chance to restore the device setting (such as cache enable/
1775 	 * disable or other resettable stuff).
1776 	 */
1777 	/*
1778 	 * It is unnecessary to use specific flags to indicate
1779 	 * reset_in_progress for a pmport. While mopping, all command will be
1780 	 * mopped so that the entire HBA port is being dealt as a single
1781 	 * object.
1782 	 */
1783 	if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1784 		ahci_portp->ahciport_reset_in_progress = 0;
1785 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1786 		    "ahci_tran_start [CLEAR] the "
1787 		    "reset_in_progress for port: %d", port);
1788 	}
1789 
1790 	if (ahci_portp->ahciport_reset_in_progress &&
1791 	    ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1792 	    ! ddi_in_panic()) {
1793 		spkt->satapkt_reason = SATA_PKT_BUSY;
1794 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1795 		    "ahci_tran_start returning BUSY while "
1796 		    "reset in progress: port: %d", port);
1797 		mutex_exit(&ahci_portp->ahciport_mutex);
1798 		return (SATA_TRAN_BUSY);
1799 	}
1800 
1801 #ifdef AHCI_DEBUG
1802 	if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1803 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1804 		    "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1805 		    spkt, port);
1806 	}
1807 #endif
1808 
1809 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1810 		spkt->satapkt_reason = SATA_PKT_BUSY;
1811 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1812 		    "ahci_tran_start returning BUSY while "
1813 		    "mopping in progress: port: %d", port);
1814 		mutex_exit(&ahci_portp->ahciport_mutex);
1815 		return (SATA_TRAN_BUSY);
1816 	}
1817 
1818 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1819 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1820 		    DDI_SERVICE_UNAFFECTED);
1821 		mutex_exit(&ahci_portp->ahciport_mutex);
1822 		return (SATA_TRAN_BUSY);
1823 	}
1824 
1825 	if (spkt->satapkt_op_mode &
1826 	    (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1827 		/*
1828 		 * If a SYNC command to be executed in interrupt context,
1829 		 * bounce it back to sata module.
1830 		 */
1831 		if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1832 		    servicing_interrupt()) {
1833 			spkt->satapkt_reason = SATA_PKT_BUSY;
1834 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1835 			    "ahci_tran_start returning BUSY while "
1836 			    "sending SYNC mode under interrupt context: "
1837 			    "port : %d", port);
1838 			mutex_exit(&ahci_portp->ahciport_mutex);
1839 			return (SATA_TRAN_BUSY);
1840 		}
1841 
1842 		/* We need to do the sync start now */
1843 		if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1844 		    spkt) == AHCI_FAILURE) {
1845 			goto fail_out;
1846 		}
1847 	} else {
1848 		/* Async start, using interrupt */
1849 		if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1850 		    == AHCI_FAILURE) {
1851 			spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1852 			goto fail_out;
1853 		}
1854 	}
1855 
1856 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1857 	    "sata tran accepted: port %s", portstr);
1858 
1859 	mutex_exit(&ahci_portp->ahciport_mutex);
1860 	return (SATA_TRAN_ACCEPTED);
1861 
1862 fail_out:
1863 	/*
1864 	 * Failed to deliver packet to the controller.
1865 	 * Check if it's caused by invalid handles.
1866 	 */
1867 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1868 	    ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1869 		spkt->satapkt_device.satadev_type =
1870 		    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1871 		spkt->satapkt_device.satadev_state =
1872 		    AHCIPORT_GET_STATE(ahci_portp, &addr);
1873 		spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1874 		mutex_exit(&ahci_portp->ahciport_mutex);
1875 		return (SATA_TRAN_PORT_ERROR);
1876 	}
1877 
1878 	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1879 	    "return QUEUE_FULL: port %d", port);
1880 	mutex_exit(&ahci_portp->ahciport_mutex);
1881 	return (SATA_TRAN_QUEUE_FULL);
1882 }
1883 
1884 /*
1885  * SATA_OPMODE_SYNCH flag is set
1886  *
1887  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1888  * without interrupt, otherwise we can still use the interrupt.
1889  */
1890 static int
ahci_do_sync_start(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_pkt_t * spkt)1891 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1892     ahci_addr_t *addrp, sata_pkt_t *spkt)
1893 {
1894 	int pkt_timeout_ticks;
1895 	uint32_t timeout_tags;
1896 	int rval;
1897 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1898 	uint8_t port = addrp->aa_port;
1899 
1900 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1901 
1902 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1903 	    "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1904 
1905 	if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1906 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1907 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1908 		    addrp, spkt)) == AHCI_FAILURE) {
1909 			ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1910 			return (rval);
1911 		}
1912 
1913 		pkt_timeout_ticks =
1914 		    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1915 
1916 		while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1917 			/* Simulate the interrupt */
1918 			mutex_exit(&ahci_portp->ahciport_mutex);
1919 			ahci_port_intr(ahci_ctlp, ahci_portp, port);
1920 			mutex_enter(&ahci_portp->ahciport_mutex);
1921 
1922 			if (spkt->satapkt_reason != SATA_PKT_BUSY)
1923 				break;
1924 
1925 			mutex_exit(&ahci_portp->ahciport_mutex);
1926 			drv_usecwait(AHCI_1MS_USECS);
1927 			mutex_enter(&ahci_portp->ahciport_mutex);
1928 
1929 			pkt_timeout_ticks -= AHCI_1MS_TICKS;
1930 			if (pkt_timeout_ticks < 0) {
1931 				cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1932 				    "port %d satapkt 0x%p timed out\n",
1933 				    instance, port, (void *)spkt);
1934 				timeout_tags = (0x1 << rval);
1935 				mutex_exit(&ahci_portp->ahciport_mutex);
1936 				ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1937 				    port, timeout_tags);
1938 				mutex_enter(&ahci_portp->ahciport_mutex);
1939 			}
1940 		}
1941 
1942 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1943 		return (AHCI_SUCCESS);
1944 
1945 	} else {
1946 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1947 		    addrp, spkt)) == AHCI_FAILURE)
1948 			return (rval);
1949 
1950 #if AHCI_DEBUG
1951 		/*
1952 		 * Note that the driver always uses the slot 0 to deliver
1953 		 * REQUEST SENSE or READ LOG EXT command
1954 		 */
1955 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1956 			ASSERT(rval == 0);
1957 #endif
1958 
1959 		while (spkt->satapkt_reason == SATA_PKT_BUSY)
1960 			cv_wait(&ahci_portp->ahciport_cv,
1961 			    &ahci_portp->ahciport_mutex);
1962 
1963 		return (AHCI_SUCCESS);
1964 	}
1965 }
1966 
1967 /*
1968  * Searches for and claims a free command slot.
1969  *
1970  * Returns value:
1971  *
1972  * AHCI_FAILURE returned only if
1973  *	1. No empty slot left
1974  *	2. Non-queued command requested while queued command(s) is outstanding
1975  *	3. Queued command requested while non-queued command(s) is outstanding
1976  *	4. HBA doesn't support multiple-use of command list while already a
1977  *	   non-queued command is oustanding
1978  *	5. Queued command requested while some queued command(s) has been
1979  *	   outstanding on a different port multiplier port. (AHCI spec 1.2,
1980  *	   9.1.2)
1981  *
1982  * claimed slot number returned if succeeded
1983  *
1984  * NOTE: it will always return slot 0 for following commands to simplify the
1985  * algorithm.
1986  *	1. REQUEST SENSE or READ LOG EXT command during error recovery process
1987  *	2. READ/WRITE PORTMULT command
1988  */
1989 static int
ahci_claim_free_slot(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,int command_type)1990 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1991     ahci_addr_t *addrp, int command_type)
1992 {
1993 	uint32_t port_cmd_issue;
1994 	uint32_t free_slots;
1995 	int slot;
1996 
1997 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1998 
1999 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
2000 	    "ahciport_pending_tags = 0x%x "
2001 	    "ahciport_pending_ncq_tags = 0x%x",
2002 	    ahci_portp->ahciport_pending_tags,
2003 	    ahci_portp->ahciport_pending_ncq_tags);
2004 
2005 	free_slots = 0;
2006 	/*
2007 	 * According to the AHCI spec, system software is responsible to
2008 	 * ensure that queued and non-queued commands are not mixed in
2009 	 * the command list.
2010 	 */
2011 	if (command_type == AHCI_NON_NCQ_CMD) {
2012 		/* Non-NCQ command request */
2013 		if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2014 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2015 			    "ahci_claim_free_slot: there is still pending "
2016 			    "queued command(s) in the command list, "
2017 			    "so no available slot for the non-queued "
2018 			    "command", NULL);
2019 			return (AHCI_FAILURE);
2020 		}
2021 		if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
2022 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2023 			    "ahci_claim_free_slot: there is still pending "
2024 			    "read/write port-mult command(s) in command list, "
2025 			    "so no available slot for the non-queued command",
2026 			    NULL);
2027 			return (AHCI_FAILURE);
2028 		}
2029 		if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
2030 		    NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2031 			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2032 			    "ahci_claim_free_slot: HBA cannot support multiple-"
2033 			    "use of the command list for non-queued commands",
2034 			    NULL);
2035 			return (AHCI_FAILURE);
2036 		}
2037 		free_slots = (~ahci_portp->ahciport_pending_tags) &
2038 		    AHCI_SLOT_MASK(ahci_ctlp);
2039 	} else if (command_type == AHCI_NCQ_CMD) {
2040 		/* NCQ command request */
2041 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2042 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2043 			    "ahci_claim_free_slot: there is still pending "
2044 			    "non-queued command(s) in the command list, "
2045 			    "so no available slot for the queued command",
2046 			    NULL);
2047 			return (AHCI_FAILURE);
2048 		}
2049 
2050 		/*
2051 		 * NCQ commands cannot be sent to different port multiplier
2052 		 * ports in Command-Based Switching mode
2053 		 */
2054 		/*
2055 		 * NOTE: In Command-Based Switching mode, AHCI controller
2056 		 * usually reports a 'Handshake Error' when multiple NCQ
2057 		 * commands are outstanding simultaneously.
2058 		 */
2059 		if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
2060 			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2061 			if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
2062 			    NCQ_CMD_IN_PROGRESS(ahci_portp) &&
2063 			    AHCIPORT_NCQ_PMPORT(ahci_portp) !=
2064 			    addrp->aa_pmport) {
2065 				AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2066 				    "ahci_claim_free_slot: there is still "
2067 				    "pending queued command(s) in the "
2068 				    "command list for another Port Multiplier "
2069 				    "port, so no available slot.", NULL);
2070 				return (AHCI_FAILURE);
2071 			}
2072 		}
2073 
2074 		free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
2075 		    AHCI_NCQ_SLOT_MASK(ahci_portp);
2076 	} else if (command_type == AHCI_ERR_RETRI_CMD) {
2077 		/* Error retrieval command request */
2078 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2079 		    "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
2080 		    "SENSE or READ LOG EXT command", NULL);
2081 		slot = 0;
2082 		goto out;
2083 	} else if (command_type == AHCI_RDWR_PMULT_CMD) {
2084 		/*
2085 		 * An extra check on PxCI. Sometimes PxCI bits may not be
2086 		 * cleared during hot-plug or error recovery process.
2087 		 */
2088 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2089 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
2090 
2091 		if (port_cmd_issue != 0) {
2092 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2093 			    "ahci_claim_free_slot: there is still pending "
2094 			    "command(s) in command list (0x%x/0x%x, PxCI %x),"
2095 			    "so no available slot for R/W PMULT command.",
2096 			    NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
2097 			    NCQ_CMD_IN_PROGRESS(ahci_portp),
2098 			    port_cmd_issue);
2099 			return (AHCI_FAILURE);
2100 		}
2101 
2102 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2103 		    "ahci_claim_free_slot: slot 0 is allocated for "
2104 		    "READ/WRITE PORTMULT command", NULL);
2105 		slot = 0;
2106 		goto out;
2107 	}
2108 
2109 	slot = ddi_ffs(free_slots) - 1;
2110 	if (slot == -1) {
2111 		AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2112 		    "ahci_claim_free_slot: no empty slots", NULL);
2113 		return (AHCI_FAILURE);
2114 	}
2115 
2116 	/*
2117 	 * According to the AHCI spec, to allow a simple mechanism for the
2118 	 * HBA to map command list slots to queue entries, software must
2119 	 * match the tag number it uses to the slot it is placing the command
2120 	 * in. For example, if a queued command is placed in slot 5, the tag
2121 	 * for that command must be 5.
2122 	 */
2123 	if (command_type == AHCI_NCQ_CMD) {
2124 		ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2125 		if (AHCI_ADDR_IS_PMPORT(addrp)) {
2126 			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2127 			AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2128 		}
2129 	}
2130 
2131 	ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2132 
2133 out:
2134 	AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2135 	    "ahci_claim_free_slot: found slot: 0x%x", slot);
2136 
2137 	return (slot);
2138 }
2139 
2140 /*
2141  * Builds the Command Table for the sata packet and delivers it to controller.
2142  *
2143  * Returns:
2144  *	slot number if we can obtain a slot successfully
2145  *	otherwise, return AHCI_FAILURE
2146  */
2147 static int
ahci_deliver_satapkt(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_pkt_t * spkt)2148 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2149     ahci_addr_t *addrp, sata_pkt_t *spkt)
2150 {
2151 	int cmd_slot;
2152 	sata_cmd_t *scmd;
2153 	ahci_fis_h2d_register_t *h2d_register_fisp;
2154 	ahci_cmd_table_t *cmd_table;
2155 	ahci_cmd_header_t *cmd_header;
2156 	int ncookies;
2157 	int i;
2158 	int command_type = AHCI_NON_NCQ_CMD;
2159 	int ncq_qdepth;
2160 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2161 	uint8_t port, pmport;
2162 #if AHCI_DEBUG
2163 	uint32_t *ptr;
2164 	uint8_t *ptr2;
2165 #endif
2166 
2167 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2168 
2169 	port = addrp->aa_port;
2170 	pmport = addrp->aa_pmport;
2171 
2172 	spkt->satapkt_reason = SATA_PKT_BUSY;
2173 
2174 	scmd = &spkt->satapkt_cmd;
2175 
2176 	/* Check if the command is a NCQ command */
2177 	if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2178 	    scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2179 		command_type = AHCI_NCQ_CMD;
2180 
2181 		/*
2182 		 * When NCQ is support, system software must determine the
2183 		 * maximum tag allowed by the device and the HBA, and it
2184 		 * must use a value not beyond of the lower bound of the two.
2185 		 *
2186 		 * Sata module is going to calculate the qdepth and send
2187 		 * down to HBA driver via sata_cmd.
2188 		 */
2189 		ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2190 
2191 		/*
2192 		 * At the moment, the driver doesn't support the dynamic
2193 		 * setting of the maximum ncq depth, and the value can be
2194 		 * set either during the attach or after hot-plug insertion.
2195 		 */
2196 		if (ahci_portp->ahciport_max_ncq_tags == 0) {
2197 			ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2198 			AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2199 			    "ahci_deliver_satapkt: port %d the max tags for "
2200 			    "NCQ command is %d", port, ncq_qdepth);
2201 		} else {
2202 			if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2203 				cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2204 				    " port %d the max tag for NCQ command is "
2205 				    "requested to change from %d to %d, at the"
2206 				    " moment the driver doesn't support the "
2207 				    "dynamic change so it's going to "
2208 				    "still use the previous tag value",
2209 				    instance, port,
2210 				    ahci_portp->ahciport_max_ncq_tags,
2211 				    ncq_qdepth);
2212 			}
2213 		}
2214 	}
2215 
2216 	/* Check if the command is an error retrieval command */
2217 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2218 		command_type = AHCI_ERR_RETRI_CMD;
2219 
2220 	/* Check if the command is an read/write pmult command */
2221 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2222 		command_type = AHCI_RDWR_PMULT_CMD;
2223 
2224 	/* Check if there is an empty command slot */
2225 	cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2226 	    addrp, command_type);
2227 	if (cmd_slot == AHCI_FAILURE) {
2228 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2229 		return (AHCI_FAILURE);
2230 	}
2231 
2232 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2233 	    "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2234 	    "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2235 	    cmd_slot, port, (void *)spkt);
2236 
2237 	cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2238 	bzero((void *)cmd_table, ahci_cmd_table_size);
2239 
2240 	/* For data transfer operations, it is the H2D Register FIS */
2241 	h2d_register_fisp =
2242 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2243 
2244 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2245 
2246 	/*
2247 	 * PMP field only make sense when target is a port multiplier or a
2248 	 * device behind a port multiplier. Otherwise should set it to 0.
2249 	 */
2250 	if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2251 		SET_FIS_PMP(h2d_register_fisp, pmport);
2252 
2253 	SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2254 	SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2255 	SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2256 	SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2257 
2258 	switch (scmd->satacmd_addr_type) {
2259 
2260 	case 0:
2261 		/*
2262 		 * satacmd_addr_type will be 0 for the commands below:
2263 		 *	ATAPI command
2264 		 *	SATAC_IDLE_IM
2265 		 *	SATAC_STANDBY_IM
2266 		 *	SATAC_DOWNLOAD_MICROCODE
2267 		 *	SATAC_FLUSH_CACHE
2268 		 *	SATAC_SET_FEATURES
2269 		 *	SATAC_SMART
2270 		 *	SATAC_ID_PACKET_DEVICE
2271 		 *	SATAC_ID_DEVICE
2272 		 *	SATAC_READ_PORTMULT
2273 		 *	SATAC_WRITE_PORTMULT
2274 		 */
2275 		/* FALLTHRU */
2276 
2277 	case ATA_ADDR_LBA:
2278 		/* FALLTHRU */
2279 
2280 	case ATA_ADDR_LBA28:
2281 		/* LBA[7:0] */
2282 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2283 
2284 		/* LBA[15:8] */
2285 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2286 
2287 		/* LBA[23:16] */
2288 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2289 
2290 		/* LBA [27:24] (also called dev_head) */
2291 		SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2292 
2293 		break;
2294 
2295 	case ATA_ADDR_LBA48:
2296 		/* LBA[7:0] */
2297 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2298 
2299 		/* LBA[15:8] */
2300 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2301 
2302 		/* LBA[23:16] */
2303 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2304 
2305 		/* LBA [31:24] */
2306 		SET_FIS_SECTOR_EXP(h2d_register_fisp,
2307 		    scmd->satacmd_lba_low_msb);
2308 
2309 		/* LBA [39:32] */
2310 		SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2311 		    scmd->satacmd_lba_mid_msb);
2312 
2313 		/* LBA [47:40] */
2314 		SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2315 		    scmd->satacmd_lba_high_msb);
2316 
2317 		/* Set dev_head */
2318 		SET_FIS_DEV_HEAD(h2d_register_fisp,
2319 		    scmd->satacmd_device_reg);
2320 
2321 		/* Set the extended sector count and features */
2322 		SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2323 		    scmd->satacmd_sec_count_msb);
2324 		SET_FIS_FEATURES_EXP(h2d_register_fisp,
2325 		    scmd->satacmd_features_reg_ext);
2326 		break;
2327 	}
2328 
2329 	/*
2330 	 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2331 	 * filled into features field, and sector count 8:15 is filled into
2332 	 * features (exp) field. The hba driver doesn't need to anything
2333 	 * special with regard to this, since sata framework has already
2334 	 * done so.
2335 	 *
2336 	 * However the driver needs to make sure TAG is filled into sector
2337 	 * field.
2338 	 */
2339 	if (command_type == AHCI_NCQ_CMD) {
2340 		SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2341 		    (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2342 	}
2343 
2344 	ncookies = scmd->satacmd_num_dma_cookies;
2345 	AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2346 	    "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2347 	    ncookies, ahci_dma_prdt_number);
2348 
2349 	ASSERT(ncookies <= ahci_dma_prdt_number);
2350 	ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2351 
2352 	/* *** now fill the scatter gather list ******* */
2353 	for (i = 0; i < ncookies; i++) {
2354 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2355 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2356 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2357 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2358 		cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2359 		    scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2360 		ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2361 		    scmd->satacmd_dma_cookie_list[i].dmac_size;
2362 	}
2363 
2364 	AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2365 	    "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2366 	    ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2367 
2368 	/* The ACMD field is filled in for ATAPI command */
2369 	if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2370 		bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2371 		    SATA_ATAPI_MAX_CDB_LEN);
2372 	}
2373 
2374 	/* Set Command Header in Command List */
2375 	cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2376 	BZERO_DESCR_INFO(cmd_header);
2377 	BZERO_PRD_BYTE_COUNT(cmd_header);
2378 
2379 	/* Set the number of entries in the PRD table */
2380 	SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2381 
2382 	/* Set the length of the command in the CFIS area */
2383 	SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2384 
2385 	/*
2386 	 * PMP field only make sense when target is a port multiplier or a
2387 	 * device behind a port multiplier. Otherwise should set it to 0.
2388 	 */
2389 	if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2390 		SET_PORT_MULTI_PORT(cmd_header, pmport);
2391 
2392 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2393 	    "sata_data_direction = 0x%x",
2394 	    scmd->satacmd_flags.sata_data_direction);
2395 
2396 	/* Set A bit if it is an ATAPI command */
2397 	if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2398 		SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2399 
2400 	/* Set W bit if data is going to the device */
2401 	if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2402 		SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2403 
2404 	/*
2405 	 * Set the prefetchable bit - this bit is only valid if the PRDTL
2406 	 * field is non-zero or the ATAPI 'A' bit is set in the command
2407 	 * header. This bit cannot be set when using native command
2408 	 * queuing commands or when using FIS-based switching with a Port
2409 	 * multiplier.
2410 	 */
2411 	if (command_type != AHCI_NCQ_CMD)
2412 		SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2413 
2414 	/*
2415 	 * Now remember the sata packet in ahciport_slot_pkts[].
2416 	 * Error retrieval command and r/w port multiplier command will
2417 	 * be stored specifically for each port.
2418 	 */
2419 	if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2420 	    !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2421 		ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2422 
2423 	/*
2424 	 * Keep the timeout value
2425 	 */
2426 	ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2427 
2428 	/*
2429 	 * If the intial timout is less than 1 tick, then make it longer by
2430 	 * 1 tick to avoid immediate timeout
2431 	 */
2432 	if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2433 	    ahci_watchdog_timeout)
2434 		ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2435 		    ahci_watchdog_timeout;
2436 
2437 #if AHCI_DEBUG
2438 	if (ahci_debug_flags & AHCIDBG_ATACMD &&
2439 	    scmd->satacmd_cmd_reg != SATAC_PACKET ||
2440 	    ahci_debug_flags & AHCIDBG_ATAPICMD &&
2441 	    scmd->satacmd_cmd_reg == SATAC_PACKET) {
2442 
2443 		/* Dump the command header and table */
2444 		ahci_log(ahci_ctlp, CE_WARN, "\n");
2445 		ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2446 		    "0x%p cmd_reg 0x%x port %d", spkt,
2447 		    scmd->satacmd_cmd_reg, port);
2448 		ptr = (uint32_t *)cmd_header;
2449 		ahci_log(ahci_ctlp, CE_WARN,
2450 		    "  Command Header:%8x %8x %8x %8x",
2451 		    ptr[0], ptr[1], ptr[2], ptr[3]);
2452 
2453 		/* Dump the H2D register FIS */
2454 		ptr = (uint32_t *)h2d_register_fisp;
2455 		ahci_log(ahci_ctlp, CE_WARN,
2456 		    "  Command FIS:   %8x %8x %8x %8x",
2457 		    ptr[0], ptr[1], ptr[2], ptr[3]);
2458 
2459 		/* Dump the ACMD register FIS */
2460 		ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2461 		for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2462 			if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2463 				ahci_log(ahci_ctlp, CE_WARN,
2464 				    "  ATAPI command: %2x %2x %2x %2x "
2465 				    "%2x %2x %2x %2x",
2466 				    ptr2[8 * i], ptr2[8 * i + 1],
2467 				    ptr2[8 * i + 2], ptr2[8 * i + 3],
2468 				    ptr2[8 * i + 4], ptr2[8 * i + 5],
2469 				    ptr2[8 * i + 6], ptr2[8 * i + 7]);
2470 
2471 		/* Dump the PRDT */
2472 		for (i = 0; i < ncookies; i++) {
2473 			ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2474 			ahci_log(ahci_ctlp, CE_WARN,
2475 			    "  Cookie %d:      %8x %8x %8x %8x",
2476 			    i, ptr[0], ptr[1], ptr[2], ptr[3]);
2477 		}
2478 	}
2479 #endif
2480 
2481 	(void) ddi_dma_sync(
2482 	    ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2483 	    0,
2484 	    ahci_cmd_table_size,
2485 	    DDI_DMA_SYNC_FORDEV);
2486 
2487 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2488 	    cmd_slot * sizeof (ahci_cmd_header_t),
2489 	    sizeof (ahci_cmd_header_t),
2490 	    DDI_DMA_SYNC_FORDEV);
2491 
2492 	if ((ahci_check_dma_handle(ahci_portp->
2493 	    ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2494 	    ahci_check_dma_handle(ahci_portp->
2495 	    ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2496 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2497 		    DDI_SERVICE_UNAFFECTED);
2498 		return (AHCI_FAILURE);
2499 	}
2500 
2501 	/* Set the corresponding bit in the PxSACT.DS for queued command */
2502 	if (command_type == AHCI_NCQ_CMD) {
2503 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2504 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2505 		    (0x1 << cmd_slot));
2506 	}
2507 
2508 	/* Indicate to the HBA that a command is active. */
2509 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2510 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2511 	    (0x1 << cmd_slot));
2512 
2513 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2514 	    "exit: port %d", port);
2515 
2516 	/* Make sure the command is started by the PxSACT/PxCI */
2517 	if (ahci_check_acc_handle(ahci_ctlp->
2518 	    ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2519 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2520 		    DDI_SERVICE_UNAFFECTED);
2521 		return (AHCI_FAILURE);
2522 	}
2523 
2524 	return (cmd_slot);
2525 }
2526 
2527 /*
2528  * Called by the sata framework to abort the previously sent packet(s).
2529  *
2530  * Reset device to abort commands.
2531  */
2532 static int
ahci_tran_abort(dev_info_t * dip,sata_pkt_t * spkt,int flag)2533 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2534 {
2535 	ahci_ctl_t *ahci_ctlp;
2536 	ahci_port_t *ahci_portp;
2537 	uint32_t slot_status = 0;
2538 	uint32_t aborted_tags = 0;
2539 	uint32_t finished_tags = 0;
2540 	uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2541 	uint8_t port;
2542 	int tmp_slot;
2543 	int instance = ddi_get_instance(dip);
2544 
2545 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2546 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
2547 
2548 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2549 	    "ahci_tran_abort enter: port %d", port);
2550 
2551 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
2552 	mutex_enter(&ahci_portp->ahciport_mutex);
2553 
2554 	/*
2555 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2556 	 * commands are being mopped, therefore there is nothing else to do
2557 	 */
2558 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2559 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2560 		    "ahci_tran_abort: port %d is in "
2561 		    "mopping process, so just return directly ", port);
2562 		mutex_exit(&ahci_portp->ahciport_mutex);
2563 		return (SATA_SUCCESS);
2564 	}
2565 
2566 	/*
2567 	 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2568 	 * command is being executed so no other commands is outstanding,
2569 	 * nothing to do.
2570 	 */
2571 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2572 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2573 		    "ahci_tran_abort: port %d is reading/writing "
2574 		    "port multiplier, so just return directly ", port);
2575 		mutex_exit(&ahci_portp->ahciport_mutex);
2576 		return (SATA_SUCCESS);
2577 	}
2578 
2579 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2580 	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2581 	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2582 		/*
2583 		 * In case the targer driver would send the request before
2584 		 * sata framework can have the opportunity to process those
2585 		 * event reports.
2586 		 */
2587 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2588 		spkt->satapkt_device.satadev_state =
2589 		    ahci_portp->ahciport_port_state;
2590 		ahci_update_sata_registers(ahci_ctlp, port,
2591 		    &spkt->satapkt_device);
2592 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2593 		    "ahci_tran_abort returning SATA_FAILURE while "
2594 		    "port in FAILED/SHUTDOWN/PWROFF state: "
2595 		    "port: %d", port);
2596 		mutex_exit(&ahci_portp->ahciport_mutex);
2597 		return (SATA_FAILURE);
2598 	}
2599 
2600 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2601 		/*
2602 		 * ahci_intr_phyrdy_change() may have rendered it to
2603 		 * AHCI_PORT_TYPE_NODEV.
2604 		 */
2605 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2606 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2607 		spkt->satapkt_device.satadev_state =
2608 		    ahci_portp->ahciport_port_state;
2609 		ahci_update_sata_registers(ahci_ctlp, port,
2610 		    &spkt->satapkt_device);
2611 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2612 		    "ahci_tran_abort returning SATA_FAILURE while "
2613 		    "no device attached: port: %d", port);
2614 		mutex_exit(&ahci_portp->ahciport_mutex);
2615 		return (SATA_FAILURE);
2616 	}
2617 
2618 	if (flag == SATA_ABORT_ALL_PACKETS) {
2619 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2620 			aborted_tags = ahci_portp->ahciport_pending_tags;
2621 		else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2622 			aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2623 
2624 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2625 		    instance, port);
2626 	} else {
2627 		aborted_tags = 0xffffffff;
2628 		/*
2629 		 * Aborting one specific packet, first search the
2630 		 * ahciport_slot_pkts[] list for matching spkt.
2631 		 */
2632 		for (tmp_slot = 0;
2633 		    tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2634 			if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2635 				aborted_tags = (0x1 << tmp_slot);
2636 				break;
2637 			}
2638 		}
2639 
2640 		if (aborted_tags == 0xffffffff) {
2641 			/* request packet is not on the pending list */
2642 			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2643 			    "Cannot find the aborting pkt 0x%p on the "
2644 			    "pending list", (void *)spkt);
2645 			ahci_update_sata_registers(ahci_ctlp, port,
2646 			    &spkt->satapkt_device);
2647 			mutex_exit(&ahci_portp->ahciport_mutex);
2648 			return (SATA_FAILURE);
2649 		}
2650 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2651 		    instance, port, (void *)spkt);
2652 	}
2653 
2654 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2655 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2656 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2657 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2658 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2659 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2660 
2661 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2662 	ahci_portp->ahciport_mop_in_progress++;
2663 
2664 	/*
2665 	 * To abort the packet(s), first we are trying to clear PxCMD.ST
2666 	 * to stop the port, and if the port can be stopped
2667 	 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2668 	 * then we just send back the aborted packet(s) with ABORTED flag
2669 	 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2670 	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2671 	 * perform a COMRESET.
2672 	 */
2673 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2674 	    ahci_portp, port, 0, NULL);
2675 
2676 	/*
2677 	 * Compute which have finished and which need to be retried.
2678 	 *
2679 	 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2680 	 * minus the slot_status. The aborted_tags has to be deducted by
2681 	 * finished_tags since we can't possibly abort a tag which had finished
2682 	 * already.
2683 	 */
2684 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2685 		finished_tags = ahci_portp->ahciport_pending_tags &
2686 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2687 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2688 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2689 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2690 
2691 	aborted_tags &= ~finished_tags;
2692 
2693 	ahci_mop_commands(ahci_ctlp,
2694 	    ahci_portp,
2695 	    slot_status,
2696 	    0, /* failed tags */
2697 	    0, /* timeout tags */
2698 	    aborted_tags,
2699 	    0); /* reset tags */
2700 
2701 	ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2702 	mutex_exit(&ahci_portp->ahciport_mutex);
2703 
2704 	return (SATA_SUCCESS);
2705 }
2706 
2707 /*
2708  * Used to do device reset and reject all the pending packets on a device
2709  * during the reset operation.
2710  *
2711  * NOTE: ONLY called by ahci_tran_reset_dport
2712  */
2713 static int
ahci_reset_device_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2714 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2715     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2716 {
2717 	uint32_t slot_status = 0;
2718 	uint32_t reset_tags = 0;
2719 	uint32_t finished_tags = 0;
2720 	uint8_t port = addrp->aa_port;
2721 	sata_device_t sdevice;
2722 	int ret;
2723 
2724 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2725 
2726 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2727 	    "ahci_reset_device_reject_pkts on port: %d", port);
2728 
2729 	/*
2730 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2731 	 * commands are being mopped, therefore there is nothing else to do
2732 	 */
2733 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2734 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2735 		    "ahci_reset_device_reject_pkts: port %d is in "
2736 		    "mopping process, so return directly ", port);
2737 		return (SATA_SUCCESS);
2738 	}
2739 
2740 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2741 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2742 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2743 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2744 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2745 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2746 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2747 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2748 	}
2749 
2750 	if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2751 	    != AHCI_SUCCESS) {
2752 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2753 		    "Try to do a port reset after software "
2754 		    "reset failed", port);
2755 		ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2756 		if (ret != AHCI_SUCCESS) {
2757 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2758 			    "ahci_reset_device_reject_pkts: port %d "
2759 			    "failed", port);
2760 			return (SATA_FAILURE);
2761 		}
2762 	}
2763 	/* Set the reset in progress flag */
2764 	ahci_portp->ahciport_reset_in_progress = 1;
2765 
2766 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2767 	ahci_portp->ahciport_mop_in_progress++;
2768 
2769 	/* Indicate to the framework that a reset has happened */
2770 	bzero((void *)&sdevice, sizeof (sata_device_t));
2771 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2772 	sdevice.satadev_addr.pmport = 0;
2773 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2774 	sdevice.satadev_state = SATA_DSTATE_RESET |
2775 	    SATA_DSTATE_PWR_ACTIVE;
2776 	mutex_exit(&ahci_portp->ahciport_mutex);
2777 	sata_hba_event_notify(
2778 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2779 	    &sdevice,
2780 	    SATA_EVNT_DEVICE_RESET);
2781 	mutex_enter(&ahci_portp->ahciport_mutex);
2782 
2783 	AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2784 	    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2785 
2786 	/* Next try to mop the pending commands */
2787 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2788 		finished_tags = ahci_portp->ahciport_pending_tags &
2789 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2790 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2791 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2792 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2793 
2794 	reset_tags &= ~finished_tags;
2795 
2796 	ahci_mop_commands(ahci_ctlp,
2797 	    ahci_portp,
2798 	    slot_status,
2799 	    0, /* failed tags */
2800 	    0, /* timeout tags */
2801 	    0, /* aborted tags */
2802 	    reset_tags); /* reset tags */
2803 
2804 	return (SATA_SUCCESS);
2805 }
2806 
2807 /*
2808  * Used to do device reset and reject all the pending packets on a device
2809  * during the reset operation.
2810  *
2811  * NOTE: ONLY called by ahci_tran_reset_dport
2812  */
2813 static int
ahci_reset_pmdevice_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2814 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2815     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2816 {
2817 	uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2818 	uint8_t port = addrp->aa_port;
2819 	uint8_t pmport = addrp->aa_pmport;
2820 	sata_device_t sdevice;
2821 
2822 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2823 
2824 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2825 	    "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2826 
2827 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2828 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2829 		    "ahci_reset_pmdevice_reject_pkts: port %d is in "
2830 		    "mopping process, so return directly ", port);
2831 		return (SATA_SUCCESS);
2832 	}
2833 
2834 	/* Checking for outstanding commands */
2835 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2836 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2837 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2838 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2839 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2840 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2841 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2842 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2843 	}
2844 
2845 	/* Issue SOFTWARE reset command. */
2846 	if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2847 	    != AHCI_SUCCESS) {
2848 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2849 		    "Try to do a port reset after software "
2850 		    "reset failed", port);
2851 		return (SATA_FAILURE);
2852 	}
2853 
2854 	/* Set the reset in progress flag */
2855 	ahci_portp->ahciport_reset_in_progress = 1;
2856 
2857 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2858 	ahci_portp->ahciport_mop_in_progress++;
2859 
2860 	/* Indicate to the framework that a reset has happened */
2861 	bzero((void *)&sdevice, sizeof (sata_device_t));
2862 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2863 	sdevice.satadev_addr.pmport = pmport;
2864 	if (AHCI_ADDR_IS_PMULT(addrp))
2865 		sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2866 	else
2867 		sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2868 	sdevice.satadev_state = SATA_DSTATE_RESET |
2869 	    SATA_DSTATE_PWR_ACTIVE;
2870 	mutex_exit(&ahci_portp->ahciport_mutex);
2871 	sata_hba_event_notify(
2872 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2873 	    &sdevice,
2874 	    SATA_EVNT_DEVICE_RESET);
2875 	mutex_enter(&ahci_portp->ahciport_mutex);
2876 
2877 	AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2878 	    "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2879 	    port, pmport);
2880 
2881 	/* Next try to mop the pending commands */
2882 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2883 		finished_tags = ahci_portp->ahciport_pending_tags &
2884 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2885 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2886 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2887 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2888 	reset_tags &= ~finished_tags;
2889 
2890 	AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2891 	    "reset_tags = %x, finished_tags = %x, slot_status = %x",
2892 	    reset_tags, finished_tags, slot_status);
2893 
2894 	/*
2895 	 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2896 	 * try to reset a single device behind a port multiplier will
2897 	 * terminate all the commands on that HBA port. We need mop these
2898 	 * commands as well.
2899 	 */
2900 	ahci_mop_commands(ahci_ctlp,
2901 	    ahci_portp,
2902 	    slot_status,
2903 	    0, /* failed tags */
2904 	    0, /* timeout tags */
2905 	    0, /* aborted tags */
2906 	    reset_tags); /* reset tags */
2907 
2908 	return (SATA_SUCCESS);
2909 }
2910 
2911 /*
2912  * Used to do port reset and reject all the pending packets on a port during
2913  * the reset operation.
2914  */
2915 static int
ahci_reset_port_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2916 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2917     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2918 {
2919 	uint32_t slot_status = 0;
2920 	uint32_t reset_tags = 0;
2921 	uint32_t finished_tags = 0;
2922 	uint8_t port = addrp->aa_port;
2923 
2924 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2925 
2926 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2927 	    "ahci_reset_port_reject_pkts at port: %d", port);
2928 
2929 	/*
2930 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2931 	 * commands are being mopped, therefore there is nothing else to do
2932 	 */
2933 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2934 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2935 		    "ahci_reset_port_reject_pkts: port %d is in "
2936 		    "mopping process, so return directly ", port);
2937 		return (SATA_SUCCESS);
2938 	}
2939 
2940 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2941 	ahci_portp->ahciport_mop_in_progress++;
2942 
2943 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2944 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2945 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2946 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2947 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2948 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2949 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2950 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2951 	}
2952 
2953 	if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2954 	    ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2955 	    NULL) != AHCI_SUCCESS) {
2956 
2957 		/* Clear mop flag */
2958 		ahci_portp->ahciport_mop_in_progress--;
2959 		if (ahci_portp->ahciport_mop_in_progress == 0)
2960 			ahci_portp->ahciport_flags &=
2961 			    ~AHCI_PORT_FLAG_MOPPING;
2962 		return (SATA_FAILURE);
2963 	}
2964 
2965 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2966 		finished_tags = ahci_portp->ahciport_pending_tags &
2967 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2968 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2969 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2970 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2971 
2972 	reset_tags &= ~finished_tags;
2973 
2974 	ahci_mop_commands(ahci_ctlp,
2975 	    ahci_portp,
2976 	    slot_status,
2977 	    0, /* failed tags */
2978 	    0, /* timeout tags */
2979 	    0, /* aborted tags */
2980 	    reset_tags); /* reset tags */
2981 
2982 	return (SATA_SUCCESS);
2983 }
2984 
2985 /*
2986  * Used to do hba reset and reject all the pending packets on all ports
2987  * during the reset operation.
2988  */
2989 static int
ahci_reset_hba_reject_pkts(ahci_ctl_t * ahci_ctlp)2990 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2991 {
2992 	ahci_port_t *ahci_portp;
2993 	uint32_t slot_status[AHCI_MAX_PORTS];
2994 	uint32_t reset_tags[AHCI_MAX_PORTS];
2995 	uint32_t finished_tags[AHCI_MAX_PORTS];
2996 	int port;
2997 	int ret = SATA_SUCCESS;
2998 
2999 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3000 	    "ahci_reset_hba_reject_pkts enter", NULL);
3001 
3002 	bzero(slot_status, sizeof (slot_status));
3003 	bzero(reset_tags, sizeof (reset_tags));
3004 	bzero(finished_tags, sizeof (finished_tags));
3005 
3006 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3007 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3008 			continue;
3009 		}
3010 
3011 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3012 
3013 		mutex_enter(&ahci_portp->ahciport_mutex);
3014 		ahci_portp->ahciport_reset_in_progress = 1;
3015 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3016 			slot_status[port] = ddi_get32(
3017 			    ahci_ctlp->ahcictl_ahci_acc_handle,
3018 			    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3019 			reset_tags[port] = slot_status[port] &
3020 			    AHCI_SLOT_MASK(ahci_ctlp);
3021 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3022 			    "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3023 			    port, reset_tags[port],
3024 			    ahci_portp->ahciport_pending_tags);
3025 		} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3026 			slot_status[port] = ddi_get32(
3027 			    ahci_ctlp->ahcictl_ahci_acc_handle,
3028 			    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3029 			reset_tags[port] = slot_status[port] &
3030 			    AHCI_NCQ_SLOT_MASK(ahci_portp);
3031 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3032 			    "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3033 			    port, reset_tags[port],
3034 			    ahci_portp->ahciport_pending_tags);
3035 		}
3036 		mutex_exit(&ahci_portp->ahciport_mutex);
3037 	}
3038 
3039 	if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
3040 		ret = SATA_FAILURE;
3041 	}
3042 
3043 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3044 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3045 			continue;
3046 		}
3047 
3048 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3049 
3050 		mutex_enter(&ahci_portp->ahciport_mutex);
3051 		/*
3052 		 * To prevent recursive enter to ahci_mop_commands, we need
3053 		 * check AHCI_PORT_FLAG_MOPPING flag.
3054 		 */
3055 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3056 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3057 			    "ahci_reset_hba_reject_pkts: port %d is in "
3058 			    "mopping process, so return directly ", port);
3059 			mutex_exit(&ahci_portp->ahciport_mutex);
3060 			continue;
3061 		}
3062 
3063 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3064 		ahci_portp->ahciport_mop_in_progress++;
3065 
3066 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
3067 			finished_tags[port]  =
3068 			    ahci_portp->ahciport_pending_tags &
3069 			    ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
3070 		else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
3071 			finished_tags[port] =
3072 			    ahci_portp->ahciport_pending_ncq_tags &
3073 			    ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
3074 
3075 		reset_tags[port] &= ~finished_tags[port];
3076 
3077 		ahci_mop_commands(ahci_ctlp,
3078 		    ahci_portp,
3079 		    slot_status[port],
3080 		    0, /* failed tags */
3081 		    0, /* timeout tags */
3082 		    0, /* aborted tags */
3083 		    reset_tags[port]); /* reset tags */
3084 		mutex_exit(&ahci_portp->ahciport_mutex);
3085 	}
3086 out:
3087 	return (ret);
3088 }
3089 
3090 /*
3091  * Called by sata framework to reset a port(s) or device.
3092  */
3093 static int
ahci_tran_reset_dport(dev_info_t * dip,sata_device_t * sd)3094 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
3095 {
3096 	ahci_ctl_t *ahci_ctlp;
3097 	ahci_port_t *ahci_portp;
3098 	ahci_addr_t addr;
3099 	uint8_t cport = sd->satadev_addr.cport;
3100 	uint8_t pmport = sd->satadev_addr.pmport;
3101 	uint8_t port;
3102 	int ret = SATA_SUCCESS;
3103 	int instance = ddi_get_instance(dip);
3104 
3105 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3106 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3107 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3108 
3109 	ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3110 
3111 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3112 	    "ahci_tran_reset_dport enter: cport %d", cport);
3113 
3114 	switch (sd->satadev_addr.qual) {
3115 	case SATA_ADDR_PMPORT:
3116 		/*
3117 		 * If we want to issue a COMRESET on a pmport, we need to
3118 		 * reject the outstanding commands on that pmport. According
3119 		 * to AHCI spec, PxCI register could only be cleared by
3120 		 * clearing PxCMD.ST, which will halt the controller port - as
3121 		 * well as other pmports.
3122 		 *
3123 		 * Therefore we directly reset the controller port for
3124 		 * simplicity. ahci_tran_probe_port() will handle reset stuff
3125 		 * like initializing the given pmport.
3126 		 */
3127 		/* FALLTHRU */
3128 	case SATA_ADDR_CPORT:
3129 		/* Port reset */
3130 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3131 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3132 		    "port %d reset port", instance, port);
3133 
3134 		mutex_enter(&ahci_portp->ahciport_mutex);
3135 		ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3136 		mutex_exit(&ahci_portp->ahciport_mutex);
3137 
3138 		break;
3139 
3140 	case SATA_ADDR_DPMPORT:
3141 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3142 		    "port %d:%d reset device", instance, port, pmport);
3143 		/* FALLTHRU */
3144 	case SATA_ADDR_DCPORT:
3145 		/* Device reset */
3146 		if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3147 			cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3148 			    "port %d reset device", instance, port);
3149 
3150 		mutex_enter(&ahci_portp->ahciport_mutex);
3151 		/*
3152 		 * software reset request must be sent to SATA_PMULT_HOSTPORT
3153 		 * if target is a port multiplier:
3154 		 */
3155 		if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3156 		    ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3157 			AHCI_ADDR_SET_PMULT(&addr, port);
3158 
3159 		if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3160 		    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3161 		    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3162 			/*
3163 			 * In case the targer driver would send the request
3164 			 * before sata framework can have the opportunity to
3165 			 * process those event reports.
3166 			 */
3167 			sd->satadev_state = ahci_portp->ahciport_port_state;
3168 			ahci_update_sata_registers(ahci_ctlp, port, sd);
3169 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3170 			    "ahci_tran_reset_dport returning SATA_FAILURE "
3171 			    "while port in FAILED/SHUTDOWN/PWROFF state: "
3172 			    "port: %d", port);
3173 			mutex_exit(&ahci_portp->ahciport_mutex);
3174 			ret = SATA_FAILURE;
3175 			break;
3176 		}
3177 
3178 		if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3179 		    SATA_DTYPE_NONE) {
3180 			/*
3181 			 * ahci_intr_phyrdy_change() may have rendered it to
3182 			 * AHCI_PORT_TYPE_NODEV.
3183 			 */
3184 			sd->satadev_type = SATA_DTYPE_NONE;
3185 			sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3186 			    &addr);
3187 			ahci_update_sata_registers(ahci_ctlp, port, sd);
3188 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3189 			    "ahci_tran_reset_dport returning SATA_FAILURE "
3190 			    "while no device attached: port: %d", port);
3191 			mutex_exit(&ahci_portp->ahciport_mutex);
3192 			ret = SATA_FAILURE;
3193 			break;
3194 		}
3195 
3196 		if (AHCI_ADDR_IS_PORT(&addr)) {
3197 			ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3198 			    ahci_portp, &addr);
3199 		} else {
3200 			ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3201 			    ahci_portp, &addr);
3202 		}
3203 
3204 		mutex_exit(&ahci_portp->ahciport_mutex);
3205 		break;
3206 
3207 	case SATA_ADDR_CNTRL:
3208 		/* Reset the whole controller */
3209 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3210 		    "reset the whole hba", instance);
3211 		ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3212 		break;
3213 
3214 	default:
3215 		ret = SATA_FAILURE;
3216 	}
3217 
3218 	return (ret);
3219 }
3220 
3221 /*
3222  * Called by sata framework to activate a port as part of hotplug.
3223  * (cfgadm -c connect satax/y)
3224  * Support port multiplier.
3225  */
3226 static int
ahci_tran_hotplug_port_activate(dev_info_t * dip,sata_device_t * satadev)3227 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3228 {
3229 	ahci_ctl_t *ahci_ctlp;
3230 	ahci_port_t *ahci_portp;
3231 	ahci_addr_t addr;
3232 	uint8_t	cport = satadev->satadev_addr.cport;
3233 	uint8_t	pmport = satadev->satadev_addr.pmport;
3234 	uint8_t port;
3235 	int instance = ddi_get_instance(dip);
3236 
3237 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3238 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3239 
3240 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3241 	    "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3242 
3243 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3244 
3245 	mutex_enter(&ahci_portp->ahciport_mutex);
3246 	ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3247 	ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3248 
3249 	if (AHCI_ADDR_IS_PORT(&addr)) {
3250 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3251 		    instance, port);
3252 
3253 		/* Enable the interrupts on the port */
3254 		ahci_enable_port_intrs(ahci_ctlp, port);
3255 
3256 		/*
3257 		 * Reset the port so that the PHY communication would be
3258 		 * re-established.  But this reset is an internal operation
3259 		 * and the sata module doesn't need to know about it.
3260 		 * Moreover, the port with a device attached will be started
3261 		 * too.
3262 		 */
3263 		(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3264 		    ahci_portp, port,
3265 		    AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3266 		    NULL);
3267 
3268 		/*
3269 		 * Need to check the link status and device status of the port
3270 		 * and consider raising power if the port was in D3 state
3271 		 */
3272 		ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3273 		ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3274 		ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3275 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3276 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3277 		    instance, port, pmport);
3278 		/* AHCI_ADDR_PMPORT */
3279 		AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3280 		AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3281 		    ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3282 	}
3283 
3284 	satadev->satadev_state = ahci_portp->ahciport_port_state;
3285 
3286 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
3287 
3288 	mutex_exit(&ahci_portp->ahciport_mutex);
3289 	return (SATA_SUCCESS);
3290 }
3291 
3292 /*
3293  * Called by sata framework to deactivate a port as part of hotplug.
3294  * (cfgadm -c disconnect satax/y)
3295  * Support port multiplier.
3296  */
3297 static int
ahci_tran_hotplug_port_deactivate(dev_info_t * dip,sata_device_t * satadev)3298 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3299 {
3300 	ahci_ctl_t *ahci_ctlp;
3301 	ahci_port_t *ahci_portp;
3302 	ahci_addr_t addr;
3303 	uint8_t	cport = satadev->satadev_addr.cport;
3304 	uint8_t	pmport = satadev->satadev_addr.pmport;
3305 	uint8_t port;
3306 	uint32_t port_scontrol;
3307 	int instance = ddi_get_instance(dip);
3308 
3309 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3310 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3311 
3312 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3313 	    "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3314 
3315 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3316 	mutex_enter(&ahci_portp->ahciport_mutex);
3317 	ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3318 	ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3319 
3320 	if (AHCI_ADDR_IS_PORT(&addr)) {
3321 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3322 		    instance, port);
3323 
3324 		/* Disable the interrupts on the port */
3325 		ahci_disable_port_intrs(ahci_ctlp, port);
3326 
3327 		if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3328 
3329 			/* First to abort all the pending commands */
3330 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3331 
3332 			/* Then stop the port */
3333 			(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3334 			    ahci_portp, port);
3335 		}
3336 
3337 		/* Next put the PHY offline */
3338 		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3339 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3340 		SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3341 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3342 		    AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3343 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3344 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3345 		    instance, port, pmport);
3346 
3347 		ahci_disable_port_intrs(ahci_ctlp, port);
3348 		if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3349 		    != SATA_DTYPE_NONE)
3350 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3351 
3352 		/* Re-enable the interrupts for the other pmports */
3353 		ahci_enable_port_intrs(ahci_ctlp, port);
3354 	}
3355 
3356 	/* Update port state */
3357 	AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3358 	satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3359 
3360 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
3361 
3362 	mutex_exit(&ahci_portp->ahciport_mutex);
3363 	return (SATA_SUCCESS);
3364 }
3365 
3366 /*
3367  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3368  * when a device is unplugged or a port is deactivated.
3369  */
3370 static void
ahci_reject_all_abort_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)3371 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3372     ahci_port_t *ahci_portp, uint8_t port)
3373 {
3374 	uint32_t slot_status = 0;
3375 	uint32_t abort_tags = 0;
3376 
3377 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3378 
3379 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3380 	    "ahci_reject_all_abort_pkts at port: %d", port);
3381 
3382 	/* Read/write port multiplier command takes highest priority */
3383 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3384 		slot_status = 0x1;
3385 		abort_tags = 0x1;
3386 		goto out;
3387 	}
3388 
3389 	/*
3390 	 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3391 	 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3392 	 * to get the error data, if yes when the device is removed, the
3393 	 * command needs to be aborted too.
3394 	 */
3395 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3396 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3397 			slot_status = 0x1;
3398 			abort_tags = 0x1;
3399 			goto out;
3400 		} else {
3401 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3402 			    "ahci_reject_all_abort_pkts return directly "
3403 			    "port %d no needs to reject any outstanding "
3404 			    "commands", port);
3405 			return;
3406 		}
3407 	}
3408 
3409 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3410 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3411 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3412 		abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3413 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3414 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3415 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3416 		abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3417 	}
3418 
3419 out:
3420 	/* No need to do mop when there is no outstanding commands */
3421 	if (slot_status != 0) {
3422 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3423 		ahci_portp->ahciport_mop_in_progress++;
3424 
3425 		ahci_mop_commands(ahci_ctlp,
3426 		    ahci_portp,
3427 		    slot_status,
3428 		    0, /* failed tags */
3429 		    0, /* timeout tags */
3430 		    abort_tags, /* aborting tags */
3431 		    0); /* reset tags */
3432 	}
3433 }
3434 
3435 #if defined(__lock_lint)
3436 static int
ahci_selftest(dev_info_t * dip,sata_device_t * device)3437 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3438 {
3439 	return (SATA_SUCCESS);
3440 }
3441 #endif
3442 
3443 /*
3444  * Initialize fma capabilities and register with IO fault services.
3445  */
3446 static void
ahci_fm_init(ahci_ctl_t * ahci_ctlp)3447 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3448 {
3449 	/*
3450 	 * Need to change iblock to priority for new MSI intr
3451 	 */
3452 	ddi_iblock_cookie_t fm_ibc;
3453 
3454 	ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3455 	    ahci_ctlp->ahcictl_dip,
3456 	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3457 	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3458 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3459 
3460 	/* Only register with IO Fault Services if we have some capability */
3461 	if (ahci_ctlp->ahcictl_fm_cap) {
3462 		/* Adjust access and dma attributes for FMA */
3463 		accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3464 		buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3465 		rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3466 		cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3467 		cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3468 
3469 		/*
3470 		 * Register capabilities with IO Fault Services.
3471 		 * ahcictl_fm_cap will be updated to indicate
3472 		 * capabilities actually supported (not requested.)
3473 		 */
3474 		ddi_fm_init(ahci_ctlp->ahcictl_dip,
3475 		    &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3476 
3477 		if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3478 			cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3479 			    ddi_get_instance(ahci_ctlp->ahcictl_dip));
3480 			return;
3481 		}
3482 		/*
3483 		 * Initialize pci ereport capabilities if ereport
3484 		 * capable (should always be.)
3485 		 */
3486 		if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3487 		    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3488 			pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3489 		}
3490 
3491 		/*
3492 		 * Register error callback if error callback capable.
3493 		 */
3494 		if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3495 			ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3496 			    ahci_fm_error_cb, (void *) ahci_ctlp);
3497 		}
3498 
3499 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3500 		    "ahci_fm_fini: fma enabled.", NULL);
3501 	}
3502 }
3503 
3504 /*
3505  * Releases fma capabilities and un-registers with IO fault services.
3506  */
3507 static void
ahci_fm_fini(ahci_ctl_t * ahci_ctlp)3508 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3509 {
3510 	/* Only unregister FMA capabilities if registered */
3511 	if (ahci_ctlp->ahcictl_fm_cap) {
3512 		/*
3513 		 * Un-register error callback if error callback capable.
3514 		 */
3515 		if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3516 			ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3517 		}
3518 
3519 		/*
3520 		 * Release any resources allocated by pci_ereport_setup()
3521 		 */
3522 		if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3523 		    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3524 			pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3525 		}
3526 
3527 		/* Unregister from IO Fault Services */
3528 		ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3529 
3530 		/* Adjust access and dma attributes for FMA */
3531 		accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3532 		buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3533 		rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3534 		cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3535 		cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3536 
3537 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3538 		    "ahci_fm_fini: fma disabled.", NULL);
3539 	}
3540 }
3541 
3542 /*ARGSUSED*/
3543 static int
ahci_fm_error_cb(dev_info_t * dip,ddi_fm_error_t * err,const void * impl_data)3544 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3545 {
3546 	/*
3547 	 * as the driver can always deal with an error in any dma or
3548 	 * access handle, we can just return the fme_status value.
3549 	 */
3550 	pci_ereport_post(dip, err, NULL);
3551 	return (err->fme_status);
3552 }
3553 
3554 int
ahci_check_acc_handle(ddi_acc_handle_t handle)3555 ahci_check_acc_handle(ddi_acc_handle_t handle)
3556 {
3557 	ddi_fm_error_t de;
3558 
3559 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3560 	return (de.fme_status);
3561 }
3562 
3563 int
ahci_check_dma_handle(ddi_dma_handle_t handle)3564 ahci_check_dma_handle(ddi_dma_handle_t handle)
3565 {
3566 	ddi_fm_error_t de;
3567 
3568 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3569 	return (de.fme_status);
3570 }
3571 
3572 /*
3573  * Generate an ereport
3574  */
3575 void
ahci_fm_ereport(ahci_ctl_t * ahci_ctlp,char * detail)3576 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3577 {
3578 	uint64_t ena;
3579 	char buf[FM_MAX_CLASS];
3580 
3581 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3582 	ena = fm_ena_generate(0, FM_ENA_FMT1);
3583 	if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3584 		ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3585 		    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3586 		    FM_EREPORT_VERSION, NULL);
3587 	}
3588 }
3589 
3590 /*
3591  * Check if all handles are correctly allocated.
3592  */
3593 static int
ahci_check_all_handle(ahci_ctl_t * ahci_ctlp)3594 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3595 {
3596 	int port;
3597 
3598 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3599 		return (DDI_FAILURE);
3600 	}
3601 
3602 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3603 		ahci_port_t *ahci_portp;
3604 
3605 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3606 			continue;
3607 
3608 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3609 
3610 		mutex_enter(&ahci_portp->ahciport_mutex);
3611 		if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3612 			mutex_exit(&ahci_portp->ahciport_mutex);
3613 			return (DDI_FAILURE);
3614 		}
3615 		mutex_exit(&ahci_portp->ahciport_mutex);
3616 	}
3617 
3618 	return (DDI_SUCCESS);
3619 }
3620 
3621 /*
3622  * Check the access handles for the controller. Note that
3623  * ahcictl_pci_conf_handle is only used in attach process.
3624  */
3625 static int
ahci_check_ctl_handle(ahci_ctl_t * ahci_ctlp)3626 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3627 {
3628 	if ((ahci_check_acc_handle(ahci_ctlp->
3629 	    ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3630 	    (ahci_check_acc_handle(ahci_ctlp->
3631 	    ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3632 		return (DDI_FAILURE);
3633 	}
3634 	return (DDI_SUCCESS);
3635 }
3636 
3637 /*
3638  * Check the DMA handles and the access handles of a controller port.
3639  */
3640 static int
ahci_check_port_handle(ahci_ctl_t * ahci_ctlp,int port)3641 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3642 {
3643 	ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3644 	int slot;
3645 
3646 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3647 
3648 	if ((ahci_check_dma_handle(ahci_portp->
3649 	    ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3650 	    (ahci_check_dma_handle(ahci_portp->
3651 	    ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3652 	    (ahci_check_acc_handle(ahci_portp->
3653 	    ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3654 	    (ahci_check_acc_handle(ahci_portp->
3655 	    ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3656 		return (DDI_FAILURE);
3657 	}
3658 	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3659 		if (ahci_check_slot_handle(ahci_portp, slot)
3660 		    != DDI_SUCCESS) {
3661 			return (DDI_FAILURE);
3662 		}
3663 	}
3664 	return (DDI_SUCCESS);
3665 }
3666 
3667 /*
3668  * Check the DMA handles and the access handles of a cmd table slot.
3669  */
3670 static int
ahci_check_slot_handle(ahci_port_t * ahci_portp,int slot)3671 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3672 {
3673 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3674 
3675 	if ((ahci_check_acc_handle(ahci_portp->
3676 	    ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3677 	    (ahci_check_dma_handle(ahci_portp->
3678 	    ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3679 		return (DDI_FAILURE);
3680 	}
3681 	return (DDI_SUCCESS);
3682 }
3683 
3684 /*
3685  * Allocate the ports structure, only called by ahci_attach
3686  */
3687 static int
ahci_alloc_ports_state(ahci_ctl_t * ahci_ctlp)3688 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3689 {
3690 	int port, cport = 0;
3691 
3692 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3693 	    "ahci_alloc_ports_state enter", NULL);
3694 
3695 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3696 
3697 	/* Allocate structures only for the implemented ports */
3698 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3699 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3700 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3701 			    "hba port %d not implemented", port);
3702 			continue;
3703 		}
3704 
3705 		ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3706 		ahci_ctlp->ahcictl_port_to_cport[port] =
3707 		    (uint8_t)cport++;
3708 
3709 		if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3710 			goto err_out;
3711 		}
3712 	}
3713 
3714 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3715 	return (AHCI_SUCCESS);
3716 
3717 err_out:
3718 	for (port--; port >= 0; port--) {
3719 		if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3720 			ahci_dealloc_port_state(ahci_ctlp, port);
3721 		}
3722 	}
3723 
3724 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3725 	return (AHCI_FAILURE);
3726 }
3727 
3728 /*
3729  * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3730  */
3731 static void
ahci_dealloc_ports_state(ahci_ctl_t * ahci_ctlp)3732 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3733 {
3734 	int port;
3735 
3736 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3737 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3738 		/* if this port is implemented by the HBA */
3739 		if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3740 			ahci_dealloc_port_state(ahci_ctlp, port);
3741 	}
3742 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3743 }
3744 
3745 /*
3746  * Drain the taskq.
3747  */
3748 static void
ahci_drain_ports_taskq(ahci_ctl_t * ahci_ctlp)3749 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3750 {
3751 	ahci_port_t *ahci_portp;
3752 	int port;
3753 
3754 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3755 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3756 			continue;
3757 		}
3758 
3759 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3760 
3761 		mutex_enter(&ahci_portp->ahciport_mutex);
3762 		ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3763 		mutex_exit(&ahci_portp->ahciport_mutex);
3764 	}
3765 }
3766 
3767 /*
3768  * Initialize the controller and all ports. And then try to start the ports
3769  * if there are devices attached.
3770  *
3771  * This routine can be called from three seperate cases: DDI_ATTACH,
3772  * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3773  * other two cases; device signature probing are attempted only during
3774  * DDI_ATTACH case.
3775  */
3776 static int
ahci_initialize_controller(ahci_ctl_t * ahci_ctlp)3777 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3778 {
3779 	ahci_port_t *ahci_portp;
3780 	ahci_addr_t addr;
3781 	int port;
3782 
3783 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3784 	    "ahci_initialize_controller enter", NULL);
3785 
3786 	/* Disable the whole controller interrupts */
3787 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3788 	ahci_disable_all_intrs(ahci_ctlp);
3789 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3790 
3791 	/* Initialize the implemented ports and structures */
3792 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3793 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3794 			continue;
3795 		}
3796 
3797 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3798 		mutex_enter(&ahci_portp->ahciport_mutex);
3799 
3800 		/*
3801 		 * Ensure that the controller is not in the running state
3802 		 * by checking every implemented port's PxCMD register
3803 		 */
3804 		AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3805 
3806 		if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3807 		    != AHCI_SUCCESS) {
3808 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3809 			    "ahci_initialize_controller: failed to "
3810 			    "initialize port %d", port);
3811 			/*
3812 			 * Set the port state to SATA_PSTATE_FAILED if
3813 			 * failed to initialize it.
3814 			 */
3815 			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3816 		}
3817 
3818 		mutex_exit(&ahci_portp->ahciport_mutex);
3819 	}
3820 
3821 	/* Enable the whole controller interrupts */
3822 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3823 	ahci_enable_all_intrs(ahci_ctlp);
3824 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3825 
3826 	return (AHCI_SUCCESS);
3827 }
3828 
3829 /*
3830  * Reverse of ahci_initialize_controller()
3831  *
3832  * We only need to stop the ports and disable the interrupt.
3833  */
3834 static void
ahci_uninitialize_controller(ahci_ctl_t * ahci_ctlp)3835 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3836 {
3837 	ahci_port_t *ahci_portp;
3838 	int port;
3839 
3840 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3841 	    "ahci_uninitialize_controller enter", NULL);
3842 
3843 	/* disable all the interrupts. */
3844 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3845 	ahci_disable_all_intrs(ahci_ctlp);
3846 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3847 
3848 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3849 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3850 			continue;
3851 		}
3852 
3853 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3854 
3855 		/* Stop the port by clearing PxCMD.ST */
3856 		mutex_enter(&ahci_portp->ahciport_mutex);
3857 
3858 		/*
3859 		 * Here we must disable the port interrupt because
3860 		 * ahci_disable_all_intrs only clear GHC.IE, and IS
3861 		 * register will be still set if PxIE is enabled.
3862 		 * When ahci shares one IRQ with other drivers, the
3863 		 * intr handler may claim the intr mistakenly.
3864 		 */
3865 		ahci_disable_port_intrs(ahci_ctlp, port);
3866 		(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3867 		    ahci_portp, port);
3868 		mutex_exit(&ahci_portp->ahciport_mutex);
3869 	}
3870 }
3871 
3872 /*
3873  * ahci_alloc_pmult()
3874  * 1. Setting HBA port registers which are necessary for a port multiplier.
3875  *    (Set PxCMD.PMA while PxCMD.ST is '0')
3876  * 2. Allocate ahci_pmult_info structure.
3877  *
3878  * NOTE: Must stop port before the function is called.
3879  */
3880 static void
ahci_alloc_pmult(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)3881 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3882 {
3883 	uint32_t port_cmd_status;
3884 	uint8_t port = ahci_portp->ahciport_port_num;
3885 
3886 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3887 
3888 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3889 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3890 
3891 	/* The port must have been stopped before. */
3892 	ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3893 
3894 	if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3895 		/* set PMA bit */
3896 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3897 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3898 		    port_cmd_status|AHCI_CMD_STATUS_PMA);
3899 
3900 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3901 		    "ahci_alloc_pmult: "
3902 		    "PxCMD.PMA bit set at port %d.", port);
3903 	}
3904 
3905 	/* Allocate port multiplier information structure */
3906 	if (ahci_portp->ahciport_pmult_info == NULL) {
3907 		ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3908 		    kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3909 	}
3910 
3911 	ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3912 }
3913 
3914 /*
3915  * ahci_dealloc_pmult()
3916  * 1. Clearing related registers when a port multiplier is detached.
3917  *    (Clear PxCMD.PMA while PxCMD.ST is '0')
3918  * 2. Deallocate ahci_pmult_info structure.
3919  *
3920  * NOTE: Must stop port before the function is called.
3921  */
3922 static void
ahci_dealloc_pmult(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)3923 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3924 {
3925 	uint32_t port_cmd_status;
3926 	uint8_t port = ahci_portp->ahciport_port_num;
3927 
3928 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3929 
3930 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3931 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3932 
3933 	if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3934 		/* Clear PMA bit */
3935 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3936 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3937 		    (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3938 
3939 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3940 		    "ahci_dealloc_pmult: "
3941 		    "PxCMD.PMA bit cleared at port %d.", port);
3942 	}
3943 
3944 	/* Release port multiplier information structure */
3945 	if (ahci_portp->ahciport_pmult_info != NULL) {
3946 		kmem_free(ahci_portp->ahciport_pmult_info,
3947 		    sizeof (ahci_pmult_info_t));
3948 		ahci_portp->ahciport_pmult_info = NULL;
3949 	}
3950 }
3951 
3952 /*
3953  * Staggered Spin-up.
3954  */
3955 static void
ahci_staggered_spin_up(ahci_ctl_t * ahci_ctlp,uint8_t port)3956 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3957 {
3958 	uint32_t cap_status;
3959 	uint32_t port_cmd_status;
3960 
3961 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3962 
3963 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3964 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3965 
3966 	/* Check for staggered spin-up support */
3967 	if (!(cap_status & AHCI_HBA_CAP_SSS))
3968 		return;
3969 
3970 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3971 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3972 
3973 	/* If PxCMD.SUD == 1, no staggered spin-up is needed */
3974 	if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3975 		return;
3976 
3977 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3978 
3979 	/* Set PxCMD.SUD */
3980 	port_cmd_status |= AHCI_CMD_STATUS_SUD;
3981 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3982 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3983 	    port_cmd_status);
3984 }
3985 
3986 /*
3987  * The routine is to initialize a port. First put the port in NotRunning
3988  * state, then enable port interrupt and clear Serror register. And under
3989  * AHCI_ATTACH case, find device signature and then try to start the port.
3990  *
3991  * Called by
3992  *    1. ahci_initialize_controller
3993  *    2. ahci_intr_phyrdy_change (hotplug)
3994  */
3995 static int
ahci_initialize_port(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)3996 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3997     ahci_addr_t *addrp)
3998 {
3999 	uint32_t port_sstatus, port_task_file, port_cmd_status;
4000 	uint8_t port = addrp->aa_port;
4001 	boolean_t resuming = B_TRUE;	/*  processing DDI_RESUME */
4002 	int ret;
4003 
4004 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4005 
4006 	/* AHCI_ADDR_PORT: We've no idea of the attached device here.  */
4007 	ASSERT(AHCI_ADDR_IS_PORT(addrp));
4008 
4009 	/*
4010 	 * At the time being, only probe ports/devices and get the types of
4011 	 * attached devices during DDI_ATTACH. In fact, the device can be
4012 	 * changed during power state changes, but at the time being, we
4013 	 * don't support the situation.
4014 	 */
4015 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4016 		resuming = B_FALSE;
4017 	} else {
4018 		/* check for DDI_RESUME case */
4019 		mutex_exit(&ahci_portp->ahciport_mutex);
4020 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
4021 		if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
4022 			resuming = B_FALSE;
4023 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
4024 		mutex_enter(&ahci_portp->ahciport_mutex);
4025 	}
4026 
4027 	if (resuming) {
4028 		/*
4029 		 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
4030 		 * and PxFBU registers in case these registers were cleared
4031 		 * during the suspend.
4032 		 */
4033 		AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4034 		    "ahci_initialize_port: port %d "
4035 		    "set PxCLB, PxCLBU, PxFB and PxFBU "
4036 		    "during resume", port);
4037 
4038 		if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
4039 		    AHCI_SUCCESS)
4040 			return (AHCI_FAILURE);
4041 	}
4042 
4043 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4044 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4045 
4046 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4047 	    "ahci_initialize_port: port %d ", port);
4048 
4049 	/*
4050 	 * Check whether the port is in NotRunning state, if not,
4051 	 * put the port in NotRunning state
4052 	 */
4053 	if (port_cmd_status &
4054 	    (AHCI_CMD_STATUS_ST |
4055 	    AHCI_CMD_STATUS_CR |
4056 	    AHCI_CMD_STATUS_FRE |
4057 	    AHCI_CMD_STATUS_FR)) {
4058 		(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
4059 		    ahci_portp, port);
4060 	}
4061 
4062 	/* Make sure the drive is spun-up */
4063 	ahci_staggered_spin_up(ahci_ctlp, port);
4064 
4065 	/* Disable interrupt */
4066 	ahci_disable_port_intrs(ahci_ctlp, port);
4067 
4068 	/* Device is unknown at first */
4069 	AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4070 
4071 	/* Disable the interface power management */
4072 	ahci_disable_interface_pm(ahci_ctlp, port);
4073 
4074 	port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4075 	    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
4076 	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4077 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4078 
4079 	/* Check physcial link status */
4080 	if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
4081 	    SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
4082 
4083 	    /* Check interface status */
4084 	    port_task_file & AHCI_TFD_STS_BSY ||
4085 	    port_task_file & AHCI_TFD_STS_DRQ ||
4086 
4087 	    /* Check whether port reset must be executed */
4088 	    ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
4089 
4090 	    /* Always reset port on RESUME */
4091 	    resuming != B_FALSE) {
4092 
4093 		/* Something went wrong, we need do some reset things */
4094 		ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
4095 
4096 		/* Does port reset succeed on HBA port? */
4097 		if (ret != AHCI_SUCCESS) {
4098 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4099 			    "ahci_initialize_port:"
4100 			    "port reset failed at port %d", port);
4101 			return (AHCI_FAILURE);
4102 		}
4103 
4104 		/* Is port failed? */
4105 		if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4106 		    SATA_PSTATE_FAILED) {
4107 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4108 			    "ahci_initialize_port: port %d state 0x%x",
4109 			    port, ahci_portp->ahciport_port_state);
4110 			return (AHCI_FAILURE);
4111 		}
4112 	}
4113 
4114 	AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4115 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4116 
4117 	/*
4118 	 * Try to get the device signature if the port is not empty.
4119 	 */
4120 	if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4121 	    SATA_DTYPE_NONE)
4122 		ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4123 
4124 	/* Return directly if no device connected */
4125 	if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4126 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4127 		    "No device connected to port %d", port);
4128 		goto out;
4129 	}
4130 
4131 	/* If this is a port multiplier, we need do some initialization */
4132 	if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4133 		AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4134 		    "Port multiplier found at port %d", port);
4135 		ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4136 	}
4137 
4138 	/* Try to start the port */
4139 	if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4140 	    != AHCI_SUCCESS) {
4141 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4142 		    "failed to start port %d", port);
4143 		return (AHCI_FAILURE);
4144 	}
4145 out:
4146 	/* Enable port interrupts */
4147 	ahci_enable_port_intrs(ahci_ctlp, port);
4148 
4149 	return (AHCI_SUCCESS);
4150 }
4151 
4152 /*
4153  *  Handle hardware defect, and check the capabilities. For example,
4154  *  power management capabilty and MSI capability.
4155  */
4156 static int
ahci_config_space_init(ahci_ctl_t * ahci_ctlp)4157 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4158 {
4159 	ushort_t caps_ptr, cap_count, cap;
4160 #if AHCI_DEBUG
4161 	ushort_t pmcap, pmcsr;
4162 	ushort_t msimc;
4163 #endif
4164 	uint8_t revision;
4165 
4166 	ahci_ctlp->ahcictl_venid =
4167 	    pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4168 	    PCI_CONF_VENID);
4169 
4170 	ahci_ctlp->ahcictl_devid =
4171 	    pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4172 	    PCI_CONF_DEVID);
4173 
4174 	/*
4175 	 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4176 	 * controllers with 0x00 revision id work on 4-byte aligned buffer,
4177 	 * which is a bug and was fixed after 0x00 revision id controllers.
4178 	 *
4179 	 * Moreover, VT8251 cannot use multiple command slots in the command
4180 	 * list for non-queued commands because the previous register content
4181 	 * of PxCI can be re-written in the register write, so a flag will be
4182 	 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4183 	 *
4184 	 * For VT8251, software reset also has the same defect as the below
4185 	 * AMD/ATI chipset. That is, software reset will get failed if 0xf
4186 	 * is filled in pmport field. Therefore, another software reset need
4187 	 * to be done with 0 filled in pmport field.
4188 	 */
4189 	if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4190 		revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4191 		    PCI_CONF_REVID);
4192 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4193 		    "revision id = 0x%x", revision);
4194 		if (revision == 0x00) {
4195 			ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4196 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4197 			    "change ddi_attr_align to 0x4", NULL);
4198 		}
4199 
4200 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4201 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4202 		    "VT8251 cannot use multiple command lists for "
4203 		    "non-queued commands", NULL);
4204 
4205 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4206 	}
4207 
4208 	/*
4209 	 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4210 	 * DMA addressing for communication memory descriptors though S64A bit
4211 	 * of CAP register declares it supports. Even though 64-bit DMA for
4212 	 * data buffer works on ASUS M2A-VM with newer BIOS, three other
4213 	 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4214 	 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4215 	 *
4216 	 * Due to certain hardware issue, the chipset must do port reset during
4217 	 * initialization, otherwise, when retrieving device signature,
4218 	 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4219 	 * need to set.
4220 	 *
4221 	 * For this chipset software reset will get failure if the pmport of
4222 	 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4223 	 * multiplier is connected to the port. In order to fix the issue,
4224 	 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4225 	 * reset got failure, the driver will try to do another software reset
4226 	 * with pmport 0.
4227 	 */
4228 	if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4229 	    ahci_ctlp->ahcictl_devid == 0x4380) {
4230 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4231 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4232 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4233 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4234 
4235 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4236 		    "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4237 		    "communication memory descriptors though CAP indicates "
4238 		    "support, so force it to use 32-bit DMA", NULL);
4239 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4240 		    "ATI SB600 need to do a port reset during initialization",
4241 		    NULL);
4242 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4243 		    "ATI SB600 will get software reset failure if pmport "
4244 		    "is set 0xf and no port multiplier is attached", NULL);
4245 	}
4246 
4247 	/*
4248 	 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4249 	 * vendor ID and device ID (0x1002,0x4391).
4250 	 *
4251 	 * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4252 	 * DMA addressing for communication memory descriptors though S64A bit
4253 	 * of CAP register declares the support. However, it does support
4254 	 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4255 	 * set for this controller.
4256 	 *
4257 	 * SB710 has the same initialization issue as SB600, so it also need
4258 	 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4259 	 *
4260 	 * SB700 also has the same issue about software reset, and thus
4261 	 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4262 	 */
4263 	if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4264 	    ahci_ctlp->ahcictl_devid == 0x4391) {
4265 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4266 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4267 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4268 
4269 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4270 		    "ATI SB700/750 cannot do 64-bit DMA for communication "
4271 		    "memory descriptors though CAP indicates support, "
4272 		    "so force it to use 32-bit DMA", NULL);
4273 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4274 		    "ATI SB710 need to do a port reset during initialization",
4275 		    NULL);
4276 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4277 		    "ATI SB700 will get software reset failure if pmport "
4278 		    "is set 0xf and no port multiplier is attached", NULL);
4279 	}
4280 
4281 	/*
4282 	 * Check if capabilities list is supported and if so,
4283 	 * get initial capabilities pointer and clear bits 0,1.
4284 	 */
4285 	if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4286 	    PCI_CONF_STAT) & PCI_STAT_CAP) {
4287 		caps_ptr = P2ALIGN(pci_config_get8(
4288 		    ahci_ctlp->ahcictl_pci_conf_handle,
4289 		    PCI_CONF_CAP_PTR), 4);
4290 	} else {
4291 		caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4292 	}
4293 
4294 	/*
4295 	 * Walk capabilities if supported.
4296 	 */
4297 	for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4298 
4299 		/*
4300 		 * Check that we haven't exceeded the maximum number of
4301 		 * capabilities and that the pointer is in a valid range.
4302 		 */
4303 		if (++cap_count > PCI_CAP_MAX_PTR) {
4304 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4305 			    "too many device capabilities", NULL);
4306 			return (AHCI_FAILURE);
4307 		}
4308 		if (caps_ptr < PCI_CAP_PTR_OFF) {
4309 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4310 			    "capabilities pointer 0x%x out of range",
4311 			    caps_ptr);
4312 			return (AHCI_FAILURE);
4313 		}
4314 
4315 		/*
4316 		 * Get next capability and check that it is valid.
4317 		 * For now, we only support power management.
4318 		 */
4319 		cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4320 		    caps_ptr);
4321 		switch (cap) {
4322 		case PCI_CAP_ID_PM:
4323 
4324 			/* power management supported */
4325 			ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4326 
4327 			/* Save PMCSR offset */
4328 			ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4329 
4330 #if AHCI_DEBUG
4331 			pmcap = pci_config_get16(
4332 			    ahci_ctlp->ahcictl_pci_conf_handle,
4333 			    caps_ptr + PCI_PMCAP);
4334 			pmcsr = pci_config_get16(
4335 			    ahci_ctlp->ahcictl_pci_conf_handle,
4336 			    ahci_ctlp->ahcictl_pmcsr_offset);
4337 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4338 			    "Power Management capability found PCI_PMCAP "
4339 			    "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4340 			if ((pmcap & 0x3) == 0x3)
4341 				AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4342 				    "PCI Power Management Interface "
4343 				    "spec 1.2 compliant", NULL);
4344 #endif
4345 			break;
4346 
4347 		case PCI_CAP_ID_MSI:
4348 #if AHCI_DEBUG
4349 			msimc = pci_config_get16(
4350 			    ahci_ctlp->ahcictl_pci_conf_handle,
4351 			    caps_ptr + PCI_MSI_CTRL);
4352 			AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4353 			    "Message Signaled Interrupt capability found "
4354 			    "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4355 #endif
4356 			AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4357 			    "MSI capability found", NULL);
4358 			break;
4359 
4360 		case PCI_CAP_ID_PCIX:
4361 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4362 			    "PCI-X capability found", NULL);
4363 			break;
4364 
4365 		case PCI_CAP_ID_PCI_E:
4366 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4367 			    "PCI Express capability found", NULL);
4368 			break;
4369 
4370 		case PCI_CAP_ID_MSI_X:
4371 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4372 			    "MSI-X capability found", NULL);
4373 			break;
4374 
4375 		case PCI_CAP_ID_SATA:
4376 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4377 			    "SATA capability found", NULL);
4378 			break;
4379 
4380 		case PCI_CAP_ID_VS:
4381 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4382 			    "Vendor Specific capability found", NULL);
4383 			break;
4384 
4385 		default:
4386 			AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4387 			    "unrecognized capability 0x%x", cap);
4388 			break;
4389 		}
4390 
4391 		/*
4392 		 * Get next capabilities pointer and clear bits 0,1.
4393 		 */
4394 		caps_ptr = P2ALIGN(pci_config_get8(
4395 		    ahci_ctlp->ahcictl_pci_conf_handle,
4396 		    (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4397 	}
4398 
4399 	return (AHCI_SUCCESS);
4400 }
4401 
4402 /*
4403  * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4404  * PORTMULT command. SYNC & POLLING mode is used.
4405  */
4406 static int
ahci_rdwr_pmult(ahci_ctl_t * ahci_ctlp,ahci_addr_t * addrp,uint8_t regn,uint32_t * pregv,uint8_t type)4407 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4408     uint8_t regn, uint32_t *pregv, uint8_t type)
4409 {
4410 	ahci_port_t *ahci_portp;
4411 	ahci_addr_t pmult_addr;
4412 	sata_pkt_t *spkt;
4413 	sata_cmd_t *scmd;
4414 	sata_device_t sata_device;
4415 	uint8_t port = addrp->aa_port;
4416 	uint8_t pmport = addrp->aa_pmport;
4417 	uint8_t cport;
4418 	uint32_t intr_mask;
4419 	int rval;
4420 	char portstr[10];
4421 
4422 	SET_PORTSTR(portstr, addrp);
4423 	cport = ahci_ctlp->ahcictl_port_to_cport[port];
4424 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
4425 
4426 	ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4427 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4428 
4429 	/* Check the existence of the port multiplier */
4430 	if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4431 		return (AHCI_FAILURE);
4432 
4433 	/* Request a READ/WRITE PORTMULT sata packet. */
4434 	bzero(&sata_device, sizeof (sata_device_t));
4435 	sata_device.satadev_addr.cport = cport;
4436 	sata_device.satadev_addr.pmport = pmport;
4437 	sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4438 	sata_device.satadev_rev = SATA_DEVICE_REV;
4439 
4440 	/*
4441 	 * Make sure no command is outstanding here. All R/W PMULT requests
4442 	 * come from
4443 	 *
4444 	 * 1. ahci_attach()
4445 	 *    The port should be empty.
4446 	 *
4447 	 * 2. ahci_tran_probe_port()
4448 	 *    Any request from SATA framework (via ahci_tran_start) should be
4449 	 *    rejected if R/W PMULT command is outstanding.
4450 	 *
4451 	 *    If we are doing mopping, do not check those flags because no
4452 	 *    command will be actually outstanding.
4453 	 *
4454 	 *    If the port has been occupied by any other commands, the probe
4455 	 *    function will return a SATA_RETRY. SATA framework will retry
4456 	 *    later.
4457 	 */
4458 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4459 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4460 		    "R/W PMULT failed: R/W PMULT in progress at port %d.",
4461 		    port, ahci_portp->ahciport_flags);
4462 		return (AHCI_FAILURE);
4463 	}
4464 
4465 	if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4466 	    ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4467 	    NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4468 	    NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4469 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4470 		    "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4471 		    port, ahci_portp->ahciport_flags);
4472 		return (AHCI_FAILURE);
4473 	}
4474 
4475 	/*
4476 	 * The port multiplier is gone. This may happen when
4477 	 * 1. Cutting off the power of an enclosure. The device lose the power
4478 	 *    before port multiplier.
4479 	 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4480 	 *
4481 	 * The issued command should be aborted and the following command
4482 	 * should not be continued.
4483 	 */
4484 	if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4485 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4486 		    "READ/WRITE PMULT failed: "
4487 		    "port-mult is removed from port %d", port);
4488 		return (AHCI_FAILURE);
4489 	}
4490 
4491 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4492 
4493 	spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4494 	    &sata_device, regn, *pregv, type);
4495 
4496 	/*
4497 	 * READ/WRITE PORTMULT command is intended to sent to the control port
4498 	 * of the port multiplier.
4499 	 */
4500 	AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4501 
4502 	ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4503 
4504 	/* No interrupt here. Store the interrupt enable mask. */
4505 	intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4506 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4507 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4508 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4509 
4510 	rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4511 
4512 	if (rval == AHCI_SUCCESS &&
4513 	    spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4514 		if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4515 			scmd = &spkt->satapkt_cmd;
4516 			*pregv = scmd->satacmd_lba_high_lsb << 24 |
4517 			    scmd->satacmd_lba_mid_lsb << 16 |
4518 			    scmd->satacmd_lba_low_lsb << 8 |
4519 			    scmd->satacmd_sec_count_lsb;
4520 		}
4521 	} else {
4522 		/* Failed or not completed. */
4523 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4524 		    "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4525 		    type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4526 		    AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4527 		rval = AHCI_FAILURE;
4528 	}
4529 out:
4530 	/* Restore the interrupt mask */
4531 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4532 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4533 
4534 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4535 	ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4536 	sata_free_rdwr_pmult_pkt(spkt);
4537 	return (rval);
4538 }
4539 
4540 static int
ahci_read_pmult(ahci_ctl_t * ahci_ctlp,ahci_addr_t * addrp,uint8_t regn,uint32_t * pregv)4541 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4542     uint8_t regn, uint32_t *pregv)
4543 {
4544 	return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4545 	    SATA_RDWR_PMULT_PKT_TYPE_READ);
4546 }
4547 
4548 static int
ahci_write_pmult(ahci_ctl_t * ahci_ctlp,ahci_addr_t * addrp,uint8_t regn,uint32_t regv)4549 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4550     uint8_t regn, uint32_t regv)
4551 {
4552 	return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, &regv,
4553 	    SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4554 }
4555 
4556 #define	READ_PMULT(addrp, r, pv, out)					\
4557 	if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS)	\
4558 		goto out;
4559 
4560 #define	WRITE_PMULT(addrp, r, v, out)					\
4561 	if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS)	\
4562 		goto out;
4563 
4564 /*
4565  * Update sata registers on port multiplier, including GSCR/PSCR registers.
4566  * ahci_update_pmult_gscr()
4567  * ahci_update_pmult_pscr()
4568  */
4569 static int
ahci_update_pmult_gscr(ahci_ctl_t * ahci_ctlp,ahci_addr_t * addrp,sata_pmult_gscr_t * sg)4570 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4571     sata_pmult_gscr_t *sg)
4572 {
4573 	ASSERT(MUTEX_HELD(
4574 	    &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4575 
4576 	READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4577 	READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4578 	READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4579 	READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4580 
4581 	return (AHCI_SUCCESS);
4582 
4583 err:	/* R/W PMULT error */
4584 	return (AHCI_FAILURE);
4585 }
4586 
4587 static int
ahci_update_pmult_pscr(ahci_ctl_t * ahci_ctlp,ahci_addr_t * addrp,sata_device_t * sd)4588 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4589     sata_device_t *sd)
4590 {
4591 	ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4592 	ASSERT(MUTEX_HELD(
4593 	    &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4594 
4595 	READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4596 	READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4597 	READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4598 	READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4599 
4600 	return (AHCI_SUCCESS);
4601 
4602 err:	/* R/W PMULT error */
4603 	return (AHCI_FAILURE);
4604 }
4605 
4606 /*
4607  * ahci_initialize_pmult()
4608  *
4609  * Initialize a port multiplier, including
4610  * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4611  * 2. Redefine MASK register. (SATA Chap 16.?)
4612  */
4613 static int
ahci_initialize_pmult(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_device_t * sd)4614 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4615     ahci_addr_t *addrp, sata_device_t *sd)
4616 {
4617 	sata_pmult_gscr_t sg;
4618 	uint32_t gscr64;
4619 	uint8_t port = addrp->aa_port;
4620 
4621 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4622 
4623 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4624 	    "[Initialize] Port-multiplier at port %d.", port);
4625 
4626 	/*
4627 	 * Enable features of port multiplier. Currently only
4628 	 * Asynchronous Notification is enabled.
4629 	 */
4630 	/* Check gscr64 for supported features. */
4631 	READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4632 
4633 	if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4634 		AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4635 		    "port %d: Port Multiplier supports "
4636 		    "Asynchronous Notification.", port);
4637 
4638 		/* Write to gscr96 to enabled features */
4639 		WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4640 		    SATA_PMULT_CAP_SNOTIF, err);
4641 
4642 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4643 		    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4644 		    AHCI_SNOTIF_CLEAR_ALL);
4645 		AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4646 		    "port %d: PMult PxSNTF cleared.", port);
4647 
4648 	}
4649 
4650 	/*
4651 	 * Now we need to update gscr33 register to enable hot-plug interrupt
4652 	 * for sub devices behind port multiplier.
4653 	 */
4654 	WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4655 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4656 	    "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4657 
4658 	/*
4659 	 * Fetch the number of device ports of the port multiplier
4660 	 */
4661 	if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4662 		return (AHCI_FAILURE);
4663 
4664 	/* Register the port multiplier to SATA Framework. */
4665 	mutex_exit(&ahci_portp->ahciport_mutex);
4666 	sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4667 	mutex_enter(&ahci_portp->ahciport_mutex);
4668 
4669 	ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4670 	    sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4671 
4672 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4673 	    "port %d: pmult sub-port number updated to %x.", port,
4674 	    ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4675 
4676 	/* Till now port-mult is successfully initialized */
4677 	ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4678 	return (AHCI_SUCCESS);
4679 
4680 err:	/* R/W PMULT error */
4681 	return (AHCI_FAILURE);
4682 }
4683 
4684 /*
4685  * Initialize a port multiplier port. According to spec, firstly we need
4686  * issue a COMRESET, then a software reset to get its signature.
4687  *
4688  * NOTE: This function should only be called in ahci_probe_pmport()
4689  */
4690 static int
ahci_initialize_pmport(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)4691 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4692     ahci_addr_t *addrp)
4693 {
4694 	uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4695 	uint8_t port = addrp->aa_port;
4696 	uint8_t pmport = addrp->aa_pmport;
4697 	int ret = AHCI_FAILURE;
4698 
4699 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4700 	ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4701 
4702 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4703 	    "ahci_initialize_pmport: port %d:%d", port, pmport);
4704 
4705 	/* Check HBA port state */
4706 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4707 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4708 		    "ahci_initialize_pmport:"
4709 		    "port %d:%d Port Multiplier is failed.",
4710 		    port, pmport);
4711 		return (AHCI_FAILURE);
4712 	}
4713 
4714 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4715 		return (AHCI_FAILURE);
4716 	}
4717 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4718 
4719 	/* Checking for outstanding commands */
4720 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4721 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4722 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4723 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4724 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4725 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4726 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4727 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4728 	}
4729 
4730 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4731 	ahci_portp->ahciport_mop_in_progress++;
4732 
4733 	/* Clear status */
4734 	AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4735 
4736 	/* Firstly assume an unknown device */
4737 	AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4738 
4739 	ahci_disable_port_intrs(ahci_ctlp, port);
4740 
4741 	/* port reset is necessary for port multiplier port */
4742 	if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4743 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4744 		    "ahci_initialize_pmport:"
4745 		    "port reset failed at port %d:%d",
4746 		    port, pmport);
4747 		goto out;
4748 	}
4749 
4750 	/* Is port failed? */
4751 	if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4752 	    SATA_PSTATE_FAILED) {
4753 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4754 		    "ahci_initialize_pmport: port %d:%d failed. "
4755 		    "state = 0x%x", port, pmport,
4756 		    ahci_portp->ahciport_port_state);
4757 		goto out;
4758 	}
4759 
4760 	/* Is there any device attached? */
4761 	if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4762 	    == SATA_DTYPE_NONE) {
4763 		/* Do not waste time on an empty port */
4764 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4765 		    "ahci_initialize_pmport: No device is found "
4766 		    "at port %d:%d", port, pmport);
4767 		ret = AHCI_SUCCESS;
4768 		goto out;
4769 	}
4770 
4771 	AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4772 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4773 	    "port %d:%d is ready now.", port, pmport);
4774 
4775 	/*
4776 	 * Till now we can assure a device attached to that HBA port and work
4777 	 * correctly. Now try to get the device signature. This is an optional
4778 	 * step. If failed, unknown device is assumed, then SATA module will
4779 	 * continue to use IDENTIFY DEVICE to get the information of the
4780 	 * device.
4781 	 */
4782 	ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4783 
4784 	ret = AHCI_SUCCESS;
4785 
4786 out:
4787 	/* Next try to mop the pending commands */
4788 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4789 		finished_tags = ahci_portp->ahciport_pending_tags &
4790 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4791 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4792 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4793 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4794 	reset_tags &= ~finished_tags;
4795 
4796 	ahci_mop_commands(ahci_ctlp,
4797 	    ahci_portp,
4798 	    slot_status,
4799 	    0, /* failed tags */
4800 	    0, /* timeout tags */
4801 	    0, /* aborted tags */
4802 	    reset_tags); /* reset tags */
4803 
4804 	/* Clear PxSNTF register if supported. */
4805 	if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4806 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4807 		    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4808 		    AHCI_SNOTIF_CLEAR_ALL);
4809 	}
4810 
4811 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4812 	ahci_enable_port_intrs(ahci_ctlp, port);
4813 	return (ret);
4814 }
4815 
4816 /*
4817  * ahci_probe_pmult()
4818  *
4819  * This function will be called to probe a port multiplier, which will
4820  * handle hotplug events on port multiplier ports.
4821  *
4822  * NOTE: Only called from ahci_tran_probe_port()
4823  */
4824 static int
ahci_probe_pmult(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)4825 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4826     ahci_addr_t *addrp)
4827 {
4828 	sata_device_t sdevice;
4829 	ahci_addr_t pmport_addr;
4830 	uint32_t gscr32, port_hotplug_tags;
4831 	uint32_t pmport_sstatus;
4832 	int dev_exists_now = 0, dev_existed_previously = 0;
4833 	uint8_t port = addrp->aa_port;
4834 	int npmport;
4835 
4836 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4837 
4838 	/* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4839 	READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4840 	port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4841 
4842 	do {
4843 		npmport = ddi_ffs(port_hotplug_tags) - 1;
4844 		if (npmport == -1)
4845 			/* no pending hot plug events. */
4846 			return (AHCI_SUCCESS);
4847 
4848 		AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4849 		    "hot-plug event at port %d:%d", port, npmport);
4850 
4851 		AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4852 
4853 		/* Check previous device at that port */
4854 		if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4855 		    != SATA_DTYPE_NONE)
4856 			dev_existed_previously = 1;
4857 
4858 		/* PxSStatus tells the presence of device. */
4859 		READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4860 		    &pmport_sstatus, err);
4861 
4862 		if (SSTATUS_GET_DET(pmport_sstatus) ==
4863 		    SSTATUS_DET_DEVPRE_PHYCOM)
4864 			dev_exists_now = 1;
4865 
4866 		/*
4867 		 * Clear PxSERR is critical. The transition from 0 to 1 will
4868 		 * emit a FIS which generates an asynchronous notification
4869 		 * event at controller. If we fail to clear the PxSERR, the
4870 		 * Async Notif events will no longer be activated on this
4871 		 * pmport.
4872 		 */
4873 		WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4874 		    AHCI_SERROR_CLEAR_ALL, err);
4875 
4876 		bzero((void *)&sdevice, sizeof (sata_device_t));
4877 		sdevice.satadev_addr.cport = ahci_ctlp->
4878 		    ahcictl_port_to_cport[port];
4879 		sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4880 		sdevice.satadev_addr.pmport = (uint8_t)npmport;
4881 		sdevice.satadev_state = SATA_PSTATE_PWRON;
4882 
4883 		AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4884 		    "[Existence] %d -> %d", dev_existed_previously,
4885 		    dev_exists_now);
4886 
4887 		if (dev_exists_now) {
4888 			if (dev_existed_previously) {
4889 				/* Link (may) not change: Exist -> Exist * */
4890 				AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4891 				    "ahci_probe_pmult: port %d:%d "
4892 				    "device link lost/established",
4893 				    port, npmport);
4894 
4895 				mutex_exit(&ahci_portp->ahciport_mutex);
4896 				sata_hba_event_notify(
4897 				    ahci_ctlp->ahcictl_sata_hba_tran->
4898 				    sata_tran_hba_dip,
4899 				    &sdevice,
4900 				    SATA_EVNT_LINK_LOST|
4901 				    SATA_EVNT_LINK_ESTABLISHED);
4902 				mutex_enter(&ahci_portp->ahciport_mutex);
4903 			} else {
4904 				/* Link change: None -> Exist */
4905 				AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4906 				    "ahci_probe_pmult: port %d:%d "
4907 				    "device link established", port, npmport);
4908 
4909 				/* Clear port state */
4910 				AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4911 				    SATA_STATE_UNKNOWN);
4912 				AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4913 				    "ahci_probe_pmult: port %d "
4914 				    "ahciport_port_state [Cleared].", port);
4915 
4916 				mutex_exit(&ahci_portp->ahciport_mutex);
4917 				sata_hba_event_notify(
4918 				    ahci_ctlp->ahcictl_sata_hba_tran->
4919 				    sata_tran_hba_dip,
4920 				    &sdevice,
4921 				    SATA_EVNT_LINK_ESTABLISHED);
4922 				mutex_enter(&ahci_portp->ahciport_mutex);
4923 			}
4924 		} else { /* No device exists now */
4925 			if (dev_existed_previously) {
4926 
4927 				/* Link change: Exist -> None */
4928 				AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4929 				    "ahci_probe_pmult: port %d:%d "
4930 				    "device link lost", port, npmport);
4931 
4932 				/* An existing device is lost. */
4933 				AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4934 				    SATA_STATE_UNKNOWN);
4935 				AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4936 				    SATA_DTYPE_NONE);
4937 
4938 				mutex_exit(&ahci_portp->ahciport_mutex);
4939 				sata_hba_event_notify(
4940 				    ahci_ctlp->ahcictl_sata_hba_tran->
4941 				    sata_tran_hba_dip,
4942 				    &sdevice,
4943 				    SATA_EVNT_LINK_LOST);
4944 				mutex_enter(&ahci_portp->ahciport_mutex);
4945 			}
4946 		}
4947 
4948 		CLEAR_BIT(port_hotplug_tags, npmport);
4949 	} while (port_hotplug_tags != 0);
4950 
4951 	return (AHCI_SUCCESS);
4952 
4953 err:	/* R/W PMULT error */
4954 	return (AHCI_FAILURE);
4955 }
4956 
4957 /*
4958  * Probe and initialize a port multiplier port.
4959  * A port multiplier port could only be initilaizer here.
4960  */
4961 static int
ahci_probe_pmport(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_device_t * sd)4962 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4963     ahci_addr_t *addrp, sata_device_t *sd)
4964 {
4965 	uint32_t port_state;
4966 	uint8_t port = addrp->aa_port;
4967 	ahci_addr_t addr_pmult;
4968 
4969 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4970 
4971 	/*
4972 	 * Check the parent - port multiplier first.
4973 	 */
4974 
4975 	/*
4976 	 * Parent port multiplier might have been removed. This event will be
4977 	 * ignored and failure.
4978 	 */
4979 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4980 	    ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4981 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4982 		    "ahci_tran_probe_port: "
4983 		    "parent device removed, ignore event.", NULL);
4984 
4985 		return (AHCI_FAILURE);
4986 	}
4987 
4988 	/* The port is ready? */
4989 	port_state = ahci_portp->ahciport_port_state;
4990 	if (!(port_state & SATA_STATE_READY)) {
4991 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4992 		    "ahci_tran_probe_port: "
4993 		    "parent port-mult is NOT ready.", NULL);
4994 
4995 		if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4996 		    ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4997 		    AHCI_SUCCESS) {
4998 			AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4999 			    "ahci_tran_probe_port: "
5000 			    "restart port-mult failed.", NULL);
5001 			return (AHCI_FAILURE);
5002 		}
5003 	}
5004 
5005 	/*
5006 	 * If port-mult is restarted due to some reason, we need
5007 	 * re-initialized the PMult.
5008 	 */
5009 	if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
5010 		/* Initialize registers on a port multiplier */
5011 		AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
5012 		if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
5013 		    &addr_pmult, sd) != AHCI_SUCCESS)
5014 			return (AHCI_FAILURE);
5015 	}
5016 
5017 	/*
5018 	 * Then we check the port-mult port
5019 	 */
5020 	/* Is this pmport initialized? */
5021 	port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5022 	if (!(port_state & SATA_STATE_READY)) {
5023 
5024 		/* ahci_initialize_pmport() will set READY state */
5025 		if (ahci_initialize_pmport(ahci_ctlp,
5026 		    ahci_portp, addrp) != AHCI_SUCCESS)
5027 			return (AHCI_FAILURE);
5028 	}
5029 
5030 	return (AHCI_SUCCESS);
5031 }
5032 
5033 /*
5034  * AHCI device reset ...; a single device on one of the ports is reset,
5035  * but the HBA and physical communication remain intact. This is the
5036  * least intrusive.
5037  *
5038  * When issuing a software reset sequence, there should not be other
5039  * commands in the command list, so we will first clear and then re-set
5040  * PxCMD.ST to clear PxCI. And before issuing the software reset,
5041  * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
5042  * cleared unless command list override (PxCMD.CLO) is supported.
5043  */
5044 static int
ahci_software_reset(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)5045 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5046     ahci_addr_t *addrp)
5047 {
5048 	ahci_fis_h2d_register_t *h2d_register_fisp;
5049 	ahci_cmd_table_t *cmd_table;
5050 	ahci_cmd_header_t *cmd_header;
5051 	uint32_t port_cmd_status, port_cmd_issue, port_task_file;
5052 	int slot, loop_count;
5053 	uint8_t port = addrp->aa_port;
5054 	uint8_t pmport = addrp->aa_pmport;
5055 	int rval = AHCI_FAILURE;
5056 
5057 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5058 
5059 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5060 	    "port %d:%d device software resetting (FIS)", port, pmport);
5061 
5062 	/* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
5063 	if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5064 	    port) != AHCI_SUCCESS) {
5065 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5066 		    "ahci_software_reset: cannot stop HBA port %d.", port);
5067 		goto out;
5068 	}
5069 
5070 	/* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
5071 	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5072 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5073 
5074 	if (port_task_file & AHCI_TFD_STS_BSY ||
5075 	    port_task_file & AHCI_TFD_STS_DRQ) {
5076 		if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
5077 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5078 			    "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
5079 			    "cannot issue a software reset.", port_task_file);
5080 			goto out;
5081 		}
5082 
5083 		/*
5084 		 * If HBA Support CLO, as Command List Override (CAP.SCLO is
5085 		 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
5086 		 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
5087 		 */
5088 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5089 		    "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
5090 
5091 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5092 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5093 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5094 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5095 		    port_cmd_status|AHCI_CMD_STATUS_CLO);
5096 
5097 		/* Waiting till PxCMD.SCLO bit is cleared */
5098 		loop_count = 0;
5099 		do {
5100 			/* Wait for 10 millisec */
5101 			drv_usecwait(AHCI_10MS_USECS);
5102 
5103 			/* We are effectively timing out after 1 sec. */
5104 			if (loop_count++ > 100) {
5105 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5106 				    "SCLO time out. port %d is busy.", port);
5107 				goto out;
5108 			}
5109 
5110 			port_cmd_status =
5111 			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5112 			    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5113 		} while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5114 
5115 		/* Re-check */
5116 		port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5117 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5118 		if (port_task_file & AHCI_TFD_STS_BSY ||
5119 		    port_task_file & AHCI_TFD_STS_DRQ) {
5120 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5121 			    "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5122 			    port_task_file);
5123 			goto out;
5124 		}
5125 	}
5126 
5127 	/* Then start port */
5128 	if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5129 	    != AHCI_SUCCESS) {
5130 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5131 		    "ahci_software_reset: cannot start AHCI port %d.", port);
5132 		goto out;
5133 	}
5134 
5135 	/*
5136 	 * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5137 	 * ahci_port.ahciport_pending_ncq_tags may fail
5138 	 * ahci_claim_free_slot(). Actually according to spec, by clearing
5139 	 * PxCMD.ST there is no command outstanding while executing software
5140 	 * reseting. Hence we directly use slot 0 instead of
5141 	 * ahci_claim_free_slot().
5142 	 */
5143 	slot = 0;
5144 
5145 	/* Now send the first H2D Register FIS with SRST set to 1 */
5146 	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5147 	bzero((void *)cmd_table, ahci_cmd_table_size);
5148 
5149 	h2d_register_fisp =
5150 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5151 
5152 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5153 	SET_FIS_PMP(h2d_register_fisp, pmport);
5154 	SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5155 
5156 	/* Set Command Header in Command List */
5157 	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5158 	BZERO_DESCR_INFO(cmd_header);
5159 	BZERO_PRD_BYTE_COUNT(cmd_header);
5160 	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5161 	SET_PORT_MULTI_PORT(cmd_header, pmport);
5162 
5163 	SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5164 	SET_RESET(cmd_header, 1);
5165 	SET_WRITE(cmd_header, 1);
5166 
5167 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5168 	    0,
5169 	    ahci_cmd_table_size,
5170 	    DDI_DMA_SYNC_FORDEV);
5171 
5172 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5173 	    slot * sizeof (ahci_cmd_header_t),
5174 	    sizeof (ahci_cmd_header_t),
5175 	    DDI_DMA_SYNC_FORDEV);
5176 
5177 	/* Indicate to the HBA that a command is active. */
5178 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5179 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5180 	    (0x1 << slot));
5181 
5182 	loop_count = 0;
5183 
5184 	/* Loop till the first command is finished */
5185 	do {
5186 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5187 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5188 
5189 		/* We are effectively timing out after 1 sec. */
5190 		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5191 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5192 			    "the first SRST FIS is timed out, "
5193 			    "loop_count = %d", loop_count);
5194 			goto out;
5195 		}
5196 		/* Wait for 10 millisec */
5197 		drv_usecwait(AHCI_10MS_USECS);
5198 	} while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5199 
5200 	AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5201 	    "ahci_software_reset: 1st loop count: %d, "
5202 	    "port_cmd_issue = 0x%x, slot = 0x%x",
5203 	    loop_count, port_cmd_issue, slot);
5204 
5205 	/* According to ATA spec, we need wait at least 5 microsecs here. */
5206 	drv_usecwait(AHCI_1MS_USECS);
5207 
5208 	/* Now send the second H2D Register FIS with SRST cleard to zero */
5209 	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5210 	bzero((void *)cmd_table, ahci_cmd_table_size);
5211 
5212 	h2d_register_fisp =
5213 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5214 
5215 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5216 	SET_FIS_PMP(h2d_register_fisp, pmport);
5217 
5218 	/* Set Command Header in Command List */
5219 	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5220 	BZERO_DESCR_INFO(cmd_header);
5221 	BZERO_PRD_BYTE_COUNT(cmd_header);
5222 	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5223 	SET_PORT_MULTI_PORT(cmd_header, pmport);
5224 
5225 	SET_WRITE(cmd_header, 1);
5226 
5227 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5228 	    0,
5229 	    ahci_cmd_table_size,
5230 	    DDI_DMA_SYNC_FORDEV);
5231 
5232 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5233 	    slot * sizeof (ahci_cmd_header_t),
5234 	    sizeof (ahci_cmd_header_t),
5235 	    DDI_DMA_SYNC_FORDEV);
5236 
5237 	/* Indicate to the HBA that a command is active. */
5238 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5239 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5240 	    (0x1 << slot));
5241 
5242 	loop_count = 0;
5243 
5244 	/* Loop till the second command is finished */
5245 	do {
5246 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5247 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5248 
5249 		/* We are effectively timing out after 1 sec. */
5250 		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5251 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5252 			    "the second SRST FIS is timed out, "
5253 			    "loop_count = %d", loop_count);
5254 			goto out;
5255 		}
5256 
5257 		/* Wait for 10 millisec */
5258 		drv_usecwait(AHCI_10MS_USECS);
5259 	} while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5260 
5261 	AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5262 	    "ahci_software_reset: 2nd loop count: %d, "
5263 	    "port_cmd_issue = 0x%x, slot = 0x%x",
5264 	    loop_count, port_cmd_issue, slot);
5265 
5266 	if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5267 	    (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5268 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5269 		    DDI_SERVICE_UNAFFECTED);
5270 		goto out;
5271 	}
5272 
5273 	rval = AHCI_SUCCESS;
5274 out:
5275 	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5276 	    "ahci_software_reset: %s at port %d:%d",
5277 	    rval == AHCI_SUCCESS ? "succeed" : "failed",
5278 	    port, pmport);
5279 
5280 	return (rval);
5281 }
5282 
5283 /*
5284  * AHCI port reset ...; the physical communication between the HBA and device
5285  * on a port are disabled. This is more intrusive.
5286  *
5287  * When an HBA or port reset occurs, Phy communication is going to
5288  * be re-established with the device through a COMRESET followed by the
5289  * normal out-of-band communication sequence defined in Serial ATA. At
5290  * the end of reset, the device, if working properly, will send a D2H
5291  * Register FIS, which contains the device signature. When the HBA receives
5292  * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5293  * the PxSIG register with the signature.
5294  *
5295  * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5296  * function is called. If not, it is assumed the interface is in hung
5297  * condition.
5298  */
5299 static int
ahci_port_reset(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)5300 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5301     ahci_addr_t *addrp)
5302 {
5303 	ahci_addr_t pmult_addr;
5304 	uint32_t port_cmd_status;
5305 	uint32_t port_scontrol, port_sstatus;
5306 	uint32_t port_task_file;
5307 	uint32_t port_state;
5308 	uint8_t port = addrp->aa_port;
5309 
5310 	int loop_count;
5311 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5312 
5313 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5314 
5315 	/* Target is a port multiplier port? */
5316 	if (AHCI_ADDR_IS_PMPORT(addrp))
5317 		return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5318 
5319 	/* Otherwise it must be an HBA port. */
5320 	ASSERT(AHCI_ADDR_IS_PORT(addrp));
5321 
5322 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5323 	    "Port %d port resetting...", port);
5324 	ahci_portp->ahciport_port_state = 0;
5325 
5326 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5327 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5328 
5329 	/*
5330 	 * According to the spec, SUD bit should be set here,
5331 	 * but JMicron JMB363 doesn't follow it, so print
5332 	 * a debug message.
5333 	 */
5334 	if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5335 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5336 		    "ahci_port_reset: port %d SUD bit not set", port);
5337 
5338 	port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5339 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5340 	SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5341 
5342 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5343 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5344 	    port_scontrol);
5345 
5346 	/* Enable PxCMD.FRE to read device */
5347 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5348 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5349 	    port_cmd_status|AHCI_CMD_STATUS_FRE);
5350 
5351 	/*
5352 	 * The port enters P:StartComm state, and the HBA tells the link layer
5353 	 * to start communication, which involves sending COMRESET to the
5354 	 * device. And the HBA resets PxTFD.STS to 7Fh.
5355 	 *
5356 	 * Give time for COMRESET to percolate, according to the AHCI
5357 	 * spec, software shall wait at least 1 millisecond before
5358 	 * clearing PxSCTL.DET
5359 	 */
5360 	drv_usecwait(AHCI_1MS_USECS * 2);
5361 
5362 	/* Fetch the SCONTROL again and rewrite the DET part with 0 */
5363 	port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5364 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5365 	SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5366 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5367 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5368 	    port_scontrol);
5369 
5370 	/*
5371 	 * When a COMINIT is received from the device, then the port enters
5372 	 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5373 	 * PxSSTS.DET to 1h to indicate a device is detected but communication
5374 	 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5375 	 * a COMINIT has been received.
5376 	 */
5377 	/*
5378 	 * The DET field is valid only if IPM field indicates
5379 	 * that the interface is in active state.
5380 	 */
5381 	loop_count = 0;
5382 	for (;;) {
5383 		port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5384 		    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5385 
5386 		if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5387 			/*
5388 			 * If the interface is not active, the DET field
5389 			 * is considered not accurate. So we want to
5390 			 * continue looping.
5391 			 */
5392 			SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5393 		}
5394 
5395 		if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5396 			break;
5397 
5398 		if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5399 			/*
5400 			 * We are effectively timing out after 0.1 sec.
5401 			 */
5402 			break;
5403 		}
5404 
5405 		/* Wait for 10 millisec */
5406 		drv_usecwait(AHCI_10MS_USECS);
5407 	}
5408 
5409 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5410 	    "ahci_port_reset: 1st loop count: %d, "
5411 	    "port_sstatus = 0x%x port %d",
5412 	    loop_count, port_sstatus, port);
5413 
5414 	if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5415 		/*
5416 		 * Either the port is not active or there
5417 		 * is no device present.
5418 		 */
5419 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5420 		return (AHCI_SUCCESS);
5421 	}
5422 
5423 	/* Clear port serror register for the port */
5424 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5425 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5426 	    AHCI_SERROR_CLEAR_ALL);
5427 
5428 	/*
5429 	 * Devices should return a FIS contains its signature to HBA after
5430 	 * COMINIT signal. Check whether a D2H Register FIS is received by
5431 	 * polling PxTFD.STS.
5432 	 */
5433 	loop_count = 0;
5434 	for (;;) {
5435 		port_task_file =
5436 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5437 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5438 
5439 		if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5440 		    AHCI_TFD_STS_ERR)) == 0)
5441 			break;
5442 
5443 		if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5444 			/*
5445 			 * We are effectively timing out after 11 sec.
5446 			 */
5447 			cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5448 			    "the device hardware has been initialized and "
5449 			    "the power-up diagnostics failed",
5450 			    instance, port);
5451 
5452 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5453 			    "port %d: some or all of BSY, DRQ and ERR in "
5454 			    "PxTFD.STS are not clear. We need another "
5455 			    "software reset.", port);
5456 
5457 			/* Clear port serror register for the port */
5458 			ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5459 			    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5460 			    AHCI_SERROR_CLEAR_ALL);
5461 
5462 			AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5463 
5464 			/* Try another software reset. */
5465 			if (ahci_software_reset(ahci_ctlp, ahci_portp,
5466 			    &pmult_addr) != AHCI_SUCCESS) {
5467 				AHCIPORT_SET_STATE(ahci_portp, addrp,
5468 				    SATA_PSTATE_FAILED);
5469 				return (AHCI_FAILURE);
5470 			}
5471 			break;
5472 		}
5473 
5474 		/* Wait for 10 millisec */
5475 		drv_usecwait(AHCI_10MS_USECS);
5476 	}
5477 
5478 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5479 	    "ahci_port_reset: 2nd loop count: %d, "
5480 	    "port_task_file = 0x%x port %d",
5481 	    loop_count, port_task_file, port);
5482 
5483 	/* Clear port serror register for the port */
5484 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5485 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5486 	    AHCI_SERROR_CLEAR_ALL);
5487 
5488 	/* Set port as ready */
5489 	port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5490 	AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5491 
5492 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5493 	    "ahci_port_reset: succeed at port %d.", port);
5494 	return (AHCI_SUCCESS);
5495 }
5496 
5497 /*
5498  * COMRESET on a port multiplier port.
5499  *
5500  * NOTE: Only called in ahci_port_reset()
5501  */
5502 static int
ahci_pmport_reset(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)5503 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5504     ahci_addr_t *addrp)
5505 {
5506 	uint32_t port_scontrol, port_sstatus, port_serror;
5507 	uint32_t port_cmd_status, port_intr_status;
5508 	uint32_t port_state;
5509 	uint8_t port = addrp->aa_port;
5510 	uint8_t pmport = addrp->aa_pmport;
5511 	int loop_count;
5512 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5513 
5514 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5515 	    "port %d:%d: pmport resetting", port, pmport);
5516 
5517 	/* Initialize pmport state */
5518 	AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5519 
5520 	READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5521 	SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5522 	WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5523 
5524 	/* PxCMD.FRE should be set before. */
5525 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5526 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5527 	ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5528 	if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5529 		return (AHCI_FAILURE);
5530 
5531 	/*
5532 	 * Give time for COMRESET to percolate, according to the AHCI
5533 	 * spec, software shall wait at least 1 millisecond before
5534 	 * clearing PxSCTL.DET
5535 	 */
5536 	drv_usecwait(AHCI_1MS_USECS*2);
5537 
5538 	/*
5539 	 * Fetch the SCONTROL again and rewrite the DET part with 0
5540 	 * This will generate an Asychronous Notification events.
5541 	 */
5542 	READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5543 	SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5544 	WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5545 
5546 	/*
5547 	 * The port enters P:StartComm state, and HBA tells link layer to
5548 	 * start communication, which involves sending COMRESET to device.
5549 	 * And the HBA resets PxTFD.STS to 7Fh.
5550 	 *
5551 	 * When a COMINIT is received from the device, then the port enters
5552 	 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5553 	 * PxSSTS.DET to 1h to indicate a device is detected but communication
5554 	 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5555 	 * a COMINIT has been received.
5556 	 */
5557 	/*
5558 	 * The DET field is valid only if IPM field indicates
5559 	 * that the interface is in active state.
5560 	 */
5561 	loop_count = 0;
5562 	do {
5563 		READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5564 
5565 		if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5566 			/*
5567 			 * If the interface is not active, the DET field
5568 			 * is considered not accurate. So we want to
5569 			 * continue looping.
5570 			 */
5571 			SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5572 		}
5573 
5574 		if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5575 			/*
5576 			 * We are effectively timing out after 0.1 sec.
5577 			 */
5578 			break;
5579 		}
5580 
5581 		/* Wait for 10 millisec */
5582 		drv_usecwait(AHCI_10MS_USECS);
5583 
5584 	} while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5585 
5586 	AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5587 	    "ahci_pmport_reset: 1st loop count: %d, "
5588 	    "port_sstatus = 0x%x port %d:%d",
5589 	    loop_count, port_sstatus, port, pmport);
5590 
5591 	if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5592 	    (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5593 		/*
5594 		 * Either the port is not active or there
5595 		 * is no device present.
5596 		 */
5597 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5598 		    "ahci_pmport_reset: "
5599 		    "no device attached to port %d:%d",
5600 		    port, pmport);
5601 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5602 		return (AHCI_SUCCESS);
5603 	}
5604 
5605 	/* Now we can make sure there is a device connected to the port */
5606 	/* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5607 	READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5608 
5609 	if (!(port_serror & (1 << 26))) {
5610 		cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5611 		    "COMINIT signal from the device not received port %d:%d",
5612 		    instance, port, pmport);
5613 
5614 		AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5615 		return (AHCI_FAILURE);
5616 	}
5617 
5618 	/*
5619 	 * After clear PxSERR register, we will receive a D2H FIS.
5620 	 * Normally this FIS will cause a IPMS error according to AHCI spec
5621 	 * v1.2 because there is no command outstanding for it. So we need
5622 	 * to ignore this error.
5623 	 */
5624 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5625 	WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5626 
5627 	/* Now we need to check the D2H FIS by checking IPMS error. */
5628 	loop_count = 0;
5629 	do {
5630 		port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5631 		    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5632 
5633 		if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5634 			/*
5635 			 * No D2H FIS received. This is possible according
5636 			 * to SATA 2.6 spec.
5637 			 */
5638 			cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5639 			    "PxIS.IPMS is not set, we need another "
5640 			    "software reset.", port, pmport);
5641 
5642 			break;
5643 		}
5644 
5645 		/* Wait for 10 millisec */
5646 		mutex_exit(&ahci_portp->ahciport_mutex);
5647 		delay(AHCI_10MS_TICKS);
5648 		mutex_enter(&ahci_portp->ahciport_mutex);
5649 
5650 	} while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5651 
5652 	AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5653 	    "ahci_pmport_reset: 2st loop count: %d, "
5654 	    "port_sstatus = 0x%x port %d:%d",
5655 	    loop_count, port_sstatus, port, pmport);
5656 
5657 	/* Clear IPMS */
5658 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5659 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5660 	    AHCI_INTR_STATUS_IPMS);
5661 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5662 
5663 	/* This pmport is now ready for ahci_tran_start() */
5664 	port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5665 	AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5666 
5667 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5668 	    "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5669 	return (AHCI_SUCCESS);
5670 
5671 err:	/* R/W PMULT error */
5672 	/* IPMS flags might be set before. */
5673 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5674 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5675 	    "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5676 
5677 	return (AHCI_FAILURE);
5678 }
5679 
5680 /*
5681  * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5682  * This is the most intrusive.
5683  *
5684  * When an HBA reset occurs, Phy communication will be re-established with
5685  * the device through a COMRESET followed by the normal out-of-band
5686  * communication sequence defined in Serial ATA. At the end of reset, the
5687  * device, if working properly, will send a D2H Register FIS, which contains
5688  * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5689  * and PxTFD.ERR register fields, and updates the PxSIG register with the
5690  * signature.
5691  *
5692  * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5693  */
5694 static int
ahci_hba_reset(ahci_ctl_t * ahci_ctlp)5695 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5696 {
5697 	ahci_port_t *ahci_portp;
5698 	uint32_t ghc_control;
5699 	uint8_t port;
5700 	int loop_count;
5701 	int rval = AHCI_SUCCESS;
5702 
5703 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5704 	    NULL);
5705 
5706 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
5707 
5708 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5709 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5710 
5711 	/* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5712 	ghc_control |= AHCI_HBA_GHC_HR;
5713 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5714 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5715 
5716 	/*
5717 	 * Wait until HBA Reset complete or timeout
5718 	 */
5719 	loop_count = 0;
5720 	do {
5721 		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5722 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5723 
5724 		if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5725 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5726 			    "ahci hba reset is timing out, "
5727 			    "ghc_control = 0x%x", ghc_control);
5728 			/* We are effectively timing out after 1 sec. */
5729 			break;
5730 		}
5731 
5732 		/* Wait for 10 millisec */
5733 		drv_usecwait(AHCI_10MS_USECS);
5734 	} while (ghc_control & AHCI_HBA_GHC_HR);
5735 
5736 	AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5737 	    "ahci_hba_reset: 1st loop count: %d, "
5738 	    "ghc_control = 0x%x", loop_count, ghc_control);
5739 
5740 	if (ghc_control & AHCI_HBA_GHC_HR) {
5741 		/* The hba is not reset for some reasons */
5742 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5743 		    "hba reset failed: HBA in a hung or locked state", NULL);
5744 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
5745 		return (AHCI_FAILURE);
5746 	}
5747 
5748 	/*
5749 	 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5750 	 */
5751 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5752 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5753 	ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5754 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5755 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5756 
5757 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
5758 
5759 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5760 		/* Only check implemented ports */
5761 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5762 			continue;
5763 		}
5764 
5765 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
5766 		mutex_enter(&ahci_portp->ahciport_mutex);
5767 
5768 		/* Make sure the drive is spun-up */
5769 		ahci_staggered_spin_up(ahci_ctlp, port);
5770 
5771 		if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5772 		    port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5773 		    AHCI_SUCCESS) {
5774 			rval = AHCI_FAILURE;
5775 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5776 			    "ahci_hba_reset: port %d failed", port);
5777 			/*
5778 			 * Set the port state to SATA_PSTATE_FAILED if
5779 			 * failed to initialize it.
5780 			 */
5781 			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5782 		}
5783 
5784 		mutex_exit(&ahci_portp->ahciport_mutex);
5785 	}
5786 
5787 	return (rval);
5788 }
5789 
5790 /*
5791  * This routine is only called from AHCI_ATTACH or phyrdy change
5792  * case. It first calls software reset, then stop the port and try to
5793  * read PxSIG register to find the type of device attached to the port.
5794  *
5795  * The caller should make sure a valid device exists on specified port and
5796  * physical communication has been established so that the signature could
5797  * be retrieved by software reset.
5798  *
5799  * NOTE: The port interrupts should be disabled before the function is called.
5800  */
5801 static void
ahci_find_dev_signature(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)5802 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5803     ahci_addr_t *addrp)
5804 {
5805 	ahci_addr_t dev_addr;
5806 	uint32_t signature;
5807 	uint8_t port = addrp->aa_port;
5808 	uint8_t pmport = addrp->aa_pmport;
5809 	int rval;
5810 
5811 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5812 	ASSERT(AHCI_ADDR_IS_VALID(addrp));
5813 
5814 	/*
5815 	 * If the HBA doesn't support port multiplier, then the driver
5816 	 * doesn't need to bother to check port multiplier device.
5817 	 *
5818 	 * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5819 	 * connected to Silicon Image 4723, to which the two sata drives
5820 	 * attached can be set with RAID1, RAID0 or Spanning mode.
5821 	 *
5822 	 * We found software reset will get failure if port multiplier address
5823 	 * 0xf is used by software reset, so just ignore the check since
5824 	 * ICH7 doesn't support port multiplier device at all.
5825 	 */
5826 	if (AHCI_ADDR_IS_PORT(addrp) &&
5827 	    (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5828 		AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5829 		    "ahci_find_dev_signature enter: port %d", port);
5830 
5831 		/*
5832 		 * NOTE: when the ahci address is a HBA port, we do not know
5833 		 * it is a device or a port multiplier that attached. we need
5834 		 * try a software reset at port multiplier address (0xf
5835 		 * pmport)
5836 		 */
5837 		AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5838 	} else {
5839 		AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5840 		    "ahci_find_dev_signature enter: port %d:%d",
5841 		    port, pmport);
5842 		dev_addr = *addrp;
5843 	}
5844 
5845 	/* Assume it is unknown. */
5846 	AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5847 
5848 	/* Issue a software reset to get the signature */
5849 	rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5850 	if (rval != AHCI_SUCCESS) {
5851 
5852 		/*
5853 		 * Try to do software reset again with pmport set with 0 if
5854 		 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5855 		 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5856 		 */
5857 		if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5858 		    (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5859 			dev_addr.aa_pmport = 0;
5860 			rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5861 			    &dev_addr);
5862 		}
5863 
5864 		if (rval != AHCI_SUCCESS) {
5865 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5866 			    "ahci_find_dev_signature: software reset failed "
5867 			    "at port %d:%d, cannot get signature.",
5868 			    port, pmport);
5869 
5870 			AHCIPORT_SET_STATE(ahci_portp, addrp,
5871 			    SATA_PSTATE_FAILED);
5872 			return;
5873 		}
5874 	}
5875 
5876 	/*
5877 	 * ahci_software_reset has started the port, so we need manually stop
5878 	 * the port again.
5879 	 */
5880 	if (AHCI_ADDR_IS_PORT(addrp)) {
5881 		if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5882 		    ahci_portp, port) != AHCI_SUCCESS) {
5883 			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5884 			    "ahci_find_dev_signature: cannot stop port %d.",
5885 			    port);
5886 			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5887 			return;
5888 		}
5889 	}
5890 
5891 	/* Now we can make sure that a valid signature is received. */
5892 	signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5893 	    (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5894 
5895 	if (AHCI_ADDR_IS_PMPORT(addrp)) {
5896 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5897 		    "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5898 		    signature, port, pmport);
5899 	} else {
5900 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5901 		    "ahci_find_dev_signature: signature = 0x%x at port %d",
5902 		    signature, port);
5903 	}
5904 
5905 	/* NOTE: Only support ATAPI device at controller port. */
5906 	if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5907 		signature = SATA_DTYPE_UNKNOWN;
5908 
5909 	switch (signature) {
5910 
5911 	case AHCI_SIGNATURE_DISK:
5912 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5913 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5914 		    "Disk is found at port: %d", port);
5915 		break;
5916 
5917 	case AHCI_SIGNATURE_ATAPI:
5918 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5919 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5920 		    "ATAPI device is found at port: %d", port);
5921 		break;
5922 
5923 	case AHCI_SIGNATURE_PORT_MULTIPLIER:
5924 		/* Port Multiplier cannot recursively attached. */
5925 		ASSERT(AHCI_ADDR_IS_PORT(addrp));
5926 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5927 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5928 		    "Port Multiplier is found at port: %d", port);
5929 		break;
5930 
5931 	default:
5932 		AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5933 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5934 		    "Unknown device is found at port: %d", port);
5935 	}
5936 }
5937 
5938 /*
5939  * According to the spec, to reliably detect hot plug removals, software
5940  * must disable interface power management. Software should perform the
5941  * following initialization on a port after a device is attached:
5942  *   Set PxSCTL.IPM to 3h or 7h to disable interface state transitions
5943  *   Set PxCMD.ALPE to '0' to disable aggressive power management
5944  *   Disable device initiated interface power management by SET FEATURE
5945  *
5946  * We can ignore the last item because by default the feature is disabled
5947  */
5948 static void
ahci_disable_interface_pm(ahci_ctl_t * ahci_ctlp,uint8_t port)5949 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5950 {
5951 	uint32_t port_scontrol, port_cmd_status;
5952 	uint32_t ipm;
5953 
5954 	port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5955 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5956 	if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SDS) {
5957 		ipm = SCONTROL_IPM_DISABLE_ALL;
5958 	} else {
5959 		ipm = SCONTROL_IPM_DISABLE_BOTH;
5960 	}
5961 	SCONTROL_SET_IPM(port_scontrol, ipm);
5962 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5963 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5964 
5965 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5966 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5967 	port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5968 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5969 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5970 }
5971 
5972 /*
5973  * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5974  * to 1, then set it firstly.
5975  *
5976  * Each port contains two major DMA engines. One DMA engine walks through
5977  * the command list, and is controlled by PxCMD.ST. The second DMA engine
5978  * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5979  *
5980  * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5981  * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5982  * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5983  *
5984  * Software shall not set PxCMD.ST to '1' unless a functional device is
5985  * present on the port(as determined by PxTFD.STS.BSY = '0',
5986  * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5987  */
5988 static int
ahci_start_port(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)5989 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5990 {
5991 	uint32_t port_cmd_status;
5992 
5993 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5994 
5995 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5996 
5997 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5998 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5999 		    "the state for port %d is 0x%x",
6000 		    port, ahci_portp->ahciport_port_state);
6001 		return (AHCI_FAILURE);
6002 	}
6003 
6004 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
6005 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
6006 		    "no device is attached at port %d", port);
6007 		return (AHCI_FAILURE);
6008 	}
6009 
6010 	/* First to set PxCMD.FRE before setting PxCMD.ST. */
6011 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6012 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6013 
6014 	if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
6015 		port_cmd_status |= AHCI_CMD_STATUS_FRE;
6016 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6017 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6018 		    port_cmd_status);
6019 	}
6020 
6021 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6022 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6023 
6024 	port_cmd_status |= AHCI_CMD_STATUS_ST;
6025 
6026 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6027 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6028 	    port_cmd_status);
6029 
6030 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
6031 
6032 	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
6033 	    "PxCMD.ST set to '1' at port %d", port);
6034 
6035 	return (AHCI_SUCCESS);
6036 }
6037 
6038 /*
6039  * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
6040  * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
6041  * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
6042  */
6043 static int
ahci_setup_port_base_addresses(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6044 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6045 {
6046 	uint8_t port = ahci_portp->ahciport_port_num;
6047 	uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6048 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6049 
6050 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6051 
6052 	/* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
6053 	if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
6054 		if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
6055 		    port) != AHCI_SUCCESS)
6056 			return (AHCI_FAILURE);
6057 
6058 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6059 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6060 	}
6061 
6062 	/* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
6063 	if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
6064 		int loop_count = 0;
6065 
6066 		/* Clear PxCMD.FRE */
6067 		port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
6068 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6069 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
6070 		    port_cmd_status);
6071 
6072 		/* Wait until PxCMD.FR is cleared */
6073 		for (;;) {
6074 			port_cmd_status =
6075 			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6076 			    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
6077 
6078 			if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
6079 				break;
6080 
6081 			if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
6082 				AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
6083 				    "ahci_setup_port_base_addresses: cannot "
6084 				    "clear PxCMD.FR for port %d.", port);
6085 
6086 				/*
6087 				 * We are effectively timing out after 0.5 sec.
6088 				 * This value is specified in AHCI spec.
6089 				 */
6090 				return (AHCI_FAILURE);
6091 			}
6092 
6093 			/* Wait for 1 millisec */
6094 			drv_usecwait(AHCI_1MS_USECS);
6095 		}
6096 	}
6097 
6098 	/* Step 3: Config Port Command List Base Address */
6099 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6100 	    (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
6101 	    ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6102 
6103 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6104 	    (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
6105 	    ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
6106 
6107 	/* Step 4: Config Port Received FIS Base Address */
6108 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6109 	    (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
6110 	    ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6111 
6112 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6113 	    (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6114 	    ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6115 
6116 	return (AHCI_SUCCESS);
6117 }
6118 
6119 /*
6120  * Allocate the ahci_port_t including Received FIS and Command List.
6121  * The argument - port is the physical port number, and not logical
6122  * port number seen by the SATA framework.
6123  */
6124 static int
ahci_alloc_port_state(ahci_ctl_t * ahci_ctlp,uint8_t port)6125 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6126 {
6127 	dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6128 	ahci_port_t *ahci_portp;
6129 	char taskq_name[64] = "event_handle_taskq";
6130 
6131 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6132 
6133 	ahci_portp =
6134 	    (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6135 
6136 	ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6137 	ahci_portp->ahciport_port_num = port;
6138 
6139 	/* Initialize the port condition variable */
6140 	cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6141 
6142 	/* Initialize the port mutex */
6143 	mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6144 	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6145 
6146 	mutex_enter(&ahci_portp->ahciport_mutex);
6147 
6148 	/*
6149 	 * Allocate memory for received FIS structure and
6150 	 * command list for this port
6151 	 */
6152 	if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6153 		goto err_case1;
6154 	}
6155 
6156 	if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6157 		goto err_case2;
6158 	}
6159 
6160 	/* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6161 	if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6162 	    AHCI_SUCCESS) {
6163 		goto err_case3;
6164 	}
6165 
6166 	(void) snprintf(taskq_name + strlen(taskq_name),
6167 	    sizeof (taskq_name) - strlen(taskq_name),
6168 	    "_port%d", port);
6169 
6170 	/* Create the taskq for the port */
6171 	if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6172 	    taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6173 		cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6174 		    "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6175 		goto err_case3;
6176 	}
6177 
6178 	/* Allocate the argument for the taskq */
6179 	ahci_portp->ahciport_event_args =
6180 	    kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6181 
6182 	ahci_portp->ahciport_event_args->ahciea_addrp =
6183 	    kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6184 
6185 	/* Initialize the done queue */
6186 	ahci_portp->ahciport_doneq = NULL;
6187 	ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6188 	ahci_portp->ahciport_doneq_len = 0;
6189 
6190 	mutex_exit(&ahci_portp->ahciport_mutex);
6191 
6192 	return (AHCI_SUCCESS);
6193 
6194 err_case3:
6195 	ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6196 
6197 err_case2:
6198 	ahci_dealloc_rcvd_fis(ahci_portp);
6199 
6200 err_case1:
6201 	mutex_exit(&ahci_portp->ahciport_mutex);
6202 	mutex_destroy(&ahci_portp->ahciport_mutex);
6203 	cv_destroy(&ahci_portp->ahciport_cv);
6204 
6205 	kmem_free(ahci_portp, sizeof (ahci_port_t));
6206 
6207 	return (AHCI_FAILURE);
6208 }
6209 
6210 /*
6211  * Reverse of ahci_alloc_port_state().
6212  */
6213 static void
ahci_dealloc_port_state(ahci_ctl_t * ahci_ctlp,uint8_t port)6214 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6215 {
6216 	ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6217 
6218 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6219 	ASSERT(ahci_portp != NULL);
6220 
6221 	mutex_enter(&ahci_portp->ahciport_mutex);
6222 	kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6223 	    sizeof (ahci_addr_t));
6224 	ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6225 	kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6226 	ahci_portp->ahciport_event_args = NULL;
6227 	ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6228 	ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6229 	ahci_dealloc_rcvd_fis(ahci_portp);
6230 	ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6231 	mutex_exit(&ahci_portp->ahciport_mutex);
6232 
6233 	mutex_destroy(&ahci_portp->ahciport_mutex);
6234 	cv_destroy(&ahci_portp->ahciport_cv);
6235 
6236 	kmem_free(ahci_portp, sizeof (ahci_port_t));
6237 
6238 	ahci_ctlp->ahcictl_ports[port] = NULL;
6239 }
6240 
6241 /*
6242  * Allocates memory for the Received FIS Structure
6243  */
6244 static int
ahci_alloc_rcvd_fis(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6245 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6246 {
6247 	size_t rcvd_fis_size;
6248 	size_t ret_len;
6249 	uint_t cookie_count;
6250 
6251 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6252 
6253 	rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6254 
6255 	/* allocate rcvd FIS dma handle. */
6256 	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6257 	    &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6258 	    DDI_DMA_SLEEP,
6259 	    NULL,
6260 	    &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6261 	    DDI_SUCCESS) {
6262 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6263 		    "rcvd FIS dma handle alloc failed", NULL);
6264 
6265 		return (AHCI_FAILURE);
6266 	}
6267 
6268 	if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6269 	    rcvd_fis_size,
6270 	    &accattr,
6271 	    DDI_DMA_CONSISTENT,
6272 	    DDI_DMA_SLEEP,
6273 	    NULL,
6274 	    (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6275 	    &ret_len,
6276 	    &ahci_portp->ahciport_rcvd_fis_acc_handle) != 0) {
6277 
6278 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6279 		    "rcvd FIS dma mem alloc fail", NULL);
6280 		/* error.. free the dma handle. */
6281 		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6282 		return (AHCI_FAILURE);
6283 	}
6284 
6285 	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6286 	    NULL,
6287 	    (caddr_t)ahci_portp->ahciport_rcvd_fis,
6288 	    rcvd_fis_size,
6289 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6290 	    DDI_DMA_SLEEP,
6291 	    NULL,
6292 	    &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6293 	    &cookie_count) !=  DDI_DMA_MAPPED) {
6294 
6295 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6296 		    "rcvd FIS dma handle bind fail", NULL);
6297 		/*  error.. free the dma handle & free the memory. */
6298 		ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6299 		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6300 		return (AHCI_FAILURE);
6301 	}
6302 
6303 	bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6304 
6305 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6306 	    ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6307 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6308 	    ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6309 
6310 	return (AHCI_SUCCESS);
6311 }
6312 
6313 /*
6314  * Deallocates the Received FIS Structure
6315  */
6316 static void
ahci_dealloc_rcvd_fis(ahci_port_t * ahci_portp)6317 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6318 {
6319 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6320 
6321 	/* Unbind the cmd list dma handle first. */
6322 	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6323 
6324 	/* Then free the underlying memory. */
6325 	ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6326 
6327 	/* Now free the handle itself. */
6328 	ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6329 }
6330 
6331 /*
6332  * Allocates memory for the Command List, which contains up to 32 entries.
6333  * Each entry contains a command header, which is a 32-byte structure that
6334  * includes the pointer to the command table.
6335  */
6336 static int
ahci_alloc_cmd_list(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6337 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6338 {
6339 	size_t cmd_list_size;
6340 	size_t ret_len;
6341 	uint_t cookie_count;
6342 
6343 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6344 
6345 	cmd_list_size =
6346 	    ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6347 
6348 	/* allocate cmd list dma handle. */
6349 	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6350 	    &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6351 	    DDI_DMA_SLEEP,
6352 	    NULL,
6353 	    &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6354 
6355 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6356 		    "cmd list dma handle alloc failed", NULL);
6357 		return (AHCI_FAILURE);
6358 	}
6359 
6360 	if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6361 	    cmd_list_size,
6362 	    &accattr,
6363 	    DDI_DMA_CONSISTENT,
6364 	    DDI_DMA_SLEEP,
6365 	    NULL,
6366 	    (caddr_t *)&ahci_portp->ahciport_cmd_list,
6367 	    &ret_len,
6368 	    &ahci_portp->ahciport_cmd_list_acc_handle) != 0) {
6369 
6370 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6371 		    "cmd list dma mem alloc fail", NULL);
6372 		/* error.. free the dma handle. */
6373 		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6374 		return (AHCI_FAILURE);
6375 	}
6376 
6377 	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6378 	    NULL,
6379 	    (caddr_t)ahci_portp->ahciport_cmd_list,
6380 	    cmd_list_size,
6381 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6382 	    DDI_DMA_SLEEP,
6383 	    NULL,
6384 	    &ahci_portp->ahciport_cmd_list_dma_cookie,
6385 	    &cookie_count) !=  DDI_DMA_MAPPED) {
6386 
6387 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6388 		    "cmd list dma handle bind fail", NULL);
6389 		/*  error.. free the dma handle & free the memory. */
6390 		ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6391 		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6392 		return (AHCI_FAILURE);
6393 	}
6394 
6395 	bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6396 
6397 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6398 	    ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6399 
6400 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6401 	    ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6402 
6403 	if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6404 		goto err_out;
6405 	}
6406 
6407 	return (AHCI_SUCCESS);
6408 
6409 err_out:
6410 	/* Unbind the cmd list dma handle first. */
6411 	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6412 
6413 	/* Then free the underlying memory. */
6414 	ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6415 
6416 	/* Now free the handle itself. */
6417 	ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6418 
6419 	return (AHCI_FAILURE);
6420 }
6421 
6422 /*
6423  * Deallocates the Command List
6424  */
6425 static void
ahci_dealloc_cmd_list(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6426 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6427 {
6428 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6429 
6430 	/* First dealloc command table */
6431 	ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6432 
6433 	/* Unbind the cmd list dma handle first. */
6434 	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6435 
6436 	/* Then free the underlying memory. */
6437 	ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6438 
6439 	/* Now free the handle itself. */
6440 	ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6441 }
6442 
6443 /*
6444  * Allocates memory for all Command Tables, which contains Command FIS,
6445  * ATAPI Command and Physical Region Descriptor Table.
6446  */
6447 static int
ahci_alloc_cmd_tables(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6448 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6449 {
6450 	size_t ret_len;
6451 	ddi_dma_cookie_t cmd_table_dma_cookie;
6452 	uint_t cookie_count;
6453 	int slot;
6454 
6455 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6456 
6457 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6458 	    "ahci_alloc_cmd_tables: port %d enter",
6459 	    ahci_portp->ahciport_port_num);
6460 
6461 	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6462 		/* Allocate cmd table dma handle. */
6463 		if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6464 		    &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6465 		    DDI_DMA_SLEEP,
6466 		    NULL,
6467 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6468 		    DDI_SUCCESS) {
6469 
6470 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6471 			    "cmd table dma handle alloc failed", NULL);
6472 
6473 			goto err_out;
6474 		}
6475 
6476 		if (ddi_dma_mem_alloc(
6477 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6478 		    ahci_cmd_table_size,
6479 		    &accattr,
6480 		    DDI_DMA_CONSISTENT,
6481 		    DDI_DMA_SLEEP,
6482 		    NULL,
6483 		    (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6484 		    &ret_len,
6485 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 0) {
6486 
6487 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6488 			    "cmd table dma mem alloc fail", NULL);
6489 
6490 			/* error.. free the dma handle. */
6491 			ddi_dma_free_handle(
6492 			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6493 			goto err_out;
6494 		}
6495 
6496 		if (ddi_dma_addr_bind_handle(
6497 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6498 		    NULL,
6499 		    (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6500 		    ahci_cmd_table_size,
6501 		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6502 		    DDI_DMA_SLEEP,
6503 		    NULL,
6504 		    &cmd_table_dma_cookie,
6505 		    &cookie_count) !=  DDI_DMA_MAPPED) {
6506 
6507 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6508 			    "cmd table dma handle bind fail", NULL);
6509 			/*  error.. free the dma handle & free the memory. */
6510 			ddi_dma_mem_free(
6511 			    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6512 			ddi_dma_free_handle(
6513 			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6514 			goto err_out;
6515 		}
6516 
6517 		bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6518 		    ahci_cmd_table_size);
6519 
6520 		/* Config Port Command Table Base Address */
6521 		SET_COMMAND_TABLE_BASE_ADDR(
6522 		    (&ahci_portp->ahciport_cmd_list[slot]),
6523 		    cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6524 
6525 #ifndef __lock_lint
6526 		SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6527 		    (&ahci_portp->ahciport_cmd_list[slot]),
6528 		    cmd_table_dma_cookie.dmac_laddress >> 32);
6529 #endif
6530 	}
6531 
6532 	return (AHCI_SUCCESS);
6533 err_out:
6534 
6535 	for (slot--; slot >= 0; slot--) {
6536 		/* Unbind the cmd table dma handle first */
6537 		(void) ddi_dma_unbind_handle(
6538 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6539 
6540 		/* Then free the underlying memory */
6541 		ddi_dma_mem_free(
6542 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6543 
6544 		/* Now free the handle itself */
6545 		ddi_dma_free_handle(
6546 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6547 	}
6548 
6549 	return (AHCI_FAILURE);
6550 }
6551 
6552 /*
6553  * Deallocates memory for all Command Tables.
6554  */
6555 static void
ahci_dealloc_cmd_tables(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp)6556 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6557 {
6558 	int slot;
6559 
6560 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6561 
6562 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6563 	    "ahci_dealloc_cmd_tables: %d enter",
6564 	    ahci_portp->ahciport_port_num);
6565 
6566 	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6567 		/* Unbind the cmd table dma handle first. */
6568 		(void) ddi_dma_unbind_handle(
6569 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6570 
6571 		/* Then free the underlying memory. */
6572 		ddi_dma_mem_free(
6573 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6574 
6575 		/* Now free the handle itself. */
6576 		ddi_dma_free_handle(
6577 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6578 	}
6579 }
6580 
6581 /*
6582  * Update SATA registers at controller ports
6583  */
6584 static void
ahci_update_sata_registers(ahci_ctl_t * ahci_ctlp,uint8_t port,sata_device_t * sd)6585 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6586     sata_device_t *sd)
6587 {
6588 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6589 
6590 	sd->satadev_scr.sstatus =
6591 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6592 	    (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6593 	sd->satadev_scr.serror =
6594 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6595 	    (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6596 	sd->satadev_scr.scontrol =
6597 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6598 	    (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6599 	sd->satadev_scr.sactive =
6600 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6601 	    (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6602 }
6603 
6604 /*
6605  * For poll mode, ahci_port_intr will be called to emulate the interrupt
6606  */
6607 static void
ahci_port_intr(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)6608 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6609 {
6610 	uint32_t port_intr_status;
6611 	uint32_t port_intr_enable;
6612 
6613 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6614 	    "ahci_port_intr enter: port %d", port);
6615 
6616 	mutex_enter(&ahci_portp->ahciport_mutex);
6617 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6618 		/* For SATA_OPMODE_POLLING commands */
6619 		port_intr_enable =
6620 		    (AHCI_INTR_STATUS_DHRS |
6621 		    AHCI_INTR_STATUS_PSS |
6622 		    AHCI_INTR_STATUS_SDBS |
6623 		    AHCI_INTR_STATUS_UFS |
6624 		    AHCI_INTR_STATUS_PCS |
6625 		    AHCI_INTR_STATUS_PRCS |
6626 		    AHCI_INTR_STATUS_OFS |
6627 		    AHCI_INTR_STATUS_INFS |
6628 		    AHCI_INTR_STATUS_IFS |
6629 		    AHCI_INTR_STATUS_HBDS |
6630 		    AHCI_INTR_STATUS_HBFS |
6631 		    AHCI_INTR_STATUS_TFES);
6632 	} else {
6633 		/*
6634 		 * port_intr_enable indicates that the corresponding interrrupt
6635 		 * reporting is enabled.
6636 		 */
6637 		port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6638 		    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6639 	}
6640 
6641 	/* IPMS error in port reset should be ignored according AHCI spec. */
6642 	if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6643 		port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6644 	mutex_exit(&ahci_portp->ahciport_mutex);
6645 
6646 	/*
6647 	 * port_intr_stats indicates that the corresponding interrupt
6648 	 * condition is active.
6649 	 */
6650 	port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6651 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6652 
6653 	AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6654 	    "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6655 	    "port_intr_enable = 0x%x",
6656 	    port, port_intr_status, port_intr_enable);
6657 
6658 	port_intr_status &= port_intr_enable;
6659 
6660 	/*
6661 	 * Pending interrupt events are indicated by the PxIS register.
6662 	 * Make sure we don't miss any event.
6663 	 */
6664 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6665 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6666 		    DDI_SERVICE_UNAFFECTED);
6667 		ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6668 		    DDI_FME_VERSION);
6669 		return;
6670 	}
6671 
6672 	/* First clear the port interrupts status */
6673 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6674 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6675 	    port_intr_status);
6676 
6677 	/* Check the completed non-queued commands */
6678 	if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6679 	    AHCI_INTR_STATUS_PSS)) {
6680 		(void) ahci_intr_cmd_cmplt(ahci_ctlp,
6681 		    ahci_portp, port);
6682 	}
6683 
6684 	/* Check the completed queued commands */
6685 	if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6686 		(void) ahci_intr_set_device_bits(ahci_ctlp,
6687 		    ahci_portp, port);
6688 	}
6689 
6690 	/* Check the port connect change status interrupt bit */
6691 	if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6692 		(void) ahci_intr_port_connect_change(ahci_ctlp,
6693 		    ahci_portp, port);
6694 	}
6695 
6696 	/* Check the device mechanical presence status interrupt bit */
6697 	if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6698 		(void) ahci_intr_device_mechanical_presence_status(
6699 		    ahci_ctlp, ahci_portp, port);
6700 	}
6701 
6702 	/* Check the PhyRdy change status interrupt bit */
6703 	if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6704 		(void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6705 		    port);
6706 	}
6707 
6708 	/*
6709 	 * Check the non-fatal error interrupt bits, there are four
6710 	 * kinds of non-fatal errors at the time being:
6711 	 *
6712 	 *    PxIS.UFS - Unknown FIS Error
6713 	 *    PxIS.OFS - Overflow Error
6714 	 *    PxIS.INFS - Interface Non-Fatal Error
6715 	 *    PxIS.IPMS - Incorrect Port Multiplier Status Error
6716 	 *
6717 	 * For these non-fatal errors, the HBA can continue to operate,
6718 	 * so the driver just log the error messages.
6719 	 */
6720 	if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6721 	    AHCI_INTR_STATUS_OFS |
6722 	    AHCI_INTR_STATUS_IPMS |
6723 	    AHCI_INTR_STATUS_INFS)) {
6724 		(void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6725 		    port, port_intr_status);
6726 	}
6727 
6728 	/*
6729 	 * Check the fatal error interrupt bits, there are four kinds
6730 	 * of fatal errors for AHCI controllers:
6731 	 *
6732 	 *    PxIS.HBFS - Host Bus Fatal Error
6733 	 *    PxIS.HBDS - Host Bus Data Error
6734 	 *    PxIS.IFS - Interface Fatal Error
6735 	 *    PxIS.TFES - Task File Error
6736 	 *
6737 	 * The fatal error means the HBA can not recover from it by
6738 	 * itself, and it will try to abort the transfer, and the software
6739 	 * must intervene to restart the port.
6740 	 */
6741 	if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6742 	    AHCI_INTR_STATUS_HBDS |
6743 	    AHCI_INTR_STATUS_HBFS |
6744 	    AHCI_INTR_STATUS_TFES))
6745 		(void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6746 		    port, port_intr_status);
6747 
6748 	/* Check the cold port detect interrupt bit */
6749 	if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6750 		(void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6751 	}
6752 
6753 	/* Second clear the corresponding bit in IS.IPS */
6754 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6755 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6756 
6757 	/* Try to recover at the end of the interrupt handler. */
6758 	if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6759 	    DDI_FM_OK) {
6760 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6761 		    DDI_SERVICE_UNAFFECTED);
6762 		ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6763 		    DDI_FME_VERSION);
6764 	}
6765 }
6766 
6767 /*
6768  * Interrupt service handler
6769  */
6770 static uint_t
ahci_intr(caddr_t arg1,caddr_t arg2)6771 ahci_intr(caddr_t arg1, caddr_t arg2)
6772 {
6773 #ifndef __lock_lint
6774 	_NOTE(ARGUNUSED(arg2))
6775 #endif
6776 	/* LINTED */
6777 	ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6778 	ahci_port_t *ahci_portp;
6779 	int32_t global_intr_status;
6780 	uint8_t port;
6781 
6782 	/*
6783 	 * global_intr_status indicates that the corresponding port has
6784 	 * an interrupt pending.
6785 	 */
6786 	global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6787 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6788 
6789 	if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6790 		/* The interrupt is not ours */
6791 		return (DDI_INTR_UNCLAIMED);
6792 	}
6793 
6794 	/*
6795 	 * Check the handle after reading global_intr_status - we don't want
6796 	 * to miss any port with pending interrupts.
6797 	 */
6798 	if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6799 	    DDI_FM_OK) {
6800 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6801 		    DDI_SERVICE_UNAFFECTED);
6802 		ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6803 		    DDI_FME_VERSION);
6804 		return (DDI_INTR_UNCLAIMED);
6805 	}
6806 
6807 	/* Loop for all the ports */
6808 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6809 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6810 			continue;
6811 		}
6812 		if (!((0x1 << port) & global_intr_status)) {
6813 			continue;
6814 		}
6815 
6816 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
6817 
6818 		/* Call ahci_port_intr */
6819 		ahci_port_intr(ahci_ctlp, ahci_portp, port);
6820 	}
6821 
6822 	return (DDI_INTR_CLAIMED);
6823 }
6824 
6825 /*
6826  * For non-queued commands, when the corresponding bit in the PxCI register
6827  * is cleared, it means the command is completed successfully. And according
6828  * to the HBA state machine, there are three conditions which possibly will
6829  * try to clear the PxCI register bit.
6830  *	1. Receive one D2H Register FIS which is with 'I' bit set
6831  *	2. Update PIO Setup FIS
6832  *	3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6833  *
6834  * Process completed non-queued commands when the interrupt status bit -
6835  * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6836  *
6837  * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6838  * with the 'I' bit set. And the following commands will send thus
6839  * FIS with 'I' bit set upon the successful completion:
6840  *	1. Non-data commands
6841  *	2. DMA data-in command
6842  *	3. DMA data-out command
6843  *	4. PIO data-out command
6844  *	5. PACKET non-data commands
6845  *	6. PACKET PIO data-in command
6846  *	7. PACKET PIO data-out command
6847  *	8. PACKET DMA data-in command
6848  *	9. PACKET DMA data-out command
6849  *
6850  * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6851  * with the 'I' bit set. And the following commands will send this
6852  * FIS upon the successful completion:
6853  *	1. PIO data-in command
6854  */
6855 static int
ahci_intr_cmd_cmplt(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)6856 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6857     ahci_port_t *ahci_portp, uint8_t port)
6858 {
6859 	uint32_t port_cmd_issue = 0;
6860 	uint32_t finished_tags;
6861 	int finished_slot;
6862 	sata_pkt_t *satapkt;
6863 	ahci_fis_d2h_register_t *rcvd_fisp;
6864 #if AHCI_DEBUG
6865 	ahci_cmd_header_t *cmd_header;
6866 	uint32_t cmd_dmacount;
6867 #endif
6868 
6869 	mutex_enter(&ahci_portp->ahciport_mutex);
6870 
6871 	if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6872 	    !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6873 	    !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6874 		/*
6875 		 * Spurious interrupt. Nothing to be done.
6876 		 */
6877 		mutex_exit(&ahci_portp->ahciport_mutex);
6878 		return (AHCI_SUCCESS);
6879 	}
6880 
6881 	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6882 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6883 
6884 	/* If the PxCI corrupts, don't complete the commmands. */
6885 	if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6886 	    != DDI_FM_OK) {
6887 		mutex_exit(&ahci_portp->ahciport_mutex);
6888 		return (AHCI_FAILURE);
6889 	}
6890 
6891 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6892 		/* Slot 0 is always used during error recovery */
6893 		finished_tags = 0x1 & ~port_cmd_issue;
6894 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6895 		    "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6896 		    "retrieval is finished, and finished_tags = 0x%x",
6897 		    port, finished_tags);
6898 	} else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6899 		finished_tags = 0x1 & ~port_cmd_issue;
6900 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6901 		    "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6902 		    "port multiplier is finished, and finished_tags = 0x%x",
6903 		    port, finished_tags);
6904 
6905 	} else {
6906 
6907 		finished_tags = ahci_portp->ahciport_pending_tags &
6908 		    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6909 	}
6910 
6911 	AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6912 	    "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6913 	    "port_cmd_issue = 0x%x finished_tags = 0x%x",
6914 	    ahci_portp->ahciport_pending_tags, port_cmd_issue,
6915 	    finished_tags);
6916 
6917 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6918 	    (finished_tags == 0x1)) {
6919 		satapkt = ahci_portp->ahciport_err_retri_pkt;
6920 		ASSERT(satapkt != NULL);
6921 
6922 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6923 		    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6924 		    "with SATA_PKT_COMPLETED", (void *)satapkt);
6925 
6926 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6927 		goto out;
6928 	}
6929 
6930 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6931 	    (finished_tags == 0x1)) {
6932 		satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6933 		ASSERT(satapkt != NULL);
6934 
6935 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6936 		    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6937 		    "with SATA_PKT_COMPLETED", (void *)satapkt);
6938 
6939 		/* READ PORTMULT need copy out FIS content. */
6940 		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6941 			rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6942 			    ahcirf_d2h_register_fis);
6943 			satapkt->satapkt_cmd.satacmd_status_reg =
6944 			    GET_RFIS_STATUS(rcvd_fisp);
6945 			ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6946 		}
6947 
6948 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6949 		goto out;
6950 	}
6951 
6952 	while (finished_tags) {
6953 		finished_slot = ddi_ffs(finished_tags) - 1;
6954 		if (finished_slot == -1) {
6955 			goto out;
6956 		}
6957 
6958 		satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6959 		ASSERT(satapkt != NULL);
6960 #if AHCI_DEBUG
6961 		/*
6962 		 * For non-native queued commands, the PRD byte count field
6963 		 * shall contain an accurate count of the number of bytes
6964 		 * transferred for the command before the PxCI bit is cleared
6965 		 * to '0' for the command.
6966 		 *
6967 		 * The purpose of this field is to let software know how many
6968 		 * bytes transferred for a given operation in order to
6969 		 * determine if underflow occurred. When issuing native command
6970 		 * queuing commands, this field should not be used and is not
6971 		 * required to be valid since in this case underflow is always
6972 		 * illegal.
6973 		 *
6974 		 * For data reads, the HBA will update its PRD byte count with
6975 		 * the total number of bytes received from the last FIS, and
6976 		 * may be able to continue normally. For data writes, the
6977 		 * device will detect an error, and HBA most likely will get
6978 		 * a fatal error.
6979 		 *
6980 		 * Therefore, here just put code to debug part. And please
6981 		 * refer to the comment above ahci_intr_fatal_error for the
6982 		 * definition of underflow error.
6983 		 */
6984 		cmd_dmacount =
6985 		    ahci_portp->ahciport_prd_bytecounts[finished_slot];
6986 		if (cmd_dmacount) {
6987 			cmd_header =
6988 			    &ahci_portp->ahciport_cmd_list[finished_slot];
6989 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6990 			    "ahci_intr_cmd_cmplt: port %d, "
6991 			    "PRD Byte Count = 0x%x, "
6992 			    "ahciport_prd_bytecounts = 0x%x", port,
6993 			    cmd_header->ahcich_prd_byte_count,
6994 			    cmd_dmacount);
6995 
6996 			if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6997 				AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6998 				    "ahci_intr_cmd_cmplt: port %d, "
6999 				    "an underflow occurred", port);
7000 			}
7001 		}
7002 #endif
7003 
7004 		/*
7005 		 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
7006 		 * feature, sata_special_regs flag will be set, and the
7007 		 * driver should copy the status and the other corresponding
7008 		 * register values in the D2H Register FIS received (It's
7009 		 * working on Non-data protocol) from the device back to
7010 		 * the sata_cmd.
7011 		 *
7012 		 * For every AHCI port, there is only one Received FIS
7013 		 * structure, which contains the FISes received from the
7014 		 * device, So we're trying to copy the content of D2H
7015 		 * Register FIS in the Received FIS structure back to
7016 		 * the sata_cmd.
7017 		 */
7018 		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
7019 			rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
7020 			    ahcirf_d2h_register_fis);
7021 			satapkt->satapkt_cmd.satacmd_status_reg =
7022 			    GET_RFIS_STATUS(rcvd_fisp);
7023 			ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
7024 		}
7025 
7026 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7027 		    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
7028 		    "with SATA_PKT_COMPLETED", (void *)satapkt);
7029 
7030 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
7031 		CLEAR_BIT(finished_tags, finished_slot);
7032 		ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7033 
7034 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7035 	}
7036 out:
7037 	AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
7038 	    "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
7039 	    ahci_portp->ahciport_pending_tags);
7040 
7041 	ahci_flush_doneq(ahci_portp);
7042 
7043 	mutex_exit(&ahci_portp->ahciport_mutex);
7044 
7045 	return (AHCI_SUCCESS);
7046 }
7047 
7048 /*
7049  * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
7050  * with the 'I' bit set and has been copied into system memory. It will
7051  * be sent under the following situations:
7052  *
7053  * 1. NCQ command is completed
7054  *
7055  * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
7056  * via the Set Device Bits FIS. When such event is generated, the software
7057  * needs to read PxSACT register and compares the current value to the
7058  * list of commands previously issue by software. ahciport_pending_ncq_tags
7059  * keeps the tags of previously issued commands.
7060  *
7061  * 2. Asynchronous Notification
7062  *
7063  * Asynchronous Notification is a feature in SATA spec 2.6.
7064  *
7065  * 1) ATAPI device will send a signal to the host when media is inserted or
7066  * removed and avoids polling the device for media changes. The signal
7067  * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
7068  * set to '1'. At the moment, it's not supported yet.
7069  *
7070  * 2) Port multiplier will send a signal to the host when a hot plug event
7071  * has occured on a port multiplier port. It is used when command based
7072  * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
7073  */
7074 static int
ahci_intr_set_device_bits(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)7075 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
7076     ahci_port_t *ahci_portp, uint8_t port)
7077 {
7078 	ahci_addr_t addr;
7079 
7080 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7081 	    "ahci_intr_set_device_bits enter: port %d", port);
7082 
7083 	/* Initialize HBA port address */
7084 	AHCI_ADDR_SET_PORT(&addr, port);
7085 
7086 	/* NCQ plug handler */
7087 	(void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
7088 
7089 	/* Check port multiplier's asynchronous notification events */
7090 	if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
7091 		(void) ahci_intr_pmult_sntf_events(ahci_ctlp,
7092 		    ahci_portp, port);
7093 	}
7094 
7095 	/* ATAPI events is not supported yet */
7096 
7097 	return (AHCI_SUCCESS);
7098 }
7099 /*
7100  * NCQ interrupt handler. Called upon a NCQ command is completed.
7101  * Only be called from ahci_intr_set_device_bits().
7102  */
7103 static int
ahci_intr_ncq_events(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)7104 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7105     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7106 {
7107 	uint32_t port_sactive;
7108 	uint32_t port_cmd_issue;
7109 	uint32_t issued_tags;
7110 	int issued_slot;
7111 	uint32_t finished_tags;
7112 	int finished_slot;
7113 	uint8_t port = addrp->aa_port;
7114 	sata_pkt_t *satapkt;
7115 
7116 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7117 	    "ahci_intr_set_device_bits enter: port %d", port);
7118 
7119 	mutex_enter(&ahci_portp->ahciport_mutex);
7120 	if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7121 		mutex_exit(&ahci_portp->ahciport_mutex);
7122 		return (AHCI_SUCCESS);
7123 	}
7124 
7125 	/*
7126 	 * First the handler got which commands are finished by checking
7127 	 * PxSACT register
7128 	 */
7129 	port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7130 	    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7131 
7132 	finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7133 	    ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7134 
7135 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7136 	    "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7137 	    "port_sactive = 0x%x", port,
7138 	    ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7139 
7140 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7141 	    "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7142 
7143 	/*
7144 	 * For NCQ commands, the software can determine which command has
7145 	 * already been transmitted to the device by checking PxCI register.
7146 	 */
7147 	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7148 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7149 
7150 	issued_tags = ahci_portp->ahciport_pending_tags &
7151 	    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7152 
7153 	/* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7154 	if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7155 	    != DDI_FM_OK) {
7156 		mutex_exit(&ahci_portp->ahciport_mutex);
7157 		return (AHCI_FAILURE);
7158 	}
7159 
7160 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7161 	    "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7162 	    "port_cmd_issue = 0x%x", port,
7163 	    ahci_portp->ahciport_pending_tags, port_cmd_issue);
7164 
7165 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7166 	    "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7167 
7168 	/*
7169 	 * Clear ahciport_pending_tags bit when the corresponding command
7170 	 * is already sent down to the device.
7171 	 */
7172 	while (issued_tags) {
7173 		issued_slot = ddi_ffs(issued_tags) - 1;
7174 		if (issued_slot == -1) {
7175 			goto next;
7176 		}
7177 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7178 		CLEAR_BIT(issued_tags, issued_slot);
7179 	}
7180 
7181 next:
7182 	while (finished_tags) {
7183 		finished_slot = ddi_ffs(finished_tags) - 1;
7184 		if (finished_slot == -1) {
7185 			goto out;
7186 		}
7187 
7188 		/* The command is certainly transmitted to the device */
7189 		ASSERT(!(ahci_portp->ahciport_pending_tags &
7190 		    (0x1 << finished_slot)));
7191 
7192 		satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7193 		ASSERT(satapkt != NULL);
7194 
7195 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7196 		    "ahci_intr_set_device_bits: sending up pkt 0x%p "
7197 		    "with SATA_PKT_COMPLETED", (void *)satapkt);
7198 
7199 		CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7200 		CLEAR_BIT(finished_tags, finished_slot);
7201 		ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7202 
7203 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7204 	}
7205 out:
7206 	AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7207 	    "ahci_intr_set_device_bits: port %d "
7208 	    "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7209 	    port, ahci_portp->ahciport_pending_ncq_tags,
7210 	    ahci_portp->ahciport_pending_tags);
7211 
7212 	ahci_flush_doneq(ahci_portp);
7213 
7214 	mutex_exit(&ahci_portp->ahciport_mutex);
7215 
7216 	return (AHCI_SUCCESS);
7217 }
7218 
7219 /*
7220  * Port multiplier asynchronous notification event handler. Called upon a
7221  * device is hot plugged/pulled.
7222  *
7223  * The async-notification event will only be recorded by ahcipmi_snotif_tags
7224  * here and will be handled by ahci_probe_pmult().
7225  *
7226  * NOTE: called only from ahci_port_intr().
7227  */
7228 static int
ahci_intr_pmult_sntf_events(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)7229 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7230     uint8_t port)
7231 {
7232 	sata_device_t sdevice;
7233 
7234 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7235 	    "ahci_intr_pmult_sntf_events enter: port %d ", port);
7236 
7237 	/* no hot-plug while attaching process */
7238 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
7239 	if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7240 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
7241 		return (AHCI_SUCCESS);
7242 	}
7243 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
7244 
7245 	mutex_enter(&ahci_portp->ahciport_mutex);
7246 	if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7247 		mutex_exit(&ahci_portp->ahciport_mutex);
7248 		return (AHCI_SUCCESS);
7249 	}
7250 
7251 	ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7252 
7253 	ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7254 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7255 	    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7256 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7257 	    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7258 	    AHCI_SNOTIF_CLEAR_ALL);
7259 
7260 	if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7261 		mutex_exit(&ahci_portp->ahciport_mutex);
7262 		return (AHCI_SUCCESS);
7263 	}
7264 
7265 	/* Port Multiplier sub-device hot-plug handler */
7266 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7267 		mutex_exit(&ahci_portp->ahciport_mutex);
7268 		return (AHCI_SUCCESS);
7269 	}
7270 
7271 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7272 		/* Not allowed to re-enter. */
7273 		mutex_exit(&ahci_portp->ahciport_mutex);
7274 		return (AHCI_SUCCESS);
7275 	}
7276 
7277 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7278 
7279 	/*
7280 	 * NOTE:
7281 	 * Even if Asynchronous Notification is supported (and enabled) by
7282 	 * both controller and the port multiplier, the content of PxSNTF
7283 	 * register is always set to 0x8000 by async notification event. We
7284 	 * need to check GSCR[32] on the port multiplier to find out the
7285 	 * owner of this event.
7286 	 * This is not accord with SATA spec 2.6 and needs further
7287 	 * clarification.
7288 	 */
7289 	/* hot-plug will not reported while reseting. */
7290 	if (ahci_portp->ahciport_reset_in_progress == 1) {
7291 		AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7292 		    "port %d snotif event ignored", port);
7293 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7294 		mutex_exit(&ahci_portp->ahciport_mutex);
7295 		return (AHCI_SUCCESS);
7296 	}
7297 
7298 	AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7299 	    "PxSNTF is set to 0x%x by port multiplier",
7300 	    ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7301 
7302 	/*
7303 	 * Now we need do some necessary operation and inform SATA framework
7304 	 * that link/device events has happened.
7305 	 */
7306 	bzero((void *)&sdevice, sizeof (sata_device_t));
7307 	sdevice.satadev_addr.cport = ahci_ctlp->
7308 	    ahcictl_port_to_cport[port];
7309 	sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7310 	sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7311 	sdevice.satadev_state = SATA_PSTATE_PWRON;
7312 
7313 	/* Just reject packets, do not stop that port. */
7314 	ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7315 
7316 	mutex_exit(&ahci_portp->ahciport_mutex);
7317 	sata_hba_event_notify(
7318 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7319 	    &sdevice,
7320 	    SATA_EVNT_PMULT_LINK_CHANGED);
7321 	mutex_enter(&ahci_portp->ahciport_mutex);
7322 
7323 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7324 	mutex_exit(&ahci_portp->ahciport_mutex);
7325 
7326 	return (AHCI_SUCCESS);
7327 }
7328 
7329 /*
7330  * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7331  * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7332  * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7333  * indicates a COMINIT signal was received.
7334  *
7335  * Hot plug insertion is detected by reception of a COMINIT signal from the
7336  * device. On reception of unsolicited COMINIT, the HBA shall generate a
7337  * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7338  * begin the normal communication negotiation sequence as outlined in the
7339  * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7340  * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7341  * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7342  * sequence is complete and PhyRdy is true the PxSSTS.DET field	shall be set
7343  * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7344  * to handle hot plug insertion. In this interrupt handler, just do nothing
7345  * but print some log message and clear the bit.
7346  */
7347 static int
ahci_intr_port_connect_change(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)7348 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7349     ahci_port_t *ahci_portp, uint8_t port)
7350 {
7351 #if AHCI_DEBUG
7352 	uint32_t port_serror;
7353 #endif
7354 
7355 	mutex_enter(&ahci_portp->ahciport_mutex);
7356 
7357 #if AHCI_DEBUG
7358 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7359 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7360 
7361 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7362 	    "ahci_intr_port_connect_change: port %d, "
7363 	    "port_serror = 0x%x", port, port_serror);
7364 #endif
7365 
7366 	/* Clear PxSERR.DIAG.X to clear the interrupt bit */
7367 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7368 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7369 	    SERROR_EXCHANGED_ERR);
7370 
7371 	mutex_exit(&ahci_portp->ahciport_mutex);
7372 
7373 	return (AHCI_SUCCESS);
7374 }
7375 
7376 /*
7377  * Hot Plug Operation for platforms that support Mechanical Presence
7378  * Switches.
7379  *
7380  * When set, it indicates that a mechanical presence switch attached to this
7381  * port has been opened or closed, which may lead to a change in the connection
7382  * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7383  * are set to '1'.
7384  *
7385  * At the moment, this interrupt is not needed and disabled and we just log
7386  * the debug message.
7387  */
7388 static int
ahci_intr_device_mechanical_presence_status(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)7389 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7390     ahci_port_t *ahci_portp, uint8_t port)
7391 {
7392 	uint32_t cap_status, port_cmd_status;
7393 
7394 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7395 	    "ahci_intr_device_mechanical_presence_status enter, "
7396 	    "port %d", port);
7397 
7398 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7399 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7400 
7401 	mutex_enter(&ahci_portp->ahciport_mutex);
7402 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7403 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7404 
7405 	if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7406 	    !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7407 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7408 		    "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7409 		    "the interrupt: cap_status = 0x%x, "
7410 		    "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7411 		mutex_exit(&ahci_portp->ahciport_mutex);
7412 
7413 		return (AHCI_SUCCESS);
7414 	}
7415 
7416 #if AHCI_DEBUG
7417 	if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7418 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7419 		    "The mechanical presence switch is open: "
7420 		    "port %d, port_cmd_status = 0x%x",
7421 		    port, port_cmd_status);
7422 	} else {
7423 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7424 		    "The mechanical presence switch is close: "
7425 		    "port %d, port_cmd_status = 0x%x",
7426 		    port, port_cmd_status);
7427 	}
7428 #endif
7429 
7430 	mutex_exit(&ahci_portp->ahciport_mutex);
7431 
7432 	return (AHCI_SUCCESS);
7433 }
7434 
7435 /*
7436  * Native Hot Plug Support.
7437  *
7438  * When set, it indicates that the internal PHYRDY signal changed state.
7439  * This bit reflects the state of PxSERR.DIAG.N.
7440  *
7441  * There are three kinds of conditions to generate this interrupt event:
7442  * 1. a device is inserted
7443  * 2. a device is disconnected
7444  * 3. when the link enters/exits a Partial or Slumber interface power
7445  *    management state
7446  *
7447  * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7448  * bit may be set due to the link entering the Partial or Slumber power
7449  * management state, rather than due to a hot plug insertion or removal
7450  * event. So far, the interface power management is disabled, so the
7451  * driver can reliably get removal detection notification via the
7452  * PxSERR.DIAG.N bit.
7453  */
7454 static int
ahci_intr_phyrdy_change(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)7455 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7456     ahci_port_t *ahci_portp, uint8_t port)
7457 {
7458 	uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7459 	sata_device_t sdevice;
7460 	int dev_exists_now = 0;
7461 	int dev_existed_previously = 0;
7462 	ahci_addr_t port_addr;
7463 
7464 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7465 	    "ahci_intr_phyrdy_change enter, port %d", port);
7466 
7467 	/* Clear PxSERR.DIAG.N to clear the interrupt bit */
7468 	mutex_enter(&ahci_portp->ahciport_mutex);
7469 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7470 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7471 	    SERROR_PHY_RDY_CHG);
7472 	mutex_exit(&ahci_portp->ahciport_mutex);
7473 
7474 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
7475 	if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7476 	    (ahci_portp == NULL)) {
7477 		/* The whole controller setup is not yet done. */
7478 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
7479 		return (AHCI_SUCCESS);
7480 	}
7481 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
7482 
7483 	mutex_enter(&ahci_portp->ahciport_mutex);
7484 
7485 	/* SStatus tells the presence of device. */
7486 	port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7487 	    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7488 
7489 	if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7490 		dev_exists_now = 1;
7491 	}
7492 
7493 	if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7494 		dev_existed_previously = 1;
7495 	}
7496 
7497 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7498 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7499 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7500 		    "ahci_intr_phyrdy_change: port %d "
7501 		    "AHCI_PORT_FLAG_NODEV is cleared", port);
7502 		if (dev_exists_now == 0)
7503 			dev_existed_previously = 1;
7504 	}
7505 
7506 	bzero((void *)&sdevice, sizeof (sata_device_t));
7507 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7508 	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7509 	sdevice.satadev_addr.pmport = 0;
7510 	sdevice.satadev_state = SATA_PSTATE_PWRON;
7511 	ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7512 
7513 	AHCI_ADDR_SET_PORT(&port_addr, port);
7514 
7515 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7516 	if (dev_exists_now) {
7517 		if (dev_existed_previously) { /* 1 -> 1 */
7518 			/* Things are fine now. The loss was temporary. */
7519 			AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7520 			    "ahci_intr_phyrdy_change  port %d "
7521 			    "device link lost/established", port);
7522 
7523 			mutex_exit(&ahci_portp->ahciport_mutex);
7524 			sata_hba_event_notify(
7525 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7526 			    &sdevice,
7527 			    SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7528 			mutex_enter(&ahci_portp->ahciport_mutex);
7529 
7530 		} else { /* 0 -> 1 */
7531 			AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7532 			    "ahci_intr_phyrdy_change: port %d "
7533 			    "device link established", port);
7534 
7535 			/*
7536 			 * A new device has been detected. The new device
7537 			 * might be a port multiplier instead of a drive, so
7538 			 * we cannot update the signature directly.
7539 			 */
7540 			(void) ahci_initialize_port(ahci_ctlp,
7541 			    ahci_portp, &port_addr);
7542 
7543 			/* Try to start the port */
7544 			if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7545 			    != AHCI_SUCCESS) {
7546 				sdevice.satadev_state |= SATA_PSTATE_FAILED;
7547 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7548 				    "ahci_intr_phyrdy_change: port %d failed "
7549 				    "at start port", port);
7550 			}
7551 
7552 			/* Clear the max queue depth for inserted device */
7553 			ahci_portp->ahciport_max_ncq_tags = 0;
7554 
7555 			mutex_exit(&ahci_portp->ahciport_mutex);
7556 			sata_hba_event_notify(
7557 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7558 			    &sdevice,
7559 			    SATA_EVNT_LINK_ESTABLISHED);
7560 			mutex_enter(&ahci_portp->ahciport_mutex);
7561 
7562 		}
7563 	} else { /* No device exists now */
7564 
7565 		if (dev_existed_previously) { /* 1 -> 0 */
7566 			AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7567 			    "ahci_intr_phyrdy_change: port %d "
7568 			    "device link lost", port);
7569 
7570 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7571 			(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7572 			    ahci_portp, port);
7573 
7574 			if (ahci_portp->ahciport_device_type ==
7575 			    SATA_DTYPE_PMULT) {
7576 				ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7577 			}
7578 
7579 			/* An existing device is lost. */
7580 			ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7581 			ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7582 
7583 			mutex_exit(&ahci_portp->ahciport_mutex);
7584 			sata_hba_event_notify(
7585 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7586 			    &sdevice,
7587 			    SATA_EVNT_LINK_LOST);
7588 			mutex_enter(&ahci_portp->ahciport_mutex);
7589 		}
7590 	}
7591 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7592 
7593 	mutex_exit(&ahci_portp->ahciport_mutex);
7594 
7595 	return (AHCI_SUCCESS);
7596 }
7597 
7598 /*
7599  * PxIS.UFS - Unknown FIS Error
7600  *
7601  * This interrupt event means an unknown FIS was received and has been
7602  * copied into system memory. An unknown FIS is not considered an illegal
7603  * FIS, unless the length received is more than 64 bytes. If an unknown
7604  * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7605  * normal operation. If the unknown FIS is more than 64 bytes, then it
7606  * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7607  * a fatal error.
7608  *
7609  * PxIS.IPMS - Incorrect Port Multiplier Status
7610  *
7611  * IPMS Indicates that the HBA received a FIS from a device that did not
7612  * have a command outstanding. The IPMS bit may be set during enumeration
7613  * of devices on a Port Multiplier due to the normal Port Multiplier
7614  * enumeration process. It is recommended that IPMS only be used after
7615  * enumeration is complete on the Port Multiplier (copied from spec).
7616  *
7617  * PxIS.OFS - Overflow Error
7618  *
7619  * Command list overflow is defined as software building a command table
7620  * that has fewer total bytes than the transaction given to the device.
7621  * On device writes, the HBA will run out of data, and on reads, there
7622  * will be no room to put the data.
7623  *
7624  * For an overflow on data read, either PIO or DMA, the HBA will set
7625  * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7626  * non-fatal error when the HBA can continues. Sometimes, it will cause
7627  * a fatal error and need the software to do something.
7628  *
7629  * For an overflow on data write, setting PxIS.OFS is optional for both
7630  * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7631  * software to clean up from this serious error.
7632  *
7633  * PxIS.INFS - Interface Non-Fatal Error
7634  *
7635  * This interrupt event indicates that the HBA encountered an error on
7636  * the Serial ATA interface but was able to continue operation. The kind
7637  * of error usually occurred during a non-Data FIS, and under this condition
7638  * the FIS will be re-transmitted by HBA automatically.
7639  *
7640  * When the FMA is implemented, there should be a stat structure to
7641  * record how many every kind of error happens.
7642  */
7643 static int
ahci_intr_non_fatal_error(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,uint32_t intr_status)7644 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7645     uint8_t port, uint32_t intr_status)
7646 {
7647 	uint32_t port_serror;
7648 #if AHCI_DEBUG
7649 	uint32_t port_cmd_status;
7650 	uint32_t port_cmd_issue;
7651 	uint32_t port_sactive;
7652 	int current_slot;
7653 	uint32_t current_tags;
7654 	sata_pkt_t *satapkt;
7655 	ahci_cmd_header_t *cmd_header;
7656 	uint32_t cmd_dmacount;
7657 #endif
7658 
7659 	mutex_enter(&ahci_portp->ahciport_mutex);
7660 
7661 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7662 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7663 
7664 	AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7665 	    "ahci_intr_non_fatal_error: port %d, "
7666 	    "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7667 
7668 	ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7669 
7670 	if (intr_status & AHCI_INTR_STATUS_UFS) {
7671 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7672 		    "ahci port %d has unknown FIS error", port);
7673 
7674 		/* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7675 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7676 		    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7677 		    SERROR_FIS_TYPE);
7678 	}
7679 
7680 #if AHCI_DEBUG
7681 	if (intr_status & AHCI_INTR_STATUS_IPMS) {
7682 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7683 		    "has Incorrect Port Multiplier Status error", port);
7684 	}
7685 
7686 	if (intr_status & AHCI_INTR_STATUS_OFS) {
7687 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7688 		    "ahci port %d has overflow error", port);
7689 	}
7690 
7691 	if (intr_status & AHCI_INTR_STATUS_INFS) {
7692 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7693 		    "ahci port %d has interface non fatal error", port);
7694 	}
7695 
7696 	/*
7697 	 * Record the error occurred command's slot.
7698 	 */
7699 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7700 	    ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7701 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7702 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7703 
7704 		current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7705 		    AHCI_CMD_STATUS_CCS_SHIFT;
7706 
7707 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7708 			satapkt = ahci_portp->ahciport_err_retri_pkt;
7709 			ASSERT(satapkt != NULL);
7710 			ASSERT(current_slot == 0);
7711 		} else {
7712 			satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7713 		}
7714 
7715 		if (satapkt != NULL) {
7716 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7717 			    "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7718 			    "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7719 			    satapkt->satapkt_cmd.satacmd_cmd_reg);
7720 
7721 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7722 			    "ahci_intr_non_fatal_error: port %d, "
7723 			    "satapkt 0x%p is being processed when error occurs",
7724 			    port, (void *)satapkt);
7725 
7726 			/*
7727 			 * PRD Byte Count field of command header is not
7728 			 * required to reflect the total number of bytes
7729 			 * transferred when an overflow occurs, so here
7730 			 * just log the value.
7731 			 */
7732 			cmd_dmacount =
7733 			    ahci_portp->ahciport_prd_bytecounts[current_slot];
7734 			if (cmd_dmacount) {
7735 				cmd_header = &ahci_portp->
7736 				    ahciport_cmd_list[current_slot];
7737 				AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7738 				    "ahci_intr_non_fatal_error: port %d, "
7739 				    "PRD Byte Count = 0x%x, "
7740 				    "ahciport_prd_bytecounts = 0x%x", port,
7741 				    cmd_header->ahcich_prd_byte_count,
7742 				    cmd_dmacount);
7743 			}
7744 		}
7745 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7746 		/*
7747 		 * For queued command, list those command which have already
7748 		 * been transmitted to the device and still not completed.
7749 		 */
7750 		port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7751 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7752 
7753 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7754 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7755 
7756 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7757 		    "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7758 		    "port_sactive = 0x%x port_cmd_issue = 0x%x",
7759 		    ahci_portp->ahciport_pending_ncq_tags,
7760 		    port_sactive, port_cmd_issue);
7761 
7762 		current_tags = ahci_portp->ahciport_pending_ncq_tags &
7763 		    port_sactive & ~port_cmd_issue &
7764 		    AHCI_NCQ_SLOT_MASK(ahci_portp);
7765 
7766 		while (current_tags) {
7767 			current_slot = ddi_ffs(current_tags) - 1;
7768 			if (current_slot == -1) {
7769 				goto out;
7770 			}
7771 
7772 			satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7773 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7774 			    ahci_ctlp, "ahci_intr_non_fatal_error: "
7775 			    "port %d, satapkt 0x%p is outstanding when "
7776 			    "error occurs", port, (void *)satapkt);
7777 
7778 			CLEAR_BIT(current_tags, current_slot);
7779 		}
7780 	}
7781 out:
7782 #endif
7783 	mutex_exit(&ahci_portp->ahciport_mutex);
7784 
7785 	return (AHCI_SUCCESS);
7786 }
7787 
7788 /*
7789  * According to the AHCI spec, the error types include system memory
7790  * errors, interface errors, port multiplier errors, device errors,
7791  * command list overflow, command list underflow, native command
7792  * queuing tag errors and pio data transfer errors.
7793  *
7794  * System memory errors such as target abort, master abort, and parity
7795  * may cause the host to stop, and they are serious errors and needed
7796  * to be recovered with software intervention. When system software
7797  * has given a pointer to the HBA that doesn't exist in physical memory,
7798  * a master/target abort error occurs, and PxIS.HBFS will be set. A
7799  * data error such as CRC or parity occurs, the HBA aborts the transfer
7800  * (if necessary) and PxIS.HBDS will be set.
7801  *
7802  * Interface errors are errors that occur due to electrical issues on
7803  * the interface, or protocol miscommunication between the device and
7804  * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7805  * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7806  * causes PxIS.IFS/PxIS.INFS to be set are
7807  *	1. in PxSERR.ERR, P bit is set to '1'
7808  *	2. in PxSERR.DIAG, C or H bit is set to '1'
7809  *	3. PhyRdy drop unexpectly, N bit is set to '1'
7810  * If the error occurred during a non-data FIS, the FIS must be
7811  * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7812  * the error occurred during a data FIS, the transfer will stop, so
7813  * the error is fatal and PxIS.IFS is set.
7814  *
7815  * When a FIS arrives that updates the taskfile, the HBA checks to see
7816  * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7817  * stops processing any more commands.
7818  *
7819  * Command list overflow is defined as software building a command table
7820  * that has fewer total bytes than the transaction given to the device.
7821  * On device writes, the HBA will run out of data, and on reads, there
7822  * will be no room to put the data. For an overflow on data read, either
7823  * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7824  * For an overflow on data write, setting PxIS.OFS is optional for both
7825  * DMA and PIO, and a COMRESET is required by software to clean up from
7826  * this serious error.
7827  *
7828  * Command list underflow is defined as software building a command
7829  * table that has more total bytes than the transaction given to the
7830  * device. For data writes, both PIO and DMA, the device will detect
7831  * an error and end the transfer. And these errors are most likely going
7832  * to be fatal errors that will cause the port to be restarted. For
7833  * data reads, the HBA updates its PRD byte count, and may be
7834  * able to continue normally, but is not required to. And The HBA is
7835  * not required to detect underflow conditions for native command
7836  * queuing command.
7837  *
7838  * The HBA does not actively check incoming DMA Setup FISes to ensure
7839  * that the PxSACT register bit for that slot is set. Existing error
7840  * mechanisms, such as host bus failure, or bad protocol, are used to
7841  * recover from this case.
7842  *
7843  * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7844  * DATA FIS must be an integral number of Dwords. If the HBA receives
7845  * a request which is not an integral number of Dwords, the HBA
7846  * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7847  * software restarts the port. And the HBA ensures that the size
7848  * of the DATA FIS received during a PIO command matches the size in
7849  * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7850  * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7851  * stop running until software restarts the port.
7852  */
7853 /*
7854  * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7855  *
7856  * PxIS.IFS indicates that the hba encountered an error on the serial ata
7857  * interface which caused the transfer to stop.
7858  *
7859  * PxIS.HBDS indicates that the hba encountered a data error
7860  * (uncorrectable ecc/parity) when reading from or writing to system memory.
7861  *
7862  * PxIS.HBFS indicates that the hba encountered a host bus error that it
7863  * cannot recover from, such as a bad software pointer.
7864  *
7865  * PxIS.TFES is set whenever the status register is updated by the device
7866  * and the error bit (bit 0) is set.
7867  */
7868 static int
ahci_intr_fatal_error(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,uint32_t intr_status)7869 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7870     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7871 {
7872 	uint32_t port_cmd_status;
7873 	uint32_t port_serror;
7874 	uint32_t task_file_status;
7875 	int failed_slot;
7876 	sata_pkt_t *spkt = NULL;
7877 	uint8_t err_byte;
7878 	ahci_event_arg_t *args;
7879 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7880 	uint32_t failed_tags = 0;
7881 	int task_fail_flag = 0, task_abort_flag = 0;
7882 	uint32_t slot_status;
7883 
7884 	mutex_enter(&ahci_portp->ahciport_mutex);
7885 
7886 	/*
7887 	 * ahci_intr_phyrdy_change() may have rendered it to
7888 	 * SATA_DTYPE_NONE.
7889 	 */
7890 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7891 		AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7892 		    "ahci_intr_fatal_error: port %d no device attached, "
7893 		    "and just return without doing anything", port);
7894 		goto out0;
7895 	}
7896 
7897 	if (intr_status & AHCI_INTR_STATUS_TFES) {
7898 		task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7899 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7900 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7901 		    "ahci_intr_fatal_error: port %d "
7902 		    "task_file_status = 0x%x", port, task_file_status);
7903 		task_fail_flag = 1;
7904 
7905 		err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7906 		    >> AHCI_TFD_ERR_SHIFT;
7907 		if (err_byte == SATA_ERROR_ABORT)
7908 			task_abort_flag = 1;
7909 	}
7910 
7911 	/*
7912 	 * Here we just log the fatal error info in interrupt context.
7913 	 * Misc recovery processing will be handled in task queue.
7914 	 */
7915 	if (task_fail_flag  == 1) {
7916 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7917 			/*
7918 			 * Read PxCMD.CCS to determine the slot that the HBA
7919 			 * was processing when the error occurred.
7920 			 */
7921 			port_cmd_status = ddi_get32(
7922 			    ahci_ctlp->ahcictl_ahci_acc_handle,
7923 			    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7924 			failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7925 			    AHCI_CMD_STATUS_CCS_SHIFT;
7926 			failed_tags = 0x1 << failed_slot;
7927 
7928 			spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7929 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7930 			    "ahci_intr_fatal_error: spkt 0x%p is being "
7931 			    "processed when fatal error occurred for port %d",
7932 			    spkt, port);
7933 
7934 			/*
7935 			 * Won't emit the error message if it is an IDENTIFY
7936 			 * DEVICE command sent to an ATAPI device.
7937 			 */
7938 			if ((spkt != NULL) &&
7939 			    (spkt->satapkt_cmd.satacmd_cmd_reg ==
7940 			    SATAC_ID_DEVICE) &&
7941 			    (task_abort_flag == 1))
7942 				goto out1;
7943 
7944 			/*
7945 			 * Won't emit the error message if it is an ATAPI PACKET
7946 			 * command
7947 			 */
7948 			if ((spkt != NULL) &&
7949 			    (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7950 				goto out1;
7951 
7952 		} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7953 			slot_status = ddi_get32(
7954 			    ahci_ctlp->ahcictl_ahci_acc_handle,
7955 			    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7956 			failed_tags = slot_status &
7957 			    AHCI_NCQ_SLOT_MASK(ahci_portp);
7958 		}
7959 	}
7960 
7961 	/* print the fatal error type */
7962 	ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7963 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7964 
7965 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7966 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7967 
7968 	/* print PxSERR related error message */
7969 	ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7970 
7971 	/* print task file register value */
7972 	if (task_fail_flag == 1) {
7973 		cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7974 		    "= 0x%x", instance, port, task_file_status);
7975 		if (task_abort_flag == 1) {
7976 			cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7977 			    "port %d are aborted", instance, port);
7978 			ahci_dump_commands(ahci_ctlp, port, failed_tags);
7979 		}
7980 	}
7981 
7982 out1:
7983 	/* Prepare the argument for the taskq */
7984 	args = ahci_portp->ahciport_event_args;
7985 	args->ahciea_ctlp = (void *)ahci_ctlp;
7986 	args->ahciea_portp = (void *)ahci_portp;
7987 	args->ahciea_event = intr_status;
7988 	AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7989 
7990 	/* Start the taskq to handle error recovery */
7991 	if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7992 	    ahci_events_handler,
7993 	    (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7994 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7995 		cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7996 		    "port %d failed", instance, port);
7997 	}
7998 out0:
7999 	mutex_exit(&ahci_portp->ahciport_mutex);
8000 
8001 	return (AHCI_SUCCESS);
8002 }
8003 
8004 /*
8005  * Hot Plug Operation for platforms that support Cold Presence Detect.
8006  *
8007  * When set, a device status has changed as detected by the cold presence
8008  * detect logic. This bit can either be set due to a non-connected port
8009  * receiving a device, or a connected port having its device removed.
8010  * This bit is only valid if the port supports cold presence detect as
8011  * indicated by PxCMD.CPD set to '1'.
8012  *
8013  * At the moment, this interrupt is not needed and disabled and we just
8014  * log the debug message.
8015  */
8016 static int
ahci_intr_cold_port_detect(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)8017 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
8018     ahci_port_t *ahci_portp, uint8_t port)
8019 {
8020 	uint32_t port_cmd_status;
8021 	sata_device_t sdevice;
8022 
8023 	AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8024 	    "ahci_intr_cold_port_detect enter, port %d", port);
8025 
8026 	mutex_enter(&ahci_portp->ahciport_mutex);
8027 
8028 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8029 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8030 	if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
8031 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8032 		    "port %d does not support cold presence detect, so "
8033 		    "we just ignore this interrupt", port);
8034 		mutex_exit(&ahci_portp->ahciport_mutex);
8035 		return (AHCI_SUCCESS);
8036 	}
8037 
8038 	AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8039 	    "port %d device status has changed", port);
8040 
8041 	bzero((void *)&sdevice, sizeof (sata_device_t));
8042 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
8043 	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
8044 	sdevice.satadev_addr.pmport = 0;
8045 	sdevice.satadev_state = SATA_PSTATE_PWRON;
8046 
8047 	if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
8048 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8049 		    "port %d: a device is hot plugged", port);
8050 		mutex_exit(&ahci_portp->ahciport_mutex);
8051 		sata_hba_event_notify(
8052 		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8053 		    &sdevice,
8054 		    SATA_EVNT_DEVICE_ATTACHED);
8055 		mutex_enter(&ahci_portp->ahciport_mutex);
8056 
8057 	} else {
8058 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8059 		    "port %d: a device is hot unplugged", port);
8060 		mutex_exit(&ahci_portp->ahciport_mutex);
8061 		sata_hba_event_notify(
8062 		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8063 		    &sdevice,
8064 		    SATA_EVNT_DEVICE_DETACHED);
8065 		mutex_enter(&ahci_portp->ahciport_mutex);
8066 	}
8067 
8068 	mutex_exit(&ahci_portp->ahciport_mutex);
8069 
8070 	return (AHCI_SUCCESS);
8071 }
8072 
8073 /*
8074  * Enable the interrupts for a particular port.
8075  */
8076 static void
ahci_enable_port_intrs(ahci_ctl_t * ahci_ctlp,uint8_t port)8077 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8078 {
8079 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8080 
8081 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8082 	    "ahci_enable_port_intrs enter, port %d", port);
8083 
8084 	/*
8085 	 * Clear port interrupt status before enabling interrupt
8086 	 */
8087 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8088 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
8089 	    AHCI_PORT_INTR_MASK);
8090 
8091 	/*
8092 	 * Clear the pending bit from IS.IPS
8093 	 */
8094 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8095 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
8096 
8097 	/*
8098 	 * Enable the following interrupts:
8099 	 *	Device to Host Register FIS Interrupt (DHRS)
8100 	 *	PIO Setup FIS Interrupt (PSS)
8101 	 *	Set Device Bits Interrupt (SDBS)
8102 	 *	Unknown FIS Interrupt (UFS)
8103 	 *	Port Connect Change Status (PCS)
8104 	 *	PhyRdy Change Status (PRCS)
8105 	 *	Overflow Status (OFS)
8106 	 *	Interface Non-fatal Error Status (INFS)
8107 	 *	Interface Fatal Error Status (IFS)
8108 	 *	Host Bus Data Error Status (HBDS)
8109 	 *	Host Bus Fatal Error Status (HBFS)
8110 	 *	Task File Error Status (TFES)
8111 	 */
8112 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8113 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8114 	    (AHCI_INTR_STATUS_DHRS |
8115 	    AHCI_INTR_STATUS_PSS |
8116 	    AHCI_INTR_STATUS_SDBS |
8117 	    AHCI_INTR_STATUS_UFS |
8118 	    AHCI_INTR_STATUS_DPS |
8119 	    AHCI_INTR_STATUS_PCS |
8120 	    AHCI_INTR_STATUS_PRCS |
8121 	    AHCI_INTR_STATUS_OFS |
8122 	    AHCI_INTR_STATUS_INFS |
8123 	    AHCI_INTR_STATUS_IFS |
8124 	    AHCI_INTR_STATUS_HBDS |
8125 	    AHCI_INTR_STATUS_HBFS |
8126 	    AHCI_INTR_STATUS_TFES));
8127 }
8128 
8129 /*
8130  * Enable interrupts for all the ports.
8131  */
8132 static void
ahci_enable_all_intrs(ahci_ctl_t * ahci_ctlp)8133 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8134 {
8135 	uint32_t ghc_control;
8136 
8137 	ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8138 
8139 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8140 
8141 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8142 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8143 
8144 	ghc_control |= AHCI_HBA_GHC_IE;
8145 
8146 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8147 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8148 }
8149 
8150 /*
8151  * Disable interrupts for a particular port.
8152  */
8153 static void
ahci_disable_port_intrs(ahci_ctl_t * ahci_ctlp,uint8_t port)8154 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8155 {
8156 	ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8157 	    MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8158 
8159 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8160 	    "ahci_disable_port_intrs enter, port %d", port);
8161 
8162 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8163 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8164 }
8165 
8166 /*
8167  * Disable interrupts for the whole HBA.
8168  *
8169  * The global bit is cleared, then all interrupt sources from all
8170  * ports are disabled.
8171  */
8172 static void
ahci_disable_all_intrs(ahci_ctl_t * ahci_ctlp)8173 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8174 {
8175 	uint32_t ghc_control;
8176 
8177 	ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8178 	    MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8179 
8180 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8181 	    NULL);
8182 
8183 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8184 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8185 
8186 	ghc_control &= ~AHCI_HBA_GHC_IE;
8187 
8188 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8189 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8190 }
8191 
8192 /*
8193  * Handle FIXED or MSI interrupts.
8194  */
8195 /*
8196  * According to AHCI spec, the HBA may support several interrupt modes:
8197  *	* pin based interrupts (FIXED)
8198  *	* single MSI message interrupts
8199  *	* multiple MSI based message interrupts
8200  *
8201  * For pin based interrupts, the software interrupt handler need to check IS
8202  * register to find out which port has pending interrupts. And then check
8203  * PxIS register to find out which interrupt events happened on that port.
8204  *
8205  * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8206  * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8207  * in that software interrupt handler need to check IS register to determine
8208  * which port triggered the interrupts since it uses a single message for all
8209  * port interrupts.
8210  *
8211  * HBA may optionally support multiple MSI message for better performance. In
8212  * this mode, each port may have its own interrupt message, and thus generation
8213  * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8214  * represents a power-of-2 wrapper on the number of implemented ports, and
8215  * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8216  * maximum number of assigned interrupts. When the number of MSI messages
8217  * allocated is less than the number requested, then hardware may have two
8218  * implementation behaviors:
8219  *	* assign each ports its own interrupt and then force all additional
8220  *	  ports to share the last interrupt message, and this condition is
8221  *	  indicated by clearing GHC.MRSM to '0'
8222  *	* revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8223  * When multiple-message MSI is enabled, hardware will still set IS register
8224  * as single message case. And this IS register may be used by software when
8225  * fewer than the requested number of messages is granted in order to determine
8226  * which port had the interrupt.
8227  *
8228  * Note: The current ahci driver only supports the first two interrupt modes:
8229  * pin based interrupts and single MSI message interrupts, and the reason
8230  * is indicated in below code.
8231  */
8232 static int
ahci_add_intrs(ahci_ctl_t * ahci_ctlp,int intr_type)8233 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8234 {
8235 	dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8236 	int		count, avail, actual;
8237 	int		i, rc;
8238 
8239 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8240 	    "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8241 
8242 	/* get number of interrupts. */
8243 	rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8244 	if ((rc != DDI_SUCCESS) || (count == 0)) {
8245 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8246 		    "ddi_intr_get_nintrs() failed, "
8247 		    "rc %d count %d\n", rc, count);
8248 		return (DDI_FAILURE);
8249 	}
8250 
8251 	/* get number of available interrupts. */
8252 	rc = ddi_intr_get_navail(dip, intr_type, &avail);
8253 	if ((rc != DDI_SUCCESS) || (avail == 0)) {
8254 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8255 		    "ddi_intr_get_navail() failed, "
8256 		    "rc %d avail %d\n", rc, avail);
8257 		return (DDI_FAILURE);
8258 	}
8259 
8260 #if AHCI_DEBUG
8261 	if (avail < count) {
8262 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8263 		    "ddi_intr_get_nintrs returned %d, navail() returned %d",
8264 		    count, avail);
8265 	}
8266 #endif
8267 
8268 	/*
8269 	 * Note: So far Solaris restricts the maximum number of messages for
8270 	 * x86 to 2, that is avail is 2, so here we set the count with 1 to
8271 	 * force the driver to use single MSI message interrupt. In future if
8272 	 * Solaris remove the restriction, then we need to delete the below
8273 	 * code and try to use multiple interrupt routine to gain better
8274 	 * performance.
8275 	 */
8276 	if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8277 		AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8278 		    "force to use one interrupt routine though the "
8279 		    "HBA supports %d interrupt", count);
8280 		count = 1;
8281 	}
8282 
8283 	/* Allocate an array of interrupt handles. */
8284 	ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8285 	ahci_ctlp->ahcictl_intr_htable =
8286 	    kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8287 
8288 	/* call ddi_intr_alloc(). */
8289 	rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8290 	    intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8291 
8292 	if ((rc != DDI_SUCCESS) || (actual == 0)) {
8293 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8294 		    "ddi_intr_alloc() failed, rc %d count %d actual %d "
8295 		    "avail %d\n", rc, count, actual, avail);
8296 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
8297 		    ahci_ctlp->ahcictl_intr_size);
8298 		return (DDI_FAILURE);
8299 	}
8300 
8301 	/* use interrupt count returned */
8302 #if AHCI_DEBUG
8303 	if (actual < count) {
8304 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8305 		    "Requested: %d, Received: %d", count, actual);
8306 	}
8307 #endif
8308 
8309 	ahci_ctlp->ahcictl_intr_cnt = actual;
8310 
8311 	/*
8312 	 * Get priority for first, assume remaining are all the same.
8313 	 */
8314 	if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8315 	    &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8316 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8317 		    "ddi_intr_get_pri() failed", NULL);
8318 
8319 		/* Free already allocated intr. */
8320 		for (i = 0; i < actual; i++) {
8321 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8322 		}
8323 
8324 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
8325 		    ahci_ctlp->ahcictl_intr_size);
8326 		return (DDI_FAILURE);
8327 	}
8328 
8329 	/* Test for high level interrupt. */
8330 	if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8331 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8332 		    "ahci_add_intrs: Hi level intr not supported", NULL);
8333 
8334 		/* Free already allocated intr. */
8335 		for (i = 0; i < actual; i++) {
8336 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8337 		}
8338 
8339 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
8340 		    sizeof (ddi_intr_handle_t));
8341 
8342 		return (DDI_FAILURE);
8343 	}
8344 
8345 	/* Call ddi_intr_add_handler(). */
8346 	for (i = 0; i < actual; i++) {
8347 		if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8348 		    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8349 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8350 			    "ddi_intr_add_handler() failed", NULL);
8351 
8352 			/* Free already allocated intr. */
8353 			for (i = 0; i < actual; i++) {
8354 				(void) ddi_intr_free(
8355 				    ahci_ctlp->ahcictl_intr_htable[i]);
8356 			}
8357 
8358 			kmem_free(ahci_ctlp->ahcictl_intr_htable,
8359 			    ahci_ctlp->ahcictl_intr_size);
8360 			return (DDI_FAILURE);
8361 		}
8362 	}
8363 
8364 	if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8365 	    &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8366 		AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8367 		    "ddi_intr_get_cap() failed", NULL);
8368 
8369 		/* Free already allocated intr. */
8370 		for (i = 0; i < actual; i++) {
8371 			(void) ddi_intr_free(
8372 			    ahci_ctlp->ahcictl_intr_htable[i]);
8373 		}
8374 
8375 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
8376 		    ahci_ctlp->ahcictl_intr_size);
8377 		return (DDI_FAILURE);
8378 	}
8379 
8380 	if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8381 		/* Call ddi_intr_block_enable() for MSI. */
8382 		(void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8383 		    ahci_ctlp->ahcictl_intr_cnt);
8384 	} else {
8385 		/* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8386 		for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8387 			(void) ddi_intr_enable(
8388 			    ahci_ctlp->ahcictl_intr_htable[i]);
8389 		}
8390 	}
8391 
8392 	return (DDI_SUCCESS);
8393 }
8394 
8395 /*
8396  * Removes the registered interrupts irrespective of whether they
8397  * were legacy or MSI.
8398  *
8399  * NOTE: The controller interrupts must be disabled before calling
8400  * this routine.
8401  */
8402 static void
ahci_rem_intrs(ahci_ctl_t * ahci_ctlp)8403 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8404 {
8405 	int x;
8406 
8407 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8408 
8409 	/* Disable all interrupts. */
8410 	if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8411 	    (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8412 		/* Call ddi_intr_block_disable(). */
8413 		(void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8414 		    ahci_ctlp->ahcictl_intr_cnt);
8415 	} else {
8416 		for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8417 			(void) ddi_intr_disable(
8418 			    ahci_ctlp->ahcictl_intr_htable[x]);
8419 		}
8420 	}
8421 
8422 	/* Call ddi_intr_remove_handler(). */
8423 	for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8424 		(void) ddi_intr_remove_handler(
8425 		    ahci_ctlp->ahcictl_intr_htable[x]);
8426 		(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8427 	}
8428 
8429 	kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8430 }
8431 
8432 /*
8433  * This routine tries to put port into P:NotRunning state by clearing
8434  * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8435  * and PxCMD.CR to '0'.
8436  */
8437 static int
ahci_put_port_into_notrunning_state(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)8438 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8439     ahci_port_t *ahci_portp, uint8_t port)
8440 {
8441 	uint32_t port_cmd_status;
8442 	int loop_count;
8443 
8444 	ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8445 	    MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8446 
8447 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8448 	    "ahci_put_port_into_notrunning_state enter: port %d", port);
8449 
8450 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8451 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8452 
8453 	port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8454 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8455 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8456 
8457 	/* Wait until PxCMD.CR is cleared */
8458 	loop_count = 0;
8459 	do {
8460 		port_cmd_status =
8461 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8462 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8463 
8464 		if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8465 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8466 			    "clearing port %d CMD.CR timeout, "
8467 			    "port_cmd_status = 0x%x", port,
8468 			    port_cmd_status);
8469 			/*
8470 			 * We are effectively timing out after 0.5 sec.
8471 			 * This value is specified in AHCI spec.
8472 			 */
8473 			break;
8474 		}
8475 
8476 		/* Wait for 10 millisec */
8477 		drv_usecwait(AHCI_10MS_USECS);
8478 	} while (port_cmd_status & AHCI_CMD_STATUS_CR);
8479 
8480 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8481 
8482 	if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8483 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8484 		    "ahci_put_port_into_notrunning_state: failed to clear "
8485 		    "PxCMD.CR to '0' after loop count: %d, and "
8486 		    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8487 		return (AHCI_FAILURE);
8488 	} else {
8489 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8490 		    "ahci_put_port_into_notrunning_state: succeeded to clear "
8491 		    "PxCMD.CR to '0' after loop count: %d, and "
8492 		    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8493 		return (AHCI_SUCCESS);
8494 	}
8495 }
8496 
8497 /*
8498  * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8499  * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8500  * stable state, then set PxCMD.ST to '1' to start the port directly.
8501  * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8502  * COMRESET to the device to put it in an idle state.
8503  *
8504  * The fifth argument returns whether the port reset is involved during
8505  * the process.
8506  *
8507  * The routine will be called under following scenarios:
8508  *	+ To reset the HBA
8509  *	+ To abort the packet(s)
8510  *	+ To reset the port
8511  *	+ To activate the port
8512  *	+ Fatal error recovery
8513  *	+ To abort the timeout packet(s)
8514  *
8515  * NOTES!!! During this procedure, PxSERR register will be cleared, and
8516  * according to the spec, the clearance of three bits will also clear
8517  * three interrupt status bits.
8518  *	1. PxSERR.DIAG.F will clear PxIS.UFS
8519  *	2. PxSERR.DIAG.X will clear PxIS.PCS
8520  *	3. PxSERR.DIAG.N will clear PxIS.PRCS
8521  *
8522  * Among these three interrupt events, the driver needs to take care of
8523  * PxIS.PRCS, which is the hot plug event. When the driver found out
8524  * a device was unplugged, it will call the interrupt handler.
8525  */
8526 static int
ahci_restart_port_wait_till_ready(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,int flag,int * reset_flag)8527 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8528     ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8529 {
8530 	uint32_t port_sstatus;
8531 	uint32_t task_file_status;
8532 	sata_device_t sdevice;
8533 	int rval;
8534 	ahci_addr_t addr_port;
8535 	ahci_pmult_info_t *pminfo = NULL;
8536 	int dev_exists_begin = 0;
8537 	int dev_exists_end = 0;
8538 	uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8539 	int npmport = 0;
8540 	uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8541 
8542 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8543 
8544 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8545 	    "ahci_restart_port_wait_till_ready: port %d enter", port);
8546 
8547 	AHCI_ADDR_SET_PORT(&addr_port, port);
8548 
8549 	if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8550 		dev_exists_begin = 1;
8551 
8552 	/* First clear PxCMD.ST */
8553 	rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8554 	    port);
8555 	if (rval != AHCI_SUCCESS)
8556 		/*
8557 		 * If PxCMD.CR does not clear within a reasonable time, it
8558 		 * may assume the interface is in a hung condition and may
8559 		 * continue with issuing the port reset.
8560 		 */
8561 		goto reset;
8562 
8563 	/* Then clear PxSERR */
8564 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8565 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8566 	    AHCI_SERROR_CLEAR_ALL);
8567 
8568 	/* Then get PxTFD */
8569 	task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8570 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8571 
8572 	/*
8573 	 * Check whether the device is in a stable status, if yes,
8574 	 * then start the port directly. However for ahci_tran_reset_dport,
8575 	 * we may have to perform a port reset.
8576 	 */
8577 	if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8578 	    !(flag & AHCI_PORT_RESET))
8579 		goto out;
8580 
8581 reset:
8582 	/*
8583 	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8584 	 * a COMRESET to the device
8585 	 */
8586 	ahci_disable_port_intrs(ahci_ctlp, port);
8587 	rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8588 	ahci_enable_port_intrs(ahci_ctlp, port);
8589 
8590 #ifdef AHCI_DEBUG
8591 	if (rval != AHCI_SUCCESS)
8592 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8593 		    "ahci_restart_port_wait_till_ready: port %d failed",
8594 		    port);
8595 #endif
8596 
8597 	if (reset_flag != NULL)
8598 		*reset_flag = 1;
8599 
8600 	/* Indicate to the framework that a reset has happened. */
8601 	if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8602 	    (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8603 	    !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8604 		/* Set the reset in progress flag */
8605 		ahci_portp->ahciport_reset_in_progress = 1;
8606 
8607 		bzero((void *)&sdevice, sizeof (sata_device_t));
8608 		sdevice.satadev_addr.cport =
8609 		    ahci_ctlp->ahcictl_port_to_cport[port];
8610 		sdevice.satadev_addr.pmport = 0;
8611 		sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8612 
8613 		sdevice.satadev_state = SATA_DSTATE_RESET |
8614 		    SATA_DSTATE_PWR_ACTIVE;
8615 		if (ahci_ctlp->ahcictl_sata_hba_tran) {
8616 			mutex_exit(&ahci_portp->ahciport_mutex);
8617 			sata_hba_event_notify(
8618 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8619 			    &sdevice,
8620 			    SATA_EVNT_DEVICE_RESET);
8621 			mutex_enter(&ahci_portp->ahciport_mutex);
8622 		}
8623 
8624 		AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8625 		    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8626 	} else {
8627 		ahci_portp->ahciport_reset_in_progress = 0;
8628 	}
8629 
8630 out:
8631 	(void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8632 
8633 	/* SStatus tells the presence of device. */
8634 	port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8635 	    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8636 
8637 	if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8638 		dev_exists_end = 1;
8639 	}
8640 
8641 	if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8642 		return (rval);
8643 
8644 	/* Check whether a hot plug event happened */
8645 	if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8646 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8647 		    "ahci_restart_port_wait_till_ready: port %d "
8648 		    "device is removed", port);
8649 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8650 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8651 		    "ahci_restart_port_wait_till_ready: port %d "
8652 		    "AHCI_PORT_FLAG_NODEV flag is set", port);
8653 		mutex_exit(&ahci_portp->ahciport_mutex);
8654 		(void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8655 		mutex_enter(&ahci_portp->ahciport_mutex);
8656 
8657 		return (rval);
8658 	}
8659 
8660 
8661 	/* 0/1 -> 1 : device may change */
8662 	/*
8663 	 * May be called by ahci_fatal_error_recovery_handler, so
8664 	 * don't issue software if the previous device is ATAPI.
8665 	 */
8666 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8667 		return (rval);
8668 
8669 	/*
8670 	 * The COMRESET will make port multiplier enter legacy mode.
8671 	 * Issue a software reset to make it work again.
8672 	 */
8673 	ahci_disable_port_intrs(ahci_ctlp, port);
8674 	ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8675 	ahci_enable_port_intrs(ahci_ctlp, port);
8676 
8677 	/*
8678 	 * Following codes are specific for the port multiplier
8679 	 */
8680 	if (previous_dev_type != SATA_DTYPE_PMULT &&
8681 	    ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8682 		/* in case previous_dev_type is corrupt */
8683 		ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8684 		(void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8685 		return (rval);
8686 	}
8687 
8688 	/* Device change: PMult -> Non-PMult */
8689 	if (previous_dev_type == SATA_DTYPE_PMULT &&
8690 	    ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8691 		/*
8692 		 * This might happen because
8693 		 * 1. Software reset failed. Port multiplier is not correctly
8694 		 *    enumerated.
8695 		 * 2. Another non-port-multiplier device is attached. Perhaps
8696 		 *    the port multiplier was replaced by another device by
8697 		 *    whatever reason, but AHCI driver missed hot-plug event.
8698 		 *
8699 		 * Now that the port has been initialized, we just need to
8700 		 * update the port structure according new device, then report
8701 		 * and wait SATA framework to probe new device.
8702 		 */
8703 
8704 		/* Force to release pmult resource */
8705 		ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8706 		(void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8707 
8708 		bzero((void *)&sdevice, sizeof (sata_device_t));
8709 		sdevice.satadev_addr.cport =
8710 		    ahci_ctlp->ahcictl_port_to_cport[port];
8711 		sdevice.satadev_addr.pmport = 0;
8712 		sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8713 
8714 		sdevice.satadev_state = SATA_DSTATE_RESET |
8715 		    SATA_DSTATE_PWR_ACTIVE;
8716 
8717 		mutex_exit(&ahci_portp->ahciport_mutex);
8718 		sata_hba_event_notify(
8719 		    ahci_ctlp->ahcictl_dip,
8720 		    &sdevice,
8721 		    SATA_EVNT_DEVICE_RESET);
8722 		mutex_enter(&ahci_portp->ahciport_mutex);
8723 
8724 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8725 		    "Port multiplier is [Gone] at port %d ", port);
8726 		AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8727 		    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8728 
8729 		return (AHCI_SUCCESS);
8730 	}
8731 
8732 	/* Device change: Non-PMult -> PMult */
8733 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8734 
8735 		/* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8736 		ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8737 
8738 		(void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8739 	}
8740 	pminfo = ahci_portp->ahciport_pmult_info;
8741 	ASSERT(pminfo != NULL);
8742 
8743 	/* Device (may) change: PMult -> PMult */
8744 	/*
8745 	 * First initialize port multiplier. Set state to READY and wait for
8746 	 * probe entry point to initialize it
8747 	 */
8748 	ahci_portp->ahciport_port_state = SATA_STATE_READY;
8749 
8750 	/*
8751 	 * It's a little complicated while target is a port multiplier. we
8752 	 * need to COMRESET all pmports behind that PMult otherwise those
8753 	 * sub-links between the PMult and the sub-devices will be in an
8754 	 * inactive state (indicated by PSCR0/PxSSTS) and the following access
8755 	 * to those sub-devices will be rejected by Link-Fatal-Error.
8756 	 */
8757 	/*
8758 	 * The PxSNTF will be set soon after the pmult is plugged. While the
8759 	 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8760 	 * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8761 	 * will initialized it.
8762 	 */
8763 	for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8764 		pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8765 
8766 	/* Report reset event. */
8767 	ahci_portp->ahciport_reset_in_progress = 1;
8768 
8769 	bzero((void *)&sdevice, sizeof (sata_device_t));
8770 	sdevice.satadev_addr.cport = cport;
8771 	sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8772 	sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8773 	sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8774 	sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8775 	    SATA_EVNT_DEVICE_RESET);
8776 
8777 	return (rval);
8778 }
8779 
8780 /*
8781  * This routine may be called under four scenarios:
8782  *	a) do the recovery from fatal error
8783  *	b) or we need to timeout some commands
8784  *	c) or we need to abort some commands
8785  *	d) or we need reset device/port/controller
8786  *
8787  * In all these scenarios, we need to send any pending unfinished
8788  * commands up to sata framework.
8789  */
8790 static void
ahci_mop_commands(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint32_t slot_status,uint32_t failed_tags,uint32_t timeout_tags,uint32_t aborted_tags,uint32_t reset_tags)8791 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8792     ahci_port_t *ahci_portp,
8793     uint32_t slot_status,
8794     uint32_t failed_tags,
8795     uint32_t timeout_tags,
8796     uint32_t aborted_tags,
8797     uint32_t reset_tags)
8798 {
8799 	uint32_t finished_tags = 0;
8800 	uint32_t unfinished_tags = 0;
8801 	int tmp_slot;
8802 	sata_pkt_t *satapkt;
8803 	int ncq_cmd_in_progress = 0;
8804 	int err_retri_cmd_in_progress = 0;
8805 	int rdwr_pmult_cmd_in_progress = 0;
8806 
8807 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8808 
8809 	AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8810 	    "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8811 	    ahci_portp->ahciport_port_num, slot_status);
8812 
8813 	AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8814 	    "ahci_mop_commands: failed_tags: 0x%x, "
8815 	    "timeout_tags: 0x%x aborted_tags: 0x%x, "
8816 	    "reset_tags: 0x%x", failed_tags,
8817 	    timeout_tags, aborted_tags, reset_tags);
8818 
8819 #ifdef AHCI_DEBUG
8820 	if (ahci_debug_flags & AHCIDBG_ERRS) {
8821 		int i;
8822 		char msg_buf[200] = {0, };
8823 		for (i = 0x1f; i >= 0; i--) {
8824 			if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8825 				msg_buf[i] = 'X';
8826 			else
8827 				msg_buf[i] = '.';
8828 		}
8829 		msg_buf[0x20] = '\0';
8830 		cmn_err(CE_NOTE, "port[%d] slots: %s",
8831 		    ahci_portp->ahciport_port_num, msg_buf);
8832 		cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8833 		    (void *)ahci_portp->ahciport_err_retri_pkt,
8834 		    (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8835 	}
8836 #endif
8837 
8838 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8839 		finished_tags = ahci_portp->ahciport_pending_tags &
8840 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8841 
8842 		unfinished_tags = slot_status &
8843 		    AHCI_SLOT_MASK(ahci_ctlp) &
8844 		    ~failed_tags &
8845 		    ~aborted_tags &
8846 		    ~reset_tags &
8847 		    ~timeout_tags;
8848 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8849 		ncq_cmd_in_progress = 1;
8850 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8851 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8852 
8853 		unfinished_tags = slot_status &
8854 		    AHCI_NCQ_SLOT_MASK(ahci_portp) &
8855 		    ~failed_tags &
8856 		    ~aborted_tags &
8857 		    ~reset_tags &
8858 		    ~timeout_tags;
8859 	} else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8860 
8861 		/*
8862 		 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8863 		 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8864 		 * complete successfully due to one of the following three
8865 		 * conditions:
8866 		 *
8867 		 *	1. Fatal error - failed_tags includes its slot
8868 		 *	2. Timed out - timeout_tags includes its slot
8869 		 *	3. Aborted when hot unplug - aborted_tags includes its
8870 		 *	   slot
8871 		 *
8872 		 * Please note that the command is always sent down in Slot 0
8873 		 */
8874 		err_retri_cmd_in_progress = 1;
8875 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8876 		    "ahci_mop_commands is called for port %d while "
8877 		    "REQUEST SENSE or READ LOG EXT for error retrieval "
8878 		    "is being executed slot_status = 0x%x",
8879 		    ahci_portp->ahciport_port_num, slot_status);
8880 		ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8881 		ASSERT(slot_status == 0x1);
8882 	} else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8883 		rdwr_pmult_cmd_in_progress = 1;
8884 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8885 		    "ahci_mop_commands is called for port %d while "
8886 		    "READ/WRITE PORTMULT command is being executed",
8887 		    ahci_portp->ahciport_port_num);
8888 
8889 		ASSERT(slot_status == 0x1);
8890 	}
8891 
8892 #ifdef AHCI_DEBUG
8893 	AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8894 	    "ahci_mop_commands: finished_tags: 0x%x, "
8895 	    "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8896 #endif
8897 
8898 	/* Send up finished packets with SATA_PKT_COMPLETED */
8899 	while (finished_tags) {
8900 		tmp_slot = ddi_ffs(finished_tags) - 1;
8901 		if (tmp_slot == -1) {
8902 			break;
8903 		}
8904 
8905 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8906 		ASSERT(satapkt != NULL);
8907 
8908 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8909 		    "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8910 		    (void *)satapkt);
8911 
8912 		/*
8913 		 * Cannot fetch the return register content since the port
8914 		 * was restarted, so the corresponding tag will be set to
8915 		 * aborted tags.
8916 		 */
8917 		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8918 			CLEAR_BIT(finished_tags, tmp_slot);
8919 			aborted_tags |= tmp_slot;
8920 			continue;
8921 		}
8922 
8923 		if (ncq_cmd_in_progress)
8924 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8925 			    tmp_slot);
8926 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8927 		CLEAR_BIT(finished_tags, tmp_slot);
8928 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8929 
8930 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8931 	}
8932 
8933 	/* Send up failed packets with SATA_PKT_DEV_ERROR. */
8934 	while (failed_tags) {
8935 		if (err_retri_cmd_in_progress) {
8936 			satapkt = ahci_portp->ahciport_err_retri_pkt;
8937 			ASSERT(satapkt != NULL);
8938 			ASSERT(failed_tags == 0x1);
8939 
8940 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8941 			    "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8942 			    (void *)satapkt);
8943 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8944 			break;
8945 		}
8946 		if (rdwr_pmult_cmd_in_progress) {
8947 			satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8948 			ASSERT(satapkt != NULL);
8949 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8950 			    "ahci_mop_commands: sending up "
8951 			    "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8952 			    (void *)satapkt);
8953 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8954 			break;
8955 		}
8956 
8957 		tmp_slot = ddi_ffs(failed_tags) - 1;
8958 		if (tmp_slot == -1) {
8959 			break;
8960 		}
8961 
8962 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8963 		ASSERT(satapkt != NULL);
8964 
8965 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8966 		    "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8967 		    (void *)satapkt);
8968 
8969 		if (ncq_cmd_in_progress)
8970 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8971 			    tmp_slot);
8972 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8973 		CLEAR_BIT(failed_tags, tmp_slot);
8974 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8975 
8976 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8977 	}
8978 
8979 	/* Send up timeout packets with SATA_PKT_TIMEOUT. */
8980 	while (timeout_tags) {
8981 		if (err_retri_cmd_in_progress) {
8982 			satapkt = ahci_portp->ahciport_err_retri_pkt;
8983 			ASSERT(satapkt != NULL);
8984 			ASSERT(timeout_tags == 0x1);
8985 
8986 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8987 			    "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8988 			    (void *)satapkt);
8989 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8990 			break;
8991 		}
8992 		if (rdwr_pmult_cmd_in_progress) {
8993 			satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8994 			ASSERT(satapkt != NULL);
8995 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8996 			    "ahci_mop_commands: sending up "
8997 			    "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8998 			    (void *)satapkt);
8999 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
9000 			break;
9001 		}
9002 
9003 		tmp_slot = ddi_ffs(timeout_tags) - 1;
9004 		if (tmp_slot == -1) {
9005 			break;
9006 		}
9007 
9008 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9009 		ASSERT(satapkt != NULL);
9010 
9011 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9012 		    "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
9013 		    (void *)satapkt);
9014 
9015 		if (ncq_cmd_in_progress)
9016 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9017 			    tmp_slot);
9018 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9019 		CLEAR_BIT(timeout_tags, tmp_slot);
9020 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9021 
9022 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
9023 	}
9024 
9025 	/* Send up aborted packets with SATA_PKT_ABORTED */
9026 	while (aborted_tags) {
9027 		if (err_retri_cmd_in_progress) {
9028 			satapkt = ahci_portp->ahciport_err_retri_pkt;
9029 			ASSERT(satapkt != NULL);
9030 			ASSERT(aborted_tags == 0x1);
9031 
9032 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9033 			    "sending up pkt 0x%p with SATA_PKT_ABORTED",
9034 			    (void *)satapkt);
9035 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9036 			break;
9037 		}
9038 		if (rdwr_pmult_cmd_in_progress) {
9039 			satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9040 			ASSERT(satapkt != NULL);
9041 			ASSERT(aborted_tags == 0x1);
9042 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9043 			    "ahci_mop_commands: sending up "
9044 			    "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
9045 			    (void *)satapkt);
9046 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9047 			break;
9048 		}
9049 
9050 		tmp_slot = ddi_ffs(aborted_tags) - 1;
9051 		if (tmp_slot == -1) {
9052 			break;
9053 		}
9054 
9055 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9056 		ASSERT(satapkt != NULL);
9057 
9058 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9059 		    "sending up pkt 0x%p with SATA_PKT_ABORTED",
9060 		    (void *)satapkt);
9061 
9062 		if (ncq_cmd_in_progress)
9063 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9064 			    tmp_slot);
9065 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9066 		CLEAR_BIT(aborted_tags, tmp_slot);
9067 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9068 
9069 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
9070 	}
9071 
9072 	/* Send up reset packets with SATA_PKT_RESET. */
9073 	while (reset_tags) {
9074 		if (rdwr_pmult_cmd_in_progress) {
9075 			satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9076 			ASSERT(satapkt != NULL);
9077 			ASSERT(aborted_tags == 0x1);
9078 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9079 			    "ahci_mop_commands: sending up "
9080 			    "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
9081 			    (void *)satapkt);
9082 			ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9083 			break;
9084 		}
9085 
9086 		tmp_slot = ddi_ffs(reset_tags) - 1;
9087 		if (tmp_slot == -1) {
9088 			break;
9089 		}
9090 
9091 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9092 		ASSERT(satapkt != NULL);
9093 
9094 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9095 		    "sending up pkt 0x%p with SATA_PKT_RESET",
9096 		    (void *)satapkt);
9097 
9098 		if (ncq_cmd_in_progress)
9099 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9100 			    tmp_slot);
9101 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9102 		CLEAR_BIT(reset_tags, tmp_slot);
9103 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9104 
9105 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9106 	}
9107 
9108 	/* Send up unfinished packets with SATA_PKT_RESET */
9109 	while (unfinished_tags) {
9110 		tmp_slot = ddi_ffs(unfinished_tags) - 1;
9111 		if (tmp_slot == -1) {
9112 			break;
9113 		}
9114 
9115 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9116 		ASSERT(satapkt != NULL);
9117 
9118 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9119 		    "sending up pkt 0x%p with SATA_PKT_RESET",
9120 		    (void *)satapkt);
9121 
9122 		if (ncq_cmd_in_progress)
9123 			CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9124 			    tmp_slot);
9125 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9126 		CLEAR_BIT(unfinished_tags, tmp_slot);
9127 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9128 
9129 		ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9130 	}
9131 
9132 	ahci_portp->ahciport_mop_in_progress--;
9133 	ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9134 
9135 	if (ahci_portp->ahciport_mop_in_progress == 0)
9136 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9137 
9138 	ahci_flush_doneq(ahci_portp);
9139 }
9140 
9141 /*
9142  * This routine is going to first request a READ LOG EXT sata pkt from sata
9143  * module, and then deliver it to the HBA to get the ncq failure context.
9144  * The return value is the exactly failed tags.
9145  */
9146 static uint32_t
ahci_get_rdlogext_data(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)9147 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9148     uint8_t port)
9149 {
9150 	sata_device_t	sdevice;
9151 	sata_pkt_t	*rdlog_spkt, *spkt;
9152 	ddi_dma_handle_t buf_dma_handle;
9153 	ahci_addr_t	addr;
9154 	int		loop_count;
9155 	int		rval;
9156 	int		failed_slot;
9157 	uint32_t	failed_tags = 0;
9158 	struct sata_ncq_error_recovery_page *ncq_err_page;
9159 
9160 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9161 	    "ahci_get_rdlogext_data enter: port %d", port);
9162 
9163 	/* Prepare the sdevice data */
9164 	bzero((void *)&sdevice, sizeof (sata_device_t));
9165 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9166 
9167 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9168 	sdevice.satadev_addr.pmport = 0;
9169 
9170 	/* Translate sata_device.satadev_addr -> ahci_addr */
9171 	ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9172 
9173 	/*
9174 	 * Call the sata hba interface to get a rdlog spkt
9175 	 */
9176 	loop_count = 0;
9177 loop:
9178 	rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9179 	    &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9180 	if (rdlog_spkt == NULL) {
9181 		if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9182 			/* Sleep for a while */
9183 			drv_usecwait(AHCI_10MS_USECS);
9184 			goto loop;
9185 		}
9186 		/* Timed out after 1s */
9187 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9188 		    "failed to get rdlog spkt for port %d", port);
9189 		return (failed_tags);
9190 	}
9191 
9192 	ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9193 
9194 	/*
9195 	 * This flag is used to handle the specific error recovery when the
9196 	 * READ LOG EXT command gets a failure (fatal error or time-out).
9197 	 */
9198 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9199 
9200 	/*
9201 	 * This start is not supposed to fail because after port is restarted,
9202 	 * the whole command list is empty.
9203 	 */
9204 	ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9205 	(void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9206 	ahci_portp->ahciport_err_retri_pkt = NULL;
9207 
9208 	/* Remove the flag after READ LOG EXT command is completed */
9209 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9210 
9211 	if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9212 		/* Update the request log data */
9213 		buf_dma_handle = *(ddi_dma_handle_t *)
9214 		    (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9215 		rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9216 		    DDI_DMA_SYNC_FORKERNEL);
9217 		if (rval == DDI_SUCCESS) {
9218 			ncq_err_page =
9219 			    (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9220 			    satapkt_cmd.satacmd_bp->b_un.b_addr;
9221 
9222 			/* Get the failed tag */
9223 			failed_slot = ncq_err_page->ncq_tag;
9224 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9225 			    "ahci_get_rdlogext_data: port %d "
9226 			    "failed slot %d", port, failed_slot);
9227 			if (failed_slot & NQ) {
9228 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9229 				    "the failed slot is not a valid tag", NULL);
9230 				goto out;
9231 			}
9232 
9233 			failed_slot &= NCQ_TAG_MASK;
9234 			spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9235 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9236 			    "ahci_get_rdlogext_data: failed spkt 0x%p",
9237 			    (void *)spkt);
9238 			if (spkt == NULL) {
9239 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9240 				    "the failed slot spkt is NULL", NULL);
9241 				goto out;
9242 			}
9243 
9244 			failed_tags = 0x1 << failed_slot;
9245 
9246 			/* Fill out the error context */
9247 			ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9248 			    ncq_err_page);
9249 			ahci_update_sata_registers(ahci_ctlp, port,
9250 			    &spkt->satapkt_device);
9251 		}
9252 	}
9253 out:
9254 	sata_free_error_retrieval_pkt(rdlog_spkt);
9255 
9256 	return (failed_tags);
9257 }
9258 
9259 /*
9260  * This routine is going to first request a REQUEST SENSE sata pkt from sata
9261  * module, and then deliver it to the HBA to get the sense data and copy
9262  * the sense data back to the orignal failed sata pkt, and free the REQUEST
9263  * SENSE sata pkt later.
9264  */
9265 static void
ahci_get_rqsense_data(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,sata_pkt_t * spkt)9266 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9267     uint8_t port, sata_pkt_t *spkt)
9268 {
9269 	sata_device_t	sdevice;
9270 	sata_pkt_t	*rs_spkt;
9271 	sata_cmd_t	*sata_cmd;
9272 	ddi_dma_handle_t buf_dma_handle;
9273 	ahci_addr_t	addr;
9274 	int		loop_count;
9275 #if AHCI_DEBUG
9276 	struct scsi_extended_sense *rqsense;
9277 #endif
9278 
9279 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9280 	    "ahci_get_rqsense_data enter: port %d", port);
9281 
9282 	/* Prepare the sdevice data */
9283 	bzero((void *)&sdevice, sizeof (sata_device_t));
9284 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9285 
9286 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9287 	sdevice.satadev_addr.pmport = 0;
9288 
9289 	/* Translate sata_device.satadev_addr -> ahci_addr */
9290 	ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9291 
9292 	sata_cmd = &spkt->satapkt_cmd;
9293 
9294 	/*
9295 	 * Call the sata hba interface to get a rs spkt
9296 	 */
9297 	loop_count = 0;
9298 loop:
9299 	rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9300 	    &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9301 	if (rs_spkt == NULL) {
9302 		if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9303 			/* Sleep for a while */
9304 			drv_usecwait(AHCI_10MS_USECS);
9305 			goto loop;
9306 
9307 		}
9308 		/* Timed out after 1s */
9309 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9310 		    "failed to get rs spkt for port %d", port);
9311 		return;
9312 	}
9313 
9314 	ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9315 
9316 	/*
9317 	 * This flag is used to handle the specific error recovery when the
9318 	 * REQUEST SENSE command gets a faiure (fatal error or time-out).
9319 	 */
9320 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9321 
9322 	/*
9323 	 * This start is not supposed to fail because after port is restarted,
9324 	 * the whole command list is empty.
9325 	 */
9326 	ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9327 	(void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9328 	ahci_portp->ahciport_err_retri_pkt = NULL;
9329 
9330 	/* Remove the flag after REQUEST SENSE command is completed */
9331 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9332 
9333 	if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9334 		/* Update the request sense data */
9335 		buf_dma_handle = *(ddi_dma_handle_t *)
9336 		    (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9337 		(void) ddi_dma_sync(buf_dma_handle, 0, 0,
9338 		    DDI_DMA_SYNC_FORKERNEL);
9339 		/* Copy the request sense data */
9340 		bcopy(rs_spkt->
9341 		    satapkt_cmd.satacmd_bp->b_un.b_addr,
9342 		    &sata_cmd->satacmd_rqsense,
9343 		    SATA_ATAPI_MIN_RQSENSE_LEN);
9344 #if AHCI_DEBUG
9345 		rqsense = (struct scsi_extended_sense *)
9346 		    sata_cmd->satacmd_rqsense;
9347 
9348 		/* Dump the sense data */
9349 		AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9350 		AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9351 		    "Sense data for satapkt %p ATAPI cmd 0x%x",
9352 		    spkt, sata_cmd->satacmd_acdb[0]);
9353 		AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9354 		    "  es_code 0x%x es_class 0x%x "
9355 		    "es_key 0x%x es_add_code 0x%x "
9356 		    "es_qual_code 0x%x",
9357 		    rqsense->es_code, rqsense->es_class,
9358 		    rqsense->es_key, rqsense->es_add_code,
9359 		    rqsense->es_qual_code);
9360 #endif
9361 	}
9362 
9363 	sata_free_error_retrieval_pkt(rs_spkt);
9364 }
9365 
9366 /*
9367  * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9368  * the port must be restarted. When the HBA detects thus error, it may try
9369  * to abort a transfer. And if the transfer was aborted, the device is
9370  * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9371  * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9372  * that the device is in a stable status and transfers may be restarted without
9373  * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9374  * then the software will send the COMRESET to do the port reset.
9375  *
9376  * Software should perform the appropriate error recovery actions based on
9377  * whether non-queued commands were being issued or natived command queuing
9378  * commands were being issued.
9379  *
9380  * And software will complete the command that had the error with error mark
9381  * to higher level software.
9382  *
9383  * Fatal errors include the following:
9384  *	PxIS.IFS - Interface Fatal Error Status
9385  *	PxIS.HBDS - Host Bus Data Error Status
9386  *	PxIS.HBFS - Host Bus Fatal Error Status
9387  *	PxIS.TFES - Task File Error Status
9388  */
9389 static void
ahci_fatal_error_recovery_handler(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,uint32_t intr_status)9390 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9391     ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9392 {
9393 	uint32_t	port_cmd_status;
9394 	uint32_t	slot_status = 0;
9395 	uint32_t	failed_tags = 0;
9396 	int		failed_slot;
9397 	int		reset_flag = 0, flag = 0;
9398 	ahci_fis_d2h_register_t	*ahci_rcvd_fisp;
9399 	sata_cmd_t	*sata_cmd = NULL;
9400 	sata_pkt_t	*spkt = NULL;
9401 #if AHCI_DEBUG
9402 	ahci_cmd_header_t *cmd_header;
9403 #endif
9404 	uint8_t		port = addrp->aa_port;
9405 	int		instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9406 	int		rval;
9407 
9408 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9409 
9410 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9411 	    "ahci_fatal_error_recovery_handler enter: port %d", port);
9412 
9413 	/* Port multiplier error */
9414 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9415 		/* FBS code is neither completed nor tested. */
9416 		ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9417 		    port, intr_status);
9418 
9419 		/* Force a port reset */
9420 		flag = AHCI_PORT_RESET;
9421 	}
9422 
9423 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9424 	    ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9425 
9426 		/* Read PxCI to see which commands are still outstanding */
9427 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9428 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9429 
9430 		/*
9431 		 * Read PxCMD.CCS to determine the slot that the HBA
9432 		 * was processing when the error occurred.
9433 		 */
9434 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9435 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9436 		failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9437 		    AHCI_CMD_STATUS_CCS_SHIFT;
9438 
9439 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9440 			spkt = ahci_portp->ahciport_err_retri_pkt;
9441 			ASSERT(spkt != NULL);
9442 		} else {
9443 			spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9444 			if (spkt == NULL) {
9445 				/* May happen when interface errors occur? */
9446 				goto next;
9447 			}
9448 		}
9449 
9450 #if AHCI_DEBUG
9451 		/*
9452 		 * Debugging purpose...
9453 		 */
9454 		if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9455 			cmd_header =
9456 			    &ahci_portp->ahciport_cmd_list[failed_slot];
9457 			AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9458 			    "ahci_fatal_error_recovery_handler: port %d, "
9459 			    "PRD Byte Count = 0x%x, "
9460 			    "ahciport_prd_bytecounts = 0x%x", port,
9461 			    cmd_header->ahcich_prd_byte_count,
9462 			    ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9463 		}
9464 #endif
9465 
9466 		sata_cmd = &spkt->satapkt_cmd;
9467 
9468 		/* Fill out the status and error registers for PxIS.TFES */
9469 		if (intr_status & AHCI_INTR_STATUS_TFES) {
9470 			ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9471 			    ahcirf_d2h_register_fis);
9472 
9473 			/* Copy the error context back to the sata_cmd */
9474 			ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9475 		}
9476 
9477 		/* The failed command must be one of the outstanding commands */
9478 		failed_tags = 0x1 << failed_slot;
9479 		ASSERT(failed_tags & slot_status);
9480 
9481 		/* Update the sata registers, especially PxSERR register */
9482 		ahci_update_sata_registers(ahci_ctlp, port,
9483 		    &spkt->satapkt_device);
9484 
9485 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9486 		/* Read PxSACT to see which commands are still outstanding */
9487 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9488 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9489 	}
9490 next:
9491 
9492 #if AHCI_DEBUG
9493 	/*
9494 	 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9495 	 * set, it means a fatal error happened after REQUEST SENSE command
9496 	 * or READ LOG EXT command is delivered to the HBA during the error
9497 	 * recovery process. At this time, the only outstanding command is
9498 	 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9499 	 */
9500 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9501 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9502 		    "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9503 		    "command or READ LOG EXT command for error data retrieval "
9504 		    "failed", port);
9505 		ASSERT(slot_status == 0x1);
9506 		ASSERT(failed_slot == 0);
9507 		ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9508 		    SCMD_REQUEST_SENSE ||
9509 		    spkt->satapkt_cmd.satacmd_cmd_reg ==
9510 		    SATAC_READ_LOG_EXT);
9511 	}
9512 #endif
9513 
9514 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9515 	ahci_portp->ahciport_mop_in_progress++;
9516 
9517 	rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9518 	    port, flag, &reset_flag);
9519 
9520 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9521 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9522 		if (rval == AHCI_SUCCESS)
9523 			cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9524 			    "succeed", instance, port);
9525 		else
9526 			cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9527 			    "failed", instance, port);
9528 	}
9529 
9530 	/*
9531 	 * Won't retrieve error information:
9532 	 * 1. Port reset was involved to recover
9533 	 * 2. Device is gone
9534 	 * 3. IDENTIFY DEVICE command sent to ATAPI device
9535 	 * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9536 	 */
9537 	if (reset_flag ||
9538 	    ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9539 	    spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9540 	    ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9541 		goto out;
9542 
9543 	/*
9544 	 * Deliver READ LOG EXT to gather information about the error when
9545 	 * a COMRESET has not been performed as part of the error recovery
9546 	 * during NCQ command processing.
9547 	 */
9548 	if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9549 		failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9550 		    ahci_portp, port);
9551 		goto out;
9552 	}
9553 
9554 	/*
9555 	 * Deliver REQUEST SENSE for ATAPI command to gather information about
9556 	 * the error when a COMRESET has not been performed as part of the
9557 	 * error recovery.
9558 	 */
9559 	if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9560 		ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9561 out:
9562 	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9563 	    "ahci_fatal_error_recovery_handler: port %d fatal error "
9564 	    "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9565 	    "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9566 	    port, slot_status, ahci_portp->ahciport_pending_tags,
9567 	    ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9568 
9569 	ahci_mop_commands(ahci_ctlp,
9570 	    ahci_portp,
9571 	    slot_status,
9572 	    failed_tags, /* failed tags */
9573 	    0, /* timeout tags */
9574 	    0, /* aborted tags */
9575 	    0); /* reset tags */
9576 }
9577 
9578 /*
9579  * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9580  *	1. device specific.PxFBS.SDE=1
9581  *	2. Non Device specific.
9582  * Nothing will be done when Command-based switching is employed.
9583  *
9584  * Currently code is neither completed nor tested.
9585  */
9586 static void
ahci_pmult_error_recovery_handler(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,uint32_t intr_status)9587 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9588     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9589 {
9590 #ifndef __lock_lint
9591 	_NOTE(ARGUNUSED(intr_status))
9592 #endif
9593 	uint32_t	port_fbs_ctrl;
9594 	int loop_count = 0;
9595 	ahci_addr_t	addr;
9596 
9597 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9598 
9599 	/* Nothing will be done under Command-based switching. */
9600 	if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9601 		return;
9602 
9603 	port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9604 	    (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9605 
9606 	if (!(port_fbs_ctrl & AHCI_FBS_EN))
9607 		/* FBS is not enabled. */
9608 		return;
9609 
9610 	/* Problem's getting complicated now. */
9611 	/*
9612 	 * If FIS-based switching is used, we need to check
9613 	 * the PxFBS to see the error type.
9614 	 */
9615 	port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9616 	    (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9617 
9618 	/* Refer to spec(v1.2) 9.3.6.1 */
9619 	if (port_fbs_ctrl & AHCI_FBS_SDE) {
9620 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9621 		    "A Device Sepcific Error: port %d", port);
9622 		/*
9623 		 * Controller has paused commands for all other
9624 		 * sub-devices until PxFBS.DEC is set.
9625 		 */
9626 		ahci_reject_all_abort_pkts(ahci_ctlp,
9627 		    ahci_portp, 0);
9628 
9629 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9630 		    (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9631 		    port_fbs_ctrl | AHCI_FBS_DEC);
9632 
9633 		/*
9634 		 * Wait controller clear PxFBS.DEC,
9635 		 * then we can continue.
9636 		 */
9637 		loop_count = 0;
9638 		do {
9639 			port_fbs_ctrl = ddi_get32(ahci_ctlp->
9640 			    ahcictl_ahci_acc_handle, (uint32_t *)
9641 			    AHCI_PORT_PxFBS(ahci_ctlp, port));
9642 
9643 			if (loop_count++ > 1000)
9644 				/*
9645 				 * Esclate the error. Follow
9646 				 * non-device specific error
9647 				 * procedure.
9648 				 */
9649 				return;
9650 
9651 			drv_usecwait(AHCI_100US_USECS);
9652 		} while (port_fbs_ctrl & AHCI_FBS_DEC);
9653 
9654 		/*
9655 		 * Issue a software reset to ensure drive is in
9656 		 * a known state.
9657 		 */
9658 		(void) ahci_software_reset(ahci_ctlp,
9659 		    ahci_portp, &addr);
9660 
9661 	} else {
9662 
9663 		/* Process Non-Device Specific Error. */
9664 		/* This will be handled later on. */
9665 		cmn_err(CE_NOTE, "!FBS is not supported now.");
9666 	}
9667 }
9668 /*
9669  * Handle events - fatal error recovery
9670  */
9671 static void
ahci_events_handler(void * args)9672 ahci_events_handler(void *args)
9673 {
9674 	ahci_event_arg_t *ahci_event_arg;
9675 	ahci_ctl_t *ahci_ctlp;
9676 	ahci_port_t *ahci_portp;
9677 	ahci_addr_t *addrp;
9678 	uint32_t event;
9679 	int instance;
9680 
9681 	ahci_event_arg = (ahci_event_arg_t *)args;
9682 
9683 	ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9684 	ahci_portp = ahci_event_arg->ahciea_portp;
9685 	addrp = ahci_event_arg->ahciea_addrp;
9686 	event = ahci_event_arg->ahciea_event;
9687 	instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9688 
9689 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9690 	    "ahci_events_handler enter: port %d intr_status = 0x%x",
9691 	    ahci_portp->ahciport_port_num, event);
9692 
9693 	mutex_enter(&ahci_portp->ahciport_mutex);
9694 
9695 	/*
9696 	 * ahci_intr_phyrdy_change() may have rendered it to
9697 	 * SATA_DTYPE_NONE.
9698 	 */
9699 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9700 		AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9701 		    "ahci_events_handler: port %d no device attached, "
9702 		    "and just return without doing anything",
9703 		    ahci_portp->ahciport_port_num);
9704 
9705 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9706 			ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9707 			cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9708 			    "succeed", instance, ahci_portp->ahciport_port_num);
9709 		}
9710 
9711 		goto out;
9712 	}
9713 
9714 	if (event & (AHCI_INTR_STATUS_IFS |
9715 	    AHCI_INTR_STATUS_HBDS |
9716 	    AHCI_INTR_STATUS_HBFS |
9717 	    AHCI_INTR_STATUS_TFES))
9718 		ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9719 		    addrp, event);
9720 
9721 out:
9722 	mutex_exit(&ahci_portp->ahciport_mutex);
9723 }
9724 
9725 /*
9726  * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9727  * detect there are some commands which are timed out.
9728  */
9729 static void
ahci_timeout_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port,uint32_t tmp_timeout_tags)9730 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9731     uint8_t port, uint32_t tmp_timeout_tags)
9732 {
9733 	uint32_t slot_status = 0;
9734 	uint32_t finished_tags = 0;
9735 	uint32_t timeout_tags = 0;
9736 
9737 	AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9738 	    "ahci_timeout_pkts enter: port %d", port);
9739 
9740 	mutex_enter(&ahci_portp->ahciport_mutex);
9741 
9742 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9743 	    RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9744 	    ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9745 		/* Read PxCI to see which commands are still outstanding */
9746 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9747 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9748 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9749 		/* Read PxSACT to see which commands are still outstanding */
9750 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9751 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9752 	}
9753 
9754 #if AHCI_DEBUG
9755 	/*
9756 	 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9757 	 * set, it means a fatal error happened after REQUEST SENSE command
9758 	 * or READ LOG EXT command is delivered to the HBA during the error
9759 	 * recovery process. At this time, the only outstanding command is
9760 	 * supposed to be REQUEST SENSE command or READ LOG EXT command.
9761 	 */
9762 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9763 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9764 		    "ahci_timeout_pkts called while REQUEST SENSE "
9765 		    "command or READ LOG EXT command for error recovery "
9766 		    "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9767 		    "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9768 		    tmp_timeout_tags, slot_status,
9769 		    ahci_portp->ahciport_pending_tags,
9770 		    ahci_portp->ahciport_pending_ncq_tags);
9771 		ASSERT(slot_status == 0x1);
9772 	} else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9773 		AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9774 		    "ahci_timeout_pkts called while executing R/W PMULT "
9775 		    "command timeout_tags = 0x%x, slot_status = 0x%x",
9776 		    tmp_timeout_tags, slot_status);
9777 		ASSERT(slot_status == 0x1);
9778 	}
9779 #endif
9780 
9781 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9782 	ahci_portp->ahciport_mop_in_progress++;
9783 
9784 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9785 	    port, AHCI_PORT_RESET, NULL);
9786 
9787 	/*
9788 	 * Re-identify timeout tags because some previously checked commands
9789 	 * could already complete.
9790 	 */
9791 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9792 		finished_tags = ahci_portp->ahciport_pending_tags &
9793 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9794 		timeout_tags = tmp_timeout_tags & ~finished_tags;
9795 
9796 		AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9797 		    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9798 		    "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9799 		    "pending_tags = 0x%x ",
9800 		    port, finished_tags, timeout_tags,
9801 		    slot_status, ahci_portp->ahciport_pending_tags);
9802 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9803 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9804 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9805 		timeout_tags = tmp_timeout_tags & ~finished_tags;
9806 
9807 		AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9808 		    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9809 		    "timeout_tags = 0x%x, port_sactive = 0x%x, "
9810 		    "pending_ncq_tags = 0x%x ",
9811 		    port, finished_tags, timeout_tags,
9812 		    slot_status, ahci_portp->ahciport_pending_ncq_tags);
9813 	} else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9814 	    RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9815 		timeout_tags = tmp_timeout_tags;
9816 	}
9817 
9818 	ahci_mop_commands(ahci_ctlp,
9819 	    ahci_portp,
9820 	    slot_status,
9821 	    0,			/* failed tags */
9822 	    timeout_tags,	/* timeout tags */
9823 	    0,			/* aborted tags */
9824 	    0);			/* reset tags */
9825 
9826 	mutex_exit(&ahci_portp->ahciport_mutex);
9827 }
9828 
9829 /*
9830  * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9831  * for long time.
9832  */
9833 static void
ahci_watchdog_handler(ahci_ctl_t * ahci_ctlp)9834 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9835 {
9836 	ahci_port_t *ahci_portp;
9837 	sata_pkt_t *spkt;
9838 	uint32_t pending_tags;
9839 	uint32_t timeout_tags;
9840 	uint32_t port_cmd_status;
9841 	uint32_t port_sactive;
9842 	uint8_t port;
9843 	int tmp_slot;
9844 	int current_slot;
9845 	uint32_t current_tags;
9846 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9847 
9848 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
9849 
9850 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9851 	    "ahci_watchdog_handler entered", NULL);
9852 
9853 	current_slot = 0;
9854 	current_tags = 0;
9855 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9856 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9857 			continue;
9858 		}
9859 
9860 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
9861 
9862 		mutex_enter(&ahci_portp->ahciport_mutex);
9863 		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9864 			mutex_exit(&ahci_portp->ahciport_mutex);
9865 			continue;
9866 		}
9867 
9868 		/* Skip the check for those ports in error recovery */
9869 		if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9870 		    !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9871 			mutex_exit(&ahci_portp->ahciport_mutex);
9872 			continue;
9873 		}
9874 
9875 		pending_tags = 0;
9876 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9877 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9878 
9879 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9880 		    RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9881 			current_slot = 0;
9882 			pending_tags = 0x1;
9883 		} else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9884 			current_slot =
9885 			    (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9886 			    AHCI_CMD_STATUS_CCS_SHIFT;
9887 			pending_tags = ahci_portp->ahciport_pending_tags;
9888 		} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9889 			port_sactive = ddi_get32(
9890 			    ahci_ctlp->ahcictl_ahci_acc_handle,
9891 			    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9892 			current_tags = port_sactive &
9893 			    ~port_cmd_status &
9894 			    AHCI_NCQ_SLOT_MASK(ahci_portp);
9895 			pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9896 		}
9897 
9898 		timeout_tags = 0;
9899 		while (pending_tags) {
9900 			tmp_slot = ddi_ffs(pending_tags) - 1;
9901 			if (tmp_slot == -1) {
9902 				break;
9903 			}
9904 
9905 			if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9906 				spkt = ahci_portp->ahciport_err_retri_pkt;
9907 			else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9908 				spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9909 			else
9910 				spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9911 
9912 			if ((spkt != NULL) && spkt->satapkt_time &&
9913 			    !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9914 				/*
9915 				 * If a packet has survived for more than it's
9916 				 * max life cycles, it is a candidate for time
9917 				 * out.
9918 				 */
9919 				ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9920 				    ahci_watchdog_timeout;
9921 
9922 				if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9923 				    > 0)
9924 					goto next;
9925 
9926 #if AHCI_DEBUG
9927 				if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9928 					AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9929 					    ahci_ctlp, "watchdog: the current "
9930 					    "tags is 0x%x", current_tags);
9931 				} else {
9932 					AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9933 					    ahci_ctlp, "watchdog: the current "
9934 					    "slot is %d", current_slot);
9935 				}
9936 #endif
9937 
9938 				/*
9939 				 * We need to check whether the HBA has
9940 				 * begun to execute the command, if not,
9941 				 * then re-set the timer of the command.
9942 				 */
9943 				if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9944 				    (tmp_slot != current_slot) ||
9945 				    NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9946 				    ((0x1 << tmp_slot) & current_tags)) {
9947 					ahci_portp->ahciport_slot_timeout \
9948 					    [tmp_slot] = spkt->satapkt_time;
9949 				} else {
9950 					timeout_tags |= (0x1 << tmp_slot);
9951 					cmn_err(CE_WARN, "!ahci%d: watchdog "
9952 					    "port %d satapkt 0x%p timed out\n",
9953 					    instance, port, (void *)spkt);
9954 				}
9955 			}
9956 next:
9957 			CLEAR_BIT(pending_tags, tmp_slot);
9958 		}
9959 
9960 		if (timeout_tags) {
9961 			mutex_exit(&ahci_portp->ahciport_mutex);
9962 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
9963 			ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9964 			    port, timeout_tags);
9965 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
9966 			mutex_enter(&ahci_portp->ahciport_mutex);
9967 		}
9968 
9969 		mutex_exit(&ahci_portp->ahciport_mutex);
9970 	}
9971 
9972 	/* Re-install the watchdog timeout handler */
9973 	if (ahci_ctlp->ahcictl_timeout_id != 0) {
9974 		ahci_ctlp->ahcictl_timeout_id =
9975 		    timeout((void (*)(void *))ahci_watchdog_handler,
9976 		    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9977 	}
9978 
9979 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
9980 }
9981 
9982 /*
9983  * Fill the error context into sata_cmd for non-queued command error.
9984  */
9985 static void
ahci_copy_err_cnxt(sata_cmd_t * scmd,ahci_fis_d2h_register_t * rfisp)9986 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9987 {
9988 	scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9989 	scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9990 	scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9991 	scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9992 	scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9993 	scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9994 	scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9995 
9996 	if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9997 		scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9998 		scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9999 		scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
10000 		scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
10001 	}
10002 }
10003 
10004 /*
10005  * Fill the ncq error page into sata_cmd for queued command error.
10006  */
10007 static void
ahci_copy_ncq_err_page(sata_cmd_t * scmd,struct sata_ncq_error_recovery_page * ncq_err_page)10008 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
10009     struct sata_ncq_error_recovery_page *ncq_err_page)
10010 {
10011 	scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
10012 	scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
10013 	scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
10014 	scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
10015 	scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
10016 	scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
10017 	scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
10018 	scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
10019 	scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
10020 	scmd->satacmd_status_reg = ncq_err_page->ncq_status;
10021 	scmd->satacmd_error_reg = ncq_err_page->ncq_error;
10022 }
10023 
10024 /*
10025  * Put the respective register value to sata_cmd_t for satacmd_flags.
10026  */
10027 static void
ahci_copy_out_regs(sata_cmd_t * scmd,ahci_fis_d2h_register_t * rfisp)10028 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
10029 {
10030 	if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
10031 		scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
10032 	if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
10033 		scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
10034 	if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
10035 		scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
10036 	if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
10037 		scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
10038 	if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
10039 		scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
10040 	if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
10041 		scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
10042 	if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
10043 		scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
10044 	if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
10045 		scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
10046 	if (scmd->satacmd_flags.sata_copy_out_device_reg)
10047 		scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
10048 	if (scmd->satacmd_flags.sata_copy_out_error_reg)
10049 		scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
10050 }
10051 
10052 static void
ahci_log_fatal_error_message(ahci_ctl_t * ahci_ctlp,uint8_t port,uint32_t intr_status)10053 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10054     uint32_t intr_status)
10055 {
10056 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
10057 
10058 	if (intr_status & AHCI_INTR_STATUS_IFS)
10059 		cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
10060 		    "error", instance, port);
10061 
10062 	if (intr_status & AHCI_INTR_STATUS_HBDS)
10063 		cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
10064 		    instance, port);
10065 
10066 	if (intr_status & AHCI_INTR_STATUS_HBFS)
10067 		cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
10068 		    instance, port);
10069 
10070 	if (intr_status & AHCI_INTR_STATUS_TFES)
10071 		cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
10072 		    instance, port);
10073 
10074 	cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
10075 	    "recovery", instance, port);
10076 }
10077 
10078 static void
ahci_dump_commands(ahci_ctl_t * ahci_ctlp,uint8_t port,uint32_t slot_tags)10079 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
10080     uint32_t slot_tags)
10081 {
10082 	ahci_port_t *ahci_portp;
10083 	int tmp_slot;
10084 	sata_pkt_t *spkt;
10085 	sata_cmd_t cmd;
10086 
10087 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
10088 	ASSERT(ahci_portp != NULL);
10089 
10090 	while (slot_tags) {
10091 		tmp_slot = ddi_ffs(slot_tags) - 1;
10092 		if (tmp_slot == -1) {
10093 			break;
10094 		}
10095 
10096 		spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
10097 		if (spkt != NULL) {
10098 			cmd = spkt->satapkt_cmd;
10099 
10100 			cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
10101 			    "features_reg = 0x%x sec_count_msb = 0x%x "
10102 			    "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
10103 			    "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
10104 			    "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
10105 			    "lba_high_lsb = 0x%x device_reg = 0x%x "
10106 			    "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10107 			    cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10108 			    cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10109 			    cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10110 			    cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10111 			    cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10112 			    cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10113 			    *((uint32_t *)&(cmd.satacmd_flags)));
10114 		}
10115 
10116 		CLEAR_BIT(slot_tags, tmp_slot);
10117 	}
10118 }
10119 
10120 /*
10121  * Dump the serror message to the log.
10122  */
10123 static void
ahci_log_serror_message(ahci_ctl_t * ahci_ctlp,uint8_t port,uint32_t port_serror,int debug_only)10124 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10125     uint32_t port_serror, int debug_only)
10126 {
10127 	static char err_buf[512];
10128 	static char err_msg_header[16];
10129 	char *err_msg = err_buf;
10130 
10131 	*err_buf = '\0';
10132 	*err_msg_header = '\0';
10133 
10134 	if (port_serror & SERROR_DATA_ERR_FIXED) {
10135 		err_msg = strcat(err_msg,
10136 		    "\tRecovered Data Integrity Error (I)\n");
10137 	}
10138 
10139 	if (port_serror & SERROR_COMM_ERR_FIXED) {
10140 		err_msg = strcat(err_msg,
10141 		    "\tRecovered Communication Error (M)\n");
10142 	}
10143 
10144 	if (port_serror & SERROR_DATA_ERR) {
10145 		err_msg = strcat(err_msg,
10146 		    "\tTransient Data Integrity Error (T)\n");
10147 	}
10148 
10149 	if (port_serror & SERROR_PERSISTENT_ERR) {
10150 		err_msg = strcat(err_msg,
10151 		    "\tPersistent Communication or Data Integrity Error (C)\n");
10152 	}
10153 
10154 	if (port_serror & SERROR_PROTOCOL_ERR) {
10155 		err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10156 	}
10157 
10158 	if (port_serror & SERROR_INT_ERR) {
10159 		err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10160 	}
10161 
10162 	if (port_serror & SERROR_PHY_RDY_CHG) {
10163 		err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10164 	}
10165 
10166 	if (port_serror & SERROR_PHY_INT_ERR) {
10167 		err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10168 	}
10169 
10170 	if (port_serror & SERROR_COMM_WAKE) {
10171 		err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10172 	}
10173 
10174 	if (port_serror & SERROR_10B_TO_8B_ERR) {
10175 		err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10176 	}
10177 
10178 	if (port_serror & SERROR_DISPARITY_ERR) {
10179 		err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10180 	}
10181 
10182 	if (port_serror & SERROR_CRC_ERR) {
10183 		err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10184 	}
10185 
10186 	if (port_serror & SERROR_HANDSHAKE_ERR) {
10187 		err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10188 	}
10189 
10190 	if (port_serror & SERROR_LINK_SEQ_ERR) {
10191 		err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10192 	}
10193 
10194 	if (port_serror & SERROR_TRANS_ERR) {
10195 		err_msg = strcat(err_msg,
10196 		    "\tTransport state transition error (T)\n");
10197 	}
10198 
10199 	if (port_serror & SERROR_FIS_TYPE) {
10200 		err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10201 	}
10202 
10203 	if (port_serror & SERROR_EXCHANGED_ERR) {
10204 		err_msg = strcat(err_msg, "\tExchanged (X)\n");
10205 	}
10206 
10207 	if (*err_msg == '\0')
10208 		return;
10209 
10210 	if (debug_only) {
10211 		(void) sprintf(err_msg_header, "port %d", port);
10212 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10213 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10214 	} else if (ahci_ctlp) {
10215 		cmn_err(CE_WARN, "!ahci%d: %s %s",
10216 		    ddi_get_instance(ahci_ctlp->ahcictl_dip),
10217 		    err_msg_header, err_msg);
10218 
10219 		/* sata trace debug */
10220 		sata_trace_debug(ahci_ctlp->ahcictl_dip,
10221 		    "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10222 		    err_msg_header, err_msg);
10223 	} else {
10224 		cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10225 
10226 		/* sata trace debug */
10227 		sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10228 	}
10229 }
10230 
10231 /*
10232  * Translate the sata_address_t type into the ahci_addr_t type.
10233  * sata_device.satadev_addr structure is used as source.
10234  */
10235 static void
ahci_get_ahci_addr(ahci_ctl_t * ahci_ctlp,sata_device_t * sd,ahci_addr_t * ahci_addrp)10236 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10237     ahci_addr_t *ahci_addrp)
10238 {
10239 	sata_address_t *sata_addrp = &sd->satadev_addr;
10240 	ahci_addrp->aa_port =
10241 	    ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10242 	ahci_addrp->aa_pmport = sata_addrp->pmport;
10243 
10244 	switch (sata_addrp->qual) {
10245 	case SATA_ADDR_DCPORT:
10246 	case SATA_ADDR_CPORT:
10247 		ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10248 		break;
10249 	case SATA_ADDR_PMULT:
10250 	case SATA_ADDR_PMULT_SPEC:
10251 		ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10252 		break;
10253 	case SATA_ADDR_DPMPORT:
10254 	case SATA_ADDR_PMPORT:
10255 		ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10256 		break;
10257 	case SATA_ADDR_NULL:
10258 	default:
10259 		/* something went wrong */
10260 		ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10261 		break;
10262 	}
10263 }
10264 
10265 /*
10266  * This routine is to calculate the total number of ports implemented
10267  * by the HBA.
10268  */
10269 static int
ahci_get_num_implemented_ports(uint32_t ports_implemented)10270 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10271 {
10272 	uint8_t i;
10273 	int num = 0;
10274 
10275 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
10276 		if (((uint32_t)0x1 << i) & ports_implemented)
10277 			num++;
10278 	}
10279 
10280 	return (num);
10281 }
10282 
10283 #if AHCI_DEBUG
10284 static void
ahci_log(ahci_ctl_t * ahci_ctlp,uint_t level,char * fmt,...)10285 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10286 {
10287 	static char name[16];
10288 	va_list ap;
10289 
10290 	mutex_enter(&ahci_log_mutex);
10291 
10292 	va_start(ap, fmt);
10293 	if (ahci_ctlp) {
10294 		(void) sprintf(name, "ahci%d: ",
10295 		    ddi_get_instance(ahci_ctlp->ahcictl_dip));
10296 	} else {
10297 		(void) sprintf(name, "ahci: ");
10298 	}
10299 
10300 	(void) vsprintf(ahci_log_buf, fmt, ap);
10301 	va_end(ap);
10302 
10303 	cmn_err(level, "%s%s", name, ahci_log_buf);
10304 
10305 	mutex_exit(&ahci_log_mutex);
10306 }
10307 #endif
10308 
10309 /*
10310  * quiesce(9E) entry point.
10311  *
10312  * This function is called when the system is single-threaded at high
10313  * PIL with preemption disabled. Therefore, this function must not be
10314  * blocked. Because no taskqs are running, there is no need for us to
10315  * take any action for enclosure services which are running in the
10316  * taskq context, especially as no interrupts are generated by it nor
10317  * are any messages expected to come in.
10318  *
10319  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10320  * DDI_FAILURE indicates an error condition and should almost never happen.
10321  */
10322 static int
ahci_quiesce(dev_info_t * dip)10323 ahci_quiesce(dev_info_t *dip)
10324 {
10325 	ahci_ctl_t *ahci_ctlp;
10326 	ahci_port_t *ahci_portp;
10327 	int instance, port;
10328 
10329 	instance = ddi_get_instance(dip);
10330 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10331 
10332 	if (ahci_ctlp == NULL)
10333 		return (DDI_FAILURE);
10334 
10335 #if AHCI_DEBUG
10336 	ahci_debug_flags = 0;
10337 #endif
10338 
10339 	ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10340 
10341 	/* disable all the interrupts. */
10342 	ahci_disable_all_intrs(ahci_ctlp);
10343 
10344 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10345 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10346 			continue;
10347 		}
10348 
10349 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
10350 
10351 		/*
10352 		 * Stop the port by clearing PxCMD.ST
10353 		 *
10354 		 * Here we must disable the port interrupt because
10355 		 * ahci_disable_all_intrs only clear GHC.IE, and IS
10356 		 * register will be still set if PxIE is enabled.
10357 		 * When ahci shares one IRQ with other drivers, the
10358 		 * intr handler may claim the intr mistakenly.
10359 		 */
10360 		ahci_disable_port_intrs(ahci_ctlp, port);
10361 		(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10362 		    ahci_portp, port);
10363 	}
10364 
10365 	ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10366 
10367 	return (DDI_SUCCESS);
10368 }
10369 
10370 /*
10371  * The function will add a sata packet to the done queue.
10372  */
10373 static void
ahci_add_doneq(ahci_port_t * ahci_portp,sata_pkt_t * satapkt,int reason)10374 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10375 {
10376 	ASSERT(satapkt != NULL);
10377 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10378 
10379 	/* set the reason for all packets */
10380 	satapkt->satapkt_reason = reason;
10381 	satapkt->satapkt_hba_driver_private = NULL;
10382 
10383 	if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10384 	    satapkt->satapkt_comp) {
10385 		/*
10386 		 * only add to queue when mode is not synch and there is
10387 		 * completion callback
10388 		 */
10389 		*ahci_portp->ahciport_doneqtail = satapkt;
10390 		ahci_portp->ahciport_doneqtail =
10391 		    (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10392 		ahci_portp->ahciport_doneq_len++;
10393 
10394 	} else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10395 	    ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10396 		/*
10397 		 * for sync/non-poll mode, just call cv_broadcast
10398 		 */
10399 		cv_broadcast(&ahci_portp->ahciport_cv);
10400 }
10401 
10402 /*
10403  * The function will call completion callback of sata packet on the
10404  * completed queue
10405  */
10406 static void
ahci_flush_doneq(ahci_port_t * ahci_portp)10407 ahci_flush_doneq(ahci_port_t *ahci_portp)
10408 {
10409 	sata_pkt_t *satapkt, *next;
10410 
10411 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10412 
10413 	if (ahci_portp->ahciport_doneq) {
10414 		satapkt = ahci_portp->ahciport_doneq;
10415 
10416 		ahci_portp->ahciport_doneq = NULL;
10417 		ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10418 		ahci_portp->ahciport_doneq_len = 0;
10419 
10420 		mutex_exit(&ahci_portp->ahciport_mutex);
10421 
10422 		while (satapkt != NULL) {
10423 			next = satapkt->satapkt_hba_driver_private;
10424 			satapkt->satapkt_hba_driver_private = NULL;
10425 
10426 			/* Call the callback */
10427 			(*satapkt->satapkt_comp)(satapkt);
10428 
10429 			satapkt = next;
10430 		}
10431 
10432 		mutex_enter(&ahci_portp->ahciport_mutex);
10433 	}
10434 }
10435 
10436 /*
10437  * Sets the state for the specified port on the controller to desired state.
10438  * This must be run in the context of the enclosure taskq which ensures that
10439  * only one event is outstanding at any time.
10440  */
10441 static boolean_t
ahci_em_set_led(ahci_ctl_t * ahci_ctlp,uint8_t port,ahci_em_led_state_t desire)10442 ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire)
10443 {
10444 	ahci_em_led_msg_t msg;
10445 	ahci_em_msg_hdr_t hdr;
10446 	uint32_t msgval, hdrval;
10447 	uint_t i, max_delay = ahci_em_tx_delay_count;
10448 
10449 	msg.alm_hba = port;
10450 	msg.alm_pminfo = 0;
10451 	msg.alm_value = 0;
10452 
10453 	if (desire & AHCI_EM_LED_IDENT_ENABLE) {
10454 		msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF;
10455 	}
10456 
10457 	if (desire & AHCI_EM_LED_FAULT_ENABLE) {
10458 		msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF;
10459 	}
10460 
10461 	if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 &&
10462 	    (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) {
10463 		msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF;
10464 	}
10465 
10466 	hdr.aemh_rsvd = 0;
10467 	hdr.aemh_mlen = sizeof (ahci_em_led_msg_t);
10468 	hdr.aemh_dlen = 0;
10469 	hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED;
10470 
10471 	bcopy(&msg, &msgval, sizeof (msgval));
10472 	bcopy(&hdr, &hdrval, sizeof (hdrval));
10473 
10474 	/*
10475 	 * First, make sure we can transmit. We should not have been placed in a
10476 	 * situation where an outstanding transmission is going on.
10477 	 */
10478 	for (i = 0; i < max_delay; i++) {
10479 		uint32_t val;
10480 
10481 		val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10482 		    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10483 		if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10484 			break;
10485 
10486 		delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10487 	}
10488 
10489 	if (i == max_delay)
10490 		return (B_FALSE);
10491 
10492 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10493 	    (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval);
10494 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10495 	    (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval);
10496 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10497 	    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM);
10498 
10499 	for (i = 0; i < max_delay; i++) {
10500 		uint32_t val;
10501 
10502 		val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10503 		    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10504 		if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
10505 			break;
10506 
10507 		delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
10508 	}
10509 
10510 	if (i == max_delay)
10511 		return (B_FALSE);
10512 
10513 	return (B_TRUE);
10514 }
10515 
10516 typedef struct ahci_em_led_task_arg {
10517 	ahci_ctl_t		*aelta_ctl;
10518 	uint8_t			aelta_port;
10519 	uint_t			aelta_op;
10520 	ahci_em_led_state_t	aelta_state;
10521 	uint_t			aelta_ret;
10522 	kcondvar_t		aelta_cv;
10523 	uint_t			aelta_ref;
10524 } ahci_em_led_task_arg_t;
10525 
10526 static void
ahci_em_led_task_free(ahci_em_led_task_arg_t * task)10527 ahci_em_led_task_free(ahci_em_led_task_arg_t *task)
10528 {
10529 	ASSERT3U(task->aelta_ref, ==, 0);
10530 	cv_destroy(&task->aelta_cv);
10531 	kmem_free(task, sizeof (*task));
10532 }
10533 
10534 static void
ahci_em_led_task(void * arg)10535 ahci_em_led_task(void *arg)
10536 {
10537 	boolean_t ret, cleanup = B_FALSE;
10538 	ahci_em_led_task_arg_t *led = arg;
10539 	ahci_em_led_state_t state;
10540 
10541 	mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10542 	if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) {
10543 		led->aelta_ret = EIO;
10544 		mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10545 		return;
10546 	}
10547 
10548 	state = led->aelta_ctl->ahcictl_em_state[led->aelta_port];
10549 	mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10550 
10551 	switch (led->aelta_op) {
10552 	case AHCI_EM_IOC_SET_OP_ADD:
10553 		state |= led->aelta_state;
10554 		break;
10555 	case AHCI_EM_IOC_SET_OP_REM:
10556 		state &= ~led->aelta_state;
10557 		break;
10558 	case AHCI_EM_IOC_SET_OP_SET:
10559 		state = led->aelta_state;
10560 		break;
10561 	default:
10562 		led->aelta_ret = ENOTSUP;
10563 		return;
10564 	}
10565 
10566 	ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state);
10567 
10568 	mutex_enter(&led->aelta_ctl->ahcictl_mutex);
10569 	if (ret) {
10570 		led->aelta_ctl->ahcictl_em_state[led->aelta_port] = state;
10571 		led->aelta_ret = 0;
10572 	} else {
10573 		led->aelta_ret = EIO;
10574 		led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10575 	}
10576 	led->aelta_ref--;
10577 	if (led->aelta_ref > 0) {
10578 		cv_signal(&led->aelta_cv);
10579 	} else {
10580 		cleanup = B_TRUE;
10581 	}
10582 	mutex_exit(&led->aelta_ctl->ahcictl_mutex);
10583 
10584 	if (cleanup) {
10585 		ahci_em_led_task_free(led);
10586 	}
10587 }
10588 
10589 static void
ahci_em_reset(void * arg)10590 ahci_em_reset(void *arg)
10591 {
10592 	uint_t i, max_delay = ahci_em_reset_delay_count;
10593 	ahci_ctl_t *ahci_ctlp = arg;
10594 
10595 	/*
10596 	 * We've been asked to reset the device. The caller should have set the
10597 	 * resetting flag. Make sure that we don't have a request to quiesce.
10598 	 */
10599 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10600 	ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING);
10601 	if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) {
10602 		ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10603 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
10604 		return;
10605 	}
10606 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10607 
10608 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
10609 	    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST);
10610 	for (i = 0; i < max_delay; i++) {
10611 		uint32_t val;
10612 
10613 		val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
10614 		    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
10615 		if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0)
10616 			break;
10617 
10618 		delay(drv_usectohz(ahci_em_reset_delay_ms * 1000));
10619 	}
10620 
10621 	if (i == max_delay) {
10622 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
10623 		ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10624 		ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10625 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
10626 		cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting",
10627 		    ddi_get_instance(ahci_ctlp->ahcictl_dip));
10628 		return;
10629 	}
10630 
10631 	for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10632 
10633 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i))
10634 			continue;
10635 
10636 		/*
10637 		 * Try to flush all the LEDs as part of reset. If it fails,
10638 		 * drive on.
10639 		 */
10640 		if (!ahci_em_set_led(ahci_ctlp, i,
10641 		    ahci_ctlp->ahcictl_em_state[i])) {
10642 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
10643 			ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10644 			ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
10645 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
10646 			cmn_err(CE_WARN, "!ahci%d: enclosure timed out "
10647 			    "setting port %u",
10648 			    ddi_get_instance(ahci_ctlp->ahcictl_dip), i);
10649 			return;
10650 		}
10651 	}
10652 
10653 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10654 	ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
10655 	ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY;
10656 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10657 }
10658 
10659 static boolean_t
ahci_em_init(ahci_ctl_t * ahci_ctlp)10660 ahci_em_init(ahci_ctl_t *ahci_ctlp)
10661 {
10662 	char name[128];
10663 
10664 	/*
10665 	 * First make sure we actually have enclosure services and if so, that
10666 	 * we have the hardware support that we care about for this.
10667 	 */
10668 	if (ahci_ctlp->ahcictl_em_loc == 0 ||
10669 	    (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0)
10670 		return (B_TRUE);
10671 
10672 	/*
10673 	 * Next, make sure that the buffer is large enough for us. We need two
10674 	 * dwords or 8 bytes. The location register is stored in dwords.
10675 	 */
10676 	if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) <
10677 	    AHCI_EM_BUFFER_MIN) {
10678 		return (B_TRUE);
10679 	}
10680 
10681 	ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc &
10682 	    AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4;
10683 	ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr;
10684 
10685 	bzero(ahci_ctlp->ahcictl_em_state,
10686 	    sizeof (ahci_ctlp->ahcictl_em_state));
10687 
10688 	(void) snprintf(name, sizeof (name), "ahcti_em_taskq%d",
10689 	    ddi_get_instance(ahci_ctlp->ahcictl_dip));
10690 	if ((ahci_ctlp->ahcictl_em_taskq =
10691 	    ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1,
10692 	    TASKQ_DEFAULTPRI, 0)) == NULL) {
10693 		cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em "
10694 		    "services", ddi_get_instance(ahci_ctlp->ahcictl_dip));
10695 		return (B_FALSE);
10696 	}
10697 
10698 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10699 	ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT | AHCI_EM_RESETTING;
10700 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10701 	(void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10702 	    ahci_ctlp, DDI_SLEEP);
10703 
10704 	return (B_TRUE);
10705 }
10706 
10707 static int
ahci_em_ioctl_get(ahci_ctl_t * ahci_ctlp,intptr_t arg)10708 ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10709 {
10710 	int i;
10711 	ahci_ioc_em_get_t get;
10712 
10713 	if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10714 		return (ENOTSUP);
10715 	}
10716 
10717 	bzero(&get, sizeof (get));
10718 	get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented;
10719 	if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) {
10720 		get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY;
10721 	}
10722 
10723 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10724 	for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
10725 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) {
10726 			continue;
10727 		}
10728 		get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i];
10729 	}
10730 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10731 
10732 	if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0)
10733 		return (EFAULT);
10734 
10735 	return (0);
10736 }
10737 
10738 static int
ahci_em_ioctl_set(ahci_ctl_t * ahci_ctlp,intptr_t arg)10739 ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg)
10740 {
10741 	int ret;
10742 	ahci_ioc_em_set_t set;
10743 	ahci_em_led_task_arg_t *task;
10744 	boolean_t signal, cleanup;
10745 
10746 	if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0)
10747 		return (EFAULT);
10748 
10749 	if (set.aiems_port > ahci_ctlp->ahcictl_num_ports)
10750 		return (EINVAL);
10751 
10752 	if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) {
10753 		return (EINVAL);
10754 	}
10755 
10756 	if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE |
10757 	    AHCI_EM_LED_FAULT_ENABLE |
10758 	    AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) {
10759 		return (EINVAL);
10760 	}
10761 
10762 	switch (set.aiems_op) {
10763 	case AHCI_EM_IOC_SET_OP_ADD:
10764 	case AHCI_EM_IOC_SET_OP_REM:
10765 	case AHCI_EM_IOC_SET_OP_SET:
10766 		break;
10767 	default:
10768 		return (EINVAL);
10769 	}
10770 
10771 	if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10772 		return (ENOTSUP);
10773 	}
10774 
10775 	if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 &&
10776 	    ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) {
10777 		return (ENOTSUP);
10778 	}
10779 
10780 	task = kmem_alloc(sizeof (*task), KM_NOSLEEP_LAZY);
10781 	if (task == NULL) {
10782 		return (ENOMEM);
10783 	}
10784 
10785 	task->aelta_ctl = ahci_ctlp;
10786 	task->aelta_port = (uint8_t)set.aiems_port;
10787 	task->aelta_op = set.aiems_op;
10788 	task->aelta_state = set.aiems_leds;
10789 
10790 	cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL);
10791 
10792 	/*
10793 	 * Initialize the reference count to two. One for us and one for the
10794 	 * taskq. This will be used in case we get canceled.
10795 	 */
10796 	task->aelta_ref = 2;
10797 
10798 	/*
10799 	 * Once dispatched, the task state is protected by our global mutex.
10800 	 */
10801 	(void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq,
10802 	    ahci_em_led_task, task, DDI_SLEEP);
10803 
10804 	signal = B_FALSE;
10805 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10806 	while (task->aelta_ref > 1) {
10807 		if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) ==
10808 		    0) {
10809 			signal = B_TRUE;
10810 			break;
10811 		}
10812 	}
10813 
10814 	/*
10815 	 * Remove our reference count. If we were woken up because of a signal
10816 	 * then the taskq may still be dispatched. In which case we shouldn't
10817 	 * free this memory until it is done. In that case, the taskq will take
10818 	 * care of it.
10819 	 */
10820 	task->aelta_ref--;
10821 	cleanup = (task->aelta_ref == 0);
10822 	if (signal) {
10823 		ret = EINTR;
10824 	} else {
10825 		ret = task->aelta_ret;
10826 	}
10827 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10828 
10829 	if (cleanup) {
10830 		ahci_em_led_task_free(task);
10831 	}
10832 
10833 	return (ret);
10834 }
10835 
10836 static int
ahci_em_ioctl(dev_info_t * dip,int cmd,intptr_t arg)10837 ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg)
10838 {
10839 	int inst;
10840 	ahci_ctl_t *ahci_ctlp;
10841 
10842 	inst = ddi_get_instance(dip);
10843 	if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) {
10844 		return (ENXIO);
10845 	}
10846 
10847 	switch (cmd) {
10848 	case AHCI_EM_IOC_GET:
10849 		return (ahci_em_ioctl_get(ahci_ctlp, arg));
10850 	case AHCI_EM_IOC_SET:
10851 		return (ahci_em_ioctl_set(ahci_ctlp, arg));
10852 	default:
10853 		return (ENOTTY);
10854 	}
10855 
10856 }
10857 
10858 static void
ahci_em_quiesce(ahci_ctl_t * ahci_ctlp)10859 ahci_em_quiesce(ahci_ctl_t *ahci_ctlp)
10860 {
10861 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10862 	if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10863 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
10864 		return;
10865 	}
10866 	ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE;
10867 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10868 
10869 	ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq);
10870 }
10871 
10872 static void
ahci_em_suspend(ahci_ctl_t * ahci_ctlp)10873 ahci_em_suspend(ahci_ctl_t *ahci_ctlp)
10874 {
10875 	ahci_em_quiesce(ahci_ctlp);
10876 
10877 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10878 	ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY;
10879 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10880 }
10881 
10882 static void
ahci_em_resume(ahci_ctl_t * ahci_ctlp)10883 ahci_em_resume(ahci_ctl_t *ahci_ctlp)
10884 {
10885 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
10886 	if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10887 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
10888 		return;
10889 	}
10890 	ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
10891 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
10892 
10893 	(void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
10894 	    ahci_ctlp, DDI_SLEEP);
10895 }
10896 
10897 static void
ahci_em_fini(ahci_ctl_t * ahci_ctlp)10898 ahci_em_fini(ahci_ctl_t *ahci_ctlp)
10899 {
10900 	if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
10901 		return;
10902 	}
10903 
10904 	ahci_em_quiesce(ahci_ctlp);
10905 	ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq);
10906 	ahci_ctlp->ahcictl_em_taskq = NULL;
10907 }
10908