xref: /freebsd/sys/powerpc/mpc85xx/fsl_sata.c (revision 5ab1c5846ff41be24b1f6beb0317bf8258cd4409)
1 /*-
2  * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org>
3  * Copyright (c) 2017 Justin Hibbits <jhibbits@FreeBSD.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer,
11  *    without modification, immediately at the beginning of the file.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include <sys/param.h>
32 #include <sys/module.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/endian.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/mutex.h>
41 #include <sys/rman.h>
42 
43 #include <cam/cam.h>
44 #include <cam/cam_ccb.h>
45 #include <cam/cam_sim.h>
46 #include <cam/cam_xpt_sim.h>
47 #include <cam/cam_debug.h>
48 
49 #include <dev/ofw/ofw_bus_subr.h>
50 
51 #include <machine/bus.h>
52 #include <machine/resource.h>
53 
54 #include "fsl_sata.h"
55 
56 struct fsl_sata_channel;
57 struct fsl_sata_slot;
58 enum fsl_sata_err_type;
59 struct fsl_sata_cmd_tab;
60 
61 
62 /* local prototypes */
63 static int fsl_sata_init(device_t dev);
64 static int fsl_sata_deinit(device_t dev);
65 static int fsl_sata_suspend(device_t dev);
66 static int fsl_sata_resume(device_t dev);
67 static void fsl_sata_pm(void *arg);
68 static void fsl_sata_intr(void *arg);
69 static void fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus);
70 static void fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb);
71 static void fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
72 static void fsl_sata_execute_transaction(struct fsl_sata_slot *slot);
73 static void fsl_sata_timeout(void *arg);
74 static void fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et);
75 static int fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag);
76 static void fsl_sata_dmainit(device_t dev);
77 static void fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
78 static void fsl_sata_dmafini(device_t dev);
79 static void fsl_sata_slotsalloc(device_t dev);
80 static void fsl_sata_slotsfree(device_t dev);
81 static void fsl_sata_reset(struct fsl_sata_channel *ch);
82 static void fsl_sata_start(struct fsl_sata_channel *ch);
83 static void fsl_sata_stop(struct fsl_sata_channel *ch);
84 
85 static void fsl_sata_issue_recovery(struct fsl_sata_channel *ch);
86 static void fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb);
87 static void fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb);
88 
89 static void fsl_sataaction(struct cam_sim *sim, union ccb *ccb);
90 static void fsl_satapoll(struct cam_sim *sim);
91 
92 static MALLOC_DEFINE(M_FSL_SATA, "FSL SATA driver", "FSL SATA driver data buffers");
93 
94 #define	recovery_type		spriv_field0
95 #define	RECOVERY_NONE		0
96 #define	RECOVERY_READ_LOG	1
97 #define	RECOVERY_REQUEST_SENSE	2
98 #define	recovery_slot		spriv_field1
99 
100 #define	FSL_SATA_P_CQR		0x0
101 #define	FSL_SATA_P_CAR		0x4
102 #define	FSL_SATA_P_CCR		0x10
103 #define	FSL_SATA_P_CER			0x18
104 #define	FSL_SATA_P_DER		0x20
105 #define	FSL_SATA_P_CHBA		0x24
106 #define	FSL_SATA_P_HSTS		0x28
107 #define	  FSL_SATA_P_HSTS_HS_ON	  0x80000000
108 #define	  FSL_SATA_P_HSTS_ME	  0x00040000
109 #define	  FSL_SATA_P_HSTS_DLM	  0x00001000
110 #define	  FSL_SATA_P_HSTS_FOT	  0x00000200
111 #define	  FSL_SATA_P_HSTS_FOR	  0x00000100
112 #define	  FSL_SATA_P_HSTS_FE	  0x00000020
113 #define	  FSL_SATA_P_HSTS_PR	  0x00000010
114 #define	  FSL_SATA_P_HSTS_SNTFU	  0x00000004
115 #define	  FSL_SATA_P_HSTS_DE	  0x00000002
116 #define	FSL_SATA_P_HCTRL	0x2c
117 #define	  FSL_SATA_P_HCTRL_HC_ON  0x80000000
118 #define	  FSL_SATA_P_HCTRL_HC_FORCE_OFF  0x40000000
119 #define	  FSL_SATA_P_HCTRL_ENT	  0x10000000
120 #define	  FSL_SATA_P_HCTRL_SNOOP  0x00000400
121 #define	  FSL_SATA_P_HCTRL_PM	  0x00000200
122 #define	  FSL_SATA_P_HCTRL_FATAL  0x00000020
123 #define	  FSL_SATA_P_HCTRL_PHYRDY 0x00000010
124 #define	  FSL_SATA_P_HCTRL_SIG	  0x00000008
125 #define	  FSL_SATA_P_HCTRL_SNTFY  0x00000004
126 #define	  FSL_SATA_P_HCTRL_DE	  0x00000002
127 #define	  FSL_SATA_P_HCTRL_CC	  0x00000001
128 #define	  FSL_SATA_P_HCTRL_INT_MASK	0x0000003f
129 #define	FSL_SATA_P_CQPMP	0x30
130 #define	FSL_SATA_P_SIG		0x34
131 #define	FSL_SATA_P_ICC		0x38
132 #define	  FSL_SATA_P_ICC_ITC_M	  0x1f000000
133 #define	  FSL_SATA_P_ICC_ITC_S	  24
134 #define	  FSL_SATA_P_ICC_ITTCV_M	  0x0007ffff
135 #define	FSL_SATA_P_PCC		0x15c
136 #define	  FSL_SATA_P_PCC_SLUMBER	  0x0000000c
137 #define	  FSL_SATA_P_PCC_PARTIAL	  0x0000000a
138 #define	  FSL_SATA_PCC_LPB_EN		  0x0000000e
139 
140 #define	FSL_SATA_MAX_SLOTS		16
141 /* FSL_SATA register defines */
142 
143 #define	FSL_SATA_P_SSTS		0x100
144 #define	FSL_SATA_P_SERR		0x104
145 #define	FSL_SATA_P_SCTL		0x108
146 #define	FSL_SATA_P_SNTF		0x10c
147 
148 /* Pessimistic prognosis on number of required S/G entries */
149 #define	FSL_SATA_SG_ENTRIES	63
150 /* Command list. 16 commands. First, 1Kbyte aligned. */
151 #define	FSL_SATA_CL_OFFSET	0
152 #define	FSL_SATA_CL_SIZE	16
153 /* Command tables. Up to 32 commands, Each, 4-byte aligned. */
154 #define	FSL_SATA_CT_OFFSET	(FSL_SATA_CL_OFFSET + FSL_SATA_CL_SIZE * FSL_SATA_MAX_SLOTS)
155 #define	FSL_SATA_CT_SIZE	(96 + FSL_SATA_SG_ENTRIES * 16)
156 /* Total main work area. */
157 #define	FSL_SATA_WORK_SIZE	(FSL_SATA_CT_OFFSET + FSL_SATA_CT_SIZE * FSL_SATA_MAX_SLOTS)
158 #define	FSL_SATA_MAX_XFER	(64 * 1024 * 1024)
159 
160 /* Some convenience macros for getting the CTP and CLP */
161 #define	FSL_SATA_CTP_BUS(ch, slot)	\
162     ((ch->dma.work_bus + FSL_SATA_CT_OFFSET + (FSL_SATA_CT_SIZE * slot->slot)))
163 #define FSL_SATA_PRD_OFFSET(prd) (96 + (prd) * 16)
164 #define	FSL_SATA_CTP(ch, slot)		\
165     ((struct fsl_sata_cmd_tab *)(ch->dma.work + FSL_SATA_CT_OFFSET + \
166      (FSL_SATA_CT_SIZE * slot->slot)))
167 #define	FSL_SATA_CLP(ch, slot)		\
168 	((struct fsl_sata_cmd_list *) (ch->dma.work + FSL_SATA_CL_OFFSET + \
169 	 (FSL_SATA_CL_SIZE * slot->slot)))
170 
171 struct fsl_sata_dma_prd {
172 	uint32_t		dba;
173 	uint32_t		reserved;
174 	uint32_t		reserved2;
175 	uint32_t		dwc_flg;		/* 0 based */
176 #define	FSL_SATA_PRD_MASK		0x01fffffc	/* max 32MB */
177 #define	FSL_SATA_PRD_MAX		(FSL_SATA_PRD_MASK + 4)
178 #define	FSL_SATA_PRD_SNOOP		0x10000000
179 #define	FSL_SATA_PRD_EXT		0x80000000
180 } __packed;
181 
182 struct fsl_sata_cmd_tab {
183 	uint8_t			cfis[32];
184 	uint8_t			sfis[32];
185 	uint8_t			acmd[16];
186 	uint8_t			reserved[16];
187 	struct fsl_sata_dma_prd	prd_tab[FSL_SATA_SG_ENTRIES];
188 #define	FSL_SATA_PRD_EXT_INDEX	15
189 #define FSL_SATA_PRD_MAX_DIRECT	16
190 } __packed;
191 
192 struct fsl_sata_cmd_list {
193 	uint32_t			cda;		/* word aligned */
194 	uint16_t			fis_length;	/* length in bytes (aligned to words) */
195 	uint16_t			prd_length;	/* PRD entries */
196 	uint32_t			ttl;
197 	uint32_t			cmd_flags;
198 #define	FSL_SATA_CMD_TAG_MASK		0x001f
199 #define	FSL_SATA_CMD_ATAPI		0x0020
200 #define	FSL_SATA_CMD_BIST		0x0040
201 #define	FSL_SATA_CMD_RESET		0x0080
202 #define	FSL_SATA_CMD_QUEUED		0x0100
203 #define	FSL_SATA_CMD_SNOOP		0x0200
204 #define	FSL_SATA_CMD_VBIST		0x0400
205 #define	FSL_SATA_CMD_WRITE		0x0800
206 
207 } __packed;
208 
209 /* misc defines */
210 #define	ATA_IRQ_RID		0
211 #define	ATA_INTR_FLAGS		(INTR_MPSAFE|INTR_TYPE_BIO|INTR_ENTROPY)
212 
213 struct ata_dmaslot {
214 	bus_dmamap_t		data_map;	/* data DMA map */
215 	int			nsegs;		/* Number of segs loaded */
216 };
217 
218 /* structure holding DMA related information */
219 struct ata_dma {
220 	bus_dma_tag_t		 work_tag;	/* workspace DMA tag */
221 	bus_dmamap_t		 work_map;	/* workspace DMA map */
222 	uint8_t			*work;		/* workspace */
223 	bus_addr_t		 work_bus;	/* bus address of work */
224 	bus_dma_tag_t		data_tag;	/* data DMA tag */
225 };
226 
227 enum fsl_sata_slot_states {
228 	FSL_SATA_SLOT_EMPTY,
229 	FSL_SATA_SLOT_LOADING,
230 	FSL_SATA_SLOT_RUNNING,
231 	FSL_SATA_SLOT_EXECUTING
232 };
233 
234 struct fsl_sata_slot {
235 	struct fsl_sata_channel		*ch;		/* Channel */
236 	uint8_t				 slot;		/* Number of this slot */
237 	enum fsl_sata_slot_states	 state;	/* Slot state */
238 	union ccb			*ccb;		/* CCB occupying slot */
239 	struct ata_dmaslot		 dma;	/* DMA data of this slot */
240 	struct callout			 timeout;	/* Execution timeout */
241 	uint32_t			 ttl;
242 };
243 
244 struct fsl_sata_device {
245 	int			revision;
246 	int			mode;
247 	u_int			bytecount;
248 	u_int			atapi;
249 	u_int			tags;
250 	u_int			caps;
251 };
252 
253 /* structure describing an ATA channel */
254 struct fsl_sata_channel {
255 	device_t		dev;		/* Device handle */
256 	int			unit;		/* Physical channel */
257 	struct resource		*r_mem;		/* Memory of this channel */
258 	struct resource		*r_irq;		/* Interrupt of this channel */
259 	void			*ih;		/* Interrupt handle */
260 	struct ata_dma		dma;		/* DMA data */
261 	struct cam_sim		*sim;
262 	struct cam_path		*path;
263 	uint32_t		caps;		/* Controller capabilities */
264 	int			pm_level;	/* power management level */
265 	int			devices;	/* What is present */
266 	int			pm_present;	/* PM presence reported */
267 
268 	union ccb		*hold[FSL_SATA_MAX_SLOTS];
269 	struct fsl_sata_slot	slot[FSL_SATA_MAX_SLOTS];
270 	uint32_t		oslots;		/* Occupied slots */
271 	uint32_t		rslots;		/* Running slots */
272 	uint32_t		aslots;		/* Slots with atomic commands  */
273 	uint32_t		eslots;		/* Slots in error */
274 	uint32_t		toslots;	/* Slots in timeout */
275 	int			lastslot;	/* Last used slot */
276 	int			taggedtarget;	/* Last tagged target */
277 	int			numrslots;	/* Number of running slots */
278 	int			numrslotspd[16];/* Number of running slots per dev */
279 	int			numtslots;	/* Number of tagged slots */
280 	int			numtslotspd[16];/* Number of tagged slots per dev */
281 	int			numhslots;	/* Number of held slots */
282 	int			recoverycmd;	/* Our READ LOG active */
283 	int			fatalerr;	/* Fatal error happend */
284 	int			resetting;	/* Hard-reset in progress. */
285 	int			resetpolldiv;	/* Hard-reset poll divider. */
286 	union ccb		*frozen;	/* Frozen command */
287 	struct callout		pm_timer;	/* Power management events */
288 	struct callout		reset_timer;	/* Hard-reset timeout */
289 
290 	struct fsl_sata_device	user[16];	/* User-specified settings */
291 	struct fsl_sata_device	curr[16];	/* Current settings */
292 
293 	struct mtx_padalign	mtx;		/* state lock */
294 	STAILQ_HEAD(, ccb_hdr)	doneq;		/* queue of completed CCBs */
295 	int			batch;		/* doneq is in use */
296 };
297 
298 enum fsl_sata_err_type {
299 	FSL_SATA_ERR_NONE,		/* No error */
300 	FSL_SATA_ERR_INVALID,	/* Error detected by us before submitting. */
301 	FSL_SATA_ERR_INNOCENT,	/* Innocent victim. */
302 	FSL_SATA_ERR_TFE,		/* Task File Error. */
303 	FSL_SATA_ERR_SATA,		/* SATA error. */
304 	FSL_SATA_ERR_TIMEOUT,	/* Command execution timeout. */
305 	FSL_SATA_ERR_NCQ,		/* NCQ command error. CCB should be put on hold
306 				 * until READ LOG executed to reveal error. */
307 };
308 
309 /* macros to hide busspace uglyness */
310 #define	ATA_INB(res, offset) \
311 	bus_read_1((res), (offset))
312 #define	ATA_INW(res, offset) \
313 	bus_read_2((res), (offset))
314 #define	ATA_INL(res, offset) \
315 	bus_read_4((res), (offset))
316 #define	ATA_INSW(res, offset, addr, count) \
317 	bus_read_multi_2((res), (offset), (addr), (count))
318 #define	ATA_INSW_STRM(res, offset, addr, count) \
319 	bus_read_multi_stream_2((res), (offset), (addr), (count))
320 #define	ATA_INSL(res, offset, addr, count) \
321 	bus_read_multi_4((res), (offset), (addr), (count))
322 #define	ATA_INSL_STRM(res, offset, addr, count) \
323 	bus_read_multi_stream_4((res), (offset), (addr), (count))
324 #define	ATA_OUTB(res, offset, value) \
325 	bus_write_1((res), (offset), (value))
326 #define	ATA_OUTW(res, offset, value) \
327 	bus_write_2((res), (offset), (value))
328 #define	ATA_OUTL(res, offset, value) \
329 	bus_write_4((res), (offset), (value))
330 #define	ATA_OUTSW(res, offset, addr, count) \
331 	bus_write_multi_2((res), (offset), (addr), (count))
332 #define	ATA_OUTSW_STRM(res, offset, addr, count) \
333 	bus_write_multi_stream_2((res), (offset), (addr), (count))
334 #define	ATA_OUTSL(res, offset, addr, count) \
335 	bus_write_multi_4((res), (offset), (addr), (count))
336 #define	ATA_OUTSL_STRM(res, offset, addr, count) \
337 	bus_write_multi_stream_4((res), (offset), (addr), (count))
338 
339 static int
340 fsl_sata_probe(device_t dev)
341 {
342 
343 	if (!ofw_bus_is_compatible(dev, "fsl,pq-sata-v2") &&
344 	    !ofw_bus_is_compatible(dev, "fsl,pq-sata"))
345 		return (ENXIO);
346 
347 	device_set_desc_copy(dev, "Freescale Integrated SATA Controller");
348 	return (BUS_PROBE_DEFAULT);
349 }
350 
351 static int
352 fsl_sata_attach(device_t dev)
353 {
354 	struct fsl_sata_channel *ch = device_get_softc(dev);
355 	struct cam_devq *devq;
356 	int rid, error, i, sata_rev = 0;
357 
358 	ch->dev = dev;
359 	ch->unit = (intptr_t)device_get_ivars(dev);
360 	mtx_init(&ch->mtx, "FSL SATA channel lock", NULL, MTX_DEF);
361 	ch->pm_level = 0;
362 	resource_int_value(device_get_name(dev),
363 	    device_get_unit(dev), "pm_level", &ch->pm_level);
364 	STAILQ_INIT(&ch->doneq);
365 	if (ch->pm_level > 3)
366 		callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
367 	resource_int_value(device_get_name(dev),
368 	    device_get_unit(dev), "sata_rev", &sata_rev);
369 	for (i = 0; i < 16; i++) {
370 		ch->user[i].revision = sata_rev;
371 		ch->user[i].mode = 0;
372 		ch->user[i].bytecount = 8192;
373 		ch->user[i].tags = FSL_SATA_MAX_SLOTS;
374 		ch->user[i].caps = 0;
375 		ch->curr[i] = ch->user[i];
376 		if (ch->pm_level) {
377 			ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
378 			    CTS_SATA_CAPS_D_PMREQ;
379 		}
380 		ch->user[i].caps |= CTS_SATA_CAPS_H_AN;
381 	}
382 	rid = 0;
383 	if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
384 	    &rid, RF_ACTIVE)))
385 		return (ENXIO);
386 	rman_set_bustag(ch->r_mem, &bs_le_tag);
387 	fsl_sata_dmainit(dev);
388 	fsl_sata_slotsalloc(dev);
389 	fsl_sata_init(dev);
390 	rid = ATA_IRQ_RID;
391 	if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
392 	    &rid, RF_SHAREABLE | RF_ACTIVE))) {
393 		device_printf(dev, "Unable to map interrupt\n");
394 		error = ENXIO;
395 		goto err0;
396 	}
397 	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
398 	    fsl_sata_intr, ch, &ch->ih))) {
399 		device_printf(dev, "Unable to setup interrupt\n");
400 		error = ENXIO;
401 		goto err1;
402 	}
403 	mtx_lock(&ch->mtx);
404 	/* Create the device queue for our SIM. */
405 	devq = cam_simq_alloc(FSL_SATA_MAX_SLOTS);
406 	if (devq == NULL) {
407 		device_printf(dev, "Unable to allocate simq\n");
408 		error = ENOMEM;
409 		goto err1;
410 	}
411 	/* Construct SIM entry */
412 	ch->sim = cam_sim_alloc(fsl_sataaction, fsl_satapoll, "fslsata", ch,
413 	    device_get_unit(dev), (struct mtx *)&ch->mtx, 2, FSL_SATA_MAX_SLOTS,
414 	    devq);
415 	if (ch->sim == NULL) {
416 		cam_simq_free(devq);
417 		device_printf(dev, "unable to allocate sim\n");
418 		error = ENOMEM;
419 		goto err1;
420 	}
421 	if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
422 		device_printf(dev, "unable to register xpt bus\n");
423 		error = ENXIO;
424 		goto err2;
425 	}
426 	if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
427 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
428 		device_printf(dev, "unable to create path\n");
429 		error = ENXIO;
430 		goto err3;
431 	}
432 	if (ch->pm_level > 3) {
433 		callout_reset(&ch->pm_timer,
434 		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
435 		    fsl_sata_pm, ch);
436 	}
437 	mtx_unlock(&ch->mtx);
438 	return (0);
439 
440 err3:
441 	xpt_bus_deregister(cam_sim_path(ch->sim));
442 err2:
443 	cam_sim_free(ch->sim, /*free_devq*/TRUE);
444 err1:
445 	mtx_unlock(&ch->mtx);
446 	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
447 err0:
448 	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
449 	mtx_destroy(&ch->mtx);
450 	return (error);
451 }
452 
453 static int
454 fsl_sata_detach(device_t dev)
455 {
456 	struct fsl_sata_channel *ch = device_get_softc(dev);
457 
458 	mtx_lock(&ch->mtx);
459 	xpt_async(AC_LOST_DEVICE, ch->path, NULL);
460 
461 	xpt_free_path(ch->path);
462 	xpt_bus_deregister(cam_sim_path(ch->sim));
463 	cam_sim_free(ch->sim, /*free_devq*/TRUE);
464 	mtx_unlock(&ch->mtx);
465 
466 	if (ch->pm_level > 3)
467 		callout_drain(&ch->pm_timer);
468 	bus_teardown_intr(dev, ch->r_irq, ch->ih);
469 	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
470 
471 	fsl_sata_deinit(dev);
472 	fsl_sata_slotsfree(dev);
473 	fsl_sata_dmafini(dev);
474 
475 	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
476 	mtx_destroy(&ch->mtx);
477 	return (0);
478 }
479 
480 static int
481 fsl_sata_wait_register(struct fsl_sata_channel *ch, bus_size_t off,
482     unsigned int mask, unsigned int val, int t)
483 {
484 	int timeout = 0;
485 	uint32_t rval;
486 
487 	while (((rval = ATA_INL(ch->r_mem, off)) & mask) != val) {
488 		if (timeout > t) {
489 			return (EBUSY);
490 		}
491 		DELAY(1000);
492 		timeout++;
493 	}
494 	return (0);
495 }
496 
497 static int
498 fsl_sata_init(device_t dev)
499 {
500 	struct fsl_sata_channel *ch = device_get_softc(dev);
501 	uint64_t work;
502 	uint32_t r;
503 
504 	/* Disable port interrupts */
505 	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
506 	r &= ~FSL_SATA_P_HCTRL_HC_ON;
507 	r |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
508 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
509 	fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
510 	    FSL_SATA_P_HSTS_HS_ON, 0, 1000);
511 	/* Setup work areas */
512 	work = ch->dma.work_bus + FSL_SATA_CL_OFFSET;
513 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CHBA, work);
514 	r &= ~FSL_SATA_P_HCTRL_ENT;
515 	r &= ~FSL_SATA_P_HCTRL_PM;
516 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r);
517 	r = ATA_INL(ch->r_mem, FSL_SATA_P_PCC);
518 	ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, r & ~FSL_SATA_PCC_LPB_EN);
519 	ATA_OUTL(ch->r_mem, FSL_SATA_P_ICC, (1 << FSL_SATA_P_ICC_ITC_S));
520 	fsl_sata_start(ch);
521 	return (0);
522 }
523 
524 static int
525 fsl_sata_deinit(device_t dev)
526 {
527 	struct fsl_sata_channel *ch = device_get_softc(dev);
528 	uint32_t r;
529 
530 	/* Disable port interrupts. */
531 	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
532 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK);
533 	/* Reset command register. */
534 	fsl_sata_stop(ch);
535 	/* Allow everything, including partial and slumber modes. */
536 	ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, 0);
537 	DELAY(100);
538 	/* Disable PHY. */
539 	ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, ATA_SC_DET_DISABLE);
540 	r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
541 	/* Turn off the controller. */
542 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_HC_ON);
543 	return (0);
544 }
545 
546 static int
547 fsl_sata_suspend(device_t dev)
548 {
549 	struct fsl_sata_channel *ch = device_get_softc(dev);
550 
551 	mtx_lock(&ch->mtx);
552 	xpt_freeze_simq(ch->sim, 1);
553 	while (ch->oslots)
554 		msleep(ch, &ch->mtx, PRIBIO, "fsl_satasusp", hz/100);
555 	fsl_sata_deinit(dev);
556 	mtx_unlock(&ch->mtx);
557 	return (0);
558 }
559 
560 static int
561 fsl_sata_resume(device_t dev)
562 {
563 	struct fsl_sata_channel *ch = device_get_softc(dev);
564 
565 	mtx_lock(&ch->mtx);
566 	fsl_sata_init(dev);
567 	fsl_sata_reset(ch);
568 	xpt_release_simq(ch->sim, TRUE);
569 	mtx_unlock(&ch->mtx);
570 	return (0);
571 }
572 
573 devclass_t fsl_satach_devclass;
574 static device_method_t fsl_satach_methods[] = {
575 	DEVMETHOD(device_probe,     fsl_sata_probe),
576 	DEVMETHOD(device_attach,    fsl_sata_attach),
577 	DEVMETHOD(device_detach,    fsl_sata_detach),
578 	DEVMETHOD(device_suspend,   fsl_sata_suspend),
579 	DEVMETHOD(device_resume,    fsl_sata_resume),
580 	DEVMETHOD_END
581 };
582 static driver_t fsl_satach_driver = {
583 	"fslsata",
584 	fsl_satach_methods,
585 	sizeof(struct fsl_sata_channel)
586 };
587 DRIVER_MODULE(fsl_satach, simplebus, fsl_satach_driver, fsl_satach_devclass, NULL, NULL);
588 
589 struct fsl_sata_dc_cb_args {
590 	bus_addr_t maddr;
591 	int error;
592 };
593 
594 static void
595 fsl_sata_dmainit(device_t dev)
596 {
597 	struct fsl_sata_channel *ch = device_get_softc(dev);
598 	struct fsl_sata_dc_cb_args dcba;
599 
600 	/* Command area. */
601 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
602 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
603 	    NULL, NULL, FSL_SATA_WORK_SIZE, 1, FSL_SATA_WORK_SIZE,
604 	    0, NULL, NULL, &ch->dma.work_tag))
605 		goto error;
606 	if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work,
607 	    BUS_DMA_ZERO, &ch->dma.work_map))
608 		goto error;
609 	if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
610 	    FSL_SATA_WORK_SIZE, fsl_sata_dmasetupc_cb, &dcba, 0) || dcba.error) {
611 		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
612 		goto error;
613 	}
614 	ch->dma.work_bus = dcba.maddr;
615 	/* Data area. */
616 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
617 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
618 	    NULL, NULL, FSL_SATA_MAX_XFER,
619 	    FSL_SATA_SG_ENTRIES - 1, FSL_SATA_PRD_MAX,
620 	    0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
621 		goto error;
622 	}
623 	if (bootverbose)
624 		device_printf(dev, "work area: %p\n", ch->dma.work);
625 	return;
626 
627 error:
628 	device_printf(dev, "WARNING - DMA initialization failed\n");
629 	fsl_sata_dmafini(dev);
630 }
631 
632 static void
633 fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
634 {
635 	struct fsl_sata_dc_cb_args *dcba = (struct fsl_sata_dc_cb_args *)xsc;
636 
637 	if (!(dcba->error = error))
638 		dcba->maddr = segs[0].ds_addr;
639 }
640 
641 static void
642 fsl_sata_dmafini(device_t dev)
643 {
644 	struct fsl_sata_channel *ch = device_get_softc(dev);
645 
646 	if (ch->dma.data_tag) {
647 		bus_dma_tag_destroy(ch->dma.data_tag);
648 		ch->dma.data_tag = NULL;
649 	}
650 	if (ch->dma.work_bus) {
651 		bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
652 		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
653 		ch->dma.work_bus = 0;
654 		ch->dma.work = NULL;
655 	}
656 	if (ch->dma.work_tag) {
657 		bus_dma_tag_destroy(ch->dma.work_tag);
658 		ch->dma.work_tag = NULL;
659 	}
660 }
661 
662 static void
663 fsl_sata_slotsalloc(device_t dev)
664 {
665 	struct fsl_sata_channel *ch = device_get_softc(dev);
666 	int i;
667 
668 	/* Alloc and setup command/dma slots */
669 	bzero(ch->slot, sizeof(ch->slot));
670 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
671 		struct fsl_sata_slot *slot = &ch->slot[i];
672 
673 		slot->ch = ch;
674 		slot->slot = i;
675 		slot->state = FSL_SATA_SLOT_EMPTY;
676 		slot->ccb = NULL;
677 		callout_init_mtx(&slot->timeout, &ch->mtx, 0);
678 
679 		if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
680 			device_printf(ch->dev, "FAILURE - create data_map\n");
681 	}
682 }
683 
684 static void
685 fsl_sata_slotsfree(device_t dev)
686 {
687 	struct fsl_sata_channel *ch = device_get_softc(dev);
688 	int i;
689 
690 	/* Free all dma slots */
691 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
692 		struct fsl_sata_slot *slot = &ch->slot[i];
693 
694 		callout_drain(&slot->timeout);
695 		if (slot->dma.data_map) {
696 			bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
697 			slot->dma.data_map = NULL;
698 		}
699 	}
700 }
701 
702 static int
703 fsl_sata_phy_check_events(struct fsl_sata_channel *ch, u_int32_t serr)
704 {
705 
706 	if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
707 	    ((ch->pm_level != 0) && (serr & ATA_SE_EXCHANGED))) {
708 		u_int32_t status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS);
709 		union ccb *ccb;
710 
711 		if (bootverbose) {
712 			if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE)
713 				device_printf(ch->dev, "CONNECT requested\n");
714 			else
715 				device_printf(ch->dev, "DISCONNECT requested\n");
716 		}
717 		/* Issue soft reset */
718 		xpt_async(AC_BUS_RESET, ch->path, NULL);
719 		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
720 			return (0);
721 		if (xpt_create_path(&ccb->ccb_h.path, NULL,
722 		    cam_sim_path(ch->sim),
723 		    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
724 			xpt_free_ccb(ccb);
725 			return (0);
726 		}
727 		xpt_rescan(ccb);
728 		return (1);
729 	}
730 	return (0);
731 }
732 
733 static void
734 fsl_sata_notify_events(struct fsl_sata_channel *ch, u_int32_t status)
735 {
736 	struct cam_path *dpath;
737 	int i;
738 
739 	ATA_OUTL(ch->r_mem, FSL_SATA_P_SNTF, status);
740 	if (bootverbose)
741 		device_printf(ch->dev, "SNTF 0x%04x\n", status);
742 	for (i = 0; i < 16; i++) {
743 		if ((status & (1 << i)) == 0)
744 			continue;
745 		if (xpt_create_path(&dpath, NULL,
746 		    xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) {
747 			xpt_async(AC_SCSI_AEN, dpath, NULL);
748 			xpt_free_path(dpath);
749 		}
750 	}
751 }
752 
753 static void
754 fsl_sata_done(struct fsl_sata_channel *ch, union ccb *ccb)
755 {
756 
757 	mtx_assert(&ch->mtx, MA_OWNED);
758 	if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 ||
759 	    ch->batch == 0) {
760 		xpt_done(ccb);
761 		return;
762 	}
763 
764 	STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe);
765 }
766 
767 static void
768 fsl_sata_intr(void *arg)
769 {
770 	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
771 	struct ccb_hdr *ccb_h;
772 	uint32_t istatus;
773 	STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq);
774 
775 	/* Read interrupt statuses. */
776 	istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & 0x7ffff;
777 	if ((istatus & 0x3f) == 0)
778 		return;
779 
780 	mtx_lock(&ch->mtx);
781 	ch->batch = 1;
782 	fsl_sata_intr_main(ch, istatus);
783 	ch->batch = 0;
784 	/*
785 	 * Prevent the possibility of issues caused by processing the queue
786 	 * while unlocked below by moving the contents to a local queue.
787 	 */
788 	STAILQ_CONCAT(&tmp_doneq, &ch->doneq);
789 	mtx_unlock(&ch->mtx);
790 	while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) {
791 		STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe);
792 		xpt_done_direct((union ccb *)ccb_h);
793 	}
794 	/* Clear interrupt statuses. */
795 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, istatus & 0x3f);
796 
797 }
798 
799 static void
800 fsl_sata_pm(void *arg)
801 {
802 	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg;
803 	uint32_t work;
804 
805 	if (ch->numrslots != 0)
806 		return;
807 	work = ATA_INL(ch->r_mem, FSL_SATA_P_PCC) & ~FSL_SATA_PCC_LPB_EN;
808 	if (ch->pm_level == 4)
809 		work |= FSL_SATA_P_PCC_PARTIAL;
810 	else
811 		work |= FSL_SATA_P_PCC_SLUMBER;
812 	ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, work);
813 }
814 
815 /* XXX: interrupt todo */
816 static void
817 fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus)
818 {
819 	uint32_t cer, der, serr = 0, sntf = 0, ok, err;
820 	enum fsl_sata_err_type et;
821 	int i;
822 
823 	/* Complete all successful commands. */
824 	ok = ATA_INL(ch->r_mem, FSL_SATA_P_CCR);
825 	/* Mark all commands complete, to complete the interrupt. */
826 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, ok);
827 	if (ch->aslots == 0 && ok != 0) {
828 		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
829 			if (((ok >> i) & 1) && ch->slot[i].ccb != NULL)
830 				fsl_sata_end_transaction(&ch->slot[i],
831 				    FSL_SATA_ERR_NONE);
832 		}
833 	}
834 	/* Read command statuses. */
835 	if (istatus & FSL_SATA_P_HSTS_SNTFU)
836 		sntf = ATA_INL(ch->r_mem, FSL_SATA_P_SNTF);
837 	/* XXX: Process PHY events */
838 	serr = ATA_INL(ch->r_mem, FSL_SATA_P_SERR);
839 	ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, serr);
840 	if (istatus & (FSL_SATA_P_HSTS_PR)) {
841 		if (serr) {
842 			fsl_sata_phy_check_events(ch, serr);
843 		}
844 	}
845 	/* Process command errors */
846 	err = (istatus & (FSL_SATA_P_HSTS_FE | FSL_SATA_P_HSTS_DE));
847 	cer = ATA_INL(ch->r_mem, FSL_SATA_P_CER);
848 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, cer);
849 	der = ATA_INL(ch->r_mem, FSL_SATA_P_DER);
850 	ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, der);
851 	/* On error, complete the rest of commands with error statuses. */
852 	if (err) {
853 		if (ch->frozen) {
854 			union ccb *fccb = ch->frozen;
855 			ch->frozen = NULL;
856 			fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
857 			if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
858 				xpt_freeze_devq(fccb->ccb_h.path, 1);
859 				fccb->ccb_h.status |= CAM_DEV_QFRZN;
860 			}
861 			fsl_sata_done(ch, fccb);
862 		}
863 		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
864 			if (ch->slot[i].ccb == NULL)
865 				continue;
866 			if ((cer & (1 << i)) != 0)
867 				et = FSL_SATA_ERR_TFE;
868 			else if ((der & (1 << ch->slot[i].ccb->ccb_h.target_id)) != 0)
869 				et = FSL_SATA_ERR_SATA;
870 			else
871 				et = FSL_SATA_ERR_INVALID;
872 			fsl_sata_end_transaction(&ch->slot[i], et);
873 		}
874 	}
875 	/* Process NOTIFY events */
876 	if (sntf)
877 		fsl_sata_notify_events(ch, sntf);
878 }
879 
880 /* Must be called with channel locked. */
881 static int
882 fsl_sata_check_collision(struct fsl_sata_channel *ch, union ccb *ccb)
883 {
884 	int t = ccb->ccb_h.target_id;
885 
886 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
887 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
888 		/* Tagged command while we have no supported tag free. */
889 		if (((~ch->oslots) & (0xffff >> (16 - ch->curr[t].tags))) == 0)
890 			return (1);
891 		/* Tagged command while untagged are active. */
892 		if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0)
893 			return (1);
894 	} else {
895 		/* Untagged command while tagged are active. */
896 		if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0)
897 			return (1);
898 	}
899 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
900 	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) {
901 		/* Atomic command while anything active. */
902 		if (ch->numrslots != 0)
903 			return (1);
904 	}
905 	/* We have some atomic command running. */
906 	if (ch->aslots != 0)
907 		return (1);
908 	return (0);
909 }
910 
911 /* Must be called with channel locked. */
912 static void
913 fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb)
914 {
915 	struct fsl_sata_slot *slot;
916 	int tag, tags;
917 
918 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
919 	    ("fsl_sata_begin_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
920 	/* Choose empty slot. */
921 	tags = FSL_SATA_MAX_SLOTS;
922 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
923 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
924 		tags = ch->curr[ccb->ccb_h.target_id].tags;
925 	if (ch->lastslot + 1 < tags)
926 		tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
927 	else
928 		tag = 0;
929 	if (tag == 0 || tag + ch->lastslot >= tags)
930 		tag = ffs(~ch->oslots) - 1;
931 	else
932 		tag += ch->lastslot;
933 	ch->lastslot = tag;
934 	/* Occupy chosen slot. */
935 	slot = &ch->slot[tag];
936 	slot->ccb = ccb;
937 	slot->ttl = 0;
938 	/* Stop PM timer. */
939 	if (ch->numrslots == 0 && ch->pm_level > 3)
940 		callout_stop(&ch->pm_timer);
941 	/* Update channel stats. */
942 	ch->oslots |= (1 << tag);
943 	ch->numrslots++;
944 	ch->numrslotspd[ccb->ccb_h.target_id]++;
945 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
946 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
947 		ch->numtslots++;
948 		ch->numtslotspd[ccb->ccb_h.target_id]++;
949 		ch->taggedtarget = ccb->ccb_h.target_id;
950 	}
951 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
952 	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
953 		ch->aslots |= (1 << tag);
954 	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
955 		slot->state = FSL_SATA_SLOT_LOADING;
956 		bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
957 		    fsl_sata_dmasetprd, slot, 0);
958 	} else {
959 		slot->dma.nsegs = 0;
960 		fsl_sata_execute_transaction(slot);
961 	}
962 
963 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
964 	    ("fsl_sata_begin_transaction exit\n"));
965 }
966 
967 /* Locked by busdma engine. */
968 static void
969 fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
970 {
971 	struct fsl_sata_slot *slot = arg;
972 	struct fsl_sata_channel *ch = slot->ch;
973 	struct fsl_sata_cmd_tab *ctp;
974 	struct fsl_sata_dma_prd *prd;
975 	int i, j, len, extlen;
976 
977 	if (error) {
978 		device_printf(ch->dev, "DMA load error %d\n", error);
979 		fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
980 		return;
981 	}
982 	KASSERT(nsegs <= FSL_SATA_SG_ENTRIES - 1,
983 	    ("too many DMA segment entries\n"));
984 	/* Get a piece of the workspace for this request */
985 	ctp = FSL_SATA_CTP(ch, slot);
986 	/* Fill S/G table */
987 	prd = &ctp->prd_tab[0];
988 	for (i = 0, j = 0; i < nsegs; i++, j++) {
989 		if (j == FSL_SATA_PRD_EXT_INDEX &&
990 		    FSL_SATA_PRD_MAX_DIRECT < nsegs) {
991 			prd[j].dba = htole32(FSL_SATA_CTP_BUS(ch, slot) +
992 				     FSL_SATA_PRD_OFFSET(j+1));
993 			j++;
994 			extlen = 0;
995 		}
996 		len = segs[i].ds_len;
997 		len = roundup2(len, sizeof(uint32_t));
998 		prd[j].dba = htole32((uint32_t)segs[i].ds_addr);
999 		prd[j].dwc_flg = htole32(FSL_SATA_PRD_SNOOP | len);
1000 		slot->ttl += len;
1001 		if (j > FSL_SATA_PRD_MAX_DIRECT)
1002 			extlen += len;
1003 	}
1004 	slot->dma.nsegs = j;
1005 	if (j > FSL_SATA_PRD_MAX_DIRECT)
1006 		prd[FSL_SATA_PRD_EXT_INDEX].dwc_flg =
1007 		    htole32(FSL_SATA_PRD_SNOOP | FSL_SATA_PRD_EXT | extlen);
1008 	bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1009 	    ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
1010 	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
1011 	fsl_sata_execute_transaction(slot);
1012 }
1013 
1014 /* Must be called with channel locked. */
1015 static void
1016 fsl_sata_execute_transaction(struct fsl_sata_slot *slot)
1017 {
1018 	struct fsl_sata_channel *ch = slot->ch;
1019 	struct fsl_sata_cmd_tab *ctp;
1020 	struct fsl_sata_cmd_list *clp;
1021 	union ccb *ccb = slot->ccb;
1022 	int port = ccb->ccb_h.target_id & 0x0f;
1023 	int fis_size, i, softreset;
1024 	uint32_t tmp;
1025 	uint32_t cmd_flags = FSL_SATA_CMD_WRITE | FSL_SATA_CMD_SNOOP;
1026 
1027 	softreset = 0;
1028 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
1029 	    ("fsl_sata_execute_transaction func_code=0x%x\n", ccb->ccb_h.func_code));
1030 	/* Get a piece of the workspace for this request */
1031 	ctp = FSL_SATA_CTP(ch, slot);
1032 	/* Setup the FIS for this request */
1033 	if (!(fis_size = fsl_sata_setup_fis(ch, ctp, ccb, slot->slot))) {
1034 		device_printf(ch->dev, "Setting up SATA FIS failed\n");
1035 		fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID);
1036 		return;
1037 	}
1038 	/* Setup the command list entry */
1039 	clp = FSL_SATA_CLP(ch, slot);
1040 	clp->fis_length = htole16(fis_size);
1041 	clp->prd_length = htole16(slot->dma.nsegs);
1042 	/* Special handling for Soft Reset command. */
1043 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1044 	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
1045 		if (ccb->ataio.cmd.control & ATA_A_RESET) {
1046 			softreset = 1;
1047 			cmd_flags |= FSL_SATA_CMD_RESET;
1048 		} else {
1049 			/* Prepare FIS receive area for check. */
1050 			for (i = 0; i < 32; i++)
1051 				ctp->sfis[i] = 0xff;
1052 			softreset = 2;
1053 		}
1054 	}
1055 	if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)
1056 		cmd_flags |= FSL_SATA_CMD_QUEUED;
1057 	clp->cmd_flags = htole32(cmd_flags |
1058 	    (ccb->ccb_h.func_code == XPT_SCSI_IO ?  FSL_SATA_CMD_ATAPI : 0) |
1059 	    slot->slot);
1060 	clp->ttl = htole32(slot->ttl);
1061 	clp->cda = htole32(FSL_SATA_CTP_BUS(ch, slot));
1062 	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1063 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1064 	/* Issue command to the controller. */
1065 	slot->state = FSL_SATA_SLOT_RUNNING;
1066 	ch->rslots |= (1 << slot->slot);
1067 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CQPMP, port);
1068 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CQR, (1 << slot->slot));
1069 	/* Device reset commands don't interrupt. Poll them. */
1070 	if (ccb->ccb_h.func_code == XPT_ATA_IO &&
1071 	    (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) {
1072 		int count, timeout = ccb->ccb_h.timeout * 100;
1073 		enum fsl_sata_err_type et = FSL_SATA_ERR_NONE;
1074 
1075 		for (count = 0; count < timeout; count++) {
1076 			DELAY(10);
1077 			tmp = 0;
1078 			if (softreset == 2) {
1079 				tmp = ATA_INL(ch->r_mem, FSL_SATA_P_SIG);
1080 				if (tmp != 0 && tmp != 0xffffffff)
1081 					break;
1082 				continue;
1083 			}
1084 			if ((ATA_INL(ch->r_mem, FSL_SATA_P_CCR) & (1 << slot->slot)) != 0)
1085 				break;
1086 		}
1087 
1088 		if (timeout && (count >= timeout)) {
1089 			device_printf(ch->dev, "Poll timeout on slot %d port %d (round %d)\n",
1090 			    slot->slot, port, softreset);
1091 			device_printf(ch->dev, "hsts %08x cqr %08x ccr %08x ss %08x "
1092 			    "rs %08x cer %08x der %08x serr %08x car %08x sig %08x\n",
1093 			    ATA_INL(ch->r_mem, FSL_SATA_P_HSTS),
1094 			    ATA_INL(ch->r_mem, FSL_SATA_P_CQR),
1095 			    ATA_INL(ch->r_mem, FSL_SATA_P_CCR),
1096 			    ATA_INL(ch->r_mem, FSL_SATA_P_SSTS), ch->rslots,
1097 			    ATA_INL(ch->r_mem, FSL_SATA_P_CER),
1098 			    ATA_INL(ch->r_mem, FSL_SATA_P_DER),
1099 			    ATA_INL(ch->r_mem, FSL_SATA_P_SERR),
1100 			    ATA_INL(ch->r_mem, FSL_SATA_P_CAR),
1101 			    ATA_INL(ch->r_mem, FSL_SATA_P_SIG));
1102 			et = FSL_SATA_ERR_TIMEOUT;
1103 		}
1104 
1105 		fsl_sata_end_transaction(slot, et);
1106 		return;
1107 	}
1108 	/* Start command execution timeout */
1109 	callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2,
1110 	    0, fsl_sata_timeout, slot, 0);
1111 	return;
1112 }
1113 
1114 /* Must be called with channel locked. */
1115 static void
1116 fsl_sata_process_timeout(struct fsl_sata_channel *ch)
1117 {
1118 	int i;
1119 
1120 	mtx_assert(&ch->mtx, MA_OWNED);
1121 	/* Handle the rest of commands. */
1122 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1123 		/* Do we have a running request on slot? */
1124 		if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
1125 			continue;
1126 		fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_TIMEOUT);
1127 	}
1128 }
1129 
1130 /* Must be called with channel locked. */
1131 static void
1132 fsl_sata_rearm_timeout(struct fsl_sata_channel *ch)
1133 {
1134 	int i;
1135 
1136 	mtx_assert(&ch->mtx, MA_OWNED);
1137 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1138 		struct fsl_sata_slot *slot = &ch->slot[i];
1139 
1140 		/* Do we have a running request on slot? */
1141 		if (slot->state < FSL_SATA_SLOT_RUNNING)
1142 			continue;
1143 		if ((ch->toslots & (1 << i)) == 0)
1144 			continue;
1145 		callout_reset_sbt(&slot->timeout,
1146  	    	    SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1147 		    fsl_sata_timeout, slot, 0);
1148 	}
1149 }
1150 
1151 /* Locked by callout mechanism. */
1152 static void
1153 fsl_sata_timeout(void *arg)
1154 {
1155 	struct fsl_sata_slot *slot = arg;
1156 	struct fsl_sata_channel *ch = slot->ch;
1157 	device_t dev = ch->dev;
1158 	uint32_t sstatus;
1159 
1160 	/* Check for stale timeout. */
1161 	if (slot->state < FSL_SATA_SLOT_RUNNING)
1162 		return;
1163 
1164 	/* Check if slot was not being executed last time we checked. */
1165 	if (slot->state < FSL_SATA_SLOT_EXECUTING) {
1166 		/* Check if slot started executing. */
1167 		sstatus = ATA_INL(ch->r_mem, FSL_SATA_P_CAR);
1168 		if ((sstatus & (1 << slot->slot)) != 0)
1169 			slot->state = FSL_SATA_SLOT_EXECUTING;
1170 
1171 		callout_reset_sbt(&slot->timeout,
1172 	    	    SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1173 		    fsl_sata_timeout, slot, 0);
1174 		return;
1175 	}
1176 
1177 	device_printf(dev, "Timeout on slot %d port %d\n",
1178 	    slot->slot, slot->ccb->ccb_h.target_id & 0x0f);
1179 
1180 	/* Handle frozen command. */
1181 	if (ch->frozen) {
1182 		union ccb *fccb = ch->frozen;
1183 		ch->frozen = NULL;
1184 		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1185 		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1186 			xpt_freeze_devq(fccb->ccb_h.path, 1);
1187 			fccb->ccb_h.status |= CAM_DEV_QFRZN;
1188 		}
1189 		fsl_sata_done(ch, fccb);
1190 	}
1191 	if (ch->toslots == 0)
1192 		xpt_freeze_simq(ch->sim, 1);
1193 	ch->toslots |= (1 << slot->slot);
1194 	if ((ch->rslots & ~ch->toslots) == 0)
1195 		fsl_sata_process_timeout(ch);
1196 	else
1197 		device_printf(dev, " ... waiting for slots %08x\n",
1198 		    ch->rslots & ~ch->toslots);
1199 }
1200 
1201 /* Must be called with channel locked. */
1202 static void
1203 fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et)
1204 {
1205 	struct fsl_sata_channel *ch = slot->ch;
1206 	union ccb *ccb = slot->ccb;
1207 	struct fsl_sata_cmd_list *clp;
1208 	int lastto;
1209 	uint32_t sig;
1210 
1211 	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1212 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1213 	clp = FSL_SATA_CLP(ch, slot);
1214 	/* Read result registers to the result struct */
1215 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1216 		struct ata_res *res = &ccb->ataio.res;
1217 
1218 		if ((et == FSL_SATA_ERR_TFE) ||
1219 		    (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
1220 			struct fsl_sata_cmd_tab *ctp = FSL_SATA_CTP(ch, slot);
1221 			uint8_t *fis = ctp->sfis;
1222 
1223 			res->status = fis[2];
1224 			res->error = fis[3];
1225 			res->lba_low = fis[4];
1226 			res->lba_mid = fis[5];
1227 			res->lba_high = fis[6];
1228 			res->device = fis[7];
1229 			res->lba_low_exp = fis[8];
1230 			res->lba_mid_exp = fis[9];
1231 			res->lba_high_exp = fis[10];
1232 			res->sector_count = fis[12];
1233 			res->sector_count_exp = fis[13];
1234 
1235 			if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1236 			    (ccb->ataio.cmd.control & ATA_A_RESET) == 0) {
1237 				sig = ATA_INL(ch->r_mem,  FSL_SATA_P_SIG);
1238 				res->lba_high = sig >> 24;
1239 				res->lba_mid = sig >> 16;
1240 				res->lba_low = sig >> 8;
1241 				res->sector_count = sig;
1242 			}
1243 		} else
1244 			bzero(res, sizeof(*res));
1245 		if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 &&
1246 		    (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1247 			ccb->ataio.resid =
1248 			    ccb->ataio.dxfer_len - le32toh(clp->ttl);
1249 		}
1250 	} else {
1251 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1252 			ccb->csio.resid =
1253 			    ccb->csio.dxfer_len - le32toh(clp->ttl);
1254 		}
1255 	}
1256 	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1257 		bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1258 		    (ccb->ccb_h.flags & CAM_DIR_IN) ?
1259 		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1260 		bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
1261 	}
1262 	if (et != FSL_SATA_ERR_NONE)
1263 		ch->eslots |= (1 << slot->slot);
1264 	/* In case of error, freeze device for proper recovery. */
1265 	if ((et != FSL_SATA_ERR_NONE) && (!ch->recoverycmd) &&
1266 	    !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
1267 		xpt_freeze_devq(ccb->ccb_h.path, 1);
1268 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
1269 	}
1270 	/* Set proper result status. */
1271 	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1272 	switch (et) {
1273 	case FSL_SATA_ERR_NONE:
1274 		ccb->ccb_h.status |= CAM_REQ_CMP;
1275 		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1276 			ccb->csio.scsi_status = SCSI_STATUS_OK;
1277 		break;
1278 	case FSL_SATA_ERR_INVALID:
1279 		ch->fatalerr = 1;
1280 		ccb->ccb_h.status |= CAM_REQ_INVALID;
1281 		break;
1282 	case FSL_SATA_ERR_INNOCENT:
1283 		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1284 		break;
1285 	case FSL_SATA_ERR_TFE:
1286 	case FSL_SATA_ERR_NCQ:
1287 		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1288 			ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1289 			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1290 		} else {
1291 			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
1292 		}
1293 		break;
1294 	case FSL_SATA_ERR_SATA:
1295 		ch->fatalerr = 1;
1296 		if (!ch->recoverycmd) {
1297 			xpt_freeze_simq(ch->sim, 1);
1298 			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1299 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1300 		}
1301 		ccb->ccb_h.status |= CAM_UNCOR_PARITY;
1302 		break;
1303 	case FSL_SATA_ERR_TIMEOUT:
1304 		if (!ch->recoverycmd) {
1305 			xpt_freeze_simq(ch->sim, 1);
1306 			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1307 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1308 		}
1309 		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
1310 		break;
1311 	default:
1312 		ch->fatalerr = 1;
1313 		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1314 	}
1315 	/* Free slot. */
1316 	ch->oslots &= ~(1 << slot->slot);
1317 	ch->rslots &= ~(1 << slot->slot);
1318 	ch->aslots &= ~(1 << slot->slot);
1319 	slot->state = FSL_SATA_SLOT_EMPTY;
1320 	slot->ccb = NULL;
1321 	/* Update channel stats. */
1322 	ch->numrslots--;
1323 	ch->numrslotspd[ccb->ccb_h.target_id]--;
1324 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, 1 << slot->slot);
1325 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1326 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1327 		ch->numtslots--;
1328 		ch->numtslotspd[ccb->ccb_h.target_id]--;
1329 	}
1330 	/* Cancel timeout state if request completed normally. */
1331 	if (et != FSL_SATA_ERR_TIMEOUT) {
1332 		lastto = (ch->toslots == (1 << slot->slot));
1333 		ch->toslots &= ~(1 << slot->slot);
1334 		if (lastto)
1335 			xpt_release_simq(ch->sim, TRUE);
1336 	}
1337 	/* If it was first request of reset sequence and there is no error,
1338 	 * proceed to second request. */
1339 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1340 	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1341 	    (ccb->ataio.cmd.control & ATA_A_RESET) &&
1342 	    et == FSL_SATA_ERR_NONE) {
1343 		ccb->ataio.cmd.control &= ~ATA_A_RESET;
1344 		fsl_sata_begin_transaction(ch, ccb);
1345 		return;
1346 	}
1347 	/* If it was our READ LOG command - process it. */
1348 	if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
1349 		fsl_sata_process_read_log(ch, ccb);
1350 	/* If it was our REQUEST SENSE command - process it. */
1351 	} else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
1352 		fsl_sata_process_request_sense(ch, ccb);
1353 	/* If it was NCQ or ATAPI command error, put result on hold. */
1354 	} else if (et == FSL_SATA_ERR_NCQ ||
1355 	    ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
1356 	     (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) {
1357 		ch->hold[slot->slot] = ccb;
1358 		ch->numhslots++;
1359 	} else
1360 		fsl_sata_done(ch, ccb);
1361 	/* If we have no other active commands, ... */
1362 	if (ch->rslots == 0) {
1363 		/* if there was fatal error - reset port. */
1364 		if (ch->toslots != 0 || ch->fatalerr) {
1365 			fsl_sata_reset(ch);
1366 		} else {
1367 			/* if we have slots in error, we can reinit port. */
1368 			if (ch->eslots != 0) {
1369 				fsl_sata_stop(ch);
1370 				fsl_sata_start(ch);
1371 			}
1372 			/* if there commands on hold, we can do READ LOG. */
1373 			if (!ch->recoverycmd && ch->numhslots)
1374 				fsl_sata_issue_recovery(ch);
1375 		}
1376 	/* If all the rest of commands are in timeout - give them chance. */
1377 	} else if ((ch->rslots & ~ch->toslots) == 0 &&
1378 	    et != FSL_SATA_ERR_TIMEOUT)
1379 		fsl_sata_rearm_timeout(ch);
1380 	/* Unfreeze frozen command. */
1381 	if (ch->frozen && !fsl_sata_check_collision(ch, ch->frozen)) {
1382 		union ccb *fccb = ch->frozen;
1383 		ch->frozen = NULL;
1384 		fsl_sata_begin_transaction(ch, fccb);
1385 		xpt_release_simq(ch->sim, TRUE);
1386 	}
1387 	/* Start PM timer. */
1388 	if (ch->numrslots == 0 && ch->pm_level > 3 &&
1389 	    (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
1390 		callout_schedule(&ch->pm_timer,
1391 		    (ch->pm_level == 4) ? hz / 1000 : hz / 8);
1392 	}
1393 }
1394 
1395 static void
1396 fsl_sata_issue_recovery(struct fsl_sata_channel *ch)
1397 {
1398 	union ccb *ccb;
1399 	struct ccb_ataio *ataio;
1400 	struct ccb_scsiio *csio;
1401 	int i;
1402 
1403 	/* Find some held command. */
1404 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1405 		if (ch->hold[i])
1406 			break;
1407 	}
1408 	ccb = xpt_alloc_ccb_nowait();
1409 	if (ccb == NULL) {
1410 		device_printf(ch->dev, "Unable to allocate recovery command\n");
1411 completeall:
1412 		/* We can't do anything -- complete held commands. */
1413 		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1414 			if (ch->hold[i] == NULL)
1415 				continue;
1416 			ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1417 			ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
1418 			fsl_sata_done(ch, ch->hold[i]);
1419 			ch->hold[i] = NULL;
1420 			ch->numhslots--;
1421 		}
1422 		fsl_sata_reset(ch);
1423 		return;
1424 	}
1425 	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
1426 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1427 		/* READ LOG */
1428 		ccb->ccb_h.recovery_type = RECOVERY_READ_LOG;
1429 		ccb->ccb_h.func_code = XPT_ATA_IO;
1430 		ccb->ccb_h.flags = CAM_DIR_IN;
1431 		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
1432 		ataio = &ccb->ataio;
1433 		ataio->data_ptr = malloc(512, M_FSL_SATA, M_NOWAIT);
1434 		if (ataio->data_ptr == NULL) {
1435 			xpt_free_ccb(ccb);
1436 			device_printf(ch->dev,
1437 			    "Unable to allocate memory for READ LOG command\n");
1438 			goto completeall;
1439 		}
1440 		ataio->dxfer_len = 512;
1441 		bzero(&ataio->cmd, sizeof(ataio->cmd));
1442 		ataio->cmd.flags = CAM_ATAIO_48BIT;
1443 		ataio->cmd.command = 0x2F;	/* READ LOG EXT */
1444 		ataio->cmd.sector_count = 1;
1445 		ataio->cmd.sector_count_exp = 0;
1446 		ataio->cmd.lba_low = 0x10;
1447 		ataio->cmd.lba_mid = 0;
1448 		ataio->cmd.lba_mid_exp = 0;
1449 	} else {
1450 		/* REQUEST SENSE */
1451 		ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE;
1452 		ccb->ccb_h.recovery_slot = i;
1453 		ccb->ccb_h.func_code = XPT_SCSI_IO;
1454 		ccb->ccb_h.flags = CAM_DIR_IN;
1455 		ccb->ccb_h.status = 0;
1456 		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
1457 		csio = &ccb->csio;
1458 		csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
1459 		csio->dxfer_len = ch->hold[i]->csio.sense_len;
1460 		csio->cdb_len = 6;
1461 		bzero(&csio->cdb_io, sizeof(csio->cdb_io));
1462 		csio->cdb_io.cdb_bytes[0] = 0x03;
1463 		csio->cdb_io.cdb_bytes[4] = csio->dxfer_len;
1464 	}
1465 	/* Freeze SIM while doing recovery. */
1466 	ch->recoverycmd = 1;
1467 	xpt_freeze_simq(ch->sim, 1);
1468 	fsl_sata_begin_transaction(ch, ccb);
1469 }
1470 
1471 static void
1472 fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb)
1473 {
1474 	uint8_t *data;
1475 	struct ata_res *res;
1476 	int i;
1477 
1478 	ch->recoverycmd = 0;
1479 
1480 	data = ccb->ataio.data_ptr;
1481 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
1482 	    (data[0] & 0x80) == 0) {
1483 		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1484 			if (!ch->hold[i])
1485 				continue;
1486 			if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
1487 				continue;
1488 			if ((data[0] & 0x1F) == i) {
1489 				res = &ch->hold[i]->ataio.res;
1490 				res->status = data[2];
1491 				res->error = data[3];
1492 				res->lba_low = data[4];
1493 				res->lba_mid = data[5];
1494 				res->lba_high = data[6];
1495 				res->device = data[7];
1496 				res->lba_low_exp = data[8];
1497 				res->lba_mid_exp = data[9];
1498 				res->lba_high_exp = data[10];
1499 				res->sector_count = data[12];
1500 				res->sector_count_exp = data[13];
1501 			} else {
1502 				ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1503 				ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
1504 			}
1505 			fsl_sata_done(ch, ch->hold[i]);
1506 			ch->hold[i] = NULL;
1507 			ch->numhslots--;
1508 		}
1509 	} else {
1510 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
1511 			device_printf(ch->dev, "Error while READ LOG EXT\n");
1512 		else if ((data[0] & 0x80) == 0) {
1513 			device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
1514 		}
1515 		for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1516 			if (!ch->hold[i])
1517 				continue;
1518 			if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO)
1519 				continue;
1520 			fsl_sata_done(ch, ch->hold[i]);
1521 			ch->hold[i] = NULL;
1522 			ch->numhslots--;
1523 		}
1524 	}
1525 	free(ccb->ataio.data_ptr, M_FSL_SATA);
1526 	xpt_free_ccb(ccb);
1527 	xpt_release_simq(ch->sim, TRUE);
1528 }
1529 
1530 static void
1531 fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb)
1532 {
1533 	int i;
1534 
1535 	ch->recoverycmd = 0;
1536 
1537 	i = ccb->ccb_h.recovery_slot;
1538 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
1539 		ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
1540 	} else {
1541 		ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1542 		ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
1543 	}
1544 	fsl_sata_done(ch, ch->hold[i]);
1545 	ch->hold[i] = NULL;
1546 	ch->numhslots--;
1547 	xpt_free_ccb(ccb);
1548 	xpt_release_simq(ch->sim, TRUE);
1549 }
1550 
1551 static void
1552 fsl_sata_start(struct fsl_sata_channel *ch)
1553 {
1554 	u_int32_t cmd;
1555 
1556 	/* Clear SATA error register */
1557 	ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, 0xFFFFFFFF);
1558 	/* Clear any interrupts pending on this channel */
1559 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, 0x3F);
1560 	ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, 0xFFFF);
1561 	ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, 0xFFFF);
1562 	/* Start operations on this channel */
1563 	cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
1564 	cmd |= FSL_SATA_P_HCTRL_HC_ON | FSL_SATA_P_HCTRL_SNOOP;
1565 	cmd &= ~FSL_SATA_P_HCTRL_HC_FORCE_OFF;
1566 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd |
1567 	    (ch->pm_present ? FSL_SATA_P_HCTRL_PM : 0));
1568 	fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
1569 	    FSL_SATA_P_HSTS_PR, FSL_SATA_P_HSTS_PR, 500);
1570 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS,
1571 	    ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & FSL_SATA_P_HSTS_PR);
1572 }
1573 
1574 static void
1575 fsl_sata_stop(struct fsl_sata_channel *ch)
1576 {
1577 	uint32_t cmd;
1578 	int i;
1579 
1580 	/* Kill all activity on this channel */
1581 	cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL);
1582 	cmd &= ~FSL_SATA_P_HCTRL_HC_ON;
1583 
1584 	for (i = 0; i < 2; i++) {
1585 		ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd);
1586 		if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS,
1587 		    FSL_SATA_P_HSTS_HS_ON, 0, 500)) {
1588 			if (i != 0)
1589 				device_printf(ch->dev,
1590 				    "stopping FSL SATA engine failed\n");
1591 			cmd |= FSL_SATA_P_HCTRL_HC_FORCE_OFF;
1592 		} else
1593 			break;
1594 	}
1595 	ch->eslots = 0;
1596 }
1597 
1598 static void
1599 fsl_sata_reset(struct fsl_sata_channel *ch)
1600 {
1601 	uint32_t ctrl;
1602 	int i;
1603 
1604 	xpt_freeze_simq(ch->sim, 1);
1605 	if (bootverbose)
1606 		device_printf(ch->dev, "FSL SATA reset...\n");
1607 
1608 	/* Requeue freezed command. */
1609 	if (ch->frozen) {
1610 		union ccb *fccb = ch->frozen;
1611 		ch->frozen = NULL;
1612 		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1613 		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1614 			xpt_freeze_devq(fccb->ccb_h.path, 1);
1615 			fccb->ccb_h.status |= CAM_DEV_QFRZN;
1616 		}
1617 		fsl_sata_done(ch, fccb);
1618 	}
1619 	/* Kill the engine and requeue all running commands. */
1620 	fsl_sata_stop(ch);
1621 	DELAY(1000);	/* sleep for 1ms */
1622 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1623 		/* Do we have a running request on slot? */
1624 		if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING)
1625 			continue;
1626 		/* XXX; Commands in loading state. */
1627 		fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_INNOCENT);
1628 	}
1629 	for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) {
1630 		if (!ch->hold[i])
1631 			continue;
1632 		fsl_sata_done(ch, ch->hold[i]);
1633 		ch->hold[i] = NULL;
1634 		ch->numhslots--;
1635 	}
1636 	if (ch->toslots != 0)
1637 		xpt_release_simq(ch->sim, TRUE);
1638 	ch->eslots = 0;
1639 	ch->toslots = 0;
1640 	ch->fatalerr = 0;
1641 	/* Tell the XPT about the event */
1642 	xpt_async(AC_BUS_RESET, ch->path, NULL);
1643 	/* Disable port interrupts */
1644 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1645 	    ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f);
1646 	/* Reset and reconnect PHY, */
1647 	fsl_sata_start(ch);
1648 	if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 0x08, 0x08, 500)) {
1649 		if (bootverbose)
1650 			device_printf(ch->dev,
1651 			    "FSL SATA reset: device not found\n");
1652 		ch->devices = 0;
1653 		/* Enable wanted port interrupts */
1654 		ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1655 		    ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) | FSL_SATA_P_HCTRL_PHYRDY);
1656 		xpt_release_simq(ch->sim, TRUE);
1657 		return;
1658 	}
1659 	if (bootverbose)
1660 		device_printf(ch->dev, "FSL SATA reset: device found\n");
1661 	ch->devices = 1;
1662 	/* Enable wanted port interrupts */
1663 	ctrl = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f;
1664 	ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL,
1665 	    ctrl | FSL_SATA_P_HCTRL_FATAL | FSL_SATA_P_HCTRL_PHYRDY |
1666 	    FSL_SATA_P_HCTRL_SIG | FSL_SATA_P_HCTRL_SNTFY |
1667 	    FSL_SATA_P_HCTRL_DE | FSL_SATA_P_HCTRL_CC);
1668 	xpt_release_simq(ch->sim, TRUE);
1669 }
1670 
1671 static int
1672 fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag)
1673 {
1674 	uint8_t *fis = &ctp->cfis[0];
1675 
1676 	bzero(fis, 32);
1677 	fis[0] = 0x27;  		/* host to device */
1678 	fis[1] = (ccb->ccb_h.target_id & 0x0f);
1679 	if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1680 		fis[1] |= 0x80;
1681 		fis[2] = ATA_PACKET_CMD;
1682 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1683 		    ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA)
1684 			fis[3] = ATA_F_DMA;
1685 		else {
1686 			fis[5] = ccb->csio.dxfer_len;
1687 			fis[6] = ccb->csio.dxfer_len >> 8;
1688 		}
1689 		fis[7] = ATA_D_LBA;
1690 		fis[15] = ATA_A_4BIT;
1691 		bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1692 		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes,
1693 		    ctp->acmd, ccb->csio.cdb_len);
1694 		bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len);
1695 	} else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) {
1696 		fis[1] |= 0x80;
1697 		fis[2] = ccb->ataio.cmd.command;
1698 		fis[3] = ccb->ataio.cmd.features;
1699 		fis[4] = ccb->ataio.cmd.lba_low;
1700 		fis[5] = ccb->ataio.cmd.lba_mid;
1701 		fis[6] = ccb->ataio.cmd.lba_high;
1702 		fis[7] = ccb->ataio.cmd.device;
1703 		fis[8] = ccb->ataio.cmd.lba_low_exp;
1704 		fis[9] = ccb->ataio.cmd.lba_mid_exp;
1705 		fis[10] = ccb->ataio.cmd.lba_high_exp;
1706 		fis[11] = ccb->ataio.cmd.features_exp;
1707 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1708 			fis[12] = tag << 3;
1709 			fis[13] = 0;
1710 		} else {
1711 			fis[12] = ccb->ataio.cmd.sector_count;
1712 			fis[13] = ccb->ataio.cmd.sector_count_exp;
1713 		}
1714 		fis[15] = ATA_A_4BIT;
1715 	} else {
1716 		fis[15] = ccb->ataio.cmd.control;
1717 	}
1718 	return (20);
1719 }
1720 
1721 static int
1722 fsl_sata_check_ids(struct fsl_sata_channel *ch, union ccb *ccb)
1723 {
1724 
1725 	if (ccb->ccb_h.target_id > 15) {
1726 		ccb->ccb_h.status = CAM_TID_INVALID;
1727 		fsl_sata_done(ch, ccb);
1728 		return (-1);
1729 	}
1730 	if (ccb->ccb_h.target_lun != 0) {
1731 		ccb->ccb_h.status = CAM_LUN_INVALID;
1732 		fsl_sata_done(ch, ccb);
1733 		return (-1);
1734 	}
1735 	return (0);
1736 }
1737 
1738 static void
1739 fsl_sataaction(struct cam_sim *sim, union ccb *ccb)
1740 {
1741 	struct fsl_sata_channel *ch;
1742 
1743 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
1744 	    ("fsl_sataaction func_code=0x%x\n", ccb->ccb_h.func_code));
1745 
1746 	ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
1747 	switch (ccb->ccb_h.func_code) {
1748 	/* Common cases first */
1749 	case XPT_ATA_IO:	/* Execute the requested I/O operation */
1750 	case XPT_SCSI_IO:
1751 		if (fsl_sata_check_ids(ch, ccb))
1752 			return;
1753 		if (ch->devices == 0 ||
1754 		    (ch->pm_present == 0 &&
1755 		     ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
1756 			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
1757 			break;
1758 		}
1759 		ccb->ccb_h.recovery_type = RECOVERY_NONE;
1760 		/* Check for command collision. */
1761 		if (fsl_sata_check_collision(ch, ccb)) {
1762 			/* Freeze command. */
1763 			ch->frozen = ccb;
1764 			/* We have only one frozen slot, so freeze simq also. */
1765 			xpt_freeze_simq(ch->sim, 1);
1766 			return;
1767 		}
1768 		fsl_sata_begin_transaction(ch, ccb);
1769 		return;
1770 	case XPT_ABORT:			/* Abort the specified CCB */
1771 		/* XXX Implement */
1772 		ccb->ccb_h.status = CAM_REQ_INVALID;
1773 		break;
1774 	case XPT_SET_TRAN_SETTINGS:
1775 	{
1776 		struct	ccb_trans_settings *cts = &ccb->cts;
1777 		struct	fsl_sata_device *d;
1778 
1779 		if (fsl_sata_check_ids(ch, ccb))
1780 			return;
1781 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
1782 			d = &ch->curr[ccb->ccb_h.target_id];
1783 		else
1784 			d = &ch->user[ccb->ccb_h.target_id];
1785 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
1786 			d->revision = cts->xport_specific.sata.revision;
1787 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
1788 			d->mode = cts->xport_specific.sata.mode;
1789 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
1790 			d->bytecount = min(8192, cts->xport_specific.sata.bytecount);
1791 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
1792 			d->tags = min(FSL_SATA_MAX_SLOTS, cts->xport_specific.sata.tags);
1793 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
1794 			ch->pm_present = cts->xport_specific.sata.pm_present;
1795 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
1796 			d->atapi = cts->xport_specific.sata.atapi;
1797 		ccb->ccb_h.status = CAM_REQ_CMP;
1798 		break;
1799 	}
1800 	case XPT_GET_TRAN_SETTINGS:
1801 	/* Get default/user set transfer settings for the target */
1802 	{
1803 		struct	ccb_trans_settings *cts = &ccb->cts;
1804 		struct  fsl_sata_device *d;
1805 		uint32_t status;
1806 
1807 		if (fsl_sata_check_ids(ch, ccb))
1808 			return;
1809 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
1810 			d = &ch->curr[ccb->ccb_h.target_id];
1811 		else
1812 			d = &ch->user[ccb->ccb_h.target_id];
1813 		cts->protocol = PROTO_UNSPECIFIED;
1814 		cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
1815 		cts->transport = XPORT_SATA;
1816 		cts->transport_version = XPORT_VERSION_UNSPECIFIED;
1817 		cts->proto_specific.valid = 0;
1818 		cts->xport_specific.sata.valid = 0;
1819 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
1820 		    (ccb->ccb_h.target_id == 15 ||
1821 		    (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
1822 			status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS) & ATA_SS_SPD_MASK;
1823 			if (status & 0x0f0) {
1824 				cts->xport_specific.sata.revision =
1825 				    (status & 0x0f0) >> 4;
1826 				cts->xport_specific.sata.valid |=
1827 				    CTS_SATA_VALID_REVISION;
1828 			}
1829 			cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
1830 			if (ch->pm_level) {
1831 				cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
1832 			}
1833 			cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN;
1834 			cts->xport_specific.sata.caps &=
1835 			    ch->user[ccb->ccb_h.target_id].caps;
1836 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
1837 		} else {
1838 			cts->xport_specific.sata.revision = d->revision;
1839 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
1840 			cts->xport_specific.sata.caps = d->caps;
1841 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
1842 		}
1843 		cts->xport_specific.sata.mode = d->mode;
1844 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
1845 		cts->xport_specific.sata.bytecount = d->bytecount;
1846 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
1847 		cts->xport_specific.sata.pm_present = ch->pm_present;
1848 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
1849 		cts->xport_specific.sata.tags = d->tags;
1850 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
1851 		cts->xport_specific.sata.atapi = d->atapi;
1852 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
1853 		ccb->ccb_h.status = CAM_REQ_CMP;
1854 		break;
1855 	}
1856 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
1857 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
1858 		fsl_sata_reset(ch);
1859 		ccb->ccb_h.status = CAM_REQ_CMP;
1860 		break;
1861 	case XPT_TERM_IO:		/* Terminate the I/O process */
1862 		/* XXX Implement */
1863 		ccb->ccb_h.status = CAM_REQ_INVALID;
1864 		break;
1865 	case XPT_PATH_INQ:		/* Path routing inquiry */
1866 	{
1867 		struct ccb_pathinq *cpi = &ccb->cpi;
1868 
1869 		cpi->version_num = 1; /* XXX??? */
1870 		cpi->hba_inquiry = PI_SDTR_ABLE;
1871 		cpi->hba_inquiry |= PI_TAG_ABLE;
1872 #if 0
1873 		/*
1874 		 * XXX: CAM tries to reset port 15 if it sees port multiplier
1875 		 * support.  Disable it for now.
1876 		 */
1877 		cpi->hba_inquiry |= PI_SATAPM;
1878 #endif
1879 		cpi->target_sprt = 0;
1880 		cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
1881 		cpi->hba_eng_cnt = 0;
1882 		/*
1883 		 * XXX: This should be 15, since hardware *does* support a port
1884 		 * multiplier.  See above.
1885 		 */
1886 		cpi->max_target = 0;
1887 		cpi->max_lun = 0;
1888 		cpi->initiator_id = 0;
1889 		cpi->bus_id = cam_sim_bus(sim);
1890 		cpi->base_transfer_speed = 150000;
1891 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1892 		strncpy(cpi->hba_vid, "FSL SATA", HBA_IDLEN);
1893 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1894 		cpi->unit_number = cam_sim_unit(sim);
1895 		cpi->transport = XPORT_SATA;
1896 		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
1897 		cpi->protocol = PROTO_ATA;
1898 		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
1899 		cpi->maxio = MAXPHYS;
1900 		cpi->ccb_h.status = CAM_REQ_CMP;
1901 		break;
1902 	}
1903 	default:
1904 		ccb->ccb_h.status = CAM_REQ_INVALID;
1905 		break;
1906 	}
1907 	fsl_sata_done(ch, ccb);
1908 }
1909 
1910 static void
1911 fsl_satapoll(struct cam_sim *sim)
1912 {
1913 	struct fsl_sata_channel *ch = (struct fsl_sata_channel *)cam_sim_softc(sim);
1914 	uint32_t istatus;
1915 
1916 	/* Read interrupt statuses and process if any. */
1917 	istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS);
1918 	if (istatus != 0)
1919 		fsl_sata_intr_main(ch, istatus);
1920 }
1921 MODULE_VERSION(fsl_sata, 1);
1922 MODULE_DEPEND(fsl_sata, cam, 1, 1, 1);
1923