xref: /freebsd/sys/dev/mvs/mvs.c (revision 39ee7a7a6bdd1557b1c3532abf60d139798ac88b)
1 /*-
2  * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/module.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/ata.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/endian.h>
38 #include <sys/malloc.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <vm/uma.h>
42 #include <machine/stdarg.h>
43 #include <machine/resource.h>
44 #include <machine/bus.h>
45 #include <sys/rman.h>
46 #include <dev/pci/pcivar.h>
47 #include "mvs.h"
48 
49 #include <cam/cam.h>
50 #include <cam/cam_ccb.h>
51 #include <cam/cam_sim.h>
52 #include <cam/cam_xpt_sim.h>
53 #include <cam/cam_debug.h>
54 
55 /* local prototypes */
56 static int mvs_ch_init(device_t dev);
57 static int mvs_ch_deinit(device_t dev);
58 static int mvs_ch_suspend(device_t dev);
59 static int mvs_ch_resume(device_t dev);
60 static void mvs_dmainit(device_t dev);
61 static void mvs_dmasetupc_cb(void *xsc,
62 	bus_dma_segment_t *segs, int nsegs, int error);
63 static void mvs_dmafini(device_t dev);
64 static void mvs_slotsalloc(device_t dev);
65 static void mvs_slotsfree(device_t dev);
66 static void mvs_setup_edma_queues(device_t dev);
67 static void mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode);
68 static void mvs_ch_pm(void *arg);
69 static void mvs_ch_intr_locked(void *data);
70 static void mvs_ch_intr(void *data);
71 static void mvs_reset(device_t dev);
72 static void mvs_softreset(device_t dev, union ccb *ccb);
73 
74 static int mvs_sata_connect(struct mvs_channel *ch);
75 static int mvs_sata_phy_reset(device_t dev);
76 static int mvs_wait(device_t dev, u_int s, u_int c, int t);
77 static void mvs_tfd_read(device_t dev, union ccb *ccb);
78 static void mvs_tfd_write(device_t dev, union ccb *ccb);
79 static void mvs_legacy_intr(device_t dev, int poll);
80 static void mvs_crbq_intr(device_t dev);
81 static void mvs_begin_transaction(device_t dev, union ccb *ccb);
82 static void mvs_legacy_execute_transaction(struct mvs_slot *slot);
83 static void mvs_timeout(struct mvs_slot *slot);
84 static void mvs_dmasetprd(void *arg,
85 	bus_dma_segment_t *segs, int nsegs, int error);
86 static void mvs_requeue_frozen(device_t dev);
87 static void mvs_execute_transaction(struct mvs_slot *slot);
88 static void mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et);
89 
90 static void mvs_issue_recovery(device_t dev);
91 static void mvs_process_read_log(device_t dev, union ccb *ccb);
92 static void mvs_process_request_sense(device_t dev, union ccb *ccb);
93 
94 static void mvsaction(struct cam_sim *sim, union ccb *ccb);
95 static void mvspoll(struct cam_sim *sim);
96 
97 static MALLOC_DEFINE(M_MVS, "MVS driver", "MVS driver data buffers");
98 
99 #define recovery_type		spriv_field0
100 #define RECOVERY_NONE		0
101 #define RECOVERY_READ_LOG	1
102 #define RECOVERY_REQUEST_SENSE	2
103 #define recovery_slot		spriv_field1
104 
105 static int
106 mvs_ch_probe(device_t dev)
107 {
108 
109 	device_set_desc_copy(dev, "Marvell SATA channel");
110 	return (BUS_PROBE_DEFAULT);
111 }
112 
113 static int
114 mvs_ch_attach(device_t dev)
115 {
116 	struct mvs_controller *ctlr = device_get_softc(device_get_parent(dev));
117 	struct mvs_channel *ch = device_get_softc(dev);
118 	struct cam_devq *devq;
119 	int rid, error, i, sata_rev = 0;
120 
121 	ch->dev = dev;
122 	ch->unit = (intptr_t)device_get_ivars(dev);
123 	ch->quirks = ctlr->quirks;
124 	mtx_init(&ch->mtx, "MVS channel lock", NULL, MTX_DEF);
125 	ch->pm_level = 0;
126 	resource_int_value(device_get_name(dev),
127 	    device_get_unit(dev), "pm_level", &ch->pm_level);
128 	if (ch->pm_level > 3)
129 		callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
130 	callout_init_mtx(&ch->reset_timer, &ch->mtx, 0);
131 	resource_int_value(device_get_name(dev),
132 	    device_get_unit(dev), "sata_rev", &sata_rev);
133 	for (i = 0; i < 16; i++) {
134 		ch->user[i].revision = sata_rev;
135 		ch->user[i].mode = 0;
136 		ch->user[i].bytecount = (ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048;
137 		ch->user[i].tags = MVS_MAX_SLOTS;
138 		ch->curr[i] = ch->user[i];
139 		if (ch->pm_level) {
140 			ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
141 			    CTS_SATA_CAPS_H_APST |
142 			    CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST;
143 		}
144 		ch->user[i].caps |= CTS_SATA_CAPS_H_AN;
145 	}
146 	rid = ch->unit;
147 	if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
148 	    &rid, RF_ACTIVE)))
149 		return (ENXIO);
150 	mvs_dmainit(dev);
151 	mvs_slotsalloc(dev);
152 	mvs_ch_init(dev);
153 	mtx_lock(&ch->mtx);
154 	rid = ATA_IRQ_RID;
155 	if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
156 	    &rid, RF_SHAREABLE | RF_ACTIVE))) {
157 		device_printf(dev, "Unable to map interrupt\n");
158 		error = ENXIO;
159 		goto err0;
160 	}
161 	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
162 	    mvs_ch_intr_locked, dev, &ch->ih))) {
163 		device_printf(dev, "Unable to setup interrupt\n");
164 		error = ENXIO;
165 		goto err1;
166 	}
167 	/* Create the device queue for our SIM. */
168 	devq = cam_simq_alloc(MVS_MAX_SLOTS - 1);
169 	if (devq == NULL) {
170 		device_printf(dev, "Unable to allocate simq\n");
171 		error = ENOMEM;
172 		goto err1;
173 	}
174 	/* Construct SIM entry */
175 	ch->sim = cam_sim_alloc(mvsaction, mvspoll, "mvsch", ch,
176 	    device_get_unit(dev), &ch->mtx,
177 	    2, (ch->quirks & MVS_Q_GENI) ? 0 : MVS_MAX_SLOTS - 1,
178 	    devq);
179 	if (ch->sim == NULL) {
180 		cam_simq_free(devq);
181 		device_printf(dev, "unable to allocate sim\n");
182 		error = ENOMEM;
183 		goto err1;
184 	}
185 	if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
186 		device_printf(dev, "unable to register xpt bus\n");
187 		error = ENXIO;
188 		goto err2;
189 	}
190 	if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
191 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
192 		device_printf(dev, "unable to create path\n");
193 		error = ENXIO;
194 		goto err3;
195 	}
196 	if (ch->pm_level > 3) {
197 		callout_reset(&ch->pm_timer,
198 		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
199 		    mvs_ch_pm, dev);
200 	}
201 	mtx_unlock(&ch->mtx);
202 	return (0);
203 
204 err3:
205 	xpt_bus_deregister(cam_sim_path(ch->sim));
206 err2:
207 	cam_sim_free(ch->sim, /*free_devq*/TRUE);
208 err1:
209 	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
210 err0:
211 	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
212 	mtx_unlock(&ch->mtx);
213 	mtx_destroy(&ch->mtx);
214 	return (error);
215 }
216 
217 static int
218 mvs_ch_detach(device_t dev)
219 {
220 	struct mvs_channel *ch = device_get_softc(dev);
221 
222 	mtx_lock(&ch->mtx);
223 	xpt_async(AC_LOST_DEVICE, ch->path, NULL);
224 	/* Forget about reset. */
225 	if (ch->resetting) {
226 		ch->resetting = 0;
227 		xpt_release_simq(ch->sim, TRUE);
228 	}
229 	xpt_free_path(ch->path);
230 	xpt_bus_deregister(cam_sim_path(ch->sim));
231 	cam_sim_free(ch->sim, /*free_devq*/TRUE);
232 	mtx_unlock(&ch->mtx);
233 
234 	if (ch->pm_level > 3)
235 		callout_drain(&ch->pm_timer);
236 	callout_drain(&ch->reset_timer);
237 	bus_teardown_intr(dev, ch->r_irq, ch->ih);
238 	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
239 
240 	mvs_ch_deinit(dev);
241 	mvs_slotsfree(dev);
242 	mvs_dmafini(dev);
243 
244 	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
245 	mtx_destroy(&ch->mtx);
246 	return (0);
247 }
248 
249 static int
250 mvs_ch_init(device_t dev)
251 {
252 	struct mvs_channel *ch = device_get_softc(dev);
253 	uint32_t reg;
254 
255 	/* Disable port interrupts */
256 	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
257 	/* Stop EDMA */
258 	ch->curr_mode = MVS_EDMA_UNKNOWN;
259 	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
260 	/* Clear and configure FIS interrupts. */
261 	ATA_OUTL(ch->r_mem, SATA_FISIC, 0);
262 	reg = ATA_INL(ch->r_mem, SATA_FISC);
263 	reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
264 	ATA_OUTL(ch->r_mem, SATA_FISC, reg);
265 	reg = ATA_INL(ch->r_mem, SATA_FISIM);
266 	reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
267 	ATA_OUTL(ch->r_mem, SATA_FISC, reg);
268 	/* Clear SATA error register. */
269 	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
270 	/* Clear any outstanding error interrupts. */
271 	ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
272 	/* Unmask all error interrupts */
273 	ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
274 	return (0);
275 }
276 
277 static int
278 mvs_ch_deinit(device_t dev)
279 {
280 	struct mvs_channel *ch = device_get_softc(dev);
281 
282 	/* Stop EDMA */
283 	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
284 	/* Disable port interrupts. */
285 	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
286 	return (0);
287 }
288 
289 static int
290 mvs_ch_suspend(device_t dev)
291 {
292 	struct mvs_channel *ch = device_get_softc(dev);
293 
294 	mtx_lock(&ch->mtx);
295 	xpt_freeze_simq(ch->sim, 1);
296 	while (ch->oslots)
297 		msleep(ch, &ch->mtx, PRIBIO, "mvssusp", hz/100);
298 	/* Forget about reset. */
299 	if (ch->resetting) {
300 		ch->resetting = 0;
301 		callout_stop(&ch->reset_timer);
302 		xpt_release_simq(ch->sim, TRUE);
303 	}
304 	mvs_ch_deinit(dev);
305 	mtx_unlock(&ch->mtx);
306 	return (0);
307 }
308 
309 static int
310 mvs_ch_resume(device_t dev)
311 {
312 	struct mvs_channel *ch = device_get_softc(dev);
313 
314 	mtx_lock(&ch->mtx);
315 	mvs_ch_init(dev);
316 	mvs_reset(dev);
317 	xpt_release_simq(ch->sim, TRUE);
318 	mtx_unlock(&ch->mtx);
319 	return (0);
320 }
321 
322 struct mvs_dc_cb_args {
323 	bus_addr_t maddr;
324 	int error;
325 };
326 
327 static void
328 mvs_dmainit(device_t dev)
329 {
330 	struct mvs_channel *ch = device_get_softc(dev);
331 	struct mvs_dc_cb_args dcba;
332 
333 	/* EDMA command request area. */
334 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
335 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
336 	    NULL, NULL, MVS_WORKRQ_SIZE, 1, MVS_WORKRQ_SIZE,
337 	    0, NULL, NULL, &ch->dma.workrq_tag))
338 		goto error;
339 	if (bus_dmamem_alloc(ch->dma.workrq_tag, (void **)&ch->dma.workrq, 0,
340 	    &ch->dma.workrq_map))
341 		goto error;
342 	if (bus_dmamap_load(ch->dma.workrq_tag, ch->dma.workrq_map,
343 	    ch->dma.workrq, MVS_WORKRQ_SIZE, mvs_dmasetupc_cb, &dcba, 0) ||
344 	    dcba.error) {
345 		bus_dmamem_free(ch->dma.workrq_tag,
346 		    ch->dma.workrq, ch->dma.workrq_map);
347 		goto error;
348 	}
349 	ch->dma.workrq_bus = dcba.maddr;
350 	/* EDMA command response area. */
351 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 256, 0,
352 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
353 	    NULL, NULL, MVS_WORKRP_SIZE, 1, MVS_WORKRP_SIZE,
354 	    0, NULL, NULL, &ch->dma.workrp_tag))
355 		goto error;
356 	if (bus_dmamem_alloc(ch->dma.workrp_tag, (void **)&ch->dma.workrp, 0,
357 	    &ch->dma.workrp_map))
358 		goto error;
359 	if (bus_dmamap_load(ch->dma.workrp_tag, ch->dma.workrp_map,
360 	    ch->dma.workrp, MVS_WORKRP_SIZE, mvs_dmasetupc_cb, &dcba, 0) ||
361 	    dcba.error) {
362 		bus_dmamem_free(ch->dma.workrp_tag,
363 		    ch->dma.workrp, ch->dma.workrp_map);
364 		goto error;
365 	}
366 	ch->dma.workrp_bus = dcba.maddr;
367 	/* Data area. */
368 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, MVS_EPRD_MAX,
369 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
370 	    NULL, NULL,
371 	    MVS_SG_ENTRIES * PAGE_SIZE * MVS_MAX_SLOTS,
372 	    MVS_SG_ENTRIES, MVS_EPRD_MAX,
373 	    0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
374 		goto error;
375 	}
376 	return;
377 
378 error:
379 	device_printf(dev, "WARNING - DMA initialization failed\n");
380 	mvs_dmafini(dev);
381 }
382 
383 static void
384 mvs_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
385 {
386 	struct mvs_dc_cb_args *dcba = (struct mvs_dc_cb_args *)xsc;
387 
388 	if (!(dcba->error = error))
389 		dcba->maddr = segs[0].ds_addr;
390 }
391 
392 static void
393 mvs_dmafini(device_t dev)
394 {
395 	struct mvs_channel *ch = device_get_softc(dev);
396 
397 	if (ch->dma.data_tag) {
398 		bus_dma_tag_destroy(ch->dma.data_tag);
399 		ch->dma.data_tag = NULL;
400 	}
401 	if (ch->dma.workrp_bus) {
402 		bus_dmamap_unload(ch->dma.workrp_tag, ch->dma.workrp_map);
403 		bus_dmamem_free(ch->dma.workrp_tag,
404 		    ch->dma.workrp, ch->dma.workrp_map);
405 		ch->dma.workrp_bus = 0;
406 		ch->dma.workrp = NULL;
407 	}
408 	if (ch->dma.workrp_tag) {
409 		bus_dma_tag_destroy(ch->dma.workrp_tag);
410 		ch->dma.workrp_tag = NULL;
411 	}
412 	if (ch->dma.workrq_bus) {
413 		bus_dmamap_unload(ch->dma.workrq_tag, ch->dma.workrq_map);
414 		bus_dmamem_free(ch->dma.workrq_tag,
415 		    ch->dma.workrq, ch->dma.workrq_map);
416 		ch->dma.workrq_bus = 0;
417 		ch->dma.workrq = NULL;
418 	}
419 	if (ch->dma.workrq_tag) {
420 		bus_dma_tag_destroy(ch->dma.workrq_tag);
421 		ch->dma.workrq_tag = NULL;
422 	}
423 }
424 
425 static void
426 mvs_slotsalloc(device_t dev)
427 {
428 	struct mvs_channel *ch = device_get_softc(dev);
429 	int i;
430 
431 	/* Alloc and setup command/dma slots */
432 	bzero(ch->slot, sizeof(ch->slot));
433 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
434 		struct mvs_slot *slot = &ch->slot[i];
435 
436 		slot->dev = dev;
437 		slot->slot = i;
438 		slot->state = MVS_SLOT_EMPTY;
439 		slot->ccb = NULL;
440 		callout_init_mtx(&slot->timeout, &ch->mtx, 0);
441 
442 		if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
443 			device_printf(ch->dev, "FAILURE - create data_map\n");
444 	}
445 }
446 
447 static void
448 mvs_slotsfree(device_t dev)
449 {
450 	struct mvs_channel *ch = device_get_softc(dev);
451 	int i;
452 
453 	/* Free all dma slots */
454 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
455 		struct mvs_slot *slot = &ch->slot[i];
456 
457 		callout_drain(&slot->timeout);
458 		if (slot->dma.data_map) {
459 			bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
460 			slot->dma.data_map = NULL;
461 		}
462 	}
463 }
464 
465 static void
466 mvs_setup_edma_queues(device_t dev)
467 {
468 	struct mvs_channel *ch = device_get_softc(dev);
469 	uint64_t work;
470 
471 	/* Requests queue. */
472 	work = ch->dma.workrq_bus;
473 	ATA_OUTL(ch->r_mem, EDMA_REQQBAH, work >> 32);
474 	ATA_OUTL(ch->r_mem, EDMA_REQQIP, work & 0xffffffff);
475 	ATA_OUTL(ch->r_mem, EDMA_REQQOP, work & 0xffffffff);
476 	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
477 	    BUS_DMASYNC_PREWRITE);
478 	/* Reponses queue. */
479 	memset(ch->dma.workrp, 0xff, MVS_WORKRP_SIZE);
480 	work = ch->dma.workrp_bus;
481 	ATA_OUTL(ch->r_mem, EDMA_RESQBAH, work >> 32);
482 	ATA_OUTL(ch->r_mem, EDMA_RESQIP, work & 0xffffffff);
483 	ATA_OUTL(ch->r_mem, EDMA_RESQOP, work & 0xffffffff);
484 	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
485 	    BUS_DMASYNC_PREREAD);
486 	ch->out_idx = 0;
487 	ch->in_idx = 0;
488 }
489 
490 static void
491 mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode)
492 {
493 	struct mvs_channel *ch = device_get_softc(dev);
494 	int timeout;
495 	uint32_t ecfg, fcfg, hc, ltm, unkn;
496 
497 	if (mode == ch->curr_mode)
498 		return;
499 	/* If we are running, we should stop first. */
500 	if (ch->curr_mode != MVS_EDMA_OFF) {
501 		ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EDSEDMA);
502 		timeout = 0;
503 		while (ATA_INL(ch->r_mem, EDMA_CMD) & EDMA_CMD_EENEDMA) {
504 			DELAY(1000);
505 			if (timeout++ > 1000) {
506 				device_printf(dev, "stopping EDMA engine failed\n");
507 				break;
508 			}
509 		};
510 	}
511 	ch->curr_mode = mode;
512 	ch->fbs_enabled = 0;
513 	ch->fake_busy = 0;
514 	/* Report mode to controller. Needed for correct CCC operation. */
515 	MVS_EDMA(device_get_parent(dev), dev, mode);
516 	/* Configure new mode. */
517 	ecfg = EDMA_CFG_RESERVED | EDMA_CFG_RESERVED2 | EDMA_CFG_EHOSTQUEUECACHEEN;
518 	if (ch->pm_present) {
519 		ecfg |= EDMA_CFG_EMASKRXPM;
520 		if (ch->quirks & MVS_Q_GENIIE) {
521 			ecfg |= EDMA_CFG_EEDMAFBS;
522 			ch->fbs_enabled = 1;
523 		}
524 	}
525 	if (ch->quirks & MVS_Q_GENI)
526 		ecfg |= EDMA_CFG_ERDBSZ;
527 	else if (ch->quirks & MVS_Q_GENII)
528 		ecfg |= EDMA_CFG_ERDBSZEXT | EDMA_CFG_EWRBUFFERLEN;
529 	if (ch->quirks & MVS_Q_CT)
530 		ecfg |= EDMA_CFG_ECUTTHROUGHEN;
531 	if (mode != MVS_EDMA_OFF)
532 		ecfg |= EDMA_CFG_EEARLYCOMPLETIONEN;
533 	if (mode == MVS_EDMA_QUEUED)
534 		ecfg |= EDMA_CFG_EQUE;
535 	else if (mode == MVS_EDMA_NCQ)
536 		ecfg |= EDMA_CFG_ESATANATVCMDQUE;
537 	ATA_OUTL(ch->r_mem, EDMA_CFG, ecfg);
538 	mvs_setup_edma_queues(dev);
539 	if (ch->quirks & MVS_Q_GENIIE) {
540 		/* Configure FBS-related registers */
541 		fcfg = ATA_INL(ch->r_mem, SATA_FISC);
542 		ltm = ATA_INL(ch->r_mem, SATA_LTM);
543 		hc = ATA_INL(ch->r_mem, EDMA_HC);
544 		if (ch->fbs_enabled) {
545 			fcfg |= SATA_FISC_FISDMAACTIVATESYNCRESP;
546 			if (mode == MVS_EDMA_NCQ) {
547 				fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
548 				hc &= ~EDMA_IE_EDEVERR;
549 			} else {
550 				fcfg |= SATA_FISC_FISWAIT4HOSTRDYEN_B0;
551 				hc |= EDMA_IE_EDEVERR;
552 			}
553 			ltm |= (1 << 8);
554 		} else {
555 			fcfg &= ~SATA_FISC_FISDMAACTIVATESYNCRESP;
556 			fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
557 			hc |= EDMA_IE_EDEVERR;
558 			ltm &= ~(1 << 8);
559 		}
560 		ATA_OUTL(ch->r_mem, SATA_FISC, fcfg);
561 		ATA_OUTL(ch->r_mem, SATA_LTM, ltm);
562 		ATA_OUTL(ch->r_mem, EDMA_HC, hc);
563 		/* This is some magic, required to handle several DRQs
564 		 * with basic DMA. */
565 		unkn = ATA_INL(ch->r_mem, EDMA_UNKN_RESD);
566 		if (mode == MVS_EDMA_OFF)
567 			unkn |= 1;
568 		else
569 			unkn &= ~1;
570 		ATA_OUTL(ch->r_mem, EDMA_UNKN_RESD, unkn);
571 	}
572 	/* Run EDMA. */
573 	if (mode != MVS_EDMA_OFF)
574 		ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EENEDMA);
575 }
576 
577 devclass_t mvs_devclass;
578 devclass_t mvsch_devclass;
579 static device_method_t mvsch_methods[] = {
580 	DEVMETHOD(device_probe,     mvs_ch_probe),
581 	DEVMETHOD(device_attach,    mvs_ch_attach),
582 	DEVMETHOD(device_detach,    mvs_ch_detach),
583 	DEVMETHOD(device_suspend,   mvs_ch_suspend),
584 	DEVMETHOD(device_resume,    mvs_ch_resume),
585 	{ 0, 0 }
586 };
587 static driver_t mvsch_driver = {
588         "mvsch",
589         mvsch_methods,
590         sizeof(struct mvs_channel)
591 };
592 DRIVER_MODULE(mvsch, mvs, mvsch_driver, mvsch_devclass, 0, 0);
593 DRIVER_MODULE(mvsch, sata, mvsch_driver, mvsch_devclass, 0, 0);
594 
595 static void
596 mvs_phy_check_events(device_t dev, u_int32_t serr)
597 {
598 	struct mvs_channel *ch = device_get_softc(dev);
599 
600 	if (ch->pm_level == 0) {
601 		u_int32_t status = ATA_INL(ch->r_mem, SATA_SS);
602 		union ccb *ccb;
603 
604 		if (bootverbose) {
605 			if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
606 			    ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
607 			    ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE)) {
608 				device_printf(dev, "CONNECT requested\n");
609 			} else
610 				device_printf(dev, "DISCONNECT requested\n");
611 		}
612 		mvs_reset(dev);
613 		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
614 			return;
615 		if (xpt_create_path(&ccb->ccb_h.path, NULL,
616 		    cam_sim_path(ch->sim),
617 		    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
618 			xpt_free_ccb(ccb);
619 			return;
620 		}
621 		xpt_rescan(ccb);
622 	}
623 }
624 
625 static void
626 mvs_notify_events(device_t dev)
627 {
628 	struct mvs_channel *ch = device_get_softc(dev);
629 	struct cam_path *dpath;
630 	uint32_t fis;
631 	int d;
632 
633 	/* Try to read PMP field from SDB FIS. Present only for Gen-IIe. */
634 	fis = ATA_INL(ch->r_mem, SATA_FISDW0);
635 	if ((fis & 0x80ff) == 0x80a1)
636 		d = (fis & 0x0f00) >> 8;
637 	else
638 		d = ch->pm_present ? 15 : 0;
639 	if (bootverbose)
640 		device_printf(dev, "SNTF %d\n", d);
641 	if (xpt_create_path(&dpath, NULL,
642 	    xpt_path_path_id(ch->path), d, 0) == CAM_REQ_CMP) {
643 		xpt_async(AC_SCSI_AEN, dpath, NULL);
644 		xpt_free_path(dpath);
645 	}
646 }
647 
648 static void
649 mvs_ch_intr_locked(void *data)
650 {
651 	struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
652 	device_t dev = (device_t)arg->arg;
653 	struct mvs_channel *ch = device_get_softc(dev);
654 
655 	mtx_lock(&ch->mtx);
656 	mvs_ch_intr(data);
657 	mtx_unlock(&ch->mtx);
658 }
659 
660 static void
661 mvs_ch_pm(void *arg)
662 {
663 	device_t dev = (device_t)arg;
664 	struct mvs_channel *ch = device_get_softc(dev);
665 	uint32_t work;
666 
667 	if (ch->numrslots != 0)
668 		return;
669 	/* If we are idle - request power state transition. */
670 	work = ATA_INL(ch->r_mem, SATA_SC);
671 	work &= ~SATA_SC_SPM_MASK;
672 	if (ch->pm_level == 4)
673 		work |= SATA_SC_SPM_PARTIAL;
674 	else
675 		work |= SATA_SC_SPM_SLUMBER;
676 	ATA_OUTL(ch->r_mem, SATA_SC, work);
677 }
678 
679 static void
680 mvs_ch_pm_wake(device_t dev)
681 {
682 	struct mvs_channel *ch = device_get_softc(dev);
683 	uint32_t work;
684 	int timeout = 0;
685 
686 	work = ATA_INL(ch->r_mem, SATA_SS);
687 	if (work & SATA_SS_IPM_ACTIVE)
688 		return;
689 	/* If we are not in active state - request power state transition. */
690 	work = ATA_INL(ch->r_mem, SATA_SC);
691 	work &= ~SATA_SC_SPM_MASK;
692 	work |= SATA_SC_SPM_ACTIVE;
693 	ATA_OUTL(ch->r_mem, SATA_SC, work);
694 	/* Wait for transition to happen. */
695 	while ((ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_IPM_ACTIVE) == 0 &&
696 	    timeout++ < 100) {
697 		DELAY(100);
698 	}
699 }
700 
701 static void
702 mvs_ch_intr(void *data)
703 {
704 	struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
705 	device_t dev = (device_t)arg->arg;
706 	struct mvs_channel *ch = device_get_softc(dev);
707 	uint32_t iec, serr = 0, fisic = 0;
708 	enum mvs_err_type et;
709 	int i, ccs, port = -1, selfdis = 0;
710 	int edma = (ch->numtslots != 0 || ch->numdslots != 0);
711 
712 	/* New item in response queue. */
713 	if ((arg->cause & 2) && edma)
714 		mvs_crbq_intr(dev);
715 	/* Some error or special event. */
716 	if (arg->cause & 1) {
717 		iec = ATA_INL(ch->r_mem, EDMA_IEC);
718 		if (iec & EDMA_IE_SERRINT) {
719 			serr = ATA_INL(ch->r_mem, SATA_SE);
720 			ATA_OUTL(ch->r_mem, SATA_SE, serr);
721 		}
722 		/* EDMA self-disabled due to error. */
723 		if (iec & EDMA_IE_ESELFDIS)
724 			selfdis = 1;
725 		/* Transport interrupt. */
726 		if (iec & EDMA_IE_ETRANSINT) {
727 			/* For Gen-I this bit means self-disable. */
728 			if (ch->quirks & MVS_Q_GENI)
729 				selfdis = 1;
730 			/* For Gen-II this bit means SDB-N. */
731 			else if (ch->quirks & MVS_Q_GENII)
732 				fisic = SATA_FISC_FISWAIT4HOSTRDYEN_B1;
733 			else	/* For Gen-IIe - read FIS interrupt cause. */
734 				fisic = ATA_INL(ch->r_mem, SATA_FISIC);
735 		}
736 		if (selfdis)
737 			ch->curr_mode = MVS_EDMA_UNKNOWN;
738 		ATA_OUTL(ch->r_mem, EDMA_IEC, ~iec);
739 		/* Interface errors or Device error. */
740 		if (iec & (0xfc1e9000 | EDMA_IE_EDEVERR)) {
741 			port = -1;
742 			if (ch->numpslots != 0) {
743 				ccs = 0;
744 			} else {
745 				if (ch->quirks & MVS_Q_GENIIE)
746 					ccs = EDMA_S_EIOID(ATA_INL(ch->r_mem, EDMA_S));
747 				else
748 					ccs = EDMA_S_EDEVQUETAG(ATA_INL(ch->r_mem, EDMA_S));
749 				/* Check if error is one-PMP-port-specific, */
750 				if (ch->fbs_enabled) {
751 					/* Which ports were active. */
752 					for (i = 0; i < 16; i++) {
753 						if (ch->numrslotspd[i] == 0)
754 							continue;
755 						if (port == -1)
756 							port = i;
757 						else if (port != i) {
758 							port = -2;
759 							break;
760 						}
761 					}
762 					/* If several ports were active and EDMA still enabled -
763 					 * other ports are probably unaffected and may continue.
764 					 */
765 					if (port == -2 && !selfdis) {
766 						uint16_t p = ATA_INL(ch->r_mem, SATA_SATAITC) >> 16;
767 						port = ffs(p) - 1;
768 						if (port != (fls(p) - 1))
769 							port = -2;
770 					}
771 				}
772 			}
773 			mvs_requeue_frozen(dev);
774 			for (i = 0; i < MVS_MAX_SLOTS; i++) {
775 				/* XXX: reqests in loading state. */
776 				if (((ch->rslots >> i) & 1) == 0)
777 					continue;
778 				if (port >= 0 &&
779 				    ch->slot[i].ccb->ccb_h.target_id != port)
780 					continue;
781 				if (iec & EDMA_IE_EDEVERR) { /* Device error. */
782 				    if (port != -2) {
783 					if (ch->numtslots == 0) {
784 						/* Untagged operation. */
785 						if (i == ccs)
786 							et = MVS_ERR_TFE;
787 						else
788 							et = MVS_ERR_INNOCENT;
789 					} else {
790 						/* Tagged operation. */
791 						et = MVS_ERR_NCQ;
792 					}
793 				    } else {
794 					et = MVS_ERR_TFE;
795 					ch->fatalerr = 1;
796 				    }
797 				} else if (iec & 0xfc1e9000) {
798 					if (ch->numtslots == 0 &&
799 					    i != ccs && port != -2)
800 						et = MVS_ERR_INNOCENT;
801 					else
802 						et = MVS_ERR_SATA;
803 				} else
804 					et = MVS_ERR_INVALID;
805 				mvs_end_transaction(&ch->slot[i], et);
806 			}
807 		}
808 		/* Process SDB-N. */
809 		if (fisic & SATA_FISC_FISWAIT4HOSTRDYEN_B1)
810 			mvs_notify_events(dev);
811 		if (fisic)
812 			ATA_OUTL(ch->r_mem, SATA_FISIC, ~fisic);
813 		/* Process hot-plug. */
814 		if ((iec & (EDMA_IE_EDEVDIS | EDMA_IE_EDEVCON)) ||
815 		    (serr & SATA_SE_PHY_CHANGED))
816 			mvs_phy_check_events(dev, serr);
817 	}
818 	/* Legacy mode device interrupt. */
819 	if ((arg->cause & 2) && !edma)
820 		mvs_legacy_intr(dev, arg->cause & 4);
821 }
822 
823 static uint8_t
824 mvs_getstatus(device_t dev, int clear)
825 {
826 	struct mvs_channel *ch = device_get_softc(dev);
827 	uint8_t status = ATA_INB(ch->r_mem, clear ? ATA_STATUS : ATA_ALTSTAT);
828 
829 	if (ch->fake_busy) {
830 		if (status & (ATA_S_BUSY | ATA_S_DRQ | ATA_S_ERROR))
831 			ch->fake_busy = 0;
832 		else
833 			status |= ATA_S_BUSY;
834 	}
835 	return (status);
836 }
837 
838 static void
839 mvs_legacy_intr(device_t dev, int poll)
840 {
841 	struct mvs_channel *ch = device_get_softc(dev);
842 	struct mvs_slot *slot = &ch->slot[0]; /* PIO is always in slot 0. */
843 	union ccb *ccb = slot->ccb;
844 	enum mvs_err_type et = MVS_ERR_NONE;
845 	int port;
846 	u_int length, resid, size;
847 	uint8_t buf[2];
848 	uint8_t status, ireason;
849 
850 	/* Clear interrupt and get status. */
851 	status = mvs_getstatus(dev, 1);
852 	if (slot->state < MVS_SLOT_RUNNING)
853 	    return;
854 	port = ccb->ccb_h.target_id & 0x0f;
855 	/* Wait a bit for late !BUSY status update. */
856 	if (status & ATA_S_BUSY) {
857 		if (poll)
858 			return;
859 		DELAY(100);
860 		if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY) {
861 			DELAY(1000);
862 			if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY)
863 				return;
864 		}
865 	}
866 	/* If we got an error, we are done. */
867 	if (status & ATA_S_ERROR) {
868 		et = MVS_ERR_TFE;
869 		goto end_finished;
870 	}
871 	if (ccb->ccb_h.func_code == XPT_ATA_IO) { /* ATA PIO */
872 		ccb->ataio.res.status = status;
873 		/* Are we moving data? */
874 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
875 		    /* If data read command - get them. */
876 		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
877 			if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
878 			    device_printf(dev, "timeout waiting for read DRQ\n");
879 			    et = MVS_ERR_TIMEOUT;
880 			    xpt_freeze_simq(ch->sim, 1);
881 			    ch->toslots |= (1 << slot->slot);
882 			    goto end_finished;
883 			}
884 			ATA_INSW_STRM(ch->r_mem, ATA_DATA,
885 			   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
886 			   ch->transfersize / 2);
887 		    }
888 		    /* Update how far we've gotten. */
889 		    ch->donecount += ch->transfersize;
890 		    /* Do we need more? */
891 		    if (ccb->ataio.dxfer_len > ch->donecount) {
892 			/* Set this transfer size according to HW capabilities */
893 			ch->transfersize = min(ccb->ataio.dxfer_len - ch->donecount,
894 			    ch->transfersize);
895 			/* If data write command - put them */
896 			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
897 				if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
898 				    device_printf(dev,
899 					"timeout waiting for write DRQ\n");
900 				    et = MVS_ERR_TIMEOUT;
901 				    xpt_freeze_simq(ch->sim, 1);
902 				    ch->toslots |= (1 << slot->slot);
903 				    goto end_finished;
904 				}
905 				ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
906 				   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
907 				   ch->transfersize / 2);
908 				return;
909 			}
910 			/* If data read command, return & wait for interrupt */
911 			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
912 				return;
913 		    }
914 		}
915 	} else if (ch->basic_dma) {	/* ATAPI DMA */
916 		if (status & ATA_S_DWF)
917 			et = MVS_ERR_TFE;
918 		else if (ATA_INL(ch->r_mem, DMA_S) & DMA_S_ERR)
919 			et = MVS_ERR_TFE;
920 		/* Stop basic DMA. */
921 		ATA_OUTL(ch->r_mem, DMA_C, 0);
922 		goto end_finished;
923 	} else {			/* ATAPI PIO */
924 		length = ATA_INB(ch->r_mem,ATA_CYL_LSB) |
925 		    (ATA_INB(ch->r_mem,ATA_CYL_MSB) << 8);
926 		size = min(ch->transfersize, length);
927 		ireason = ATA_INB(ch->r_mem,ATA_IREASON);
928 		switch ((ireason & (ATA_I_CMD | ATA_I_IN)) |
929 			(status & ATA_S_DRQ)) {
930 
931 		case ATAPI_P_CMDOUT:
932 		    device_printf(dev, "ATAPI CMDOUT\n");
933 		    /* Return wait for interrupt */
934 		    return;
935 
936 		case ATAPI_P_WRITE:
937 		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
938 			device_printf(dev, "trying to write on read buffer\n");
939 			et = MVS_ERR_TFE;
940 			goto end_finished;
941 			break;
942 		    }
943 		    ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
944 			(uint16_t *)(ccb->csio.data_ptr + ch->donecount),
945 			(size + 1) / 2);
946 		    for (resid = ch->transfersize + (size & 1);
947 			resid < length; resid += sizeof(int16_t))
948 			    ATA_OUTW(ch->r_mem, ATA_DATA, 0);
949 		    ch->donecount += length;
950 		    /* Set next transfer size according to HW capabilities */
951 		    ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
952 			    ch->curr[ccb->ccb_h.target_id].bytecount);
953 		    /* Return wait for interrupt */
954 		    return;
955 
956 		case ATAPI_P_READ:
957 		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
958 			device_printf(dev, "trying to read on write buffer\n");
959 			et = MVS_ERR_TFE;
960 			goto end_finished;
961 		    }
962 		    if (size >= 2) {
963 			ATA_INSW_STRM(ch->r_mem, ATA_DATA,
964 			    (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
965 			    size / 2);
966 		    }
967 		    if (size & 1) {
968 			ATA_INSW_STRM(ch->r_mem, ATA_DATA, (void*)buf, 1);
969 			((uint8_t *)ccb->csio.data_ptr + ch->donecount +
970 			    (size & ~1))[0] = buf[0];
971 		    }
972 		    for (resid = ch->transfersize + (size & 1);
973 			resid < length; resid += sizeof(int16_t))
974 			    ATA_INW(ch->r_mem, ATA_DATA);
975 		    ch->donecount += length;
976 		    /* Set next transfer size according to HW capabilities */
977 		    ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
978 			    ch->curr[ccb->ccb_h.target_id].bytecount);
979 		    /* Return wait for interrupt */
980 		    return;
981 
982 		case ATAPI_P_DONEDRQ:
983 		    device_printf(dev,
984 			  "WARNING - DONEDRQ non conformant device\n");
985 		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
986 			ATA_INSW_STRM(ch->r_mem, ATA_DATA,
987 			    (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
988 			    length / 2);
989 			ch->donecount += length;
990 		    }
991 		    else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
992 			ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
993 			    (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
994 			    length / 2);
995 			ch->donecount += length;
996 		    }
997 		    else
998 			et = MVS_ERR_TFE;
999 		    /* FALLTHROUGH */
1000 
1001 		case ATAPI_P_ABORT:
1002 		case ATAPI_P_DONE:
1003 		    if (status & (ATA_S_ERROR | ATA_S_DWF))
1004 			et = MVS_ERR_TFE;
1005 		    goto end_finished;
1006 
1007 		default:
1008 		    device_printf(dev, "unknown transfer phase"
1009 			" (status %02x, ireason %02x)\n",
1010 			status, ireason);
1011 		    et = MVS_ERR_TFE;
1012 		}
1013 	}
1014 
1015 end_finished:
1016 	mvs_end_transaction(slot, et);
1017 }
1018 
1019 static void
1020 mvs_crbq_intr(device_t dev)
1021 {
1022 	struct mvs_channel *ch = device_get_softc(dev);
1023 	struct mvs_crpb *crpb;
1024 	union ccb *ccb;
1025 	int in_idx, fin_idx, cin_idx, slot;
1026 	uint32_t val;
1027 	uint16_t flags;
1028 
1029 	val = ATA_INL(ch->r_mem, EDMA_RESQIP);
1030 	if (val == 0)
1031 		val = ATA_INL(ch->r_mem, EDMA_RESQIP);
1032 	in_idx = (val & EDMA_RESQP_ERPQP_MASK) >>
1033 	    EDMA_RESQP_ERPQP_SHIFT;
1034 	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1035 	    BUS_DMASYNC_POSTREAD);
1036 	fin_idx = cin_idx = ch->in_idx;
1037 	ch->in_idx = in_idx;
1038 	while (in_idx != cin_idx) {
1039 		crpb = (struct mvs_crpb *)
1040 		    (ch->dma.workrp + MVS_CRPB_OFFSET +
1041 		    (MVS_CRPB_SIZE * cin_idx));
1042 		slot = le16toh(crpb->id) & MVS_CRPB_TAG_MASK;
1043 		flags = le16toh(crpb->rspflg);
1044 		/*
1045 		 * Handle only successfull completions here.
1046 		 * Errors will be handled by main intr handler.
1047 		 */
1048 #if defined(__i386__) || defined(__amd64__)
1049 		if (crpb->id == 0xffff && crpb->rspflg == 0xffff) {
1050 			device_printf(dev, "Unfilled CRPB "
1051 			    "%d (%d->%d) tag %d flags %04x rs %08x\n",
1052 			    cin_idx, fin_idx, in_idx, slot, flags, ch->rslots);
1053 		} else
1054 #endif
1055 		if (ch->numtslots != 0 ||
1056 		    (flags & EDMA_IE_EDEVERR) == 0) {
1057 #if defined(__i386__) || defined(__amd64__)
1058 			crpb->id = 0xffff;
1059 			crpb->rspflg = 0xffff;
1060 #endif
1061 			if (ch->slot[slot].state >= MVS_SLOT_RUNNING) {
1062 				ccb = ch->slot[slot].ccb;
1063 				ccb->ataio.res.status =
1064 				    (flags & MVS_CRPB_ATASTS_MASK) >>
1065 				    MVS_CRPB_ATASTS_SHIFT;
1066 				mvs_end_transaction(&ch->slot[slot], MVS_ERR_NONE);
1067 			} else {
1068 				device_printf(dev, "Unused tag in CRPB "
1069 				    "%d (%d->%d) tag %d flags %04x rs %08x\n",
1070 				    cin_idx, fin_idx, in_idx, slot, flags,
1071 				    ch->rslots);
1072 			}
1073 		} else {
1074 			device_printf(dev,
1075 			    "CRPB with error %d tag %d flags %04x\n",
1076 			    cin_idx, slot, flags);
1077 		}
1078 		cin_idx = (cin_idx + 1) & (MVS_MAX_SLOTS - 1);
1079 	}
1080 	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1081 	    BUS_DMASYNC_PREREAD);
1082 	if (cin_idx == ch->in_idx) {
1083 		ATA_OUTL(ch->r_mem, EDMA_RESQOP,
1084 		    ch->dma.workrp_bus | (cin_idx << EDMA_RESQP_ERPQP_SHIFT));
1085 	}
1086 }
1087 
1088 /* Must be called with channel locked. */
1089 static int
1090 mvs_check_collision(device_t dev, union ccb *ccb)
1091 {
1092 	struct mvs_channel *ch = device_get_softc(dev);
1093 
1094 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1095 		/* NCQ DMA */
1096 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1097 			/* Can't mix NCQ and non-NCQ DMA commands. */
1098 			if (ch->numdslots != 0)
1099 				return (1);
1100 			/* Can't mix NCQ and PIO commands. */
1101 			if (ch->numpslots != 0)
1102 				return (1);
1103 			/* If we have no FBS */
1104 			if (!ch->fbs_enabled) {
1105 				/* Tagged command while tagged to other target is active. */
1106 				if (ch->numtslots != 0 &&
1107 				    ch->taggedtarget != ccb->ccb_h.target_id)
1108 					return (1);
1109 			}
1110 		/* Non-NCQ DMA */
1111 		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1112 			/* Can't mix non-NCQ DMA and NCQ commands. */
1113 			if (ch->numtslots != 0)
1114 				return (1);
1115 			/* Can't mix non-NCQ DMA and PIO commands. */
1116 			if (ch->numpslots != 0)
1117 				return (1);
1118 		/* PIO */
1119 		} else {
1120 			/* Can't mix PIO with anything. */
1121 			if (ch->numrslots != 0)
1122 				return (1);
1123 		}
1124 		if (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
1125 			/* Atomic command while anything active. */
1126 			if (ch->numrslots != 0)
1127 				return (1);
1128 		}
1129 	} else { /* ATAPI */
1130 		/* ATAPI goes without EDMA, so can't mix it with anything. */
1131 		if (ch->numrslots != 0)
1132 			return (1);
1133 	}
1134 	/* We have some atomic command running. */
1135 	if (ch->aslots != 0)
1136 		return (1);
1137 	return (0);
1138 }
1139 
1140 static void
1141 mvs_tfd_read(device_t dev, union ccb *ccb)
1142 {
1143 	struct mvs_channel *ch = device_get_softc(dev);
1144 	struct ata_res *res = &ccb->ataio.res;
1145 
1146 	res->status = ATA_INB(ch->r_mem, ATA_ALTSTAT);
1147 	res->error =  ATA_INB(ch->r_mem, ATA_ERROR);
1148 	res->device = ATA_INB(ch->r_mem, ATA_DRIVE);
1149 	ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_HOB);
1150 	res->sector_count_exp = ATA_INB(ch->r_mem, ATA_COUNT);
1151 	res->lba_low_exp = ATA_INB(ch->r_mem, ATA_SECTOR);
1152 	res->lba_mid_exp = ATA_INB(ch->r_mem, ATA_CYL_LSB);
1153 	res->lba_high_exp = ATA_INB(ch->r_mem, ATA_CYL_MSB);
1154 	ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
1155 	res->sector_count = ATA_INB(ch->r_mem, ATA_COUNT);
1156 	res->lba_low = ATA_INB(ch->r_mem, ATA_SECTOR);
1157 	res->lba_mid = ATA_INB(ch->r_mem, ATA_CYL_LSB);
1158 	res->lba_high = ATA_INB(ch->r_mem, ATA_CYL_MSB);
1159 }
1160 
1161 static void
1162 mvs_tfd_write(device_t dev, union ccb *ccb)
1163 {
1164 	struct mvs_channel *ch = device_get_softc(dev);
1165 	struct ata_cmd *cmd = &ccb->ataio.cmd;
1166 
1167 	ATA_OUTB(ch->r_mem, ATA_DRIVE, cmd->device);
1168 	ATA_OUTB(ch->r_mem, ATA_CONTROL, cmd->control);
1169 	ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features_exp);
1170 	ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features);
1171 	ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count_exp);
1172 	ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count);
1173 	ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low_exp);
1174 	ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low);
1175 	ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid_exp);
1176 	ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid);
1177 	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high_exp);
1178 	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high);
1179 	ATA_OUTB(ch->r_mem, ATA_COMMAND, cmd->command);
1180 }
1181 
1182 
1183 /* Must be called with channel locked. */
1184 static void
1185 mvs_begin_transaction(device_t dev, union ccb *ccb)
1186 {
1187 	struct mvs_channel *ch = device_get_softc(dev);
1188 	struct mvs_slot *slot;
1189 	int slotn, tag;
1190 
1191 	if (ch->pm_level > 0)
1192 		mvs_ch_pm_wake(dev);
1193 	/* Softreset is a special case. */
1194 	if (ccb->ccb_h.func_code == XPT_ATA_IO &&
1195 	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
1196 		mvs_softreset(dev, ccb);
1197 		return;
1198 	}
1199 	/* Choose empty slot. */
1200 	slotn = ffs(~ch->oslots) - 1;
1201 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1202 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1203 		if (ch->quirks & MVS_Q_GENIIE)
1204 			tag = ffs(~ch->otagspd[ccb->ccb_h.target_id]) - 1;
1205 		else
1206 			tag = slotn;
1207 	} else
1208 		tag = 0;
1209 	/* Occupy chosen slot. */
1210 	slot = &ch->slot[slotn];
1211 	slot->ccb = ccb;
1212 	slot->tag = tag;
1213 	/* Stop PM timer. */
1214 	if (ch->numrslots == 0 && ch->pm_level > 3)
1215 		callout_stop(&ch->pm_timer);
1216 	/* Update channel stats. */
1217 	ch->oslots |= (1 << slot->slot);
1218 	ch->numrslots++;
1219 	ch->numrslotspd[ccb->ccb_h.target_id]++;
1220 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1221 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1222 			ch->otagspd[ccb->ccb_h.target_id] |= (1 << slot->tag);
1223 			ch->numtslots++;
1224 			ch->numtslotspd[ccb->ccb_h.target_id]++;
1225 			ch->taggedtarget = ccb->ccb_h.target_id;
1226 			mvs_set_edma_mode(dev, MVS_EDMA_NCQ);
1227 		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1228 			ch->numdslots++;
1229 			mvs_set_edma_mode(dev, MVS_EDMA_ON);
1230 		} else {
1231 			ch->numpslots++;
1232 			mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1233 		}
1234 		if (ccb->ataio.cmd.flags &
1235 		    (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
1236 			ch->aslots |= (1 << slot->slot);
1237 		}
1238 	} else {
1239 		uint8_t *cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1240 		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
1241 		ch->numpslots++;
1242 		/* Use ATAPI DMA only for commands without under-/overruns. */
1243 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1244 		    ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA &&
1245 		    (ch->quirks & MVS_Q_SOC) == 0 &&
1246 		    (cdb[0] == 0x08 ||
1247 		     cdb[0] == 0x0a ||
1248 		     cdb[0] == 0x28 ||
1249 		     cdb[0] == 0x2a ||
1250 		     cdb[0] == 0x88 ||
1251 		     cdb[0] == 0x8a ||
1252 		     cdb[0] == 0xa8 ||
1253 		     cdb[0] == 0xaa ||
1254 		     cdb[0] == 0xbe)) {
1255 			ch->basic_dma = 1;
1256 		}
1257 		mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1258 	}
1259 	if (ch->numpslots == 0 || ch->basic_dma) {
1260 		slot->state = MVS_SLOT_LOADING;
1261 		bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map,
1262 		    ccb, mvs_dmasetprd, slot, 0);
1263 	} else
1264 		mvs_legacy_execute_transaction(slot);
1265 }
1266 
1267 /* Locked by busdma engine. */
1268 static void
1269 mvs_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1270 {
1271 	struct mvs_slot *slot = arg;
1272 	struct mvs_channel *ch = device_get_softc(slot->dev);
1273 	struct mvs_eprd *eprd;
1274 	int i;
1275 
1276 	if (error) {
1277 		device_printf(slot->dev, "DMA load error\n");
1278 		mvs_end_transaction(slot, MVS_ERR_INVALID);
1279 		return;
1280 	}
1281 	KASSERT(nsegs <= MVS_SG_ENTRIES, ("too many DMA segment entries\n"));
1282 	/* If there is only one segment - no need to use S/G table on Gen-IIe. */
1283 	if (nsegs == 1 && ch->basic_dma == 0 && (ch->quirks & MVS_Q_GENIIE)) {
1284 		slot->dma.addr = segs[0].ds_addr;
1285 		slot->dma.len = segs[0].ds_len;
1286 	} else {
1287 		slot->dma.addr = 0;
1288 		/* Get a piece of the workspace for this EPRD */
1289 		eprd = (struct mvs_eprd *)
1290 		    (ch->dma.workrq + MVS_EPRD_OFFSET + (MVS_EPRD_SIZE * slot->slot));
1291 		/* Fill S/G table */
1292 		for (i = 0; i < nsegs; i++) {
1293 			eprd[i].prdbal = htole32(segs[i].ds_addr);
1294 			eprd[i].bytecount = htole32(segs[i].ds_len & MVS_EPRD_MASK);
1295 			eprd[i].prdbah = htole32((segs[i].ds_addr >> 16) >> 16);
1296 		}
1297 		eprd[i - 1].bytecount |= htole32(MVS_EPRD_EOF);
1298 	}
1299 	bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1300 	    ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
1301 	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
1302 	if (ch->basic_dma)
1303 		mvs_legacy_execute_transaction(slot);
1304 	else
1305 		mvs_execute_transaction(slot);
1306 }
1307 
1308 static void
1309 mvs_legacy_execute_transaction(struct mvs_slot *slot)
1310 {
1311 	device_t dev = slot->dev;
1312 	struct mvs_channel *ch = device_get_softc(dev);
1313 	bus_addr_t eprd;
1314 	union ccb *ccb = slot->ccb;
1315 	int port = ccb->ccb_h.target_id & 0x0f;
1316 	int timeout;
1317 
1318 	slot->state = MVS_SLOT_RUNNING;
1319 	ch->rslots |= (1 << slot->slot);
1320 	ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
1321 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1322 		mvs_tfd_write(dev, ccb);
1323 		/* Device reset doesn't interrupt. */
1324 		if (ccb->ataio.cmd.command == ATA_DEVICE_RESET) {
1325 			int timeout = 1000000;
1326 			do {
1327 			    DELAY(10);
1328 			    ccb->ataio.res.status = ATA_INB(ch->r_mem, ATA_STATUS);
1329 			} while (ccb->ataio.res.status & ATA_S_BUSY && timeout--);
1330 			mvs_legacy_intr(dev, 1);
1331 			return;
1332 		}
1333 		ch->donecount = 0;
1334 		if (ccb->ataio.cmd.command == ATA_READ_MUL ||
1335 		    ccb->ataio.cmd.command == ATA_READ_MUL48 ||
1336 		    ccb->ataio.cmd.command == ATA_WRITE_MUL ||
1337 		    ccb->ataio.cmd.command == ATA_WRITE_MUL48) {
1338 			ch->transfersize = min(ccb->ataio.dxfer_len,
1339 			    ch->curr[port].bytecount);
1340 		} else
1341 			ch->transfersize = min(ccb->ataio.dxfer_len, 512);
1342 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE)
1343 			ch->fake_busy = 1;
1344 		/* If data write command - output the data */
1345 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1346 			if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
1347 				device_printf(dev,
1348 				    "timeout waiting for write DRQ\n");
1349 				xpt_freeze_simq(ch->sim, 1);
1350 				ch->toslots |= (1 << slot->slot);
1351 				mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1352 				return;
1353 			}
1354 			ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
1355 			   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
1356 			   ch->transfersize / 2);
1357 		}
1358 	} else {
1359 		ch->donecount = 0;
1360 		ch->transfersize = min(ccb->csio.dxfer_len,
1361 		    ch->curr[port].bytecount);
1362 		/* Write ATA PACKET command. */
1363 		if (ch->basic_dma) {
1364 			ATA_OUTB(ch->r_mem, ATA_FEATURE, ATA_F_DMA);
1365 			ATA_OUTB(ch->r_mem, ATA_CYL_LSB, 0);
1366 		    	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, 0);
1367 		} else {
1368 			ATA_OUTB(ch->r_mem, ATA_FEATURE, 0);
1369 			ATA_OUTB(ch->r_mem, ATA_CYL_LSB, ch->transfersize);
1370 		    	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, ch->transfersize >> 8);
1371 		}
1372 		ATA_OUTB(ch->r_mem, ATA_COMMAND, ATA_PACKET_CMD);
1373 		ch->fake_busy = 1;
1374 		/* Wait for ready to write ATAPI command block */
1375 		if (mvs_wait(dev, 0, ATA_S_BUSY, 1000) < 0) {
1376 			device_printf(dev, "timeout waiting for ATAPI !BUSY\n");
1377 			xpt_freeze_simq(ch->sim, 1);
1378 			ch->toslots |= (1 << slot->slot);
1379 			mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1380 			return;
1381 		}
1382 		timeout = 5000;
1383 		while (timeout--) {
1384 		    int reason = ATA_INB(ch->r_mem, ATA_IREASON);
1385 		    int status = ATA_INB(ch->r_mem, ATA_STATUS);
1386 
1387 		    if (((reason & (ATA_I_CMD | ATA_I_IN)) |
1388 			 (status & (ATA_S_DRQ | ATA_S_BUSY))) == ATAPI_P_CMDOUT)
1389 			break;
1390 		    DELAY(20);
1391 		}
1392 		if (timeout <= 0) {
1393 			device_printf(dev,
1394 			    "timeout waiting for ATAPI command ready\n");
1395 			xpt_freeze_simq(ch->sim, 1);
1396 			ch->toslots |= (1 << slot->slot);
1397 			mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1398 			return;
1399 		}
1400 		/* Write ATAPI command. */
1401 		ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
1402 		   (uint16_t *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1403 		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes),
1404 		   ch->curr[port].atapi / 2);
1405 		DELAY(10);
1406 		if (ch->basic_dma) {
1407 			/* Start basic DMA. */
1408 			eprd = ch->dma.workrq_bus + MVS_EPRD_OFFSET +
1409 			    (MVS_EPRD_SIZE * slot->slot);
1410 			ATA_OUTL(ch->r_mem, DMA_DTLBA, eprd);
1411 			ATA_OUTL(ch->r_mem, DMA_DTHBA, (eprd >> 16) >> 16);
1412 			ATA_OUTL(ch->r_mem, DMA_C, DMA_C_START |
1413 			    (((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) ?
1414 			    DMA_C_READ : 0));
1415 		}
1416 	}
1417 	/* Start command execution timeout */
1418 	callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout, 0,
1419 	    (timeout_t*)mvs_timeout, slot, 0);
1420 }
1421 
1422 /* Must be called with channel locked. */
1423 static void
1424 mvs_execute_transaction(struct mvs_slot *slot)
1425 {
1426 	device_t dev = slot->dev;
1427 	struct mvs_channel *ch = device_get_softc(dev);
1428 	bus_addr_t eprd;
1429 	struct mvs_crqb *crqb;
1430 	struct mvs_crqb_gen2e *crqb2e;
1431 	union ccb *ccb = slot->ccb;
1432 	int port = ccb->ccb_h.target_id & 0x0f;
1433 	int i;
1434 
1435 	/* Get address of the prepared EPRD */
1436 	eprd = ch->dma.workrq_bus + MVS_EPRD_OFFSET + (MVS_EPRD_SIZE * slot->slot);
1437 	/* Prepare CRQB. Gen IIe uses different CRQB format. */
1438 	if (ch->quirks & MVS_Q_GENIIE) {
1439 		crqb2e = (struct mvs_crqb_gen2e *)
1440 		    (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1441 		crqb2e->ctrlflg = htole32(
1442 		    ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB2E_READ : 0) |
1443 		    (slot->tag << MVS_CRQB2E_DTAG_SHIFT) |
1444 		    (port << MVS_CRQB2E_PMP_SHIFT) |
1445 		    (slot->slot << MVS_CRQB2E_HTAG_SHIFT));
1446 		/* If there is only one segment - no need to use S/G table. */
1447 		if (slot->dma.addr != 0) {
1448 			eprd = slot->dma.addr;
1449 			crqb2e->ctrlflg |= htole32(MVS_CRQB2E_CPRD);
1450 			crqb2e->drbc = slot->dma.len;
1451 		}
1452 		crqb2e->cprdbl = htole32(eprd);
1453 		crqb2e->cprdbh = htole32((eprd >> 16) >> 16);
1454 		crqb2e->cmd[0] = 0;
1455 		crqb2e->cmd[1] = 0;
1456 		crqb2e->cmd[2] = ccb->ataio.cmd.command;
1457 		crqb2e->cmd[3] = ccb->ataio.cmd.features;
1458 		crqb2e->cmd[4] = ccb->ataio.cmd.lba_low;
1459 		crqb2e->cmd[5] = ccb->ataio.cmd.lba_mid;
1460 		crqb2e->cmd[6] = ccb->ataio.cmd.lba_high;
1461 		crqb2e->cmd[7] = ccb->ataio.cmd.device;
1462 		crqb2e->cmd[8] = ccb->ataio.cmd.lba_low_exp;
1463 		crqb2e->cmd[9] = ccb->ataio.cmd.lba_mid_exp;
1464 		crqb2e->cmd[10] = ccb->ataio.cmd.lba_high_exp;
1465 		crqb2e->cmd[11] = ccb->ataio.cmd.features_exp;
1466 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1467 			crqb2e->cmd[12] = slot->tag << 3;
1468 			crqb2e->cmd[13] = 0;
1469 		} else {
1470 			crqb2e->cmd[12] = ccb->ataio.cmd.sector_count;
1471 			crqb2e->cmd[13] = ccb->ataio.cmd.sector_count_exp;
1472 		}
1473 		crqb2e->cmd[14] = 0;
1474 		crqb2e->cmd[15] = 0;
1475 	} else {
1476 		crqb = (struct mvs_crqb *)
1477 		    (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1478 		crqb->cprdbl = htole32(eprd);
1479 		crqb->cprdbh = htole32((eprd >> 16) >> 16);
1480 		crqb->ctrlflg = htole16(
1481 		    ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB_READ : 0) |
1482 		    (slot->slot << MVS_CRQB_TAG_SHIFT) |
1483 		    (port << MVS_CRQB_PMP_SHIFT));
1484 		i = 0;
1485 		/*
1486 		 * Controller can handle only 11 of 12 ATA registers,
1487 		 * so we have to choose which one to skip.
1488 		 */
1489 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1490 			crqb->cmd[i++] = ccb->ataio.cmd.features_exp;
1491 			crqb->cmd[i++] = 0x11;
1492 		}
1493 		crqb->cmd[i++] = ccb->ataio.cmd.features;
1494 		crqb->cmd[i++] = 0x11;
1495 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1496 			crqb->cmd[i++] = slot->tag << 3;
1497 			crqb->cmd[i++] = 0x12;
1498 		} else {
1499 			crqb->cmd[i++] = ccb->ataio.cmd.sector_count_exp;
1500 			crqb->cmd[i++] = 0x12;
1501 			crqb->cmd[i++] = ccb->ataio.cmd.sector_count;
1502 			crqb->cmd[i++] = 0x12;
1503 		}
1504 		crqb->cmd[i++] = ccb->ataio.cmd.lba_low_exp;
1505 		crqb->cmd[i++] = 0x13;
1506 		crqb->cmd[i++] = ccb->ataio.cmd.lba_low;
1507 		crqb->cmd[i++] = 0x13;
1508 		crqb->cmd[i++] = ccb->ataio.cmd.lba_mid_exp;
1509 		crqb->cmd[i++] = 0x14;
1510 		crqb->cmd[i++] = ccb->ataio.cmd.lba_mid;
1511 		crqb->cmd[i++] = 0x14;
1512 		crqb->cmd[i++] = ccb->ataio.cmd.lba_high_exp;
1513 		crqb->cmd[i++] = 0x15;
1514 		crqb->cmd[i++] = ccb->ataio.cmd.lba_high;
1515 		crqb->cmd[i++] = 0x15;
1516 		crqb->cmd[i++] = ccb->ataio.cmd.device;
1517 		crqb->cmd[i++] = 0x16;
1518 		crqb->cmd[i++] = ccb->ataio.cmd.command;
1519 		crqb->cmd[i++] = 0x97;
1520 	}
1521 	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
1522 	    BUS_DMASYNC_PREWRITE);
1523 	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1524 	    BUS_DMASYNC_PREREAD);
1525 	slot->state = MVS_SLOT_RUNNING;
1526 	ch->rslots |= (1 << slot->slot);
1527 	/* Issue command to the controller. */
1528 	ch->out_idx = (ch->out_idx + 1) & (MVS_MAX_SLOTS - 1);
1529 	ATA_OUTL(ch->r_mem, EDMA_REQQIP,
1530 	    ch->dma.workrq_bus + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1531 	/* Start command execution timeout */
1532 	callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout, 0,
1533 	    (timeout_t*)mvs_timeout, slot, 0);
1534 	return;
1535 }
1536 
1537 /* Must be called with channel locked. */
1538 static void
1539 mvs_process_timeout(device_t dev)
1540 {
1541 	struct mvs_channel *ch = device_get_softc(dev);
1542 	int i;
1543 
1544 	mtx_assert(&ch->mtx, MA_OWNED);
1545 	/* Handle the rest of commands. */
1546 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
1547 		/* Do we have a running request on slot? */
1548 		if (ch->slot[i].state < MVS_SLOT_RUNNING)
1549 			continue;
1550 		mvs_end_transaction(&ch->slot[i], MVS_ERR_TIMEOUT);
1551 	}
1552 }
1553 
1554 /* Must be called with channel locked. */
1555 static void
1556 mvs_rearm_timeout(device_t dev)
1557 {
1558 	struct mvs_channel *ch = device_get_softc(dev);
1559 	int i;
1560 
1561 	mtx_assert(&ch->mtx, MA_OWNED);
1562 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
1563 		struct mvs_slot *slot = &ch->slot[i];
1564 
1565 		/* Do we have a running request on slot? */
1566 		if (slot->state < MVS_SLOT_RUNNING)
1567 			continue;
1568 		if ((ch->toslots & (1 << i)) == 0)
1569 			continue;
1570 		callout_reset_sbt(&slot->timeout,
1571 		    SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1572 		    (timeout_t*)mvs_timeout, slot, 0);
1573 	}
1574 }
1575 
1576 /* Locked by callout mechanism. */
1577 static void
1578 mvs_timeout(struct mvs_slot *slot)
1579 {
1580 	device_t dev = slot->dev;
1581 	struct mvs_channel *ch = device_get_softc(dev);
1582 
1583 	/* Check for stale timeout. */
1584 	if (slot->state < MVS_SLOT_RUNNING)
1585 		return;
1586 	device_printf(dev, "Timeout on slot %d\n", slot->slot);
1587 	device_printf(dev, "iec %08x sstat %08x serr %08x edma_s %08x "
1588 	    "dma_c %08x dma_s %08x rs %08x status %02x\n",
1589 	    ATA_INL(ch->r_mem, EDMA_IEC),
1590 	    ATA_INL(ch->r_mem, SATA_SS), ATA_INL(ch->r_mem, SATA_SE),
1591 	    ATA_INL(ch->r_mem, EDMA_S), ATA_INL(ch->r_mem, DMA_C),
1592 	    ATA_INL(ch->r_mem, DMA_S), ch->rslots,
1593 	    ATA_INB(ch->r_mem, ATA_ALTSTAT));
1594 	/* Handle frozen command. */
1595 	mvs_requeue_frozen(dev);
1596 	/* We wait for other commands timeout and pray. */
1597 	if (ch->toslots == 0)
1598 		xpt_freeze_simq(ch->sim, 1);
1599 	ch->toslots |= (1 << slot->slot);
1600 	if ((ch->rslots & ~ch->toslots) == 0)
1601 		mvs_process_timeout(dev);
1602 	else
1603 		device_printf(dev, " ... waiting for slots %08x\n",
1604 		    ch->rslots & ~ch->toslots);
1605 }
1606 
1607 /* Must be called with channel locked. */
1608 static void
1609 mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et)
1610 {
1611 	device_t dev = slot->dev;
1612 	struct mvs_channel *ch = device_get_softc(dev);
1613 	union ccb *ccb = slot->ccb;
1614 	int lastto;
1615 
1616 	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
1617 	    BUS_DMASYNC_POSTWRITE);
1618 	/* Read result registers to the result struct
1619 	 * May be incorrect if several commands finished same time,
1620 	 * so read only when sure or have to.
1621 	 */
1622 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1623 		struct ata_res *res = &ccb->ataio.res;
1624 
1625 		if ((et == MVS_ERR_TFE) ||
1626 		    (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
1627 			mvs_tfd_read(dev, ccb);
1628 		} else
1629 			bzero(res, sizeof(*res));
1630 	} else {
1631 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1632 		    ch->basic_dma == 0)
1633 			ccb->csio.resid = ccb->csio.dxfer_len - ch->donecount;
1634 	}
1635 	if (ch->numpslots == 0 || ch->basic_dma) {
1636 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1637 			bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1638 			    (ccb->ccb_h.flags & CAM_DIR_IN) ?
1639 			    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1640 			bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
1641 		}
1642 	}
1643 	if (et != MVS_ERR_NONE)
1644 		ch->eslots |= (1 << slot->slot);
1645 	/* In case of error, freeze device for proper recovery. */
1646 	if ((et != MVS_ERR_NONE) && (!ch->recoverycmd) &&
1647 	    !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
1648 		xpt_freeze_devq(ccb->ccb_h.path, 1);
1649 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
1650 	}
1651 	/* Set proper result status. */
1652 	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1653 	switch (et) {
1654 	case MVS_ERR_NONE:
1655 		ccb->ccb_h.status |= CAM_REQ_CMP;
1656 		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1657 			ccb->csio.scsi_status = SCSI_STATUS_OK;
1658 		break;
1659 	case MVS_ERR_INVALID:
1660 		ch->fatalerr = 1;
1661 		ccb->ccb_h.status |= CAM_REQ_INVALID;
1662 		break;
1663 	case MVS_ERR_INNOCENT:
1664 		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1665 		break;
1666 	case MVS_ERR_TFE:
1667 	case MVS_ERR_NCQ:
1668 		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1669 			ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1670 			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1671 		} else {
1672 			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
1673 		}
1674 		break;
1675 	case MVS_ERR_SATA:
1676 		ch->fatalerr = 1;
1677 		if (!ch->recoverycmd) {
1678 			xpt_freeze_simq(ch->sim, 1);
1679 			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1680 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1681 		}
1682 		ccb->ccb_h.status |= CAM_UNCOR_PARITY;
1683 		break;
1684 	case MVS_ERR_TIMEOUT:
1685 		if (!ch->recoverycmd) {
1686 			xpt_freeze_simq(ch->sim, 1);
1687 			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1688 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1689 		}
1690 		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
1691 		break;
1692 	default:
1693 		ch->fatalerr = 1;
1694 		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1695 	}
1696 	/* Free slot. */
1697 	ch->oslots &= ~(1 << slot->slot);
1698 	ch->rslots &= ~(1 << slot->slot);
1699 	ch->aslots &= ~(1 << slot->slot);
1700 	slot->state = MVS_SLOT_EMPTY;
1701 	slot->ccb = NULL;
1702 	/* Update channel stats. */
1703 	ch->numrslots--;
1704 	ch->numrslotspd[ccb->ccb_h.target_id]--;
1705 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1706 		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1707 			ch->otagspd[ccb->ccb_h.target_id] &= ~(1 << slot->tag);
1708 			ch->numtslots--;
1709 			ch->numtslotspd[ccb->ccb_h.target_id]--;
1710 		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1711 			ch->numdslots--;
1712 		} else {
1713 			ch->numpslots--;
1714 		}
1715 	} else {
1716 		ch->numpslots--;
1717 		ch->basic_dma = 0;
1718 	}
1719 	/* Cancel timeout state if request completed normally. */
1720 	if (et != MVS_ERR_TIMEOUT) {
1721 		lastto = (ch->toslots == (1 << slot->slot));
1722 		ch->toslots &= ~(1 << slot->slot);
1723 		if (lastto)
1724 			xpt_release_simq(ch->sim, TRUE);
1725 	}
1726 	/* If it was our READ LOG command - process it. */
1727 	if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
1728 		mvs_process_read_log(dev, ccb);
1729 	/* If it was our REQUEST SENSE command - process it. */
1730 	} else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
1731 		mvs_process_request_sense(dev, ccb);
1732 	/* If it was NCQ or ATAPI command error, put result on hold. */
1733 	} else if (et == MVS_ERR_NCQ ||
1734 	    ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
1735 	     (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) {
1736 		ch->hold[slot->slot] = ccb;
1737 		ch->holdtag[slot->slot] = slot->tag;
1738 		ch->numhslots++;
1739 	} else
1740 		xpt_done(ccb);
1741 	/* If we have no other active commands, ... */
1742 	if (ch->rslots == 0) {
1743 		/* if there was fatal error - reset port. */
1744 		if (ch->toslots != 0 || ch->fatalerr) {
1745 			mvs_reset(dev);
1746 		} else {
1747 			/* if we have slots in error, we can reinit port. */
1748 			if (ch->eslots != 0) {
1749 				mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1750 				ch->eslots = 0;
1751 			}
1752 			/* if there commands on hold, we can do READ LOG. */
1753 			if (!ch->recoverycmd && ch->numhslots)
1754 				mvs_issue_recovery(dev);
1755 		}
1756 	/* If all the rest of commands are in timeout - give them chance. */
1757 	} else if ((ch->rslots & ~ch->toslots) == 0 &&
1758 	    et != MVS_ERR_TIMEOUT)
1759 		mvs_rearm_timeout(dev);
1760 	/* Unfreeze frozen command. */
1761 	if (ch->frozen && !mvs_check_collision(dev, ch->frozen)) {
1762 		union ccb *fccb = ch->frozen;
1763 		ch->frozen = NULL;
1764 		mvs_begin_transaction(dev, fccb);
1765 		xpt_release_simq(ch->sim, TRUE);
1766 	}
1767 	/* Start PM timer. */
1768 	if (ch->numrslots == 0 && ch->pm_level > 3 &&
1769 	    (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
1770 		callout_schedule(&ch->pm_timer,
1771 		    (ch->pm_level == 4) ? hz / 1000 : hz / 8);
1772 	}
1773 }
1774 
1775 static void
1776 mvs_issue_recovery(device_t dev)
1777 {
1778 	struct mvs_channel *ch = device_get_softc(dev);
1779 	union ccb *ccb;
1780 	struct ccb_ataio *ataio;
1781 	struct ccb_scsiio *csio;
1782 	int i;
1783 
1784 	/* Find some held command. */
1785 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
1786 		if (ch->hold[i])
1787 			break;
1788 	}
1789 	ccb = xpt_alloc_ccb_nowait();
1790 	if (ccb == NULL) {
1791 		device_printf(dev, "Unable to allocate recovery command\n");
1792 completeall:
1793 		/* We can't do anything -- complete held commands. */
1794 		for (i = 0; i < MVS_MAX_SLOTS; i++) {
1795 			if (ch->hold[i] == NULL)
1796 				continue;
1797 			ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1798 			ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
1799 			xpt_done(ch->hold[i]);
1800 			ch->hold[i] = NULL;
1801 			ch->numhslots--;
1802 		}
1803 		mvs_reset(dev);
1804 		return;
1805 	}
1806 	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
1807 	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1808 		/* READ LOG */
1809 		ccb->ccb_h.recovery_type = RECOVERY_READ_LOG;
1810 		ccb->ccb_h.func_code = XPT_ATA_IO;
1811 		ccb->ccb_h.flags = CAM_DIR_IN;
1812 		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
1813 		ataio = &ccb->ataio;
1814 		ataio->data_ptr = malloc(512, M_MVS, M_NOWAIT);
1815 		if (ataio->data_ptr == NULL) {
1816 			xpt_free_ccb(ccb);
1817 			device_printf(dev,
1818 			    "Unable to allocate memory for READ LOG command\n");
1819 			goto completeall;
1820 		}
1821 		ataio->dxfer_len = 512;
1822 		bzero(&ataio->cmd, sizeof(ataio->cmd));
1823 		ataio->cmd.flags = CAM_ATAIO_48BIT;
1824 		ataio->cmd.command = 0x2F;	/* READ LOG EXT */
1825 		ataio->cmd.sector_count = 1;
1826 		ataio->cmd.sector_count_exp = 0;
1827 		ataio->cmd.lba_low = 0x10;
1828 		ataio->cmd.lba_mid = 0;
1829 		ataio->cmd.lba_mid_exp = 0;
1830 	} else {
1831 		/* REQUEST SENSE */
1832 		ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE;
1833 		ccb->ccb_h.recovery_slot = i;
1834 		ccb->ccb_h.func_code = XPT_SCSI_IO;
1835 		ccb->ccb_h.flags = CAM_DIR_IN;
1836 		ccb->ccb_h.status = 0;
1837 		ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
1838 		csio = &ccb->csio;
1839 		csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
1840 		csio->dxfer_len = ch->hold[i]->csio.sense_len;
1841 		csio->cdb_len = 6;
1842 		bzero(&csio->cdb_io, sizeof(csio->cdb_io));
1843 		csio->cdb_io.cdb_bytes[0] = 0x03;
1844 		csio->cdb_io.cdb_bytes[4] = csio->dxfer_len;
1845 	}
1846 	/* Freeze SIM while doing recovery. */
1847 	ch->recoverycmd = 1;
1848 	xpt_freeze_simq(ch->sim, 1);
1849 	mvs_begin_transaction(dev, ccb);
1850 }
1851 
1852 static void
1853 mvs_process_read_log(device_t dev, union ccb *ccb)
1854 {
1855 	struct mvs_channel *ch = device_get_softc(dev);
1856 	uint8_t *data;
1857 	struct ata_res *res;
1858 	int i;
1859 
1860 	ch->recoverycmd = 0;
1861 
1862 	data = ccb->ataio.data_ptr;
1863 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
1864 	    (data[0] & 0x80) == 0) {
1865 		for (i = 0; i < MVS_MAX_SLOTS; i++) {
1866 			if (!ch->hold[i])
1867 				continue;
1868 			if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
1869 				continue;
1870 			if ((data[0] & 0x1F) == ch->holdtag[i]) {
1871 				res = &ch->hold[i]->ataio.res;
1872 				res->status = data[2];
1873 				res->error = data[3];
1874 				res->lba_low = data[4];
1875 				res->lba_mid = data[5];
1876 				res->lba_high = data[6];
1877 				res->device = data[7];
1878 				res->lba_low_exp = data[8];
1879 				res->lba_mid_exp = data[9];
1880 				res->lba_high_exp = data[10];
1881 				res->sector_count = data[12];
1882 				res->sector_count_exp = data[13];
1883 			} else {
1884 				ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1885 				ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
1886 			}
1887 			xpt_done(ch->hold[i]);
1888 			ch->hold[i] = NULL;
1889 			ch->numhslots--;
1890 		}
1891 	} else {
1892 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
1893 			device_printf(dev, "Error while READ LOG EXT\n");
1894 		else if ((data[0] & 0x80) == 0) {
1895 			device_printf(dev,
1896 			    "Non-queued command error in READ LOG EXT\n");
1897 		}
1898 		for (i = 0; i < MVS_MAX_SLOTS; i++) {
1899 			if (!ch->hold[i])
1900 				continue;
1901 			if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
1902 				continue;
1903 			xpt_done(ch->hold[i]);
1904 			ch->hold[i] = NULL;
1905 			ch->numhslots--;
1906 		}
1907 	}
1908 	free(ccb->ataio.data_ptr, M_MVS);
1909 	xpt_free_ccb(ccb);
1910 	xpt_release_simq(ch->sim, TRUE);
1911 }
1912 
1913 static void
1914 mvs_process_request_sense(device_t dev, union ccb *ccb)
1915 {
1916 	struct mvs_channel *ch = device_get_softc(dev);
1917 	int i;
1918 
1919 	ch->recoverycmd = 0;
1920 
1921 	i = ccb->ccb_h.recovery_slot;
1922 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
1923 		ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
1924 	} else {
1925 		ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1926 		ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
1927 	}
1928 	xpt_done(ch->hold[i]);
1929 	ch->hold[i] = NULL;
1930 	ch->numhslots--;
1931 	xpt_free_ccb(ccb);
1932 	xpt_release_simq(ch->sim, TRUE);
1933 }
1934 
1935 static int
1936 mvs_wait(device_t dev, u_int s, u_int c, int t)
1937 {
1938 	int timeout = 0;
1939 	uint8_t st;
1940 
1941 	while (((st =  mvs_getstatus(dev, 0)) & (s | c)) != s) {
1942 		if (timeout >= t) {
1943 			if (t != 0)
1944 				device_printf(dev, "Wait status %02x\n", st);
1945 			return (-1);
1946 		}
1947 		DELAY(1000);
1948 		timeout++;
1949 	}
1950 	return (timeout);
1951 }
1952 
1953 static void
1954 mvs_requeue_frozen(device_t dev)
1955 {
1956 	struct mvs_channel *ch = device_get_softc(dev);
1957 	union ccb *fccb = ch->frozen;
1958 
1959 	if (fccb) {
1960 		ch->frozen = NULL;
1961 		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1962 		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1963 			xpt_freeze_devq(fccb->ccb_h.path, 1);
1964 			fccb->ccb_h.status |= CAM_DEV_QFRZN;
1965 		}
1966 		xpt_done(fccb);
1967 	}
1968 }
1969 
1970 static void
1971 mvs_reset_to(void *arg)
1972 {
1973 	device_t dev = arg;
1974 	struct mvs_channel *ch = device_get_softc(dev);
1975 	int t;
1976 
1977 	if (ch->resetting == 0)
1978 		return;
1979 	ch->resetting--;
1980 	if ((t = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, 0)) >= 0) {
1981 		if (bootverbose) {
1982 			device_printf(dev,
1983 			    "MVS reset: device ready after %dms\n",
1984 			    (310 - ch->resetting) * 100);
1985 		}
1986 		ch->resetting = 0;
1987 		xpt_release_simq(ch->sim, TRUE);
1988 		return;
1989 	}
1990 	if (ch->resetting == 0) {
1991 		device_printf(dev,
1992 		    "MVS reset: device not ready after 31000ms\n");
1993 		xpt_release_simq(ch->sim, TRUE);
1994 		return;
1995 	}
1996 	callout_schedule(&ch->reset_timer, hz / 10);
1997 }
1998 
1999 static void
2000 mvs_errata(device_t dev)
2001 {
2002 	struct mvs_channel *ch = device_get_softc(dev);
2003 	uint32_t val;
2004 
2005 	if (ch->quirks & MVS_Q_SOC65) {
2006 		val = ATA_INL(ch->r_mem, SATA_PHYM3);
2007 		val &= ~(0x3 << 27);	/* SELMUPF = 1 */
2008 		val |= (0x1 << 27);
2009 		val &= ~(0x3 << 29);	/* SELMUPI = 1 */
2010 		val |= (0x1 << 29);
2011 		ATA_OUTL(ch->r_mem, SATA_PHYM3, val);
2012 
2013 		val = ATA_INL(ch->r_mem, SATA_PHYM4);
2014 		val &= ~0x1;		/* SATU_OD8 = 0 */
2015 		val |= (0x1 << 16);	/* reserved bit 16 = 1 */
2016 		ATA_OUTL(ch->r_mem, SATA_PHYM4, val);
2017 
2018 		val = ATA_INL(ch->r_mem, SATA_PHYM9_GEN2);
2019 		val &= ~0xf;		/* TXAMP[3:0] = 8 */
2020 		val |= 0x8;
2021 		val &= ~(0x1 << 14);	/* TXAMP[4] = 0 */
2022 		ATA_OUTL(ch->r_mem, SATA_PHYM9_GEN2, val);
2023 
2024 		val = ATA_INL(ch->r_mem, SATA_PHYM9_GEN1);
2025 		val &= ~0xf;		/* TXAMP[3:0] = 8 */
2026 		val |= 0x8;
2027 		val &= ~(0x1 << 14);	/* TXAMP[4] = 0 */
2028 		ATA_OUTL(ch->r_mem, SATA_PHYM9_GEN1, val);
2029 	}
2030 }
2031 
2032 static void
2033 mvs_reset(device_t dev)
2034 {
2035 	struct mvs_channel *ch = device_get_softc(dev);
2036 	int i;
2037 
2038 	xpt_freeze_simq(ch->sim, 1);
2039 	if (bootverbose)
2040 		device_printf(dev, "MVS reset...\n");
2041 	/* Forget about previous reset. */
2042 	if (ch->resetting) {
2043 		ch->resetting = 0;
2044 		callout_stop(&ch->reset_timer);
2045 		xpt_release_simq(ch->sim, TRUE);
2046 	}
2047 	/* Requeue freezed command. */
2048 	mvs_requeue_frozen(dev);
2049 	/* Kill the engine and requeue all running commands. */
2050 	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
2051 	ATA_OUTL(ch->r_mem, DMA_C, 0);
2052 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
2053 		/* Do we have a running request on slot? */
2054 		if (ch->slot[i].state < MVS_SLOT_RUNNING)
2055 			continue;
2056 		/* XXX; Commands in loading state. */
2057 		mvs_end_transaction(&ch->slot[i], MVS_ERR_INNOCENT);
2058 	}
2059 	for (i = 0; i < MVS_MAX_SLOTS; i++) {
2060 		if (!ch->hold[i])
2061 			continue;
2062 		xpt_done(ch->hold[i]);
2063 		ch->hold[i] = NULL;
2064 		ch->numhslots--;
2065 	}
2066 	if (ch->toslots != 0)
2067 		xpt_release_simq(ch->sim, TRUE);
2068 	ch->eslots = 0;
2069 	ch->toslots = 0;
2070 	ch->fatalerr = 0;
2071 	ch->fake_busy = 0;
2072 	/* Tell the XPT about the event */
2073 	xpt_async(AC_BUS_RESET, ch->path, NULL);
2074 	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
2075 	ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EATARST);
2076 	DELAY(25);
2077 	ATA_OUTL(ch->r_mem, EDMA_CMD, 0);
2078 	mvs_errata(dev);
2079 	/* Reset and reconnect PHY, */
2080 	if (!mvs_sata_phy_reset(dev)) {
2081 		if (bootverbose)
2082 			device_printf(dev, "MVS reset: device not found\n");
2083 		ch->devices = 0;
2084 		ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2085 		ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
2086 		ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
2087 		xpt_release_simq(ch->sim, TRUE);
2088 		return;
2089 	}
2090 	if (bootverbose)
2091 		device_printf(dev, "MVS reset: device found\n");
2092 	/* Wait for clearing busy status. */
2093 	if ((i = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ,
2094 	    dumping ? 31000 : 0)) < 0) {
2095 		if (dumping) {
2096 			device_printf(dev,
2097 			    "MVS reset: device not ready after 31000ms\n");
2098 		} else
2099 			ch->resetting = 310;
2100 	} else if (bootverbose)
2101 		device_printf(dev, "MVS reset: device ready after %dms\n", i);
2102 	ch->devices = 1;
2103 	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2104 	ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
2105 	ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
2106 	if (ch->resetting)
2107 		callout_reset(&ch->reset_timer, hz / 10, mvs_reset_to, dev);
2108 	else
2109 		xpt_release_simq(ch->sim, TRUE);
2110 }
2111 
2112 static void
2113 mvs_softreset(device_t dev, union ccb *ccb)
2114 {
2115 	struct mvs_channel *ch = device_get_softc(dev);
2116 	int port = ccb->ccb_h.target_id & 0x0f;
2117 	int i, stuck;
2118 	uint8_t status;
2119 
2120 	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
2121 	ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
2122 	ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_RESET);
2123 	DELAY(10000);
2124 	ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
2125 	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2126 	/* Wait for clearing busy status. */
2127 	if ((i = mvs_wait(dev, 0, ATA_S_BUSY, ccb->ccb_h.timeout)) < 0) {
2128 		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
2129 		stuck = 1;
2130 	} else {
2131 		status = mvs_getstatus(dev, 0);
2132 		if (status & ATA_S_ERROR)
2133 			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
2134 		else
2135 			ccb->ccb_h.status |= CAM_REQ_CMP;
2136 		if (status & ATA_S_DRQ)
2137 			stuck = 1;
2138 		else
2139 			stuck = 0;
2140 	}
2141 	mvs_tfd_read(dev, ccb);
2142 
2143 	/*
2144 	 * XXX: If some device on PMP failed to soft-reset,
2145 	 * try to recover by sending dummy soft-reset to PMP.
2146 	 */
2147 	if (stuck && ch->pm_present && port != 15) {
2148 		ATA_OUTB(ch->r_mem, SATA_SATAICTL,
2149 		    15 << SATA_SATAICTL_PMPTX_SHIFT);
2150 		ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_RESET);
2151 		DELAY(10000);
2152 		ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
2153 		mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, ccb->ccb_h.timeout);
2154 	}
2155 
2156 	xpt_done(ccb);
2157 }
2158 
2159 static int
2160 mvs_sata_connect(struct mvs_channel *ch)
2161 {
2162 	u_int32_t status;
2163 	int timeout, found = 0;
2164 
2165 	/* Wait up to 100ms for "connect well" */
2166 	for (timeout = 0; timeout < 1000 ; timeout++) {
2167 		status = ATA_INL(ch->r_mem, SATA_SS);
2168 		if ((status & SATA_SS_DET_MASK) != SATA_SS_DET_NO_DEVICE)
2169 			found = 1;
2170 		if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
2171 		    ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
2172 		    ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE))
2173 			break;
2174 		if ((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_OFFLINE) {
2175 			if (bootverbose) {
2176 				device_printf(ch->dev, "SATA offline status=%08x\n",
2177 				    status);
2178 			}
2179 			return (0);
2180 		}
2181 		if (found == 0 && timeout >= 100)
2182 			break;
2183 		DELAY(100);
2184 	}
2185 	if (timeout >= 1000 || !found) {
2186 		if (bootverbose) {
2187 			device_printf(ch->dev,
2188 			    "SATA connect timeout time=%dus status=%08x\n",
2189 			    timeout * 100, status);
2190 		}
2191 		return (0);
2192 	}
2193 	if (bootverbose) {
2194 		device_printf(ch->dev, "SATA connect time=%dus status=%08x\n",
2195 		    timeout * 100, status);
2196 	}
2197 	/* Clear SATA error register */
2198 	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2199 	return (1);
2200 }
2201 
2202 static int
2203 mvs_sata_phy_reset(device_t dev)
2204 {
2205 	struct mvs_channel *ch = device_get_softc(dev);
2206 	int sata_rev;
2207 	uint32_t val;
2208 
2209 	sata_rev = ch->user[ch->pm_present ? 15 : 0].revision;
2210 	if (sata_rev == 1)
2211 		val = SATA_SC_SPD_SPEED_GEN1;
2212 	else if (sata_rev == 2)
2213 		val = SATA_SC_SPD_SPEED_GEN2;
2214 	else if (sata_rev == 3)
2215 		val = SATA_SC_SPD_SPEED_GEN3;
2216 	else
2217 		val = 0;
2218 	ATA_OUTL(ch->r_mem, SATA_SC,
2219 	    SATA_SC_DET_RESET | val |
2220 	    SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER);
2221 	DELAY(1000);
2222 	ATA_OUTL(ch->r_mem, SATA_SC,
2223 	    SATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 :
2224 	    (SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER)));
2225 	if (!mvs_sata_connect(ch)) {
2226 		if (ch->pm_level > 0)
2227 			ATA_OUTL(ch->r_mem, SATA_SC, SATA_SC_DET_DISABLE);
2228 		return (0);
2229 	}
2230 	return (1);
2231 }
2232 
2233 static int
2234 mvs_check_ids(device_t dev, union ccb *ccb)
2235 {
2236 	struct mvs_channel *ch = device_get_softc(dev);
2237 
2238 	if (ccb->ccb_h.target_id > ((ch->quirks & MVS_Q_GENI) ? 0 : 15)) {
2239 		ccb->ccb_h.status = CAM_TID_INVALID;
2240 		xpt_done(ccb);
2241 		return (-1);
2242 	}
2243 	if (ccb->ccb_h.target_lun != 0) {
2244 		ccb->ccb_h.status = CAM_LUN_INVALID;
2245 		xpt_done(ccb);
2246 		return (-1);
2247 	}
2248 	return (0);
2249 }
2250 
2251 static void
2252 mvsaction(struct cam_sim *sim, union ccb *ccb)
2253 {
2254 	device_t dev, parent;
2255 	struct mvs_channel *ch;
2256 
2257 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("mvsaction func_code=%x\n",
2258 	    ccb->ccb_h.func_code));
2259 
2260 	ch = (struct mvs_channel *)cam_sim_softc(sim);
2261 	dev = ch->dev;
2262 	switch (ccb->ccb_h.func_code) {
2263 	/* Common cases first */
2264 	case XPT_ATA_IO:	/* Execute the requested I/O operation */
2265 	case XPT_SCSI_IO:
2266 		if (mvs_check_ids(dev, ccb))
2267 			return;
2268 		if (ch->devices == 0 ||
2269 		    (ch->pm_present == 0 &&
2270 		     ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
2271 			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2272 			break;
2273 		}
2274 		ccb->ccb_h.recovery_type = RECOVERY_NONE;
2275 		/* Check for command collision. */
2276 		if (mvs_check_collision(dev, ccb)) {
2277 			/* Freeze command. */
2278 			ch->frozen = ccb;
2279 			/* We have only one frozen slot, so freeze simq also. */
2280 			xpt_freeze_simq(ch->sim, 1);
2281 			return;
2282 		}
2283 		mvs_begin_transaction(dev, ccb);
2284 		return;
2285 	case XPT_EN_LUN:		/* Enable LUN as a target */
2286 	case XPT_TARGET_IO:		/* Execute target I/O request */
2287 	case XPT_ACCEPT_TARGET_IO:	/* Accept Host Target Mode CDB */
2288 	case XPT_CONT_TARGET_IO:	/* Continue Host Target I/O Connection*/
2289 	case XPT_ABORT:			/* Abort the specified CCB */
2290 		/* XXX Implement */
2291 		ccb->ccb_h.status = CAM_REQ_INVALID;
2292 		break;
2293 	case XPT_SET_TRAN_SETTINGS:
2294 	{
2295 		struct	ccb_trans_settings *cts = &ccb->cts;
2296 		struct	mvs_device *d;
2297 
2298 		if (mvs_check_ids(dev, ccb))
2299 			return;
2300 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2301 			d = &ch->curr[ccb->ccb_h.target_id];
2302 		else
2303 			d = &ch->user[ccb->ccb_h.target_id];
2304 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
2305 			d->revision = cts->xport_specific.sata.revision;
2306 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
2307 			d->mode = cts->xport_specific.sata.mode;
2308 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) {
2309 			d->bytecount = min((ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048,
2310 			    cts->xport_specific.sata.bytecount);
2311 		}
2312 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
2313 			d->tags = min(MVS_MAX_SLOTS, cts->xport_specific.sata.tags);
2314 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
2315 			ch->pm_present = cts->xport_specific.sata.pm_present;
2316 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
2317 			d->atapi = cts->xport_specific.sata.atapi;
2318 		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
2319 			d->caps = cts->xport_specific.sata.caps;
2320 		ccb->ccb_h.status = CAM_REQ_CMP;
2321 		break;
2322 	}
2323 	case XPT_GET_TRAN_SETTINGS:
2324 	/* Get default/user set transfer settings for the target */
2325 	{
2326 		struct	ccb_trans_settings *cts = &ccb->cts;
2327 		struct  mvs_device *d;
2328 		uint32_t status;
2329 
2330 		if (mvs_check_ids(dev, ccb))
2331 			return;
2332 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2333 			d = &ch->curr[ccb->ccb_h.target_id];
2334 		else
2335 			d = &ch->user[ccb->ccb_h.target_id];
2336 		cts->protocol = PROTO_UNSPECIFIED;
2337 		cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
2338 		cts->transport = XPORT_SATA;
2339 		cts->transport_version = XPORT_VERSION_UNSPECIFIED;
2340 		cts->proto_specific.valid = 0;
2341 		cts->xport_specific.sata.valid = 0;
2342 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
2343 		    (ccb->ccb_h.target_id == 15 ||
2344 		    (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
2345 			status = ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_SPD_MASK;
2346 			if (status & 0x0f0) {
2347 				cts->xport_specific.sata.revision =
2348 				    (status & 0x0f0) >> 4;
2349 				cts->xport_specific.sata.valid |=
2350 				    CTS_SATA_VALID_REVISION;
2351 			}
2352 			cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
2353 //			if (ch->pm_level)
2354 //				cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
2355 			cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN;
2356 			cts->xport_specific.sata.caps &=
2357 			    ch->user[ccb->ccb_h.target_id].caps;
2358 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
2359 		} else {
2360 			cts->xport_specific.sata.revision = d->revision;
2361 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
2362 			cts->xport_specific.sata.caps = d->caps;
2363 			if (cts->type == CTS_TYPE_CURRENT_SETTINGS/* &&
2364 			    (ch->quirks & MVS_Q_GENIIE) == 0*/)
2365 				cts->xport_specific.sata.caps &= ~CTS_SATA_CAPS_H_AN;
2366 			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
2367 		}
2368 		cts->xport_specific.sata.mode = d->mode;
2369 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
2370 		cts->xport_specific.sata.bytecount = d->bytecount;
2371 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
2372 		cts->xport_specific.sata.pm_present = ch->pm_present;
2373 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
2374 		cts->xport_specific.sata.tags = d->tags;
2375 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
2376 		cts->xport_specific.sata.atapi = d->atapi;
2377 		cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
2378 		ccb->ccb_h.status = CAM_REQ_CMP;
2379 		break;
2380 	}
2381 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
2382 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
2383 		mvs_reset(dev);
2384 		ccb->ccb_h.status = CAM_REQ_CMP;
2385 		break;
2386 	case XPT_TERM_IO:		/* Terminate the I/O process */
2387 		/* XXX Implement */
2388 		ccb->ccb_h.status = CAM_REQ_INVALID;
2389 		break;
2390 	case XPT_PATH_INQ:		/* Path routing inquiry */
2391 	{
2392 		struct ccb_pathinq *cpi = &ccb->cpi;
2393 
2394 		parent = device_get_parent(dev);
2395 		cpi->version_num = 1; /* XXX??? */
2396 		cpi->hba_inquiry = PI_SDTR_ABLE;
2397 		if (!(ch->quirks & MVS_Q_GENI)) {
2398 			cpi->hba_inquiry |= PI_SATAPM;
2399 			/* Gen-II is extremely slow with NCQ on PMP. */
2400 			if ((ch->quirks & MVS_Q_GENIIE) || ch->pm_present == 0)
2401 				cpi->hba_inquiry |= PI_TAG_ABLE;
2402 		}
2403 		cpi->target_sprt = 0;
2404 		cpi->hba_misc = PIM_SEQSCAN;
2405 		cpi->hba_eng_cnt = 0;
2406 		if (!(ch->quirks & MVS_Q_GENI))
2407 			cpi->max_target = 15;
2408 		else
2409 			cpi->max_target = 0;
2410 		cpi->max_lun = 0;
2411 		cpi->initiator_id = 0;
2412 		cpi->bus_id = cam_sim_bus(sim);
2413 		cpi->base_transfer_speed = 150000;
2414 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2415 		strncpy(cpi->hba_vid, "Marvell", HBA_IDLEN);
2416 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2417 		cpi->unit_number = cam_sim_unit(sim);
2418 		cpi->transport = XPORT_SATA;
2419 		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
2420 		cpi->protocol = PROTO_ATA;
2421 		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
2422 		cpi->maxio = MAXPHYS;
2423 		if ((ch->quirks & MVS_Q_SOC) == 0) {
2424 			cpi->hba_vendor = pci_get_vendor(parent);
2425 			cpi->hba_device = pci_get_device(parent);
2426 			cpi->hba_subvendor = pci_get_subvendor(parent);
2427 			cpi->hba_subdevice = pci_get_subdevice(parent);
2428 		}
2429 		cpi->ccb_h.status = CAM_REQ_CMP;
2430 		break;
2431 	}
2432 	default:
2433 		ccb->ccb_h.status = CAM_REQ_INVALID;
2434 		break;
2435 	}
2436 	xpt_done(ccb);
2437 }
2438 
2439 static void
2440 mvspoll(struct cam_sim *sim)
2441 {
2442 	struct mvs_channel *ch = (struct mvs_channel *)cam_sim_softc(sim);
2443 	struct mvs_intr_arg arg;
2444 
2445 	arg.arg = ch->dev;
2446 	arg.cause = 2 | 4; /* XXX */
2447 	mvs_ch_intr(&arg);
2448 	if (ch->resetting != 0 &&
2449 	    (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) {
2450 		ch->resetpolldiv = 1000;
2451 		mvs_reset_to(ch->dev);
2452 	}
2453 }
2454 
2455