xref: /freebsd/sys/powerpc/ps3/ps3disk.c (revision a7623790fb345e6dc986dfd31df0ace115e6f2e4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2011 glevand (geoffrey.levand@mail.ru)
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  *
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/sysctl.h>
35 #include <sys/disk.h>
36 #include <sys/bio.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/module.h>
44 #include <sys/mutex.h>
45 
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 
49 #include <machine/pio.h>
50 #include <machine/bus.h>
51 #include <machine/platform.h>
52 #include <machine/resource.h>
53 #include <sys/bus.h>
54 #include <sys/rman.h>
55 
56 #include <geom/geom_disk.h>
57 
58 #include "ps3bus.h"
59 #include "ps3-hvcall.h"
60 
61 #define PS3DISK_LOCK_INIT(_sc)		\
62 	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), "ps3disk", MTX_DEF)
63 #define PS3DISK_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
64 #define PS3DISK_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
65 #define	PS3DISK_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
66 #define PS3DISK_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
67 #define PS3DISK_ASSERT_UNLOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
68 
69 #define LV1_STORAGE_ATA_HDDOUT 		0x23
70 
71 static SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
72     "PS3 Disk driver parameters");
73 
74 #ifdef PS3DISK_DEBUG
75 static int ps3disk_debug = 0;
76 SYSCTL_INT(_hw_ps3disk, OID_AUTO, debug, CTLFLAG_RW, &ps3disk_debug,
77 	0, "control debugging printfs");
78 TUNABLE_INT("hw.ps3disk.debug", &ps3disk_debug);
79 enum {
80 	PS3DISK_DEBUG_INTR	= 0x00000001,
81 	PS3DISK_DEBUG_TASK	= 0x00000002,
82 	PS3DISK_DEBUG_READ	= 0x00000004,
83 	PS3DISK_DEBUG_WRITE	= 0x00000008,
84 	PS3DISK_DEBUG_FLUSH	= 0x00000010,
85 	PS3DISK_DEBUG_ANY	= 0xffffffff
86 };
87 #define	DPRINTF(sc, m, fmt, ...)				\
88 do {								\
89 	if (sc->sc_debug & (m))					\
90 		printf(fmt, __VA_ARGS__);			\
91 } while (0)
92 #else
93 #define	DPRINTF(sc, m, fmt, ...)
94 #endif
95 
96 struct ps3disk_region {
97 	uint64_t r_id;
98 	uint64_t r_start;
99 	uint64_t r_size;
100 	uint64_t r_flags;
101 };
102 
103 struct ps3disk_softc {
104 	device_t sc_dev;
105 
106 	struct mtx sc_mtx;
107 
108 	uint64_t sc_blksize;
109 	uint64_t sc_nblocks;
110 
111 	uint64_t sc_nregs;
112 	struct ps3disk_region *sc_reg;
113 
114 	int sc_irqid;
115 	struct resource	*sc_irq;
116 	void *sc_irqctx;
117 
118 	struct disk **sc_disk;
119 
120 	struct bio_queue_head sc_bioq;
121 	struct bio_queue_head sc_deferredq;
122 	struct proc *sc_task;
123 
124 	bus_dma_tag_t sc_dmatag;
125 
126 	int sc_running;
127 	int sc_debug;
128 };
129 
130 static int ps3disk_open(struct disk *dp);
131 static int ps3disk_close(struct disk *dp);
132 static void ps3disk_strategy(struct bio *bp);
133 
134 static void ps3disk_task(void *arg);
135 static void ps3disk_intr(void *arg);
136 static int ps3disk_get_disk_geometry(struct ps3disk_softc *sc);
137 static int ps3disk_enum_regions(struct ps3disk_softc *sc);
138 static void ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs,
139     int error);
140 
141 static void ps3disk_sysctlattach(struct ps3disk_softc *sc);
142 
143 static MALLOC_DEFINE(M_PS3DISK, "ps3disk", "PS3 Disk");
144 
145 static int
146 ps3disk_probe(device_t dev)
147 {
148 	if (ps3bus_get_bustype(dev) != PS3_BUSTYPE_STORAGE ||
149 	    ps3bus_get_devtype(dev) != PS3_DEVTYPE_DISK)
150 		return (ENXIO);
151 
152 	device_set_desc(dev, "Playstation 3 Disk");
153 
154 	return (BUS_PROBE_SPECIFIC);
155 }
156 
157 static int
158 ps3disk_attach(device_t dev)
159 {
160 	struct ps3disk_softc *sc;
161 	struct disk *d;
162 	intmax_t mb;
163 	uint64_t junk;
164 	char unit;
165 	int i, err;
166 
167 	sc = device_get_softc(dev);
168 	sc->sc_dev = dev;
169 
170 	PS3DISK_LOCK_INIT(sc);
171 
172 	err = ps3disk_get_disk_geometry(sc);
173 	if (err) {
174 		device_printf(dev, "Could not get disk geometry\n");
175 		err = ENXIO;
176 		goto fail_destroy_lock;
177 	}
178 
179 	device_printf(dev, "block size %lu total blocks %lu\n",
180 	    sc->sc_blksize, sc->sc_nblocks);
181 
182 	err = ps3disk_enum_regions(sc);
183 	if (err) {
184 		device_printf(dev, "Could not enumerate disk regions\n");
185 		err = ENXIO;
186 		goto fail_destroy_lock;
187 	}
188 
189 	device_printf(dev, "Found %lu regions\n", sc->sc_nregs);
190 
191 	if (!sc->sc_nregs) {
192 		err = ENXIO;
193 		goto fail_destroy_lock;
194 	}
195 
196 	/* Setup interrupt handler */
197 	sc->sc_irqid = 0;
198 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqid,
199 	    RF_ACTIVE);
200 	if (!sc->sc_irq) {
201 		device_printf(dev, "Could not allocate IRQ\n");
202 		err = ENXIO;
203 		goto fail_free_regions;
204 	}
205 
206 	err = bus_setup_intr(dev, sc->sc_irq,
207 	    INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY,
208 	    NULL, ps3disk_intr, sc, &sc->sc_irqctx);
209 	if (err) {
210 		device_printf(dev, "Could not setup IRQ\n");
211 		err = ENXIO;
212 		goto fail_release_intr;
213 	}
214 
215 	/* Setup DMA */
216 	err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0,
217 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
218 	    BUS_SPACE_UNRESTRICTED, 1, PAGE_SIZE, 0,
219 	    busdma_lock_mutex, &sc->sc_mtx, &sc->sc_dmatag);
220 	if (err) {
221 		device_printf(dev, "Could not create DMA tag\n");
222 		err = ENXIO;
223 		goto fail_teardown_intr;
224 	}
225 
226 	/* Setup disks */
227 
228 	sc->sc_disk = malloc(sc->sc_nregs * sizeof(struct disk *),
229 	    M_PS3DISK, M_ZERO | M_WAITOK);
230 	if (!sc->sc_disk) {
231 		device_printf(dev, "Could not allocate disk(s)\n");
232 		err = ENOMEM;
233 		goto fail_teardown_intr;
234 	}
235 
236 	for (i = 0; i < sc->sc_nregs; i++) {
237 		struct ps3disk_region *rp = &sc->sc_reg[i];
238 
239 		d = sc->sc_disk[i] = disk_alloc();
240 		d->d_open = ps3disk_open;
241 		d->d_close = ps3disk_close;
242 		d->d_strategy = ps3disk_strategy;
243 		d->d_name = "ps3disk";
244 		d->d_drv1 = sc;
245 		d->d_maxsize = PAGE_SIZE;
246 		d->d_sectorsize = sc->sc_blksize;
247 		d->d_unit = i;
248 		d->d_mediasize = sc->sc_reg[i].r_size * sc->sc_blksize;
249 		d->d_flags |= DISKFLAG_CANFLUSHCACHE;
250 
251 		mb = d->d_mediasize >> 20;
252 		unit = 'M';
253 		if (mb >= 10240) {
254 			unit = 'G';
255 			mb /= 1024;
256 		}
257 
258 		/* Test to see if we can read this region */
259 		err = lv1_storage_read(ps3bus_get_device(dev), d->d_unit,
260 		    0, 0, rp->r_flags, 0, &junk);
261 		device_printf(dev, "region %d %ju%cB%s\n", i, mb, unit,
262 		    (err == LV1_DENIED_BY_POLICY) ?  " (hypervisor protected)"
263 		    : "");
264 
265 		if (err != LV1_DENIED_BY_POLICY)
266 			disk_create(d, DISK_VERSION);
267 	}
268 	err = 0;
269 
270 	bioq_init(&sc->sc_bioq);
271 	bioq_init(&sc->sc_deferredq);
272 	kproc_create(&ps3disk_task, sc, &sc->sc_task, 0, 0, "ps3disk");
273 
274 	ps3disk_sysctlattach(sc);
275 	sc->sc_running = 1;
276 	return (0);
277 
278 fail_teardown_intr:
279 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
280 fail_release_intr:
281 	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
282 fail_free_regions:
283 	free(sc->sc_reg, M_PS3DISK);
284 fail_destroy_lock:
285 	PS3DISK_LOCK_DESTROY(sc);
286 	return (err);
287 }
288 
289 static int
290 ps3disk_detach(device_t dev)
291 {
292 	struct ps3disk_softc *sc = device_get_softc(dev);
293 	int i;
294 
295 	for (i = 0; i < sc->sc_nregs; i++)
296 		disk_destroy(sc->sc_disk[i]);
297 
298 	bus_dma_tag_destroy(sc->sc_dmatag);
299 
300 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
301 	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
302 
303 	free(sc->sc_disk, M_PS3DISK);
304 	free(sc->sc_reg, M_PS3DISK);
305 
306 	PS3DISK_LOCK_DESTROY(sc);
307 
308 	return (0);
309 }
310 
311 static int
312 ps3disk_open(struct disk *dp)
313 {
314 	return (0);
315 }
316 
317 static int
318 ps3disk_close(struct disk *dp)
319 {
320 	return (0);
321 }
322 
323 /* Process deferred blocks */
324 static void
325 ps3disk_task(void *arg)
326 {
327 	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
328 	struct bio *bp;
329 
330 	while (1) {
331 		kproc_suspend_check(sc->sc_task);
332 		tsleep(&sc->sc_deferredq, PRIBIO, "ps3disk", 10);
333 
334 		PS3DISK_LOCK(sc);
335 		bp = bioq_takefirst(&sc->sc_deferredq);
336 		PS3DISK_UNLOCK(sc);
337 
338 		if (bp == NULL)
339 			continue;
340 
341 		if (bp->bio_driver1 != NULL) {
342 			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
343 			    bp->bio_driver1);
344 			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
345 			    bp->bio_driver1);
346 		}
347 
348 		ps3disk_strategy(bp);
349 	}
350 
351 	kproc_exit(0);
352 }
353 
354 static void
355 ps3disk_strategy(struct bio *bp)
356 {
357 	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
358 	int err;
359 
360 	if (sc == NULL) {
361 		bp->bio_flags |= BIO_ERROR;
362 		bp->bio_error = EINVAL;
363 		biodone(bp);
364 		return;
365 	}
366 
367 	PS3DISK_LOCK(sc);
368 	bp->bio_resid = bp->bio_bcount;
369 	bioq_insert_tail(&sc->sc_bioq, bp);
370 
371 	DPRINTF(sc, PS3DISK_DEBUG_TASK, "%s: bio_cmd 0x%02x\n",
372 	    __func__, bp->bio_cmd);
373 
374 	err = 0;
375 	if (bp->bio_cmd == BIO_FLUSH) {
376 		bp->bio_driver1 = 0;
377 		err = lv1_storage_send_device_command(
378 		    ps3bus_get_device(sc->sc_dev), LV1_STORAGE_ATA_HDDOUT,
379 		    0, 0, 0, 0, (uint64_t *)&bp->bio_driver2);
380 		if (err == LV1_BUSY)
381 			err = EAGAIN;
382 	} else if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
383 		if (bp->bio_bcount % sc->sc_blksize != 0) {
384 			err = EINVAL;
385 		} else {
386 			bus_dmamap_create(sc->sc_dmatag, BUS_DMA_COHERENT,
387 			    (bus_dmamap_t *)(&bp->bio_driver1));
388 			err = bus_dmamap_load(sc->sc_dmatag,
389 			    (bus_dmamap_t)(bp->bio_driver1), bp->bio_data,
390 			    bp->bio_bcount, ps3disk_transfer, bp, 0);
391 			if (err == EINPROGRESS)
392 				err = 0;
393 		}
394 	} else {
395 		err = EINVAL;
396 	}
397 
398 	if (err == EAGAIN) {
399 		bioq_remove(&sc->sc_bioq, bp);
400 		bioq_insert_tail(&sc->sc_deferredq, bp);
401 	} else if (err != 0) {
402 		bp->bio_error = err;
403 		bp->bio_flags |= BIO_ERROR;
404 		bioq_remove(&sc->sc_bioq, bp);
405 		disk_err(bp, "hard error", -1, 1);
406 		biodone(bp);
407 	}
408 
409 	PS3DISK_UNLOCK(sc);
410 }
411 
412 static void
413 ps3disk_intr(void *arg)
414 {
415 	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
416 	device_t dev = sc->sc_dev;
417 	uint64_t devid = ps3bus_get_device(dev);
418 	struct bio *bp;
419 	uint64_t tag, status;
420 
421 	if (lv1_storage_get_async_status(devid, &tag, &status) != 0)
422 		return;
423 
424 	PS3DISK_LOCK(sc);
425 
426 	DPRINTF(sc, PS3DISK_DEBUG_INTR, "%s: tag 0x%016lx "
427 	    "status 0x%016lx\n", __func__, tag, status);
428 
429 	/* Locate the matching request */
430 	TAILQ_FOREACH(bp, &sc->sc_bioq.queue, bio_queue) {
431 		if ((uint64_t)bp->bio_driver2 != tag)
432 			continue;
433 
434 		if (status != 0) {
435 			device_printf(sc->sc_dev, "%s error (%#lx)\n",
436 			    (bp->bio_cmd == BIO_READ) ? "Read" : "Write",
437 			    status);
438 			bp->bio_error = EIO;
439 			bp->bio_flags |= BIO_ERROR;
440 		} else {
441 			bp->bio_error = 0;
442 			bp->bio_resid = 0;
443 			bp->bio_flags |= BIO_DONE;
444 		}
445 
446 		if (bp->bio_driver1 != NULL) {
447 			if (bp->bio_cmd == BIO_READ)
448 				bus_dmamap_sync(sc->sc_dmatag, (bus_dmamap_t)
449 				    bp->bio_driver1, BUS_DMASYNC_POSTREAD);
450 			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
451 			    bp->bio_driver1);
452 			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
453 			    bp->bio_driver1);
454 		}
455 
456 		bioq_remove(&sc->sc_bioq, bp);
457 		biodone(bp);
458 		break;
459 	}
460 
461 	if (bioq_first(&sc->sc_deferredq) != NULL)
462 		wakeup(&sc->sc_deferredq);
463 
464 	PS3DISK_UNLOCK(sc);
465 }
466 
467 static int
468 ps3disk_get_disk_geometry(struct ps3disk_softc *sc)
469 {
470 	device_t dev = sc->sc_dev;
471 	uint64_t bus_index = ps3bus_get_busidx(dev);
472 	uint64_t dev_index = ps3bus_get_devidx(dev);
473 	uint64_t junk;
474 	int err;
475 
476 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
477 	    (lv1_repository_string("bus") >> 32) | bus_index,
478 	    lv1_repository_string("dev") | dev_index,
479 	    lv1_repository_string("blk_size"), 0, &sc->sc_blksize, &junk);
480 	if (err) {
481 		device_printf(dev, "Could not get block size (0x%08x)\n", err);
482 		return (ENXIO);
483 	}
484 
485 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
486 	    (lv1_repository_string("bus") >> 32) | bus_index,
487 	    lv1_repository_string("dev") | dev_index,
488 	    lv1_repository_string("n_blocks"), 0, &sc->sc_nblocks, &junk);
489 	if (err) {
490 		device_printf(dev, "Could not get total number of blocks "
491 		    "(0x%08x)\n", err);
492 		err = ENXIO;
493 	}
494 
495 	return (err);
496 }
497 
498 static int
499 ps3disk_enum_regions(struct ps3disk_softc *sc)
500 {
501 	device_t dev = sc->sc_dev;
502 	uint64_t bus_index = ps3bus_get_busidx(dev);
503 	uint64_t dev_index = ps3bus_get_devidx(dev);
504 	uint64_t junk;
505 	int i, err;
506 
507 	/* Read number of regions */
508 
509 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
510 	    (lv1_repository_string("bus") >> 32) | bus_index,
511 	    lv1_repository_string("dev") | dev_index,
512 	    lv1_repository_string("n_regs"), 0, &sc->sc_nregs, &junk);
513 	if (err) {
514 		device_printf(dev, "Could not get number of regions (0x%08x)\n",
515 		    err);
516 		err = ENXIO;
517 		goto fail;
518 	}
519 
520 	if (!sc->sc_nregs)
521 		return 0;
522 
523 	sc->sc_reg = malloc(sc->sc_nregs * sizeof(struct ps3disk_region),
524 	    M_PS3DISK, M_ZERO | M_WAITOK);
525 	if (!sc->sc_reg) {
526 		err = ENOMEM;
527 		goto fail;
528 	}
529 
530 	/* Setup regions */
531 
532 	for (i = 0; i < sc->sc_nregs; i++) {
533 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
534 		    (lv1_repository_string("bus") >> 32) | bus_index,
535 		    lv1_repository_string("dev") | dev_index,
536 		    lv1_repository_string("region") | i,
537 		    lv1_repository_string("id"), &sc->sc_reg[i].r_id, &junk);
538 		if (err) {
539 			device_printf(dev, "Could not get region id (0x%08x)\n",
540 			    err);
541 			err = ENXIO;
542 			goto fail;
543 		}
544 
545 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
546 		    (lv1_repository_string("bus") >> 32) | bus_index,
547 		    lv1_repository_string("dev") | dev_index,
548 		    lv1_repository_string("region") | i,
549 		    lv1_repository_string("start"), &sc->sc_reg[i].r_start,
550 		    &junk);
551 		if (err) {
552 			device_printf(dev, "Could not get region start "
553 			    "(0x%08x)\n", err);
554 			err = ENXIO;
555 			goto fail;
556 		}
557 
558 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
559 		    (lv1_repository_string("bus") >> 32) | bus_index,
560 		    lv1_repository_string("dev") | dev_index,
561 		    lv1_repository_string("region") | i,
562 		    lv1_repository_string("size"), &sc->sc_reg[i].r_size,
563 		    &junk);
564 		if (err) {
565 			device_printf(dev, "Could not get region size "
566 			    "(0x%08x)\n", err);
567 			err = ENXIO;
568 			goto fail;
569 		}
570 
571 		if (i == 0)
572 			sc->sc_reg[i].r_flags = 0x2;
573 		else
574 			sc->sc_reg[i].r_flags = 0;
575 	}
576 
577 	return (0);
578 
579 fail:
580 
581 	sc->sc_nregs = 0;
582 	if (sc->sc_reg)
583 		free(sc->sc_reg, M_PS3DISK);
584 
585 	return (err);
586 }
587 
588 static void
589 ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
590 {
591 	struct bio *bp = (struct bio *)(arg);
592 	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
593 	struct ps3disk_region *rp = &sc->sc_reg[bp->bio_disk->d_unit];
594 	uint64_t devid = ps3bus_get_device(sc->sc_dev);
595 	uint64_t block;
596 	int i, err;
597 
598 	/* Locks already held by busdma */
599 	PS3DISK_ASSERT_LOCKED(sc);
600 
601 	if (error) {
602 		bp->bio_error = error;
603 		bp->bio_flags |= BIO_ERROR;
604 		bioq_remove(&sc->sc_bioq, bp);
605 		biodone(bp);
606 		return;
607 	}
608 
609 	block = bp->bio_pblkno;
610 	for (i = 0; i < nsegs; i++) {
611 		KASSERT((segs[i].ds_len % sc->sc_blksize) == 0,
612 		    ("DMA fragments not blocksize multiples"));
613 
614 		if (bp->bio_cmd == BIO_READ) {
615 			err = lv1_storage_read(devid, rp->r_id,
616 			    block, segs[i].ds_len/sc->sc_blksize,
617 			    rp->r_flags, segs[i].ds_addr,
618 			    (uint64_t *)&bp->bio_driver2);
619 		} else {
620 			bus_dmamap_sync(sc->sc_dmatag,
621 			    (bus_dmamap_t)bp->bio_driver1,
622 			    BUS_DMASYNC_PREWRITE);
623 			err = lv1_storage_write(devid, rp->r_id,
624 			    block, segs[i].ds_len/sc->sc_blksize,
625 			    rp->r_flags, segs[i].ds_addr,
626 			    (uint64_t *)&bp->bio_driver2);
627 		}
628 
629 		if (err) {
630 			if (err == LV1_BUSY) {
631 				bioq_remove(&sc->sc_bioq, bp);
632 				bioq_insert_tail(&sc->sc_deferredq, bp);
633 			} else {
634 				bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
635 				    bp->bio_driver1);
636 				bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
637 				    bp->bio_driver1);
638 				device_printf(sc->sc_dev, "Could not read "
639 				    "sectors (0x%08x)\n", err);
640 				bp->bio_error = EINVAL;
641 				bp->bio_flags |= BIO_ERROR;
642 				bioq_remove(&sc->sc_bioq, bp);
643 				biodone(bp);
644 			}
645 
646 			break;
647 		}
648 
649 		DPRINTF(sc, PS3DISK_DEBUG_READ, "%s: tag 0x%016lx\n",
650 		    __func__, sc->sc_bounce_tag);
651 	}
652 }
653 
654 #ifdef PS3DISK_DEBUG
655 static int
656 ps3disk_sysctl_debug(SYSCTL_HANDLER_ARGS)
657 {
658 	struct ps3disk_softc *sc = arg1;
659 	int debug, error;
660 
661 	debug = sc->sc_debug;
662 
663 	error = sysctl_handle_int(oidp, &debug, 0, req);
664 	if (error || !req->newptr)
665 		return error;
666 
667 	sc->sc_debug = debug;
668 
669 	return 0;
670 }
671 #endif
672 
673 static void
674 ps3disk_sysctlattach(struct ps3disk_softc *sc)
675 {
676 #ifdef PS3DISK_DEBUG
677 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
678 	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
679 
680 	sc->sc_debug = ps3disk_debug;
681 
682 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
683 	    "debug", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
684 	    ps3disk_sysctl_debug, "I", "control debugging printfs");
685 #endif
686 }
687 
688 static device_method_t ps3disk_methods[] = {
689 	DEVMETHOD(device_probe,		ps3disk_probe),
690 	DEVMETHOD(device_attach,	ps3disk_attach),
691 	DEVMETHOD(device_detach,	ps3disk_detach),
692 	{0, 0},
693 };
694 
695 static driver_t ps3disk_driver = {
696 	"ps3disk",
697 	ps3disk_methods,
698 	sizeof(struct ps3disk_softc),
699 };
700 
701 static devclass_t ps3disk_devclass;
702 
703 DRIVER_MODULE(ps3disk, ps3bus, ps3disk_driver, ps3disk_devclass, 0, 0);
704