xref: /freebsd/sys/dev/virtio/mmio/virtio_mmio.c (revision c6db8143eda5c775467145ac73e8ebec47afdd8f)
1 /*-
2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3  * All rights reserved.
4  *
5  * This software was developed by SRI International and the University of
6  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
7  * ("CTSRD"), as part of the DARPA CRASH research programme.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 /*
32  * VirtIO MMIO interface.
33  * This driver is heavily based on VirtIO PCI interface driver.
34  */
35 
36 /*
37  * FDT example:
38  *		virtio_block@1000 {
39  *			compatible = "virtio,mmio";
40  *			reg = <0x1000 0x100>;
41  *			interrupts = <63>;
42  *			interrupt-parent = <&GIC>;
43  *		};
44  */
45 
46 #include <sys/cdefs.h>
47 __FBSDID("$FreeBSD$");
48 
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/bus.h>
52 #include <sys/kernel.h>
53 #include <sys/module.h>
54 #include <sys/malloc.h>
55 #include <sys/rman.h>
56 
57 #include <machine/bus.h>
58 #include <machine/resource.h>
59 
60 #include <dev/fdt/fdt_common.h>
61 #include <dev/ofw/openfirm.h>
62 #include <dev/ofw/ofw_bus.h>
63 #include <dev/ofw/ofw_bus_subr.h>
64 
65 #include <dev/virtio/virtio.h>
66 #include <dev/virtio/virtqueue.h>
67 #include <dev/virtio/mmio/virtio_mmio.h>
68 
69 #include "virtio_mmio_if.h"
70 #include "virtio_bus_if.h"
71 #include "virtio_if.h"
72 
73 #define	PAGE_SHIFT	12
74 
75 struct vtmmio_virtqueue {
76 	struct virtqueue	*vtv_vq;
77 	int			 vtv_no_intr;
78 };
79 
80 struct vtmmio_softc {
81 	device_t			dev;
82 	device_t			platform;
83 	struct resource			*res[2];
84 
85 	uint64_t			vtmmio_features;
86 	uint32_t			vtmmio_flags;
87 
88 	/* This "bus" will only ever have one child. */
89 	device_t			vtmmio_child_dev;
90 	struct virtio_feature_desc	*vtmmio_child_feat_desc;
91 
92 	int				vtmmio_nvqs;
93 	struct vtmmio_virtqueue		*vtmmio_vqs;
94 	void				*ih;
95 };
96 
97 static int	vtmmio_probe(device_t);
98 static int	vtmmio_attach(device_t);
99 static int	vtmmio_detach(device_t);
100 static int	vtmmio_suspend(device_t);
101 static int	vtmmio_resume(device_t);
102 static int	vtmmio_shutdown(device_t);
103 static void	vtmmio_driver_added(device_t, driver_t *);
104 static void	vtmmio_child_detached(device_t, device_t);
105 static int	vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
106 static int	vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
107 static uint64_t	vtmmio_negotiate_features(device_t, uint64_t);
108 static int	vtmmio_with_feature(device_t, uint64_t);
109 static int	vtmmio_alloc_virtqueues(device_t, int, int,
110 		    struct vq_alloc_info *);
111 static int	vtmmio_setup_intr(device_t, enum intr_type);
112 static void	vtmmio_stop(device_t);
113 static int	vtmmio_reinit(device_t, uint64_t);
114 static void	vtmmio_reinit_complete(device_t);
115 static void	vtmmio_notify_virtqueue(device_t, uint16_t);
116 static uint8_t	vtmmio_get_status(device_t);
117 static void	vtmmio_set_status(device_t, uint8_t);
118 static void	vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
119 static void	vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
120 static void	vtmmio_describe_features(struct vtmmio_softc *, const char *,
121 		    uint64_t);
122 static void	vtmmio_probe_and_attach_child(struct vtmmio_softc *);
123 static int	vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
124 static void	vtmmio_free_interrupts(struct vtmmio_softc *);
125 static void	vtmmio_free_virtqueues(struct vtmmio_softc *);
126 static void	vtmmio_release_child_resources(struct vtmmio_softc *);
127 static void	vtmmio_reset(struct vtmmio_softc *);
128 static void	vtmmio_select_virtqueue(struct vtmmio_softc *, int);
129 static void	vtmmio_vq_intr(void *);
130 
131 /*
132  * I/O port read/write wrappers.
133  */
134 #define vtmmio_write_config_1(sc, o, v) \
135 	bus_write_1((sc)->res[0], (o), (v)); \
136 	VIRTIO_MMIO_NOTE(sc->platform, (o))
137 #define vtmmio_write_config_2(sc, o, v) \
138 	bus_write_2((sc)->res[0], (o), (v)); \
139 	VIRTIO_MMIO_NOTE(sc->platform, (o))
140 #define vtmmio_write_config_4(sc, o, v) \
141 	bus_write_4((sc)->res[0], (o), (v)); \
142 	VIRTIO_MMIO_NOTE(sc->platform, (o))
143 
144 #define vtmmio_read_config_1(sc, o) \
145 	bus_read_1((sc)->res[0], (o))
146 #define vtmmio_read_config_2(sc, o) \
147 	bus_read_2((sc)->res[0], (o))
148 #define vtmmio_read_config_4(sc, o) \
149 	bus_read_4((sc)->res[0], (o))
150 
151 static device_method_t vtmmio_methods[] = {
152 	/* Device interface. */
153 	DEVMETHOD(device_probe,			  vtmmio_probe),
154 	DEVMETHOD(device_attach,		  vtmmio_attach),
155 	DEVMETHOD(device_detach,		  vtmmio_detach),
156 	DEVMETHOD(device_suspend,		  vtmmio_suspend),
157 	DEVMETHOD(device_resume,		  vtmmio_resume),
158 	DEVMETHOD(device_shutdown,		  vtmmio_shutdown),
159 
160 	/* Bus interface. */
161 	DEVMETHOD(bus_driver_added,		  vtmmio_driver_added),
162 	DEVMETHOD(bus_child_detached,		  vtmmio_child_detached),
163 	DEVMETHOD(bus_read_ivar,		  vtmmio_read_ivar),
164 	DEVMETHOD(bus_write_ivar,		  vtmmio_write_ivar),
165 
166 	/* VirtIO bus interface. */
167 	DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
168 	DEVMETHOD(virtio_bus_with_feature,	  vtmmio_with_feature),
169 	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtmmio_alloc_virtqueues),
170 	DEVMETHOD(virtio_bus_setup_intr,	  vtmmio_setup_intr),
171 	DEVMETHOD(virtio_bus_stop,		  vtmmio_stop),
172 	DEVMETHOD(virtio_bus_reinit,		  vtmmio_reinit),
173 	DEVMETHOD(virtio_bus_reinit_complete,	  vtmmio_reinit_complete),
174 	DEVMETHOD(virtio_bus_notify_vq,		  vtmmio_notify_virtqueue),
175 	DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
176 	DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
177 
178 	DEVMETHOD_END
179 };
180 
181 static driver_t vtmmio_driver = {
182 	"virtio_mmio",
183 	vtmmio_methods,
184 	sizeof(struct vtmmio_softc)
185 };
186 
187 devclass_t vtmmio_devclass;
188 
189 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
190 MODULE_VERSION(virtio_mmio, 1);
191 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
192 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
193 
194 static int
195 vtmmio_setup_intr(device_t dev, enum intr_type type)
196 {
197 	struct vtmmio_softc *sc;
198 	int rid;
199 	int err;
200 
201 	sc = device_get_softc(dev);
202 
203 	err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
204 				vtmmio_vq_intr, sc);
205 	if (err == 0) {
206 		/* Okay we have backend-specific interrupts */
207 		return (0);
208 	}
209 
210 	rid = 0;
211 	sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
212 		RF_ACTIVE);
213 	if (!sc->res[1]) {
214 		device_printf(dev, "Can't allocate interrupt\n");
215 		return (ENXIO);
216 	}
217 
218 	if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
219 		NULL, vtmmio_vq_intr, sc, &sc->ih)) {
220 		device_printf(dev, "Can't setup the interrupt\n");
221 		return (ENXIO);
222 	}
223 
224 	return (0);
225 }
226 
227 static int
228 vtmmio_probe(device_t dev)
229 {
230 
231 	if (!ofw_bus_status_okay(dev))
232 		return (ENXIO);
233 
234 	if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
235 		return (ENXIO);
236 
237 	device_set_desc(dev, "VirtIO MMIO adapter");
238 	return (BUS_PROBE_DEFAULT);
239 }
240 
241 static int
242 vtmmio_setup_platform(struct vtmmio_softc *sc)
243 {
244 	phandle_t platform_node;
245 	struct fdt_ic *ic;
246 	phandle_t xref;
247 	phandle_t node;
248 
249 	sc->platform = NULL;
250 
251 	if ((node = ofw_bus_get_node(sc->dev)) == -1)
252 		return (ENXIO);
253 
254 	if (OF_searchencprop(node, "platform", &xref,
255 		sizeof(xref)) == -1) {
256 		return (ENXIO);
257 	}
258 
259 	platform_node = OF_node_from_xref(xref);
260 
261 	SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
262 		if (ic->iph == platform_node) {
263 			sc->platform = ic->dev;
264 			break;
265 		}
266 	}
267 
268 	if (sc->platform == NULL) {
269 		/* No platform-specific device. Ignore it. */
270 	}
271 
272 	return (0);
273 }
274 
275 static int
276 vtmmio_attach(device_t dev)
277 {
278 	struct vtmmio_softc *sc;
279 	device_t child;
280 	int rid;
281 
282 	sc = device_get_softc(dev);
283 	sc->dev = dev;
284 
285 	vtmmio_setup_platform(sc);
286 
287 	rid = 0;
288 	sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
289 			RF_ACTIVE);
290 	if (!sc->res[0]) {
291 		device_printf(dev, "Cannot allocate memory window.\n");
292 		return (ENXIO);
293 	}
294 
295 	vtmmio_reset(sc);
296 
297 	/* Tell the host we've noticed this device. */
298 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
299 
300 	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
301 		device_printf(dev, "Cannot create child device.\n");
302 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
303 		vtmmio_detach(dev);
304 		return (ENOMEM);
305 	}
306 
307 	sc->vtmmio_child_dev = child;
308 	vtmmio_probe_and_attach_child(sc);
309 
310 	return (0);
311 }
312 
313 static int
314 vtmmio_detach(device_t dev)
315 {
316 	struct vtmmio_softc *sc;
317 	device_t child;
318 	int error;
319 
320 	sc = device_get_softc(dev);
321 
322 	if ((child = sc->vtmmio_child_dev) != NULL) {
323 		error = device_delete_child(dev, child);
324 		if (error)
325 			return (error);
326 		sc->vtmmio_child_dev = NULL;
327 	}
328 
329 	vtmmio_reset(sc);
330 
331 	if (sc->res[0] != NULL) {
332 		bus_release_resource(dev, SYS_RES_MEMORY, 0,
333 		    sc->res[0]);
334 		sc->res[0] = NULL;
335 	}
336 
337 	return (0);
338 }
339 
340 static int
341 vtmmio_suspend(device_t dev)
342 {
343 
344 	return (bus_generic_suspend(dev));
345 }
346 
347 static int
348 vtmmio_resume(device_t dev)
349 {
350 
351 	return (bus_generic_resume(dev));
352 }
353 
354 static int
355 vtmmio_shutdown(device_t dev)
356 {
357 
358 	(void) bus_generic_shutdown(dev);
359 
360 	/* Forcibly stop the host device. */
361 	vtmmio_stop(dev);
362 
363 	return (0);
364 }
365 
366 static void
367 vtmmio_driver_added(device_t dev, driver_t *driver)
368 {
369 	struct vtmmio_softc *sc;
370 
371 	sc = device_get_softc(dev);
372 
373 	vtmmio_probe_and_attach_child(sc);
374 }
375 
376 static void
377 vtmmio_child_detached(device_t dev, device_t child)
378 {
379 	struct vtmmio_softc *sc;
380 
381 	sc = device_get_softc(dev);
382 
383 	vtmmio_reset(sc);
384 	vtmmio_release_child_resources(sc);
385 }
386 
387 static int
388 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
389 {
390 	struct vtmmio_softc *sc;
391 
392 	sc = device_get_softc(dev);
393 
394 	if (sc->vtmmio_child_dev != child)
395 		return (ENOENT);
396 
397 	switch (index) {
398 	case VIRTIO_IVAR_DEVTYPE:
399 	case VIRTIO_IVAR_SUBDEVICE:
400 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
401 		break;
402 	case VIRTIO_IVAR_VENDOR:
403 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
404 		break;
405 	default:
406 		return (ENOENT);
407 	}
408 
409 	return (0);
410 }
411 
412 static int
413 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
414 {
415 	struct vtmmio_softc *sc;
416 
417 	sc = device_get_softc(dev);
418 
419 	if (sc->vtmmio_child_dev != child)
420 		return (ENOENT);
421 
422 	switch (index) {
423 	case VIRTIO_IVAR_FEATURE_DESC:
424 		sc->vtmmio_child_feat_desc = (void *) value;
425 		break;
426 	default:
427 		return (ENOENT);
428 	}
429 
430 	return (0);
431 }
432 
433 static uint64_t
434 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
435 {
436 	struct vtmmio_softc *sc;
437 	uint64_t host_features, features;
438 
439 	sc = device_get_softc(dev);
440 
441 	host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
442 	vtmmio_describe_features(sc, "host", host_features);
443 
444 	/*
445 	 * Limit negotiated features to what the driver, virtqueue, and
446 	 * host all support.
447 	 */
448 	features = host_features & child_features;
449 	features = virtqueue_filter_features(features);
450 	sc->vtmmio_features = features;
451 
452 	vtmmio_describe_features(sc, "negotiated", features);
453 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
454 
455 	return (features);
456 }
457 
458 static int
459 vtmmio_with_feature(device_t dev, uint64_t feature)
460 {
461 	struct vtmmio_softc *sc;
462 
463 	sc = device_get_softc(dev);
464 
465 	return ((sc->vtmmio_features & feature) != 0);
466 }
467 
468 static int
469 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
470     struct vq_alloc_info *vq_info)
471 {
472 	struct vtmmio_virtqueue *vqx;
473 	struct vq_alloc_info *info;
474 	struct vtmmio_softc *sc;
475 	struct virtqueue *vq;
476 	int idx, error;
477 	uint16_t size;
478 
479 	sc = device_get_softc(dev);
480 
481 	if (sc->vtmmio_nvqs != 0)
482 		return (EALREADY);
483 	if (nvqs <= 0)
484 		return (EINVAL);
485 
486 	sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
487 	    M_DEVBUF, M_NOWAIT | M_ZERO);
488 	if (sc->vtmmio_vqs == NULL)
489 		return (ENOMEM);
490 
491 	for (idx = 0; idx < nvqs; idx++) {
492 		vqx = &sc->vtmmio_vqs[idx];
493 		info = &vq_info[idx];
494 
495 		vtmmio_select_virtqueue(sc, idx);
496 		size = vtmmio_read_config_2(sc, VIRTIO_MMIO_QUEUE_NUM);
497 
498 		error = virtqueue_alloc(dev, idx, size,
499 		    VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
500 		if (error) {
501 			device_printf(dev,
502 			    "cannot allocate virtqueue %d: %d\n",
503 			    idx, error);
504 			break;
505 		}
506 #if 0
507 		device_printf(dev, "virtqueue paddr 0x%08lx\n",
508 				(uint64_t)virtqueue_paddr(vq));
509 #endif
510 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
511 			virtqueue_paddr(vq) >> PAGE_SHIFT);
512 
513 		vqx->vtv_vq = *info->vqai_vq = vq;
514 		vqx->vtv_no_intr = info->vqai_intr == NULL;
515 
516 		sc->vtmmio_nvqs++;
517 	}
518 
519 	if (error)
520 		vtmmio_free_virtqueues(sc);
521 
522 	return (error);
523 }
524 
525 static void
526 vtmmio_stop(device_t dev)
527 {
528 
529 	vtmmio_reset(device_get_softc(dev));
530 }
531 
532 static int
533 vtmmio_reinit(device_t dev, uint64_t features)
534 {
535 	struct vtmmio_softc *sc;
536 	int idx, error;
537 
538 	sc = device_get_softc(dev);
539 
540 	if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
541 		vtmmio_stop(dev);
542 
543 	/*
544 	 * Quickly drive the status through ACK and DRIVER. The device
545 	 * does not become usable again until vtmmio_reinit_complete().
546 	 */
547 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
548 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
549 
550 	vtmmio_negotiate_features(dev, features);
551 
552 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
553 		error = vtmmio_reinit_virtqueue(sc, idx);
554 		if (error)
555 			return (error);
556 	}
557 
558 	return (0);
559 }
560 
561 static void
562 vtmmio_reinit_complete(device_t dev)
563 {
564 
565 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
566 }
567 
568 static void
569 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
570 {
571 	struct vtmmio_softc *sc;
572 
573 	sc = device_get_softc(dev);
574 
575 	vtmmio_write_config_2(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
576 }
577 
578 static uint8_t
579 vtmmio_get_status(device_t dev)
580 {
581 	struct vtmmio_softc *sc;
582 
583 	sc = device_get_softc(dev);
584 
585 	return (vtmmio_read_config_1(sc, VIRTIO_MMIO_STATUS));
586 }
587 
588 static void
589 vtmmio_set_status(device_t dev, uint8_t status)
590 {
591 	struct vtmmio_softc *sc;
592 
593 	sc = device_get_softc(dev);
594 
595 	if (status != VIRTIO_CONFIG_STATUS_RESET)
596 		status |= vtmmio_get_status(dev);
597 
598 	vtmmio_write_config_1(sc, VIRTIO_MMIO_STATUS, status);
599 }
600 
601 static void
602 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
603     void *dst, int length)
604 {
605 	struct vtmmio_softc *sc;
606 	bus_size_t off;
607 	uint8_t *d;
608 	int size;
609 
610 	sc = device_get_softc(dev);
611 	off = VIRTIO_MMIO_CONFIG + offset;
612 
613 	for (d = dst; length > 0; d += size, off += size, length -= size) {
614 		if (length >= 4) {
615 			size = 4;
616 			*(uint32_t *)d = vtmmio_read_config_4(sc, off);
617 		} else if (length >= 2) {
618 			size = 2;
619 			*(uint16_t *)d = vtmmio_read_config_2(sc, off);
620 		} else {
621 			size = 1;
622 			*d = vtmmio_read_config_1(sc, off);
623 		}
624 	}
625 }
626 
627 static void
628 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
629     void *src, int length)
630 {
631 	struct vtmmio_softc *sc;
632 	bus_size_t off;
633 	uint8_t *s;
634 	int size;
635 
636 	sc = device_get_softc(dev);
637 	off = VIRTIO_MMIO_CONFIG + offset;
638 
639 	for (s = src; length > 0; s += size, off += size, length -= size) {
640 		if (length >= 4) {
641 			size = 4;
642 			vtmmio_write_config_4(sc, off, *(uint32_t *)s);
643 		} else if (length >= 2) {
644 			size = 2;
645 			vtmmio_write_config_2(sc, off, *(uint16_t *)s);
646 		} else {
647 			size = 1;
648 			vtmmio_write_config_1(sc, off, *s);
649 		}
650 	}
651 }
652 
653 static void
654 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
655     uint64_t features)
656 {
657 	device_t dev, child;
658 
659 	dev = sc->dev;
660 	child = sc->vtmmio_child_dev;
661 
662 	if (device_is_attached(child) && bootverbose == 0)
663 		return;
664 
665 	virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
666 }
667 
668 static void
669 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
670 {
671 	device_t dev, child;
672 
673 	dev = sc->dev;
674 	child = sc->vtmmio_child_dev;
675 
676 	if (child == NULL)
677 		return;
678 
679 	if (device_get_state(child) != DS_NOTPRESENT) {
680 		return;
681 	}
682 
683 	if (device_probe(child) != 0) {
684 		return;
685 	}
686 
687 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
688 	if (device_attach(child) != 0) {
689 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
690 		vtmmio_reset(sc);
691 		vtmmio_release_child_resources(sc);
692 		/* Reset status for future attempt. */
693 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
694 	} else {
695 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
696 		VIRTIO_ATTACH_COMPLETED(child);
697 	}
698 }
699 
700 static int
701 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
702 {
703 	struct vtmmio_virtqueue *vqx;
704 	struct virtqueue *vq;
705 	int error;
706 	uint16_t size;
707 
708 	vqx = &sc->vtmmio_vqs[idx];
709 	vq = vqx->vtv_vq;
710 
711 	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
712 
713 	vtmmio_select_virtqueue(sc, idx);
714 	size = vtmmio_read_config_2(sc, VIRTIO_MMIO_QUEUE_NUM);
715 
716 	error = virtqueue_reinit(vq, size);
717 	if (error)
718 		return (error);
719 
720 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
721 	    virtqueue_paddr(vq) >> PAGE_SHIFT);
722 
723 	return (0);
724 }
725 
726 static void
727 vtmmio_free_interrupts(struct vtmmio_softc *sc)
728 {
729 
730 	if (sc->ih != NULL)
731 		bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
732 
733 	if (sc->res[1] != NULL)
734 		bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
735 }
736 
737 static void
738 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
739 {
740 	struct vtmmio_virtqueue *vqx;
741 	int idx;
742 
743 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
744 		vqx = &sc->vtmmio_vqs[idx];
745 
746 		vtmmio_select_virtqueue(sc, idx);
747 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
748 
749 		virtqueue_free(vqx->vtv_vq);
750 		vqx->vtv_vq = NULL;
751 	}
752 
753 	free(sc->vtmmio_vqs, M_DEVBUF);
754 	sc->vtmmio_vqs = NULL;
755 	sc->vtmmio_nvqs = 0;
756 }
757 
758 static void
759 vtmmio_release_child_resources(struct vtmmio_softc *sc)
760 {
761 
762 	vtmmio_free_interrupts(sc);
763 	vtmmio_free_virtqueues(sc);
764 }
765 
766 static void
767 vtmmio_reset(struct vtmmio_softc *sc)
768 {
769 
770 	/*
771 	 * Setting the status to RESET sets the host device to
772 	 * the original, uninitialized state.
773 	 */
774 	vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
775 }
776 
777 static void
778 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
779 {
780 
781 	vtmmio_write_config_2(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
782 }
783 
784 static void
785 vtmmio_vq_intr(void *arg)
786 {
787 	struct vtmmio_virtqueue *vqx;
788 	struct vtmmio_softc *sc;
789 	struct virtqueue *vq;
790 	int idx;
791 
792 	sc = arg;
793 
794 	/* Notify all virtqueues. */
795 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
796 		vqx = &sc->vtmmio_vqs[idx];
797 		vq = vqx->vtv_vq;
798 		virtqueue_intr(vq);
799 	};
800 }
801