xref: /freebsd/sys/dev/virtio/mmio/virtio_mmio.c (revision 952364486a4b9d135e4b28f7f88a8703a74eae6f)
1 /*-
2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3  * Copyright (c) 2014 The FreeBSD Foundation
4  * All rights reserved.
5  *
6  * This software was developed by SRI International and the University of
7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8  * ("CTSRD"), as part of the DARPA CRASH research programme.
9  *
10  * Portions of this software were developed by Andrew Turner
11  * under sponsorship from the FreeBSD Foundation.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 
35 /*
36  * VirtIO MMIO interface.
37  * This driver is heavily based on VirtIO PCI interface driver.
38  */
39 
40 /*
41  * FDT example:
42  *		virtio_block@1000 {
43  *			compatible = "virtio,mmio";
44  *			reg = <0x1000 0x100>;
45  *			interrupts = <63>;
46  *			interrupt-parent = <&GIC>;
47  *		};
48  */
49 
50 #include <sys/cdefs.h>
51 __FBSDID("$FreeBSD$");
52 
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/bus.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/malloc.h>
59 #include <sys/rman.h>
60 
61 #include <machine/bus.h>
62 #include <machine/resource.h>
63 
64 #include <dev/fdt/fdt_common.h>
65 #include <dev/ofw/openfirm.h>
66 #include <dev/ofw/ofw_bus.h>
67 #include <dev/ofw/ofw_bus_subr.h>
68 
69 #include <dev/virtio/virtio.h>
70 #include <dev/virtio/virtqueue.h>
71 #include <dev/virtio/mmio/virtio_mmio.h>
72 
73 #include "virtio_mmio_if.h"
74 #include "virtio_bus_if.h"
75 #include "virtio_if.h"
76 
77 #define	PAGE_SHIFT	12
78 
79 struct vtmmio_virtqueue {
80 	struct virtqueue	*vtv_vq;
81 	int			 vtv_no_intr;
82 };
83 
84 struct vtmmio_softc {
85 	device_t			dev;
86 	device_t			platform;
87 	struct resource			*res[2];
88 
89 	uint64_t			vtmmio_features;
90 	uint32_t			vtmmio_flags;
91 
92 	/* This "bus" will only ever have one child. */
93 	device_t			vtmmio_child_dev;
94 	struct virtio_feature_desc	*vtmmio_child_feat_desc;
95 
96 	int				vtmmio_nvqs;
97 	struct vtmmio_virtqueue		*vtmmio_vqs;
98 	void				*ih;
99 };
100 
101 static int	vtmmio_probe(device_t);
102 static int	vtmmio_attach(device_t);
103 static int	vtmmio_detach(device_t);
104 static int	vtmmio_suspend(device_t);
105 static int	vtmmio_resume(device_t);
106 static int	vtmmio_shutdown(device_t);
107 static void	vtmmio_driver_added(device_t, driver_t *);
108 static void	vtmmio_child_detached(device_t, device_t);
109 static int	vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
110 static int	vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
111 static uint64_t	vtmmio_negotiate_features(device_t, uint64_t);
112 static int	vtmmio_with_feature(device_t, uint64_t);
113 static int	vtmmio_alloc_virtqueues(device_t, int, int,
114 		    struct vq_alloc_info *);
115 static int	vtmmio_setup_intr(device_t, enum intr_type);
116 static void	vtmmio_stop(device_t);
117 static void	vtmmio_poll(device_t);
118 static int	vtmmio_reinit(device_t, uint64_t);
119 static void	vtmmio_reinit_complete(device_t);
120 static void	vtmmio_notify_virtqueue(device_t, uint16_t);
121 static uint8_t	vtmmio_get_status(device_t);
122 static void	vtmmio_set_status(device_t, uint8_t);
123 static void	vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
124 static void	vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
125 static void	vtmmio_describe_features(struct vtmmio_softc *, const char *,
126 		    uint64_t);
127 static void	vtmmio_probe_and_attach_child(struct vtmmio_softc *);
128 static int	vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
129 static void	vtmmio_free_interrupts(struct vtmmio_softc *);
130 static void	vtmmio_free_virtqueues(struct vtmmio_softc *);
131 static void	vtmmio_release_child_resources(struct vtmmio_softc *);
132 static void	vtmmio_reset(struct vtmmio_softc *);
133 static void	vtmmio_select_virtqueue(struct vtmmio_softc *, int);
134 static void	vtmmio_vq_intr(void *);
135 
136 /*
137  * I/O port read/write wrappers.
138  */
139 #define vtmmio_write_config_1(sc, o, v)				\
140 do {								\
141 	bus_write_1((sc)->res[0], (o), (v)); 			\
142 	if (sc->platform != NULL)				\
143 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
144 } while (0)
145 #define vtmmio_write_config_2(sc, o, v)				\
146 do {								\
147 	bus_write_2((sc)->res[0], (o), (v));			\
148 	if (sc->platform != NULL)				\
149 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
150 } while (0)
151 #define vtmmio_write_config_4(sc, o, v)				\
152 do {								\
153 	bus_write_4((sc)->res[0], (o), (v));			\
154 	if (sc->platform != NULL)				\
155 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
156 } while (0)
157 
158 #define vtmmio_read_config_1(sc, o) \
159 	bus_read_1((sc)->res[0], (o))
160 #define vtmmio_read_config_2(sc, o) \
161 	bus_read_2((sc)->res[0], (o))
162 #define vtmmio_read_config_4(sc, o) \
163 	bus_read_4((sc)->res[0], (o))
164 
165 static device_method_t vtmmio_methods[] = {
166 	/* Device interface. */
167 	DEVMETHOD(device_probe,			  vtmmio_probe),
168 	DEVMETHOD(device_attach,		  vtmmio_attach),
169 	DEVMETHOD(device_detach,		  vtmmio_detach),
170 	DEVMETHOD(device_suspend,		  vtmmio_suspend),
171 	DEVMETHOD(device_resume,		  vtmmio_resume),
172 	DEVMETHOD(device_shutdown,		  vtmmio_shutdown),
173 
174 	/* Bus interface. */
175 	DEVMETHOD(bus_driver_added,		  vtmmio_driver_added),
176 	DEVMETHOD(bus_child_detached,		  vtmmio_child_detached),
177 	DEVMETHOD(bus_read_ivar,		  vtmmio_read_ivar),
178 	DEVMETHOD(bus_write_ivar,		  vtmmio_write_ivar),
179 
180 	/* VirtIO bus interface. */
181 	DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
182 	DEVMETHOD(virtio_bus_with_feature,	  vtmmio_with_feature),
183 	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtmmio_alloc_virtqueues),
184 	DEVMETHOD(virtio_bus_setup_intr,	  vtmmio_setup_intr),
185 	DEVMETHOD(virtio_bus_stop,		  vtmmio_stop),
186 	DEVMETHOD(virtio_bus_poll,		  vtmmio_poll),
187 	DEVMETHOD(virtio_bus_reinit,		  vtmmio_reinit),
188 	DEVMETHOD(virtio_bus_reinit_complete,	  vtmmio_reinit_complete),
189 	DEVMETHOD(virtio_bus_notify_vq,		  vtmmio_notify_virtqueue),
190 	DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
191 	DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
192 
193 	DEVMETHOD_END
194 };
195 
196 static driver_t vtmmio_driver = {
197 	"virtio_mmio",
198 	vtmmio_methods,
199 	sizeof(struct vtmmio_softc)
200 };
201 
202 devclass_t vtmmio_devclass;
203 
204 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
205 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
206 MODULE_VERSION(virtio_mmio, 1);
207 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
208 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
209 
210 static int
211 vtmmio_setup_intr(device_t dev, enum intr_type type)
212 {
213 	struct vtmmio_softc *sc;
214 	int rid;
215 	int err;
216 
217 	sc = device_get_softc(dev);
218 
219 	if (sc->platform != NULL) {
220 		err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
221 					vtmmio_vq_intr, sc);
222 		if (err == 0) {
223 			/* Okay we have backend-specific interrupts */
224 			return (0);
225 		}
226 	}
227 
228 	rid = 0;
229 	sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
230 		RF_ACTIVE);
231 	if (!sc->res[1]) {
232 		device_printf(dev, "Can't allocate interrupt\n");
233 		return (ENXIO);
234 	}
235 
236 	if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
237 		NULL, vtmmio_vq_intr, sc, &sc->ih)) {
238 		device_printf(dev, "Can't setup the interrupt\n");
239 		return (ENXIO);
240 	}
241 
242 	return (0);
243 }
244 
245 static int
246 vtmmio_probe(device_t dev)
247 {
248 
249 	if (!ofw_bus_status_okay(dev))
250 		return (ENXIO);
251 
252 	if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
253 		return (ENXIO);
254 
255 	device_set_desc(dev, "VirtIO MMIO adapter");
256 	return (BUS_PROBE_DEFAULT);
257 }
258 
259 static int
260 vtmmio_setup_platform(struct vtmmio_softc *sc)
261 {
262 	phandle_t platform_node;
263 	struct fdt_ic *ic;
264 	phandle_t xref;
265 	phandle_t node;
266 
267 	sc->platform = NULL;
268 
269 	if ((node = ofw_bus_get_node(sc->dev)) == -1)
270 		return (ENXIO);
271 
272 	if (OF_searchencprop(node, "platform", &xref,
273 		sizeof(xref)) == -1) {
274 		return (ENXIO);
275 	}
276 
277 	platform_node = OF_node_from_xref(xref);
278 
279 	SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
280 		if (ic->iph == platform_node) {
281 			sc->platform = ic->dev;
282 			break;
283 		}
284 	}
285 
286 	if (sc->platform == NULL) {
287 		/* No platform-specific device. Ignore it. */
288 	}
289 
290 	return (0);
291 }
292 
293 static int
294 vtmmio_attach(device_t dev)
295 {
296 	struct vtmmio_softc *sc;
297 	device_t child;
298 	int rid;
299 
300 	sc = device_get_softc(dev);
301 	sc->dev = dev;
302 
303 	vtmmio_setup_platform(sc);
304 
305 	rid = 0;
306 	sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
307 			RF_ACTIVE);
308 	if (!sc->res[0]) {
309 		device_printf(dev, "Cannot allocate memory window.\n");
310 		return (ENXIO);
311 	}
312 
313 	vtmmio_reset(sc);
314 
315 	/* Tell the host we've noticed this device. */
316 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
317 
318 	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
319 		device_printf(dev, "Cannot create child device.\n");
320 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
321 		vtmmio_detach(dev);
322 		return (ENOMEM);
323 	}
324 
325 	sc->vtmmio_child_dev = child;
326 	vtmmio_probe_and_attach_child(sc);
327 
328 	return (0);
329 }
330 
331 static int
332 vtmmio_detach(device_t dev)
333 {
334 	struct vtmmio_softc *sc;
335 	device_t child;
336 	int error;
337 
338 	sc = device_get_softc(dev);
339 
340 	if ((child = sc->vtmmio_child_dev) != NULL) {
341 		error = device_delete_child(dev, child);
342 		if (error)
343 			return (error);
344 		sc->vtmmio_child_dev = NULL;
345 	}
346 
347 	vtmmio_reset(sc);
348 
349 	if (sc->res[0] != NULL) {
350 		bus_release_resource(dev, SYS_RES_MEMORY, 0,
351 		    sc->res[0]);
352 		sc->res[0] = NULL;
353 	}
354 
355 	return (0);
356 }
357 
358 static int
359 vtmmio_suspend(device_t dev)
360 {
361 
362 	return (bus_generic_suspend(dev));
363 }
364 
365 static int
366 vtmmio_resume(device_t dev)
367 {
368 
369 	return (bus_generic_resume(dev));
370 }
371 
372 static int
373 vtmmio_shutdown(device_t dev)
374 {
375 
376 	(void) bus_generic_shutdown(dev);
377 
378 	/* Forcibly stop the host device. */
379 	vtmmio_stop(dev);
380 
381 	return (0);
382 }
383 
384 static void
385 vtmmio_driver_added(device_t dev, driver_t *driver)
386 {
387 	struct vtmmio_softc *sc;
388 
389 	sc = device_get_softc(dev);
390 
391 	vtmmio_probe_and_attach_child(sc);
392 }
393 
394 static void
395 vtmmio_child_detached(device_t dev, device_t child)
396 {
397 	struct vtmmio_softc *sc;
398 
399 	sc = device_get_softc(dev);
400 
401 	vtmmio_reset(sc);
402 	vtmmio_release_child_resources(sc);
403 }
404 
405 static int
406 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
407 {
408 	struct vtmmio_softc *sc;
409 
410 	sc = device_get_softc(dev);
411 
412 	if (sc->vtmmio_child_dev != child)
413 		return (ENOENT);
414 
415 	switch (index) {
416 	case VIRTIO_IVAR_DEVTYPE:
417 	case VIRTIO_IVAR_SUBDEVICE:
418 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
419 		break;
420 	case VIRTIO_IVAR_VENDOR:
421 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
422 		break;
423 	default:
424 		return (ENOENT);
425 	}
426 
427 	return (0);
428 }
429 
430 static int
431 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
432 {
433 	struct vtmmio_softc *sc;
434 
435 	sc = device_get_softc(dev);
436 
437 	if (sc->vtmmio_child_dev != child)
438 		return (ENOENT);
439 
440 	switch (index) {
441 	case VIRTIO_IVAR_FEATURE_DESC:
442 		sc->vtmmio_child_feat_desc = (void *) value;
443 		break;
444 	default:
445 		return (ENOENT);
446 	}
447 
448 	return (0);
449 }
450 
451 static uint64_t
452 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
453 {
454 	struct vtmmio_softc *sc;
455 	uint64_t host_features, features;
456 
457 	sc = device_get_softc(dev);
458 
459 	host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
460 	vtmmio_describe_features(sc, "host", host_features);
461 
462 	/*
463 	 * Limit negotiated features to what the driver, virtqueue, and
464 	 * host all support.
465 	 */
466 	features = host_features & child_features;
467 	features = virtqueue_filter_features(features);
468 	sc->vtmmio_features = features;
469 
470 	vtmmio_describe_features(sc, "negotiated", features);
471 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
472 
473 	return (features);
474 }
475 
476 static int
477 vtmmio_with_feature(device_t dev, uint64_t feature)
478 {
479 	struct vtmmio_softc *sc;
480 
481 	sc = device_get_softc(dev);
482 
483 	return ((sc->vtmmio_features & feature) != 0);
484 }
485 
486 static int
487 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
488     struct vq_alloc_info *vq_info)
489 {
490 	struct vtmmio_virtqueue *vqx;
491 	struct vq_alloc_info *info;
492 	struct vtmmio_softc *sc;
493 	struct virtqueue *vq;
494 	uint32_t size;
495 	int idx, error;
496 
497 	sc = device_get_softc(dev);
498 
499 	if (sc->vtmmio_nvqs != 0)
500 		return (EALREADY);
501 	if (nvqs <= 0)
502 		return (EINVAL);
503 
504 	sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
505 	    M_DEVBUF, M_NOWAIT | M_ZERO);
506 	if (sc->vtmmio_vqs == NULL)
507 		return (ENOMEM);
508 
509 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT);
510 
511 	for (idx = 0; idx < nvqs; idx++) {
512 		vqx = &sc->vtmmio_vqs[idx];
513 		info = &vq_info[idx];
514 
515 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
516 
517 		vtmmio_select_virtqueue(sc, idx);
518 		size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
519 
520 		error = virtqueue_alloc(dev, idx, size,
521 		    VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
522 		if (error) {
523 			device_printf(dev,
524 			    "cannot allocate virtqueue %d: %d\n",
525 			    idx, error);
526 			break;
527 		}
528 
529 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
530 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
531 		    VIRTIO_MMIO_VRING_ALIGN);
532 #if 0
533 		device_printf(dev, "virtqueue paddr 0x%08lx\n",
534 				(uint64_t)virtqueue_paddr(vq));
535 #endif
536 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
537 			virtqueue_paddr(vq) >> PAGE_SHIFT);
538 
539 		vqx->vtv_vq = *info->vqai_vq = vq;
540 		vqx->vtv_no_intr = info->vqai_intr == NULL;
541 
542 		sc->vtmmio_nvqs++;
543 	}
544 
545 	if (error)
546 		vtmmio_free_virtqueues(sc);
547 
548 	return (error);
549 }
550 
551 static void
552 vtmmio_stop(device_t dev)
553 {
554 
555 	vtmmio_reset(device_get_softc(dev));
556 }
557 
558 static void
559 vtmmio_poll(device_t dev)
560 {
561 	struct vtmmio_softc *sc;
562 
563 	sc = device_get_softc(dev);
564 
565 	if (sc->platform != NULL)
566 		VIRTIO_MMIO_POLL(sc->platform);
567 }
568 
569 static int
570 vtmmio_reinit(device_t dev, uint64_t features)
571 {
572 	struct vtmmio_softc *sc;
573 	int idx, error;
574 
575 	sc = device_get_softc(dev);
576 
577 	if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
578 		vtmmio_stop(dev);
579 
580 	/*
581 	 * Quickly drive the status through ACK and DRIVER. The device
582 	 * does not become usable again until vtmmio_reinit_complete().
583 	 */
584 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
585 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
586 
587 	vtmmio_negotiate_features(dev, features);
588 
589 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
590 		error = vtmmio_reinit_virtqueue(sc, idx);
591 		if (error)
592 			return (error);
593 	}
594 
595 	return (0);
596 }
597 
598 static void
599 vtmmio_reinit_complete(device_t dev)
600 {
601 
602 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
603 }
604 
605 static void
606 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
607 {
608 	struct vtmmio_softc *sc;
609 
610 	sc = device_get_softc(dev);
611 
612 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
613 }
614 
615 static uint8_t
616 vtmmio_get_status(device_t dev)
617 {
618 	struct vtmmio_softc *sc;
619 
620 	sc = device_get_softc(dev);
621 
622 	return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
623 }
624 
625 static void
626 vtmmio_set_status(device_t dev, uint8_t status)
627 {
628 	struct vtmmio_softc *sc;
629 
630 	sc = device_get_softc(dev);
631 
632 	if (status != VIRTIO_CONFIG_STATUS_RESET)
633 		status |= vtmmio_get_status(dev);
634 
635 	vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
636 }
637 
638 static void
639 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
640     void *dst, int length)
641 {
642 	struct vtmmio_softc *sc;
643 	bus_size_t off;
644 	uint8_t *d;
645 	int size;
646 
647 	sc = device_get_softc(dev);
648 	off = VIRTIO_MMIO_CONFIG + offset;
649 
650 	for (d = dst; length > 0; d += size, off += size, length -= size) {
651 #ifdef ALLOW_WORD_ALIGNED_ACCESS
652 		if (length >= 4) {
653 			size = 4;
654 			*(uint32_t *)d = vtmmio_read_config_4(sc, off);
655 		} else if (length >= 2) {
656 			size = 2;
657 			*(uint16_t *)d = vtmmio_read_config_2(sc, off);
658 		} else
659 #endif
660 		{
661 			size = 1;
662 			*d = vtmmio_read_config_1(sc, off);
663 		}
664 	}
665 }
666 
667 static void
668 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
669     void *src, int length)
670 {
671 	struct vtmmio_softc *sc;
672 	bus_size_t off;
673 	uint8_t *s;
674 	int size;
675 
676 	sc = device_get_softc(dev);
677 	off = VIRTIO_MMIO_CONFIG + offset;
678 
679 	for (s = src; length > 0; s += size, off += size, length -= size) {
680 #ifdef ALLOW_WORD_ALIGNED_ACCESS
681 		if (length >= 4) {
682 			size = 4;
683 			vtmmio_write_config_4(sc, off, *(uint32_t *)s);
684 		} else if (length >= 2) {
685 			size = 2;
686 			vtmmio_write_config_2(sc, off, *(uint16_t *)s);
687 		} else
688 #endif
689 		{
690 			size = 1;
691 			vtmmio_write_config_1(sc, off, *s);
692 		}
693 	}
694 }
695 
696 static void
697 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
698     uint64_t features)
699 {
700 	device_t dev, child;
701 
702 	dev = sc->dev;
703 	child = sc->vtmmio_child_dev;
704 
705 	if (device_is_attached(child) || bootverbose == 0)
706 		return;
707 
708 	virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
709 }
710 
711 static void
712 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
713 {
714 	device_t dev, child;
715 
716 	dev = sc->dev;
717 	child = sc->vtmmio_child_dev;
718 
719 	if (child == NULL)
720 		return;
721 
722 	if (device_get_state(child) != DS_NOTPRESENT) {
723 		return;
724 	}
725 
726 	if (device_probe(child) != 0) {
727 		return;
728 	}
729 
730 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
731 	if (device_attach(child) != 0) {
732 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
733 		vtmmio_reset(sc);
734 		vtmmio_release_child_resources(sc);
735 		/* Reset status for future attempt. */
736 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
737 	} else {
738 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
739 		VIRTIO_ATTACH_COMPLETED(child);
740 	}
741 }
742 
743 static int
744 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
745 {
746 	struct vtmmio_virtqueue *vqx;
747 	struct virtqueue *vq;
748 	int error;
749 	uint16_t size;
750 
751 	vqx = &sc->vtmmio_vqs[idx];
752 	vq = vqx->vtv_vq;
753 
754 	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
755 
756 	vtmmio_select_virtqueue(sc, idx);
757 	size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
758 
759 	error = virtqueue_reinit(vq, size);
760 	if (error)
761 		return (error);
762 
763 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
764 	    virtqueue_paddr(vq) >> PAGE_SHIFT);
765 
766 	return (0);
767 }
768 
769 static void
770 vtmmio_free_interrupts(struct vtmmio_softc *sc)
771 {
772 
773 	if (sc->ih != NULL)
774 		bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
775 
776 	if (sc->res[1] != NULL)
777 		bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
778 }
779 
780 static void
781 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
782 {
783 	struct vtmmio_virtqueue *vqx;
784 	int idx;
785 
786 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
787 		vqx = &sc->vtmmio_vqs[idx];
788 
789 		vtmmio_select_virtqueue(sc, idx);
790 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
791 
792 		virtqueue_free(vqx->vtv_vq);
793 		vqx->vtv_vq = NULL;
794 	}
795 
796 	free(sc->vtmmio_vqs, M_DEVBUF);
797 	sc->vtmmio_vqs = NULL;
798 	sc->vtmmio_nvqs = 0;
799 }
800 
801 static void
802 vtmmio_release_child_resources(struct vtmmio_softc *sc)
803 {
804 
805 	vtmmio_free_interrupts(sc);
806 	vtmmio_free_virtqueues(sc);
807 }
808 
809 static void
810 vtmmio_reset(struct vtmmio_softc *sc)
811 {
812 
813 	/*
814 	 * Setting the status to RESET sets the host device to
815 	 * the original, uninitialized state.
816 	 */
817 	vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
818 }
819 
820 static void
821 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
822 {
823 
824 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
825 }
826 
827 static void
828 vtmmio_vq_intr(void *arg)
829 {
830 	struct vtmmio_virtqueue *vqx;
831 	struct vtmmio_softc *sc;
832 	struct virtqueue *vq;
833 	uint32_t status;
834 	int idx;
835 
836 	sc = arg;
837 
838 	status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
839 	vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
840 
841 	/* The config changed */
842 	if (status & VIRTIO_MMIO_INT_CONFIG)
843 		if (sc->vtmmio_child_dev != NULL)
844 			VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
845 
846 	/* Notify all virtqueues. */
847 	if (status & VIRTIO_MMIO_INT_VRING) {
848 		for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
849 			vqx = &sc->vtmmio_vqs[idx];
850 			if (vqx->vtv_no_intr == 0) {
851 				vq = vqx->vtv_vq;
852 				virtqueue_intr(vq);
853 			}
854 		}
855 	}
856 }
857