xref: /freebsd/sys/dev/virtio/mmio/virtio_mmio.c (revision 190cef3d52236565eb22e18b33e9e865ec634aa3)
1 /*-
2  * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3  * Copyright (c) 2014 The FreeBSD Foundation
4  * All rights reserved.
5  *
6  * This software was developed by SRI International and the University of
7  * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8  * ("CTSRD"), as part of the DARPA CRASH research programme.
9  *
10  * Portions of this software were developed by Andrew Turner
11  * under sponsorship from the FreeBSD Foundation.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34 
35 /*
36  * VirtIO MMIO interface.
37  * This driver is heavily based on VirtIO PCI interface driver.
38  */
39 
40 /*
41  * FDT example:
42  *		virtio_block@1000 {
43  *			compatible = "virtio,mmio";
44  *			reg = <0x1000 0x100>;
45  *			interrupts = <63>;
46  *			interrupt-parent = <&GIC>;
47  *		};
48  */
49 
50 #include <sys/cdefs.h>
51 __FBSDID("$FreeBSD$");
52 
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/bus.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/malloc.h>
59 #include <sys/rman.h>
60 
61 #include <machine/bus.h>
62 #include <machine/resource.h>
63 
64 #include <dev/fdt/fdt_common.h>
65 #include <dev/ofw/openfirm.h>
66 #include <dev/ofw/ofw_bus.h>
67 #include <dev/ofw/ofw_bus_subr.h>
68 
69 #include <dev/virtio/virtio.h>
70 #include <dev/virtio/virtqueue.h>
71 #include <dev/virtio/mmio/virtio_mmio.h>
72 
73 #include "virtio_mmio_if.h"
74 #include "virtio_bus_if.h"
75 #include "virtio_if.h"
76 
77 #define	PAGE_SHIFT	12
78 
79 struct vtmmio_virtqueue {
80 	struct virtqueue	*vtv_vq;
81 	int			 vtv_no_intr;
82 };
83 
84 struct vtmmio_softc {
85 	device_t			dev;
86 	device_t			platform;
87 	struct resource			*res[2];
88 
89 	uint64_t			vtmmio_features;
90 	uint32_t			vtmmio_flags;
91 
92 	/* This "bus" will only ever have one child. */
93 	device_t			vtmmio_child_dev;
94 	struct virtio_feature_desc	*vtmmio_child_feat_desc;
95 
96 	int				vtmmio_nvqs;
97 	struct vtmmio_virtqueue		*vtmmio_vqs;
98 	void				*ih;
99 };
100 
101 static int	vtmmio_probe(device_t);
102 static int	vtmmio_attach(device_t);
103 static int	vtmmio_detach(device_t);
104 static int	vtmmio_suspend(device_t);
105 static int	vtmmio_resume(device_t);
106 static int	vtmmio_shutdown(device_t);
107 static void	vtmmio_driver_added(device_t, driver_t *);
108 static void	vtmmio_child_detached(device_t, device_t);
109 static int	vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
110 static int	vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
111 static uint64_t	vtmmio_negotiate_features(device_t, uint64_t);
112 static int	vtmmio_with_feature(device_t, uint64_t);
113 static int	vtmmio_alloc_virtqueues(device_t, int, int,
114 		    struct vq_alloc_info *);
115 static int	vtmmio_setup_intr(device_t, enum intr_type);
116 static void	vtmmio_stop(device_t);
117 static void	vtmmio_poll(device_t);
118 static int	vtmmio_reinit(device_t, uint64_t);
119 static void	vtmmio_reinit_complete(device_t);
120 static void	vtmmio_notify_virtqueue(device_t, uint16_t);
121 static uint8_t	vtmmio_get_status(device_t);
122 static void	vtmmio_set_status(device_t, uint8_t);
123 static void	vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
124 static void	vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
125 static void	vtmmio_describe_features(struct vtmmio_softc *, const char *,
126 		    uint64_t);
127 static void	vtmmio_probe_and_attach_child(struct vtmmio_softc *);
128 static int	vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
129 static void	vtmmio_free_interrupts(struct vtmmio_softc *);
130 static void	vtmmio_free_virtqueues(struct vtmmio_softc *);
131 static void	vtmmio_release_child_resources(struct vtmmio_softc *);
132 static void	vtmmio_reset(struct vtmmio_softc *);
133 static void	vtmmio_select_virtqueue(struct vtmmio_softc *, int);
134 static void	vtmmio_vq_intr(void *);
135 
136 /*
137  * I/O port read/write wrappers.
138  */
139 #define vtmmio_write_config_1(sc, o, v)				\
140 do {								\
141 	if (sc->platform != NULL)				\
142 		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
143 	bus_write_1((sc)->res[0], (o), (v)); 			\
144 	if (sc->platform != NULL)				\
145 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
146 } while (0)
147 #define vtmmio_write_config_2(sc, o, v)				\
148 do {								\
149 	if (sc->platform != NULL)				\
150 		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
151 	bus_write_2((sc)->res[0], (o), (v));			\
152 	if (sc->platform != NULL)				\
153 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
154 } while (0)
155 #define vtmmio_write_config_4(sc, o, v)				\
156 do {								\
157 	if (sc->platform != NULL)				\
158 		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
159 	bus_write_4((sc)->res[0], (o), (v));			\
160 	if (sc->platform != NULL)				\
161 		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
162 } while (0)
163 
164 #define vtmmio_read_config_1(sc, o) \
165 	bus_read_1((sc)->res[0], (o))
166 #define vtmmio_read_config_2(sc, o) \
167 	bus_read_2((sc)->res[0], (o))
168 #define vtmmio_read_config_4(sc, o) \
169 	bus_read_4((sc)->res[0], (o))
170 
171 static device_method_t vtmmio_methods[] = {
172 	/* Device interface. */
173 	DEVMETHOD(device_probe,			  vtmmio_probe),
174 	DEVMETHOD(device_attach,		  vtmmio_attach),
175 	DEVMETHOD(device_detach,		  vtmmio_detach),
176 	DEVMETHOD(device_suspend,		  vtmmio_suspend),
177 	DEVMETHOD(device_resume,		  vtmmio_resume),
178 	DEVMETHOD(device_shutdown,		  vtmmio_shutdown),
179 
180 	/* Bus interface. */
181 	DEVMETHOD(bus_driver_added,		  vtmmio_driver_added),
182 	DEVMETHOD(bus_child_detached,		  vtmmio_child_detached),
183 	DEVMETHOD(bus_read_ivar,		  vtmmio_read_ivar),
184 	DEVMETHOD(bus_write_ivar,		  vtmmio_write_ivar),
185 
186 	/* VirtIO bus interface. */
187 	DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
188 	DEVMETHOD(virtio_bus_with_feature,	  vtmmio_with_feature),
189 	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtmmio_alloc_virtqueues),
190 	DEVMETHOD(virtio_bus_setup_intr,	  vtmmio_setup_intr),
191 	DEVMETHOD(virtio_bus_stop,		  vtmmio_stop),
192 	DEVMETHOD(virtio_bus_poll,		  vtmmio_poll),
193 	DEVMETHOD(virtio_bus_reinit,		  vtmmio_reinit),
194 	DEVMETHOD(virtio_bus_reinit_complete,	  vtmmio_reinit_complete),
195 	DEVMETHOD(virtio_bus_notify_vq,		  vtmmio_notify_virtqueue),
196 	DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
197 	DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
198 
199 	DEVMETHOD_END
200 };
201 
202 static driver_t vtmmio_driver = {
203 	"virtio_mmio",
204 	vtmmio_methods,
205 	sizeof(struct vtmmio_softc)
206 };
207 
208 devclass_t vtmmio_devclass;
209 
210 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
211 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
212 MODULE_VERSION(virtio_mmio, 1);
213 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
214 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
215 
216 static int
217 vtmmio_setup_intr(device_t dev, enum intr_type type)
218 {
219 	struct vtmmio_softc *sc;
220 	int rid;
221 	int err;
222 
223 	sc = device_get_softc(dev);
224 
225 	if (sc->platform != NULL) {
226 		err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
227 					vtmmio_vq_intr, sc);
228 		if (err == 0) {
229 			/* Okay we have backend-specific interrupts */
230 			return (0);
231 		}
232 	}
233 
234 	rid = 0;
235 	sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
236 		RF_ACTIVE);
237 	if (!sc->res[1]) {
238 		device_printf(dev, "Can't allocate interrupt\n");
239 		return (ENXIO);
240 	}
241 
242 	if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
243 		NULL, vtmmio_vq_intr, sc, &sc->ih)) {
244 		device_printf(dev, "Can't setup the interrupt\n");
245 		return (ENXIO);
246 	}
247 
248 	return (0);
249 }
250 
251 static int
252 vtmmio_probe(device_t dev)
253 {
254 
255 	if (!ofw_bus_status_okay(dev))
256 		return (ENXIO);
257 
258 	if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
259 		return (ENXIO);
260 
261 	device_set_desc(dev, "VirtIO MMIO adapter");
262 	return (BUS_PROBE_DEFAULT);
263 }
264 
265 static int
266 vtmmio_setup_platform(struct vtmmio_softc *sc)
267 {
268 	phandle_t platform_node;
269 	struct fdt_ic *ic;
270 	phandle_t xref;
271 	phandle_t node;
272 
273 	sc->platform = NULL;
274 
275 	if ((node = ofw_bus_get_node(sc->dev)) == -1)
276 		return (ENXIO);
277 
278 	if (OF_searchencprop(node, "platform", &xref,
279 		sizeof(xref)) == -1) {
280 		return (ENXIO);
281 	}
282 
283 	platform_node = OF_node_from_xref(xref);
284 
285 	SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
286 		if (ic->iph == platform_node) {
287 			sc->platform = ic->dev;
288 			break;
289 		}
290 	}
291 
292 	if (sc->platform == NULL) {
293 		/* No platform-specific device. Ignore it. */
294 	}
295 
296 	return (0);
297 }
298 
299 static int
300 vtmmio_attach(device_t dev)
301 {
302 	struct vtmmio_softc *sc;
303 	device_t child;
304 	int rid;
305 
306 	sc = device_get_softc(dev);
307 	sc->dev = dev;
308 
309 	vtmmio_setup_platform(sc);
310 
311 	rid = 0;
312 	sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
313 			RF_ACTIVE);
314 	if (!sc->res[0]) {
315 		device_printf(dev, "Cannot allocate memory window.\n");
316 		return (ENXIO);
317 	}
318 
319 	vtmmio_reset(sc);
320 
321 	/* Tell the host we've noticed this device. */
322 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
323 
324 	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
325 		device_printf(dev, "Cannot create child device.\n");
326 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
327 		vtmmio_detach(dev);
328 		return (ENOMEM);
329 	}
330 
331 	sc->vtmmio_child_dev = child;
332 	vtmmio_probe_and_attach_child(sc);
333 
334 	return (0);
335 }
336 
337 static int
338 vtmmio_detach(device_t dev)
339 {
340 	struct vtmmio_softc *sc;
341 	device_t child;
342 	int error;
343 
344 	sc = device_get_softc(dev);
345 
346 	if ((child = sc->vtmmio_child_dev) != NULL) {
347 		error = device_delete_child(dev, child);
348 		if (error)
349 			return (error);
350 		sc->vtmmio_child_dev = NULL;
351 	}
352 
353 	vtmmio_reset(sc);
354 
355 	if (sc->res[0] != NULL) {
356 		bus_release_resource(dev, SYS_RES_MEMORY, 0,
357 		    sc->res[0]);
358 		sc->res[0] = NULL;
359 	}
360 
361 	return (0);
362 }
363 
364 static int
365 vtmmio_suspend(device_t dev)
366 {
367 
368 	return (bus_generic_suspend(dev));
369 }
370 
371 static int
372 vtmmio_resume(device_t dev)
373 {
374 
375 	return (bus_generic_resume(dev));
376 }
377 
378 static int
379 vtmmio_shutdown(device_t dev)
380 {
381 
382 	(void) bus_generic_shutdown(dev);
383 
384 	/* Forcibly stop the host device. */
385 	vtmmio_stop(dev);
386 
387 	return (0);
388 }
389 
390 static void
391 vtmmio_driver_added(device_t dev, driver_t *driver)
392 {
393 	struct vtmmio_softc *sc;
394 
395 	sc = device_get_softc(dev);
396 
397 	vtmmio_probe_and_attach_child(sc);
398 }
399 
400 static void
401 vtmmio_child_detached(device_t dev, device_t child)
402 {
403 	struct vtmmio_softc *sc;
404 
405 	sc = device_get_softc(dev);
406 
407 	vtmmio_reset(sc);
408 	vtmmio_release_child_resources(sc);
409 }
410 
411 static int
412 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
413 {
414 	struct vtmmio_softc *sc;
415 
416 	sc = device_get_softc(dev);
417 
418 	if (sc->vtmmio_child_dev != child)
419 		return (ENOENT);
420 
421 	switch (index) {
422 	case VIRTIO_IVAR_DEVTYPE:
423 	case VIRTIO_IVAR_SUBDEVICE:
424 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
425 		break;
426 	case VIRTIO_IVAR_VENDOR:
427 		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
428 		break;
429 	default:
430 		return (ENOENT);
431 	}
432 
433 	return (0);
434 }
435 
436 static int
437 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
438 {
439 	struct vtmmio_softc *sc;
440 
441 	sc = device_get_softc(dev);
442 
443 	if (sc->vtmmio_child_dev != child)
444 		return (ENOENT);
445 
446 	switch (index) {
447 	case VIRTIO_IVAR_FEATURE_DESC:
448 		sc->vtmmio_child_feat_desc = (void *) value;
449 		break;
450 	default:
451 		return (ENOENT);
452 	}
453 
454 	return (0);
455 }
456 
457 static uint64_t
458 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
459 {
460 	struct vtmmio_softc *sc;
461 	uint64_t host_features, features;
462 
463 	sc = device_get_softc(dev);
464 
465 	host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
466 	vtmmio_describe_features(sc, "host", host_features);
467 
468 	/*
469 	 * Limit negotiated features to what the driver, virtqueue, and
470 	 * host all support.
471 	 */
472 	features = host_features & child_features;
473 	features = virtqueue_filter_features(features);
474 	sc->vtmmio_features = features;
475 
476 	vtmmio_describe_features(sc, "negotiated", features);
477 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
478 
479 	return (features);
480 }
481 
482 static int
483 vtmmio_with_feature(device_t dev, uint64_t feature)
484 {
485 	struct vtmmio_softc *sc;
486 
487 	sc = device_get_softc(dev);
488 
489 	return ((sc->vtmmio_features & feature) != 0);
490 }
491 
492 static int
493 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
494     struct vq_alloc_info *vq_info)
495 {
496 	struct vtmmio_virtqueue *vqx;
497 	struct vq_alloc_info *info;
498 	struct vtmmio_softc *sc;
499 	struct virtqueue *vq;
500 	uint32_t size;
501 	int idx, error;
502 
503 	sc = device_get_softc(dev);
504 
505 	if (sc->vtmmio_nvqs != 0)
506 		return (EALREADY);
507 	if (nvqs <= 0)
508 		return (EINVAL);
509 
510 	sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
511 	    M_DEVBUF, M_NOWAIT | M_ZERO);
512 	if (sc->vtmmio_vqs == NULL)
513 		return (ENOMEM);
514 
515 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
516 	    (1 << PAGE_SHIFT));
517 
518 	for (idx = 0; idx < nvqs; idx++) {
519 		vqx = &sc->vtmmio_vqs[idx];
520 		info = &vq_info[idx];
521 
522 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
523 
524 		vtmmio_select_virtqueue(sc, idx);
525 		size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
526 
527 		error = virtqueue_alloc(dev, idx, size,
528 		    VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
529 		if (error) {
530 			device_printf(dev,
531 			    "cannot allocate virtqueue %d: %d\n",
532 			    idx, error);
533 			break;
534 		}
535 
536 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
537 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
538 		    VIRTIO_MMIO_VRING_ALIGN);
539 #if 0
540 		device_printf(dev, "virtqueue paddr 0x%08lx\n",
541 		    (uint64_t)virtqueue_paddr(vq));
542 #endif
543 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
544 		    virtqueue_paddr(vq) >> PAGE_SHIFT);
545 
546 		vqx->vtv_vq = *info->vqai_vq = vq;
547 		vqx->vtv_no_intr = info->vqai_intr == NULL;
548 
549 		sc->vtmmio_nvqs++;
550 	}
551 
552 	if (error)
553 		vtmmio_free_virtqueues(sc);
554 
555 	return (error);
556 }
557 
558 static void
559 vtmmio_stop(device_t dev)
560 {
561 
562 	vtmmio_reset(device_get_softc(dev));
563 }
564 
565 static void
566 vtmmio_poll(device_t dev)
567 {
568 	struct vtmmio_softc *sc;
569 
570 	sc = device_get_softc(dev);
571 
572 	if (sc->platform != NULL)
573 		VIRTIO_MMIO_POLL(sc->platform);
574 }
575 
576 static int
577 vtmmio_reinit(device_t dev, uint64_t features)
578 {
579 	struct vtmmio_softc *sc;
580 	int idx, error;
581 
582 	sc = device_get_softc(dev);
583 
584 	if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
585 		vtmmio_stop(dev);
586 
587 	/*
588 	 * Quickly drive the status through ACK and DRIVER. The device
589 	 * does not become usable again until vtmmio_reinit_complete().
590 	 */
591 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
592 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
593 
594 	vtmmio_negotiate_features(dev, features);
595 
596 	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
597 	    (1 << PAGE_SHIFT));
598 
599 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
600 		error = vtmmio_reinit_virtqueue(sc, idx);
601 		if (error)
602 			return (error);
603 	}
604 
605 	return (0);
606 }
607 
608 static void
609 vtmmio_reinit_complete(device_t dev)
610 {
611 
612 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
613 }
614 
615 static void
616 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
617 {
618 	struct vtmmio_softc *sc;
619 
620 	sc = device_get_softc(dev);
621 
622 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
623 }
624 
625 static uint8_t
626 vtmmio_get_status(device_t dev)
627 {
628 	struct vtmmio_softc *sc;
629 
630 	sc = device_get_softc(dev);
631 
632 	return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
633 }
634 
635 static void
636 vtmmio_set_status(device_t dev, uint8_t status)
637 {
638 	struct vtmmio_softc *sc;
639 
640 	sc = device_get_softc(dev);
641 
642 	if (status != VIRTIO_CONFIG_STATUS_RESET)
643 		status |= vtmmio_get_status(dev);
644 
645 	vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
646 }
647 
648 static void
649 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
650     void *dst, int length)
651 {
652 	struct vtmmio_softc *sc;
653 	bus_size_t off;
654 	uint8_t *d;
655 	int size;
656 
657 	sc = device_get_softc(dev);
658 	off = VIRTIO_MMIO_CONFIG + offset;
659 
660 	for (d = dst; length > 0; d += size, off += size, length -= size) {
661 #ifdef ALLOW_WORD_ALIGNED_ACCESS
662 		if (length >= 4) {
663 			size = 4;
664 			*(uint32_t *)d = vtmmio_read_config_4(sc, off);
665 		} else if (length >= 2) {
666 			size = 2;
667 			*(uint16_t *)d = vtmmio_read_config_2(sc, off);
668 		} else
669 #endif
670 		{
671 			size = 1;
672 			*d = vtmmio_read_config_1(sc, off);
673 		}
674 	}
675 }
676 
677 static void
678 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
679     void *src, int length)
680 {
681 	struct vtmmio_softc *sc;
682 	bus_size_t off;
683 	uint8_t *s;
684 	int size;
685 
686 	sc = device_get_softc(dev);
687 	off = VIRTIO_MMIO_CONFIG + offset;
688 
689 	for (s = src; length > 0; s += size, off += size, length -= size) {
690 #ifdef ALLOW_WORD_ALIGNED_ACCESS
691 		if (length >= 4) {
692 			size = 4;
693 			vtmmio_write_config_4(sc, off, *(uint32_t *)s);
694 		} else if (length >= 2) {
695 			size = 2;
696 			vtmmio_write_config_2(sc, off, *(uint16_t *)s);
697 		} else
698 #endif
699 		{
700 			size = 1;
701 			vtmmio_write_config_1(sc, off, *s);
702 		}
703 	}
704 }
705 
706 static void
707 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
708     uint64_t features)
709 {
710 	device_t dev, child;
711 
712 	dev = sc->dev;
713 	child = sc->vtmmio_child_dev;
714 
715 	if (device_is_attached(child) || bootverbose == 0)
716 		return;
717 
718 	virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
719 }
720 
721 static void
722 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
723 {
724 	device_t dev, child;
725 
726 	dev = sc->dev;
727 	child = sc->vtmmio_child_dev;
728 
729 	if (child == NULL)
730 		return;
731 
732 	if (device_get_state(child) != DS_NOTPRESENT) {
733 		return;
734 	}
735 
736 	if (device_probe(child) != 0) {
737 		return;
738 	}
739 
740 	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
741 	if (device_attach(child) != 0) {
742 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
743 		vtmmio_reset(sc);
744 		vtmmio_release_child_resources(sc);
745 		/* Reset status for future attempt. */
746 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
747 	} else {
748 		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
749 		VIRTIO_ATTACH_COMPLETED(child);
750 	}
751 }
752 
753 static int
754 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
755 {
756 	struct vtmmio_virtqueue *vqx;
757 	struct virtqueue *vq;
758 	int error;
759 	uint16_t size;
760 
761 	vqx = &sc->vtmmio_vqs[idx];
762 	vq = vqx->vtv_vq;
763 
764 	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
765 
766 	vtmmio_select_virtqueue(sc, idx);
767 	size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
768 
769 	error = virtqueue_reinit(vq, size);
770 	if (error)
771 		return (error);
772 
773 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
774 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
775 	    VIRTIO_MMIO_VRING_ALIGN);
776 #if 0
777 	device_printf(sc->dev, "virtqueue paddr 0x%08lx\n",
778 	    (uint64_t)virtqueue_paddr(vq));
779 #endif
780 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
781 	    virtqueue_paddr(vq) >> PAGE_SHIFT);
782 
783 	return (0);
784 }
785 
786 static void
787 vtmmio_free_interrupts(struct vtmmio_softc *sc)
788 {
789 
790 	if (sc->ih != NULL)
791 		bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
792 
793 	if (sc->res[1] != NULL)
794 		bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
795 }
796 
797 static void
798 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
799 {
800 	struct vtmmio_virtqueue *vqx;
801 	int idx;
802 
803 	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
804 		vqx = &sc->vtmmio_vqs[idx];
805 
806 		vtmmio_select_virtqueue(sc, idx);
807 		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
808 
809 		virtqueue_free(vqx->vtv_vq);
810 		vqx->vtv_vq = NULL;
811 	}
812 
813 	free(sc->vtmmio_vqs, M_DEVBUF);
814 	sc->vtmmio_vqs = NULL;
815 	sc->vtmmio_nvqs = 0;
816 }
817 
818 static void
819 vtmmio_release_child_resources(struct vtmmio_softc *sc)
820 {
821 
822 	vtmmio_free_interrupts(sc);
823 	vtmmio_free_virtqueues(sc);
824 }
825 
826 static void
827 vtmmio_reset(struct vtmmio_softc *sc)
828 {
829 
830 	/*
831 	 * Setting the status to RESET sets the host device to
832 	 * the original, uninitialized state.
833 	 */
834 	vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
835 }
836 
837 static void
838 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
839 {
840 
841 	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
842 }
843 
844 static void
845 vtmmio_vq_intr(void *arg)
846 {
847 	struct vtmmio_virtqueue *vqx;
848 	struct vtmmio_softc *sc;
849 	struct virtqueue *vq;
850 	uint32_t status;
851 	int idx;
852 
853 	sc = arg;
854 
855 	status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
856 	vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
857 
858 	/* The config changed */
859 	if (status & VIRTIO_MMIO_INT_CONFIG)
860 		if (sc->vtmmio_child_dev != NULL)
861 			VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
862 
863 	/* Notify all virtqueues. */
864 	if (status & VIRTIO_MMIO_INT_VRING) {
865 		for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
866 			vqx = &sc->vtmmio_vqs[idx];
867 			if (vqx->vtv_no_intr == 0) {
868 				vq = vqx->vtv_vq;
869 				virtqueue_intr(vq);
870 			}
871 		}
872 	}
873 }
874