xref: /freebsd/sys/dev/virtio/pci/virtio_pci.c (revision 9e5787d2284e187abb5b654d924394a65772e004)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for the VirtIO PCI interface. */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/bus.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/malloc.h>
40 #include <sys/endian.h>
41 
42 #include <machine/bus.h>
43 #include <machine/resource.h>
44 #include <sys/bus.h>
45 #include <sys/rman.h>
46 
47 #include <dev/pci/pcivar.h>
48 #include <dev/pci/pcireg.h>
49 
50 #include <dev/virtio/virtio.h>
51 #include <dev/virtio/virtqueue.h>
52 #include <dev/virtio/pci/virtio_pci.h>
53 
54 #include "virtio_bus_if.h"
55 #include "virtio_if.h"
56 
57 struct vtpci_interrupt {
58 	struct resource		*vti_irq;
59 	int			 vti_rid;
60 	void			*vti_handler;
61 };
62 
63 struct vtpci_virtqueue {
64 	struct virtqueue	*vtv_vq;
65 	int			 vtv_no_intr;
66 };
67 
68 struct vtpci_softc {
69 	device_t			 vtpci_dev;
70 	struct resource			*vtpci_res;
71 	struct resource			*vtpci_msix_res;
72 	uint64_t			 vtpci_features;
73 	uint32_t			 vtpci_flags;
74 #define VTPCI_FLAG_NO_MSI		0x0001
75 #define VTPCI_FLAG_NO_MSIX		0x0002
76 #define VTPCI_FLAG_LEGACY		0x1000
77 #define VTPCI_FLAG_MSI			0x2000
78 #define VTPCI_FLAG_MSIX			0x4000
79 #define VTPCI_FLAG_SHARED_MSIX		0x8000
80 #define VTPCI_FLAG_ITYPE_MASK		0xF000
81 
82 	/* This "bus" will only ever have one child. */
83 	device_t			 vtpci_child_dev;
84 	struct virtio_feature_desc	*vtpci_child_feat_desc;
85 
86 	int				 vtpci_nvqs;
87 	struct vtpci_virtqueue		*vtpci_vqs;
88 
89 	/*
90 	 * Ideally, each virtqueue that the driver provides a callback for will
91 	 * receive its own MSIX vector. If there are not sufficient vectors
92 	 * available, then attempt to have all the VQs share one vector. For
93 	 * MSIX, the configuration changed notifications must be on their own
94 	 * vector.
95 	 *
96 	 * If MSIX is not available, we will attempt to have the whole device
97 	 * share one MSI vector, and then, finally, one legacy interrupt.
98 	 */
99 	struct vtpci_interrupt		 vtpci_device_interrupt;
100 	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
101 	int				 vtpci_nmsix_resources;
102 };
103 
104 static int	vtpci_probe(device_t);
105 static int	vtpci_attach(device_t);
106 static int	vtpci_detach(device_t);
107 static int	vtpci_suspend(device_t);
108 static int	vtpci_resume(device_t);
109 static int	vtpci_shutdown(device_t);
110 static void	vtpci_driver_added(device_t, driver_t *);
111 static void	vtpci_child_detached(device_t, device_t);
112 static int	vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
113 static int	vtpci_write_ivar(device_t, device_t, int, uintptr_t);
114 
115 static uint64_t	vtpci_negotiate_features(device_t, uint64_t);
116 static int	vtpci_with_feature(device_t, uint64_t);
117 static int	vtpci_alloc_virtqueues(device_t, int, int,
118 		    struct vq_alloc_info *);
119 static int	vtpci_setup_intr(device_t, enum intr_type);
120 static void	vtpci_stop(device_t);
121 static int	vtpci_reinit(device_t, uint64_t);
122 static void	vtpci_reinit_complete(device_t);
123 static void	vtpci_notify_virtqueue(device_t, uint16_t);
124 static uint8_t	vtpci_get_status(device_t);
125 static void	vtpci_set_status(device_t, uint8_t);
126 static void	vtpci_read_dev_config(device_t, bus_size_t, void *, int);
127 static void	vtpci_write_dev_config(device_t, bus_size_t, void *, int);
128 
129 static void	vtpci_describe_features(struct vtpci_softc *, const char *,
130 		    uint64_t);
131 static void	vtpci_probe_and_attach_child(struct vtpci_softc *);
132 
133 static int	vtpci_alloc_msix(struct vtpci_softc *, int);
134 static int	vtpci_alloc_msi(struct vtpci_softc *);
135 static int	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
136 static int	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
137 static int	vtpci_alloc_intr_msi(struct vtpci_softc *);
138 static int	vtpci_alloc_intr_legacy(struct vtpci_softc *);
139 static int	vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
140 		    struct vtpci_interrupt *);
141 static int	vtpci_alloc_intr_resources(struct vtpci_softc *);
142 
143 static int	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
144 		    enum intr_type);
145 static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
146 		    enum intr_type);
147 static int	vtpci_setup_msix_interrupts(struct vtpci_softc *,
148 		    enum intr_type);
149 static int	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
150 
151 static int	vtpci_register_msix_vector(struct vtpci_softc *, int,
152 		    struct vtpci_interrupt *);
153 static int	vtpci_set_host_msix_vectors(struct vtpci_softc *);
154 static int	vtpci_reinit_virtqueue(struct vtpci_softc *, int);
155 
156 static void	vtpci_free_interrupt(struct vtpci_softc *,
157 		    struct vtpci_interrupt *);
158 static void	vtpci_free_interrupts(struct vtpci_softc *);
159 static void	vtpci_free_virtqueues(struct vtpci_softc *);
160 static void	vtpci_release_child_resources(struct vtpci_softc *);
161 static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
162 static void	vtpci_reset(struct vtpci_softc *);
163 
164 static void	vtpci_select_virtqueue(struct vtpci_softc *, int);
165 
166 static void	vtpci_legacy_intr(void *);
167 static int	vtpci_vq_shared_intr_filter(void *);
168 static void	vtpci_vq_shared_intr(void *);
169 static int	vtpci_vq_intr_filter(void *);
170 static void	vtpci_vq_intr(void *);
171 static void	vtpci_config_intr(void *);
172 
173 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
174 
175 #define VIRTIO_PCI_CONFIG(_sc) \
176     VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
177 
178 /*
179  * I/O port read/write wrappers.
180  */
181 #define vtpci_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
182 #define vtpci_write_config_1(sc, o, v)	bus_write_1((sc)->vtpci_res, (o), (v))
183 
184 /*
185  * Virtio-pci specifies that PCI Configuration area is guest endian. However,
186  * since PCI devices are inherently little-endian, on BE systems the bus layer
187  * transparently converts it to BE. For virtio-legacy, this conversion is
188  * undesired, so an extra byte swap is required to fix it.
189  */
190 #define vtpci_read_config_2(sc, o)      le16toh(bus_read_2((sc)->vtpci_res, (o)))
191 #define vtpci_read_config_4(sc, o)      le32toh(bus_read_4((sc)->vtpci_res, (o)))
192 #define vtpci_write_config_2(sc, o, v)  bus_write_2((sc)->vtpci_res, (o), (htole16(v)))
193 #define vtpci_write_config_4(sc, o, v)  bus_write_4((sc)->vtpci_res, (o), (htole32(v)))
194 
195 /* PCI Header LE. On BE systems the bus layer takes care of byte swapping */
196 #define vtpci_read_header_2(sc, o)      (bus_read_2((sc)->vtpci_res, (o)))
197 #define vtpci_read_header_4(sc, o)      (bus_read_4((sc)->vtpci_res, (o)))
198 #define vtpci_write_header_2(sc, o, v)  bus_write_2((sc)->vtpci_res, (o), (v))
199 #define vtpci_write_header_4(sc, o, v)  bus_write_4((sc)->vtpci_res, (o), (v))
200 
201 /* Tunables. */
202 static int vtpci_disable_msix = 0;
203 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
204 
205 static device_method_t vtpci_methods[] = {
206 	/* Device interface. */
207 	DEVMETHOD(device_probe,			  vtpci_probe),
208 	DEVMETHOD(device_attach,		  vtpci_attach),
209 	DEVMETHOD(device_detach,		  vtpci_detach),
210 	DEVMETHOD(device_suspend,		  vtpci_suspend),
211 	DEVMETHOD(device_resume,		  vtpci_resume),
212 	DEVMETHOD(device_shutdown,		  vtpci_shutdown),
213 
214 	/* Bus interface. */
215 	DEVMETHOD(bus_driver_added,		  vtpci_driver_added),
216 	DEVMETHOD(bus_child_detached,		  vtpci_child_detached),
217 	DEVMETHOD(bus_child_pnpinfo_str,	  virtio_child_pnpinfo_str),
218 	DEVMETHOD(bus_read_ivar,		  vtpci_read_ivar),
219 	DEVMETHOD(bus_write_ivar,		  vtpci_write_ivar),
220 
221 	/* VirtIO bus interface. */
222 	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_negotiate_features),
223 	DEVMETHOD(virtio_bus_with_feature,	  vtpci_with_feature),
224 	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_alloc_virtqueues),
225 	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_setup_intr),
226 	DEVMETHOD(virtio_bus_stop,		  vtpci_stop),
227 	DEVMETHOD(virtio_bus_reinit,		  vtpci_reinit),
228 	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_reinit_complete),
229 	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_notify_virtqueue),
230 	DEVMETHOD(virtio_bus_read_device_config,  vtpci_read_dev_config),
231 	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
232 
233 	DEVMETHOD_END
234 };
235 
236 static driver_t vtpci_driver = {
237 	"virtio_pci",
238 	vtpci_methods,
239 	sizeof(struct vtpci_softc)
240 };
241 
242 devclass_t vtpci_devclass;
243 
244 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
245 MODULE_VERSION(virtio_pci, 1);
246 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
247 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
248 
249 static int
250 vtpci_probe(device_t dev)
251 {
252 	char desc[36];
253 	const char *name;
254 
255 	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
256 		return (ENXIO);
257 
258 	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
259 	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
260 		return (ENXIO);
261 
262 	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
263 		return (ENXIO);
264 
265 	name = virtio_device_name(pci_get_subdevice(dev));
266 	if (name == NULL)
267 		name = "Unknown";
268 
269 	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
270 	device_set_desc_copy(dev, desc);
271 
272 	return (BUS_PROBE_DEFAULT);
273 }
274 
275 static int
276 vtpci_attach(device_t dev)
277 {
278 	struct vtpci_softc *sc;
279 	device_t child;
280 	int rid;
281 
282 	sc = device_get_softc(dev);
283 	sc->vtpci_dev = dev;
284 
285 	pci_enable_busmaster(dev);
286 
287 	rid = PCIR_BAR(0);
288 	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
289 	    RF_ACTIVE);
290 	if (sc->vtpci_res == NULL) {
291 		device_printf(dev, "cannot map I/O space\n");
292 		return (ENXIO);
293 	}
294 
295 	if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
296 		sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
297 
298 	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
299 		rid = PCIR_BAR(1);
300 		sc->vtpci_msix_res = bus_alloc_resource_any(dev,
301 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
302 	}
303 
304 	if (sc->vtpci_msix_res == NULL)
305 		sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
306 
307 	vtpci_reset(sc);
308 
309 	/* Tell the host we've noticed this device. */
310 	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
311 
312 	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
313 		device_printf(dev, "cannot create child device\n");
314 		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
315 		vtpci_detach(dev);
316 		return (ENOMEM);
317 	}
318 
319 	sc->vtpci_child_dev = child;
320 	vtpci_probe_and_attach_child(sc);
321 
322 	return (0);
323 }
324 
325 static int
326 vtpci_detach(device_t dev)
327 {
328 	struct vtpci_softc *sc;
329 	device_t child;
330 	int error;
331 
332 	sc = device_get_softc(dev);
333 
334 	if ((child = sc->vtpci_child_dev) != NULL) {
335 		error = device_delete_child(dev, child);
336 		if (error)
337 			return (error);
338 		sc->vtpci_child_dev = NULL;
339 	}
340 
341 	vtpci_reset(sc);
342 
343 	if (sc->vtpci_msix_res != NULL) {
344 		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
345 		    sc->vtpci_msix_res);
346 		sc->vtpci_msix_res = NULL;
347 	}
348 
349 	if (sc->vtpci_res != NULL) {
350 		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
351 		    sc->vtpci_res);
352 		sc->vtpci_res = NULL;
353 	}
354 
355 	return (0);
356 }
357 
358 static int
359 vtpci_suspend(device_t dev)
360 {
361 
362 	return (bus_generic_suspend(dev));
363 }
364 
365 static int
366 vtpci_resume(device_t dev)
367 {
368 
369 	return (bus_generic_resume(dev));
370 }
371 
372 static int
373 vtpci_shutdown(device_t dev)
374 {
375 
376 	(void) bus_generic_shutdown(dev);
377 	/* Forcibly stop the host device. */
378 	vtpci_stop(dev);
379 
380 	return (0);
381 }
382 
383 static void
384 vtpci_driver_added(device_t dev, driver_t *driver)
385 {
386 	struct vtpci_softc *sc;
387 
388 	sc = device_get_softc(dev);
389 
390 	vtpci_probe_and_attach_child(sc);
391 }
392 
393 static void
394 vtpci_child_detached(device_t dev, device_t child)
395 {
396 	struct vtpci_softc *sc;
397 
398 	sc = device_get_softc(dev);
399 
400 	vtpci_reset(sc);
401 	vtpci_release_child_resources(sc);
402 }
403 
404 static int
405 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
406 {
407 	struct vtpci_softc *sc;
408 
409 	sc = device_get_softc(dev);
410 
411 	if (sc->vtpci_child_dev != child)
412 		return (ENOENT);
413 
414 	switch (index) {
415 	case VIRTIO_IVAR_DEVTYPE:
416 	case VIRTIO_IVAR_SUBDEVICE:
417 		*result = pci_get_subdevice(dev);
418 		break;
419 	case VIRTIO_IVAR_VENDOR:
420 		*result = pci_get_vendor(dev);
421 		break;
422 	case VIRTIO_IVAR_DEVICE:
423 		*result = pci_get_device(dev);
424 		break;
425 	case VIRTIO_IVAR_SUBVENDOR:
426 		*result = pci_get_subvendor(dev);
427 		break;
428 	default:
429 		return (ENOENT);
430 	}
431 
432 	return (0);
433 }
434 
435 static int
436 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
437 {
438 	struct vtpci_softc *sc;
439 
440 	sc = device_get_softc(dev);
441 
442 	if (sc->vtpci_child_dev != child)
443 		return (ENOENT);
444 
445 	switch (index) {
446 	case VIRTIO_IVAR_FEATURE_DESC:
447 		sc->vtpci_child_feat_desc = (void *) value;
448 		break;
449 	default:
450 		return (ENOENT);
451 	}
452 
453 	return (0);
454 }
455 
456 static uint64_t
457 vtpci_negotiate_features(device_t dev, uint64_t child_features)
458 {
459 	struct vtpci_softc *sc;
460 	uint64_t host_features, features;
461 
462 	sc = device_get_softc(dev);
463 
464 	host_features = vtpci_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES);
465 	vtpci_describe_features(sc, "host", host_features);
466 
467 	/*
468 	 * Limit negotiated features to what the driver, virtqueue, and
469 	 * host all support.
470 	 */
471 	features = host_features & child_features;
472 	features = virtqueue_filter_features(features);
473 	sc->vtpci_features = features;
474 
475 	vtpci_describe_features(sc, "negotiated", features);
476 	vtpci_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
477 
478 	return (features);
479 }
480 
481 static int
482 vtpci_with_feature(device_t dev, uint64_t feature)
483 {
484 	struct vtpci_softc *sc;
485 
486 	sc = device_get_softc(dev);
487 
488 	return ((sc->vtpci_features & feature) != 0);
489 }
490 
491 static int
492 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
493     struct vq_alloc_info *vq_info)
494 {
495 	struct vtpci_softc *sc;
496 	struct virtqueue *vq;
497 	struct vtpci_virtqueue *vqx;
498 	struct vq_alloc_info *info;
499 	int idx, error;
500 	uint16_t size;
501 
502 	sc = device_get_softc(dev);
503 
504 	if (sc->vtpci_nvqs != 0)
505 		return (EALREADY);
506 	if (nvqs <= 0)
507 		return (EINVAL);
508 
509 	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
510 	    M_DEVBUF, M_NOWAIT | M_ZERO);
511 	if (sc->vtpci_vqs == NULL)
512 		return (ENOMEM);
513 
514 	for (idx = 0; idx < nvqs; idx++) {
515 		vqx = &sc->vtpci_vqs[idx];
516 		info = &vq_info[idx];
517 
518 		vtpci_select_virtqueue(sc, idx);
519 		size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM);
520 
521 		error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
522 		    ~(vm_paddr_t)0, info, &vq);
523 		if (error) {
524 			device_printf(dev,
525 			    "cannot allocate virtqueue %d: %d\n", idx, error);
526 			break;
527 		}
528 
529 		vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN,
530 		    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
531 
532 		vqx->vtv_vq = *info->vqai_vq = vq;
533 		vqx->vtv_no_intr = info->vqai_intr == NULL;
534 
535 		sc->vtpci_nvqs++;
536 	}
537 
538 	if (error)
539 		vtpci_free_virtqueues(sc);
540 
541 	return (error);
542 }
543 
544 static int
545 vtpci_setup_intr(device_t dev, enum intr_type type)
546 {
547 	struct vtpci_softc *sc;
548 	int attempt, error;
549 
550 	sc = device_get_softc(dev);
551 
552 	for (attempt = 0; attempt < 5; attempt++) {
553 		/*
554 		 * Start with the most desirable interrupt configuration and
555 		 * fallback towards less desirable ones.
556 		 */
557 		switch (attempt) {
558 		case 0:
559 			error = vtpci_alloc_intr_msix_pervq(sc);
560 			break;
561 		case 1:
562 			error = vtpci_alloc_intr_msix_shared(sc);
563 			break;
564 		case 2:
565 			error = vtpci_alloc_intr_msi(sc);
566 			break;
567 		case 3:
568 			error = vtpci_alloc_intr_legacy(sc);
569 			break;
570 		default:
571 			device_printf(dev,
572 			    "exhausted all interrupt allocation attempts\n");
573 			return (ENXIO);
574 		}
575 
576 		if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
577 			break;
578 
579 		vtpci_cleanup_setup_intr_attempt(sc);
580 	}
581 
582 	if (bootverbose) {
583 		if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
584 			device_printf(dev, "using legacy interrupt\n");
585 		else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
586 			device_printf(dev, "using MSI interrupt\n");
587 		else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
588 			device_printf(dev, "using shared MSIX interrupts\n");
589 		else
590 			device_printf(dev, "using per VQ MSIX interrupts\n");
591 	}
592 
593 	return (0);
594 }
595 
596 static void
597 vtpci_stop(device_t dev)
598 {
599 
600 	vtpci_reset(device_get_softc(dev));
601 }
602 
603 static int
604 vtpci_reinit(device_t dev, uint64_t features)
605 {
606 	struct vtpci_softc *sc;
607 	int idx, error;
608 
609 	sc = device_get_softc(dev);
610 
611 	/*
612 	 * Redrive the device initialization. This is a bit of an abuse of
613 	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
614 	 * play nice.
615 	 *
616 	 * We do not allow the host device to change from what was originally
617 	 * negotiated beyond what the guest driver changed. MSIX state should
618 	 * not change, number of virtqueues and their size remain the same, etc.
619 	 * This will need to be rethought when we want to support migration.
620 	 */
621 
622 	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
623 		vtpci_stop(dev);
624 
625 	/*
626 	 * Quickly drive the status through ACK and DRIVER. The device
627 	 * does not become usable again until vtpci_reinit_complete().
628 	 */
629 	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
630 	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
631 
632 	vtpci_negotiate_features(dev, features);
633 
634 	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
635 		error = vtpci_reinit_virtqueue(sc, idx);
636 		if (error)
637 			return (error);
638 	}
639 
640 	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
641 		error = vtpci_set_host_msix_vectors(sc);
642 		if (error)
643 			return (error);
644 	}
645 
646 	return (0);
647 }
648 
649 static void
650 vtpci_reinit_complete(device_t dev)
651 {
652 
653 	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
654 }
655 
656 static void
657 vtpci_notify_virtqueue(device_t dev, uint16_t queue)
658 {
659 	struct vtpci_softc *sc;
660 
661 	sc = device_get_softc(dev);
662 
663 	vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
664 }
665 
666 static uint8_t
667 vtpci_get_status(device_t dev)
668 {
669 	struct vtpci_softc *sc;
670 
671 	sc = device_get_softc(dev);
672 
673 	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
674 }
675 
676 static void
677 vtpci_set_status(device_t dev, uint8_t status)
678 {
679 	struct vtpci_softc *sc;
680 
681 	sc = device_get_softc(dev);
682 
683 	if (status != VIRTIO_CONFIG_STATUS_RESET)
684 		status |= vtpci_get_status(dev);
685 
686 	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
687 }
688 
689 static void
690 vtpci_read_dev_config(device_t dev, bus_size_t offset,
691     void *dst, int length)
692 {
693 	struct vtpci_softc *sc;
694 	bus_size_t off;
695 	uint8_t *d;
696 	int size;
697 
698 	sc = device_get_softc(dev);
699 	off = VIRTIO_PCI_CONFIG(sc) + offset;
700 
701 	for (d = dst; length > 0; d += size, off += size, length -= size) {
702 		if (length >= 4) {
703 			size = 4;
704 			*(uint32_t *)d = vtpci_read_config_4(sc, off);
705 		} else if (length >= 2) {
706 			size = 2;
707 			*(uint16_t *)d = vtpci_read_config_2(sc, off);
708 		} else {
709 			size = 1;
710 			*d = vtpci_read_config_1(sc, off);
711 		}
712 	}
713 }
714 
715 static void
716 vtpci_write_dev_config(device_t dev, bus_size_t offset,
717     void *src, int length)
718 {
719 	struct vtpci_softc *sc;
720 	bus_size_t off;
721 	uint8_t *s;
722 	int size;
723 
724 	sc = device_get_softc(dev);
725 	off = VIRTIO_PCI_CONFIG(sc) + offset;
726 
727 	for (s = src; length > 0; s += size, off += size, length -= size) {
728 		if (length >= 4) {
729 			size = 4;
730 			vtpci_write_config_4(sc, off, *(uint32_t *)s);
731 		} else if (length >= 2) {
732 			size = 2;
733 			vtpci_write_config_2(sc, off, *(uint16_t *)s);
734 		} else {
735 			size = 1;
736 			vtpci_write_config_1(sc, off, *s);
737 		}
738 	}
739 }
740 
741 static void
742 vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
743     uint64_t features)
744 {
745 	device_t dev, child;
746 
747 	dev = sc->vtpci_dev;
748 	child = sc->vtpci_child_dev;
749 
750 	if (device_is_attached(child) || bootverbose == 0)
751 		return;
752 
753 	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
754 }
755 
756 static void
757 vtpci_probe_and_attach_child(struct vtpci_softc *sc)
758 {
759 	device_t dev, child;
760 
761 	dev = sc->vtpci_dev;
762 	child = sc->vtpci_child_dev;
763 
764 	if (child == NULL)
765 		return;
766 
767 	if (device_get_state(child) != DS_NOTPRESENT)
768 		return;
769 
770 	if (device_probe(child) != 0)
771 		return;
772 
773 	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
774 	if (device_attach(child) != 0) {
775 		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
776 		vtpci_reset(sc);
777 		vtpci_release_child_resources(sc);
778 		/* Reset status for future attempt. */
779 		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
780 	} else {
781 		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
782 		VIRTIO_ATTACH_COMPLETED(child);
783 	}
784 }
785 
786 static int
787 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
788 {
789 	device_t dev;
790 	int nmsix, cnt, required;
791 
792 	dev = sc->vtpci_dev;
793 
794 	/* Allocate an additional vector for the config changes. */
795 	required = nvectors + 1;
796 
797 	nmsix = pci_msix_count(dev);
798 	if (nmsix < required)
799 		return (1);
800 
801 	cnt = required;
802 	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
803 		sc->vtpci_nmsix_resources = required;
804 		return (0);
805 	}
806 
807 	pci_release_msi(dev);
808 
809 	return (1);
810 }
811 
812 static int
813 vtpci_alloc_msi(struct vtpci_softc *sc)
814 {
815 	device_t dev;
816 	int nmsi, cnt, required;
817 
818 	dev = sc->vtpci_dev;
819 	required = 1;
820 
821 	nmsi = pci_msi_count(dev);
822 	if (nmsi < required)
823 		return (1);
824 
825 	cnt = required;
826 	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
827 		return (0);
828 
829 	pci_release_msi(dev);
830 
831 	return (1);
832 }
833 
834 static int
835 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
836 {
837 	int i, nvectors, error;
838 
839 	if (vtpci_disable_msix != 0 ||
840 	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
841 		return (ENOTSUP);
842 
843 	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
844 		if (sc->vtpci_vqs[i].vtv_no_intr == 0)
845 			nvectors++;
846 	}
847 
848 	error = vtpci_alloc_msix(sc, nvectors);
849 	if (error)
850 		return (error);
851 
852 	sc->vtpci_flags |= VTPCI_FLAG_MSIX;
853 
854 	return (0);
855 }
856 
857 static int
858 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
859 {
860 	int error;
861 
862 	if (vtpci_disable_msix != 0 ||
863 	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
864 		return (ENOTSUP);
865 
866 	error = vtpci_alloc_msix(sc, 1);
867 	if (error)
868 		return (error);
869 
870 	sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
871 
872 	return (0);
873 }
874 
875 static int
876 vtpci_alloc_intr_msi(struct vtpci_softc *sc)
877 {
878 	int error;
879 
880 	/* Only BHyVe supports MSI. */
881 	if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
882 		return (ENOTSUP);
883 
884 	error = vtpci_alloc_msi(sc);
885 	if (error)
886 		return (error);
887 
888 	sc->vtpci_flags |= VTPCI_FLAG_MSI;
889 
890 	return (0);
891 }
892 
893 static int
894 vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
895 {
896 
897 	sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
898 
899 	return (0);
900 }
901 
902 static int
903 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
904     struct vtpci_interrupt *intr)
905 {
906 	struct resource *irq;
907 
908 	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
909 	if (irq == NULL)
910 		return (ENXIO);
911 
912 	intr->vti_irq = irq;
913 	intr->vti_rid = rid;
914 
915 	return (0);
916 }
917 
918 static int
919 vtpci_alloc_intr_resources(struct vtpci_softc *sc)
920 {
921 	struct vtpci_interrupt *intr;
922 	int i, rid, flags, nvq_intrs, error;
923 
924 	rid = 0;
925 	flags = RF_ACTIVE;
926 
927 	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
928 		flags |= RF_SHAREABLE;
929 	else
930 		rid = 1;
931 
932 	/*
933 	 * For legacy and MSI interrupts, this single resource handles all
934 	 * interrupts. For MSIX, this resource is used for the configuration
935 	 * changed interrupt.
936 	 */
937 	intr = &sc->vtpci_device_interrupt;
938 	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
939 	if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
940 		return (error);
941 
942 	/* Subtract one for the configuration changed interrupt. */
943 	nvq_intrs = sc->vtpci_nmsix_resources - 1;
944 
945 	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
946 	    sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
947 	if (sc->vtpci_msix_vq_interrupts == NULL)
948 		return (ENOMEM);
949 
950 	for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
951 		error = vtpci_alloc_interrupt(sc, rid, flags, intr);
952 		if (error)
953 			return (error);
954 	}
955 
956 	return (0);
957 }
958 
959 static int
960 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
961 {
962 	struct vtpci_interrupt *intr;
963 	int error;
964 
965 	intr = &sc->vtpci_device_interrupt;
966 	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
967 	    vtpci_legacy_intr, sc, &intr->vti_handler);
968 
969 	return (error);
970 }
971 
972 static int
973 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
974 {
975 	struct vtpci_virtqueue *vqx;
976 	struct vtpci_interrupt *intr;
977 	int i, error;
978 
979 	intr = sc->vtpci_msix_vq_interrupts;
980 
981 	for (i = 0; i < sc->vtpci_nvqs; i++) {
982 		vqx = &sc->vtpci_vqs[i];
983 
984 		if (vqx->vtv_no_intr)
985 			continue;
986 
987 		error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
988 		    vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
989 		    &intr->vti_handler);
990 		if (error)
991 			return (error);
992 
993 		intr++;
994 	}
995 
996 	return (0);
997 }
998 
999 static int
1000 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
1001 {
1002 	device_t dev;
1003 	struct vtpci_interrupt *intr;
1004 	int error;
1005 
1006 	dev = sc->vtpci_dev;
1007 	intr = &sc->vtpci_device_interrupt;
1008 
1009 	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
1010 	    vtpci_config_intr, sc, &intr->vti_handler);
1011 	if (error)
1012 		return (error);
1013 
1014 	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
1015 		intr = sc->vtpci_msix_vq_interrupts;
1016 		error = bus_setup_intr(dev, intr->vti_irq, type,
1017 		    vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
1018 		    &intr->vti_handler);
1019 	} else
1020 		error = vtpci_setup_pervq_msix_interrupts(sc, type);
1021 
1022 	return (error ? error : vtpci_set_host_msix_vectors(sc));
1023 }
1024 
1025 static int
1026 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1027 {
1028 	int error;
1029 
1030 	type |= INTR_MPSAFE;
1031 	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1032 	    ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1033 
1034 	error = vtpci_alloc_intr_resources(sc);
1035 	if (error)
1036 		return (error);
1037 
1038 	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1039 		error = vtpci_setup_legacy_interrupt(sc, type);
1040 	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1041 		error = vtpci_setup_msi_interrupt(sc, type);
1042 	else
1043 		error = vtpci_setup_msix_interrupts(sc, type);
1044 
1045 	return (error);
1046 }
1047 
1048 static int
1049 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1050     struct vtpci_interrupt *intr)
1051 {
1052 	device_t dev;
1053 	uint16_t vector;
1054 
1055 	dev = sc->vtpci_dev;
1056 
1057 	if (intr != NULL) {
1058 		/* Map from guest rid to host vector. */
1059 		vector = intr->vti_rid - 1;
1060 	} else
1061 		vector = VIRTIO_MSI_NO_VECTOR;
1062 
1063 	vtpci_write_header_2(sc, offset, vector);
1064 
1065 	/* Read vector to determine if the host had sufficient resources. */
1066 	if (vtpci_read_header_2(sc, offset) != vector) {
1067 		device_printf(dev,
1068 		    "insufficient host resources for MSIX interrupts\n");
1069 		return (ENODEV);
1070 	}
1071 
1072 	return (0);
1073 }
1074 
1075 static int
1076 vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1077 {
1078 	struct vtpci_interrupt *intr, *tintr;
1079 	int idx, offset, error;
1080 
1081 	intr = &sc->vtpci_device_interrupt;
1082 	offset = VIRTIO_MSI_CONFIG_VECTOR;
1083 
1084 	error = vtpci_register_msix_vector(sc, offset, intr);
1085 	if (error)
1086 		return (error);
1087 
1088 	intr = sc->vtpci_msix_vq_interrupts;
1089 	offset = VIRTIO_MSI_QUEUE_VECTOR;
1090 
1091 	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1092 		vtpci_select_virtqueue(sc, idx);
1093 
1094 		if (sc->vtpci_vqs[idx].vtv_no_intr)
1095 			tintr = NULL;
1096 		else
1097 			tintr = intr;
1098 
1099 		error = vtpci_register_msix_vector(sc, offset, tintr);
1100 		if (error)
1101 			break;
1102 
1103 		/*
1104 		 * For shared MSIX, all the virtqueues share the first
1105 		 * interrupt.
1106 		 */
1107 		if (!sc->vtpci_vqs[idx].vtv_no_intr &&
1108 		    (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1109 			intr++;
1110 	}
1111 
1112 	return (error);
1113 }
1114 
1115 static int
1116 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1117 {
1118 	struct vtpci_virtqueue *vqx;
1119 	struct virtqueue *vq;
1120 	int error;
1121 	uint16_t size;
1122 
1123 	vqx = &sc->vtpci_vqs[idx];
1124 	vq = vqx->vtv_vq;
1125 
1126 	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1127 
1128 	vtpci_select_virtqueue(sc, idx);
1129 	size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM);
1130 
1131 	error = virtqueue_reinit(vq, size);
1132 	if (error)
1133 		return (error);
1134 
1135 	vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN,
1136 	    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1137 
1138 	return (0);
1139 }
1140 
1141 static void
1142 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1143 {
1144 	device_t dev;
1145 
1146 	dev = sc->vtpci_dev;
1147 
1148 	if (intr->vti_handler != NULL) {
1149 		bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1150 		intr->vti_handler = NULL;
1151 	}
1152 
1153 	if (intr->vti_irq != NULL) {
1154 		bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1155 		    intr->vti_irq);
1156 		intr->vti_irq = NULL;
1157 		intr->vti_rid = -1;
1158 	}
1159 }
1160 
1161 static void
1162 vtpci_free_interrupts(struct vtpci_softc *sc)
1163 {
1164 	struct vtpci_interrupt *intr;
1165 	int i, nvq_intrs;
1166 
1167 	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1168 
1169 	if (sc->vtpci_nmsix_resources != 0) {
1170 		nvq_intrs = sc->vtpci_nmsix_resources - 1;
1171 		sc->vtpci_nmsix_resources = 0;
1172 
1173 		intr = sc->vtpci_msix_vq_interrupts;
1174 		if (intr != NULL) {
1175 			for (i = 0; i < nvq_intrs; i++, intr++)
1176 				vtpci_free_interrupt(sc, intr);
1177 
1178 			free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1179 			sc->vtpci_msix_vq_interrupts = NULL;
1180 		}
1181 	}
1182 
1183 	if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1184 		pci_release_msi(sc->vtpci_dev);
1185 
1186 	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1187 }
1188 
1189 static void
1190 vtpci_free_virtqueues(struct vtpci_softc *sc)
1191 {
1192 	struct vtpci_virtqueue *vqx;
1193 	int idx;
1194 
1195 	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1196 		vqx = &sc->vtpci_vqs[idx];
1197 
1198 		vtpci_select_virtqueue(sc, idx);
1199 		vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1200 
1201 		virtqueue_free(vqx->vtv_vq);
1202 		vqx->vtv_vq = NULL;
1203 	}
1204 
1205 	free(sc->vtpci_vqs, M_DEVBUF);
1206 	sc->vtpci_vqs = NULL;
1207 	sc->vtpci_nvqs = 0;
1208 }
1209 
1210 static void
1211 vtpci_release_child_resources(struct vtpci_softc *sc)
1212 {
1213 
1214 	vtpci_free_interrupts(sc);
1215 	vtpci_free_virtqueues(sc);
1216 }
1217 
1218 static void
1219 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1220 {
1221 	int idx;
1222 
1223 	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1224 		vtpci_write_header_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1225 		    VIRTIO_MSI_NO_VECTOR);
1226 
1227 		for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1228 			vtpci_select_virtqueue(sc, idx);
1229 			vtpci_write_header_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1230 			    VIRTIO_MSI_NO_VECTOR);
1231 		}
1232 	}
1233 
1234 	vtpci_free_interrupts(sc);
1235 }
1236 
1237 static void
1238 vtpci_reset(struct vtpci_softc *sc)
1239 {
1240 
1241 	/*
1242 	 * Setting the status to RESET sets the host device to
1243 	 * the original, uninitialized state.
1244 	 */
1245 	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1246 }
1247 
1248 static void
1249 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1250 {
1251 
1252 	vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1253 }
1254 
1255 static void
1256 vtpci_legacy_intr(void *xsc)
1257 {
1258 	struct vtpci_softc *sc;
1259 	struct vtpci_virtqueue *vqx;
1260 	int i;
1261 	uint8_t isr;
1262 
1263 	sc = xsc;
1264 	vqx = &sc->vtpci_vqs[0];
1265 
1266 	/* Reading the ISR also clears it. */
1267 	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1268 
1269 	if (isr & VIRTIO_PCI_ISR_CONFIG)
1270 		vtpci_config_intr(sc);
1271 
1272 	if (isr & VIRTIO_PCI_ISR_INTR) {
1273 		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1274 			if (vqx->vtv_no_intr == 0)
1275 				virtqueue_intr(vqx->vtv_vq);
1276 		}
1277 	}
1278 }
1279 
1280 static int
1281 vtpci_vq_shared_intr_filter(void *xsc)
1282 {
1283 	struct vtpci_softc *sc;
1284 	struct vtpci_virtqueue *vqx;
1285 	int i, rc;
1286 
1287 	rc = 0;
1288 	sc = xsc;
1289 	vqx = &sc->vtpci_vqs[0];
1290 
1291 	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1292 		if (vqx->vtv_no_intr == 0)
1293 			rc |= virtqueue_intr_filter(vqx->vtv_vq);
1294 	}
1295 
1296 	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1297 }
1298 
1299 static void
1300 vtpci_vq_shared_intr(void *xsc)
1301 {
1302 	struct vtpci_softc *sc;
1303 	struct vtpci_virtqueue *vqx;
1304 	int i;
1305 
1306 	sc = xsc;
1307 	vqx = &sc->vtpci_vqs[0];
1308 
1309 	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1310 		if (vqx->vtv_no_intr == 0)
1311 			virtqueue_intr(vqx->vtv_vq);
1312 	}
1313 }
1314 
1315 static int
1316 vtpci_vq_intr_filter(void *xvq)
1317 {
1318 	struct virtqueue *vq;
1319 	int rc;
1320 
1321 	vq = xvq;
1322 	rc = virtqueue_intr_filter(vq);
1323 
1324 	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1325 }
1326 
1327 static void
1328 vtpci_vq_intr(void *xvq)
1329 {
1330 	struct virtqueue *vq;
1331 
1332 	vq = xvq;
1333 	virtqueue_intr(vq);
1334 }
1335 
1336 static void
1337 vtpci_config_intr(void *xsc)
1338 {
1339 	struct vtpci_softc *sc;
1340 	device_t child;
1341 
1342 	sc = xsc;
1343 	child = sc->vtpci_child_dev;
1344 
1345 	if (child != NULL)
1346 		VIRTIO_CONFIG_CHANGE(child);
1347 }
1348