xref: /freebsd/sys/dev/virtio/pci/virtio_pci_modern.c (revision de140d60d5282b0dbf79979d04d5b6b11511e25d)
19da9560cSBryan Venteicher /*-
24d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
39da9560cSBryan Venteicher  *
49da9560cSBryan Venteicher  * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
59da9560cSBryan Venteicher  * All rights reserved.
69da9560cSBryan Venteicher  *
79da9560cSBryan Venteicher  * Redistribution and use in source and binary forms, with or without
89da9560cSBryan Venteicher  * modification, are permitted provided that the following conditions
99da9560cSBryan Venteicher  * are met:
109da9560cSBryan Venteicher  * 1. Redistributions of source code must retain the above copyright
119da9560cSBryan Venteicher  *    notice unmodified, this list of conditions, and the following
129da9560cSBryan Venteicher  *    disclaimer.
139da9560cSBryan Venteicher  * 2. Redistributions in binary form must reproduce the above copyright
149da9560cSBryan Venteicher  *    notice, this list of conditions and the following disclaimer in the
159da9560cSBryan Venteicher  *    documentation and/or other materials provided with the distribution.
169da9560cSBryan Venteicher  *
179da9560cSBryan Venteicher  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
189da9560cSBryan Venteicher  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
199da9560cSBryan Venteicher  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
209da9560cSBryan Venteicher  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
219da9560cSBryan Venteicher  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
229da9560cSBryan Venteicher  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
239da9560cSBryan Venteicher  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
249da9560cSBryan Venteicher  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
259da9560cSBryan Venteicher  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
269da9560cSBryan Venteicher  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
279da9560cSBryan Venteicher  */
289da9560cSBryan Venteicher 
299da9560cSBryan Venteicher /* Driver for the modern VirtIO PCI interface. */
309da9560cSBryan Venteicher 
319da9560cSBryan Venteicher #include <sys/param.h>
329da9560cSBryan Venteicher #include <sys/systm.h>
339da9560cSBryan Venteicher #include <sys/bus.h>
349da9560cSBryan Venteicher #include <sys/lock.h>
359da9560cSBryan Venteicher #include <sys/kernel.h>
369da9560cSBryan Venteicher #include <sys/module.h>
379da9560cSBryan Venteicher 
389da9560cSBryan Venteicher #include <machine/bus.h>
399da9560cSBryan Venteicher #include <machine/cpu.h>
409da9560cSBryan Venteicher #include <machine/resource.h>
419da9560cSBryan Venteicher #include <sys/bus.h>
429da9560cSBryan Venteicher #include <sys/rman.h>
439da9560cSBryan Venteicher 
449da9560cSBryan Venteicher #include <dev/pci/pcivar.h>
459da9560cSBryan Venteicher #include <dev/pci/pcireg.h>
469da9560cSBryan Venteicher 
479da9560cSBryan Venteicher #include <dev/virtio/virtio.h>
489da9560cSBryan Venteicher #include <dev/virtio/virtqueue.h>
499da9560cSBryan Venteicher #include <dev/virtio/pci/virtio_pci.h>
509da9560cSBryan Venteicher #include <dev/virtio/pci/virtio_pci_modern_var.h>
519da9560cSBryan Venteicher 
529da9560cSBryan Venteicher #include "virtio_bus_if.h"
539da9560cSBryan Venteicher #include "virtio_pci_if.h"
549da9560cSBryan Venteicher #include "virtio_if.h"
559da9560cSBryan Venteicher 
569da9560cSBryan Venteicher struct vtpci_modern_resource_map {
579da9560cSBryan Venteicher 	struct resource_map	vtrm_map;
589da9560cSBryan Venteicher 	int			vtrm_cap_offset;
599da9560cSBryan Venteicher 	int			vtrm_bar;
609da9560cSBryan Venteicher 	int			vtrm_offset;
619da9560cSBryan Venteicher 	int			vtrm_length;
629da9560cSBryan Venteicher 	int			vtrm_type;	/* SYS_RES_{MEMORY, IOPORT} */
639da9560cSBryan Venteicher };
649da9560cSBryan Venteicher 
659da9560cSBryan Venteicher struct vtpci_modern_bar_resource {
669da9560cSBryan Venteicher 	struct resource		*vtbr_res;
679da9560cSBryan Venteicher 	int			 vtbr_type;
689da9560cSBryan Venteicher };
699da9560cSBryan Venteicher 
709da9560cSBryan Venteicher struct vtpci_modern_softc {
719da9560cSBryan Venteicher 	device_t			 vtpci_dev;
729da9560cSBryan Venteicher 	struct vtpci_common		 vtpci_common;
739da9560cSBryan Venteicher 	uint32_t			 vtpci_notify_offset_multiplier;
749da9560cSBryan Venteicher 	uint16_t			 vtpci_devid;
759da9560cSBryan Venteicher 	int				 vtpci_msix_bar;
769da9560cSBryan Venteicher 	struct resource			*vtpci_msix_res;
779da9560cSBryan Venteicher 
789da9560cSBryan Venteicher 	struct vtpci_modern_resource_map vtpci_common_res_map;
799da9560cSBryan Venteicher 	struct vtpci_modern_resource_map vtpci_notify_res_map;
809da9560cSBryan Venteicher 	struct vtpci_modern_resource_map vtpci_isr_res_map;
819da9560cSBryan Venteicher 	struct vtpci_modern_resource_map vtpci_device_res_map;
829da9560cSBryan Venteicher 
839da9560cSBryan Venteicher #define VTPCI_MODERN_MAX_BARS		6
849da9560cSBryan Venteicher 	struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS];
859da9560cSBryan Venteicher };
869da9560cSBryan Venteicher 
879da9560cSBryan Venteicher static int	vtpci_modern_probe(device_t);
889da9560cSBryan Venteicher static int	vtpci_modern_attach(device_t);
899da9560cSBryan Venteicher static int	vtpci_modern_detach(device_t);
909da9560cSBryan Venteicher static int	vtpci_modern_suspend(device_t);
919da9560cSBryan Venteicher static int	vtpci_modern_resume(device_t);
929da9560cSBryan Venteicher static int	vtpci_modern_shutdown(device_t);
939da9560cSBryan Venteicher 
949da9560cSBryan Venteicher static void	vtpci_modern_driver_added(device_t, driver_t *);
959da9560cSBryan Venteicher static void	vtpci_modern_child_detached(device_t, device_t);
969da9560cSBryan Venteicher static int	vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *);
979da9560cSBryan Venteicher static int	vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t);
989da9560cSBryan Venteicher 
999da9560cSBryan Venteicher static uint8_t	vtpci_modern_read_isr(device_t);
1009da9560cSBryan Venteicher static uint16_t	vtpci_modern_get_vq_size(device_t, int);
1019da9560cSBryan Venteicher static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int);
1029da9560cSBryan Venteicher static void	vtpci_modern_set_vq(device_t, struct virtqueue *);
1039da9560cSBryan Venteicher static void	vtpci_modern_disable_vq(device_t, int);
1049da9560cSBryan Venteicher static int	vtpci_modern_register_msix(struct vtpci_modern_softc *, int,
1059da9560cSBryan Venteicher 		    struct vtpci_interrupt *);
1069da9560cSBryan Venteicher static int	vtpci_modern_register_cfg_msix(device_t,
1079da9560cSBryan Venteicher 		    struct vtpci_interrupt *);
1089da9560cSBryan Venteicher static int	vtpci_modern_register_vq_msix(device_t, int idx,
1099da9560cSBryan Venteicher 		    struct vtpci_interrupt *);
1109da9560cSBryan Venteicher 
1119da9560cSBryan Venteicher static uint64_t	vtpci_modern_negotiate_features(device_t, uint64_t);
1129da9560cSBryan Venteicher static int	vtpci_modern_finalize_features(device_t);
113ccb576a8SMina Galić static bool	vtpci_modern_with_feature(device_t, uint64_t);
114180c0240SMina Galić static int	vtpci_modern_alloc_virtqueues(device_t, int,
1159da9560cSBryan Venteicher 		    struct vq_alloc_info *);
1169da9560cSBryan Venteicher static int	vtpci_modern_setup_interrupts(device_t, enum intr_type);
1179da9560cSBryan Venteicher static void	vtpci_modern_stop(device_t);
1189da9560cSBryan Venteicher static int	vtpci_modern_reinit(device_t, uint64_t);
1199da9560cSBryan Venteicher static void	vtpci_modern_reinit_complete(device_t);
1209da9560cSBryan Venteicher static void	vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t);
1219da9560cSBryan Venteicher static int	vtpci_modern_config_generation(device_t);
1229da9560cSBryan Venteicher static void	vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int);
1236c4f9516SAlex Richardson static void	vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int);
1249da9560cSBryan Venteicher 
1259da9560cSBryan Venteicher static int	vtpci_modern_probe_configs(device_t);
1269da9560cSBryan Venteicher static int	vtpci_modern_find_cap(device_t, uint8_t, int *);
1279da9560cSBryan Venteicher static int	vtpci_modern_map_configs(struct vtpci_modern_softc *);
1289da9560cSBryan Venteicher static void	vtpci_modern_unmap_configs(struct vtpci_modern_softc *);
1299da9560cSBryan Venteicher static int	vtpci_modern_find_cap_resource(struct vtpci_modern_softc *,
1309da9560cSBryan Venteicher 		     uint8_t, int, int, struct vtpci_modern_resource_map *);
1319da9560cSBryan Venteicher static int	vtpci_modern_bar_type(struct vtpci_modern_softc *, int);
1329da9560cSBryan Venteicher static struct resource *vtpci_modern_get_bar_resource(
1339da9560cSBryan Venteicher 		    struct vtpci_modern_softc *, int, int);
1349da9560cSBryan Venteicher static struct resource *vtpci_modern_alloc_bar_resource(
1359da9560cSBryan Venteicher 		    struct vtpci_modern_softc *, int, int);
1369da9560cSBryan Venteicher static void	vtpci_modern_free_bar_resources(struct vtpci_modern_softc *);
1379da9560cSBryan Venteicher static int	vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *,
1389da9560cSBryan Venteicher 		    struct vtpci_modern_resource_map *);
1399da9560cSBryan Venteicher static void	vtpci_modern_free_resource_map(struct vtpci_modern_softc *,
1409da9560cSBryan Venteicher 		    struct vtpci_modern_resource_map *);
1419da9560cSBryan Venteicher static void	vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *);
1429da9560cSBryan Venteicher static void	vtpci_modern_free_msix_resource(struct vtpci_modern_softc *);
1439da9560cSBryan Venteicher 
1449da9560cSBryan Venteicher static void	vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *);
1459da9560cSBryan Venteicher 
1469da9560cSBryan Venteicher static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *);
1479da9560cSBryan Venteicher static void	vtpci_modern_write_features(struct vtpci_modern_softc *,
1489da9560cSBryan Venteicher 		    uint64_t);
1499da9560cSBryan Venteicher static void	vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int);
1509da9560cSBryan Venteicher static uint8_t	vtpci_modern_get_status(struct vtpci_modern_softc *);
1519da9560cSBryan Venteicher static void	vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t);
1529da9560cSBryan Venteicher static void	vtpci_modern_reset(struct vtpci_modern_softc *);
1539da9560cSBryan Venteicher static void	vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *);
1549da9560cSBryan Venteicher 
1559da9560cSBryan Venteicher static uint8_t	vtpci_modern_read_common_1(struct vtpci_modern_softc *,
1569da9560cSBryan Venteicher 		    bus_size_t);
1579da9560cSBryan Venteicher static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *,
1589da9560cSBryan Venteicher 		    bus_size_t);
1599da9560cSBryan Venteicher static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *,
1609da9560cSBryan Venteicher 		    bus_size_t);
1619da9560cSBryan Venteicher static void	vtpci_modern_write_common_1(struct vtpci_modern_softc *,
1629da9560cSBryan Venteicher 		     bus_size_t, uint8_t);
1639da9560cSBryan Venteicher static void	vtpci_modern_write_common_2(struct vtpci_modern_softc *,
1649da9560cSBryan Venteicher 		     bus_size_t, uint16_t);
1659da9560cSBryan Venteicher static void	vtpci_modern_write_common_4(struct vtpci_modern_softc *,
1669da9560cSBryan Venteicher 		    bus_size_t, uint32_t);
1679da9560cSBryan Venteicher static void	vtpci_modern_write_common_8(struct vtpci_modern_softc *,
1689da9560cSBryan Venteicher 		    bus_size_t, uint64_t);
1699da9560cSBryan Venteicher static void	vtpci_modern_write_notify_2(struct vtpci_modern_softc *,
1709da9560cSBryan Venteicher 		    bus_size_t, uint16_t);
1719da9560cSBryan Venteicher static uint8_t  vtpci_modern_read_isr_1(struct vtpci_modern_softc *,
1729da9560cSBryan Venteicher 		    bus_size_t);
1739da9560cSBryan Venteicher static uint8_t	vtpci_modern_read_device_1(struct vtpci_modern_softc *,
1749da9560cSBryan Venteicher 		    bus_size_t);
1759da9560cSBryan Venteicher static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *,
1769da9560cSBryan Venteicher 		    bus_size_t);
1779da9560cSBryan Venteicher static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *,
1789da9560cSBryan Venteicher 		    bus_size_t);
1799da9560cSBryan Venteicher static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *,
1809da9560cSBryan Venteicher 		    bus_size_t);
1819da9560cSBryan Venteicher static void	vtpci_modern_write_device_1(struct vtpci_modern_softc *,
1829da9560cSBryan Venteicher 		    bus_size_t, uint8_t);
1839da9560cSBryan Venteicher static void	vtpci_modern_write_device_2(struct vtpci_modern_softc *,
1849da9560cSBryan Venteicher 		    bus_size_t, uint16_t);
1859da9560cSBryan Venteicher static void	vtpci_modern_write_device_4(struct vtpci_modern_softc *,
1869da9560cSBryan Venteicher 		    bus_size_t, uint32_t);
1879da9560cSBryan Venteicher static void	vtpci_modern_write_device_8(struct vtpci_modern_softc *,
1889da9560cSBryan Venteicher 		    bus_size_t, uint64_t);
1899da9560cSBryan Venteicher 
1909da9560cSBryan Venteicher /* Tunables. */
1919da9560cSBryan Venteicher static int vtpci_modern_transitional = 0;
1929da9560cSBryan Venteicher TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional);
1939da9560cSBryan Venteicher 
1949da9560cSBryan Venteicher static device_method_t vtpci_modern_methods[] = {
1959da9560cSBryan Venteicher 	/* Device interface. */
1969da9560cSBryan Venteicher 	DEVMETHOD(device_probe,			vtpci_modern_probe),
1979da9560cSBryan Venteicher 	DEVMETHOD(device_attach,		vtpci_modern_attach),
1989da9560cSBryan Venteicher 	DEVMETHOD(device_detach,		vtpci_modern_detach),
1999da9560cSBryan Venteicher 	DEVMETHOD(device_suspend,		vtpci_modern_suspend),
2009da9560cSBryan Venteicher 	DEVMETHOD(device_resume,		vtpci_modern_resume),
2019da9560cSBryan Venteicher 	DEVMETHOD(device_shutdown,		vtpci_modern_shutdown),
2029da9560cSBryan Venteicher 
2039da9560cSBryan Venteicher 	/* Bus interface. */
2049da9560cSBryan Venteicher 	DEVMETHOD(bus_driver_added,		vtpci_modern_driver_added),
2059da9560cSBryan Venteicher 	DEVMETHOD(bus_child_detached,		vtpci_modern_child_detached),
206ddfc9c4cSWarner Losh 	DEVMETHOD(bus_child_pnpinfo,		virtio_child_pnpinfo),
2079da9560cSBryan Venteicher 	DEVMETHOD(bus_read_ivar,		vtpci_modern_read_ivar),
2089da9560cSBryan Venteicher 	DEVMETHOD(bus_write_ivar,		vtpci_modern_write_ivar),
2099da9560cSBryan Venteicher 
2109da9560cSBryan Venteicher 	/* VirtIO PCI interface. */
2119da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_read_isr,		 vtpci_modern_read_isr),
2129da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_get_vq_size,	 vtpci_modern_get_vq_size),
2139da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_get_vq_notify_off,	 vtpci_modern_get_vq_notify_off),
2149da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_set_vq,		 vtpci_modern_set_vq),
2159da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_disable_vq,	 vtpci_modern_disable_vq),
2169da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_register_cfg_msix,	 vtpci_modern_register_cfg_msix),
2179da9560cSBryan Venteicher 	DEVMETHOD(virtio_pci_register_vq_msix,	 vtpci_modern_register_vq_msix),
2189da9560cSBryan Venteicher 
2199da9560cSBryan Venteicher 	/* VirtIO bus interface. */
2209da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_modern_negotiate_features),
2219da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_finalize_features,	  vtpci_modern_finalize_features),
2229da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_with_feature,	  vtpci_modern_with_feature),
2239da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_modern_alloc_virtqueues),
2249da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_modern_setup_interrupts),
2259da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_stop,		  vtpci_modern_stop),
2269da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_reinit,		  vtpci_modern_reinit),
2279da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_modern_reinit_complete),
2289da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_modern_notify_vq),
2299da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_config_generation,	  vtpci_modern_config_generation),
2309da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_read_device_config,  vtpci_modern_read_dev_config),
2319da9560cSBryan Venteicher 	DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config),
2329da9560cSBryan Venteicher 
2339da9560cSBryan Venteicher 	DEVMETHOD_END
2349da9560cSBryan Venteicher };
2359da9560cSBryan Venteicher 
2369da9560cSBryan Venteicher static driver_t vtpci_modern_driver = {
2379da9560cSBryan Venteicher 	.name = "virtio_pci",
2389da9560cSBryan Venteicher 	.methods = vtpci_modern_methods,
2399da9560cSBryan Venteicher 	.size = sizeof(struct vtpci_modern_softc)
2409da9560cSBryan Venteicher };
2419da9560cSBryan Venteicher 
2425c4c96d3SJohn Baldwin DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0);
2439da9560cSBryan Venteicher 
2449da9560cSBryan Venteicher static int
2459da9560cSBryan Venteicher vtpci_modern_probe(device_t dev)
2469da9560cSBryan Venteicher {
2479da9560cSBryan Venteicher 	const char *name;
2489da9560cSBryan Venteicher 	uint16_t devid;
2499da9560cSBryan Venteicher 
2509da9560cSBryan Venteicher 	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
2519da9560cSBryan Venteicher 		return (ENXIO);
2529da9560cSBryan Venteicher 
2539da9560cSBryan Venteicher 	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
2549da9560cSBryan Venteicher 	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX)
2559da9560cSBryan Venteicher 		return (ENXIO);
2569da9560cSBryan Venteicher 
2579da9560cSBryan Venteicher 	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) {
2589da9560cSBryan Venteicher 		if (!vtpci_modern_transitional)
2599da9560cSBryan Venteicher 			return (ENXIO);
2609da9560cSBryan Venteicher 		devid = pci_get_subdevice(dev);
2619da9560cSBryan Venteicher 	} else
2629da9560cSBryan Venteicher 		devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN;
2639da9560cSBryan Venteicher 
2649da9560cSBryan Venteicher 	if (vtpci_modern_probe_configs(dev) != 0)
2659da9560cSBryan Venteicher 		return (ENXIO);
2669da9560cSBryan Venteicher 
2679da9560cSBryan Venteicher 	name = virtio_device_name(devid);
2689da9560cSBryan Venteicher 	if (name == NULL)
2699da9560cSBryan Venteicher 		name = "Unknown";
2709da9560cSBryan Venteicher 
271*de140d60SMark Johnston 	device_set_descf(dev, "VirtIO PCI (modern) %s adapter", name);
2729da9560cSBryan Venteicher 
2739da9560cSBryan Venteicher 	return (BUS_PROBE_DEFAULT);
2749da9560cSBryan Venteicher }
2759da9560cSBryan Venteicher 
2769da9560cSBryan Venteicher static int
2779da9560cSBryan Venteicher vtpci_modern_attach(device_t dev)
2789da9560cSBryan Venteicher {
2799da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
2809da9560cSBryan Venteicher 	int error;
2819da9560cSBryan Venteicher 
2829da9560cSBryan Venteicher 	sc = device_get_softc(dev);
2839da9560cSBryan Venteicher 	sc->vtpci_dev = dev;
2849da9560cSBryan Venteicher 	vtpci_init(&sc->vtpci_common, dev, true);
2859da9560cSBryan Venteicher 
2869da9560cSBryan Venteicher 	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN)
2879da9560cSBryan Venteicher 		sc->vtpci_devid = pci_get_subdevice(dev);
2889da9560cSBryan Venteicher 	else
2899da9560cSBryan Venteicher 		sc->vtpci_devid = pci_get_device(dev) -
2909da9560cSBryan Venteicher 		    VIRTIO_PCI_DEVICEID_MODERN_MIN;
2919da9560cSBryan Venteicher 
2929da9560cSBryan Venteicher 	error = vtpci_modern_map_configs(sc);
2939da9560cSBryan Venteicher 	if (error) {
2949da9560cSBryan Venteicher 		device_printf(dev, "cannot map configs\n");
2959da9560cSBryan Venteicher 		vtpci_modern_unmap_configs(sc);
2969da9560cSBryan Venteicher 		return (error);
2979da9560cSBryan Venteicher 	}
2989da9560cSBryan Venteicher 
2999da9560cSBryan Venteicher 	vtpci_modern_reset(sc);
3009da9560cSBryan Venteicher 
3019da9560cSBryan Venteicher 	/* Tell the host we've noticed this device. */
3029da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
3039da9560cSBryan Venteicher 
3049da9560cSBryan Venteicher 	error = vtpci_add_child(&sc->vtpci_common);
3059da9560cSBryan Venteicher 	if (error)
3069da9560cSBryan Venteicher 		goto fail;
3079da9560cSBryan Venteicher 
3089da9560cSBryan Venteicher 	vtpci_modern_probe_and_attach_child(sc);
3099da9560cSBryan Venteicher 
3109da9560cSBryan Venteicher 	return (0);
3119da9560cSBryan Venteicher 
3129da9560cSBryan Venteicher fail:
3139da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
3149da9560cSBryan Venteicher 	vtpci_modern_detach(dev);
3159da9560cSBryan Venteicher 
3169da9560cSBryan Venteicher 	return (error);
3179da9560cSBryan Venteicher }
3189da9560cSBryan Venteicher 
3199da9560cSBryan Venteicher static int
3209da9560cSBryan Venteicher vtpci_modern_detach(device_t dev)
3219da9560cSBryan Venteicher {
3229da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
3239da9560cSBryan Venteicher 	int error;
3249da9560cSBryan Venteicher 
3259da9560cSBryan Venteicher 	sc = device_get_softc(dev);
3269da9560cSBryan Venteicher 
3279da9560cSBryan Venteicher 	error = vtpci_delete_child(&sc->vtpci_common);
3289da9560cSBryan Venteicher 	if (error)
3299da9560cSBryan Venteicher 		return (error);
3309da9560cSBryan Venteicher 
3319da9560cSBryan Venteicher 	vtpci_modern_reset(sc);
3329da9560cSBryan Venteicher 	vtpci_modern_unmap_configs(sc);
3339da9560cSBryan Venteicher 
3349da9560cSBryan Venteicher 	return (0);
3359da9560cSBryan Venteicher }
3369da9560cSBryan Venteicher 
3379da9560cSBryan Venteicher static int
3389da9560cSBryan Venteicher vtpci_modern_suspend(device_t dev)
3399da9560cSBryan Venteicher {
3409da9560cSBryan Venteicher 	return (bus_generic_suspend(dev));
3419da9560cSBryan Venteicher }
3429da9560cSBryan Venteicher 
3439da9560cSBryan Venteicher static int
3449da9560cSBryan Venteicher vtpci_modern_resume(device_t dev)
3459da9560cSBryan Venteicher {
3469da9560cSBryan Venteicher 	return (bus_generic_resume(dev));
3479da9560cSBryan Venteicher }
3489da9560cSBryan Venteicher 
3499da9560cSBryan Venteicher static int
3509da9560cSBryan Venteicher vtpci_modern_shutdown(device_t dev)
3519da9560cSBryan Venteicher {
3529da9560cSBryan Venteicher 	(void) bus_generic_shutdown(dev);
3539da9560cSBryan Venteicher 	/* Forcibly stop the host device. */
3549da9560cSBryan Venteicher 	vtpci_modern_stop(dev);
3559da9560cSBryan Venteicher 
3569da9560cSBryan Venteicher 	return (0);
3579da9560cSBryan Venteicher }
3589da9560cSBryan Venteicher 
3599da9560cSBryan Venteicher static void
3609da9560cSBryan Venteicher vtpci_modern_driver_added(device_t dev, driver_t *driver)
3619da9560cSBryan Venteicher {
3629da9560cSBryan Venteicher 	vtpci_modern_probe_and_attach_child(device_get_softc(dev));
3639da9560cSBryan Venteicher }
3649da9560cSBryan Venteicher 
3659da9560cSBryan Venteicher static void
3669da9560cSBryan Venteicher vtpci_modern_child_detached(device_t dev, device_t child)
3679da9560cSBryan Venteicher {
3689da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
3699da9560cSBryan Venteicher 
3709da9560cSBryan Venteicher 	sc = device_get_softc(dev);
3719da9560cSBryan Venteicher 
3729da9560cSBryan Venteicher 	vtpci_modern_reset(sc);
3739da9560cSBryan Venteicher 	vtpci_child_detached(&sc->vtpci_common);
3749da9560cSBryan Venteicher 
3759da9560cSBryan Venteicher 	/* After the reset, retell the host we've noticed this device. */
3769da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
3779da9560cSBryan Venteicher }
3789da9560cSBryan Venteicher 
3799da9560cSBryan Venteicher static int
3809da9560cSBryan Venteicher vtpci_modern_read_ivar(device_t dev, device_t child, int index,
3819da9560cSBryan Venteicher     uintptr_t *result)
3829da9560cSBryan Venteicher {
3839da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
3849da9560cSBryan Venteicher 	struct vtpci_common *cn;
3859da9560cSBryan Venteicher 
3869da9560cSBryan Venteicher 	sc = device_get_softc(dev);
3879da9560cSBryan Venteicher 	cn = &sc->vtpci_common;
3889da9560cSBryan Venteicher 
3899da9560cSBryan Venteicher 	if (vtpci_child_device(cn) != child)
3909da9560cSBryan Venteicher 		return (ENOENT);
3919da9560cSBryan Venteicher 
3929da9560cSBryan Venteicher 	switch (index) {
3939da9560cSBryan Venteicher 	case VIRTIO_IVAR_DEVTYPE:
3949da9560cSBryan Venteicher 		*result = sc->vtpci_devid;
3959da9560cSBryan Venteicher 		break;
3969da9560cSBryan Venteicher 	default:
3979da9560cSBryan Venteicher 		return (vtpci_read_ivar(cn, index, result));
3989da9560cSBryan Venteicher 	}
3999da9560cSBryan Venteicher 
4009da9560cSBryan Venteicher 	return (0);
4019da9560cSBryan Venteicher }
4029da9560cSBryan Venteicher 
4039da9560cSBryan Venteicher static int
4049da9560cSBryan Venteicher vtpci_modern_write_ivar(device_t dev, device_t child, int index,
4059da9560cSBryan Venteicher     uintptr_t value)
4069da9560cSBryan Venteicher {
4079da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
4089da9560cSBryan Venteicher 	struct vtpci_common *cn;
4099da9560cSBryan Venteicher 
4109da9560cSBryan Venteicher 	sc = device_get_softc(dev);
4119da9560cSBryan Venteicher 	cn = &sc->vtpci_common;
4129da9560cSBryan Venteicher 
4139da9560cSBryan Venteicher 	if (vtpci_child_device(cn) != child)
4149da9560cSBryan Venteicher 		return (ENOENT);
4159da9560cSBryan Venteicher 
4169da9560cSBryan Venteicher 	switch (index) {
4179da9560cSBryan Venteicher 	default:
4189da9560cSBryan Venteicher 		return (vtpci_write_ivar(cn, index, value));
4199da9560cSBryan Venteicher 	}
4209da9560cSBryan Venteicher 
4219da9560cSBryan Venteicher 	return (0);
4229da9560cSBryan Venteicher }
4239da9560cSBryan Venteicher 
4249da9560cSBryan Venteicher static uint64_t
4259da9560cSBryan Venteicher vtpci_modern_negotiate_features(device_t dev, uint64_t child_features)
4269da9560cSBryan Venteicher {
4279da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
4289da9560cSBryan Venteicher 	uint64_t host_features, features;
4299da9560cSBryan Venteicher 
4309da9560cSBryan Venteicher 	sc = device_get_softc(dev);
4319da9560cSBryan Venteicher 	host_features = vtpci_modern_read_features(sc);
4329da9560cSBryan Venteicher 
4339da9560cSBryan Venteicher 	/*
4349da9560cSBryan Venteicher 	 * Since the driver was added as a child of the modern PCI bus,
4359da9560cSBryan Venteicher 	 * always add the V1 flag.
4369da9560cSBryan Venteicher 	 */
4379da9560cSBryan Venteicher 	child_features |= VIRTIO_F_VERSION_1;
4389da9560cSBryan Venteicher 
4399da9560cSBryan Venteicher 	features = vtpci_negotiate_features(&sc->vtpci_common,
4409da9560cSBryan Venteicher 	    child_features, host_features);
4419da9560cSBryan Venteicher 	vtpci_modern_write_features(sc, features);
4429da9560cSBryan Venteicher 
4439da9560cSBryan Venteicher 	return (features);
4449da9560cSBryan Venteicher }
4459da9560cSBryan Venteicher 
4469da9560cSBryan Venteicher static int
4479da9560cSBryan Venteicher vtpci_modern_finalize_features(device_t dev)
4489da9560cSBryan Venteicher {
4499da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
4509da9560cSBryan Venteicher 	uint8_t status;
4519da9560cSBryan Venteicher 
4529da9560cSBryan Venteicher 	sc = device_get_softc(dev);
4539da9560cSBryan Venteicher 
4549da9560cSBryan Venteicher 	/*
4559da9560cSBryan Venteicher 	 * Must re-read the status after setting it to verify the negotiated
4569da9560cSBryan Venteicher 	 * features were accepted by the device.
4579da9560cSBryan Venteicher 	 */
4589da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK);
4599da9560cSBryan Venteicher 
4609da9560cSBryan Venteicher 	status = vtpci_modern_get_status(sc);
4619da9560cSBryan Venteicher 	if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) {
4629da9560cSBryan Venteicher 		device_printf(dev, "desired features were not accepted\n");
4639da9560cSBryan Venteicher 		return (ENOTSUP);
4649da9560cSBryan Venteicher 	}
4659da9560cSBryan Venteicher 
4669da9560cSBryan Venteicher 	return (0);
4679da9560cSBryan Venteicher }
4689da9560cSBryan Venteicher 
469ccb576a8SMina Galić static bool
4709da9560cSBryan Venteicher vtpci_modern_with_feature(device_t dev, uint64_t feature)
4719da9560cSBryan Venteicher {
4729da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
4739da9560cSBryan Venteicher 
4749da9560cSBryan Venteicher 	sc = device_get_softc(dev);
4759da9560cSBryan Venteicher 
4769da9560cSBryan Venteicher 	return (vtpci_with_feature(&sc->vtpci_common, feature));
4779da9560cSBryan Venteicher }
4789da9560cSBryan Venteicher 
4799da9560cSBryan Venteicher static uint64_t
4809da9560cSBryan Venteicher vtpci_modern_read_features(struct vtpci_modern_softc *sc)
4819da9560cSBryan Venteicher {
4829da9560cSBryan Venteicher 	uint32_t features0, features1;
4839da9560cSBryan Venteicher 
4849da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0);
4859da9560cSBryan Venteicher 	features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
4869da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1);
4879da9560cSBryan Venteicher 	features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
4889da9560cSBryan Venteicher 
4899da9560cSBryan Venteicher 	return (((uint64_t) features1 << 32) | features0);
4909da9560cSBryan Venteicher }
4919da9560cSBryan Venteicher 
4929da9560cSBryan Venteicher static void
4939da9560cSBryan Venteicher vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features)
4949da9560cSBryan Venteicher {
4959da9560cSBryan Venteicher 	uint32_t features0, features1;
4969da9560cSBryan Venteicher 
4979da9560cSBryan Venteicher 	features0 = features;
4989da9560cSBryan Venteicher 	features1 = features >> 32;
4999da9560cSBryan Venteicher 
5009da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0);
5019da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0);
5029da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1);
5039da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1);
5049da9560cSBryan Venteicher }
5059da9560cSBryan Venteicher 
5069da9560cSBryan Venteicher static int
507180c0240SMina Galić vtpci_modern_alloc_virtqueues(device_t dev, int nvqs,
5089da9560cSBryan Venteicher     struct vq_alloc_info *vq_info)
5099da9560cSBryan Venteicher {
5109da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
5119da9560cSBryan Venteicher 	struct vtpci_common *cn;
5129da9560cSBryan Venteicher 	uint16_t max_nvqs;
5139da9560cSBryan Venteicher 
5149da9560cSBryan Venteicher 	sc = device_get_softc(dev);
5159da9560cSBryan Venteicher 	cn = &sc->vtpci_common;
5169da9560cSBryan Venteicher 
5179da9560cSBryan Venteicher 	max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ);
5189da9560cSBryan Venteicher 	if (nvqs > max_nvqs) {
5199da9560cSBryan Venteicher 		device_printf(sc->vtpci_dev, "requested virtqueue count %d "
5209da9560cSBryan Venteicher 		    "exceeds max %d\n", nvqs, max_nvqs);
5219da9560cSBryan Venteicher 		return (E2BIG);
5229da9560cSBryan Venteicher 	}
5239da9560cSBryan Venteicher 
524180c0240SMina Galić 	return (vtpci_alloc_virtqueues(cn, nvqs, vq_info));
5259da9560cSBryan Venteicher }
5269da9560cSBryan Venteicher 
5279da9560cSBryan Venteicher static int
5289da9560cSBryan Venteicher vtpci_modern_setup_interrupts(device_t dev, enum intr_type type)
5299da9560cSBryan Venteicher {
5309da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
5319da9560cSBryan Venteicher 	int error;
5329da9560cSBryan Venteicher 
5339da9560cSBryan Venteicher 	sc = device_get_softc(dev);
5349da9560cSBryan Venteicher 
5359da9560cSBryan Venteicher 	error = vtpci_setup_interrupts(&sc->vtpci_common, type);
5369da9560cSBryan Venteicher 	if (error == 0)
5379da9560cSBryan Venteicher 		vtpci_modern_enable_virtqueues(sc);
5389da9560cSBryan Venteicher 
5399da9560cSBryan Venteicher 	return (error);
5409da9560cSBryan Venteicher }
5419da9560cSBryan Venteicher 
5429da9560cSBryan Venteicher static void
5439da9560cSBryan Venteicher vtpci_modern_stop(device_t dev)
5449da9560cSBryan Venteicher {
5459da9560cSBryan Venteicher 	vtpci_modern_reset(device_get_softc(dev));
5469da9560cSBryan Venteicher }
5479da9560cSBryan Venteicher 
5489da9560cSBryan Venteicher static int
5499da9560cSBryan Venteicher vtpci_modern_reinit(device_t dev, uint64_t features)
5509da9560cSBryan Venteicher {
5519da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
5529da9560cSBryan Venteicher 	struct vtpci_common *cn;
5539da9560cSBryan Venteicher 	int error;
5549da9560cSBryan Venteicher 
5559da9560cSBryan Venteicher 	sc = device_get_softc(dev);
5569da9560cSBryan Venteicher 	cn = &sc->vtpci_common;
5579da9560cSBryan Venteicher 
5589da9560cSBryan Venteicher 	/*
5599da9560cSBryan Venteicher 	 * Redrive the device initialization. This is a bit of an abuse of
5609da9560cSBryan Venteicher 	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
5619da9560cSBryan Venteicher 	 * play nice.
5629da9560cSBryan Venteicher 	 *
5639da9560cSBryan Venteicher 	 * We do not allow the host device to change from what was originally
5649da9560cSBryan Venteicher 	 * negotiated beyond what the guest driver changed. MSIX state should
5659da9560cSBryan Venteicher 	 * not change, number of virtqueues and their size remain the same, etc.
5669da9560cSBryan Venteicher 	 * This will need to be rethought when we want to support migration.
5679da9560cSBryan Venteicher 	 */
5689da9560cSBryan Venteicher 
5699da9560cSBryan Venteicher 	if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
5709da9560cSBryan Venteicher 		vtpci_modern_stop(dev);
5719da9560cSBryan Venteicher 
5729da9560cSBryan Venteicher 	/*
5739da9560cSBryan Venteicher 	 * Quickly drive the status through ACK and DRIVER. The device does
5749da9560cSBryan Venteicher 	 * not become usable again until DRIVER_OK in reinit complete.
5759da9560cSBryan Venteicher 	 */
5769da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
5779da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
5789da9560cSBryan Venteicher 
5799da9560cSBryan Venteicher 	/*
5809da9560cSBryan Venteicher 	 * TODO: Check that features are not added as to what was
5819da9560cSBryan Venteicher 	 * originally negotiated.
5829da9560cSBryan Venteicher 	 */
5839da9560cSBryan Venteicher 	vtpci_modern_negotiate_features(dev, features);
5849da9560cSBryan Venteicher 	error = vtpci_modern_finalize_features(dev);
5859da9560cSBryan Venteicher 	if (error) {
5869da9560cSBryan Venteicher 		device_printf(dev, "cannot finalize features during reinit\n");
5879da9560cSBryan Venteicher 		return (error);
5889da9560cSBryan Venteicher 	}
5899da9560cSBryan Venteicher 
5909da9560cSBryan Venteicher 	error = vtpci_reinit(cn);
5919da9560cSBryan Venteicher 	if (error)
5929da9560cSBryan Venteicher 		return (error);
5939da9560cSBryan Venteicher 
5949da9560cSBryan Venteicher 	return (0);
5959da9560cSBryan Venteicher }
5969da9560cSBryan Venteicher 
5979da9560cSBryan Venteicher static void
5989da9560cSBryan Venteicher vtpci_modern_reinit_complete(device_t dev)
5999da9560cSBryan Venteicher {
6009da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
6019da9560cSBryan Venteicher 
6029da9560cSBryan Venteicher 	sc = device_get_softc(dev);
6039da9560cSBryan Venteicher 
6049da9560cSBryan Venteicher 	vtpci_modern_enable_virtqueues(sc);
6059da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
6069da9560cSBryan Venteicher }
6079da9560cSBryan Venteicher 
6089da9560cSBryan Venteicher static void
6099da9560cSBryan Venteicher vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
6109da9560cSBryan Venteicher {
6119da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
6129da9560cSBryan Venteicher 
6139da9560cSBryan Venteicher 	sc = device_get_softc(dev);
6149da9560cSBryan Venteicher 
6159da9560cSBryan Venteicher 	vtpci_modern_write_notify_2(sc, offset, queue);
6169da9560cSBryan Venteicher }
6179da9560cSBryan Venteicher 
6189da9560cSBryan Venteicher static uint8_t
6199da9560cSBryan Venteicher vtpci_modern_get_status(struct vtpci_modern_softc *sc)
6209da9560cSBryan Venteicher {
6219da9560cSBryan Venteicher 	return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS));
6229da9560cSBryan Venteicher }
6239da9560cSBryan Venteicher 
6249da9560cSBryan Venteicher static void
6259da9560cSBryan Venteicher vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status)
6269da9560cSBryan Venteicher {
6279da9560cSBryan Venteicher 	if (status != VIRTIO_CONFIG_STATUS_RESET)
6289da9560cSBryan Venteicher 		status |= vtpci_modern_get_status(sc);
6299da9560cSBryan Venteicher 
6309da9560cSBryan Venteicher 	vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status);
6319da9560cSBryan Venteicher }
6329da9560cSBryan Venteicher 
6339da9560cSBryan Venteicher static int
6349da9560cSBryan Venteicher vtpci_modern_config_generation(device_t dev)
6359da9560cSBryan Venteicher {
6369da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
6379da9560cSBryan Venteicher 	uint8_t gen;
6389da9560cSBryan Venteicher 
6399da9560cSBryan Venteicher 	sc = device_get_softc(dev);
6409da9560cSBryan Venteicher 	gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION);
6419da9560cSBryan Venteicher 
6429da9560cSBryan Venteicher 	return (gen);
6439da9560cSBryan Venteicher }
6449da9560cSBryan Venteicher 
6459da9560cSBryan Venteicher static void
6469da9560cSBryan Venteicher vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst,
6479da9560cSBryan Venteicher     int length)
6489da9560cSBryan Venteicher {
6499da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
6509da9560cSBryan Venteicher 
6519da9560cSBryan Venteicher 	sc = device_get_softc(dev);
6529da9560cSBryan Venteicher 
6539da9560cSBryan Venteicher 	if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
6549da9560cSBryan Venteicher 		panic("%s: attempt to read dev config but not present",
6559da9560cSBryan Venteicher 		    __func__);
6569da9560cSBryan Venteicher 	}
6579da9560cSBryan Venteicher 
6589da9560cSBryan Venteicher 	switch (length) {
6599da9560cSBryan Venteicher 	case 1:
6609da9560cSBryan Venteicher 		*(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset);
6619da9560cSBryan Venteicher 		break;
6629da9560cSBryan Venteicher 	case 2:
6639da9560cSBryan Venteicher 		*(uint16_t *) dst = virtio_htog16(true,
6649da9560cSBryan Venteicher 		    vtpci_modern_read_device_2(sc, offset));
6659da9560cSBryan Venteicher 		break;
6669da9560cSBryan Venteicher 	case 4:
6679da9560cSBryan Venteicher 		*(uint32_t *) dst = virtio_htog32(true,
6689da9560cSBryan Venteicher 		    vtpci_modern_read_device_4(sc, offset));
6699da9560cSBryan Venteicher 		break;
6709da9560cSBryan Venteicher 	case 8:
6719da9560cSBryan Venteicher 		*(uint64_t *) dst = virtio_htog64(true,
6729da9560cSBryan Venteicher 		    vtpci_modern_read_device_8(sc, offset));
6739da9560cSBryan Venteicher 		break;
6749da9560cSBryan Venteicher 	default:
6759da9560cSBryan Venteicher 		panic("%s: device %s invalid device read length %d offset %d",
6769da9560cSBryan Venteicher 		    __func__, device_get_nameunit(dev), length, (int) offset);
6779da9560cSBryan Venteicher 	}
6789da9560cSBryan Venteicher }
6799da9560cSBryan Venteicher 
6809da9560cSBryan Venteicher static void
6816c4f9516SAlex Richardson vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src,
6829da9560cSBryan Venteicher     int length)
6839da9560cSBryan Venteicher {
6849da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
6859da9560cSBryan Venteicher 
6869da9560cSBryan Venteicher 	sc = device_get_softc(dev);
6879da9560cSBryan Venteicher 
6889da9560cSBryan Venteicher 	if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
6899da9560cSBryan Venteicher 		panic("%s: attempt to write dev config but not present",
6909da9560cSBryan Venteicher 		    __func__);
6919da9560cSBryan Venteicher 	}
6929da9560cSBryan Venteicher 
6939da9560cSBryan Venteicher 	switch (length) {
6949da9560cSBryan Venteicher 	case 1:
6956c4f9516SAlex Richardson 		vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src);
6969da9560cSBryan Venteicher 		break;
6979da9560cSBryan Venteicher 	case 2: {
6986c4f9516SAlex Richardson 		uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src);
6999da9560cSBryan Venteicher 		vtpci_modern_write_device_2(sc, offset, val);
7009da9560cSBryan Venteicher 		break;
7019da9560cSBryan Venteicher 	}
7029da9560cSBryan Venteicher 	case 4: {
7036c4f9516SAlex Richardson 		uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src);
7049da9560cSBryan Venteicher 		vtpci_modern_write_device_4(sc, offset, val);
7059da9560cSBryan Venteicher 		break;
7069da9560cSBryan Venteicher 	}
7079da9560cSBryan Venteicher 	case 8: {
7086c4f9516SAlex Richardson 		uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src);
7099da9560cSBryan Venteicher 		vtpci_modern_write_device_8(sc, offset, val);
7109da9560cSBryan Venteicher 		break;
7119da9560cSBryan Venteicher 	}
7129da9560cSBryan Venteicher 	default:
7139da9560cSBryan Venteicher 		panic("%s: device %s invalid device write length %d offset %d",
7149da9560cSBryan Venteicher 		    __func__, device_get_nameunit(dev), length, (int) offset);
7159da9560cSBryan Venteicher 	}
7169da9560cSBryan Venteicher }
7179da9560cSBryan Venteicher 
7189da9560cSBryan Venteicher static int
7199da9560cSBryan Venteicher vtpci_modern_probe_configs(device_t dev)
7209da9560cSBryan Venteicher {
7219da9560cSBryan Venteicher 	int error;
7229da9560cSBryan Venteicher 
7239da9560cSBryan Venteicher 	/*
7249da9560cSBryan Venteicher 	 * These config capabilities must be present. The DEVICE_CFG
7259da9560cSBryan Venteicher 	 * capability is only present if the device requires it.
7269da9560cSBryan Venteicher 	 */
7279da9560cSBryan Venteicher 
7289da9560cSBryan Venteicher 	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL);
7299da9560cSBryan Venteicher 	if (error) {
7309da9560cSBryan Venteicher 		device_printf(dev, "cannot find COMMON_CFG capability\n");
7319da9560cSBryan Venteicher 		return (error);
7329da9560cSBryan Venteicher 	}
7339da9560cSBryan Venteicher 
7349da9560cSBryan Venteicher 	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL);
7359da9560cSBryan Venteicher 	if (error) {
7369da9560cSBryan Venteicher 		device_printf(dev, "cannot find NOTIFY_CFG capability\n");
7379da9560cSBryan Venteicher 		return (error);
7389da9560cSBryan Venteicher 	}
7399da9560cSBryan Venteicher 
7409da9560cSBryan Venteicher 	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL);
7419da9560cSBryan Venteicher 	if (error) {
7429da9560cSBryan Venteicher 		device_printf(dev, "cannot find ISR_CFG capability\n");
7439da9560cSBryan Venteicher 		return (error);
7449da9560cSBryan Venteicher 	}
7459da9560cSBryan Venteicher 
7469da9560cSBryan Venteicher 	return (0);
7479da9560cSBryan Venteicher }
7489da9560cSBryan Venteicher 
7499da9560cSBryan Venteicher static int
7509da9560cSBryan Venteicher vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset)
7519da9560cSBryan Venteicher {
7529da9560cSBryan Venteicher 	uint32_t type, bar;
7539da9560cSBryan Venteicher 	int capreg, error;
7549da9560cSBryan Venteicher 
7559da9560cSBryan Venteicher 	for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg);
7569da9560cSBryan Venteicher 	     error == 0;
7579da9560cSBryan Venteicher 	     error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) {
7589da9560cSBryan Venteicher 
7599da9560cSBryan Venteicher 		type = pci_read_config(dev, capreg +
7609da9560cSBryan Venteicher 		    offsetof(struct virtio_pci_cap, cfg_type), 1);
7619da9560cSBryan Venteicher 		bar = pci_read_config(dev, capreg +
7629da9560cSBryan Venteicher 		    offsetof(struct virtio_pci_cap, bar), 1);
7639da9560cSBryan Venteicher 
7649da9560cSBryan Venteicher 		/* Must ignore reserved BARs. */
7659da9560cSBryan Venteicher 		if (bar >= VTPCI_MODERN_MAX_BARS)
7669da9560cSBryan Venteicher 			continue;
7679da9560cSBryan Venteicher 
7689da9560cSBryan Venteicher 		if (type == cfg_type) {
7699da9560cSBryan Venteicher 			if (cap_offset != NULL)
7709da9560cSBryan Venteicher 				*cap_offset = capreg;
7719da9560cSBryan Venteicher 			break;
7729da9560cSBryan Venteicher 		}
7739da9560cSBryan Venteicher 	}
7749da9560cSBryan Venteicher 
7759da9560cSBryan Venteicher 	return (error);
7769da9560cSBryan Venteicher }
7779da9560cSBryan Venteicher 
7789da9560cSBryan Venteicher static int
7799da9560cSBryan Venteicher vtpci_modern_map_common_config(struct vtpci_modern_softc *sc)
7809da9560cSBryan Venteicher {
7819da9560cSBryan Venteicher 	device_t dev;
7829da9560cSBryan Venteicher 	int error;
7839da9560cSBryan Venteicher 
7849da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
7859da9560cSBryan Venteicher 
7869da9560cSBryan Venteicher 	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG,
7879da9560cSBryan Venteicher 	    sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map);
7889da9560cSBryan Venteicher 	if (error) {
7899da9560cSBryan Venteicher 		device_printf(dev, "cannot find cap COMMON_CFG resource\n");
7909da9560cSBryan Venteicher 		return (error);
7919da9560cSBryan Venteicher 	}
7929da9560cSBryan Venteicher 
7939da9560cSBryan Venteicher 	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map);
7949da9560cSBryan Venteicher 	if (error) {
7959da9560cSBryan Venteicher 		device_printf(dev, "cannot alloc resource for COMMON_CFG\n");
7969da9560cSBryan Venteicher 		return (error);
7979da9560cSBryan Venteicher 	}
7989da9560cSBryan Venteicher 
7999da9560cSBryan Venteicher 	return (0);
8009da9560cSBryan Venteicher }
8019da9560cSBryan Venteicher 
8029da9560cSBryan Venteicher static int
8039da9560cSBryan Venteicher vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc)
8049da9560cSBryan Venteicher {
8059da9560cSBryan Venteicher 	device_t dev;
8069da9560cSBryan Venteicher 	int cap_offset, error;
8079da9560cSBryan Venteicher 
8089da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
8099da9560cSBryan Venteicher 
8109da9560cSBryan Venteicher 	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG,
8119da9560cSBryan Venteicher 	    -1, 2, &sc->vtpci_notify_res_map);
8129da9560cSBryan Venteicher 	if (error) {
8139da9560cSBryan Venteicher 		device_printf(dev, "cannot find cap NOTIFY_CFG resource\n");
8149da9560cSBryan Venteicher 		return (error);
8159da9560cSBryan Venteicher 	}
8169da9560cSBryan Venteicher 
8179da9560cSBryan Venteicher 	cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset;
8189da9560cSBryan Venteicher 
8199da9560cSBryan Venteicher 	sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset +
8209da9560cSBryan Venteicher 	    offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4);
8219da9560cSBryan Venteicher 
8229da9560cSBryan Venteicher 	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map);
8239da9560cSBryan Venteicher 	if (error) {
8249da9560cSBryan Venteicher 		device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n");
8259da9560cSBryan Venteicher 		return (error);
8269da9560cSBryan Venteicher 	}
8279da9560cSBryan Venteicher 
8289da9560cSBryan Venteicher 	return (0);
8299da9560cSBryan Venteicher }
8309da9560cSBryan Venteicher 
8319da9560cSBryan Venteicher static int
8329da9560cSBryan Venteicher vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc)
8339da9560cSBryan Venteicher {
8349da9560cSBryan Venteicher 	device_t dev;
8359da9560cSBryan Venteicher 	int error;
8369da9560cSBryan Venteicher 
8379da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
8389da9560cSBryan Venteicher 
8399da9560cSBryan Venteicher 	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG,
8409da9560cSBryan Venteicher 	    sizeof(uint8_t), 1, &sc->vtpci_isr_res_map);
8419da9560cSBryan Venteicher 	if (error) {
8429da9560cSBryan Venteicher 		device_printf(dev, "cannot find cap ISR_CFG resource\n");
8439da9560cSBryan Venteicher 		return (error);
8449da9560cSBryan Venteicher 	}
8459da9560cSBryan Venteicher 
8469da9560cSBryan Venteicher 	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map);
8479da9560cSBryan Venteicher 	if (error) {
8489da9560cSBryan Venteicher 		device_printf(dev, "cannot alloc resource for ISR_CFG\n");
8499da9560cSBryan Venteicher 		return (error);
8509da9560cSBryan Venteicher 	}
8519da9560cSBryan Venteicher 
8529da9560cSBryan Venteicher 	return (0);
8539da9560cSBryan Venteicher }
8549da9560cSBryan Venteicher 
8559da9560cSBryan Venteicher static int
8569da9560cSBryan Venteicher vtpci_modern_map_device_config(struct vtpci_modern_softc *sc)
8579da9560cSBryan Venteicher {
8589da9560cSBryan Venteicher 	device_t dev;
8599da9560cSBryan Venteicher 	int error;
8609da9560cSBryan Venteicher 
8619da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
8629da9560cSBryan Venteicher 
8639da9560cSBryan Venteicher 	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG,
8649da9560cSBryan Venteicher 	    -1, 4, &sc->vtpci_device_res_map);
8659da9560cSBryan Venteicher 	if (error == ENOENT) {
8669da9560cSBryan Venteicher 		/* Device configuration is optional depending on device. */
8679da9560cSBryan Venteicher 		return (0);
8689da9560cSBryan Venteicher 	} else if (error) {
8699da9560cSBryan Venteicher 		device_printf(dev, "cannot find cap DEVICE_CFG resource\n");
8709da9560cSBryan Venteicher 		return (error);
8719da9560cSBryan Venteicher 	}
8729da9560cSBryan Venteicher 
8739da9560cSBryan Venteicher 	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map);
8749da9560cSBryan Venteicher 	if (error) {
8759da9560cSBryan Venteicher 		device_printf(dev, "cannot alloc resource for DEVICE_CFG\n");
8769da9560cSBryan Venteicher 		return (error);
8779da9560cSBryan Venteicher 	}
8789da9560cSBryan Venteicher 
8799da9560cSBryan Venteicher 	return (0);
8809da9560cSBryan Venteicher }
8819da9560cSBryan Venteicher 
8829da9560cSBryan Venteicher static int
8839da9560cSBryan Venteicher vtpci_modern_map_configs(struct vtpci_modern_softc *sc)
8849da9560cSBryan Venteicher {
8859da9560cSBryan Venteicher 	int error;
8869da9560cSBryan Venteicher 
8879da9560cSBryan Venteicher 	error = vtpci_modern_map_common_config(sc);
8889da9560cSBryan Venteicher 	if (error)
8899da9560cSBryan Venteicher 		return (error);
8909da9560cSBryan Venteicher 
8919da9560cSBryan Venteicher 	error = vtpci_modern_map_notify_config(sc);
8929da9560cSBryan Venteicher 	if (error)
8939da9560cSBryan Venteicher 		return (error);
8949da9560cSBryan Venteicher 
8959da9560cSBryan Venteicher 	error = vtpci_modern_map_isr_config(sc);
8969da9560cSBryan Venteicher 	if (error)
8979da9560cSBryan Venteicher 		return (error);
8989da9560cSBryan Venteicher 
8999da9560cSBryan Venteicher 	error = vtpci_modern_map_device_config(sc);
9009da9560cSBryan Venteicher 	if (error)
9019da9560cSBryan Venteicher 		return (error);
9029da9560cSBryan Venteicher 
9039da9560cSBryan Venteicher 	vtpci_modern_alloc_msix_resource(sc);
9049da9560cSBryan Venteicher 
9059da9560cSBryan Venteicher 	return (0);
9069da9560cSBryan Venteicher }
9079da9560cSBryan Venteicher 
9089da9560cSBryan Venteicher static void
9099da9560cSBryan Venteicher vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc)
9109da9560cSBryan Venteicher {
9119da9560cSBryan Venteicher 
9129da9560cSBryan Venteicher 	vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map);
9139da9560cSBryan Venteicher 	vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map);
9149da9560cSBryan Venteicher 	vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map);
9159da9560cSBryan Venteicher 	vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map);
9169da9560cSBryan Venteicher 
9179da9560cSBryan Venteicher 	vtpci_modern_free_bar_resources(sc);
9189da9560cSBryan Venteicher 	vtpci_modern_free_msix_resource(sc);
9199da9560cSBryan Venteicher 
9209da9560cSBryan Venteicher 	sc->vtpci_notify_offset_multiplier = 0;
9219da9560cSBryan Venteicher }
9229da9560cSBryan Venteicher 
9239da9560cSBryan Venteicher static int
9249da9560cSBryan Venteicher vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type,
9259da9560cSBryan Venteicher     int min_size, int alignment, struct vtpci_modern_resource_map *res)
9269da9560cSBryan Venteicher {
9279da9560cSBryan Venteicher 	device_t dev;
9289da9560cSBryan Venteicher 	int cap_offset, offset, length, error;
9299da9560cSBryan Venteicher 	uint8_t bar, cap_length;
9309da9560cSBryan Venteicher 
9319da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
9329da9560cSBryan Venteicher 
9339da9560cSBryan Venteicher 	error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset);
9349da9560cSBryan Venteicher 	if (error)
9359da9560cSBryan Venteicher 		return (error);
9369da9560cSBryan Venteicher 
9379da9560cSBryan Venteicher 	cap_length = pci_read_config(dev,
9389da9560cSBryan Venteicher 	    cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1);
9399da9560cSBryan Venteicher 
9409da9560cSBryan Venteicher 	if (cap_length < sizeof(struct virtio_pci_cap)) {
9419da9560cSBryan Venteicher 		device_printf(dev, "cap %u length %d less than expected\n",
9429da9560cSBryan Venteicher 		    cfg_type, cap_length);
9439da9560cSBryan Venteicher 		return (ENXIO);
9449da9560cSBryan Venteicher 	}
9459da9560cSBryan Venteicher 
9469da9560cSBryan Venteicher 	bar = pci_read_config(dev,
9479da9560cSBryan Venteicher 	    cap_offset + offsetof(struct virtio_pci_cap, bar), 1);
9489da9560cSBryan Venteicher 	offset = pci_read_config(dev,
9499da9560cSBryan Venteicher 	    cap_offset + offsetof(struct virtio_pci_cap, offset), 4);
9509da9560cSBryan Venteicher 	length = pci_read_config(dev,
9519da9560cSBryan Venteicher 	    cap_offset + offsetof(struct virtio_pci_cap, length), 4);
9529da9560cSBryan Venteicher 
9539da9560cSBryan Venteicher 	if (min_size != -1 && length < min_size) {
9549da9560cSBryan Venteicher 		device_printf(dev, "cap %u struct length %d less than min %d\n",
9559da9560cSBryan Venteicher 		    cfg_type, length, min_size);
9569da9560cSBryan Venteicher 		return (ENXIO);
9579da9560cSBryan Venteicher 	}
9589da9560cSBryan Venteicher 
9599da9560cSBryan Venteicher 	if (offset % alignment) {
9609da9560cSBryan Venteicher 		device_printf(dev, "cap %u struct offset %d not aligned to %d\n",
9619da9560cSBryan Venteicher 		    cfg_type, offset, alignment);
9629da9560cSBryan Venteicher 		return (ENXIO);
9639da9560cSBryan Venteicher 	}
9649da9560cSBryan Venteicher 
9659da9560cSBryan Venteicher 	/* BMV: TODO Can we determine the size of the BAR here? */
9669da9560cSBryan Venteicher 
9679da9560cSBryan Venteicher 	res->vtrm_cap_offset = cap_offset;
9689da9560cSBryan Venteicher 	res->vtrm_bar = bar;
9699da9560cSBryan Venteicher 	res->vtrm_offset = offset;
9709da9560cSBryan Venteicher 	res->vtrm_length = length;
9719da9560cSBryan Venteicher 	res->vtrm_type = vtpci_modern_bar_type(sc, bar);
9729da9560cSBryan Venteicher 
9739da9560cSBryan Venteicher 	return (0);
9749da9560cSBryan Venteicher }
9759da9560cSBryan Venteicher 
9769da9560cSBryan Venteicher static int
9779da9560cSBryan Venteicher vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar)
9789da9560cSBryan Venteicher {
9799da9560cSBryan Venteicher 	uint32_t val;
9809da9560cSBryan Venteicher 
9819da9560cSBryan Venteicher 	/*
9829da9560cSBryan Venteicher 	 * The BAR described by a config capability may be either an IOPORT or
9839da9560cSBryan Venteicher 	 * MEM, but we must know the type when calling bus_alloc_resource().
9849da9560cSBryan Venteicher 	 */
9859da9560cSBryan Venteicher 	val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4);
9869da9560cSBryan Venteicher 	if (PCI_BAR_IO(val))
9879da9560cSBryan Venteicher 		return (SYS_RES_IOPORT);
9889da9560cSBryan Venteicher 	else
9899da9560cSBryan Venteicher 		return (SYS_RES_MEMORY);
9909da9560cSBryan Venteicher }
9919da9560cSBryan Venteicher 
9929da9560cSBryan Venteicher static struct resource *
9939da9560cSBryan Venteicher vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type)
9949da9560cSBryan Venteicher {
9959da9560cSBryan Venteicher 	struct resource *res;
9969da9560cSBryan Venteicher 
9979da9560cSBryan Venteicher 	MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
9989da9560cSBryan Venteicher 	res = sc->vtpci_bar_res[bar].vtbr_res;
9999da9560cSBryan Venteicher 	MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type);
10009da9560cSBryan Venteicher 
10019da9560cSBryan Venteicher 	return (res);
10029da9560cSBryan Venteicher }
10039da9560cSBryan Venteicher 
10049da9560cSBryan Venteicher static struct resource *
10059da9560cSBryan Venteicher vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar,
10069da9560cSBryan Venteicher     int type)
10079da9560cSBryan Venteicher {
10089da9560cSBryan Venteicher 	struct resource *res;
10099da9560cSBryan Venteicher 	int rid;
10109da9560cSBryan Venteicher 
10119da9560cSBryan Venteicher 	MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
10129da9560cSBryan Venteicher 	MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT);
10139da9560cSBryan Venteicher 
10149da9560cSBryan Venteicher 	res = sc->vtpci_bar_res[bar].vtbr_res;
10159da9560cSBryan Venteicher 	if (res != NULL) {
10169da9560cSBryan Venteicher 		MPASS(sc->vtpci_bar_res[bar].vtbr_type == type);
10179da9560cSBryan Venteicher 		return (res);
10189da9560cSBryan Venteicher 	}
10199da9560cSBryan Venteicher 
10209da9560cSBryan Venteicher 	rid = PCIR_BAR(bar);
10219da9560cSBryan Venteicher 	res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid,
10229da9560cSBryan Venteicher 	    RF_ACTIVE | RF_UNMAPPED);
10239da9560cSBryan Venteicher 	if (res != NULL) {
10249da9560cSBryan Venteicher 		sc->vtpci_bar_res[bar].vtbr_res = res;
10259da9560cSBryan Venteicher 		sc->vtpci_bar_res[bar].vtbr_type = type;
10269da9560cSBryan Venteicher 	}
10279da9560cSBryan Venteicher 
10289da9560cSBryan Venteicher 	return (res);
10299da9560cSBryan Venteicher }
10309da9560cSBryan Venteicher 
10319da9560cSBryan Venteicher static void
10329da9560cSBryan Venteicher vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc)
10339da9560cSBryan Venteicher {
10349da9560cSBryan Venteicher 	device_t dev;
10359da9560cSBryan Venteicher 	struct resource *res;
10369da9560cSBryan Venteicher 	int bar, rid, type;
10379da9560cSBryan Venteicher 
10389da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
10399da9560cSBryan Venteicher 
10409da9560cSBryan Venteicher 	for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) {
10419da9560cSBryan Venteicher 		res = sc->vtpci_bar_res[bar].vtbr_res;
10429da9560cSBryan Venteicher 		type = sc->vtpci_bar_res[bar].vtbr_type;
10439da9560cSBryan Venteicher 
10449da9560cSBryan Venteicher 		if (res != NULL) {
10459da9560cSBryan Venteicher 			rid = PCIR_BAR(bar);
10469da9560cSBryan Venteicher 			bus_release_resource(dev, type, rid, res);
10479da9560cSBryan Venteicher 			sc->vtpci_bar_res[bar].vtbr_res = NULL;
10489da9560cSBryan Venteicher 			sc->vtpci_bar_res[bar].vtbr_type = 0;
10499da9560cSBryan Venteicher 		}
10509da9560cSBryan Venteicher 	}
10519da9560cSBryan Venteicher }
10529da9560cSBryan Venteicher 
10539da9560cSBryan Venteicher static int
10549da9560cSBryan Venteicher vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc,
10559da9560cSBryan Venteicher     struct vtpci_modern_resource_map *map)
10569da9560cSBryan Venteicher {
10579da9560cSBryan Venteicher 	struct resource_map_request req;
10589da9560cSBryan Venteicher 	struct resource *res;
10599da9560cSBryan Venteicher 	int type;
10609da9560cSBryan Venteicher 
10619da9560cSBryan Venteicher 	type = map->vtrm_type;
10629da9560cSBryan Venteicher 
10639da9560cSBryan Venteicher 	res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type);
10649da9560cSBryan Venteicher 	if (res == NULL)
10659da9560cSBryan Venteicher 		return (ENXIO);
10669da9560cSBryan Venteicher 
10679da9560cSBryan Venteicher 	resource_init_map_request(&req);
10689da9560cSBryan Venteicher 	req.offset = map->vtrm_offset;
10699da9560cSBryan Venteicher 	req.length = map->vtrm_length;
10709da9560cSBryan Venteicher 
10719da9560cSBryan Venteicher 	return (bus_map_resource(sc->vtpci_dev, type, res, &req,
10729da9560cSBryan Venteicher 	    &map->vtrm_map));
10739da9560cSBryan Venteicher }
10749da9560cSBryan Venteicher 
10759da9560cSBryan Venteicher static void
10769da9560cSBryan Venteicher vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc,
10779da9560cSBryan Venteicher     struct vtpci_modern_resource_map *map)
10789da9560cSBryan Venteicher {
10799da9560cSBryan Venteicher 	struct resource *res;
10809da9560cSBryan Venteicher 	int type;
10819da9560cSBryan Venteicher 
10829da9560cSBryan Venteicher 	type = map->vtrm_type;
10839da9560cSBryan Venteicher 	res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type);
10849da9560cSBryan Venteicher 
10859da9560cSBryan Venteicher 	if (res != NULL && map->vtrm_map.r_size != 0) {
10869da9560cSBryan Venteicher 		bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map);
10879da9560cSBryan Venteicher 		bzero(map, sizeof(struct vtpci_modern_resource_map));
10889da9560cSBryan Venteicher 	}
10899da9560cSBryan Venteicher }
10909da9560cSBryan Venteicher 
10919da9560cSBryan Venteicher static void
10929da9560cSBryan Venteicher vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc)
10939da9560cSBryan Venteicher {
10949da9560cSBryan Venteicher 	device_t dev;
10959da9560cSBryan Venteicher 	int bar;
10969da9560cSBryan Venteicher 
10979da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
10989da9560cSBryan Venteicher 
10999da9560cSBryan Venteicher 	if (!vtpci_is_msix_available(&sc->vtpci_common) ||
11009da9560cSBryan Venteicher 	    (bar = pci_msix_table_bar(dev)) == -1)
11019da9560cSBryan Venteicher 		return;
11029da9560cSBryan Venteicher 
11039da9560cSBryan Venteicher 	/* TODO: Can this BAR be in the 0-5 range? */
11049da9560cSBryan Venteicher 	sc->vtpci_msix_bar = bar;
11059da9560cSBryan Venteicher 	if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
11069da9560cSBryan Venteicher 	    &bar, RF_ACTIVE)) == NULL)
11079da9560cSBryan Venteicher 		device_printf(dev, "Unable to map MSIX table\n");
11089da9560cSBryan Venteicher }
11099da9560cSBryan Venteicher 
11109da9560cSBryan Venteicher static void
11119da9560cSBryan Venteicher vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc)
11129da9560cSBryan Venteicher {
11139da9560cSBryan Venteicher 	device_t dev;
11149da9560cSBryan Venteicher 
11159da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
11169da9560cSBryan Venteicher 
11179da9560cSBryan Venteicher 	if (sc->vtpci_msix_res != NULL) {
11189da9560cSBryan Venteicher 		bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar,
11199da9560cSBryan Venteicher 		    sc->vtpci_msix_res);
11209da9560cSBryan Venteicher 		sc->vtpci_msix_bar = 0;
11219da9560cSBryan Venteicher 		sc->vtpci_msix_res = NULL;
11229da9560cSBryan Venteicher 	}
11239da9560cSBryan Venteicher }
11249da9560cSBryan Venteicher 
11259da9560cSBryan Venteicher static void
11269da9560cSBryan Venteicher vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc)
11279da9560cSBryan Venteicher {
11289da9560cSBryan Venteicher 	device_t dev, child;
11299da9560cSBryan Venteicher 
11309da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
11319da9560cSBryan Venteicher 	child = vtpci_child_device(&sc->vtpci_common);
11329da9560cSBryan Venteicher 
11339da9560cSBryan Venteicher 	if (child == NULL || device_get_state(child) != DS_NOTPRESENT)
11349da9560cSBryan Venteicher 		return;
11359da9560cSBryan Venteicher 
11369da9560cSBryan Venteicher 	if (device_probe(child) != 0)
11379da9560cSBryan Venteicher 		return;
11389da9560cSBryan Venteicher 
11399da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
11409da9560cSBryan Venteicher 
11419da9560cSBryan Venteicher 	if (device_attach(child) != 0) {
11429da9560cSBryan Venteicher 		vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
11439da9560cSBryan Venteicher 		/* Reset state for later attempt. */
11449da9560cSBryan Venteicher 		vtpci_modern_child_detached(dev, child);
11459da9560cSBryan Venteicher 	} else {
11469da9560cSBryan Venteicher 		vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
11479da9560cSBryan Venteicher 		VIRTIO_ATTACH_COMPLETED(child);
11489da9560cSBryan Venteicher 	}
11499da9560cSBryan Venteicher }
11509da9560cSBryan Venteicher 
11519da9560cSBryan Venteicher static int
11529da9560cSBryan Venteicher vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset,
11539da9560cSBryan Venteicher     struct vtpci_interrupt *intr)
11549da9560cSBryan Venteicher {
11559da9560cSBryan Venteicher 	uint16_t vector;
11569da9560cSBryan Venteicher 
11579da9560cSBryan Venteicher 	if (intr != NULL) {
11589da9560cSBryan Venteicher 		/* Map from guest rid to host vector. */
11599da9560cSBryan Venteicher 		vector = intr->vti_rid - 1;
11609da9560cSBryan Venteicher 	} else
11619da9560cSBryan Venteicher 		vector = VIRTIO_MSI_NO_VECTOR;
11629da9560cSBryan Venteicher 
11639da9560cSBryan Venteicher 	vtpci_modern_write_common_2(sc, offset, vector);
11649da9560cSBryan Venteicher 	return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV);
11659da9560cSBryan Venteicher }
11669da9560cSBryan Venteicher 
11679da9560cSBryan Venteicher static int
11689da9560cSBryan Venteicher vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
11699da9560cSBryan Venteicher {
11709da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
11719da9560cSBryan Venteicher 	int error;
11729da9560cSBryan Venteicher 
11739da9560cSBryan Venteicher 	sc = device_get_softc(dev);
11749da9560cSBryan Venteicher 
11759da9560cSBryan Venteicher 	error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr);
11769da9560cSBryan Venteicher 	if (error) {
11779da9560cSBryan Venteicher 		device_printf(dev,
11789da9560cSBryan Venteicher 		    "unable to register config MSIX interrupt\n");
11799da9560cSBryan Venteicher 		return (error);
11809da9560cSBryan Venteicher 	}
11819da9560cSBryan Venteicher 
11829da9560cSBryan Venteicher 	return (0);
11839da9560cSBryan Venteicher }
11849da9560cSBryan Venteicher 
11859da9560cSBryan Venteicher static int
11869da9560cSBryan Venteicher vtpci_modern_register_vq_msix(device_t dev, int idx,
11879da9560cSBryan Venteicher     struct vtpci_interrupt *intr)
11889da9560cSBryan Venteicher {
11899da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
11909da9560cSBryan Venteicher 	int error;
11919da9560cSBryan Venteicher 
11929da9560cSBryan Venteicher 	sc = device_get_softc(dev);
11939da9560cSBryan Venteicher 
11949da9560cSBryan Venteicher 	vtpci_modern_select_virtqueue(sc, idx);
11959da9560cSBryan Venteicher 	error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr);
11969da9560cSBryan Venteicher 	if (error) {
11979da9560cSBryan Venteicher 		device_printf(dev,
11989da9560cSBryan Venteicher 		    "unable to register virtqueue MSIX interrupt\n");
11999da9560cSBryan Venteicher 		return (error);
12009da9560cSBryan Venteicher 	}
12019da9560cSBryan Venteicher 
12029da9560cSBryan Venteicher 	return (0);
12039da9560cSBryan Venteicher }
12049da9560cSBryan Venteicher 
12059da9560cSBryan Venteicher static void
12069da9560cSBryan Venteicher vtpci_modern_reset(struct vtpci_modern_softc *sc)
12079da9560cSBryan Venteicher {
12089da9560cSBryan Venteicher 	/*
12099da9560cSBryan Venteicher 	 * Setting the status to RESET sets the host device to the
12109da9560cSBryan Venteicher 	 * original, uninitialized state. Must poll the status until
12119da9560cSBryan Venteicher 	 * the reset is complete.
12129da9560cSBryan Venteicher 	 */
12139da9560cSBryan Venteicher 	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);
12149da9560cSBryan Venteicher 
12159da9560cSBryan Venteicher 	while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
12169da9560cSBryan Venteicher 		cpu_spinwait();
12179da9560cSBryan Venteicher }
12189da9560cSBryan Venteicher 
12199da9560cSBryan Venteicher static void
12209da9560cSBryan Venteicher vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx)
12219da9560cSBryan Venteicher {
12229da9560cSBryan Venteicher 	vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx);
12239da9560cSBryan Venteicher }
12249da9560cSBryan Venteicher 
12259da9560cSBryan Venteicher static uint8_t
12269da9560cSBryan Venteicher vtpci_modern_read_isr(device_t dev)
12279da9560cSBryan Venteicher {
12289da9560cSBryan Venteicher 	return (vtpci_modern_read_isr_1(device_get_softc(dev), 0));
12299da9560cSBryan Venteicher }
12309da9560cSBryan Venteicher 
12319da9560cSBryan Venteicher static uint16_t
12329da9560cSBryan Venteicher vtpci_modern_get_vq_size(device_t dev, int idx)
12339da9560cSBryan Venteicher {
12349da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
12359da9560cSBryan Venteicher 
12369da9560cSBryan Venteicher 	sc = device_get_softc(dev);
12379da9560cSBryan Venteicher 
12389da9560cSBryan Venteicher 	vtpci_modern_select_virtqueue(sc, idx);
12399da9560cSBryan Venteicher 	return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE));
12409da9560cSBryan Venteicher }
12419da9560cSBryan Venteicher 
12429da9560cSBryan Venteicher static bus_size_t
12439da9560cSBryan Venteicher vtpci_modern_get_vq_notify_off(device_t dev, int idx)
12449da9560cSBryan Venteicher {
12459da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
12469da9560cSBryan Venteicher 	uint16_t q_notify_off;
12479da9560cSBryan Venteicher 
12489da9560cSBryan Venteicher 	sc = device_get_softc(dev);
12499da9560cSBryan Venteicher 
12509da9560cSBryan Venteicher 	vtpci_modern_select_virtqueue(sc, idx);
12519da9560cSBryan Venteicher 	q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF);
12529da9560cSBryan Venteicher 
12539da9560cSBryan Venteicher 	return (q_notify_off * sc->vtpci_notify_offset_multiplier);
12549da9560cSBryan Venteicher }
12559da9560cSBryan Venteicher 
12569da9560cSBryan Venteicher static void
12579da9560cSBryan Venteicher vtpci_modern_set_vq(device_t dev, struct virtqueue *vq)
12589da9560cSBryan Venteicher {
12599da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
12609da9560cSBryan Venteicher 
12619da9560cSBryan Venteicher 	sc = device_get_softc(dev);
12629da9560cSBryan Venteicher 
12639da9560cSBryan Venteicher 	vtpci_modern_select_virtqueue(sc, virtqueue_index(vq));
12649da9560cSBryan Venteicher 
12659da9560cSBryan Venteicher 	/* BMV: Currently we never adjust the device's proposed VQ size. */
12669da9560cSBryan Venteicher 	vtpci_modern_write_common_2(sc,
12679da9560cSBryan Venteicher 	    VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq));
12689da9560cSBryan Venteicher 
12699da9560cSBryan Venteicher 	vtpci_modern_write_common_8(sc,
12709da9560cSBryan Venteicher 	    VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq));
12719da9560cSBryan Venteicher 	vtpci_modern_write_common_8(sc,
12729da9560cSBryan Venteicher 	    VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq));
12739da9560cSBryan Venteicher         vtpci_modern_write_common_8(sc,
12749da9560cSBryan Venteicher 	    VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq));
12759da9560cSBryan Venteicher }
12769da9560cSBryan Venteicher 
12779da9560cSBryan Venteicher static void
12789da9560cSBryan Venteicher vtpci_modern_disable_vq(device_t dev, int idx)
12799da9560cSBryan Venteicher {
12809da9560cSBryan Venteicher 	struct vtpci_modern_softc *sc;
12819da9560cSBryan Venteicher 
12829da9560cSBryan Venteicher 	sc = device_get_softc(dev);
12839da9560cSBryan Venteicher 
12849da9560cSBryan Venteicher 	vtpci_modern_select_virtqueue(sc, idx);
12859da9560cSBryan Venteicher 	vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL);
12869da9560cSBryan Venteicher 	vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL);
12879da9560cSBryan Venteicher         vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL);
12889da9560cSBryan Venteicher }
12899da9560cSBryan Venteicher 
12909da9560cSBryan Venteicher static void
12919da9560cSBryan Venteicher vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc)
12929da9560cSBryan Venteicher {
12939da9560cSBryan Venteicher 	int idx;
12949da9560cSBryan Venteicher 
12959da9560cSBryan Venteicher 	for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) {
12969da9560cSBryan Venteicher 		vtpci_modern_select_virtqueue(sc, idx);
12979da9560cSBryan Venteicher 		vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1);
12989da9560cSBryan Venteicher 	}
12999da9560cSBryan Venteicher }
13009da9560cSBryan Venteicher 
13019da9560cSBryan Venteicher static uint8_t
13029da9560cSBryan Venteicher vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off)
13039da9560cSBryan Venteicher {
13049da9560cSBryan Venteicher 	return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off));
13059da9560cSBryan Venteicher }
13069da9560cSBryan Venteicher 
13079da9560cSBryan Venteicher static uint16_t
13089da9560cSBryan Venteicher vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off)
13099da9560cSBryan Venteicher {
1310fb53b42eSAlfredo Dal'Ava Junior 	return virtio_htog16(true,
1311fb53b42eSAlfredo Dal'Ava Junior 			bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off));
13129da9560cSBryan Venteicher }
13139da9560cSBryan Venteicher 
13149da9560cSBryan Venteicher static uint32_t
13159da9560cSBryan Venteicher vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off)
13169da9560cSBryan Venteicher {
1317fb53b42eSAlfredo Dal'Ava Junior 	return virtio_htog32(true,
1318fb53b42eSAlfredo Dal'Ava Junior 			bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off));
13199da9560cSBryan Venteicher }
13209da9560cSBryan Venteicher 
13219da9560cSBryan Venteicher static void
13229da9560cSBryan Venteicher vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off,
13239da9560cSBryan Venteicher     uint8_t val)
13249da9560cSBryan Venteicher {
13259da9560cSBryan Venteicher 	bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val);
13269da9560cSBryan Venteicher }
13279da9560cSBryan Venteicher 
13289da9560cSBryan Venteicher static void
13299da9560cSBryan Venteicher vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off,
13309da9560cSBryan Venteicher     uint16_t val)
13319da9560cSBryan Venteicher {
1332fb53b42eSAlfredo Dal'Ava Junior 	bus_write_2(&sc->vtpci_common_res_map.vtrm_map,
1333fb53b42eSAlfredo Dal'Ava Junior 			off, virtio_gtoh16(true, val));
13349da9560cSBryan Venteicher }
13359da9560cSBryan Venteicher 
13369da9560cSBryan Venteicher static void
13379da9560cSBryan Venteicher vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off,
13389da9560cSBryan Venteicher     uint32_t val)
13399da9560cSBryan Venteicher {
1340fb53b42eSAlfredo Dal'Ava Junior 	bus_write_4(&sc->vtpci_common_res_map.vtrm_map,
1341fb53b42eSAlfredo Dal'Ava Junior 			off, virtio_gtoh32(true, val));
13429da9560cSBryan Venteicher }
13439da9560cSBryan Venteicher 
13449da9560cSBryan Venteicher static void
13459da9560cSBryan Venteicher vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off,
13469da9560cSBryan Venteicher     uint64_t val)
13479da9560cSBryan Venteicher {
13489da9560cSBryan Venteicher 	uint32_t val0, val1;
13499da9560cSBryan Venteicher 
13509da9560cSBryan Venteicher 	val0 = (uint32_t) val;
13519da9560cSBryan Venteicher 	val1 = val >> 32;
13529da9560cSBryan Venteicher 
13539da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, off, val0);
13549da9560cSBryan Venteicher 	vtpci_modern_write_common_4(sc, off + 4, val1);
13559da9560cSBryan Venteicher }
13569da9560cSBryan Venteicher 
13579da9560cSBryan Venteicher static void
13589da9560cSBryan Venteicher vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off,
13599da9560cSBryan Venteicher     uint16_t val)
13609da9560cSBryan Venteicher {
13619da9560cSBryan Venteicher 	bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val);
13629da9560cSBryan Venteicher }
13639da9560cSBryan Venteicher 
13649da9560cSBryan Venteicher static uint8_t
13659da9560cSBryan Venteicher vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off)
13669da9560cSBryan Venteicher {
13679da9560cSBryan Venteicher 	return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off));
13689da9560cSBryan Venteicher }
13699da9560cSBryan Venteicher 
13709da9560cSBryan Venteicher static uint8_t
13719da9560cSBryan Venteicher vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off)
13729da9560cSBryan Venteicher {
13739da9560cSBryan Venteicher 	return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off));
13749da9560cSBryan Venteicher }
13759da9560cSBryan Venteicher 
13769da9560cSBryan Venteicher static uint16_t
13779da9560cSBryan Venteicher vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off)
13789da9560cSBryan Venteicher {
13799da9560cSBryan Venteicher 	return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off));
13809da9560cSBryan Venteicher }
13819da9560cSBryan Venteicher 
13829da9560cSBryan Venteicher static uint32_t
13839da9560cSBryan Venteicher vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off)
13849da9560cSBryan Venteicher {
13859da9560cSBryan Venteicher 	return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off));
13869da9560cSBryan Venteicher }
13879da9560cSBryan Venteicher 
13889da9560cSBryan Venteicher static uint64_t
13899da9560cSBryan Venteicher vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off)
13909da9560cSBryan Venteicher {
13919da9560cSBryan Venteicher 	device_t dev;
13929da9560cSBryan Venteicher 	int gen;
13939da9560cSBryan Venteicher 	uint32_t val0, val1;
13949da9560cSBryan Venteicher 
13959da9560cSBryan Venteicher 	dev = sc->vtpci_dev;
13969da9560cSBryan Venteicher 
13979da9560cSBryan Venteicher 	/*
13989da9560cSBryan Venteicher 	 * Treat the 64-bit field as two 32-bit fields. Use the generation
13999da9560cSBryan Venteicher 	 * to ensure a consistent read.
14009da9560cSBryan Venteicher 	 */
14019da9560cSBryan Venteicher 	do {
14029da9560cSBryan Venteicher 		gen = vtpci_modern_config_generation(dev);
14039da9560cSBryan Venteicher 		val0 = vtpci_modern_read_device_4(sc, off);
14049da9560cSBryan Venteicher 		val1 = vtpci_modern_read_device_4(sc, off + 4);
14059da9560cSBryan Venteicher 	} while (gen != vtpci_modern_config_generation(dev));
14069da9560cSBryan Venteicher 
14079da9560cSBryan Venteicher 	return (((uint64_t) val1 << 32) | val0);
14089da9560cSBryan Venteicher }
14099da9560cSBryan Venteicher 
14109da9560cSBryan Venteicher static void
14119da9560cSBryan Venteicher vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off,
14129da9560cSBryan Venteicher     uint8_t val)
14139da9560cSBryan Venteicher {
14149da9560cSBryan Venteicher 	bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val);
14159da9560cSBryan Venteicher }
14169da9560cSBryan Venteicher 
14179da9560cSBryan Venteicher static void
14189da9560cSBryan Venteicher vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off,
14199da9560cSBryan Venteicher     uint16_t val)
14209da9560cSBryan Venteicher {
14219da9560cSBryan Venteicher 	bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val);
14229da9560cSBryan Venteicher }
14239da9560cSBryan Venteicher 
14249da9560cSBryan Venteicher static void
14259da9560cSBryan Venteicher vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off,
14269da9560cSBryan Venteicher     uint32_t val)
14279da9560cSBryan Venteicher {
14289da9560cSBryan Venteicher 	bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val);
14299da9560cSBryan Venteicher }
14309da9560cSBryan Venteicher 
14319da9560cSBryan Venteicher static void
14329da9560cSBryan Venteicher vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off,
14339da9560cSBryan Venteicher     uint64_t val)
14349da9560cSBryan Venteicher {
14359da9560cSBryan Venteicher 	uint32_t val0, val1;
14369da9560cSBryan Venteicher 
14379da9560cSBryan Venteicher 	val0 = (uint32_t) val;
14389da9560cSBryan Venteicher 	val1 = val >> 32;
14399da9560cSBryan Venteicher 
14409da9560cSBryan Venteicher 	vtpci_modern_write_device_4(sc, off, val0);
14419da9560cSBryan Venteicher 	vtpci_modern_write_device_4(sc, off + 4, val1);
14429da9560cSBryan Venteicher }
1443