xref: /freebsd/usr.sbin/bhyve/pci_passthru.c (revision f44ff2aba2d64d1d9312cb55008dc90275ccdc04)
1366f6083SPeter Grehan /*-
24d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
31de7b4b8SPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
6366f6083SPeter Grehan  *
7366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
8366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
9366f6083SPeter Grehan  * are met:
10366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
12366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
13366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
14366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
15366f6083SPeter Grehan  *
16366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26366f6083SPeter Grehan  * SUCH DAMAGE.
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/param.h>
3000ef17beSBartek Rutkowski #ifndef WITHOUT_CAPSICUM
3100ef17beSBartek Rutkowski #include <sys/capsicum.h>
3200ef17beSBartek Rutkowski #endif
33366f6083SPeter Grehan #include <sys/types.h>
345c40acf8SJohn Baldwin #include <sys/mman.h>
35366f6083SPeter Grehan #include <sys/pciio.h>
36366f6083SPeter Grehan #include <sys/ioctl.h>
37e47fe318SCorvin Köhne #include <sys/stat.h>
38366f6083SPeter Grehan 
39366f6083SPeter Grehan #include <dev/io/iodev.h>
402e81a7e8SNeel Natu #include <dev/pci/pcireg.h>
412e81a7e8SNeel Natu 
427fa23353SMark Johnston #include <vm/vm.h>
437fa23353SMark Johnston 
44366f6083SPeter Grehan #include <machine/iodev.h>
457fa23353SMark Johnston #include <machine/vm.h>
46366f6083SPeter Grehan 
47abfa3c39SMarcelo Araujo #ifndef WITHOUT_CAPSICUM
48abfa3c39SMarcelo Araujo #include <capsicum_helpers.h>
49abfa3c39SMarcelo Araujo #endif
50baf753ccSJohn Baldwin #include <ctype.h>
51366f6083SPeter Grehan #include <stdio.h>
52366f6083SPeter Grehan #include <stdlib.h>
53366f6083SPeter Grehan #include <string.h>
54cff92ffdSJohn Baldwin #include <err.h>
5500ef17beSBartek Rutkowski #include <errno.h>
56366f6083SPeter Grehan #include <fcntl.h>
5700ef17beSBartek Rutkowski #include <sysexits.h>
58366f6083SPeter Grehan #include <unistd.h>
59366f6083SPeter Grehan 
60366f6083SPeter Grehan #include <machine/vmm.h>
61621b5090SJohn Baldwin 
62621b5090SJohn Baldwin #include "debug.h"
634d1e669cSPeter Grehan #include "mem.h"
64563fd224SCorvin Köhne #include "pci_passthru.h"
65366f6083SPeter Grehan 
66366f6083SPeter Grehan #ifndef _PATH_DEVPCI
67366f6083SPeter Grehan #define	_PATH_DEVPCI	"/dev/pci"
68366f6083SPeter Grehan #endif
69366f6083SPeter Grehan 
70366f6083SPeter Grehan #define	LEGACY_SUPPORT	1
71366f6083SPeter Grehan 
722e81a7e8SNeel Natu #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
73cd942e0fSPeter Grehan #define MSIX_CAPLEN 12
74cd942e0fSPeter Grehan 
7593cf9317SCorvin Köhne #define PASSTHRU_MMIO_MAX 2
7693cf9317SCorvin Köhne 
77366f6083SPeter Grehan static int pcifd = -1;
78366f6083SPeter Grehan 
7990c3a1b6SCorvin Köhne SET_DECLARE(passthru_dev_set, struct passthru_dev);
8090c3a1b6SCorvin Köhne 
81366f6083SPeter Grehan struct passthru_softc {
82366f6083SPeter Grehan 	struct pci_devinst *psc_pi;
83e47fe318SCorvin Köhne 	/* ROM is handled like a BAR */
84e47fe318SCorvin Köhne 	struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1];
85366f6083SPeter Grehan 	struct {
86366f6083SPeter Grehan 		int		capoff;
87366f6083SPeter Grehan 		int		msgctrl;
88366f6083SPeter Grehan 		int		emulated;
89366f6083SPeter Grehan 	} psc_msi;
90cd942e0fSPeter Grehan 	struct {
91cd942e0fSPeter Grehan 		int		capoff;
92cd942e0fSPeter Grehan 	} psc_msix;
93366f6083SPeter Grehan 	struct pcisel psc_sel;
94931bb7bfSCorvin Köhne 
9593cf9317SCorvin Köhne 	struct passthru_mmio_mapping psc_mmio_map[PASSTHRU_MMIO_MAX];
96931bb7bfSCorvin Köhne 	cfgread_handler psc_pcir_rhandler[PCI_REGMAX + 1];
97931bb7bfSCorvin Köhne 	cfgwrite_handler psc_pcir_whandler[PCI_REGMAX + 1];
98366f6083SPeter Grehan };
99366f6083SPeter Grehan 
100366f6083SPeter Grehan static int
msi_caplen(int msgctrl)101366f6083SPeter Grehan msi_caplen(int msgctrl)
102366f6083SPeter Grehan {
103366f6083SPeter Grehan 	int len;
104366f6083SPeter Grehan 
105366f6083SPeter Grehan 	len = 10;		/* minimum length of msi capability */
106366f6083SPeter Grehan 
107366f6083SPeter Grehan 	if (msgctrl & PCIM_MSICTRL_64BIT)
108366f6083SPeter Grehan 		len += 4;
109366f6083SPeter Grehan 
110366f6083SPeter Grehan #if 0
111366f6083SPeter Grehan 	/*
112366f6083SPeter Grehan 	 * Ignore the 'mask' and 'pending' bits in the MSI capability.
113366f6083SPeter Grehan 	 * We'll let the guest manipulate them directly.
114366f6083SPeter Grehan 	 */
115366f6083SPeter Grehan 	if (msgctrl & PCIM_MSICTRL_VECTOR)
116366f6083SPeter Grehan 		len += 10;
117366f6083SPeter Grehan #endif
118366f6083SPeter Grehan 
119366f6083SPeter Grehan 	return (len);
120366f6083SPeter Grehan }
121366f6083SPeter Grehan 
122563fd224SCorvin Köhne static int
pcifd_init(void)12375ce327aSMark Johnston pcifd_init(void)
12475ce327aSMark Johnston {
125563fd224SCorvin Köhne 	pcifd = open(_PATH_DEVPCI, O_RDWR, 0);
126563fd224SCorvin Köhne 	if (pcifd < 0) {
127563fd224SCorvin Köhne 		warn("failed to open %s", _PATH_DEVPCI);
128563fd224SCorvin Köhne 		return (1);
129563fd224SCorvin Köhne 	}
130563fd224SCorvin Köhne 
131563fd224SCorvin Köhne #ifndef WITHOUT_CAPSICUM
132563fd224SCorvin Köhne 	cap_rights_t pcifd_rights;
133563fd224SCorvin Köhne 	cap_rights_init(&pcifd_rights, CAP_IOCTL, CAP_READ, CAP_WRITE);
134563fd224SCorvin Köhne 	if (caph_rights_limit(pcifd, &pcifd_rights) == -1)
135563fd224SCorvin Köhne 		errx(EX_OSERR, "Unable to apply rights for sandbox");
136563fd224SCorvin Köhne 
137563fd224SCorvin Köhne 	const cap_ioctl_t pcifd_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR,
138baf753ccSJohn Baldwin 		PCIOCBARIO, PCIOCBARMMAP, PCIOCGETCONF };
139563fd224SCorvin Köhne 	if (caph_ioctls_limit(pcifd, pcifd_ioctls, nitems(pcifd_ioctls)) == -1)
140563fd224SCorvin Köhne 		errx(EX_OSERR, "Unable to apply rights for sandbox");
141563fd224SCorvin Köhne #endif
142563fd224SCorvin Köhne 
143563fd224SCorvin Köhne 	return (0);
144563fd224SCorvin Köhne }
145563fd224SCorvin Köhne 
146563fd224SCorvin Köhne uint32_t
pci_host_read_config(const struct pcisel * sel,long reg,int width)14701d53c34SMark Johnston pci_host_read_config(const struct pcisel *sel, long reg, int width)
148366f6083SPeter Grehan {
149bcab868aSJohn Baldwin 	struct pci_io pi;
150bcab868aSJohn Baldwin 
151563fd224SCorvin Köhne 	if (pcifd < 0 && pcifd_init()) {
152563fd224SCorvin Köhne 		return (0);
153563fd224SCorvin Köhne 	}
154563fd224SCorvin Köhne 
155366f6083SPeter Grehan 	bzero(&pi, sizeof(pi));
156366f6083SPeter Grehan 	pi.pi_sel = *sel;
157366f6083SPeter Grehan 	pi.pi_reg = reg;
158366f6083SPeter Grehan 	pi.pi_width = width;
159366f6083SPeter Grehan 
160366f6083SPeter Grehan 	if (ioctl(pcifd, PCIOCREAD, &pi) < 0)
161366f6083SPeter Grehan 		return (0);				/* XXX */
162366f6083SPeter Grehan 	else
163366f6083SPeter Grehan 		return (pi.pi_data);
164366f6083SPeter Grehan }
165366f6083SPeter Grehan 
166563fd224SCorvin Köhne void
pci_host_write_config(const struct pcisel * sel,long reg,int width,uint32_t data)16701d53c34SMark Johnston pci_host_write_config(const struct pcisel *sel, long reg, int width,
16801d53c34SMark Johnston     uint32_t data)
169366f6083SPeter Grehan {
170bcab868aSJohn Baldwin 	struct pci_io pi;
171bcab868aSJohn Baldwin 
172563fd224SCorvin Köhne 	if (pcifd < 0 && pcifd_init()) {
173563fd224SCorvin Köhne 		return;
174563fd224SCorvin Köhne 	}
175563fd224SCorvin Köhne 
176366f6083SPeter Grehan 	bzero(&pi, sizeof(pi));
177366f6083SPeter Grehan 	pi.pi_sel = *sel;
178366f6083SPeter Grehan 	pi.pi_reg = reg;
179366f6083SPeter Grehan 	pi.pi_width = width;
180366f6083SPeter Grehan 	pi.pi_data = data;
181366f6083SPeter Grehan 
182366f6083SPeter Grehan 	(void)ioctl(pcifd, PCIOCWRITE, &pi);		/* XXX */
183366f6083SPeter Grehan }
184366f6083SPeter Grehan 
185366f6083SPeter Grehan #ifdef LEGACY_SUPPORT
186366f6083SPeter Grehan static int
passthru_add_msicap(struct pci_devinst * pi,int msgnum,int nextptr)187366f6083SPeter Grehan passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)
188366f6083SPeter Grehan {
189ed721684SMark Johnston 	int capoff;
190366f6083SPeter Grehan 	struct msicap msicap;
191366f6083SPeter Grehan 	u_char *capdata;
192366f6083SPeter Grehan 
193366f6083SPeter Grehan 	pci_populate_msicap(&msicap, msgnum, nextptr);
194366f6083SPeter Grehan 
195366f6083SPeter Grehan 	/*
196366f6083SPeter Grehan 	 * XXX
197366f6083SPeter Grehan 	 * Copy the msi capability structure in the last 16 bytes of the
198366f6083SPeter Grehan 	 * config space. This is wrong because it could shadow something
199366f6083SPeter Grehan 	 * useful to the device.
200366f6083SPeter Grehan 	 */
201366f6083SPeter Grehan 	capoff = 256 - roundup(sizeof(msicap), 4);
202366f6083SPeter Grehan 	capdata = (u_char *)&msicap;
203ed721684SMark Johnston 	for (size_t i = 0; i < sizeof(msicap); i++)
204366f6083SPeter Grehan 		pci_set_cfgdata8(pi, capoff + i, capdata[i]);
205366f6083SPeter Grehan 
206366f6083SPeter Grehan 	return (capoff);
207366f6083SPeter Grehan }
208366f6083SPeter Grehan #endif	/* LEGACY_SUPPORT */
209366f6083SPeter Grehan 
210366f6083SPeter Grehan static int
cfginitmsi(struct passthru_softc * sc)211366f6083SPeter Grehan cfginitmsi(struct passthru_softc *sc)
212366f6083SPeter Grehan {
2132e81a7e8SNeel Natu 	int i, ptr, capptr, cap, sts, caplen, table_size;
214366f6083SPeter Grehan 	uint32_t u32;
215366f6083SPeter Grehan 	struct pcisel sel;
216366f6083SPeter Grehan 	struct pci_devinst *pi;
217cd942e0fSPeter Grehan 	struct msixcap msixcap;
21832b21dd2SJohn Baldwin 	char *msixcap_ptr;
219366f6083SPeter Grehan 
220366f6083SPeter Grehan 	pi = sc->psc_pi;
221366f6083SPeter Grehan 	sel = sc->psc_sel;
222366f6083SPeter Grehan 
223366f6083SPeter Grehan 	/*
224366f6083SPeter Grehan 	 * Parse the capabilities and cache the location of the MSI
225cd942e0fSPeter Grehan 	 * and MSI-X capabilities.
226366f6083SPeter Grehan 	 */
22701d53c34SMark Johnston 	sts = pci_host_read_config(&sel, PCIR_STATUS, 2);
228366f6083SPeter Grehan 	if (sts & PCIM_STATUS_CAPPRESENT) {
22901d53c34SMark Johnston 		ptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1);
230366f6083SPeter Grehan 		while (ptr != 0 && ptr != 0xff) {
23101d53c34SMark Johnston 			cap = pci_host_read_config(&sel, ptr + PCICAP_ID, 1);
232366f6083SPeter Grehan 			if (cap == PCIY_MSI) {
233366f6083SPeter Grehan 				/*
234366f6083SPeter Grehan 				 * Copy the MSI capability into the config
235366f6083SPeter Grehan 				 * space of the emulated pci device
236366f6083SPeter Grehan 				 */
237366f6083SPeter Grehan 				sc->psc_msi.capoff = ptr;
23801d53c34SMark Johnston 				sc->psc_msi.msgctrl = pci_host_read_config(&sel,
239366f6083SPeter Grehan 				    ptr + 2, 2);
240366f6083SPeter Grehan 				sc->psc_msi.emulated = 0;
241366f6083SPeter Grehan 				caplen = msi_caplen(sc->psc_msi.msgctrl);
242cd942e0fSPeter Grehan 				capptr = ptr;
243366f6083SPeter Grehan 				while (caplen > 0) {
24401d53c34SMark Johnston 					u32 = pci_host_read_config(&sel, capptr,
24501d53c34SMark Johnston 					    4);
246cd942e0fSPeter Grehan 					pci_set_cfgdata32(pi, capptr, u32);
247366f6083SPeter Grehan 					caplen -= 4;
248cd942e0fSPeter Grehan 					capptr += 4;
249366f6083SPeter Grehan 				}
250cd942e0fSPeter Grehan 			} else if (cap == PCIY_MSIX) {
251cd942e0fSPeter Grehan 				/*
252cd942e0fSPeter Grehan 				 * Copy the MSI-X capability
253cd942e0fSPeter Grehan 				 */
254cd942e0fSPeter Grehan 				sc->psc_msix.capoff = ptr;
255cd942e0fSPeter Grehan 				caplen = 12;
25632b21dd2SJohn Baldwin 				msixcap_ptr = (char *)&msixcap;
257cd942e0fSPeter Grehan 				capptr = ptr;
258cd942e0fSPeter Grehan 				while (caplen > 0) {
25901d53c34SMark Johnston 					u32 = pci_host_read_config(&sel, capptr,
26001d53c34SMark Johnston 					    4);
26132b21dd2SJohn Baldwin 					memcpy(msixcap_ptr, &u32, 4);
262cd942e0fSPeter Grehan 					pci_set_cfgdata32(pi, capptr, u32);
263cd942e0fSPeter Grehan 					caplen -= 4;
264cd942e0fSPeter Grehan 					capptr += 4;
26532b21dd2SJohn Baldwin 					msixcap_ptr += 4;
266cd942e0fSPeter Grehan 				}
267366f6083SPeter Grehan 			}
26801d53c34SMark Johnston 			ptr = pci_host_read_config(&sel, ptr + PCICAP_NEXTPTR,
26901d53c34SMark Johnston 			    1);
270366f6083SPeter Grehan 		}
271366f6083SPeter Grehan 	}
272366f6083SPeter Grehan 
2734d1e669cSPeter Grehan 	if (sc->psc_msix.capoff != 0) {
2744d1e669cSPeter Grehan 		pi->pi_msix.pba_bar =
2752e81a7e8SNeel Natu 		    msixcap.pba_info & PCIM_MSIX_BIR_MASK;
2764d1e669cSPeter Grehan 		pi->pi_msix.pba_offset =
2772e81a7e8SNeel Natu 		    msixcap.pba_info & ~PCIM_MSIX_BIR_MASK;
2784d1e669cSPeter Grehan 		pi->pi_msix.table_bar =
2792e81a7e8SNeel Natu 		    msixcap.table_info & PCIM_MSIX_BIR_MASK;
2804d1e669cSPeter Grehan 		pi->pi_msix.table_offset =
2812e81a7e8SNeel Natu 		    msixcap.table_info & ~PCIM_MSIX_BIR_MASK;
282cd942e0fSPeter Grehan 		pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl);
2837a902ec0SNeel Natu 		pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count);
2842e81a7e8SNeel Natu 
2852e81a7e8SNeel Natu 		/* Allocate the emulated MSI-X table array */
2862e81a7e8SNeel Natu 		table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
287994f858aSXin LI 		pi->pi_msix.table = calloc(1, table_size);
2882e81a7e8SNeel Natu 
2892e81a7e8SNeel Natu 		/* Mask all table entries */
2902e81a7e8SNeel Natu 		for (i = 0; i < pi->pi_msix.table_count; i++) {
2912e81a7e8SNeel Natu 			pi->pi_msix.table[i].vector_control |=
2922e81a7e8SNeel Natu 						PCIM_MSIX_VCTRL_MASK;
2932e81a7e8SNeel Natu 		}
2944d1e669cSPeter Grehan 	}
295cd942e0fSPeter Grehan 
296366f6083SPeter Grehan #ifdef LEGACY_SUPPORT
297366f6083SPeter Grehan 	/*
298366f6083SPeter Grehan 	 * If the passthrough device does not support MSI then craft a
299366f6083SPeter Grehan 	 * MSI capability for it. We link the new MSI capability at the
300366f6083SPeter Grehan 	 * head of the list of capabilities.
301366f6083SPeter Grehan 	 */
302366f6083SPeter Grehan 	if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) {
303366f6083SPeter Grehan 		int origptr, msiptr;
30401d53c34SMark Johnston 		origptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1);
305366f6083SPeter Grehan 		msiptr = passthru_add_msicap(pi, 1, origptr);
306366f6083SPeter Grehan 		sc->psc_msi.capoff = msiptr;
307366f6083SPeter Grehan 		sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2);
308366f6083SPeter Grehan 		sc->psc_msi.emulated = 1;
309366f6083SPeter Grehan 		pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr);
310366f6083SPeter Grehan 	}
311366f6083SPeter Grehan #endif
312366f6083SPeter Grehan 
313cd942e0fSPeter Grehan 	/* Make sure one of the capabilities is present */
314cd942e0fSPeter Grehan 	if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0)
315366f6083SPeter Grehan 		return (-1);
316366f6083SPeter Grehan 	else
317366f6083SPeter Grehan 		return (0);
318366f6083SPeter Grehan }
319366f6083SPeter Grehan 
3204d1e669cSPeter Grehan static uint64_t
msix_table_read(struct passthru_softc * sc,uint64_t offset,int size)3214d1e669cSPeter Grehan msix_table_read(struct passthru_softc *sc, uint64_t offset, int size)
322cd942e0fSPeter Grehan {
323cd942e0fSPeter Grehan 	struct pci_devinst *pi;
3244d1e669cSPeter Grehan 	struct msix_table_entry *entry;
325cd942e0fSPeter Grehan 	uint8_t *src8;
326cd942e0fSPeter Grehan 	uint16_t *src16;
327cd942e0fSPeter Grehan 	uint32_t *src32;
328cd942e0fSPeter Grehan 	uint64_t *src64;
3294d1e669cSPeter Grehan 	uint64_t data;
3304d1e669cSPeter Grehan 	size_t entry_offset;
3317fa23353SMark Johnston 	uint32_t table_offset;
3327fa23353SMark Johnston 	int index, table_count;
333cd942e0fSPeter Grehan 
334cd942e0fSPeter Grehan 	pi = sc->psc_pi;
3357fa23353SMark Johnston 
3367fa23353SMark Johnston 	table_offset = pi->pi_msix.table_offset;
3377fa23353SMark Johnston 	table_count = pi->pi_msix.table_count;
3387fa23353SMark Johnston 	if (offset < table_offset ||
3397fa23353SMark Johnston 	    offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
3405c40acf8SJohn Baldwin 		switch (size) {
3415c40acf8SJohn Baldwin 		case 1:
3427fa23353SMark Johnston 			src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
3435c40acf8SJohn Baldwin 			data = *src8;
3445c40acf8SJohn Baldwin 			break;
3455c40acf8SJohn Baldwin 		case 2:
3467fa23353SMark Johnston 			src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
3475c40acf8SJohn Baldwin 			data = *src16;
3485c40acf8SJohn Baldwin 			break;
3495c40acf8SJohn Baldwin 		case 4:
3507fa23353SMark Johnston 			src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
3515c40acf8SJohn Baldwin 			data = *src32;
3525c40acf8SJohn Baldwin 			break;
3535c40acf8SJohn Baldwin 		case 8:
3547fa23353SMark Johnston 			src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
3555c40acf8SJohn Baldwin 			data = *src64;
3565c40acf8SJohn Baldwin 			break;
3575c40acf8SJohn Baldwin 		default:
3585c40acf8SJohn Baldwin 			return (-1);
3595c40acf8SJohn Baldwin 		}
3605c40acf8SJohn Baldwin 		return (data);
3615c40acf8SJohn Baldwin 	}
3625c40acf8SJohn Baldwin 
3637fa23353SMark Johnston 	offset -= table_offset;
364cd942e0fSPeter Grehan 	index = offset / MSIX_TABLE_ENTRY_SIZE;
3657fa23353SMark Johnston 	assert(index < table_count);
3662e81a7e8SNeel Natu 
367cd942e0fSPeter Grehan 	entry = &pi->pi_msix.table[index];
3682e81a7e8SNeel Natu 	entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
369cd942e0fSPeter Grehan 
370cd942e0fSPeter Grehan 	switch (size) {
371cd942e0fSPeter Grehan 	case 1:
3727fa23353SMark Johnston 		src8 = (uint8_t *)((uint8_t *)entry + entry_offset);
3734d1e669cSPeter Grehan 		data = *src8;
374cd942e0fSPeter Grehan 		break;
375cd942e0fSPeter Grehan 	case 2:
3767fa23353SMark Johnston 		src16 = (uint16_t *)((uint8_t *)entry + entry_offset);
3774d1e669cSPeter Grehan 		data = *src16;
378cd942e0fSPeter Grehan 		break;
379cd942e0fSPeter Grehan 	case 4:
3807fa23353SMark Johnston 		src32 = (uint32_t *)((uint8_t *)entry + entry_offset);
3814d1e669cSPeter Grehan 		data = *src32;
382cd942e0fSPeter Grehan 		break;
383cd942e0fSPeter Grehan 	case 8:
3847fa23353SMark Johnston 		src64 = (uint64_t *)((uint8_t *)entry + entry_offset);
3854d1e669cSPeter Grehan 		data = *src64;
386cd942e0fSPeter Grehan 		break;
387cd942e0fSPeter Grehan 	default:
388cd942e0fSPeter Grehan 		return (-1);
389cd942e0fSPeter Grehan 	}
390cd942e0fSPeter Grehan 
3914d1e669cSPeter Grehan 	return (data);
392cd942e0fSPeter Grehan }
393cd942e0fSPeter Grehan 
3944d1e669cSPeter Grehan static void
msix_table_write(struct passthru_softc * sc,uint64_t offset,int size,uint64_t data)3956a284cacSJohn Baldwin msix_table_write(struct passthru_softc *sc, uint64_t offset, int size,
3966a284cacSJohn Baldwin     uint64_t data)
397cd942e0fSPeter Grehan {
398cd942e0fSPeter Grehan 	struct pci_devinst *pi;
399cd942e0fSPeter Grehan 	struct msix_table_entry *entry;
4005c40acf8SJohn Baldwin 	uint8_t *dest8;
4015c40acf8SJohn Baldwin 	uint16_t *dest16;
4025c40acf8SJohn Baldwin 	uint32_t *dest32;
4035c40acf8SJohn Baldwin 	uint64_t *dest64;
4044d1e669cSPeter Grehan 	size_t entry_offset;
4057fa23353SMark Johnston 	uint32_t table_offset, vector_control;
4067fa23353SMark Johnston 	int index, table_count;
407cd942e0fSPeter Grehan 
408cd942e0fSPeter Grehan 	pi = sc->psc_pi;
4097fa23353SMark Johnston 
4107fa23353SMark Johnston 	table_offset = pi->pi_msix.table_offset;
4117fa23353SMark Johnston 	table_count = pi->pi_msix.table_count;
4127fa23353SMark Johnston 	if (offset < table_offset ||
4137fa23353SMark Johnston 	    offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
4145c40acf8SJohn Baldwin 		switch (size) {
4155c40acf8SJohn Baldwin 		case 1:
4167fa23353SMark Johnston 			dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
4175c40acf8SJohn Baldwin 			*dest8 = data;
4185c40acf8SJohn Baldwin 			break;
4195c40acf8SJohn Baldwin 		case 2:
4207fa23353SMark Johnston 			dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
4215c40acf8SJohn Baldwin 			*dest16 = data;
4225c40acf8SJohn Baldwin 			break;
4235c40acf8SJohn Baldwin 		case 4:
4247fa23353SMark Johnston 			dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
4255c40acf8SJohn Baldwin 			*dest32 = data;
4265c40acf8SJohn Baldwin 			break;
4275c40acf8SJohn Baldwin 		case 8:
4287fa23353SMark Johnston 			dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
4295c40acf8SJohn Baldwin 			*dest64 = data;
4305c40acf8SJohn Baldwin 			break;
4315c40acf8SJohn Baldwin 		}
4325c40acf8SJohn Baldwin 		return;
4335c40acf8SJohn Baldwin 	}
4345c40acf8SJohn Baldwin 
4357fa23353SMark Johnston 	offset -= table_offset;
436cd942e0fSPeter Grehan 	index = offset / MSIX_TABLE_ENTRY_SIZE;
4377fa23353SMark Johnston 	assert(index < table_count);
4382e81a7e8SNeel Natu 
439cd942e0fSPeter Grehan 	entry = &pi->pi_msix.table[index];
4402e81a7e8SNeel Natu 	entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
441cd942e0fSPeter Grehan 
442cd942e0fSPeter Grehan 	/* Only 4 byte naturally-aligned writes are supported */
4434d1e669cSPeter Grehan 	assert(size == 4);
4444d1e669cSPeter Grehan 	assert(entry_offset % 4 == 0);
4454d1e669cSPeter Grehan 
446cd942e0fSPeter Grehan 	vector_control = entry->vector_control;
44763898728SMark Johnston 	dest32 = (uint32_t *)((uint8_t *)entry + entry_offset);
4485c40acf8SJohn Baldwin 	*dest32 = data;
449cd942e0fSPeter Grehan 	/* If MSI-X hasn't been enabled, do nothing */
450cd942e0fSPeter Grehan 	if (pi->pi_msix.enabled) {
451cd942e0fSPeter Grehan 		/* If the entry is masked, don't set it up */
452cd942e0fSPeter Grehan 		if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 ||
453cd942e0fSPeter Grehan 		    (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
4547d9ef309SJohn Baldwin 			(void)vm_setup_pptdev_msix(sc->psc_pi->pi_vmctx,
45555888cfaSNeel Natu 			    sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
45655888cfaSNeel Natu 			    sc->psc_sel.pc_func, index, entry->addr,
45755888cfaSNeel Natu 			    entry->msg_data, entry->vector_control);
458cd942e0fSPeter Grehan 		}
459cd942e0fSPeter Grehan 	}
460cd942e0fSPeter Grehan }
461cd942e0fSPeter Grehan 
462cd942e0fSPeter Grehan static int
init_msix_table(struct passthru_softc * sc)4636a284cacSJohn Baldwin init_msix_table(struct passthru_softc *sc)
464cd942e0fSPeter Grehan {
465cd942e0fSPeter Grehan 	struct pci_devinst *pi = sc->psc_pi;
4667fa23353SMark Johnston 	struct pci_bar_mmap pbm;
4677fa23353SMark Johnston 	int b, s, f;
4687fa23353SMark Johnston 	uint32_t table_size, table_offset;
469cd942e0fSPeter Grehan 
470aa12663fSNeel Natu 	assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0);
471aa12663fSNeel Natu 
4722b89a044SNeel Natu 	b = sc->psc_sel.pc_bus;
4732b89a044SNeel Natu 	s = sc->psc_sel.pc_dev;
4742b89a044SNeel Natu 	f = sc->psc_sel.pc_func;
4752b89a044SNeel Natu 
476cd942e0fSPeter Grehan 	/*
4777fa23353SMark Johnston 	 * Map the region of the BAR containing the MSI-X table.  This is
4787fa23353SMark Johnston 	 * necessary for two reasons:
4797fa23353SMark Johnston 	 * 1. The PBA may reside in the first or last page containing the MSI-X
4807fa23353SMark Johnston 	 *    table.
4817fa23353SMark Johnston 	 * 2. While PCI devices are not supposed to use the page(s) containing
4827fa23353SMark Johnston 	 *    the MSI-X table for other purposes, some do in practice.
483cd942e0fSPeter Grehan 	 */
4847fa23353SMark Johnston 	memset(&pbm, 0, sizeof(pbm));
4857fa23353SMark Johnston 	pbm.pbm_sel = sc->psc_sel;
4867fa23353SMark Johnston 	pbm.pbm_flags = PCIIO_BAR_MMAP_RW;
48776b45e68SMark Johnston 	pbm.pbm_reg = PCIR_BAR(pi->pi_msix.table_bar);
4887fa23353SMark Johnston 	pbm.pbm_memattr = VM_MEMATTR_DEVICE;
4897fa23353SMark Johnston 
4907fa23353SMark Johnston 	if (ioctl(pcifd, PCIOCBARMMAP, &pbm) != 0) {
4917fa23353SMark Johnston 		warn("Failed to map MSI-X table BAR on %d/%d/%d", b, s, f);
4927fa23353SMark Johnston 		return (-1);
4937fa23353SMark Johnston 	}
4947fa23353SMark Johnston 	assert(pbm.pbm_bar_off == 0);
4957fa23353SMark Johnston 	pi->pi_msix.mapped_addr = (uint8_t *)(uintptr_t)pbm.pbm_map_base;
4967fa23353SMark Johnston 	pi->pi_msix.mapped_size = pbm.pbm_map_length;
4977fa23353SMark Johnston 
4987a902ec0SNeel Natu 	table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
4997a902ec0SNeel Natu 
5007a902ec0SNeel Natu 	table_size = pi->pi_msix.table_offset - table_offset;
5017a902ec0SNeel Natu 	table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
5027a902ec0SNeel Natu 	table_size = roundup2(table_size, 4096);
5037a902ec0SNeel Natu 
5047a902ec0SNeel Natu 	/*
5054558c11fSMark Johnston 	 * Unmap any pages not containing the table, we do not need to emulate
5067fa23353SMark Johnston 	 * accesses to them.  Avoid releasing address space to help ensure that
5077fa23353SMark Johnston 	 * a buggy out-of-bounds access causes a crash.
5087a902ec0SNeel Natu 	 */
5097fa23353SMark Johnston 	if (table_offset != 0)
5107fa23353SMark Johnston 		if (mprotect(pi->pi_msix.mapped_addr, table_offset,
5117fa23353SMark Johnston 		    PROT_NONE) != 0)
5127fa23353SMark Johnston 			warn("Failed to unmap MSI-X table BAR region");
5137fa23353SMark Johnston 	if (table_offset + table_size != pi->pi_msix.mapped_size)
5144558c11fSMark Johnston 		if (mprotect(
5154558c11fSMark Johnston 		    pi->pi_msix.mapped_addr + table_offset + table_size,
5167fa23353SMark Johnston 		    pi->pi_msix.mapped_size - (table_offset + table_size),
5177fa23353SMark Johnston 		    PROT_NONE) != 0)
5187fa23353SMark Johnston 			warn("Failed to unmap MSI-X table BAR region");
5192b89a044SNeel Natu 
5202b89a044SNeel Natu 	return (0);
521cd942e0fSPeter Grehan }
522cd942e0fSPeter Grehan 
523cd942e0fSPeter Grehan static int
cfginitbar(struct passthru_softc * sc)5246a284cacSJohn Baldwin cfginitbar(struct passthru_softc *sc)
525366f6083SPeter Grehan {
526366f6083SPeter Grehan 	int i, error;
527366f6083SPeter Grehan 	struct pci_devinst *pi;
528366f6083SPeter Grehan 	struct pci_bar_io bar;
529366f6083SPeter Grehan 	enum pcibar_type bartype;
5307a902ec0SNeel Natu 	uint64_t base, size;
531366f6083SPeter Grehan 
532366f6083SPeter Grehan 	pi = sc->psc_pi;
533366f6083SPeter Grehan 
534366f6083SPeter Grehan 	/*
535366f6083SPeter Grehan 	 * Initialize BAR registers
536366f6083SPeter Grehan 	 */
537366f6083SPeter Grehan 	for (i = 0; i <= PCI_BARMAX; i++) {
538366f6083SPeter Grehan 		bzero(&bar, sizeof(bar));
539366f6083SPeter Grehan 		bar.pbi_sel = sc->psc_sel;
540366f6083SPeter Grehan 		bar.pbi_reg = PCIR_BAR(i);
541366f6083SPeter Grehan 
542366f6083SPeter Grehan 		if (ioctl(pcifd, PCIOCGETBAR, &bar) < 0)
543366f6083SPeter Grehan 			continue;
544366f6083SPeter Grehan 
545366f6083SPeter Grehan 		if (PCI_BAR_IO(bar.pbi_base)) {
546366f6083SPeter Grehan 			bartype = PCIBAR_IO;
547366f6083SPeter Grehan 			base = bar.pbi_base & PCIM_BAR_IO_BASE;
548366f6083SPeter Grehan 		} else {
549366f6083SPeter Grehan 			switch (bar.pbi_base & PCIM_BAR_MEM_TYPE) {
550366f6083SPeter Grehan 			case PCIM_BAR_MEM_64:
551366f6083SPeter Grehan 				bartype = PCIBAR_MEM64;
552366f6083SPeter Grehan 				break;
553366f6083SPeter Grehan 			default:
554366f6083SPeter Grehan 				bartype = PCIBAR_MEM32;
555366f6083SPeter Grehan 				break;
556366f6083SPeter Grehan 			}
557366f6083SPeter Grehan 			base = bar.pbi_base & PCIM_BAR_MEM_BASE;
558366f6083SPeter Grehan 		}
5597a902ec0SNeel Natu 		size = bar.pbi_length;
5607a902ec0SNeel Natu 
5617a902ec0SNeel Natu 		if (bartype != PCIBAR_IO) {
5627a902ec0SNeel Natu 			if (((base | size) & PAGE_MASK) != 0) {
563cff92ffdSJohn Baldwin 				warnx("passthru device %d/%d/%d BAR %d: "
5647a902ec0SNeel Natu 				    "base %#lx or size %#lx not page aligned\n",
5657a902ec0SNeel Natu 				    sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
5667a902ec0SNeel Natu 				    sc->psc_sel.pc_func, i, base, size);
5677a902ec0SNeel Natu 				return (-1);
5687a902ec0SNeel Natu 			}
5697a902ec0SNeel Natu 		}
570366f6083SPeter Grehan 
571366f6083SPeter Grehan 		/* Cache information about the "real" BAR */
572366f6083SPeter Grehan 		sc->psc_bar[i].type = bartype;
5737a902ec0SNeel Natu 		sc->psc_bar[i].size = size;
574366f6083SPeter Grehan 		sc->psc_bar[i].addr = base;
575e87a6f3eSCorvin Köhne 		sc->psc_bar[i].lobits = 0;
576366f6083SPeter Grehan 
577366f6083SPeter Grehan 		/* Allocate the BAR in the guest I/O or MMIO space */
578038f5c7bSKonstantin Belousov 		error = pci_emul_alloc_bar(pi, i, bartype, size);
579366f6083SPeter Grehan 		if (error)
580366f6083SPeter Grehan 			return (-1);
581366f6083SPeter Grehan 
582e87a6f3eSCorvin Köhne 		/* Use same lobits as physical bar */
58301d53c34SMark Johnston 		uint8_t lobits = pci_host_read_config(&sc->psc_sel, PCIR_BAR(i),
58401d53c34SMark Johnston 		    0x01);
585e87a6f3eSCorvin Köhne 		if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) {
586e87a6f3eSCorvin Köhne 			lobits &= ~PCIM_BAR_MEM_BASE;
587e87a6f3eSCorvin Köhne 		} else {
588e87a6f3eSCorvin Köhne 			lobits &= ~PCIM_BAR_IO_BASE;
589e87a6f3eSCorvin Köhne 		}
590e87a6f3eSCorvin Köhne 		sc->psc_bar[i].lobits = lobits;
591e87a6f3eSCorvin Köhne 		pi->pi_bar[i].lobits = lobits;
592e87a6f3eSCorvin Köhne 
593366f6083SPeter Grehan 		/*
594366f6083SPeter Grehan 		 * 64-bit BAR takes up two slots so skip the next one.
595366f6083SPeter Grehan 		 */
596366f6083SPeter Grehan 		if (bartype == PCIBAR_MEM64) {
597366f6083SPeter Grehan 			i++;
598366f6083SPeter Grehan 			assert(i <= PCI_BARMAX);
599366f6083SPeter Grehan 			sc->psc_bar[i].type = PCIBAR_MEMHI64;
600366f6083SPeter Grehan 		}
601366f6083SPeter Grehan 	}
602366f6083SPeter Grehan 	return (0);
603366f6083SPeter Grehan }
604366f6083SPeter Grehan 
605366f6083SPeter Grehan static int
cfginit(struct pci_devinst * pi,int bus,int slot,int func)6066a284cacSJohn Baldwin cfginit(struct pci_devinst *pi, int bus, int slot, int func)
607366f6083SPeter Grehan {
608366f6083SPeter Grehan 	int error;
609366f6083SPeter Grehan 	struct passthru_softc *sc;
610*f44ff2abSJohn Baldwin 	uint16_t cmd;
611b6e67875SCorvin Köhne 	uint8_t intline, intpin;
612366f6083SPeter Grehan 
613366f6083SPeter Grehan 	error = 1;
614366f6083SPeter Grehan 	sc = pi->pi_arg;
615366f6083SPeter Grehan 
616366f6083SPeter Grehan 	bzero(&sc->psc_sel, sizeof(struct pcisel));
617366f6083SPeter Grehan 	sc->psc_sel.pc_bus = bus;
618366f6083SPeter Grehan 	sc->psc_sel.pc_dev = slot;
619366f6083SPeter Grehan 	sc->psc_sel.pc_func = func;
620366f6083SPeter Grehan 
621b6e67875SCorvin Köhne 	/*
622*f44ff2abSJohn Baldwin 	 * Copy physical PCI header to virtual config space.  COMMAND,
623*f44ff2abSJohn Baldwin 	 * INTLINE, and INTPIN shouldn't be aligned with their
624*f44ff2abSJohn Baldwin 	 * physical value and they are already set by pci_emul_init().
625b6e67875SCorvin Köhne 	 */
626*f44ff2abSJohn Baldwin 	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
627b6e67875SCorvin Köhne 	intline = pci_get_cfgdata8(pi, PCIR_INTLINE);
628b6e67875SCorvin Köhne 	intpin = pci_get_cfgdata8(pi, PCIR_INTPIN);
629b6e67875SCorvin Köhne 	for (int i = 0; i <= PCIR_MAXLAT; i += 4) {
63001d53c34SMark Johnston 		pci_set_cfgdata32(pi, i,
63101d53c34SMark Johnston 		    pci_host_read_config(&sc->psc_sel, i, 4));
632b6e67875SCorvin Köhne 	}
633*f44ff2abSJohn Baldwin 	pci_set_cfgdata16(pi, PCIR_COMMAND, cmd);
634b6e67875SCorvin Köhne 	pci_set_cfgdata8(pi, PCIR_INTLINE, intline);
635b6e67875SCorvin Köhne 	pci_set_cfgdata8(pi, PCIR_INTPIN, intpin);
636b6e67875SCorvin Köhne 
637cff92ffdSJohn Baldwin 	if (cfginitmsi(sc) != 0) {
638cff92ffdSJohn Baldwin 		warnx("failed to initialize MSI for PCI %d/%d/%d",
639cff92ffdSJohn Baldwin 		    bus, slot, func);
640cd942e0fSPeter Grehan 		goto done;
641cff92ffdSJohn Baldwin 	}
642cd942e0fSPeter Grehan 
6436a284cacSJohn Baldwin 	if (cfginitbar(sc) != 0) {
644cff92ffdSJohn Baldwin 		warnx("failed to initialize BARs for PCI %d/%d/%d",
645cff92ffdSJohn Baldwin 		    bus, slot, func);
646366f6083SPeter Grehan 		goto done;
647cff92ffdSJohn Baldwin 	}
648366f6083SPeter Grehan 
649338a1be8SCorvin Köhne 	if (pci_msix_table_bar(pi) >= 0) {
6506a284cacSJohn Baldwin 		error = init_msix_table(sc);
651f1442847SBjoern A. Zeeb 		if (error != 0) {
652338a1be8SCorvin Köhne 			warnx(
653338a1be8SCorvin Köhne 			    "failed to initialize MSI-X table for PCI %d/%d/%d: %d",
654f1442847SBjoern A. Zeeb 			    bus, slot, func, error);
655f1442847SBjoern A. Zeeb 			goto done;
656f1442847SBjoern A. Zeeb 		}
657338a1be8SCorvin Köhne 	}
658f1442847SBjoern A. Zeeb 
659338a1be8SCorvin Köhne 	error = 0;				/* success */
660366f6083SPeter Grehan done:
661366f6083SPeter Grehan 	return (error);
662366f6083SPeter Grehan }
663366f6083SPeter Grehan 
66493cf9317SCorvin Köhne struct passthru_mmio_mapping *
passthru_get_mmio(struct passthru_softc * sc,int num)66593cf9317SCorvin Köhne passthru_get_mmio(struct passthru_softc *sc, int num)
66693cf9317SCorvin Köhne {
66793cf9317SCorvin Köhne 	assert(sc != NULL);
66893cf9317SCorvin Köhne 	assert(num < PASSTHRU_MMIO_MAX);
66993cf9317SCorvin Köhne 
67093cf9317SCorvin Köhne 	return (&sc->psc_mmio_map[num]);
67193cf9317SCorvin Köhne }
67293cf9317SCorvin Köhne 
67360793ceeSCorvin Köhne struct pcisel *
passthru_get_sel(struct passthru_softc * sc)67460793ceeSCorvin Köhne passthru_get_sel(struct passthru_softc *sc)
67560793ceeSCorvin Köhne {
67660793ceeSCorvin Köhne 	assert(sc != NULL);
67760793ceeSCorvin Köhne 
67860793ceeSCorvin Köhne 	return (&sc->psc_sel);
67960793ceeSCorvin Köhne }
68060793ceeSCorvin Köhne 
681931bb7bfSCorvin Köhne int
set_pcir_handler(struct passthru_softc * sc,int reg,int len,cfgread_handler rhandler,cfgwrite_handler whandler)682931bb7bfSCorvin Köhne set_pcir_handler(struct passthru_softc *sc, int reg, int len,
683931bb7bfSCorvin Köhne     cfgread_handler rhandler, cfgwrite_handler whandler)
684931bb7bfSCorvin Köhne {
685931bb7bfSCorvin Köhne 	if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1)
686931bb7bfSCorvin Köhne 		return (-1);
687931bb7bfSCorvin Köhne 
688931bb7bfSCorvin Köhne 	for (int i = reg; i < reg + len; ++i) {
689931bb7bfSCorvin Köhne 		assert(sc->psc_pcir_rhandler[i] == NULL || rhandler == NULL);
690931bb7bfSCorvin Köhne 		assert(sc->psc_pcir_whandler[i] == NULL || whandler == NULL);
691931bb7bfSCorvin Köhne 		sc->psc_pcir_rhandler[i] = rhandler;
692931bb7bfSCorvin Köhne 		sc->psc_pcir_whandler[i] = whandler;
693931bb7bfSCorvin Köhne 	}
694931bb7bfSCorvin Köhne 
695931bb7bfSCorvin Köhne 	return (0);
696931bb7bfSCorvin Köhne }
697931bb7bfSCorvin Köhne 
698366f6083SPeter Grehan static int
passthru_legacy_config(nvlist_t * nvl,const char * opts)699621b5090SJohn Baldwin passthru_legacy_config(nvlist_t *nvl, const char *opts)
700621b5090SJohn Baldwin {
701baf753ccSJohn Baldwin 	const char *cp;
702baf753ccSJohn Baldwin 	char *tofree;
703621b5090SJohn Baldwin 	char value[16];
704621b5090SJohn Baldwin 	int bus, slot, func;
705621b5090SJohn Baldwin 
706621b5090SJohn Baldwin 	if (opts == NULL)
707621b5090SJohn Baldwin 		return (0);
708621b5090SJohn Baldwin 
709baf753ccSJohn Baldwin 	cp = strchr(opts, ',');
710621b5090SJohn Baldwin 
711baf753ccSJohn Baldwin 	if (strncmp(opts, "ppt", strlen("ppt")) == 0) {
712baf753ccSJohn Baldwin 		tofree = strndup(opts, cp - opts);
713baf753ccSJohn Baldwin 		set_config_value_node(nvl, "pptdev", tofree);
714baf753ccSJohn Baldwin 		free(tofree);
715baf753ccSJohn Baldwin 	} else if (sscanf(opts, "pci0:%d:%d:%d", &bus, &slot, &func) == 3 ||
716baf753ccSJohn Baldwin 	    sscanf(opts, "pci%d:%d:%d", &bus, &slot, &func) == 3 ||
717baf753ccSJohn Baldwin 	    sscanf(opts, "%d/%d/%d", &bus, &slot, &func) == 3) {
718621b5090SJohn Baldwin 		snprintf(value, sizeof(value), "%d", bus);
719621b5090SJohn Baldwin 		set_config_value_node(nvl, "bus", value);
720621b5090SJohn Baldwin 		snprintf(value, sizeof(value), "%d", slot);
721621b5090SJohn Baldwin 		set_config_value_node(nvl, "slot", value);
722621b5090SJohn Baldwin 		snprintf(value, sizeof(value), "%d", func);
723621b5090SJohn Baldwin 		set_config_value_node(nvl, "func", value);
724baf753ccSJohn Baldwin 	} else {
725baf753ccSJohn Baldwin 		EPRINTLN("passthru: invalid options \"%s\"", opts);
726baf753ccSJohn Baldwin 		return (-1);
727baf753ccSJohn Baldwin 	}
728e47fe318SCorvin Köhne 
729baf753ccSJohn Baldwin 	if (cp == NULL) {
7303256b7caSCorvin Köhne 		return (0);
7313256b7caSCorvin Köhne 	}
7323256b7caSCorvin Köhne 
733baf753ccSJohn Baldwin 	return (pci_parse_legacy_config(nvl, cp + 1));
734e47fe318SCorvin Köhne }
735e47fe318SCorvin Köhne 
736e47fe318SCorvin Köhne static int
passthru_init_rom(struct passthru_softc * const sc,const char * const romfile)7376a284cacSJohn Baldwin passthru_init_rom(struct passthru_softc *const sc, const char *const romfile)
738e47fe318SCorvin Köhne {
739e47fe318SCorvin Köhne 	if (romfile == NULL) {
740e47fe318SCorvin Köhne 		return (0);
741e47fe318SCorvin Köhne 	}
742e47fe318SCorvin Köhne 
743e47fe318SCorvin Köhne 	const int fd = open(romfile, O_RDONLY);
744e47fe318SCorvin Köhne 	if (fd < 0) {
745e47fe318SCorvin Köhne 		warnx("%s: can't open romfile \"%s\"", __func__, romfile);
746e47fe318SCorvin Köhne 		return (-1);
747e47fe318SCorvin Köhne 	}
748e47fe318SCorvin Köhne 
749e47fe318SCorvin Köhne 	struct stat sbuf;
750e47fe318SCorvin Köhne 	if (fstat(fd, &sbuf) < 0) {
751e47fe318SCorvin Köhne 		warnx("%s: can't fstat romfile \"%s\"", __func__, romfile);
752e47fe318SCorvin Köhne 		close(fd);
753e47fe318SCorvin Köhne 		return (-1);
754e47fe318SCorvin Köhne 	}
755e47fe318SCorvin Köhne 	const uint64_t rom_size = sbuf.st_size;
756e47fe318SCorvin Köhne 
757e47fe318SCorvin Köhne 	void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd,
758e47fe318SCorvin Köhne 	    0);
759e47fe318SCorvin Köhne 	if (rom_data == MAP_FAILED) {
760e47fe318SCorvin Köhne 		warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__,
761e47fe318SCorvin Köhne 		    romfile, errno);
762e47fe318SCorvin Köhne 		close(fd);
763e47fe318SCorvin Köhne 		return (-1);
764e47fe318SCorvin Köhne 	}
765e47fe318SCorvin Köhne 
766e47fe318SCorvin Köhne 	void *rom_addr;
767e47fe318SCorvin Köhne 	int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr);
768e47fe318SCorvin Köhne 	if (error) {
769e47fe318SCorvin Köhne 		warnx("%s: failed to alloc rom segment", __func__);
770e47fe318SCorvin Köhne 		munmap(rom_data, rom_size);
771e47fe318SCorvin Köhne 		close(fd);
772e47fe318SCorvin Köhne 		return (error);
773e47fe318SCorvin Köhne 	}
774e47fe318SCorvin Köhne 	memcpy(rom_addr, rom_data, rom_size);
775e47fe318SCorvin Köhne 
776e47fe318SCorvin Köhne 	sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM;
777e47fe318SCorvin Köhne 	sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr;
778e47fe318SCorvin Köhne 	sc->psc_bar[PCI_ROM_IDX].size = rom_size;
779e47fe318SCorvin Köhne 
780e47fe318SCorvin Köhne 	munmap(rom_data, rom_size);
781e47fe318SCorvin Köhne 	close(fd);
782e47fe318SCorvin Köhne 
783621b5090SJohn Baldwin 	return (0);
784621b5090SJohn Baldwin }
785621b5090SJohn Baldwin 
786baf753ccSJohn Baldwin static bool
passthru_lookup_pptdev(const char * name,int * bus,int * slot,int * func)787baf753ccSJohn Baldwin passthru_lookup_pptdev(const char *name, int *bus, int *slot, int *func)
788baf753ccSJohn Baldwin {
789baf753ccSJohn Baldwin 	struct pci_conf_io pc;
790baf753ccSJohn Baldwin 	struct pci_conf conf[1];
791baf753ccSJohn Baldwin 	struct pci_match_conf patterns[1];
792baf753ccSJohn Baldwin 	char *cp;
793baf753ccSJohn Baldwin 
794baf753ccSJohn Baldwin 	bzero(&pc, sizeof(struct pci_conf_io));
795baf753ccSJohn Baldwin 	pc.match_buf_len = sizeof(conf);
796baf753ccSJohn Baldwin 	pc.matches = conf;
797baf753ccSJohn Baldwin 
798baf753ccSJohn Baldwin 	bzero(&patterns, sizeof(patterns));
799baf753ccSJohn Baldwin 
800baf753ccSJohn Baldwin 	/*
801baf753ccSJohn Baldwin 	 * The pattern structure requires the unit to be split out from
802baf753ccSJohn Baldwin 	 * the driver name.  Walk backwards from the end of the name to
803baf753ccSJohn Baldwin 	 * find the start of the unit.
804baf753ccSJohn Baldwin 	 */
805baf753ccSJohn Baldwin 	cp = strchr(name, '\0');
806baf753ccSJohn Baldwin 	assert(cp != NULL);
807baf753ccSJohn Baldwin 	while (cp != name && isdigit(cp[-1]))
808baf753ccSJohn Baldwin 		cp--;
809baf753ccSJohn Baldwin 	if (cp == name || !isdigit(*cp)) {
810baf753ccSJohn Baldwin 		EPRINTLN("Invalid passthru device name %s", name);
811baf753ccSJohn Baldwin 		return (false);
812baf753ccSJohn Baldwin 	}
813baf753ccSJohn Baldwin 	if ((size_t)(cp - name) + 1 > sizeof(patterns[0].pd_name)) {
814baf753ccSJohn Baldwin 		EPRINTLN("Passthru device name %s is too long", name);
815baf753ccSJohn Baldwin 		return (false);
816baf753ccSJohn Baldwin 	}
817baf753ccSJohn Baldwin 	memcpy(patterns[0].pd_name, name, cp - name);
818baf753ccSJohn Baldwin 	patterns[0].pd_unit = strtol(cp, &cp, 10);
819baf753ccSJohn Baldwin 	if (*cp != '\0') {
820baf753ccSJohn Baldwin 		EPRINTLN("Invalid passthru device name %s", name);
821baf753ccSJohn Baldwin 		return (false);
822baf753ccSJohn Baldwin 	}
823baf753ccSJohn Baldwin 	patterns[0].flags = PCI_GETCONF_MATCH_NAME | PCI_GETCONF_MATCH_UNIT;
824baf753ccSJohn Baldwin 	pc.num_patterns = 1;
825baf753ccSJohn Baldwin 	pc.pat_buf_len = sizeof(patterns);
826baf753ccSJohn Baldwin 	pc.patterns = patterns;
827baf753ccSJohn Baldwin 
828baf753ccSJohn Baldwin 	if (ioctl(pcifd, PCIOCGETCONF, &pc) == -1) {
829baf753ccSJohn Baldwin 		EPRINTLN("ioctl(PCIOCGETCONF): %s", strerror(errno));
830baf753ccSJohn Baldwin 		return (false);
831baf753ccSJohn Baldwin 	}
832baf753ccSJohn Baldwin 	if (pc.status != PCI_GETCONF_LAST_DEVICE &&
833baf753ccSJohn Baldwin 	    pc.status != PCI_GETCONF_MORE_DEVS) {
834baf753ccSJohn Baldwin 		EPRINTLN("error returned from PCIOCGETCONF ioctl");
835baf753ccSJohn Baldwin 		return (false);
836baf753ccSJohn Baldwin 	}
837baf753ccSJohn Baldwin 	if (pc.num_matches == 0) {
838baf753ccSJohn Baldwin 		EPRINTLN("Passthru device %s not found", name);
839baf753ccSJohn Baldwin 		return (false);
840baf753ccSJohn Baldwin 	}
841baf753ccSJohn Baldwin 
842baf753ccSJohn Baldwin 	if (conf[0].pc_sel.pc_domain != 0) {
843baf753ccSJohn Baldwin 		EPRINTLN("Passthru device %s on unsupported domain", name);
844baf753ccSJohn Baldwin 		return (false);
845baf753ccSJohn Baldwin 	}
846baf753ccSJohn Baldwin 	*bus = conf[0].pc_sel.pc_bus;
847baf753ccSJohn Baldwin 	*slot = conf[0].pc_sel.pc_dev;
848baf753ccSJohn Baldwin 	*func = conf[0].pc_sel.pc_func;
849baf753ccSJohn Baldwin 	return (true);
850baf753ccSJohn Baldwin }
851baf753ccSJohn Baldwin 
852621b5090SJohn Baldwin static int
passthru_init(struct pci_devinst * pi,nvlist_t * nvl)8536a284cacSJohn Baldwin passthru_init(struct pci_devinst *pi, nvlist_t *nvl)
854366f6083SPeter Grehan {
8559b1aa8d6SNeel Natu 	int bus, slot, func, error, memflags;
856366f6083SPeter Grehan 	struct passthru_softc *sc;
85790c3a1b6SCorvin Köhne 	struct passthru_dev **devpp;
85890c3a1b6SCorvin Köhne 	struct passthru_dev *devp, *dev = NULL;
859621b5090SJohn Baldwin 	const char *value;
860366f6083SPeter Grehan 
861366f6083SPeter Grehan 	sc = NULL;
862366f6083SPeter Grehan 	error = 1;
863366f6083SPeter Grehan 
8646a284cacSJohn Baldwin 	memflags = vm_get_memflags(pi->pi_vmctx);
8659b1aa8d6SNeel Natu 	if (!(memflags & VM_MEM_F_WIRED)) {
866cff92ffdSJohn Baldwin 		warnx("passthru requires guest memory to be wired");
867dbb15211SSean Chittenden 		return (error);
8689b1aa8d6SNeel Natu 	}
8699b1aa8d6SNeel Natu 
870563fd224SCorvin Köhne 	if (pcifd < 0 && pcifd_init()) {
871dbb15211SSean Chittenden 		return (error);
872366f6083SPeter Grehan 	}
87300ef17beSBartek Rutkowski 
874621b5090SJohn Baldwin #define GET_INT_CONFIG(var, name) do {					\
875621b5090SJohn Baldwin 	value = get_config_value_node(nvl, name);			\
876621b5090SJohn Baldwin 	if (value == NULL) {						\
877621b5090SJohn Baldwin 		EPRINTLN("passthru: missing required %s setting", name); \
878621b5090SJohn Baldwin 		return (error);						\
879621b5090SJohn Baldwin 	}								\
880621b5090SJohn Baldwin 	var = atoi(value);						\
881621b5090SJohn Baldwin } while (0)
882621b5090SJohn Baldwin 
883baf753ccSJohn Baldwin 	value = get_config_value_node(nvl, "pptdev");
884baf753ccSJohn Baldwin 	if (value != NULL) {
885baf753ccSJohn Baldwin 		if (!passthru_lookup_pptdev(value, &bus, &slot, &func))
886baf753ccSJohn Baldwin 			return (error);
887baf753ccSJohn Baldwin 	} else {
888621b5090SJohn Baldwin 		GET_INT_CONFIG(bus, "bus");
889621b5090SJohn Baldwin 		GET_INT_CONFIG(slot, "slot");
890621b5090SJohn Baldwin 		GET_INT_CONFIG(func, "func");
891baf753ccSJohn Baldwin 	}
892366f6083SPeter Grehan 
8936a284cacSJohn Baldwin 	if (vm_assign_pptdev(pi->pi_vmctx, bus, slot, func) != 0) {
894cff92ffdSJohn Baldwin 		warnx("PCI device at %d/%d/%d is not using the ppt(4) driver",
895cff92ffdSJohn Baldwin 		    bus, slot, func);
896366f6083SPeter Grehan 		goto done;
897cff92ffdSJohn Baldwin 	}
898366f6083SPeter Grehan 
899994f858aSXin LI 	sc = calloc(1, sizeof(struct passthru_softc));
900366f6083SPeter Grehan 
901366f6083SPeter Grehan 	pi->pi_arg = sc;
902366f6083SPeter Grehan 	sc->psc_pi = pi;
903366f6083SPeter Grehan 
904366f6083SPeter Grehan 	/* initialize config space */
9056a284cacSJohn Baldwin 	if ((error = cfginit(pi, bus, slot, func)) != 0)
906e47fe318SCorvin Köhne 		goto done;
907e47fe318SCorvin Köhne 
908e47fe318SCorvin Köhne 	/* initialize ROM */
9096a284cacSJohn Baldwin 	if ((error = passthru_init_rom(sc,
910e47fe318SCorvin Köhne             get_config_value_node(nvl, "rom"))) != 0)
911e47fe318SCorvin Köhne 		goto done;
912e47fe318SCorvin Köhne 
913b6e67875SCorvin Köhne 	/* Emulate most PCI header register. */
914b6e67875SCorvin Köhne 	if ((error = set_pcir_handler(sc, 0, PCIR_MAXLAT + 1,
915b6e67875SCorvin Köhne 	    passthru_cfgread_emulate, passthru_cfgwrite_emulate)) != 0)
916b6e67875SCorvin Köhne 		goto done;
917b6e67875SCorvin Köhne 
918*f44ff2abSJohn Baldwin 	/* Allow access to the physical status register. */
919b6e67875SCorvin Köhne 	if ((error = set_pcir_handler(sc, PCIR_COMMAND, 0x04, NULL, NULL)) != 0)
920b6e67875SCorvin Köhne 		goto done;
921b6e67875SCorvin Köhne 
92290c3a1b6SCorvin Köhne 	SET_FOREACH(devpp, passthru_dev_set) {
92390c3a1b6SCorvin Köhne 		devp = *devpp;
92490c3a1b6SCorvin Köhne 		assert(devp->probe != NULL);
92590c3a1b6SCorvin Köhne 		if (devp->probe(pi) == 0) {
92690c3a1b6SCorvin Köhne 			dev = devp;
92790c3a1b6SCorvin Köhne 			break;
92890c3a1b6SCorvin Köhne 		}
92990c3a1b6SCorvin Köhne 	}
93090c3a1b6SCorvin Köhne 
93190c3a1b6SCorvin Köhne 	if (dev != NULL) {
93290c3a1b6SCorvin Köhne 		error = dev->init(pi, nvl);
93390c3a1b6SCorvin Köhne 		if (error != 0)
93490c3a1b6SCorvin Köhne 			goto done;
93590c3a1b6SCorvin Köhne 	}
93690c3a1b6SCorvin Köhne 
937e47fe318SCorvin Köhne 	error = 0;		/* success */
938366f6083SPeter Grehan done:
939366f6083SPeter Grehan 	if (error) {
94090c3a1b6SCorvin Köhne 		if (dev != NULL)
94190c3a1b6SCorvin Köhne 			dev->deinit(pi);
942366f6083SPeter Grehan 		free(sc);
9436a284cacSJohn Baldwin 		vm_unassign_pptdev(pi->pi_vmctx, bus, slot, func);
944366f6083SPeter Grehan 	}
945366f6083SPeter Grehan 	return (error);
946366f6083SPeter Grehan }
947366f6083SPeter Grehan 
948366f6083SPeter Grehan static int
msicap_access(struct passthru_softc * sc,int coff)949366f6083SPeter Grehan msicap_access(struct passthru_softc *sc, int coff)
950366f6083SPeter Grehan {
951366f6083SPeter Grehan 	int caplen;
952366f6083SPeter Grehan 
953366f6083SPeter Grehan 	if (sc->psc_msi.capoff == 0)
954366f6083SPeter Grehan 		return (0);
955366f6083SPeter Grehan 
956366f6083SPeter Grehan 	caplen = msi_caplen(sc->psc_msi.msgctrl);
957366f6083SPeter Grehan 
958366f6083SPeter Grehan 	if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen)
959366f6083SPeter Grehan 		return (1);
960366f6083SPeter Grehan 	else
961366f6083SPeter Grehan 		return (0);
962366f6083SPeter Grehan }
963366f6083SPeter Grehan 
964366f6083SPeter Grehan static int
msixcap_access(struct passthru_softc * sc,int coff)965cd942e0fSPeter Grehan msixcap_access(struct passthru_softc *sc, int coff)
966cd942e0fSPeter Grehan {
967cd942e0fSPeter Grehan 	if (sc->psc_msix.capoff == 0)
968cd942e0fSPeter Grehan 		return (0);
969cd942e0fSPeter Grehan 
970cd942e0fSPeter Grehan 	return (coff >= sc->psc_msix.capoff &&
971cd942e0fSPeter Grehan 	        coff < sc->psc_msix.capoff + MSIX_CAPLEN);
972cd942e0fSPeter Grehan }
973cd942e0fSPeter Grehan 
974cd942e0fSPeter Grehan static int
passthru_cfgread_default(struct passthru_softc * sc,struct pci_devinst * pi __unused,int coff,int bytes,uint32_t * rv)975931bb7bfSCorvin Köhne passthru_cfgread_default(struct passthru_softc *sc,
976931bb7bfSCorvin Köhne     struct pci_devinst *pi __unused, int coff, int bytes, uint32_t *rv)
977366f6083SPeter Grehan {
978366f6083SPeter Grehan 	/*
979b6e67875SCorvin Köhne 	 * MSI capability is emulated.
980366f6083SPeter Grehan 	 */
981b6e67875SCorvin Köhne 	if (msicap_access(sc, coff) || msixcap_access(sc, coff))
982366f6083SPeter Grehan 		return (-1);
983366f6083SPeter Grehan 
984c7ba149dSJohn Baldwin 	/*
985c7ba149dSJohn Baldwin 	 * Emulate the command register.  If a single read reads both the
986c7ba149dSJohn Baldwin 	 * command and status registers, read the status register from the
987c7ba149dSJohn Baldwin 	 * device's config space.
988c7ba149dSJohn Baldwin 	 */
989c7ba149dSJohn Baldwin 	if (coff == PCIR_COMMAND) {
990c7ba149dSJohn Baldwin 		if (bytes <= 2)
991c7ba149dSJohn Baldwin 			return (-1);
99201d53c34SMark Johnston 		*rv = pci_host_read_config(&sc->psc_sel, PCIR_STATUS, 2) << 16 |
99321368498SPeter Grehan 		    pci_get_cfgdata16(pi, PCIR_COMMAND);
994c7ba149dSJohn Baldwin 		return (0);
995c7ba149dSJohn Baldwin 	}
996c7ba149dSJohn Baldwin 
997366f6083SPeter Grehan 	/* Everything else just read from the device's config space */
99801d53c34SMark Johnston 	*rv = pci_host_read_config(&sc->psc_sel, coff, bytes);
999366f6083SPeter Grehan 
1000366f6083SPeter Grehan 	return (0);
1001366f6083SPeter Grehan }
1002366f6083SPeter Grehan 
1003b6e67875SCorvin Köhne int
passthru_cfgread_emulate(struct passthru_softc * sc __unused,struct pci_devinst * pi __unused,int coff __unused,int bytes __unused,uint32_t * rv __unused)1004b6e67875SCorvin Köhne passthru_cfgread_emulate(struct passthru_softc *sc __unused,
1005b6e67875SCorvin Köhne     struct pci_devinst *pi __unused, int coff __unused, int bytes __unused,
1006b6e67875SCorvin Köhne     uint32_t *rv __unused)
1007b6e67875SCorvin Köhne {
1008b6e67875SCorvin Köhne 	return (-1);
1009b6e67875SCorvin Köhne }
1010b6e67875SCorvin Köhne 
1011366f6083SPeter Grehan static int
passthru_cfgread(struct pci_devinst * pi,int coff,int bytes,uint32_t * rv)1012931bb7bfSCorvin Köhne passthru_cfgread(struct pci_devinst *pi, int coff, int bytes, uint32_t *rv)
1013366f6083SPeter Grehan {
1014366f6083SPeter Grehan 	struct passthru_softc *sc;
1015366f6083SPeter Grehan 
1016366f6083SPeter Grehan 	sc = pi->pi_arg;
1017366f6083SPeter Grehan 
1018931bb7bfSCorvin Köhne 	if (sc->psc_pcir_rhandler[coff] != NULL)
1019931bb7bfSCorvin Köhne 		return (sc->psc_pcir_rhandler[coff](sc, pi, coff, bytes, rv));
1020931bb7bfSCorvin Köhne 
1021931bb7bfSCorvin Köhne 	return (passthru_cfgread_default(sc, pi, coff, bytes, rv));
1022931bb7bfSCorvin Köhne }
1023931bb7bfSCorvin Köhne 
1024931bb7bfSCorvin Köhne static int
passthru_cfgwrite_default(struct passthru_softc * sc,struct pci_devinst * pi,int coff,int bytes,uint32_t val)1025931bb7bfSCorvin Köhne passthru_cfgwrite_default(struct passthru_softc *sc, struct pci_devinst *pi,
1026931bb7bfSCorvin Köhne     int coff, int bytes, uint32_t val)
1027931bb7bfSCorvin Köhne {
1028931bb7bfSCorvin Köhne 	int error, msix_table_entries, i;
1029931bb7bfSCorvin Köhne 	uint16_t cmd_old;
1030931bb7bfSCorvin Köhne 
1031366f6083SPeter Grehan 	/*
1032366f6083SPeter Grehan 	 * MSI capability is emulated
1033366f6083SPeter Grehan 	 */
1034366f6083SPeter Grehan 	if (msicap_access(sc, coff)) {
103521368498SPeter Grehan 		pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,
103621368498SPeter Grehan 		    PCIY_MSI);
10377d9ef309SJohn Baldwin 		error = vm_setup_pptdev_msi(pi->pi_vmctx, sc->psc_sel.pc_bus,
10384f8be175SNeel Natu 			sc->psc_sel.pc_dev, sc->psc_sel.pc_func,
10394f8be175SNeel Natu 			pi->pi_msi.addr, pi->pi_msi.msg_data,
10404f8be175SNeel Natu 			pi->pi_msi.maxmsgnum);
1041cff92ffdSJohn Baldwin 		if (error != 0)
1042cff92ffdSJohn Baldwin 			err(1, "vm_setup_pptdev_msi");
1043366f6083SPeter Grehan 		return (0);
1044366f6083SPeter Grehan 	}
1045366f6083SPeter Grehan 
1046cd942e0fSPeter Grehan 	if (msixcap_access(sc, coff)) {
104721368498SPeter Grehan 		pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff,
104821368498SPeter Grehan 		    PCIY_MSIX);
1049cd942e0fSPeter Grehan 		if (pi->pi_msix.enabled) {
1050cd942e0fSPeter Grehan 			msix_table_entries = pi->pi_msix.table_count;
1051cd942e0fSPeter Grehan 			for (i = 0; i < msix_table_entries; i++) {
10527d9ef309SJohn Baldwin 				error = vm_setup_pptdev_msix(pi->pi_vmctx,
10534f8be175SNeel Natu 				    sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
1054cd942e0fSPeter Grehan 				    sc->psc_sel.pc_func, i,
10554f8be175SNeel Natu 				    pi->pi_msix.table[i].addr,
1056cd942e0fSPeter Grehan 				    pi->pi_msix.table[i].msg_data,
10574f8be175SNeel Natu 				    pi->pi_msix.table[i].vector_control);
1058cd942e0fSPeter Grehan 
1059cff92ffdSJohn Baldwin 				if (error)
1060cff92ffdSJohn Baldwin 					err(1, "vm_setup_pptdev_msix");
1061cd942e0fSPeter Grehan 			}
10621925586eSJohn Baldwin 		} else {
10636a284cacSJohn Baldwin 			error = vm_disable_pptdev_msix(pi->pi_vmctx,
10646a284cacSJohn Baldwin 			    sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
10656a284cacSJohn Baldwin 			    sc->psc_sel.pc_func);
10661925586eSJohn Baldwin 			if (error)
10671925586eSJohn Baldwin 				err(1, "vm_disable_pptdev_msix");
1068cd942e0fSPeter Grehan 		}
1069cd942e0fSPeter Grehan 		return (0);
1070cd942e0fSPeter Grehan 	}
1071cd942e0fSPeter Grehan 
1072366f6083SPeter Grehan 	/*
1073*f44ff2abSJohn Baldwin 	 * The command register is emulated, but the status register
1074*f44ff2abSJohn Baldwin 	 * is passed through.
1075366f6083SPeter Grehan 	 */
1076*f44ff2abSJohn Baldwin 	if (coff == PCIR_COMMAND) {
1077*f44ff2abSJohn Baldwin 		if (bytes <= 2)
1078*f44ff2abSJohn Baldwin 			return (-1);
1079*f44ff2abSJohn Baldwin 
1080*f44ff2abSJohn Baldwin 		/* Update the physical status register. */
1081*f44ff2abSJohn Baldwin 		pci_host_write_config(&sc->psc_sel, PCIR_STATUS, val >> 16, 2);
1082*f44ff2abSJohn Baldwin 
1083*f44ff2abSJohn Baldwin 		/* Update the virtual command register. */
1084*f44ff2abSJohn Baldwin 		cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND);
1085*f44ff2abSJohn Baldwin 		pci_set_cfgdata16(pi, PCIR_COMMAND, val & 0xffff);
1086*f44ff2abSJohn Baldwin 		pci_emul_cmd_changed(pi, cmd_old);
1087*f44ff2abSJohn Baldwin 		return (0);
1088366f6083SPeter Grehan 	}
1089366f6083SPeter Grehan 
109001d53c34SMark Johnston 	pci_host_write_config(&sc->psc_sel, coff, bytes, val);
1091366f6083SPeter Grehan 
1092366f6083SPeter Grehan 	return (0);
1093366f6083SPeter Grehan }
1094366f6083SPeter Grehan 
1095b6e67875SCorvin Köhne int
passthru_cfgwrite_emulate(struct passthru_softc * sc __unused,struct pci_devinst * pi __unused,int coff __unused,int bytes __unused,uint32_t val __unused)1096b6e67875SCorvin Köhne passthru_cfgwrite_emulate(struct passthru_softc *sc __unused,
1097b6e67875SCorvin Köhne     struct pci_devinst *pi __unused, int coff __unused, int bytes __unused,
1098b6e67875SCorvin Köhne     uint32_t val __unused)
1099b6e67875SCorvin Köhne {
1100b6e67875SCorvin Köhne 	return (-1);
1101b6e67875SCorvin Köhne }
1102b6e67875SCorvin Köhne 
1103931bb7bfSCorvin Köhne static int
passthru_cfgwrite(struct pci_devinst * pi,int coff,int bytes,uint32_t val)1104931bb7bfSCorvin Köhne passthru_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val)
1105931bb7bfSCorvin Köhne {
1106931bb7bfSCorvin Köhne 	struct passthru_softc *sc;
1107931bb7bfSCorvin Köhne 
1108931bb7bfSCorvin Köhne 	sc = pi->pi_arg;
1109931bb7bfSCorvin Köhne 
1110931bb7bfSCorvin Köhne 	if (sc->psc_pcir_whandler[coff] != NULL)
1111931bb7bfSCorvin Köhne 		return (sc->psc_pcir_whandler[coff](sc, pi, coff, bytes, val));
1112931bb7bfSCorvin Köhne 
1113931bb7bfSCorvin Köhne 	return (passthru_cfgwrite_default(sc, pi, coff, bytes, val));
1114931bb7bfSCorvin Köhne }
1115931bb7bfSCorvin Köhne 
1116366f6083SPeter Grehan static void
passthru_write(struct pci_devinst * pi,int baridx,uint64_t offset,int size,uint64_t value)11176a284cacSJohn Baldwin passthru_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size,
11186a284cacSJohn Baldwin     uint64_t value)
1119366f6083SPeter Grehan {
1120366f6083SPeter Grehan 	struct passthru_softc *sc;
112142375556SMark Johnston 	struct pci_bar_ioreq pio;
1122366f6083SPeter Grehan 
1123366f6083SPeter Grehan 	sc = pi->pi_arg;
1124366f6083SPeter Grehan 
1125aa12663fSNeel Natu 	if (baridx == pci_msix_table_bar(pi)) {
11266a284cacSJohn Baldwin 		msix_table_write(sc, offset, size, value);
11274d1e669cSPeter Grehan 	} else {
11284d1e669cSPeter Grehan 		assert(pi->pi_bar[baridx].type == PCIBAR_IO);
112942375556SMark Johnston 		assert(size == 1 || size == 2 || size == 4);
113042375556SMark Johnston 		assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX);
1131366f6083SPeter Grehan 
113242375556SMark Johnston 		bzero(&pio, sizeof(pio));
113342375556SMark Johnston 		pio.pbi_sel = sc->psc_sel;
113442375556SMark Johnston 		pio.pbi_op = PCIBARIO_WRITE;
113542375556SMark Johnston 		pio.pbi_bar = baridx;
113642375556SMark Johnston 		pio.pbi_offset = (uint32_t)offset;
113742375556SMark Johnston 		pio.pbi_width = size;
113842375556SMark Johnston 		pio.pbi_value = (uint32_t)value;
113942375556SMark Johnston 
114042375556SMark Johnston 		(void)ioctl(pcifd, PCIOCBARIO, &pio);
1141366f6083SPeter Grehan 	}
11424d1e669cSPeter Grehan }
1143366f6083SPeter Grehan 
11444d1e669cSPeter Grehan static uint64_t
passthru_read(struct pci_devinst * pi,int baridx,uint64_t offset,int size)11456a284cacSJohn Baldwin passthru_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size)
1146366f6083SPeter Grehan {
1147366f6083SPeter Grehan 	struct passthru_softc *sc;
114842375556SMark Johnston 	struct pci_bar_ioreq pio;
11494d1e669cSPeter Grehan 	uint64_t val;
1150366f6083SPeter Grehan 
1151366f6083SPeter Grehan 	sc = pi->pi_arg;
1152366f6083SPeter Grehan 
1153aa12663fSNeel Natu 	if (baridx == pci_msix_table_bar(pi)) {
11544d1e669cSPeter Grehan 		val = msix_table_read(sc, offset, size);
11554d1e669cSPeter Grehan 	} else {
11564d1e669cSPeter Grehan 		assert(pi->pi_bar[baridx].type == PCIBAR_IO);
115742375556SMark Johnston 		assert(size == 1 || size == 2 || size == 4);
115842375556SMark Johnston 		assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX);
1159366f6083SPeter Grehan 
116042375556SMark Johnston 		bzero(&pio, sizeof(pio));
116142375556SMark Johnston 		pio.pbi_sel = sc->psc_sel;
116242375556SMark Johnston 		pio.pbi_op = PCIBARIO_READ;
116342375556SMark Johnston 		pio.pbi_bar = baridx;
116442375556SMark Johnston 		pio.pbi_offset = (uint32_t)offset;
116542375556SMark Johnston 		pio.pbi_width = size;
1166366f6083SPeter Grehan 
116742375556SMark Johnston 		(void)ioctl(pcifd, PCIOCBARIO, &pio);
116842375556SMark Johnston 
116942375556SMark Johnston 		val = pio.pbi_value;
11704d1e669cSPeter Grehan 	}
11714d1e669cSPeter Grehan 
11724d1e669cSPeter Grehan 	return (val);
1173366f6083SPeter Grehan }
1174366f6083SPeter Grehan 
1175f8a6ec2dSD Scott Phillips static void
passthru_msix_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)11766a284cacSJohn Baldwin passthru_msix_addr(struct pci_devinst *pi, int baridx, int enabled,
11776a284cacSJohn Baldwin     uint64_t address)
1178f8a6ec2dSD Scott Phillips {
1179f8a6ec2dSD Scott Phillips 	struct passthru_softc *sc;
1180f8a6ec2dSD Scott Phillips 	size_t remaining;
1181f8a6ec2dSD Scott Phillips 	uint32_t table_size, table_offset;
1182f8a6ec2dSD Scott Phillips 
1183f8a6ec2dSD Scott Phillips 	sc = pi->pi_arg;
1184f8a6ec2dSD Scott Phillips 	table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
1185f8a6ec2dSD Scott Phillips 	if (table_offset > 0) {
1186f8a6ec2dSD Scott Phillips 		if (!enabled) {
11876a284cacSJohn Baldwin 			if (vm_unmap_pptdev_mmio(pi->pi_vmctx,
11886a284cacSJohn Baldwin 						 sc->psc_sel.pc_bus,
1189f8a6ec2dSD Scott Phillips 						 sc->psc_sel.pc_dev,
1190f8a6ec2dSD Scott Phillips 						 sc->psc_sel.pc_func, address,
1191f8a6ec2dSD Scott Phillips 						 table_offset) != 0)
1192f8a6ec2dSD Scott Phillips 				warnx("pci_passthru: unmap_pptdev_mmio failed");
1193f8a6ec2dSD Scott Phillips 		} else {
11946a284cacSJohn Baldwin 			if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1195f8a6ec2dSD Scott Phillips 					       sc->psc_sel.pc_dev,
1196f8a6ec2dSD Scott Phillips 					       sc->psc_sel.pc_func, address,
1197f8a6ec2dSD Scott Phillips 					       table_offset,
1198f8a6ec2dSD Scott Phillips 					       sc->psc_bar[baridx].addr) != 0)
1199f8a6ec2dSD Scott Phillips 				warnx("pci_passthru: map_pptdev_mmio failed");
1200f8a6ec2dSD Scott Phillips 		}
1201f8a6ec2dSD Scott Phillips 	}
1202f8a6ec2dSD Scott Phillips 	table_size = pi->pi_msix.table_offset - table_offset;
1203f8a6ec2dSD Scott Phillips 	table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
1204f8a6ec2dSD Scott Phillips 	table_size = roundup2(table_size, 4096);
1205f8a6ec2dSD Scott Phillips 	remaining = pi->pi_bar[baridx].size - table_offset - table_size;
1206f8a6ec2dSD Scott Phillips 	if (remaining > 0) {
1207f8a6ec2dSD Scott Phillips 		address += table_offset + table_size;
1208f8a6ec2dSD Scott Phillips 		if (!enabled) {
12096a284cacSJohn Baldwin 			if (vm_unmap_pptdev_mmio(pi->pi_vmctx,
12106a284cacSJohn Baldwin 						 sc->psc_sel.pc_bus,
1211f8a6ec2dSD Scott Phillips 						 sc->psc_sel.pc_dev,
1212f8a6ec2dSD Scott Phillips 						 sc->psc_sel.pc_func, address,
1213f8a6ec2dSD Scott Phillips 						 remaining) != 0)
1214f8a6ec2dSD Scott Phillips 				warnx("pci_passthru: unmap_pptdev_mmio failed");
1215f8a6ec2dSD Scott Phillips 		} else {
12166a284cacSJohn Baldwin 			if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1217f8a6ec2dSD Scott Phillips 					       sc->psc_sel.pc_dev,
1218f8a6ec2dSD Scott Phillips 					       sc->psc_sel.pc_func, address,
1219f8a6ec2dSD Scott Phillips 					       remaining,
1220f8a6ec2dSD Scott Phillips 					       sc->psc_bar[baridx].addr +
1221f8a6ec2dSD Scott Phillips 					       table_offset + table_size) != 0)
1222f8a6ec2dSD Scott Phillips 				warnx("pci_passthru: map_pptdev_mmio failed");
1223f8a6ec2dSD Scott Phillips 		}
1224f8a6ec2dSD Scott Phillips 	}
1225f8a6ec2dSD Scott Phillips }
1226f8a6ec2dSD Scott Phillips 
1227f8a6ec2dSD Scott Phillips static void
passthru_mmio_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)12286a284cacSJohn Baldwin passthru_mmio_addr(struct pci_devinst *pi, int baridx, int enabled,
12296a284cacSJohn Baldwin     uint64_t address)
1230f8a6ec2dSD Scott Phillips {
1231f8a6ec2dSD Scott Phillips 	struct passthru_softc *sc;
1232f8a6ec2dSD Scott Phillips 
1233f8a6ec2dSD Scott Phillips 	sc = pi->pi_arg;
1234f8a6ec2dSD Scott Phillips 	if (!enabled) {
12356a284cacSJohn Baldwin 		if (vm_unmap_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1236f8a6ec2dSD Scott Phillips 					 sc->psc_sel.pc_dev,
1237f8a6ec2dSD Scott Phillips 					 sc->psc_sel.pc_func, address,
1238f8a6ec2dSD Scott Phillips 					 sc->psc_bar[baridx].size) != 0)
1239f8a6ec2dSD Scott Phillips 			warnx("pci_passthru: unmap_pptdev_mmio failed");
1240f8a6ec2dSD Scott Phillips 	} else {
12416a284cacSJohn Baldwin 		if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1242f8a6ec2dSD Scott Phillips 				       sc->psc_sel.pc_dev,
1243f8a6ec2dSD Scott Phillips 				       sc->psc_sel.pc_func, address,
1244f8a6ec2dSD Scott Phillips 				       sc->psc_bar[baridx].size,
1245f8a6ec2dSD Scott Phillips 				       sc->psc_bar[baridx].addr) != 0)
1246f8a6ec2dSD Scott Phillips 			warnx("pci_passthru: map_pptdev_mmio failed");
1247f8a6ec2dSD Scott Phillips 	}
1248f8a6ec2dSD Scott Phillips }
1249f8a6ec2dSD Scott Phillips 
1250f8a6ec2dSD Scott Phillips static void
passthru_addr_rom(struct pci_devinst * const pi,const int idx,const int enabled)1251e47fe318SCorvin Köhne passthru_addr_rom(struct pci_devinst *const pi, const int idx,
1252e47fe318SCorvin Köhne     const int enabled)
1253e47fe318SCorvin Köhne {
1254e47fe318SCorvin Köhne 	const uint64_t addr = pi->pi_bar[idx].addr;
1255e47fe318SCorvin Köhne 	const uint64_t size = pi->pi_bar[idx].size;
1256e47fe318SCorvin Köhne 
1257e47fe318SCorvin Köhne 	if (!enabled) {
1258e47fe318SCorvin Köhne 		if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) {
1259e47fe318SCorvin Köhne 			errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed",
1260e47fe318SCorvin Köhne 			    __func__, addr, addr + size);
1261e47fe318SCorvin Köhne 		}
1262e47fe318SCorvin Köhne 
1263e47fe318SCorvin Köhne 	} else {
1264e47fe318SCorvin Köhne 		if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM,
1265e47fe318SCorvin Köhne 			pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) {
126650526f52SCorvin Köhne 			errx(4, "%s: mmap_memseg @ [%016lx - %016lx]  failed",
1267e47fe318SCorvin Köhne 			    __func__, addr, addr + size);
1268e47fe318SCorvin Köhne 		}
1269e47fe318SCorvin Köhne 	}
1270e47fe318SCorvin Köhne }
1271e47fe318SCorvin Köhne 
1272e47fe318SCorvin Köhne static void
passthru_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)12736a284cacSJohn Baldwin passthru_addr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address)
1274f8a6ec2dSD Scott Phillips {
1275e47fe318SCorvin Köhne 	switch (pi->pi_bar[baridx].type) {
1276e47fe318SCorvin Köhne 	case PCIBAR_IO:
1277e47fe318SCorvin Köhne 		/* IO BARs are emulated */
1278e47fe318SCorvin Köhne 		break;
1279e47fe318SCorvin Köhne 	case PCIBAR_ROM:
1280e47fe318SCorvin Köhne 		passthru_addr_rom(pi, baridx, enabled);
1281e47fe318SCorvin Köhne 		break;
1282e47fe318SCorvin Köhne 	case PCIBAR_MEM32:
1283e47fe318SCorvin Köhne 	case PCIBAR_MEM64:
1284f8a6ec2dSD Scott Phillips 		if (baridx == pci_msix_table_bar(pi))
12856a284cacSJohn Baldwin 			passthru_msix_addr(pi, baridx, enabled, address);
1286f8a6ec2dSD Scott Phillips 		else
12876a284cacSJohn Baldwin 			passthru_mmio_addr(pi, baridx, enabled, address);
1288e47fe318SCorvin Köhne 		break;
1289e47fe318SCorvin Köhne 	default:
1290e47fe318SCorvin Köhne 		errx(4, "%s: invalid BAR type %d", __func__,
1291e47fe318SCorvin Köhne 		    pi->pi_bar[baridx].type);
1292e47fe318SCorvin Köhne 	}
1293f8a6ec2dSD Scott Phillips }
1294f8a6ec2dSD Scott Phillips 
129537045dfaSMark Johnston static const struct pci_devemu passthru = {
1296366f6083SPeter Grehan 	.pe_emu		= "passthru",
1297366f6083SPeter Grehan 	.pe_init	= passthru_init,
1298621b5090SJohn Baldwin 	.pe_legacy_config = passthru_legacy_config,
1299366f6083SPeter Grehan 	.pe_cfgwrite	= passthru_cfgwrite,
1300366f6083SPeter Grehan 	.pe_cfgread	= passthru_cfgread,
13014d1e669cSPeter Grehan 	.pe_barwrite 	= passthru_write,
13024d1e669cSPeter Grehan 	.pe_barread    	= passthru_read,
1303f8a6ec2dSD Scott Phillips 	.pe_baraddr	= passthru_addr,
1304366f6083SPeter Grehan };
1305366f6083SPeter Grehan PCI_EMUL_SET(passthru);
1306