1366f6083SPeter Grehan /*-
24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause
31de7b4b8SPedro F. Giffuni *
4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan * All rights reserved.
6366f6083SPeter Grehan *
7366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without
8366f6083SPeter Grehan * modification, are permitted provided that the following conditions
9366f6083SPeter Grehan * are met:
10366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright
11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer.
12366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright
13366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the
14366f6083SPeter Grehan * documentation and/or other materials provided with the distribution.
15366f6083SPeter Grehan *
16366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26366f6083SPeter Grehan * SUCH DAMAGE.
27366f6083SPeter Grehan */
28366f6083SPeter Grehan
29366f6083SPeter Grehan #include <sys/param.h>
3000ef17beSBartek Rutkowski #ifndef WITHOUT_CAPSICUM
3100ef17beSBartek Rutkowski #include <sys/capsicum.h>
3200ef17beSBartek Rutkowski #endif
33366f6083SPeter Grehan #include <sys/types.h>
345c40acf8SJohn Baldwin #include <sys/mman.h>
35366f6083SPeter Grehan #include <sys/pciio.h>
36366f6083SPeter Grehan #include <sys/ioctl.h>
37e47fe318SCorvin Köhne #include <sys/stat.h>
38366f6083SPeter Grehan
39366f6083SPeter Grehan #include <dev/io/iodev.h>
402e81a7e8SNeel Natu #include <dev/pci/pcireg.h>
412e81a7e8SNeel Natu
427fa23353SMark Johnston #include <vm/vm.h>
437fa23353SMark Johnston
44366f6083SPeter Grehan #include <machine/iodev.h>
457fa23353SMark Johnston #include <machine/vm.h>
46366f6083SPeter Grehan
47abfa3c39SMarcelo Araujo #ifndef WITHOUT_CAPSICUM
48abfa3c39SMarcelo Araujo #include <capsicum_helpers.h>
49abfa3c39SMarcelo Araujo #endif
50baf753ccSJohn Baldwin #include <ctype.h>
51366f6083SPeter Grehan #include <stdio.h>
52366f6083SPeter Grehan #include <stdlib.h>
53366f6083SPeter Grehan #include <string.h>
54cff92ffdSJohn Baldwin #include <err.h>
5500ef17beSBartek Rutkowski #include <errno.h>
56366f6083SPeter Grehan #include <fcntl.h>
5700ef17beSBartek Rutkowski #include <sysexits.h>
58366f6083SPeter Grehan #include <unistd.h>
59366f6083SPeter Grehan
60366f6083SPeter Grehan #include <machine/vmm.h>
61621b5090SJohn Baldwin
62621b5090SJohn Baldwin #include "debug.h"
634d1e669cSPeter Grehan #include "mem.h"
64563fd224SCorvin Köhne #include "pci_passthru.h"
65366f6083SPeter Grehan
66366f6083SPeter Grehan #ifndef _PATH_DEVPCI
67366f6083SPeter Grehan #define _PATH_DEVPCI "/dev/pci"
68366f6083SPeter Grehan #endif
69366f6083SPeter Grehan
70366f6083SPeter Grehan #define LEGACY_SUPPORT 1
71366f6083SPeter Grehan
722e81a7e8SNeel Natu #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
73cd942e0fSPeter Grehan #define MSIX_CAPLEN 12
74cd942e0fSPeter Grehan
7593cf9317SCorvin Köhne #define PASSTHRU_MMIO_MAX 2
7693cf9317SCorvin Köhne
77366f6083SPeter Grehan static int pcifd = -1;
78366f6083SPeter Grehan
7990c3a1b6SCorvin Köhne SET_DECLARE(passthru_dev_set, struct passthru_dev);
8090c3a1b6SCorvin Köhne
81366f6083SPeter Grehan struct passthru_softc {
82366f6083SPeter Grehan struct pci_devinst *psc_pi;
83e47fe318SCorvin Köhne /* ROM is handled like a BAR */
84e47fe318SCorvin Köhne struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1];
85366f6083SPeter Grehan struct {
86366f6083SPeter Grehan int capoff;
87366f6083SPeter Grehan int msgctrl;
88366f6083SPeter Grehan int emulated;
89366f6083SPeter Grehan } psc_msi;
90cd942e0fSPeter Grehan struct {
91cd942e0fSPeter Grehan int capoff;
92cd942e0fSPeter Grehan } psc_msix;
93366f6083SPeter Grehan struct pcisel psc_sel;
94931bb7bfSCorvin Köhne
9593cf9317SCorvin Köhne struct passthru_mmio_mapping psc_mmio_map[PASSTHRU_MMIO_MAX];
96931bb7bfSCorvin Köhne cfgread_handler psc_pcir_rhandler[PCI_REGMAX + 1];
97931bb7bfSCorvin Köhne cfgwrite_handler psc_pcir_whandler[PCI_REGMAX + 1];
98366f6083SPeter Grehan };
99366f6083SPeter Grehan
100366f6083SPeter Grehan static int
msi_caplen(int msgctrl)101366f6083SPeter Grehan msi_caplen(int msgctrl)
102366f6083SPeter Grehan {
103366f6083SPeter Grehan int len;
104366f6083SPeter Grehan
105366f6083SPeter Grehan len = 10; /* minimum length of msi capability */
106366f6083SPeter Grehan
107366f6083SPeter Grehan if (msgctrl & PCIM_MSICTRL_64BIT)
108366f6083SPeter Grehan len += 4;
109366f6083SPeter Grehan
110366f6083SPeter Grehan #if 0
111366f6083SPeter Grehan /*
112366f6083SPeter Grehan * Ignore the 'mask' and 'pending' bits in the MSI capability.
113366f6083SPeter Grehan * We'll let the guest manipulate them directly.
114366f6083SPeter Grehan */
115366f6083SPeter Grehan if (msgctrl & PCIM_MSICTRL_VECTOR)
116366f6083SPeter Grehan len += 10;
117366f6083SPeter Grehan #endif
118366f6083SPeter Grehan
119366f6083SPeter Grehan return (len);
120366f6083SPeter Grehan }
121366f6083SPeter Grehan
122563fd224SCorvin Köhne static int
pcifd_init(void)12375ce327aSMark Johnston pcifd_init(void)
12475ce327aSMark Johnston {
125563fd224SCorvin Köhne pcifd = open(_PATH_DEVPCI, O_RDWR, 0);
126563fd224SCorvin Köhne if (pcifd < 0) {
127563fd224SCorvin Köhne warn("failed to open %s", _PATH_DEVPCI);
128563fd224SCorvin Köhne return (1);
129563fd224SCorvin Köhne }
130563fd224SCorvin Köhne
131563fd224SCorvin Köhne #ifndef WITHOUT_CAPSICUM
132563fd224SCorvin Köhne cap_rights_t pcifd_rights;
133563fd224SCorvin Köhne cap_rights_init(&pcifd_rights, CAP_IOCTL, CAP_READ, CAP_WRITE);
134563fd224SCorvin Köhne if (caph_rights_limit(pcifd, &pcifd_rights) == -1)
135563fd224SCorvin Köhne errx(EX_OSERR, "Unable to apply rights for sandbox");
136563fd224SCorvin Köhne
137563fd224SCorvin Köhne const cap_ioctl_t pcifd_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR,
138baf753ccSJohn Baldwin PCIOCBARIO, PCIOCBARMMAP, PCIOCGETCONF };
139563fd224SCorvin Köhne if (caph_ioctls_limit(pcifd, pcifd_ioctls, nitems(pcifd_ioctls)) == -1)
140563fd224SCorvin Köhne errx(EX_OSERR, "Unable to apply rights for sandbox");
141563fd224SCorvin Köhne #endif
142563fd224SCorvin Köhne
143563fd224SCorvin Köhne return (0);
144563fd224SCorvin Köhne }
145563fd224SCorvin Köhne
146563fd224SCorvin Köhne uint32_t
pci_host_read_config(const struct pcisel * sel,long reg,int width)14701d53c34SMark Johnston pci_host_read_config(const struct pcisel *sel, long reg, int width)
148366f6083SPeter Grehan {
149bcab868aSJohn Baldwin struct pci_io pi;
150bcab868aSJohn Baldwin
151563fd224SCorvin Köhne if (pcifd < 0 && pcifd_init()) {
152563fd224SCorvin Köhne return (0);
153563fd224SCorvin Köhne }
154563fd224SCorvin Köhne
155366f6083SPeter Grehan bzero(&pi, sizeof(pi));
156366f6083SPeter Grehan pi.pi_sel = *sel;
157366f6083SPeter Grehan pi.pi_reg = reg;
158366f6083SPeter Grehan pi.pi_width = width;
159366f6083SPeter Grehan
160366f6083SPeter Grehan if (ioctl(pcifd, PCIOCREAD, &pi) < 0)
161366f6083SPeter Grehan return (0); /* XXX */
162366f6083SPeter Grehan else
163366f6083SPeter Grehan return (pi.pi_data);
164366f6083SPeter Grehan }
165366f6083SPeter Grehan
166563fd224SCorvin Köhne void
pci_host_write_config(const struct pcisel * sel,long reg,int width,uint32_t data)16701d53c34SMark Johnston pci_host_write_config(const struct pcisel *sel, long reg, int width,
16801d53c34SMark Johnston uint32_t data)
169366f6083SPeter Grehan {
170bcab868aSJohn Baldwin struct pci_io pi;
171bcab868aSJohn Baldwin
172563fd224SCorvin Köhne if (pcifd < 0 && pcifd_init()) {
173563fd224SCorvin Köhne return;
174563fd224SCorvin Köhne }
175563fd224SCorvin Köhne
176366f6083SPeter Grehan bzero(&pi, sizeof(pi));
177366f6083SPeter Grehan pi.pi_sel = *sel;
178366f6083SPeter Grehan pi.pi_reg = reg;
179366f6083SPeter Grehan pi.pi_width = width;
180366f6083SPeter Grehan pi.pi_data = data;
181366f6083SPeter Grehan
182366f6083SPeter Grehan (void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */
183366f6083SPeter Grehan }
184366f6083SPeter Grehan
185366f6083SPeter Grehan #ifdef LEGACY_SUPPORT
186366f6083SPeter Grehan static int
passthru_add_msicap(struct pci_devinst * pi,int msgnum,int nextptr)187366f6083SPeter Grehan passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)
188366f6083SPeter Grehan {
189ed721684SMark Johnston int capoff;
190366f6083SPeter Grehan struct msicap msicap;
191366f6083SPeter Grehan u_char *capdata;
192366f6083SPeter Grehan
193366f6083SPeter Grehan pci_populate_msicap(&msicap, msgnum, nextptr);
194366f6083SPeter Grehan
195366f6083SPeter Grehan /*
196366f6083SPeter Grehan * XXX
197366f6083SPeter Grehan * Copy the msi capability structure in the last 16 bytes of the
198366f6083SPeter Grehan * config space. This is wrong because it could shadow something
199366f6083SPeter Grehan * useful to the device.
200366f6083SPeter Grehan */
201366f6083SPeter Grehan capoff = 256 - roundup(sizeof(msicap), 4);
202366f6083SPeter Grehan capdata = (u_char *)&msicap;
203ed721684SMark Johnston for (size_t i = 0; i < sizeof(msicap); i++)
204366f6083SPeter Grehan pci_set_cfgdata8(pi, capoff + i, capdata[i]);
205366f6083SPeter Grehan
206366f6083SPeter Grehan return (capoff);
207366f6083SPeter Grehan }
208366f6083SPeter Grehan #endif /* LEGACY_SUPPORT */
209366f6083SPeter Grehan
210366f6083SPeter Grehan static int
cfginitmsi(struct passthru_softc * sc)211366f6083SPeter Grehan cfginitmsi(struct passthru_softc *sc)
212366f6083SPeter Grehan {
2132e81a7e8SNeel Natu int i, ptr, capptr, cap, sts, caplen, table_size;
214366f6083SPeter Grehan uint32_t u32;
215366f6083SPeter Grehan struct pcisel sel;
216366f6083SPeter Grehan struct pci_devinst *pi;
217cd942e0fSPeter Grehan struct msixcap msixcap;
21832b21dd2SJohn Baldwin char *msixcap_ptr;
219366f6083SPeter Grehan
220366f6083SPeter Grehan pi = sc->psc_pi;
221366f6083SPeter Grehan sel = sc->psc_sel;
222366f6083SPeter Grehan
223366f6083SPeter Grehan /*
224366f6083SPeter Grehan * Parse the capabilities and cache the location of the MSI
225cd942e0fSPeter Grehan * and MSI-X capabilities.
226366f6083SPeter Grehan */
22701d53c34SMark Johnston sts = pci_host_read_config(&sel, PCIR_STATUS, 2);
228366f6083SPeter Grehan if (sts & PCIM_STATUS_CAPPRESENT) {
22901d53c34SMark Johnston ptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1);
230366f6083SPeter Grehan while (ptr != 0 && ptr != 0xff) {
23101d53c34SMark Johnston cap = pci_host_read_config(&sel, ptr + PCICAP_ID, 1);
232366f6083SPeter Grehan if (cap == PCIY_MSI) {
233366f6083SPeter Grehan /*
234366f6083SPeter Grehan * Copy the MSI capability into the config
235366f6083SPeter Grehan * space of the emulated pci device
236366f6083SPeter Grehan */
237366f6083SPeter Grehan sc->psc_msi.capoff = ptr;
23801d53c34SMark Johnston sc->psc_msi.msgctrl = pci_host_read_config(&sel,
239366f6083SPeter Grehan ptr + 2, 2);
240366f6083SPeter Grehan sc->psc_msi.emulated = 0;
241366f6083SPeter Grehan caplen = msi_caplen(sc->psc_msi.msgctrl);
242cd942e0fSPeter Grehan capptr = ptr;
243366f6083SPeter Grehan while (caplen > 0) {
24401d53c34SMark Johnston u32 = pci_host_read_config(&sel, capptr,
24501d53c34SMark Johnston 4);
246cd942e0fSPeter Grehan pci_set_cfgdata32(pi, capptr, u32);
247366f6083SPeter Grehan caplen -= 4;
248cd942e0fSPeter Grehan capptr += 4;
249366f6083SPeter Grehan }
250cd942e0fSPeter Grehan } else if (cap == PCIY_MSIX) {
251cd942e0fSPeter Grehan /*
252cd942e0fSPeter Grehan * Copy the MSI-X capability
253cd942e0fSPeter Grehan */
254cd942e0fSPeter Grehan sc->psc_msix.capoff = ptr;
255cd942e0fSPeter Grehan caplen = 12;
25632b21dd2SJohn Baldwin msixcap_ptr = (char *)&msixcap;
257cd942e0fSPeter Grehan capptr = ptr;
258cd942e0fSPeter Grehan while (caplen > 0) {
25901d53c34SMark Johnston u32 = pci_host_read_config(&sel, capptr,
26001d53c34SMark Johnston 4);
26132b21dd2SJohn Baldwin memcpy(msixcap_ptr, &u32, 4);
262cd942e0fSPeter Grehan pci_set_cfgdata32(pi, capptr, u32);
263cd942e0fSPeter Grehan caplen -= 4;
264cd942e0fSPeter Grehan capptr += 4;
26532b21dd2SJohn Baldwin msixcap_ptr += 4;
266cd942e0fSPeter Grehan }
267366f6083SPeter Grehan }
26801d53c34SMark Johnston ptr = pci_host_read_config(&sel, ptr + PCICAP_NEXTPTR,
26901d53c34SMark Johnston 1);
270366f6083SPeter Grehan }
271366f6083SPeter Grehan }
272366f6083SPeter Grehan
2734d1e669cSPeter Grehan if (sc->psc_msix.capoff != 0) {
2744d1e669cSPeter Grehan pi->pi_msix.pba_bar =
2752e81a7e8SNeel Natu msixcap.pba_info & PCIM_MSIX_BIR_MASK;
2764d1e669cSPeter Grehan pi->pi_msix.pba_offset =
2772e81a7e8SNeel Natu msixcap.pba_info & ~PCIM_MSIX_BIR_MASK;
2784d1e669cSPeter Grehan pi->pi_msix.table_bar =
2792e81a7e8SNeel Natu msixcap.table_info & PCIM_MSIX_BIR_MASK;
2804d1e669cSPeter Grehan pi->pi_msix.table_offset =
2812e81a7e8SNeel Natu msixcap.table_info & ~PCIM_MSIX_BIR_MASK;
282cd942e0fSPeter Grehan pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl);
2837a902ec0SNeel Natu pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count);
2842e81a7e8SNeel Natu
2852e81a7e8SNeel Natu /* Allocate the emulated MSI-X table array */
2862e81a7e8SNeel Natu table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
287994f858aSXin LI pi->pi_msix.table = calloc(1, table_size);
2882e81a7e8SNeel Natu
2892e81a7e8SNeel Natu /* Mask all table entries */
2902e81a7e8SNeel Natu for (i = 0; i < pi->pi_msix.table_count; i++) {
2912e81a7e8SNeel Natu pi->pi_msix.table[i].vector_control |=
2922e81a7e8SNeel Natu PCIM_MSIX_VCTRL_MASK;
2932e81a7e8SNeel Natu }
2944d1e669cSPeter Grehan }
295cd942e0fSPeter Grehan
296366f6083SPeter Grehan #ifdef LEGACY_SUPPORT
297366f6083SPeter Grehan /*
298366f6083SPeter Grehan * If the passthrough device does not support MSI then craft a
299366f6083SPeter Grehan * MSI capability for it. We link the new MSI capability at the
300366f6083SPeter Grehan * head of the list of capabilities.
301366f6083SPeter Grehan */
302366f6083SPeter Grehan if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) {
303366f6083SPeter Grehan int origptr, msiptr;
30401d53c34SMark Johnston origptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1);
305366f6083SPeter Grehan msiptr = passthru_add_msicap(pi, 1, origptr);
306366f6083SPeter Grehan sc->psc_msi.capoff = msiptr;
307366f6083SPeter Grehan sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2);
308366f6083SPeter Grehan sc->psc_msi.emulated = 1;
309366f6083SPeter Grehan pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr);
310366f6083SPeter Grehan }
311366f6083SPeter Grehan #endif
312366f6083SPeter Grehan
313cd942e0fSPeter Grehan /* Make sure one of the capabilities is present */
314cd942e0fSPeter Grehan if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0)
315366f6083SPeter Grehan return (-1);
316366f6083SPeter Grehan else
317366f6083SPeter Grehan return (0);
318366f6083SPeter Grehan }
319366f6083SPeter Grehan
3204d1e669cSPeter Grehan static uint64_t
msix_table_read(struct passthru_softc * sc,uint64_t offset,int size)3214d1e669cSPeter Grehan msix_table_read(struct passthru_softc *sc, uint64_t offset, int size)
322cd942e0fSPeter Grehan {
323cd942e0fSPeter Grehan struct pci_devinst *pi;
3244d1e669cSPeter Grehan struct msix_table_entry *entry;
325cd942e0fSPeter Grehan uint8_t *src8;
326cd942e0fSPeter Grehan uint16_t *src16;
327cd942e0fSPeter Grehan uint32_t *src32;
328cd942e0fSPeter Grehan uint64_t *src64;
3294d1e669cSPeter Grehan uint64_t data;
3304d1e669cSPeter Grehan size_t entry_offset;
3317fa23353SMark Johnston uint32_t table_offset;
3327fa23353SMark Johnston int index, table_count;
333cd942e0fSPeter Grehan
334cd942e0fSPeter Grehan pi = sc->psc_pi;
3357fa23353SMark Johnston
3367fa23353SMark Johnston table_offset = pi->pi_msix.table_offset;
3377fa23353SMark Johnston table_count = pi->pi_msix.table_count;
3387fa23353SMark Johnston if (offset < table_offset ||
3397fa23353SMark Johnston offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
3405c40acf8SJohn Baldwin switch (size) {
3415c40acf8SJohn Baldwin case 1:
3427fa23353SMark Johnston src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
3435c40acf8SJohn Baldwin data = *src8;
3445c40acf8SJohn Baldwin break;
3455c40acf8SJohn Baldwin case 2:
3467fa23353SMark Johnston src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
3475c40acf8SJohn Baldwin data = *src16;
3485c40acf8SJohn Baldwin break;
3495c40acf8SJohn Baldwin case 4:
3507fa23353SMark Johnston src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
3515c40acf8SJohn Baldwin data = *src32;
3525c40acf8SJohn Baldwin break;
3535c40acf8SJohn Baldwin case 8:
3547fa23353SMark Johnston src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
3555c40acf8SJohn Baldwin data = *src64;
3565c40acf8SJohn Baldwin break;
3575c40acf8SJohn Baldwin default:
3585c40acf8SJohn Baldwin return (-1);
3595c40acf8SJohn Baldwin }
3605c40acf8SJohn Baldwin return (data);
3615c40acf8SJohn Baldwin }
3625c40acf8SJohn Baldwin
3637fa23353SMark Johnston offset -= table_offset;
364cd942e0fSPeter Grehan index = offset / MSIX_TABLE_ENTRY_SIZE;
3657fa23353SMark Johnston assert(index < table_count);
3662e81a7e8SNeel Natu
367cd942e0fSPeter Grehan entry = &pi->pi_msix.table[index];
3682e81a7e8SNeel Natu entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
369cd942e0fSPeter Grehan
370cd942e0fSPeter Grehan switch (size) {
371cd942e0fSPeter Grehan case 1:
3727fa23353SMark Johnston src8 = (uint8_t *)((uint8_t *)entry + entry_offset);
3734d1e669cSPeter Grehan data = *src8;
374cd942e0fSPeter Grehan break;
375cd942e0fSPeter Grehan case 2:
3767fa23353SMark Johnston src16 = (uint16_t *)((uint8_t *)entry + entry_offset);
3774d1e669cSPeter Grehan data = *src16;
378cd942e0fSPeter Grehan break;
379cd942e0fSPeter Grehan case 4:
3807fa23353SMark Johnston src32 = (uint32_t *)((uint8_t *)entry + entry_offset);
3814d1e669cSPeter Grehan data = *src32;
382cd942e0fSPeter Grehan break;
383cd942e0fSPeter Grehan case 8:
3847fa23353SMark Johnston src64 = (uint64_t *)((uint8_t *)entry + entry_offset);
3854d1e669cSPeter Grehan data = *src64;
386cd942e0fSPeter Grehan break;
387cd942e0fSPeter Grehan default:
388cd942e0fSPeter Grehan return (-1);
389cd942e0fSPeter Grehan }
390cd942e0fSPeter Grehan
3914d1e669cSPeter Grehan return (data);
392cd942e0fSPeter Grehan }
393cd942e0fSPeter Grehan
3944d1e669cSPeter Grehan static void
msix_table_write(struct passthru_softc * sc,uint64_t offset,int size,uint64_t data)3956a284cacSJohn Baldwin msix_table_write(struct passthru_softc *sc, uint64_t offset, int size,
3966a284cacSJohn Baldwin uint64_t data)
397cd942e0fSPeter Grehan {
398cd942e0fSPeter Grehan struct pci_devinst *pi;
399cd942e0fSPeter Grehan struct msix_table_entry *entry;
4005c40acf8SJohn Baldwin uint8_t *dest8;
4015c40acf8SJohn Baldwin uint16_t *dest16;
4025c40acf8SJohn Baldwin uint32_t *dest32;
4035c40acf8SJohn Baldwin uint64_t *dest64;
4044d1e669cSPeter Grehan size_t entry_offset;
4057fa23353SMark Johnston uint32_t table_offset, vector_control;
4067fa23353SMark Johnston int index, table_count;
407cd942e0fSPeter Grehan
408cd942e0fSPeter Grehan pi = sc->psc_pi;
4097fa23353SMark Johnston
4107fa23353SMark Johnston table_offset = pi->pi_msix.table_offset;
4117fa23353SMark Johnston table_count = pi->pi_msix.table_count;
4127fa23353SMark Johnston if (offset < table_offset ||
4137fa23353SMark Johnston offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) {
4145c40acf8SJohn Baldwin switch (size) {
4155c40acf8SJohn Baldwin case 1:
4167fa23353SMark Johnston dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset);
4175c40acf8SJohn Baldwin *dest8 = data;
4185c40acf8SJohn Baldwin break;
4195c40acf8SJohn Baldwin case 2:
4207fa23353SMark Johnston dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset);
4215c40acf8SJohn Baldwin *dest16 = data;
4225c40acf8SJohn Baldwin break;
4235c40acf8SJohn Baldwin case 4:
4247fa23353SMark Johnston dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset);
4255c40acf8SJohn Baldwin *dest32 = data;
4265c40acf8SJohn Baldwin break;
4275c40acf8SJohn Baldwin case 8:
4287fa23353SMark Johnston dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset);
4295c40acf8SJohn Baldwin *dest64 = data;
4305c40acf8SJohn Baldwin break;
4315c40acf8SJohn Baldwin }
4325c40acf8SJohn Baldwin return;
4335c40acf8SJohn Baldwin }
4345c40acf8SJohn Baldwin
4357fa23353SMark Johnston offset -= table_offset;
436cd942e0fSPeter Grehan index = offset / MSIX_TABLE_ENTRY_SIZE;
4377fa23353SMark Johnston assert(index < table_count);
4382e81a7e8SNeel Natu
439cd942e0fSPeter Grehan entry = &pi->pi_msix.table[index];
4402e81a7e8SNeel Natu entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
441cd942e0fSPeter Grehan
442cd942e0fSPeter Grehan /* Only 4 byte naturally-aligned writes are supported */
4434d1e669cSPeter Grehan assert(size == 4);
4444d1e669cSPeter Grehan assert(entry_offset % 4 == 0);
4454d1e669cSPeter Grehan
446cd942e0fSPeter Grehan vector_control = entry->vector_control;
44763898728SMark Johnston dest32 = (uint32_t *)((uint8_t *)entry + entry_offset);
4485c40acf8SJohn Baldwin *dest32 = data;
449cd942e0fSPeter Grehan /* If MSI-X hasn't been enabled, do nothing */
450cd942e0fSPeter Grehan if (pi->pi_msix.enabled) {
451cd942e0fSPeter Grehan /* If the entry is masked, don't set it up */
452cd942e0fSPeter Grehan if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 ||
453cd942e0fSPeter Grehan (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
4547d9ef309SJohn Baldwin (void)vm_setup_pptdev_msix(sc->psc_pi->pi_vmctx,
45555888cfaSNeel Natu sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
45655888cfaSNeel Natu sc->psc_sel.pc_func, index, entry->addr,
45755888cfaSNeel Natu entry->msg_data, entry->vector_control);
458cd942e0fSPeter Grehan }
459cd942e0fSPeter Grehan }
460cd942e0fSPeter Grehan }
461cd942e0fSPeter Grehan
462cd942e0fSPeter Grehan static int
init_msix_table(struct passthru_softc * sc)4636a284cacSJohn Baldwin init_msix_table(struct passthru_softc *sc)
464cd942e0fSPeter Grehan {
465cd942e0fSPeter Grehan struct pci_devinst *pi = sc->psc_pi;
4667fa23353SMark Johnston struct pci_bar_mmap pbm;
4677fa23353SMark Johnston int b, s, f;
4687fa23353SMark Johnston uint32_t table_size, table_offset;
469cd942e0fSPeter Grehan
470aa12663fSNeel Natu assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0);
471aa12663fSNeel Natu
4722b89a044SNeel Natu b = sc->psc_sel.pc_bus;
4732b89a044SNeel Natu s = sc->psc_sel.pc_dev;
4742b89a044SNeel Natu f = sc->psc_sel.pc_func;
4752b89a044SNeel Natu
476cd942e0fSPeter Grehan /*
4777fa23353SMark Johnston * Map the region of the BAR containing the MSI-X table. This is
4787fa23353SMark Johnston * necessary for two reasons:
4797fa23353SMark Johnston * 1. The PBA may reside in the first or last page containing the MSI-X
4807fa23353SMark Johnston * table.
4817fa23353SMark Johnston * 2. While PCI devices are not supposed to use the page(s) containing
4827fa23353SMark Johnston * the MSI-X table for other purposes, some do in practice.
483cd942e0fSPeter Grehan */
4847fa23353SMark Johnston memset(&pbm, 0, sizeof(pbm));
4857fa23353SMark Johnston pbm.pbm_sel = sc->psc_sel;
4867fa23353SMark Johnston pbm.pbm_flags = PCIIO_BAR_MMAP_RW;
48776b45e68SMark Johnston pbm.pbm_reg = PCIR_BAR(pi->pi_msix.table_bar);
4887fa23353SMark Johnston pbm.pbm_memattr = VM_MEMATTR_DEVICE;
4897fa23353SMark Johnston
4907fa23353SMark Johnston if (ioctl(pcifd, PCIOCBARMMAP, &pbm) != 0) {
4917fa23353SMark Johnston warn("Failed to map MSI-X table BAR on %d/%d/%d", b, s, f);
4927fa23353SMark Johnston return (-1);
4937fa23353SMark Johnston }
4947fa23353SMark Johnston assert(pbm.pbm_bar_off == 0);
4957fa23353SMark Johnston pi->pi_msix.mapped_addr = (uint8_t *)(uintptr_t)pbm.pbm_map_base;
4967fa23353SMark Johnston pi->pi_msix.mapped_size = pbm.pbm_map_length;
4977fa23353SMark Johnston
4987a902ec0SNeel Natu table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
4997a902ec0SNeel Natu
5007a902ec0SNeel Natu table_size = pi->pi_msix.table_offset - table_offset;
5017a902ec0SNeel Natu table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
5027a902ec0SNeel Natu table_size = roundup2(table_size, 4096);
5037a902ec0SNeel Natu
5047a902ec0SNeel Natu /*
5054558c11fSMark Johnston * Unmap any pages not containing the table, we do not need to emulate
5067fa23353SMark Johnston * accesses to them. Avoid releasing address space to help ensure that
5077fa23353SMark Johnston * a buggy out-of-bounds access causes a crash.
5087a902ec0SNeel Natu */
5097fa23353SMark Johnston if (table_offset != 0)
5107fa23353SMark Johnston if (mprotect(pi->pi_msix.mapped_addr, table_offset,
5117fa23353SMark Johnston PROT_NONE) != 0)
5127fa23353SMark Johnston warn("Failed to unmap MSI-X table BAR region");
5137fa23353SMark Johnston if (table_offset + table_size != pi->pi_msix.mapped_size)
5144558c11fSMark Johnston if (mprotect(
5154558c11fSMark Johnston pi->pi_msix.mapped_addr + table_offset + table_size,
5167fa23353SMark Johnston pi->pi_msix.mapped_size - (table_offset + table_size),
5177fa23353SMark Johnston PROT_NONE) != 0)
5187fa23353SMark Johnston warn("Failed to unmap MSI-X table BAR region");
5192b89a044SNeel Natu
5202b89a044SNeel Natu return (0);
521cd942e0fSPeter Grehan }
522cd942e0fSPeter Grehan
523cd942e0fSPeter Grehan static int
cfginitbar(struct passthru_softc * sc)5246a284cacSJohn Baldwin cfginitbar(struct passthru_softc *sc)
525366f6083SPeter Grehan {
526366f6083SPeter Grehan int i, error;
527366f6083SPeter Grehan struct pci_devinst *pi;
528366f6083SPeter Grehan struct pci_bar_io bar;
529366f6083SPeter Grehan enum pcibar_type bartype;
5307a902ec0SNeel Natu uint64_t base, size;
531366f6083SPeter Grehan
532366f6083SPeter Grehan pi = sc->psc_pi;
533366f6083SPeter Grehan
534366f6083SPeter Grehan /*
535366f6083SPeter Grehan * Initialize BAR registers
536366f6083SPeter Grehan */
537366f6083SPeter Grehan for (i = 0; i <= PCI_BARMAX; i++) {
538366f6083SPeter Grehan bzero(&bar, sizeof(bar));
539366f6083SPeter Grehan bar.pbi_sel = sc->psc_sel;
540366f6083SPeter Grehan bar.pbi_reg = PCIR_BAR(i);
541366f6083SPeter Grehan
542366f6083SPeter Grehan if (ioctl(pcifd, PCIOCGETBAR, &bar) < 0)
543366f6083SPeter Grehan continue;
544366f6083SPeter Grehan
545366f6083SPeter Grehan if (PCI_BAR_IO(bar.pbi_base)) {
546366f6083SPeter Grehan bartype = PCIBAR_IO;
547366f6083SPeter Grehan base = bar.pbi_base & PCIM_BAR_IO_BASE;
548366f6083SPeter Grehan } else {
549366f6083SPeter Grehan switch (bar.pbi_base & PCIM_BAR_MEM_TYPE) {
550366f6083SPeter Grehan case PCIM_BAR_MEM_64:
551366f6083SPeter Grehan bartype = PCIBAR_MEM64;
552366f6083SPeter Grehan break;
553366f6083SPeter Grehan default:
554366f6083SPeter Grehan bartype = PCIBAR_MEM32;
555366f6083SPeter Grehan break;
556366f6083SPeter Grehan }
557366f6083SPeter Grehan base = bar.pbi_base & PCIM_BAR_MEM_BASE;
558366f6083SPeter Grehan }
5597a902ec0SNeel Natu size = bar.pbi_length;
5607a902ec0SNeel Natu
5617a902ec0SNeel Natu if (bartype != PCIBAR_IO) {
5627a902ec0SNeel Natu if (((base | size) & PAGE_MASK) != 0) {
563cff92ffdSJohn Baldwin warnx("passthru device %d/%d/%d BAR %d: "
5647a902ec0SNeel Natu "base %#lx or size %#lx not page aligned\n",
5657a902ec0SNeel Natu sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
5667a902ec0SNeel Natu sc->psc_sel.pc_func, i, base, size);
5677a902ec0SNeel Natu return (-1);
5687a902ec0SNeel Natu }
5697a902ec0SNeel Natu }
570366f6083SPeter Grehan
571366f6083SPeter Grehan /* Cache information about the "real" BAR */
572366f6083SPeter Grehan sc->psc_bar[i].type = bartype;
5737a902ec0SNeel Natu sc->psc_bar[i].size = size;
574366f6083SPeter Grehan sc->psc_bar[i].addr = base;
575e87a6f3eSCorvin Köhne sc->psc_bar[i].lobits = 0;
576366f6083SPeter Grehan
577366f6083SPeter Grehan /* Allocate the BAR in the guest I/O or MMIO space */
578038f5c7bSKonstantin Belousov error = pci_emul_alloc_bar(pi, i, bartype, size);
579366f6083SPeter Grehan if (error)
580366f6083SPeter Grehan return (-1);
581366f6083SPeter Grehan
582e87a6f3eSCorvin Köhne /* Use same lobits as physical bar */
58301d53c34SMark Johnston uint8_t lobits = pci_host_read_config(&sc->psc_sel, PCIR_BAR(i),
58401d53c34SMark Johnston 0x01);
585e87a6f3eSCorvin Köhne if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) {
586e87a6f3eSCorvin Köhne lobits &= ~PCIM_BAR_MEM_BASE;
587e87a6f3eSCorvin Köhne } else {
588e87a6f3eSCorvin Köhne lobits &= ~PCIM_BAR_IO_BASE;
589e87a6f3eSCorvin Köhne }
590e87a6f3eSCorvin Köhne sc->psc_bar[i].lobits = lobits;
591e87a6f3eSCorvin Köhne pi->pi_bar[i].lobits = lobits;
592e87a6f3eSCorvin Köhne
593366f6083SPeter Grehan /*
594366f6083SPeter Grehan * 64-bit BAR takes up two slots so skip the next one.
595366f6083SPeter Grehan */
596366f6083SPeter Grehan if (bartype == PCIBAR_MEM64) {
597366f6083SPeter Grehan i++;
598366f6083SPeter Grehan assert(i <= PCI_BARMAX);
599366f6083SPeter Grehan sc->psc_bar[i].type = PCIBAR_MEMHI64;
600366f6083SPeter Grehan }
601366f6083SPeter Grehan }
602366f6083SPeter Grehan return (0);
603366f6083SPeter Grehan }
604366f6083SPeter Grehan
605366f6083SPeter Grehan static int
cfginit(struct pci_devinst * pi,int bus,int slot,int func)6066a284cacSJohn Baldwin cfginit(struct pci_devinst *pi, int bus, int slot, int func)
607366f6083SPeter Grehan {
608366f6083SPeter Grehan int error;
609366f6083SPeter Grehan struct passthru_softc *sc;
610*f44ff2abSJohn Baldwin uint16_t cmd;
611b6e67875SCorvin Köhne uint8_t intline, intpin;
612366f6083SPeter Grehan
613366f6083SPeter Grehan error = 1;
614366f6083SPeter Grehan sc = pi->pi_arg;
615366f6083SPeter Grehan
616366f6083SPeter Grehan bzero(&sc->psc_sel, sizeof(struct pcisel));
617366f6083SPeter Grehan sc->psc_sel.pc_bus = bus;
618366f6083SPeter Grehan sc->psc_sel.pc_dev = slot;
619366f6083SPeter Grehan sc->psc_sel.pc_func = func;
620366f6083SPeter Grehan
621b6e67875SCorvin Köhne /*
622*f44ff2abSJohn Baldwin * Copy physical PCI header to virtual config space. COMMAND,
623*f44ff2abSJohn Baldwin * INTLINE, and INTPIN shouldn't be aligned with their
624*f44ff2abSJohn Baldwin * physical value and they are already set by pci_emul_init().
625b6e67875SCorvin Köhne */
626*f44ff2abSJohn Baldwin cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
627b6e67875SCorvin Köhne intline = pci_get_cfgdata8(pi, PCIR_INTLINE);
628b6e67875SCorvin Köhne intpin = pci_get_cfgdata8(pi, PCIR_INTPIN);
629b6e67875SCorvin Köhne for (int i = 0; i <= PCIR_MAXLAT; i += 4) {
63001d53c34SMark Johnston pci_set_cfgdata32(pi, i,
63101d53c34SMark Johnston pci_host_read_config(&sc->psc_sel, i, 4));
632b6e67875SCorvin Köhne }
633*f44ff2abSJohn Baldwin pci_set_cfgdata16(pi, PCIR_COMMAND, cmd);
634b6e67875SCorvin Köhne pci_set_cfgdata8(pi, PCIR_INTLINE, intline);
635b6e67875SCorvin Köhne pci_set_cfgdata8(pi, PCIR_INTPIN, intpin);
636b6e67875SCorvin Köhne
637cff92ffdSJohn Baldwin if (cfginitmsi(sc) != 0) {
638cff92ffdSJohn Baldwin warnx("failed to initialize MSI for PCI %d/%d/%d",
639cff92ffdSJohn Baldwin bus, slot, func);
640cd942e0fSPeter Grehan goto done;
641cff92ffdSJohn Baldwin }
642cd942e0fSPeter Grehan
6436a284cacSJohn Baldwin if (cfginitbar(sc) != 0) {
644cff92ffdSJohn Baldwin warnx("failed to initialize BARs for PCI %d/%d/%d",
645cff92ffdSJohn Baldwin bus, slot, func);
646366f6083SPeter Grehan goto done;
647cff92ffdSJohn Baldwin }
648366f6083SPeter Grehan
649338a1be8SCorvin Köhne if (pci_msix_table_bar(pi) >= 0) {
6506a284cacSJohn Baldwin error = init_msix_table(sc);
651f1442847SBjoern A. Zeeb if (error != 0) {
652338a1be8SCorvin Köhne warnx(
653338a1be8SCorvin Köhne "failed to initialize MSI-X table for PCI %d/%d/%d: %d",
654f1442847SBjoern A. Zeeb bus, slot, func, error);
655f1442847SBjoern A. Zeeb goto done;
656f1442847SBjoern A. Zeeb }
657338a1be8SCorvin Köhne }
658f1442847SBjoern A. Zeeb
659338a1be8SCorvin Köhne error = 0; /* success */
660366f6083SPeter Grehan done:
661366f6083SPeter Grehan return (error);
662366f6083SPeter Grehan }
663366f6083SPeter Grehan
66493cf9317SCorvin Köhne struct passthru_mmio_mapping *
passthru_get_mmio(struct passthru_softc * sc,int num)66593cf9317SCorvin Köhne passthru_get_mmio(struct passthru_softc *sc, int num)
66693cf9317SCorvin Köhne {
66793cf9317SCorvin Köhne assert(sc != NULL);
66893cf9317SCorvin Köhne assert(num < PASSTHRU_MMIO_MAX);
66993cf9317SCorvin Köhne
67093cf9317SCorvin Köhne return (&sc->psc_mmio_map[num]);
67193cf9317SCorvin Köhne }
67293cf9317SCorvin Köhne
67360793ceeSCorvin Köhne struct pcisel *
passthru_get_sel(struct passthru_softc * sc)67460793ceeSCorvin Köhne passthru_get_sel(struct passthru_softc *sc)
67560793ceeSCorvin Köhne {
67660793ceeSCorvin Köhne assert(sc != NULL);
67760793ceeSCorvin Köhne
67860793ceeSCorvin Köhne return (&sc->psc_sel);
67960793ceeSCorvin Köhne }
68060793ceeSCorvin Köhne
681931bb7bfSCorvin Köhne int
set_pcir_handler(struct passthru_softc * sc,int reg,int len,cfgread_handler rhandler,cfgwrite_handler whandler)682931bb7bfSCorvin Köhne set_pcir_handler(struct passthru_softc *sc, int reg, int len,
683931bb7bfSCorvin Köhne cfgread_handler rhandler, cfgwrite_handler whandler)
684931bb7bfSCorvin Köhne {
685931bb7bfSCorvin Köhne if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1)
686931bb7bfSCorvin Köhne return (-1);
687931bb7bfSCorvin Köhne
688931bb7bfSCorvin Köhne for (int i = reg; i < reg + len; ++i) {
689931bb7bfSCorvin Köhne assert(sc->psc_pcir_rhandler[i] == NULL || rhandler == NULL);
690931bb7bfSCorvin Köhne assert(sc->psc_pcir_whandler[i] == NULL || whandler == NULL);
691931bb7bfSCorvin Köhne sc->psc_pcir_rhandler[i] = rhandler;
692931bb7bfSCorvin Köhne sc->psc_pcir_whandler[i] = whandler;
693931bb7bfSCorvin Köhne }
694931bb7bfSCorvin Köhne
695931bb7bfSCorvin Köhne return (0);
696931bb7bfSCorvin Köhne }
697931bb7bfSCorvin Köhne
698366f6083SPeter Grehan static int
passthru_legacy_config(nvlist_t * nvl,const char * opts)699621b5090SJohn Baldwin passthru_legacy_config(nvlist_t *nvl, const char *opts)
700621b5090SJohn Baldwin {
701baf753ccSJohn Baldwin const char *cp;
702baf753ccSJohn Baldwin char *tofree;
703621b5090SJohn Baldwin char value[16];
704621b5090SJohn Baldwin int bus, slot, func;
705621b5090SJohn Baldwin
706621b5090SJohn Baldwin if (opts == NULL)
707621b5090SJohn Baldwin return (0);
708621b5090SJohn Baldwin
709baf753ccSJohn Baldwin cp = strchr(opts, ',');
710621b5090SJohn Baldwin
711baf753ccSJohn Baldwin if (strncmp(opts, "ppt", strlen("ppt")) == 0) {
712baf753ccSJohn Baldwin tofree = strndup(opts, cp - opts);
713baf753ccSJohn Baldwin set_config_value_node(nvl, "pptdev", tofree);
714baf753ccSJohn Baldwin free(tofree);
715baf753ccSJohn Baldwin } else if (sscanf(opts, "pci0:%d:%d:%d", &bus, &slot, &func) == 3 ||
716baf753ccSJohn Baldwin sscanf(opts, "pci%d:%d:%d", &bus, &slot, &func) == 3 ||
717baf753ccSJohn Baldwin sscanf(opts, "%d/%d/%d", &bus, &slot, &func) == 3) {
718621b5090SJohn Baldwin snprintf(value, sizeof(value), "%d", bus);
719621b5090SJohn Baldwin set_config_value_node(nvl, "bus", value);
720621b5090SJohn Baldwin snprintf(value, sizeof(value), "%d", slot);
721621b5090SJohn Baldwin set_config_value_node(nvl, "slot", value);
722621b5090SJohn Baldwin snprintf(value, sizeof(value), "%d", func);
723621b5090SJohn Baldwin set_config_value_node(nvl, "func", value);
724baf753ccSJohn Baldwin } else {
725baf753ccSJohn Baldwin EPRINTLN("passthru: invalid options \"%s\"", opts);
726baf753ccSJohn Baldwin return (-1);
727baf753ccSJohn Baldwin }
728e47fe318SCorvin Köhne
729baf753ccSJohn Baldwin if (cp == NULL) {
7303256b7caSCorvin Köhne return (0);
7313256b7caSCorvin Köhne }
7323256b7caSCorvin Köhne
733baf753ccSJohn Baldwin return (pci_parse_legacy_config(nvl, cp + 1));
734e47fe318SCorvin Köhne }
735e47fe318SCorvin Köhne
736e47fe318SCorvin Köhne static int
passthru_init_rom(struct passthru_softc * const sc,const char * const romfile)7376a284cacSJohn Baldwin passthru_init_rom(struct passthru_softc *const sc, const char *const romfile)
738e47fe318SCorvin Köhne {
739e47fe318SCorvin Köhne if (romfile == NULL) {
740e47fe318SCorvin Köhne return (0);
741e47fe318SCorvin Köhne }
742e47fe318SCorvin Köhne
743e47fe318SCorvin Köhne const int fd = open(romfile, O_RDONLY);
744e47fe318SCorvin Köhne if (fd < 0) {
745e47fe318SCorvin Köhne warnx("%s: can't open romfile \"%s\"", __func__, romfile);
746e47fe318SCorvin Köhne return (-1);
747e47fe318SCorvin Köhne }
748e47fe318SCorvin Köhne
749e47fe318SCorvin Köhne struct stat sbuf;
750e47fe318SCorvin Köhne if (fstat(fd, &sbuf) < 0) {
751e47fe318SCorvin Köhne warnx("%s: can't fstat romfile \"%s\"", __func__, romfile);
752e47fe318SCorvin Köhne close(fd);
753e47fe318SCorvin Köhne return (-1);
754e47fe318SCorvin Köhne }
755e47fe318SCorvin Köhne const uint64_t rom_size = sbuf.st_size;
756e47fe318SCorvin Köhne
757e47fe318SCorvin Köhne void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd,
758e47fe318SCorvin Köhne 0);
759e47fe318SCorvin Köhne if (rom_data == MAP_FAILED) {
760e47fe318SCorvin Köhne warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__,
761e47fe318SCorvin Köhne romfile, errno);
762e47fe318SCorvin Köhne close(fd);
763e47fe318SCorvin Köhne return (-1);
764e47fe318SCorvin Köhne }
765e47fe318SCorvin Köhne
766e47fe318SCorvin Köhne void *rom_addr;
767e47fe318SCorvin Köhne int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr);
768e47fe318SCorvin Köhne if (error) {
769e47fe318SCorvin Köhne warnx("%s: failed to alloc rom segment", __func__);
770e47fe318SCorvin Köhne munmap(rom_data, rom_size);
771e47fe318SCorvin Köhne close(fd);
772e47fe318SCorvin Köhne return (error);
773e47fe318SCorvin Köhne }
774e47fe318SCorvin Köhne memcpy(rom_addr, rom_data, rom_size);
775e47fe318SCorvin Köhne
776e47fe318SCorvin Köhne sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM;
777e47fe318SCorvin Köhne sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr;
778e47fe318SCorvin Köhne sc->psc_bar[PCI_ROM_IDX].size = rom_size;
779e47fe318SCorvin Köhne
780e47fe318SCorvin Köhne munmap(rom_data, rom_size);
781e47fe318SCorvin Köhne close(fd);
782e47fe318SCorvin Köhne
783621b5090SJohn Baldwin return (0);
784621b5090SJohn Baldwin }
785621b5090SJohn Baldwin
786baf753ccSJohn Baldwin static bool
passthru_lookup_pptdev(const char * name,int * bus,int * slot,int * func)787baf753ccSJohn Baldwin passthru_lookup_pptdev(const char *name, int *bus, int *slot, int *func)
788baf753ccSJohn Baldwin {
789baf753ccSJohn Baldwin struct pci_conf_io pc;
790baf753ccSJohn Baldwin struct pci_conf conf[1];
791baf753ccSJohn Baldwin struct pci_match_conf patterns[1];
792baf753ccSJohn Baldwin char *cp;
793baf753ccSJohn Baldwin
794baf753ccSJohn Baldwin bzero(&pc, sizeof(struct pci_conf_io));
795baf753ccSJohn Baldwin pc.match_buf_len = sizeof(conf);
796baf753ccSJohn Baldwin pc.matches = conf;
797baf753ccSJohn Baldwin
798baf753ccSJohn Baldwin bzero(&patterns, sizeof(patterns));
799baf753ccSJohn Baldwin
800baf753ccSJohn Baldwin /*
801baf753ccSJohn Baldwin * The pattern structure requires the unit to be split out from
802baf753ccSJohn Baldwin * the driver name. Walk backwards from the end of the name to
803baf753ccSJohn Baldwin * find the start of the unit.
804baf753ccSJohn Baldwin */
805baf753ccSJohn Baldwin cp = strchr(name, '\0');
806baf753ccSJohn Baldwin assert(cp != NULL);
807baf753ccSJohn Baldwin while (cp != name && isdigit(cp[-1]))
808baf753ccSJohn Baldwin cp--;
809baf753ccSJohn Baldwin if (cp == name || !isdigit(*cp)) {
810baf753ccSJohn Baldwin EPRINTLN("Invalid passthru device name %s", name);
811baf753ccSJohn Baldwin return (false);
812baf753ccSJohn Baldwin }
813baf753ccSJohn Baldwin if ((size_t)(cp - name) + 1 > sizeof(patterns[0].pd_name)) {
814baf753ccSJohn Baldwin EPRINTLN("Passthru device name %s is too long", name);
815baf753ccSJohn Baldwin return (false);
816baf753ccSJohn Baldwin }
817baf753ccSJohn Baldwin memcpy(patterns[0].pd_name, name, cp - name);
818baf753ccSJohn Baldwin patterns[0].pd_unit = strtol(cp, &cp, 10);
819baf753ccSJohn Baldwin if (*cp != '\0') {
820baf753ccSJohn Baldwin EPRINTLN("Invalid passthru device name %s", name);
821baf753ccSJohn Baldwin return (false);
822baf753ccSJohn Baldwin }
823baf753ccSJohn Baldwin patterns[0].flags = PCI_GETCONF_MATCH_NAME | PCI_GETCONF_MATCH_UNIT;
824baf753ccSJohn Baldwin pc.num_patterns = 1;
825baf753ccSJohn Baldwin pc.pat_buf_len = sizeof(patterns);
826baf753ccSJohn Baldwin pc.patterns = patterns;
827baf753ccSJohn Baldwin
828baf753ccSJohn Baldwin if (ioctl(pcifd, PCIOCGETCONF, &pc) == -1) {
829baf753ccSJohn Baldwin EPRINTLN("ioctl(PCIOCGETCONF): %s", strerror(errno));
830baf753ccSJohn Baldwin return (false);
831baf753ccSJohn Baldwin }
832baf753ccSJohn Baldwin if (pc.status != PCI_GETCONF_LAST_DEVICE &&
833baf753ccSJohn Baldwin pc.status != PCI_GETCONF_MORE_DEVS) {
834baf753ccSJohn Baldwin EPRINTLN("error returned from PCIOCGETCONF ioctl");
835baf753ccSJohn Baldwin return (false);
836baf753ccSJohn Baldwin }
837baf753ccSJohn Baldwin if (pc.num_matches == 0) {
838baf753ccSJohn Baldwin EPRINTLN("Passthru device %s not found", name);
839baf753ccSJohn Baldwin return (false);
840baf753ccSJohn Baldwin }
841baf753ccSJohn Baldwin
842baf753ccSJohn Baldwin if (conf[0].pc_sel.pc_domain != 0) {
843baf753ccSJohn Baldwin EPRINTLN("Passthru device %s on unsupported domain", name);
844baf753ccSJohn Baldwin return (false);
845baf753ccSJohn Baldwin }
846baf753ccSJohn Baldwin *bus = conf[0].pc_sel.pc_bus;
847baf753ccSJohn Baldwin *slot = conf[0].pc_sel.pc_dev;
848baf753ccSJohn Baldwin *func = conf[0].pc_sel.pc_func;
849baf753ccSJohn Baldwin return (true);
850baf753ccSJohn Baldwin }
851baf753ccSJohn Baldwin
852621b5090SJohn Baldwin static int
passthru_init(struct pci_devinst * pi,nvlist_t * nvl)8536a284cacSJohn Baldwin passthru_init(struct pci_devinst *pi, nvlist_t *nvl)
854366f6083SPeter Grehan {
8559b1aa8d6SNeel Natu int bus, slot, func, error, memflags;
856366f6083SPeter Grehan struct passthru_softc *sc;
85790c3a1b6SCorvin Köhne struct passthru_dev **devpp;
85890c3a1b6SCorvin Köhne struct passthru_dev *devp, *dev = NULL;
859621b5090SJohn Baldwin const char *value;
860366f6083SPeter Grehan
861366f6083SPeter Grehan sc = NULL;
862366f6083SPeter Grehan error = 1;
863366f6083SPeter Grehan
8646a284cacSJohn Baldwin memflags = vm_get_memflags(pi->pi_vmctx);
8659b1aa8d6SNeel Natu if (!(memflags & VM_MEM_F_WIRED)) {
866cff92ffdSJohn Baldwin warnx("passthru requires guest memory to be wired");
867dbb15211SSean Chittenden return (error);
8689b1aa8d6SNeel Natu }
8699b1aa8d6SNeel Natu
870563fd224SCorvin Köhne if (pcifd < 0 && pcifd_init()) {
871dbb15211SSean Chittenden return (error);
872366f6083SPeter Grehan }
87300ef17beSBartek Rutkowski
874621b5090SJohn Baldwin #define GET_INT_CONFIG(var, name) do { \
875621b5090SJohn Baldwin value = get_config_value_node(nvl, name); \
876621b5090SJohn Baldwin if (value == NULL) { \
877621b5090SJohn Baldwin EPRINTLN("passthru: missing required %s setting", name); \
878621b5090SJohn Baldwin return (error); \
879621b5090SJohn Baldwin } \
880621b5090SJohn Baldwin var = atoi(value); \
881621b5090SJohn Baldwin } while (0)
882621b5090SJohn Baldwin
883baf753ccSJohn Baldwin value = get_config_value_node(nvl, "pptdev");
884baf753ccSJohn Baldwin if (value != NULL) {
885baf753ccSJohn Baldwin if (!passthru_lookup_pptdev(value, &bus, &slot, &func))
886baf753ccSJohn Baldwin return (error);
887baf753ccSJohn Baldwin } else {
888621b5090SJohn Baldwin GET_INT_CONFIG(bus, "bus");
889621b5090SJohn Baldwin GET_INT_CONFIG(slot, "slot");
890621b5090SJohn Baldwin GET_INT_CONFIG(func, "func");
891baf753ccSJohn Baldwin }
892366f6083SPeter Grehan
8936a284cacSJohn Baldwin if (vm_assign_pptdev(pi->pi_vmctx, bus, slot, func) != 0) {
894cff92ffdSJohn Baldwin warnx("PCI device at %d/%d/%d is not using the ppt(4) driver",
895cff92ffdSJohn Baldwin bus, slot, func);
896366f6083SPeter Grehan goto done;
897cff92ffdSJohn Baldwin }
898366f6083SPeter Grehan
899994f858aSXin LI sc = calloc(1, sizeof(struct passthru_softc));
900366f6083SPeter Grehan
901366f6083SPeter Grehan pi->pi_arg = sc;
902366f6083SPeter Grehan sc->psc_pi = pi;
903366f6083SPeter Grehan
904366f6083SPeter Grehan /* initialize config space */
9056a284cacSJohn Baldwin if ((error = cfginit(pi, bus, slot, func)) != 0)
906e47fe318SCorvin Köhne goto done;
907e47fe318SCorvin Köhne
908e47fe318SCorvin Köhne /* initialize ROM */
9096a284cacSJohn Baldwin if ((error = passthru_init_rom(sc,
910e47fe318SCorvin Köhne get_config_value_node(nvl, "rom"))) != 0)
911e47fe318SCorvin Köhne goto done;
912e47fe318SCorvin Köhne
913b6e67875SCorvin Köhne /* Emulate most PCI header register. */
914b6e67875SCorvin Köhne if ((error = set_pcir_handler(sc, 0, PCIR_MAXLAT + 1,
915b6e67875SCorvin Köhne passthru_cfgread_emulate, passthru_cfgwrite_emulate)) != 0)
916b6e67875SCorvin Köhne goto done;
917b6e67875SCorvin Köhne
918*f44ff2abSJohn Baldwin /* Allow access to the physical status register. */
919b6e67875SCorvin Köhne if ((error = set_pcir_handler(sc, PCIR_COMMAND, 0x04, NULL, NULL)) != 0)
920b6e67875SCorvin Köhne goto done;
921b6e67875SCorvin Köhne
92290c3a1b6SCorvin Köhne SET_FOREACH(devpp, passthru_dev_set) {
92390c3a1b6SCorvin Köhne devp = *devpp;
92490c3a1b6SCorvin Köhne assert(devp->probe != NULL);
92590c3a1b6SCorvin Köhne if (devp->probe(pi) == 0) {
92690c3a1b6SCorvin Köhne dev = devp;
92790c3a1b6SCorvin Köhne break;
92890c3a1b6SCorvin Köhne }
92990c3a1b6SCorvin Köhne }
93090c3a1b6SCorvin Köhne
93190c3a1b6SCorvin Köhne if (dev != NULL) {
93290c3a1b6SCorvin Köhne error = dev->init(pi, nvl);
93390c3a1b6SCorvin Köhne if (error != 0)
93490c3a1b6SCorvin Köhne goto done;
93590c3a1b6SCorvin Köhne }
93690c3a1b6SCorvin Köhne
937e47fe318SCorvin Köhne error = 0; /* success */
938366f6083SPeter Grehan done:
939366f6083SPeter Grehan if (error) {
94090c3a1b6SCorvin Köhne if (dev != NULL)
94190c3a1b6SCorvin Köhne dev->deinit(pi);
942366f6083SPeter Grehan free(sc);
9436a284cacSJohn Baldwin vm_unassign_pptdev(pi->pi_vmctx, bus, slot, func);
944366f6083SPeter Grehan }
945366f6083SPeter Grehan return (error);
946366f6083SPeter Grehan }
947366f6083SPeter Grehan
948366f6083SPeter Grehan static int
msicap_access(struct passthru_softc * sc,int coff)949366f6083SPeter Grehan msicap_access(struct passthru_softc *sc, int coff)
950366f6083SPeter Grehan {
951366f6083SPeter Grehan int caplen;
952366f6083SPeter Grehan
953366f6083SPeter Grehan if (sc->psc_msi.capoff == 0)
954366f6083SPeter Grehan return (0);
955366f6083SPeter Grehan
956366f6083SPeter Grehan caplen = msi_caplen(sc->psc_msi.msgctrl);
957366f6083SPeter Grehan
958366f6083SPeter Grehan if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen)
959366f6083SPeter Grehan return (1);
960366f6083SPeter Grehan else
961366f6083SPeter Grehan return (0);
962366f6083SPeter Grehan }
963366f6083SPeter Grehan
964366f6083SPeter Grehan static int
msixcap_access(struct passthru_softc * sc,int coff)965cd942e0fSPeter Grehan msixcap_access(struct passthru_softc *sc, int coff)
966cd942e0fSPeter Grehan {
967cd942e0fSPeter Grehan if (sc->psc_msix.capoff == 0)
968cd942e0fSPeter Grehan return (0);
969cd942e0fSPeter Grehan
970cd942e0fSPeter Grehan return (coff >= sc->psc_msix.capoff &&
971cd942e0fSPeter Grehan coff < sc->psc_msix.capoff + MSIX_CAPLEN);
972cd942e0fSPeter Grehan }
973cd942e0fSPeter Grehan
974cd942e0fSPeter Grehan static int
passthru_cfgread_default(struct passthru_softc * sc,struct pci_devinst * pi __unused,int coff,int bytes,uint32_t * rv)975931bb7bfSCorvin Köhne passthru_cfgread_default(struct passthru_softc *sc,
976931bb7bfSCorvin Köhne struct pci_devinst *pi __unused, int coff, int bytes, uint32_t *rv)
977366f6083SPeter Grehan {
978366f6083SPeter Grehan /*
979b6e67875SCorvin Köhne * MSI capability is emulated.
980366f6083SPeter Grehan */
981b6e67875SCorvin Köhne if (msicap_access(sc, coff) || msixcap_access(sc, coff))
982366f6083SPeter Grehan return (-1);
983366f6083SPeter Grehan
984c7ba149dSJohn Baldwin /*
985c7ba149dSJohn Baldwin * Emulate the command register. If a single read reads both the
986c7ba149dSJohn Baldwin * command and status registers, read the status register from the
987c7ba149dSJohn Baldwin * device's config space.
988c7ba149dSJohn Baldwin */
989c7ba149dSJohn Baldwin if (coff == PCIR_COMMAND) {
990c7ba149dSJohn Baldwin if (bytes <= 2)
991c7ba149dSJohn Baldwin return (-1);
99201d53c34SMark Johnston *rv = pci_host_read_config(&sc->psc_sel, PCIR_STATUS, 2) << 16 |
99321368498SPeter Grehan pci_get_cfgdata16(pi, PCIR_COMMAND);
994c7ba149dSJohn Baldwin return (0);
995c7ba149dSJohn Baldwin }
996c7ba149dSJohn Baldwin
997366f6083SPeter Grehan /* Everything else just read from the device's config space */
99801d53c34SMark Johnston *rv = pci_host_read_config(&sc->psc_sel, coff, bytes);
999366f6083SPeter Grehan
1000366f6083SPeter Grehan return (0);
1001366f6083SPeter Grehan }
1002366f6083SPeter Grehan
1003b6e67875SCorvin Köhne int
passthru_cfgread_emulate(struct passthru_softc * sc __unused,struct pci_devinst * pi __unused,int coff __unused,int bytes __unused,uint32_t * rv __unused)1004b6e67875SCorvin Köhne passthru_cfgread_emulate(struct passthru_softc *sc __unused,
1005b6e67875SCorvin Köhne struct pci_devinst *pi __unused, int coff __unused, int bytes __unused,
1006b6e67875SCorvin Köhne uint32_t *rv __unused)
1007b6e67875SCorvin Köhne {
1008b6e67875SCorvin Köhne return (-1);
1009b6e67875SCorvin Köhne }
1010b6e67875SCorvin Köhne
1011366f6083SPeter Grehan static int
passthru_cfgread(struct pci_devinst * pi,int coff,int bytes,uint32_t * rv)1012931bb7bfSCorvin Köhne passthru_cfgread(struct pci_devinst *pi, int coff, int bytes, uint32_t *rv)
1013366f6083SPeter Grehan {
1014366f6083SPeter Grehan struct passthru_softc *sc;
1015366f6083SPeter Grehan
1016366f6083SPeter Grehan sc = pi->pi_arg;
1017366f6083SPeter Grehan
1018931bb7bfSCorvin Köhne if (sc->psc_pcir_rhandler[coff] != NULL)
1019931bb7bfSCorvin Köhne return (sc->psc_pcir_rhandler[coff](sc, pi, coff, bytes, rv));
1020931bb7bfSCorvin Köhne
1021931bb7bfSCorvin Köhne return (passthru_cfgread_default(sc, pi, coff, bytes, rv));
1022931bb7bfSCorvin Köhne }
1023931bb7bfSCorvin Köhne
1024931bb7bfSCorvin Köhne static int
passthru_cfgwrite_default(struct passthru_softc * sc,struct pci_devinst * pi,int coff,int bytes,uint32_t val)1025931bb7bfSCorvin Köhne passthru_cfgwrite_default(struct passthru_softc *sc, struct pci_devinst *pi,
1026931bb7bfSCorvin Köhne int coff, int bytes, uint32_t val)
1027931bb7bfSCorvin Köhne {
1028931bb7bfSCorvin Köhne int error, msix_table_entries, i;
1029931bb7bfSCorvin Köhne uint16_t cmd_old;
1030931bb7bfSCorvin Köhne
1031366f6083SPeter Grehan /*
1032366f6083SPeter Grehan * MSI capability is emulated
1033366f6083SPeter Grehan */
1034366f6083SPeter Grehan if (msicap_access(sc, coff)) {
103521368498SPeter Grehan pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,
103621368498SPeter Grehan PCIY_MSI);
10377d9ef309SJohn Baldwin error = vm_setup_pptdev_msi(pi->pi_vmctx, sc->psc_sel.pc_bus,
10384f8be175SNeel Natu sc->psc_sel.pc_dev, sc->psc_sel.pc_func,
10394f8be175SNeel Natu pi->pi_msi.addr, pi->pi_msi.msg_data,
10404f8be175SNeel Natu pi->pi_msi.maxmsgnum);
1041cff92ffdSJohn Baldwin if (error != 0)
1042cff92ffdSJohn Baldwin err(1, "vm_setup_pptdev_msi");
1043366f6083SPeter Grehan return (0);
1044366f6083SPeter Grehan }
1045366f6083SPeter Grehan
1046cd942e0fSPeter Grehan if (msixcap_access(sc, coff)) {
104721368498SPeter Grehan pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff,
104821368498SPeter Grehan PCIY_MSIX);
1049cd942e0fSPeter Grehan if (pi->pi_msix.enabled) {
1050cd942e0fSPeter Grehan msix_table_entries = pi->pi_msix.table_count;
1051cd942e0fSPeter Grehan for (i = 0; i < msix_table_entries; i++) {
10527d9ef309SJohn Baldwin error = vm_setup_pptdev_msix(pi->pi_vmctx,
10534f8be175SNeel Natu sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
1054cd942e0fSPeter Grehan sc->psc_sel.pc_func, i,
10554f8be175SNeel Natu pi->pi_msix.table[i].addr,
1056cd942e0fSPeter Grehan pi->pi_msix.table[i].msg_data,
10574f8be175SNeel Natu pi->pi_msix.table[i].vector_control);
1058cd942e0fSPeter Grehan
1059cff92ffdSJohn Baldwin if (error)
1060cff92ffdSJohn Baldwin err(1, "vm_setup_pptdev_msix");
1061cd942e0fSPeter Grehan }
10621925586eSJohn Baldwin } else {
10636a284cacSJohn Baldwin error = vm_disable_pptdev_msix(pi->pi_vmctx,
10646a284cacSJohn Baldwin sc->psc_sel.pc_bus, sc->psc_sel.pc_dev,
10656a284cacSJohn Baldwin sc->psc_sel.pc_func);
10661925586eSJohn Baldwin if (error)
10671925586eSJohn Baldwin err(1, "vm_disable_pptdev_msix");
1068cd942e0fSPeter Grehan }
1069cd942e0fSPeter Grehan return (0);
1070cd942e0fSPeter Grehan }
1071cd942e0fSPeter Grehan
1072366f6083SPeter Grehan /*
1073*f44ff2abSJohn Baldwin * The command register is emulated, but the status register
1074*f44ff2abSJohn Baldwin * is passed through.
1075366f6083SPeter Grehan */
1076*f44ff2abSJohn Baldwin if (coff == PCIR_COMMAND) {
1077*f44ff2abSJohn Baldwin if (bytes <= 2)
1078*f44ff2abSJohn Baldwin return (-1);
1079*f44ff2abSJohn Baldwin
1080*f44ff2abSJohn Baldwin /* Update the physical status register. */
1081*f44ff2abSJohn Baldwin pci_host_write_config(&sc->psc_sel, PCIR_STATUS, val >> 16, 2);
1082*f44ff2abSJohn Baldwin
1083*f44ff2abSJohn Baldwin /* Update the virtual command register. */
1084*f44ff2abSJohn Baldwin cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND);
1085*f44ff2abSJohn Baldwin pci_set_cfgdata16(pi, PCIR_COMMAND, val & 0xffff);
1086*f44ff2abSJohn Baldwin pci_emul_cmd_changed(pi, cmd_old);
1087*f44ff2abSJohn Baldwin return (0);
1088366f6083SPeter Grehan }
1089366f6083SPeter Grehan
109001d53c34SMark Johnston pci_host_write_config(&sc->psc_sel, coff, bytes, val);
1091366f6083SPeter Grehan
1092366f6083SPeter Grehan return (0);
1093366f6083SPeter Grehan }
1094366f6083SPeter Grehan
1095b6e67875SCorvin Köhne int
passthru_cfgwrite_emulate(struct passthru_softc * sc __unused,struct pci_devinst * pi __unused,int coff __unused,int bytes __unused,uint32_t val __unused)1096b6e67875SCorvin Köhne passthru_cfgwrite_emulate(struct passthru_softc *sc __unused,
1097b6e67875SCorvin Köhne struct pci_devinst *pi __unused, int coff __unused, int bytes __unused,
1098b6e67875SCorvin Köhne uint32_t val __unused)
1099b6e67875SCorvin Köhne {
1100b6e67875SCorvin Köhne return (-1);
1101b6e67875SCorvin Köhne }
1102b6e67875SCorvin Köhne
1103931bb7bfSCorvin Köhne static int
passthru_cfgwrite(struct pci_devinst * pi,int coff,int bytes,uint32_t val)1104931bb7bfSCorvin Köhne passthru_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val)
1105931bb7bfSCorvin Köhne {
1106931bb7bfSCorvin Köhne struct passthru_softc *sc;
1107931bb7bfSCorvin Köhne
1108931bb7bfSCorvin Köhne sc = pi->pi_arg;
1109931bb7bfSCorvin Köhne
1110931bb7bfSCorvin Köhne if (sc->psc_pcir_whandler[coff] != NULL)
1111931bb7bfSCorvin Köhne return (sc->psc_pcir_whandler[coff](sc, pi, coff, bytes, val));
1112931bb7bfSCorvin Köhne
1113931bb7bfSCorvin Köhne return (passthru_cfgwrite_default(sc, pi, coff, bytes, val));
1114931bb7bfSCorvin Köhne }
1115931bb7bfSCorvin Köhne
1116366f6083SPeter Grehan static void
passthru_write(struct pci_devinst * pi,int baridx,uint64_t offset,int size,uint64_t value)11176a284cacSJohn Baldwin passthru_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size,
11186a284cacSJohn Baldwin uint64_t value)
1119366f6083SPeter Grehan {
1120366f6083SPeter Grehan struct passthru_softc *sc;
112142375556SMark Johnston struct pci_bar_ioreq pio;
1122366f6083SPeter Grehan
1123366f6083SPeter Grehan sc = pi->pi_arg;
1124366f6083SPeter Grehan
1125aa12663fSNeel Natu if (baridx == pci_msix_table_bar(pi)) {
11266a284cacSJohn Baldwin msix_table_write(sc, offset, size, value);
11274d1e669cSPeter Grehan } else {
11284d1e669cSPeter Grehan assert(pi->pi_bar[baridx].type == PCIBAR_IO);
112942375556SMark Johnston assert(size == 1 || size == 2 || size == 4);
113042375556SMark Johnston assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX);
1131366f6083SPeter Grehan
113242375556SMark Johnston bzero(&pio, sizeof(pio));
113342375556SMark Johnston pio.pbi_sel = sc->psc_sel;
113442375556SMark Johnston pio.pbi_op = PCIBARIO_WRITE;
113542375556SMark Johnston pio.pbi_bar = baridx;
113642375556SMark Johnston pio.pbi_offset = (uint32_t)offset;
113742375556SMark Johnston pio.pbi_width = size;
113842375556SMark Johnston pio.pbi_value = (uint32_t)value;
113942375556SMark Johnston
114042375556SMark Johnston (void)ioctl(pcifd, PCIOCBARIO, &pio);
1141366f6083SPeter Grehan }
11424d1e669cSPeter Grehan }
1143366f6083SPeter Grehan
11444d1e669cSPeter Grehan static uint64_t
passthru_read(struct pci_devinst * pi,int baridx,uint64_t offset,int size)11456a284cacSJohn Baldwin passthru_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size)
1146366f6083SPeter Grehan {
1147366f6083SPeter Grehan struct passthru_softc *sc;
114842375556SMark Johnston struct pci_bar_ioreq pio;
11494d1e669cSPeter Grehan uint64_t val;
1150366f6083SPeter Grehan
1151366f6083SPeter Grehan sc = pi->pi_arg;
1152366f6083SPeter Grehan
1153aa12663fSNeel Natu if (baridx == pci_msix_table_bar(pi)) {
11544d1e669cSPeter Grehan val = msix_table_read(sc, offset, size);
11554d1e669cSPeter Grehan } else {
11564d1e669cSPeter Grehan assert(pi->pi_bar[baridx].type == PCIBAR_IO);
115742375556SMark Johnston assert(size == 1 || size == 2 || size == 4);
115842375556SMark Johnston assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX);
1159366f6083SPeter Grehan
116042375556SMark Johnston bzero(&pio, sizeof(pio));
116142375556SMark Johnston pio.pbi_sel = sc->psc_sel;
116242375556SMark Johnston pio.pbi_op = PCIBARIO_READ;
116342375556SMark Johnston pio.pbi_bar = baridx;
116442375556SMark Johnston pio.pbi_offset = (uint32_t)offset;
116542375556SMark Johnston pio.pbi_width = size;
1166366f6083SPeter Grehan
116742375556SMark Johnston (void)ioctl(pcifd, PCIOCBARIO, &pio);
116842375556SMark Johnston
116942375556SMark Johnston val = pio.pbi_value;
11704d1e669cSPeter Grehan }
11714d1e669cSPeter Grehan
11724d1e669cSPeter Grehan return (val);
1173366f6083SPeter Grehan }
1174366f6083SPeter Grehan
1175f8a6ec2dSD Scott Phillips static void
passthru_msix_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)11766a284cacSJohn Baldwin passthru_msix_addr(struct pci_devinst *pi, int baridx, int enabled,
11776a284cacSJohn Baldwin uint64_t address)
1178f8a6ec2dSD Scott Phillips {
1179f8a6ec2dSD Scott Phillips struct passthru_softc *sc;
1180f8a6ec2dSD Scott Phillips size_t remaining;
1181f8a6ec2dSD Scott Phillips uint32_t table_size, table_offset;
1182f8a6ec2dSD Scott Phillips
1183f8a6ec2dSD Scott Phillips sc = pi->pi_arg;
1184f8a6ec2dSD Scott Phillips table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
1185f8a6ec2dSD Scott Phillips if (table_offset > 0) {
1186f8a6ec2dSD Scott Phillips if (!enabled) {
11876a284cacSJohn Baldwin if (vm_unmap_pptdev_mmio(pi->pi_vmctx,
11886a284cacSJohn Baldwin sc->psc_sel.pc_bus,
1189f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1190f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1191f8a6ec2dSD Scott Phillips table_offset) != 0)
1192f8a6ec2dSD Scott Phillips warnx("pci_passthru: unmap_pptdev_mmio failed");
1193f8a6ec2dSD Scott Phillips } else {
11946a284cacSJohn Baldwin if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1195f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1196f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1197f8a6ec2dSD Scott Phillips table_offset,
1198f8a6ec2dSD Scott Phillips sc->psc_bar[baridx].addr) != 0)
1199f8a6ec2dSD Scott Phillips warnx("pci_passthru: map_pptdev_mmio failed");
1200f8a6ec2dSD Scott Phillips }
1201f8a6ec2dSD Scott Phillips }
1202f8a6ec2dSD Scott Phillips table_size = pi->pi_msix.table_offset - table_offset;
1203f8a6ec2dSD Scott Phillips table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
1204f8a6ec2dSD Scott Phillips table_size = roundup2(table_size, 4096);
1205f8a6ec2dSD Scott Phillips remaining = pi->pi_bar[baridx].size - table_offset - table_size;
1206f8a6ec2dSD Scott Phillips if (remaining > 0) {
1207f8a6ec2dSD Scott Phillips address += table_offset + table_size;
1208f8a6ec2dSD Scott Phillips if (!enabled) {
12096a284cacSJohn Baldwin if (vm_unmap_pptdev_mmio(pi->pi_vmctx,
12106a284cacSJohn Baldwin sc->psc_sel.pc_bus,
1211f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1212f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1213f8a6ec2dSD Scott Phillips remaining) != 0)
1214f8a6ec2dSD Scott Phillips warnx("pci_passthru: unmap_pptdev_mmio failed");
1215f8a6ec2dSD Scott Phillips } else {
12166a284cacSJohn Baldwin if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1217f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1218f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1219f8a6ec2dSD Scott Phillips remaining,
1220f8a6ec2dSD Scott Phillips sc->psc_bar[baridx].addr +
1221f8a6ec2dSD Scott Phillips table_offset + table_size) != 0)
1222f8a6ec2dSD Scott Phillips warnx("pci_passthru: map_pptdev_mmio failed");
1223f8a6ec2dSD Scott Phillips }
1224f8a6ec2dSD Scott Phillips }
1225f8a6ec2dSD Scott Phillips }
1226f8a6ec2dSD Scott Phillips
1227f8a6ec2dSD Scott Phillips static void
passthru_mmio_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)12286a284cacSJohn Baldwin passthru_mmio_addr(struct pci_devinst *pi, int baridx, int enabled,
12296a284cacSJohn Baldwin uint64_t address)
1230f8a6ec2dSD Scott Phillips {
1231f8a6ec2dSD Scott Phillips struct passthru_softc *sc;
1232f8a6ec2dSD Scott Phillips
1233f8a6ec2dSD Scott Phillips sc = pi->pi_arg;
1234f8a6ec2dSD Scott Phillips if (!enabled) {
12356a284cacSJohn Baldwin if (vm_unmap_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1236f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1237f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1238f8a6ec2dSD Scott Phillips sc->psc_bar[baridx].size) != 0)
1239f8a6ec2dSD Scott Phillips warnx("pci_passthru: unmap_pptdev_mmio failed");
1240f8a6ec2dSD Scott Phillips } else {
12416a284cacSJohn Baldwin if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus,
1242f8a6ec2dSD Scott Phillips sc->psc_sel.pc_dev,
1243f8a6ec2dSD Scott Phillips sc->psc_sel.pc_func, address,
1244f8a6ec2dSD Scott Phillips sc->psc_bar[baridx].size,
1245f8a6ec2dSD Scott Phillips sc->psc_bar[baridx].addr) != 0)
1246f8a6ec2dSD Scott Phillips warnx("pci_passthru: map_pptdev_mmio failed");
1247f8a6ec2dSD Scott Phillips }
1248f8a6ec2dSD Scott Phillips }
1249f8a6ec2dSD Scott Phillips
1250f8a6ec2dSD Scott Phillips static void
passthru_addr_rom(struct pci_devinst * const pi,const int idx,const int enabled)1251e47fe318SCorvin Köhne passthru_addr_rom(struct pci_devinst *const pi, const int idx,
1252e47fe318SCorvin Köhne const int enabled)
1253e47fe318SCorvin Köhne {
1254e47fe318SCorvin Köhne const uint64_t addr = pi->pi_bar[idx].addr;
1255e47fe318SCorvin Köhne const uint64_t size = pi->pi_bar[idx].size;
1256e47fe318SCorvin Köhne
1257e47fe318SCorvin Köhne if (!enabled) {
1258e47fe318SCorvin Köhne if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) {
1259e47fe318SCorvin Köhne errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed",
1260e47fe318SCorvin Köhne __func__, addr, addr + size);
1261e47fe318SCorvin Köhne }
1262e47fe318SCorvin Köhne
1263e47fe318SCorvin Köhne } else {
1264e47fe318SCorvin Köhne if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM,
1265e47fe318SCorvin Köhne pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) {
126650526f52SCorvin Köhne errx(4, "%s: mmap_memseg @ [%016lx - %016lx] failed",
1267e47fe318SCorvin Köhne __func__, addr, addr + size);
1268e47fe318SCorvin Köhne }
1269e47fe318SCorvin Köhne }
1270e47fe318SCorvin Köhne }
1271e47fe318SCorvin Köhne
1272e47fe318SCorvin Köhne static void
passthru_addr(struct pci_devinst * pi,int baridx,int enabled,uint64_t address)12736a284cacSJohn Baldwin passthru_addr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address)
1274f8a6ec2dSD Scott Phillips {
1275e47fe318SCorvin Köhne switch (pi->pi_bar[baridx].type) {
1276e47fe318SCorvin Köhne case PCIBAR_IO:
1277e47fe318SCorvin Köhne /* IO BARs are emulated */
1278e47fe318SCorvin Köhne break;
1279e47fe318SCorvin Köhne case PCIBAR_ROM:
1280e47fe318SCorvin Köhne passthru_addr_rom(pi, baridx, enabled);
1281e47fe318SCorvin Köhne break;
1282e47fe318SCorvin Köhne case PCIBAR_MEM32:
1283e47fe318SCorvin Köhne case PCIBAR_MEM64:
1284f8a6ec2dSD Scott Phillips if (baridx == pci_msix_table_bar(pi))
12856a284cacSJohn Baldwin passthru_msix_addr(pi, baridx, enabled, address);
1286f8a6ec2dSD Scott Phillips else
12876a284cacSJohn Baldwin passthru_mmio_addr(pi, baridx, enabled, address);
1288e47fe318SCorvin Köhne break;
1289e47fe318SCorvin Köhne default:
1290e47fe318SCorvin Köhne errx(4, "%s: invalid BAR type %d", __func__,
1291e47fe318SCorvin Köhne pi->pi_bar[baridx].type);
1292e47fe318SCorvin Köhne }
1293f8a6ec2dSD Scott Phillips }
1294f8a6ec2dSD Scott Phillips
129537045dfaSMark Johnston static const struct pci_devemu passthru = {
1296366f6083SPeter Grehan .pe_emu = "passthru",
1297366f6083SPeter Grehan .pe_init = passthru_init,
1298621b5090SJohn Baldwin .pe_legacy_config = passthru_legacy_config,
1299366f6083SPeter Grehan .pe_cfgwrite = passthru_cfgwrite,
1300366f6083SPeter Grehan .pe_cfgread = passthru_cfgread,
13014d1e669cSPeter Grehan .pe_barwrite = passthru_write,
13024d1e669cSPeter Grehan .pe_barread = passthru_read,
1303f8a6ec2dSD Scott Phillips .pe_baraddr = passthru_addr,
1304366f6083SPeter Grehan };
1305366f6083SPeter Grehan PCI_EMUL_SET(passthru);
1306