xref: /freebsd/usr.sbin/bhyve/pci_emul.c (revision f1d705d4f431a5553aa5e64310bdc703ba27eab4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/param.h>
30 #include <sys/linker_set.h>
31 #include <sys/mman.h>
32 
33 #include <ctype.h>
34 #include <err.h>
35 #include <errno.h>
36 #include <pthread.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <string.h>
40 #include <strings.h>
41 #include <assert.h>
42 #include <stdbool.h>
43 #include <sysexits.h>
44 
45 #include <dev/vmm/vmm_mem.h>
46 #include <machine/vmm.h>
47 #include <machine/vmm_snapshot.h>
48 #include <vmmapi.h>
49 
50 #include "acpi.h"
51 #include "bhyverun.h"
52 #include "bootrom.h"
53 #include "config.h"
54 #include "debug.h"
55 #ifdef __amd64__
56 #include "amd64/inout.h"
57 #endif
58 #include "mem.h"
59 #include "pci_emul.h"
60 #ifdef __amd64__
61 #include "amd64/pci_lpc.h"
62 #include "pci_passthru.h"
63 #endif
64 #include "qemu_fwcfg.h"
65 
66 #define CONF1_ADDR_PORT	   0x0cf8
67 #define CONF1_DATA_PORT	   0x0cfc
68 
69 #define CONF1_ENABLE	   0x80000000ul
70 
71 #define	MAXBUSES	(PCI_BUSMAX + 1)
72 #define MAXSLOTS	(PCI_SLOTMAX + 1)
73 #define	MAXFUNCS	(PCI_FUNCMAX + 1)
74 
75 #define GB		(1024 * 1024 * 1024UL)
76 
77 struct funcinfo {
78 	nvlist_t *fi_config;
79 	struct pci_devemu *fi_pde;
80 	struct pci_devinst *fi_devi;
81 };
82 
83 struct intxinfo {
84 	int		ii_count;
85 	struct pci_irq	ii_irq;
86 };
87 
88 struct slotinfo {
89 	struct intxinfo si_intpins[4];
90 	struct funcinfo si_funcs[MAXFUNCS];
91 };
92 
93 struct businfo {
94 	uint16_t iobase, iolimit;		/* I/O window */
95 	uint32_t membase32, memlimit32;		/* mmio window below 4GB */
96 	uint64_t membase64, memlimit64;		/* mmio window above 4GB */
97 	struct slotinfo slotinfo[MAXSLOTS];
98 };
99 
100 static struct businfo *pci_businfo[MAXBUSES];
101 
102 SET_DECLARE(pci_devemu_set, struct pci_devemu);
103 
104 static uint64_t pci_emul_iobase;
105 static uint8_t *pci_emul_rombase;
106 static uint64_t pci_emul_romoffset;
107 static uint8_t *pci_emul_romlim;
108 static uint64_t pci_emul_membase32;
109 static uint64_t pci_emul_membase64;
110 static uint64_t pci_emul_memlim64;
111 
112 struct pci_bar_allocation {
113 	TAILQ_ENTRY(pci_bar_allocation) chain;
114 	struct pci_devinst *pdi;
115 	int idx;
116 	enum pcibar_type type;
117 	uint64_t size;
118 };
119 
120 static TAILQ_HEAD(pci_bar_list, pci_bar_allocation) pci_bars =
121     TAILQ_HEAD_INITIALIZER(pci_bars);
122 
123 struct boot_device {
124 	TAILQ_ENTRY(boot_device) boot_device_chain;
125 	struct pci_devinst *pdi;
126 	int bootindex;
127 };
128 static TAILQ_HEAD(boot_list, boot_device) boot_devices = TAILQ_HEAD_INITIALIZER(
129     boot_devices);
130 
131 #if defined(__amd64__)
132 #define	PCI_EMUL_IOBASE		0x2000
133 #define	PCI_EMUL_IOLIMIT	0x10000
134 #define	PCI_EMUL_IOMASK		0xffff
135 /*
136  * OVMF always uses 0xc0000000 as base address for 32 bit PCI MMIO. Don't
137  * change this address without changing it in OVMF.
138  */
139 #define	PCI_EMUL_MEMBASE32	0xc0000000
140 #elif defined(__aarch64__) || defined(__riscv)
141 #define	PCI_EMUL_IOBASE		0xdf000000UL
142 #define	PCI_EMUL_IOLIMIT	0xe0000000UL
143 #define	PCI_EMUL_MEMBASE32	0xa0000000UL
144 #else
145 #error Unsupported platform
146 #endif
147 
148 #define	PCI_EMUL_ROMSIZE	0x10000000
149 
150 #define	PCI_EMUL_ECFG_BASE	0xE0000000		    /* 3.5GB */
151 #define	PCI_EMUL_ECFG_SIZE	(MAXBUSES * 1024 * 1024)    /* 1MB per bus */
152 #ifdef __amd64__
153 SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE);
154 #endif
155 
156 #define	PCI_EMUL_MEMLIMIT32	PCI_EMUL_ECFG_BASE
157 #define PCI_EMUL_MEMSIZE64	(32*GB)
158 
159 static void pci_lintr_route(struct pci_devinst *pi);
160 static void pci_lintr_update(struct pci_devinst *pi);
161 
162 static struct pci_devemu *pci_emul_finddev(const char *name);
163 static void pci_cfgrw(int in, int bus, int slot, int func, int coff,
164     int bytes, uint32_t *val);
165 
166 static __inline void
CFGWRITE(struct pci_devinst * pi,int coff,uint32_t val,int bytes)167 CFGWRITE(struct pci_devinst *pi, int coff, uint32_t val, int bytes)
168 {
169 
170 	if (bytes == 1)
171 		pci_set_cfgdata8(pi, coff, val);
172 	else if (bytes == 2)
173 		pci_set_cfgdata16(pi, coff, val);
174 	else
175 		pci_set_cfgdata32(pi, coff, val);
176 }
177 
178 static __inline uint32_t
CFGREAD(struct pci_devinst * pi,int coff,int bytes)179 CFGREAD(struct pci_devinst *pi, int coff, int bytes)
180 {
181 
182 	if (bytes == 1)
183 		return (pci_get_cfgdata8(pi, coff));
184 	else if (bytes == 2)
185 		return (pci_get_cfgdata16(pi, coff));
186 	else
187 		return (pci_get_cfgdata32(pi, coff));
188 }
189 
190 static int
is_pcir_bar(int coff)191 is_pcir_bar(int coff)
192 {
193 	return (coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1));
194 }
195 
196 static int
is_pcir_bios(int coff)197 is_pcir_bios(int coff)
198 {
199 	return (coff >= PCIR_BIOS && coff < PCIR_BIOS + 4);
200 }
201 
202 /*
203  * I/O access
204  */
205 
206 /*
207  * Slot options are in the form:
208  *
209  *  <bus>:<slot>:<func>,<emul>[,<config>]
210  *  <slot>[:<func>],<emul>[,<config>]
211  *
212  *  slot is 0..31
213  *  func is 0..7
214  *  emul is a string describing the type of PCI device e.g. virtio-net
215  *  config is an optional string, depending on the device, that can be
216  *  used for configuration.
217  *   Examples are:
218  *     1,virtio-net,tap0
219  *     3:0,dummy
220  */
221 static void
pci_parse_slot_usage(char * aopt)222 pci_parse_slot_usage(char *aopt)
223 {
224 
225 	EPRINTLN("Invalid PCI slot info field \"%s\"", aopt);
226 }
227 
228 /*
229  * Helper function to parse a list of comma-separated options where
230  * each option is formatted as "name[=value]".  If no value is
231  * provided, the option is treated as a boolean and is given a value
232  * of true.
233  */
234 int
pci_parse_legacy_config(nvlist_t * nvl,const char * opt)235 pci_parse_legacy_config(nvlist_t *nvl, const char *opt)
236 {
237 	char *config, *name, *tofree, *value;
238 
239 	if (opt == NULL)
240 		return (0);
241 
242 	config = tofree = strdup(opt);
243 	while ((name = strsep(&config, ",")) != NULL) {
244 		value = strchr(name, '=');
245 		if (value != NULL) {
246 			*value = '\0';
247 			value++;
248 			set_config_value_node(nvl, name, value);
249 		} else
250 			set_config_bool_node(nvl, name, true);
251 	}
252 	free(tofree);
253 	return (0);
254 }
255 
256 /*
257  * PCI device configuration is stored in MIBs that encode the device's
258  * location:
259  *
260  * pci.<bus>.<slot>.<func>
261  *
262  * Where "bus", "slot", and "func" are all decimal values without
263  * leading zeroes.  Each valid device must have a "device" node which
264  * identifies the driver model of the device.
265  *
266  * Device backends can provide a parser for the "config" string.  If
267  * a custom parser is not provided, pci_parse_legacy_config() is used
268  * to parse the string.
269  */
270 int
pci_parse_slot(char * opt)271 pci_parse_slot(char *opt)
272 {
273 	char node_name[sizeof("pci.XXX.XX.X")];
274 	struct pci_devemu *pde;
275 	char *emul, *config, *str, *cp;
276 	int error, bnum, snum, fnum;
277 	nvlist_t *nvl;
278 
279 	error = -1;
280 	str = strdup(opt);
281 
282 	emul = config = NULL;
283 	if ((cp = strchr(str, ',')) != NULL) {
284 		*cp = '\0';
285 		emul = cp + 1;
286 		if ((cp = strchr(emul, ',')) != NULL) {
287 			*cp = '\0';
288 			config = cp + 1;
289 		}
290 	} else {
291 		pci_parse_slot_usage(opt);
292 		goto done;
293 	}
294 
295 	/* <bus>:<slot>:<func> */
296 	if (sscanf(str, "%d:%d:%d", &bnum, &snum, &fnum) != 3) {
297 		bnum = 0;
298 		/* <slot>:<func> */
299 		if (sscanf(str, "%d:%d", &snum, &fnum) != 2) {
300 			fnum = 0;
301 			/* <slot> */
302 			if (sscanf(str, "%d", &snum) != 1) {
303 				snum = -1;
304 			}
305 		}
306 	}
307 
308 	if (bnum < 0 || bnum >= MAXBUSES || snum < 0 || snum >= MAXSLOTS ||
309 	    fnum < 0 || fnum >= MAXFUNCS) {
310 		pci_parse_slot_usage(opt);
311 		goto done;
312 	}
313 
314 	pde = pci_emul_finddev(emul);
315 	if (pde == NULL) {
316 		EPRINTLN("pci slot %d:%d:%d: unknown device \"%s\"", bnum, snum,
317 		    fnum, emul);
318 		goto done;
319 	}
320 
321 	snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bnum, snum,
322 	    fnum);
323 	nvl = find_config_node(node_name);
324 	if (nvl != NULL) {
325 		EPRINTLN("pci slot %d:%d:%d already occupied!", bnum, snum,
326 		    fnum);
327 		goto done;
328 	}
329 	nvl = create_config_node(node_name);
330 	if (pde->pe_alias != NULL)
331 		set_config_value_node(nvl, "device", pde->pe_alias);
332 	else
333 		set_config_value_node(nvl, "device", pde->pe_emu);
334 
335 	if (pde->pe_legacy_config != NULL)
336 		error = pde->pe_legacy_config(nvl, config);
337 	else
338 		error = pci_parse_legacy_config(nvl, config);
339 done:
340 	free(str);
341 	return (error);
342 }
343 
344 void
pci_print_supported_devices(void)345 pci_print_supported_devices(void)
346 {
347 	struct pci_devemu **pdpp, *pdp;
348 
349 	SET_FOREACH(pdpp, pci_devemu_set) {
350 		pdp = *pdpp;
351 		printf("%s\n", pdp->pe_emu);
352 	}
353 }
354 
355 uint32_t
pci_config_read_reg(const struct pcisel * const host_sel,nvlist_t * nvl,const uint32_t reg,const uint8_t size,const uint32_t def)356 pci_config_read_reg(const struct pcisel *const host_sel, nvlist_t *nvl,
357     const uint32_t reg, const uint8_t size, const uint32_t def)
358 {
359 	const char *config;
360 	const nvlist_t *pci_regs;
361 
362 	assert(size == 1 || size == 2 || size == 4);
363 
364 	pci_regs = find_relative_config_node(nvl, "pcireg");
365 	if (pci_regs == NULL) {
366 		return def;
367 	}
368 
369 	switch (reg) {
370 	case PCIR_DEVICE:
371 		config = get_config_value_node(pci_regs, "device");
372 		break;
373 	case PCIR_VENDOR:
374 		config = get_config_value_node(pci_regs, "vendor");
375 		break;
376 	case PCIR_REVID:
377 		config = get_config_value_node(pci_regs, "revid");
378 		break;
379 	case PCIR_SUBVEND_0:
380 		config = get_config_value_node(pci_regs, "subvendor");
381 		break;
382 	case PCIR_SUBDEV_0:
383 		config = get_config_value_node(pci_regs, "subdevice");
384 		break;
385 	default:
386 		return (-1);
387 	}
388 
389 	if (config == NULL) {
390 		return def;
391 	} else if (host_sel != NULL && strcmp(config, "host") == 0) {
392 #ifdef __amd64__
393 		return pci_host_read_config(host_sel, reg, size);
394 #else
395 		errx(1, "cannot fetch host PCI configuration");
396 #endif
397 	} else {
398 		return strtol(config, NULL, 16);
399 	}
400 }
401 
402 static int
pci_valid_pba_offset(struct pci_devinst * pi,uint64_t offset)403 pci_valid_pba_offset(struct pci_devinst *pi, uint64_t offset)
404 {
405 
406 	if (offset < pi->pi_msix.pba_offset)
407 		return (0);
408 
409 	if (offset >= pi->pi_msix.pba_offset + pi->pi_msix.pba_size) {
410 		return (0);
411 	}
412 
413 	return (1);
414 }
415 
416 int
pci_emul_msix_twrite(struct pci_devinst * pi,uint64_t offset,int size,uint64_t value)417 pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size,
418 		     uint64_t value)
419 {
420 	int msix_entry_offset;
421 	int tab_index;
422 	char *dest;
423 
424 	/* support only 4 or 8 byte writes */
425 	if (size != 4 && size != 8)
426 		return (-1);
427 
428 	/*
429 	 * Return if table index is beyond what device supports
430 	 */
431 	tab_index = offset / MSIX_TABLE_ENTRY_SIZE;
432 	if (tab_index >= pi->pi_msix.table_count)
433 		return (-1);
434 
435 	msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
436 
437 	/* support only aligned writes */
438 	if ((msix_entry_offset % size) != 0)
439 		return (-1);
440 
441 	dest = (char *)(pi->pi_msix.table + tab_index);
442 	dest += msix_entry_offset;
443 
444 	if (size == 4)
445 		*((uint32_t *)dest) = value;
446 	else
447 		*((uint64_t *)dest) = value;
448 
449 	return (0);
450 }
451 
452 uint64_t
pci_emul_msix_tread(struct pci_devinst * pi,uint64_t offset,int size)453 pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size)
454 {
455 	char *dest;
456 	int msix_entry_offset;
457 	int tab_index;
458 	uint64_t retval = ~0;
459 
460 	/*
461 	 * The PCI standard only allows 4 and 8 byte accesses to the MSI-X
462 	 * table but we also allow 1 byte access to accommodate reads from
463 	 * ddb.
464 	 */
465 	if (size != 1 && size != 4 && size != 8)
466 		return (retval);
467 
468 	msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE;
469 
470 	/* support only aligned reads */
471 	if ((msix_entry_offset % size) != 0) {
472 		return (retval);
473 	}
474 
475 	tab_index = offset / MSIX_TABLE_ENTRY_SIZE;
476 
477 	if (tab_index < pi->pi_msix.table_count) {
478 		/* valid MSI-X Table access */
479 		dest = (char *)(pi->pi_msix.table + tab_index);
480 		dest += msix_entry_offset;
481 
482 		if (size == 1)
483 			retval = *((uint8_t *)dest);
484 		else if (size == 4)
485 			retval = *((uint32_t *)dest);
486 		else
487 			retval = *((uint64_t *)dest);
488 	} else if (pci_valid_pba_offset(pi, offset)) {
489 		/* return 0 for PBA access */
490 		retval = 0;
491 	}
492 
493 	return (retval);
494 }
495 
496 int
pci_msix_table_bar(struct pci_devinst * pi)497 pci_msix_table_bar(struct pci_devinst *pi)
498 {
499 
500 	if (pi->pi_msix.table != NULL)
501 		return (pi->pi_msix.table_bar);
502 	else
503 		return (-1);
504 }
505 
506 int
pci_msix_pba_bar(struct pci_devinst * pi)507 pci_msix_pba_bar(struct pci_devinst *pi)
508 {
509 
510 	if (pi->pi_msix.table != NULL)
511 		return (pi->pi_msix.pba_bar);
512 	else
513 		return (-1);
514 }
515 
516 #ifdef __amd64__
517 static int
pci_emul_io_handler(struct vmctx * ctx __unused,int in,int port,int bytes,uint32_t * eax,void * arg)518 pci_emul_io_handler(struct vmctx *ctx __unused, int in, int port,
519     int bytes, uint32_t *eax, void *arg)
520 {
521 	struct pci_devinst *pdi = arg;
522 	struct pci_devemu *pe = pdi->pi_d;
523 	uint64_t offset;
524 	int i;
525 
526 	assert(port >= 0);
527 
528 	for (i = 0; i <= PCI_BARMAX; i++) {
529 		if (pdi->pi_bar[i].type == PCIBAR_IO &&
530 		    (uint64_t)port >= pdi->pi_bar[i].addr &&
531 		    (uint64_t)port + bytes <=
532 		    pdi->pi_bar[i].addr + pdi->pi_bar[i].size) {
533 			offset = port - pdi->pi_bar[i].addr;
534 			if (in)
535 				*eax = (*pe->pe_barread)(pdi, i,
536 							 offset, bytes);
537 			else
538 				(*pe->pe_barwrite)(pdi, i, offset,
539 						   bytes, *eax);
540 			return (0);
541 		}
542 	}
543 	return (-1);
544 }
545 #else
546 static int
pci_emul_iomem_handler(struct vcpu * vcpu __unused,int dir,uint64_t addr,int size,uint64_t * val,void * arg1,long arg2)547 pci_emul_iomem_handler(struct vcpu *vcpu __unused, int dir,
548     uint64_t addr, int size, uint64_t *val, void *arg1, long arg2)
549 {
550 	struct pci_devinst *pdi = arg1;
551 	struct pci_devemu *pe = pdi->pi_d;
552 	uint64_t offset;
553 	int bidx = (int)arg2;
554 
555 	assert(bidx <= PCI_BARMAX);
556 	assert(pdi->pi_bar[bidx].type == PCIBAR_IO);
557 	assert(addr >= pdi->pi_bar[bidx].addr &&
558 	       addr + size <= pdi->pi_bar[bidx].addr + pdi->pi_bar[bidx].size);
559 	assert(size == 1 || size == 2 || size == 4);
560 
561 	offset = addr - pdi->pi_bar[bidx].addr;
562 	if (dir == MEM_F_READ)
563 		*val = (*pe->pe_barread)(pdi, bidx, offset, size);
564 	else
565 		(*pe->pe_barwrite)(pdi, bidx, offset, size, *val);
566 
567 	return (0);
568 }
569 #endif /* !__amd64__ */
570 
571 static int
pci_emul_mem_handler(struct vcpu * vcpu __unused,int dir,uint64_t addr,int size,uint64_t * val,void * arg1,long arg2)572 pci_emul_mem_handler(struct vcpu *vcpu __unused, int dir,
573     uint64_t addr, int size, uint64_t *val, void *arg1, long arg2)
574 {
575 	struct pci_devinst *pdi = arg1;
576 	struct pci_devemu *pe = pdi->pi_d;
577 	uint64_t offset;
578 	int bidx = (int)arg2;
579 
580 	assert(bidx <= PCI_BARMAX);
581 	assert(pdi->pi_bar[bidx].type == PCIBAR_MEM32 ||
582 	       pdi->pi_bar[bidx].type == PCIBAR_MEM64);
583 	assert(addr >= pdi->pi_bar[bidx].addr &&
584 	       addr + size <= pdi->pi_bar[bidx].addr + pdi->pi_bar[bidx].size);
585 
586 	offset = addr - pdi->pi_bar[bidx].addr;
587 
588 	if (dir == MEM_F_WRITE) {
589 		if (size == 8) {
590 			(*pe->pe_barwrite)(pdi, bidx, offset,
591 					   4, *val & 0xffffffff);
592 			(*pe->pe_barwrite)(pdi, bidx, offset + 4,
593 					   4, *val >> 32);
594 		} else {
595 			(*pe->pe_barwrite)(pdi, bidx, offset,
596 					   size, *val);
597 		}
598 	} else {
599 		if (size == 8) {
600 			*val = (*pe->pe_barread)(pdi, bidx,
601 						 offset, 4);
602 			*val |= (*pe->pe_barread)(pdi, bidx,
603 						  offset + 4, 4) << 32;
604 		} else {
605 			*val = (*pe->pe_barread)(pdi, bidx,
606 						 offset, size);
607 		}
608 	}
609 
610 	return (0);
611 }
612 
613 
614 static int
pci_emul_alloc_resource(uint64_t * baseptr,uint64_t limit,uint64_t size,uint64_t * addr)615 pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size,
616 			uint64_t *addr)
617 {
618 	uint64_t base;
619 
620 	assert((size & (size - 1)) == 0);	/* must be a power of 2 */
621 
622 	base = roundup2(*baseptr, size);
623 
624 	if (base + size <= limit) {
625 		*addr = base;
626 		*baseptr = base + size;
627 		return (0);
628 	} else
629 		return (-1);
630 }
631 
632 /*
633  * Register (or unregister) the MMIO or I/O region associated with the BAR
634  * register 'idx' of an emulated pci device.
635  */
636 static void
modify_bar_registration(struct pci_devinst * pi,int idx,int registration)637 modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
638 {
639 	struct pci_devemu *pe;
640 	int error;
641 	enum pcibar_type type;
642 
643 	pe = pi->pi_d;
644 	type = pi->pi_bar[idx].type;
645 	switch (type) {
646 	case PCIBAR_IO:
647 	{
648 #ifdef __amd64__
649 		struct inout_port iop;
650 
651 		bzero(&iop, sizeof(struct inout_port));
652 		iop.name = pi->pi_name;
653 		iop.port = pi->pi_bar[idx].addr;
654 		iop.size = pi->pi_bar[idx].size;
655 		if (registration) {
656 			iop.flags = IOPORT_F_INOUT;
657 			iop.handler = pci_emul_io_handler;
658 			iop.arg = pi;
659 			error = register_inout(&iop);
660 		} else
661 			error = unregister_inout(&iop);
662 #else
663 		struct mem_range mr;
664 
665 		bzero(&mr, sizeof(struct mem_range));
666 		mr.name = pi->pi_name;
667 		mr.base = pi->pi_bar[idx].addr;
668 		mr.size = pi->pi_bar[idx].size;
669 		if (registration) {
670 			mr.flags = MEM_F_RW;
671 			mr.handler = pci_emul_iomem_handler;
672 			mr.arg1 = pi;
673 			mr.arg2 = idx;
674 			error = register_mem(&mr);
675 		} else
676 			error = unregister_mem(&mr);
677 #endif
678 		break;
679 	}
680 	case PCIBAR_MEM32:
681 	case PCIBAR_MEM64:
682 	{
683 		struct mem_range mr;
684 
685 		bzero(&mr, sizeof(struct mem_range));
686 		mr.name = pi->pi_name;
687 		mr.base = pi->pi_bar[idx].addr;
688 		mr.size = pi->pi_bar[idx].size;
689 		if (registration) {
690 			mr.flags = MEM_F_RW;
691 			mr.handler = pci_emul_mem_handler;
692 			mr.arg1 = pi;
693 			mr.arg2 = idx;
694 			error = register_mem(&mr);
695 		} else
696 			error = unregister_mem(&mr);
697 		break;
698 	}
699 	case PCIBAR_ROM:
700 		error = 0;
701 		break;
702 	default:
703 		error = EINVAL;
704 		break;
705 	}
706 	assert(error == 0);
707 
708 	if (pe->pe_baraddr != NULL)
709 		(*pe->pe_baraddr)(pi, idx, registration, pi->pi_bar[idx].addr);
710 }
711 
712 static void
unregister_bar(struct pci_devinst * pi,int idx)713 unregister_bar(struct pci_devinst *pi, int idx)
714 {
715 
716 	modify_bar_registration(pi, idx, 0);
717 }
718 
719 static void
register_bar(struct pci_devinst * pi,int idx)720 register_bar(struct pci_devinst *pi, int idx)
721 {
722 
723 	modify_bar_registration(pi, idx, 1);
724 }
725 
726 /* Is the ROM enabled for the emulated pci device? */
727 static int
romen(struct pci_devinst * pi)728 romen(struct pci_devinst *pi)
729 {
730 	return (pi->pi_bar[PCI_ROM_IDX].lobits & PCIM_BIOS_ENABLE) ==
731 	    PCIM_BIOS_ENABLE;
732 }
733 
734 /* Are we decoding i/o port accesses for the emulated pci device? */
735 static int
porten(struct pci_devinst * pi)736 porten(struct pci_devinst *pi)
737 {
738 	uint16_t cmd;
739 
740 	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
741 
742 	return (cmd & PCIM_CMD_PORTEN);
743 }
744 
745 /* Are we decoding memory accesses for the emulated pci device? */
746 static int
memen(struct pci_devinst * pi)747 memen(struct pci_devinst *pi)
748 {
749 	uint16_t cmd;
750 
751 	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
752 
753 	return (cmd & PCIM_CMD_MEMEN);
754 }
755 
756 /*
757  * Update the MMIO or I/O address that is decoded by the BAR register.
758  *
759  * If the pci device has enabled the address space decoding then intercept
760  * the address range decoded by the BAR register.
761  */
762 static void
update_bar_address(struct pci_devinst * pi,uint64_t addr,int idx,int type)763 update_bar_address(struct pci_devinst *pi, uint64_t addr, int idx, int type)
764 {
765 	int decode;
766 
767 	if (pi->pi_bar[idx].type == PCIBAR_IO)
768 		decode = porten(pi);
769 	else
770 		decode = memen(pi);
771 
772 	if (decode)
773 		unregister_bar(pi, idx);
774 
775 	switch (type) {
776 	case PCIBAR_IO:
777 	case PCIBAR_MEM32:
778 		pi->pi_bar[idx].addr = addr;
779 		break;
780 	case PCIBAR_MEM64:
781 		pi->pi_bar[idx].addr &= ~0xffffffffUL;
782 		pi->pi_bar[idx].addr |= addr;
783 		break;
784 	case PCIBAR_MEMHI64:
785 		pi->pi_bar[idx].addr &= 0xffffffff;
786 		pi->pi_bar[idx].addr |= addr;
787 		break;
788 	default:
789 		assert(0);
790 	}
791 
792 	if (decode)
793 		register_bar(pi, idx);
794 }
795 
796 int
pci_emul_alloc_bar(struct pci_devinst * pdi,int idx,enum pcibar_type type,uint64_t size)797 pci_emul_alloc_bar(struct pci_devinst *pdi, int idx, enum pcibar_type type,
798     uint64_t size)
799 {
800 	assert((type == PCIBAR_ROM) || (idx >= 0 && idx <= PCI_BARMAX));
801 	assert((type != PCIBAR_ROM) || (idx == PCI_ROM_IDX));
802 
803 	if ((size & (size - 1)) != 0)
804 		size = 1UL << flsl(size);	/* round up to a power of 2 */
805 
806 	/* Enforce minimum BAR sizes required by the PCI standard */
807 	if (type == PCIBAR_IO) {
808 		if (size < 4)
809 			size = 4;
810 	} else if (type == PCIBAR_ROM) {
811 		if (size < ~PCIM_BIOS_ADDR_MASK + 1)
812 			size = ~PCIM_BIOS_ADDR_MASK + 1;
813 	} else {
814 		if (size < 16)
815 			size = 16;
816 	}
817 
818 	/*
819 	 * To reduce fragmentation of the MMIO space, we allocate the BARs by
820 	 * size. Therefore, don't allocate the BAR yet. We create a list of all
821 	 * BAR allocation which is sorted by BAR size. When all PCI devices are
822 	 * initialized, we will assign an address to the BARs.
823 	 */
824 
825 	/* create a new list entry */
826 	struct pci_bar_allocation *const new_bar = malloc(sizeof(*new_bar));
827 	memset(new_bar, 0, sizeof(*new_bar));
828 	new_bar->pdi = pdi;
829 	new_bar->idx = idx;
830 	new_bar->type = type;
831 	new_bar->size = size;
832 
833 	/*
834 	 * Search for a BAR which size is lower than the size of our newly
835 	 * allocated BAR.
836 	 */
837 	struct pci_bar_allocation *bar = NULL;
838 	TAILQ_FOREACH(bar, &pci_bars, chain) {
839 		if (bar->size < size) {
840 			break;
841 		}
842 	}
843 
844 	if (bar == NULL) {
845 		/*
846 		 * Either the list is empty or new BAR is the smallest BAR of
847 		 * the list. Append it to the end of our list.
848 		 */
849 		TAILQ_INSERT_TAIL(&pci_bars, new_bar, chain);
850 	} else {
851 		/*
852 		 * The found BAR is smaller than our new BAR. For that reason,
853 		 * insert our new BAR before the found BAR.
854 		 */
855 		TAILQ_INSERT_BEFORE(bar, new_bar, chain);
856 	}
857 
858 	/*
859 	 * Enable PCI BARs only if we don't have a boot ROM, i.e., bhyveload was
860 	 * used to load the initial guest image.  Otherwise, we rely on the boot
861 	 * ROM to handle this.
862 	 */
863 	if (!get_config_bool_default("pci.enable_bars", !bootrom_boot()))
864 		return (0);
865 
866 	/*
867 	 * pci_passthru devices synchronize their physical and virtual command
868 	 * register on init. For that reason, the virtual cmd reg should be
869 	 * updated as early as possible.
870 	 */
871 	uint16_t enbit = 0;
872 	switch (type) {
873 	case PCIBAR_IO:
874 		enbit = PCIM_CMD_PORTEN;
875 		break;
876 	case PCIBAR_MEM64:
877 	case PCIBAR_MEM32:
878 		enbit = PCIM_CMD_MEMEN;
879 		break;
880 	default:
881 		enbit = 0;
882 		break;
883 	}
884 
885 	const uint16_t cmd = pci_get_cfgdata16(pdi, PCIR_COMMAND);
886 	pci_set_cfgdata16(pdi, PCIR_COMMAND, cmd | enbit);
887 
888 	return (0);
889 }
890 
891 static int
pci_emul_assign_bar(struct pci_devinst * const pdi,const int idx,const enum pcibar_type type,const uint64_t size)892 pci_emul_assign_bar(struct pci_devinst *const pdi, const int idx,
893     const enum pcibar_type type, const uint64_t size)
894 {
895 	int error;
896 	uint64_t *baseptr, limit, addr, mask, lobits, bar;
897 
898 	switch (type) {
899 	case PCIBAR_NONE:
900 		baseptr = NULL;
901 		addr = mask = lobits = 0;
902 		break;
903 	case PCIBAR_IO:
904 		baseptr = &pci_emul_iobase;
905 		limit = PCI_EMUL_IOLIMIT;
906 		mask = PCIM_BAR_IO_BASE;
907 		lobits = PCIM_BAR_IO_SPACE;
908 		break;
909 	case PCIBAR_MEM64:
910 		/*
911 		 * XXX
912 		 * Some drivers do not work well if the 64-bit BAR is allocated
913 		 * above 4GB. Allow for this by allocating small requests under
914 		 * 4GB unless then allocation size is larger than some arbitrary
915 		 * number (128MB currently).
916 		 */
917 		if (size > 128 * 1024 * 1024) {
918 			baseptr = &pci_emul_membase64;
919 			limit = pci_emul_memlim64;
920 			mask = PCIM_BAR_MEM_BASE;
921 			lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64 |
922 				 PCIM_BAR_MEM_PREFETCH;
923 		} else {
924 			baseptr = &pci_emul_membase32;
925 			limit = PCI_EMUL_MEMLIMIT32;
926 			mask = PCIM_BAR_MEM_BASE;
927 			lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64;
928 		}
929 		break;
930 	case PCIBAR_MEM32:
931 		baseptr = &pci_emul_membase32;
932 		limit = PCI_EMUL_MEMLIMIT32;
933 		mask = PCIM_BAR_MEM_BASE;
934 		lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_32;
935 		break;
936 	case PCIBAR_ROM:
937 		/* do not claim memory for ROM. OVMF will do it for us. */
938 		baseptr = NULL;
939 		limit = 0;
940 		mask = PCIM_BIOS_ADDR_MASK;
941 		lobits = 0;
942 		break;
943 	default:
944 		printf("pci_emul_alloc_base: invalid bar type %d\n", type);
945 		assert(0);
946 	}
947 
948 	if (baseptr != NULL) {
949 		error = pci_emul_alloc_resource(baseptr, limit, size, &addr);
950 		if (error != 0)
951 			return (error);
952 	} else {
953 		addr = 0;
954 	}
955 
956 	pdi->pi_bar[idx].type = type;
957 	pdi->pi_bar[idx].addr = addr;
958 	pdi->pi_bar[idx].size = size;
959 	/*
960 	 * passthru devices are using same lobits as physical device they set
961 	 * this property
962 	 */
963 	if (pdi->pi_bar[idx].lobits != 0) {
964 		lobits = pdi->pi_bar[idx].lobits;
965 	} else {
966 		pdi->pi_bar[idx].lobits = lobits;
967 	}
968 
969 	/* Initialize the BAR register in config space */
970 	bar = (addr & mask) | lobits;
971 	pci_set_cfgdata32(pdi, PCIR_BAR(idx), bar);
972 
973 	if (type == PCIBAR_MEM64) {
974 		assert(idx + 1 <= PCI_BARMAX);
975 		pdi->pi_bar[idx + 1].type = PCIBAR_MEMHI64;
976 		pci_set_cfgdata32(pdi, PCIR_BAR(idx + 1), bar >> 32);
977 	}
978 
979 	switch (type) {
980 	case PCIBAR_IO:
981 		if (porten(pdi))
982 			register_bar(pdi, idx);
983 		break;
984 	case PCIBAR_MEM32:
985 	case PCIBAR_MEM64:
986 	case PCIBAR_MEMHI64:
987 		if (memen(pdi))
988 			register_bar(pdi, idx);
989 		break;
990 	default:
991 		break;
992 	}
993 
994 	return (0);
995 }
996 
997 int
pci_emul_alloc_rom(struct pci_devinst * const pdi,const uint64_t size,void ** const addr)998 pci_emul_alloc_rom(struct pci_devinst *const pdi, const uint64_t size,
999     void **const addr)
1000 {
1001 	/* allocate ROM space once on first call */
1002 	if (pci_emul_rombase == 0) {
1003 		pci_emul_rombase = vm_create_devmem(pdi->pi_vmctx, VM_PCIROM,
1004 		    "pcirom", PCI_EMUL_ROMSIZE);
1005 		if (pci_emul_rombase == MAP_FAILED) {
1006 			warnx("%s: failed to create rom segment", __func__);
1007 			return (-1);
1008 		}
1009 		pci_emul_romlim = pci_emul_rombase + PCI_EMUL_ROMSIZE;
1010 		pci_emul_romoffset = 0;
1011 	}
1012 
1013 	/* ROM size should be a power of 2 and greater than 2 KB */
1014 	const uint64_t rom_size = MAX(1UL << flsl(size),
1015 	    ~PCIM_BIOS_ADDR_MASK + 1);
1016 
1017 	/* check if ROM fits into ROM space */
1018 	if (pci_emul_romoffset + rom_size > PCI_EMUL_ROMSIZE) {
1019 		warnx("%s: no space left in rom segment:", __func__);
1020 		warnx("%16lu bytes left",
1021 		    PCI_EMUL_ROMSIZE - pci_emul_romoffset);
1022 		warnx("%16lu bytes required by %d/%d/%d", rom_size, pdi->pi_bus,
1023 		    pdi->pi_slot, pdi->pi_func);
1024 		return (-1);
1025 	}
1026 
1027 	/* allocate ROM BAR */
1028 	const int error = pci_emul_alloc_bar(pdi, PCI_ROM_IDX, PCIBAR_ROM,
1029 	    rom_size);
1030 	if (error)
1031 		return error;
1032 
1033 	/* return address */
1034 	*addr = pci_emul_rombase + pci_emul_romoffset;
1035 
1036 	/* save offset into ROM Space */
1037 	pdi->pi_romoffset = pci_emul_romoffset;
1038 
1039 	/* increase offset for next ROM */
1040 	pci_emul_romoffset += rom_size;
1041 
1042 	return (0);
1043 }
1044 
1045 int
pci_emul_add_boot_device(struct pci_devinst * pi,int bootindex)1046 pci_emul_add_boot_device(struct pci_devinst *pi, int bootindex)
1047 {
1048 	struct boot_device *new_device, *device;
1049 
1050 	/* don't permit a negative bootindex */
1051 	if (bootindex < 0) {
1052 		errx(4, "Invalid bootindex %d for %s", bootindex, pi->pi_name);
1053 	}
1054 
1055 	/* alloc new boot device */
1056 	new_device = calloc(1, sizeof(struct boot_device));
1057 	if (new_device == NULL) {
1058 		return (ENOMEM);
1059 	}
1060 	new_device->pdi = pi;
1061 	new_device->bootindex = bootindex;
1062 
1063 	/* search for boot device with higher boot index */
1064 	TAILQ_FOREACH(device, &boot_devices, boot_device_chain) {
1065 		if (device->bootindex == bootindex) {
1066 			errx(4,
1067 			    "Could not set bootindex %d for %s. Bootindex already occupied by %s",
1068 			    bootindex, pi->pi_name, device->pdi->pi_name);
1069 		} else if (device->bootindex > bootindex) {
1070 			break;
1071 		}
1072 	}
1073 
1074 	/* add boot device to queue */
1075 	if (device == NULL) {
1076 		TAILQ_INSERT_TAIL(&boot_devices, new_device, boot_device_chain);
1077 	} else {
1078 		TAILQ_INSERT_BEFORE(device, new_device, boot_device_chain);
1079 	}
1080 
1081 	return (0);
1082 }
1083 
1084 #define	CAP_START_OFFSET	0x40
1085 static int
pci_emul_add_capability(struct pci_devinst * pi,u_char * capdata,int caplen)1086 pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen)
1087 {
1088 	int i, capoff, reallen;
1089 	uint16_t sts;
1090 
1091 	assert(caplen > 0);
1092 
1093 	reallen = roundup2(caplen, 4);		/* dword aligned */
1094 
1095 	sts = pci_get_cfgdata16(pi, PCIR_STATUS);
1096 	if ((sts & PCIM_STATUS_CAPPRESENT) == 0)
1097 		capoff = CAP_START_OFFSET;
1098 	else
1099 		capoff = pi->pi_capend + 1;
1100 
1101 	/* Check if we have enough space */
1102 	if (capoff + reallen > PCI_REGMAX + 1)
1103 		return (-1);
1104 
1105 	/* Set the previous capability pointer */
1106 	if ((sts & PCIM_STATUS_CAPPRESENT) == 0) {
1107 		pci_set_cfgdata8(pi, PCIR_CAP_PTR, capoff);
1108 		pci_set_cfgdata16(pi, PCIR_STATUS, sts|PCIM_STATUS_CAPPRESENT);
1109 	} else
1110 		pci_set_cfgdata8(pi, pi->pi_prevcap + 1, capoff);
1111 
1112 	/* Copy the capability */
1113 	for (i = 0; i < caplen; i++)
1114 		pci_set_cfgdata8(pi, capoff + i, capdata[i]);
1115 
1116 	/* Set the next capability pointer */
1117 	pci_set_cfgdata8(pi, capoff + 1, 0);
1118 
1119 	pi->pi_prevcap = capoff;
1120 	pi->pi_capend = capoff + reallen - 1;
1121 	return (0);
1122 }
1123 
1124 static struct pci_devemu *
pci_emul_finddev(const char * name)1125 pci_emul_finddev(const char *name)
1126 {
1127 	struct pci_devemu **pdpp, *pdp;
1128 
1129 	SET_FOREACH(pdpp, pci_devemu_set) {
1130 		pdp = *pdpp;
1131 		if (!strcmp(pdp->pe_emu, name)) {
1132 			return (pdp);
1133 		}
1134 	}
1135 
1136 	return (NULL);
1137 }
1138 
1139 static int
pci_emul_init(struct vmctx * ctx,struct pci_devemu * pde,int bus,int slot,int func,struct funcinfo * fi)1140 pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int bus, int slot,
1141     int func, struct funcinfo *fi)
1142 {
1143 	struct pci_devinst *pdi;
1144 	int err;
1145 
1146 	pdi = calloc(1, sizeof(struct pci_devinst));
1147 
1148 	pdi->pi_vmctx = ctx;
1149 	pdi->pi_bus = bus;
1150 	pdi->pi_slot = slot;
1151 	pdi->pi_func = func;
1152 	pthread_mutex_init(&pdi->pi_lintr.lock, NULL);
1153 	pdi->pi_lintr.pin = 0;
1154 	pdi->pi_lintr.state = IDLE;
1155 	pci_irq_init_irq(&pdi->pi_lintr.irq);
1156 	pdi->pi_d = pde;
1157 	snprintf(pdi->pi_name, PI_NAMESZ, "%s@pci.%d.%d.%d", pde->pe_emu, bus,
1158 	    slot, func);
1159 
1160 	/* Disable legacy interrupts */
1161 	pci_set_cfgdata8(pdi, PCIR_INTLINE, 255);
1162 	pci_set_cfgdata8(pdi, PCIR_INTPIN, 0);
1163 
1164 	if (get_config_bool_default("pci.enable_bars", !bootrom_boot()))
1165 		pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN);
1166 
1167 	err = (*pde->pe_init)(pdi, fi->fi_config);
1168 	if (err == 0)
1169 		fi->fi_devi = pdi;
1170 	else
1171 		free(pdi);
1172 
1173 	return (err);
1174 }
1175 
1176 void
pci_populate_msicap(struct msicap * msicap,int msgnum,int nextptr)1177 pci_populate_msicap(struct msicap *msicap, int msgnum, int nextptr)
1178 {
1179 	int mmc;
1180 
1181 	/* Number of msi messages must be a power of 2 between 1 and 32 */
1182 	assert((msgnum & (msgnum - 1)) == 0 && msgnum >= 1 && msgnum <= 32);
1183 	mmc = ffs(msgnum) - 1;
1184 
1185 	bzero(msicap, sizeof(struct msicap));
1186 	msicap->capid = PCIY_MSI;
1187 	msicap->nextptr = nextptr;
1188 	msicap->msgctrl = PCIM_MSICTRL_64BIT | (mmc << 1);
1189 }
1190 
1191 int
pci_emul_add_msicap(struct pci_devinst * pi,int msgnum)1192 pci_emul_add_msicap(struct pci_devinst *pi, int msgnum)
1193 {
1194 	struct msicap msicap;
1195 
1196 	pci_populate_msicap(&msicap, msgnum, 0);
1197 
1198 	return (pci_emul_add_capability(pi, (u_char *)&msicap, sizeof(msicap)));
1199 }
1200 
1201 static void
pci_populate_msixcap(struct msixcap * msixcap,int msgnum,int barnum,uint32_t msix_tab_size)1202 pci_populate_msixcap(struct msixcap *msixcap, int msgnum, int barnum,
1203 		     uint32_t msix_tab_size)
1204 {
1205 
1206 	assert(msix_tab_size % 4096 == 0);
1207 
1208 	bzero(msixcap, sizeof(struct msixcap));
1209 	msixcap->capid = PCIY_MSIX;
1210 
1211 	/*
1212 	 * Message Control Register, all fields set to
1213 	 * zero except for the Table Size.
1214 	 * Note: Table size N is encoded as N-1
1215 	 */
1216 	msixcap->msgctrl = msgnum - 1;
1217 
1218 	/*
1219 	 * MSI-X BAR setup:
1220 	 * - MSI-X table start at offset 0
1221 	 * - PBA table starts at a 4K aligned offset after the MSI-X table
1222 	 */
1223 	msixcap->table_info = barnum & PCIM_MSIX_BIR_MASK;
1224 	msixcap->pba_info = msix_tab_size | (barnum & PCIM_MSIX_BIR_MASK);
1225 }
1226 
1227 static void
pci_msix_table_init(struct pci_devinst * pi,int table_entries)1228 pci_msix_table_init(struct pci_devinst *pi, int table_entries)
1229 {
1230 	int i, table_size;
1231 
1232 	assert(table_entries > 0);
1233 	assert(table_entries <= MAX_MSIX_TABLE_ENTRIES);
1234 
1235 	table_size = table_entries * MSIX_TABLE_ENTRY_SIZE;
1236 	pi->pi_msix.table = calloc(1, table_size);
1237 
1238 	/* set mask bit of vector control register */
1239 	for (i = 0; i < table_entries; i++)
1240 		pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK;
1241 }
1242 
1243 int
pci_emul_add_msixcap(struct pci_devinst * pi,int msgnum,int barnum)1244 pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum)
1245 {
1246 	uint32_t tab_size;
1247 	struct msixcap msixcap;
1248 
1249 	assert(msgnum >= 1 && msgnum <= MAX_MSIX_TABLE_ENTRIES);
1250 	assert(barnum >= 0 && barnum <= PCIR_MAX_BAR_0);
1251 
1252 	tab_size = msgnum * MSIX_TABLE_ENTRY_SIZE;
1253 
1254 	/* Align table size to nearest 4K */
1255 	tab_size = roundup2(tab_size, 4096);
1256 
1257 	pi->pi_msix.table_bar = barnum;
1258 	pi->pi_msix.pba_bar   = barnum;
1259 	pi->pi_msix.table_offset = 0;
1260 	pi->pi_msix.table_count = msgnum;
1261 	pi->pi_msix.pba_offset = tab_size;
1262 	pi->pi_msix.pba_size = PBA_SIZE(msgnum);
1263 
1264 	pci_msix_table_init(pi, msgnum);
1265 
1266 	pci_populate_msixcap(&msixcap, msgnum, barnum, tab_size);
1267 
1268 	/* allocate memory for MSI-X Table and PBA */
1269 	pci_emul_alloc_bar(pi, barnum, PCIBAR_MEM32,
1270 				tab_size + pi->pi_msix.pba_size);
1271 
1272 	return (pci_emul_add_capability(pi, (u_char *)&msixcap,
1273 					sizeof(msixcap)));
1274 }
1275 
1276 static void
msixcap_cfgwrite(struct pci_devinst * pi,int capoff,int offset,int bytes,uint32_t val)1277 msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
1278 		 int bytes, uint32_t val)
1279 {
1280 	uint16_t msgctrl, rwmask;
1281 	int off;
1282 
1283 	off = offset - capoff;
1284 	/* Message Control Register */
1285 	if (off == 2 && bytes == 2) {
1286 		rwmask = PCIM_MSIXCTRL_MSIX_ENABLE | PCIM_MSIXCTRL_FUNCTION_MASK;
1287 		msgctrl = pci_get_cfgdata16(pi, offset);
1288 		msgctrl &= ~rwmask;
1289 		msgctrl |= val & rwmask;
1290 		val = msgctrl;
1291 
1292 		pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE;
1293 		pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK;
1294 		pci_lintr_update(pi);
1295 	}
1296 
1297 	CFGWRITE(pi, offset, val, bytes);
1298 }
1299 
1300 static void
msicap_cfgwrite(struct pci_devinst * pi,int capoff,int offset,int bytes,uint32_t val)1301 msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
1302 		int bytes, uint32_t val)
1303 {
1304 	uint16_t msgctrl, rwmask, msgdata, mme;
1305 	uint32_t addrlo;
1306 
1307 	/*
1308 	 * If guest is writing to the message control register make sure
1309 	 * we do not overwrite read-only fields.
1310 	 */
1311 	if ((offset - capoff) == 2 && bytes == 2) {
1312 		rwmask = PCIM_MSICTRL_MME_MASK | PCIM_MSICTRL_MSI_ENABLE;
1313 		msgctrl = pci_get_cfgdata16(pi, offset);
1314 		msgctrl &= ~rwmask;
1315 		msgctrl |= val & rwmask;
1316 		val = msgctrl;
1317 	}
1318 	CFGWRITE(pi, offset, val, bytes);
1319 
1320 	msgctrl = pci_get_cfgdata16(pi, capoff + 2);
1321 	addrlo = pci_get_cfgdata32(pi, capoff + 4);
1322 	if (msgctrl & PCIM_MSICTRL_64BIT)
1323 		msgdata = pci_get_cfgdata16(pi, capoff + 12);
1324 	else
1325 		msgdata = pci_get_cfgdata16(pi, capoff + 8);
1326 
1327 	mme = msgctrl & PCIM_MSICTRL_MME_MASK;
1328 	pi->pi_msi.enabled = msgctrl & PCIM_MSICTRL_MSI_ENABLE ? 1 : 0;
1329 	if (pi->pi_msi.enabled) {
1330 		pi->pi_msi.addr = addrlo;
1331 		pi->pi_msi.msg_data = msgdata;
1332 		pi->pi_msi.maxmsgnum = 1 << (mme >> 4);
1333 	} else {
1334 		pi->pi_msi.maxmsgnum = 0;
1335 	}
1336 	pci_lintr_update(pi);
1337 }
1338 
1339 static void
pciecap_cfgwrite(struct pci_devinst * pi,int capoff __unused,int offset,int bytes,uint32_t val)1340 pciecap_cfgwrite(struct pci_devinst *pi, int capoff __unused, int offset,
1341     int bytes, uint32_t val)
1342 {
1343 
1344 	/* XXX don't write to the readonly parts */
1345 	CFGWRITE(pi, offset, val, bytes);
1346 }
1347 
1348 #define	PCIECAP_VERSION	0x2
1349 int
pci_emul_add_pciecap(struct pci_devinst * pi,int type)1350 pci_emul_add_pciecap(struct pci_devinst *pi, int type)
1351 {
1352 	int err;
1353 	struct pciecap pciecap;
1354 
1355 	bzero(&pciecap, sizeof(pciecap));
1356 
1357 	/*
1358 	 * Use the integrated endpoint type for endpoints on a root complex bus.
1359 	 *
1360 	 * NB: bhyve currently only supports a single PCI bus that is the root
1361 	 * complex bus, so all endpoints are integrated.
1362 	 */
1363 	if ((type == PCIEM_TYPE_ENDPOINT) && (pi->pi_bus == 0))
1364 		type = PCIEM_TYPE_ROOT_INT_EP;
1365 
1366 	pciecap.capid = PCIY_EXPRESS;
1367 	pciecap.pcie_capabilities = PCIECAP_VERSION | type;
1368 	if (type != PCIEM_TYPE_ROOT_INT_EP) {
1369 		pciecap.link_capabilities = 0x411;	/* gen1, x1 */
1370 		pciecap.link_status = 0x11;		/* gen1, x1 */
1371 	}
1372 
1373 	err = pci_emul_add_capability(pi, (u_char *)&pciecap, sizeof(pciecap));
1374 	return (err);
1375 }
1376 
1377 /*
1378  * This function assumes that 'coff' is in the capabilities region of the
1379  * config space. A capoff parameter of zero will force a search for the
1380  * offset and type.
1381  */
1382 void
pci_emul_capwrite(struct pci_devinst * pi,int offset,int bytes,uint32_t val,uint8_t capoff,int capid)1383 pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val,
1384     uint8_t capoff, int capid)
1385 {
1386 	uint8_t nextoff;
1387 
1388 	/* Do not allow un-aligned writes */
1389 	if ((offset & (bytes - 1)) != 0)
1390 		return;
1391 
1392 	if (capoff == 0) {
1393 		/* Find the capability that we want to update */
1394 		capoff = CAP_START_OFFSET;
1395 		while (1) {
1396 			nextoff = pci_get_cfgdata8(pi, capoff + 1);
1397 			if (nextoff == 0)
1398 				break;
1399 			if (offset >= capoff && offset < nextoff)
1400 				break;
1401 
1402 			capoff = nextoff;
1403 		}
1404 		assert(offset >= capoff);
1405 		capid = pci_get_cfgdata8(pi, capoff);
1406 	}
1407 
1408 	/*
1409 	 * Capability ID and Next Capability Pointer are readonly.
1410 	 * However, some o/s's do 4-byte writes that include these.
1411 	 * For this case, trim the write back to 2 bytes and adjust
1412 	 * the data.
1413 	 */
1414 	if (offset == capoff || offset == capoff + 1) {
1415 		if (offset == capoff && bytes == 4) {
1416 			bytes = 2;
1417 			offset += 2;
1418 			val >>= 16;
1419 		} else
1420 			return;
1421 	}
1422 
1423 	switch (capid) {
1424 	case PCIY_MSI:
1425 		msicap_cfgwrite(pi, capoff, offset, bytes, val);
1426 		break;
1427 	case PCIY_MSIX:
1428 		msixcap_cfgwrite(pi, capoff, offset, bytes, val);
1429 		break;
1430 	case PCIY_EXPRESS:
1431 		pciecap_cfgwrite(pi, capoff, offset, bytes, val);
1432 		break;
1433 	default:
1434 		break;
1435 	}
1436 }
1437 
1438 static int
pci_emul_iscap(struct pci_devinst * pi,int offset)1439 pci_emul_iscap(struct pci_devinst *pi, int offset)
1440 {
1441 	uint16_t sts;
1442 
1443 	sts = pci_get_cfgdata16(pi, PCIR_STATUS);
1444 	if ((sts & PCIM_STATUS_CAPPRESENT) != 0) {
1445 		if (offset >= CAP_START_OFFSET && offset <= pi->pi_capend)
1446 			return (1);
1447 	}
1448 	return (0);
1449 }
1450 
1451 static int
pci_emul_fallback_handler(struct vcpu * vcpu __unused,int dir,uint64_t addr __unused,int size __unused,uint64_t * val,void * arg1 __unused,long arg2 __unused)1452 pci_emul_fallback_handler(struct vcpu *vcpu __unused, int dir,
1453     uint64_t addr __unused, int size __unused, uint64_t *val,
1454     void *arg1 __unused, long arg2 __unused)
1455 {
1456 	/*
1457 	 * Ignore writes; return 0xff's for reads. The mem read code
1458 	 * will take care of truncating to the correct size.
1459 	 */
1460 	if (dir == MEM_F_READ) {
1461 		*val = 0xffffffffffffffff;
1462 	}
1463 
1464 	return (0);
1465 }
1466 
1467 static int
pci_emul_ecfg_handler(struct vcpu * vcpu __unused,int dir,uint64_t addr,int bytes,uint64_t * val,void * arg1 __unused,long arg2 __unused)1468 pci_emul_ecfg_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr,
1469     int bytes, uint64_t *val, void *arg1 __unused, long arg2 __unused)
1470 {
1471 	int bus, slot, func, coff, in;
1472 
1473 	coff = addr & 0xfff;
1474 	func = (addr >> 12) & 0x7;
1475 	slot = (addr >> 15) & 0x1f;
1476 	bus = (addr >> 20) & 0xff;
1477 	in = (dir == MEM_F_READ);
1478 	if (in)
1479 		*val = ~0UL;
1480 	pci_cfgrw(in, bus, slot, func, coff, bytes, (uint32_t *)val);
1481 	return (0);
1482 }
1483 
1484 uint64_t
pci_ecfg_base(void)1485 pci_ecfg_base(void)
1486 {
1487 
1488 	return (PCI_EMUL_ECFG_BASE);
1489 }
1490 
1491 static int
init_bootorder(void)1492 init_bootorder(void)
1493 {
1494 	struct boot_device *device;
1495 	FILE *fp;
1496 	char *bootorder;
1497 	size_t bootorder_len;
1498 
1499 	if (TAILQ_EMPTY(&boot_devices))
1500 		return (0);
1501 
1502 	fp = open_memstream(&bootorder, &bootorder_len);
1503 	TAILQ_FOREACH(device, &boot_devices, boot_device_chain) {
1504 		fprintf(fp, "/pci@i0cf8/pci@%d,%d\n",
1505 		    device->pdi->pi_slot, device->pdi->pi_func);
1506 	}
1507 	fclose(fp);
1508 
1509 	return (qemu_fwcfg_add_file("bootorder", bootorder_len, bootorder));
1510 }
1511 
1512 #define	BUSIO_ROUNDUP		32
1513 #define	BUSMEM32_ROUNDUP	(1024 * 1024)
1514 #define	BUSMEM64_ROUNDUP	(512 * 1024 * 1024)
1515 
1516 int
init_pci(struct vmctx * ctx)1517 init_pci(struct vmctx *ctx)
1518 {
1519 	char node_name[sizeof("pci.XXX.XX.X")];
1520 	struct mem_range mr;
1521 	struct pci_devemu *pde;
1522 	struct businfo *bi;
1523 	struct slotinfo *si;
1524 	struct funcinfo *fi;
1525 	nvlist_t *nvl;
1526 	const char *emul;
1527 	size_t lowmem;
1528 	int bus, slot, func;
1529 	int error;
1530 
1531 	if (vm_get_lowmem_limit(ctx) > PCI_EMUL_MEMBASE32)
1532 		errx(EX_OSERR, "Invalid lowmem limit");
1533 
1534 	pci_emul_iobase = PCI_EMUL_IOBASE;
1535 	pci_emul_membase32 = PCI_EMUL_MEMBASE32;
1536 
1537 	pci_emul_membase64 = vm_get_highmem_base(ctx) +
1538 	    vm_get_highmem_size(ctx);
1539 	pci_emul_membase64 = roundup2(pci_emul_membase64, PCI_EMUL_MEMSIZE64);
1540 	pci_emul_memlim64 = pci_emul_membase64 + PCI_EMUL_MEMSIZE64;
1541 
1542 	TAILQ_INIT(&boot_devices);
1543 
1544 	for (bus = 0; bus < MAXBUSES; bus++) {
1545 		snprintf(node_name, sizeof(node_name), "pci.%d", bus);
1546 		nvl = find_config_node(node_name);
1547 		if (nvl == NULL)
1548 			continue;
1549 		pci_businfo[bus] = calloc(1, sizeof(struct businfo));
1550 		bi = pci_businfo[bus];
1551 
1552 		/*
1553 		 * Keep track of the i/o and memory resources allocated to
1554 		 * this bus.
1555 		 */
1556 		bi->iobase = pci_emul_iobase;
1557 		bi->membase32 = pci_emul_membase32;
1558 		bi->membase64 = pci_emul_membase64;
1559 
1560 		/* first run: init devices */
1561 		for (slot = 0; slot < MAXSLOTS; slot++) {
1562 			si = &bi->slotinfo[slot];
1563 			for (func = 0; func < MAXFUNCS; func++) {
1564 				fi = &si->si_funcs[func];
1565 				snprintf(node_name, sizeof(node_name),
1566 				    "pci.%d.%d.%d", bus, slot, func);
1567 				nvl = find_config_node(node_name);
1568 				if (nvl == NULL)
1569 					continue;
1570 
1571 				fi->fi_config = nvl;
1572 				emul = get_config_value_node(nvl, "device");
1573 				if (emul == NULL) {
1574 					EPRINTLN("pci slot %d:%d:%d: missing "
1575 					    "\"device\" value", bus, slot, func);
1576 					return (EINVAL);
1577 				}
1578 				pde = pci_emul_finddev(emul);
1579 				if (pde == NULL) {
1580 					EPRINTLN("pci slot %d:%d:%d: unknown "
1581 					    "device \"%s\"", bus, slot, func,
1582 					    emul);
1583 					return (EINVAL);
1584 				}
1585 				if (pde->pe_alias != NULL) {
1586 					EPRINTLN("pci slot %d:%d:%d: legacy "
1587 					    "device \"%s\", use \"%s\" instead",
1588 					    bus, slot, func, emul,
1589 					    pde->pe_alias);
1590 					return (EINVAL);
1591 				}
1592 				fi->fi_pde = pde;
1593 				error = pci_emul_init(ctx, pde, bus, slot,
1594 				    func, fi);
1595 				if (error)
1596 					return (error);
1597 			}
1598 		}
1599 
1600 		/* second run: assign BARs and free list */
1601 		struct pci_bar_allocation *bar;
1602 		struct pci_bar_allocation *bar_tmp;
1603 		TAILQ_FOREACH_SAFE(bar, &pci_bars, chain, bar_tmp) {
1604 			pci_emul_assign_bar(bar->pdi, bar->idx, bar->type,
1605 			    bar->size);
1606 			free(bar);
1607 		}
1608 		TAILQ_INIT(&pci_bars);
1609 
1610 		/*
1611 		 * Add some slop to the I/O and memory resources decoded by
1612 		 * this bus to give a guest some flexibility if it wants to
1613 		 * reprogram the BARs.
1614 		 */
1615 		pci_emul_iobase += BUSIO_ROUNDUP;
1616 		pci_emul_iobase = roundup2(pci_emul_iobase, BUSIO_ROUNDUP);
1617 		bi->iolimit = pci_emul_iobase;
1618 
1619 		pci_emul_membase32 += BUSMEM32_ROUNDUP;
1620 		pci_emul_membase32 = roundup2(pci_emul_membase32,
1621 		    BUSMEM32_ROUNDUP);
1622 		bi->memlimit32 = pci_emul_membase32;
1623 
1624 		pci_emul_membase64 += BUSMEM64_ROUNDUP;
1625 		pci_emul_membase64 = roundup2(pci_emul_membase64,
1626 		    BUSMEM64_ROUNDUP);
1627 		bi->memlimit64 = pci_emul_membase64;
1628 	}
1629 
1630 	/*
1631 	 * PCI backends are initialized before routing INTx interrupts
1632 	 * so that LPC devices are able to reserve ISA IRQs before
1633 	 * routing PIRQ pins.
1634 	 */
1635 	for (bus = 0; bus < MAXBUSES; bus++) {
1636 		if ((bi = pci_businfo[bus]) == NULL)
1637 			continue;
1638 
1639 		for (slot = 0; slot < MAXSLOTS; slot++) {
1640 			si = &bi->slotinfo[slot];
1641 			for (func = 0; func < MAXFUNCS; func++) {
1642 				fi = &si->si_funcs[func];
1643 				if (fi->fi_devi == NULL)
1644 					continue;
1645 				pci_lintr_route(fi->fi_devi);
1646 			}
1647 		}
1648 	}
1649 #ifdef __amd64__
1650 	lpc_pirq_routed();
1651 #endif
1652 
1653 	if ((error = init_bootorder()) != 0) {
1654 		warnx("%s: Unable to init bootorder", __func__);
1655 		return (error);
1656 	}
1657 
1658 	/*
1659 	 * The guest physical memory map looks like the following on amd64:
1660 	 * [0,		    lowmem)		guest system memory
1661 	 * [lowmem,	    0xC0000000)		memory hole (may be absent)
1662 	 * [0xC0000000,     0xE0000000)		PCI hole (32-bit BAR allocation)
1663 	 * [0xE0000000,	    0xF0000000)		PCI extended config window
1664 	 * [0xF0000000,	    4GB)		LAPIC, IOAPIC, HPET, firmware
1665 	 * [4GB,	    4GB + highmem)	guest system memory
1666 	 * [roundup(4GB + highmem, 32GB), ...)	PCI 64-bit BAR allocation
1667 	 *
1668 	 * On arm64 the guest physical memory map looks like this:
1669 	 * [0x0DF00000,	    0x10000000)		PCI I/O memory
1670 	 * [0xA0000000,	    0xE0000000)		PCI 32-bit BAR allocation
1671 	 * [0xE0000000,	    0xF0000000)		PCI extended config window
1672 	 * [4GB,	    4GB + highmem)	guest system memory
1673 	 * [roundup(4GB + highmem, 32GB), ...)	PCI 64-bit BAR allocation
1674 	 *
1675 	 * "lowmem" is guest memory below 0xC0000000.  amd64 guests provisioned
1676 	 * with less than 3GB of RAM will have no memory above the 4GB boundary.
1677 	 * System memory for arm64 guests is all above the 4GB boundary.
1678 	 */
1679 
1680 	/*
1681 	 * Accesses to memory addresses that are not allocated to system
1682 	 * memory or PCI devices return 0xff's.
1683 	 */
1684 	lowmem = vm_get_lowmem_size(ctx);
1685 	bzero(&mr, sizeof(struct mem_range));
1686 	mr.name = "PCI hole";
1687 	mr.flags = MEM_F_RW | MEM_F_IMMUTABLE;
1688 	mr.base = lowmem;
1689 	mr.size = (4ULL * 1024 * 1024 * 1024) - lowmem;
1690 	mr.handler = pci_emul_fallback_handler;
1691 	error = register_mem_fallback(&mr);
1692 	assert(error == 0);
1693 
1694 	/* PCI extended config space */
1695 	bzero(&mr, sizeof(struct mem_range));
1696 	mr.name = "PCI ECFG";
1697 	mr.flags = MEM_F_RW | MEM_F_IMMUTABLE;
1698 	mr.base = PCI_EMUL_ECFG_BASE;
1699 	mr.size = PCI_EMUL_ECFG_SIZE;
1700 	mr.handler = pci_emul_ecfg_handler;
1701 	error = register_mem(&mr);
1702 	assert(error == 0);
1703 
1704 	return (0);
1705 }
1706 
1707 #ifdef __amd64__
1708 static void
pci_apic_prt_entry(int bus __unused,int slot,int pin,struct pci_irq * irq,void * arg __unused)1709 pci_apic_prt_entry(int bus __unused, int slot, int pin, struct pci_irq *irq,
1710     void *arg __unused)
1711 {
1712 
1713 	dsdt_line("  Package ()");
1714 	dsdt_line("  {");
1715 	dsdt_line("    0x%X,", slot << 16 | 0xffff);
1716 	dsdt_line("    0x%02X,", pin - 1);
1717 	dsdt_line("    Zero,");
1718 	dsdt_line("    0x%X", irq->ioapic_irq);
1719 	dsdt_line("  },");
1720 }
1721 
1722 static void
pci_pirq_prt_entry(int bus __unused,int slot,int pin,struct pci_irq * irq,void * arg __unused)1723 pci_pirq_prt_entry(int bus __unused, int slot, int pin, struct pci_irq *irq,
1724     void *arg __unused)
1725 {
1726 	char *name;
1727 
1728 	name = lpc_pirq_name(irq->pirq_pin);
1729 	if (name == NULL)
1730 		return;
1731 	dsdt_line("  Package ()");
1732 	dsdt_line("  {");
1733 	dsdt_line("    0x%X,", slot << 16 | 0xffff);
1734 	dsdt_line("    0x%02X,", pin - 1);
1735 	dsdt_line("    %s,", name);
1736 	dsdt_line("    0x00");
1737 	dsdt_line("  },");
1738 	free(name);
1739 }
1740 #endif
1741 
1742 /*
1743  * A bhyve virtual machine has a flat PCI hierarchy with a root port
1744  * corresponding to each PCI bus.
1745  */
1746 static void
pci_bus_write_dsdt(int bus)1747 pci_bus_write_dsdt(int bus)
1748 {
1749 	struct businfo *bi;
1750 	struct slotinfo *si;
1751 	struct pci_devinst *pi;
1752 	int func, slot;
1753 
1754 	/*
1755 	 * If there are no devices on this 'bus' then just return.
1756 	 */
1757 	if ((bi = pci_businfo[bus]) == NULL) {
1758 		/*
1759 		 * Bus 0 is special because it decodes the I/O ports used
1760 		 * for PCI config space access even if there are no devices
1761 		 * on it.
1762 		 */
1763 		if (bus != 0)
1764 			return;
1765 	}
1766 
1767 	dsdt_line("  Device (PC%02X)", bus);
1768 	dsdt_line("  {");
1769 	dsdt_line("    Name (_HID, EisaId (\"PNP0A03\"))");
1770 
1771 	dsdt_line("    Method (_BBN, 0, NotSerialized)");
1772 	dsdt_line("    {");
1773 	dsdt_line("        Return (0x%08X)", bus);
1774 	dsdt_line("    }");
1775 	dsdt_line("    Name (_CRS, ResourceTemplate ()");
1776 	dsdt_line("    {");
1777 	dsdt_line("      WordBusNumber (ResourceProducer, MinFixed, "
1778 	    "MaxFixed, PosDecode,");
1779 	dsdt_line("        0x0000,             // Granularity");
1780 	dsdt_line("        0x%04X,             // Range Minimum", bus);
1781 	dsdt_line("        0x%04X,             // Range Maximum", bus);
1782 	dsdt_line("        0x0000,             // Translation Offset");
1783 	dsdt_line("        0x0001,             // Length");
1784 	dsdt_line("        ,, )");
1785 
1786 #ifdef __amd64__
1787 	if (bus == 0) {
1788 		dsdt_indent(3);
1789 		dsdt_fixed_ioport(0xCF8, 8);
1790 		dsdt_unindent(3);
1791 
1792 		dsdt_line("      WordIO (ResourceProducer, MinFixed, MaxFixed, "
1793 		    "PosDecode, EntireRange,");
1794 		dsdt_line("        0x0000,             // Granularity");
1795 		dsdt_line("        0x0000,             // Range Minimum");
1796 		dsdt_line("        0x0CF7,             // Range Maximum");
1797 		dsdt_line("        0x0000,             // Translation Offset");
1798 		dsdt_line("        0x0CF8,             // Length");
1799 		dsdt_line("        ,, , TypeStatic)");
1800 
1801 		dsdt_line("      WordIO (ResourceProducer, MinFixed, MaxFixed, "
1802 		    "PosDecode, EntireRange,");
1803 		dsdt_line("        0x0000,             // Granularity");
1804 		dsdt_line("        0x0D00,             // Range Minimum");
1805 		dsdt_line("        0x%04X,             // Range Maximum",
1806 		    PCI_EMUL_IOBASE - 1);
1807 		dsdt_line("        0x0000,             // Translation Offset");
1808 		dsdt_line("        0x%04X,             // Length",
1809 		    PCI_EMUL_IOBASE - 0x0D00);
1810 		dsdt_line("        ,, , TypeStatic)");
1811 
1812 		if (bi == NULL) {
1813 			dsdt_line("    })");
1814 			goto done;
1815 		}
1816 	}
1817 #endif
1818 	assert(bi != NULL);
1819 
1820 	/* i/o window */
1821 	dsdt_line("      WordIO (ResourceProducer, MinFixed, MaxFixed, "
1822 	    "PosDecode, EntireRange,");
1823 	dsdt_line("        0x0000,             // Granularity");
1824 	dsdt_line("        0x%04X,             // Range Minimum", bi->iobase);
1825 	dsdt_line("        0x%04X,             // Range Maximum",
1826 	    bi->iolimit - 1);
1827 	dsdt_line("        0x0000,             // Translation Offset");
1828 	dsdt_line("        0x%04X,             // Length",
1829 	    bi->iolimit - bi->iobase);
1830 	dsdt_line("        ,, , TypeStatic)");
1831 
1832 	/* mmio window (32-bit) */
1833 	dsdt_line("      DWordMemory (ResourceProducer, PosDecode, "
1834 	    "MinFixed, MaxFixed, NonCacheable, ReadWrite,");
1835 	dsdt_line("        0x00000000,         // Granularity");
1836 	dsdt_line("        0x%08X,         // Range Minimum\n", bi->membase32);
1837 	dsdt_line("        0x%08X,         // Range Maximum\n",
1838 	    bi->memlimit32 - 1);
1839 	dsdt_line("        0x00000000,         // Translation Offset");
1840 	dsdt_line("        0x%08X,         // Length\n",
1841 	    bi->memlimit32 - bi->membase32);
1842 	dsdt_line("        ,, , AddressRangeMemory, TypeStatic)");
1843 
1844 	/* mmio window (64-bit) */
1845 	dsdt_line("      QWordMemory (ResourceProducer, PosDecode, "
1846 	    "MinFixed, MaxFixed, NonCacheable, ReadWrite,");
1847 	dsdt_line("        0x0000000000000000, // Granularity");
1848 	dsdt_line("        0x%016lX, // Range Minimum\n", bi->membase64);
1849 	dsdt_line("        0x%016lX, // Range Maximum\n",
1850 	    bi->memlimit64 - 1);
1851 	dsdt_line("        0x0000000000000000, // Translation Offset");
1852 	dsdt_line("        0x%016lX, // Length\n",
1853 	    bi->memlimit64 - bi->membase64);
1854 	dsdt_line("        ,, , AddressRangeMemory, TypeStatic)");
1855 	dsdt_line("    })");
1856 
1857 #ifdef __amd64__
1858 	if (pci_count_lintr(bus) != 0) {
1859 		dsdt_indent(2);
1860 		dsdt_line("Name (PPRT, Package ()");
1861 		dsdt_line("{");
1862 		pci_walk_lintr(bus, pci_pirq_prt_entry, NULL);
1863 		dsdt_line("})");
1864 		dsdt_line("Name (APRT, Package ()");
1865 		dsdt_line("{");
1866 		pci_walk_lintr(bus, pci_apic_prt_entry, NULL);
1867 		dsdt_line("})");
1868 		dsdt_line("Method (_PRT, 0, NotSerialized)");
1869 		dsdt_line("{");
1870 		dsdt_line("  If (PICM)");
1871 		dsdt_line("  {");
1872 		dsdt_line("    Return (APRT)");
1873 		dsdt_line("  }");
1874 		dsdt_line("  Else");
1875 		dsdt_line("  {");
1876 		dsdt_line("    Return (PPRT)");
1877 		dsdt_line("  }");
1878 		dsdt_line("}");
1879 		dsdt_unindent(2);
1880 	}
1881 #endif
1882 
1883 	dsdt_indent(2);
1884 	for (slot = 0; slot < MAXSLOTS; slot++) {
1885 		si = &bi->slotinfo[slot];
1886 		for (func = 0; func < MAXFUNCS; func++) {
1887 			pi = si->si_funcs[func].fi_devi;
1888 			if (pi != NULL && pi->pi_d->pe_write_dsdt != NULL)
1889 				pi->pi_d->pe_write_dsdt(pi);
1890 		}
1891 	}
1892 	dsdt_unindent(2);
1893 #ifdef __amd64__
1894 done:
1895 #endif
1896 	dsdt_line("  }");
1897 }
1898 
1899 void
pci_write_dsdt(void)1900 pci_write_dsdt(void)
1901 {
1902 	int bus;
1903 
1904 	dsdt_indent(1);
1905 	dsdt_line("Name (PICM, 0x00)");
1906 	dsdt_line("Method (_PIC, 1, NotSerialized)");
1907 	dsdt_line("{");
1908 	dsdt_line("  Store (Arg0, PICM)");
1909 	dsdt_line("}");
1910 	dsdt_line("");
1911 	dsdt_line("Scope (_SB)");
1912 	dsdt_line("{");
1913 	for (bus = 0; bus < MAXBUSES; bus++)
1914 		pci_bus_write_dsdt(bus);
1915 	dsdt_line("}");
1916 	dsdt_unindent(1);
1917 }
1918 
1919 int
pci_bus_configured(int bus)1920 pci_bus_configured(int bus)
1921 {
1922 	assert(bus >= 0 && bus < MAXBUSES);
1923 	return (pci_businfo[bus] != NULL);
1924 }
1925 
1926 int
pci_msi_enabled(struct pci_devinst * pi)1927 pci_msi_enabled(struct pci_devinst *pi)
1928 {
1929 	return (pi->pi_msi.enabled);
1930 }
1931 
1932 int
pci_msi_maxmsgnum(struct pci_devinst * pi)1933 pci_msi_maxmsgnum(struct pci_devinst *pi)
1934 {
1935 	if (pi->pi_msi.enabled)
1936 		return (pi->pi_msi.maxmsgnum);
1937 	else
1938 		return (0);
1939 }
1940 
1941 int
pci_msix_enabled(struct pci_devinst * pi)1942 pci_msix_enabled(struct pci_devinst *pi)
1943 {
1944 
1945 	return (pi->pi_msix.enabled && !pi->pi_msi.enabled);
1946 }
1947 
1948 void
pci_generate_msix(struct pci_devinst * pi,int index)1949 pci_generate_msix(struct pci_devinst *pi, int index)
1950 {
1951 	struct msix_table_entry *mte;
1952 
1953 	if (!pci_msix_enabled(pi))
1954 		return;
1955 
1956 	if (pi->pi_msix.function_mask)
1957 		return;
1958 
1959 	if (index >= pi->pi_msix.table_count)
1960 		return;
1961 
1962 	mte = &pi->pi_msix.table[index];
1963 	if ((mte->vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
1964 		/* XXX Set PBA bit if interrupt is disabled */
1965 		vm_raise_msi(pi->pi_vmctx, mte->addr, mte->msg_data,
1966 		    pi->pi_bus, pi->pi_slot, pi->pi_func);
1967 	}
1968 }
1969 
1970 void
pci_generate_msi(struct pci_devinst * pi,int index)1971 pci_generate_msi(struct pci_devinst *pi, int index)
1972 {
1973 
1974 	if (pci_msi_enabled(pi) && index < pci_msi_maxmsgnum(pi)) {
1975 		vm_raise_msi(pi->pi_vmctx, pi->pi_msi.addr,
1976 		    pi->pi_msi.msg_data + index,
1977 		    pi->pi_bus, pi->pi_slot, pi->pi_func);
1978 	}
1979 }
1980 
1981 static bool
pci_lintr_permitted(struct pci_devinst * pi)1982 pci_lintr_permitted(struct pci_devinst *pi)
1983 {
1984 	uint16_t cmd;
1985 
1986 	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
1987 	return (!(pi->pi_msi.enabled || pi->pi_msix.enabled ||
1988 		(cmd & PCIM_CMD_INTxDIS)));
1989 }
1990 
1991 void
pci_lintr_request(struct pci_devinst * pi)1992 pci_lintr_request(struct pci_devinst *pi)
1993 {
1994 	struct businfo *bi;
1995 	struct slotinfo *si;
1996 	int bestpin, bestcount, pin;
1997 
1998 	bi = pci_businfo[pi->pi_bus];
1999 	assert(bi != NULL);
2000 
2001 	/*
2002 	 * Just allocate a pin from our slot.  The pin will be
2003 	 * assigned IRQs later when interrupts are routed.
2004 	 */
2005 	si = &bi->slotinfo[pi->pi_slot];
2006 	bestpin = 0;
2007 	bestcount = si->si_intpins[0].ii_count;
2008 	for (pin = 1; pin < 4; pin++) {
2009 		if (si->si_intpins[pin].ii_count < bestcount) {
2010 			bestpin = pin;
2011 			bestcount = si->si_intpins[pin].ii_count;
2012 		}
2013 	}
2014 
2015 	si->si_intpins[bestpin].ii_count++;
2016 	pi->pi_lintr.pin = bestpin + 1;
2017 	pci_set_cfgdata8(pi, PCIR_INTPIN, bestpin + 1);
2018 }
2019 
2020 static void
pci_lintr_route(struct pci_devinst * pi)2021 pci_lintr_route(struct pci_devinst *pi)
2022 {
2023 	struct businfo *bi;
2024 	struct intxinfo *ii;
2025 	struct pci_irq *irq;
2026 
2027 	if (pi->pi_lintr.pin == 0)
2028 		return;
2029 
2030 	bi = pci_businfo[pi->pi_bus];
2031 	assert(bi != NULL);
2032 	ii = &bi->slotinfo[pi->pi_slot].si_intpins[pi->pi_lintr.pin - 1];
2033 	irq = &ii->ii_irq;
2034 	pci_irq_route(pi, irq);
2035 	pi->pi_lintr.irq = *irq;
2036 	pci_set_cfgdata8(pi, PCIR_INTLINE, pci_irq_intline(irq));
2037 }
2038 
2039 void
pci_lintr_assert(struct pci_devinst * pi)2040 pci_lintr_assert(struct pci_devinst *pi)
2041 {
2042 
2043 	assert(pi->pi_lintr.pin > 0);
2044 
2045 	pthread_mutex_lock(&pi->pi_lintr.lock);
2046 	if (pi->pi_lintr.state == IDLE) {
2047 		if (pci_lintr_permitted(pi)) {
2048 			pi->pi_lintr.state = ASSERTED;
2049 			pci_irq_assert(pi);
2050 		} else
2051 			pi->pi_lintr.state = PENDING;
2052 	}
2053 	pthread_mutex_unlock(&pi->pi_lintr.lock);
2054 }
2055 
2056 void
pci_lintr_deassert(struct pci_devinst * pi)2057 pci_lintr_deassert(struct pci_devinst *pi)
2058 {
2059 
2060 	assert(pi->pi_lintr.pin > 0);
2061 
2062 	pthread_mutex_lock(&pi->pi_lintr.lock);
2063 	if (pi->pi_lintr.state == ASSERTED) {
2064 		pi->pi_lintr.state = IDLE;
2065 		pci_irq_deassert(pi);
2066 	} else if (pi->pi_lintr.state == PENDING)
2067 		pi->pi_lintr.state = IDLE;
2068 	pthread_mutex_unlock(&pi->pi_lintr.lock);
2069 }
2070 
2071 static void
pci_lintr_update(struct pci_devinst * pi)2072 pci_lintr_update(struct pci_devinst *pi)
2073 {
2074 
2075 	pthread_mutex_lock(&pi->pi_lintr.lock);
2076 	if (pi->pi_lintr.state == ASSERTED && !pci_lintr_permitted(pi)) {
2077 		pci_irq_deassert(pi);
2078 		pi->pi_lintr.state = PENDING;
2079 	} else if (pi->pi_lintr.state == PENDING && pci_lintr_permitted(pi)) {
2080 		pi->pi_lintr.state = ASSERTED;
2081 		pci_irq_assert(pi);
2082 	}
2083 	pthread_mutex_unlock(&pi->pi_lintr.lock);
2084 }
2085 
2086 int
pci_count_lintr(int bus)2087 pci_count_lintr(int bus)
2088 {
2089 	int count, slot, pin;
2090 	struct slotinfo *slotinfo;
2091 
2092 	count = 0;
2093 	if (pci_businfo[bus] != NULL) {
2094 		for (slot = 0; slot < MAXSLOTS; slot++) {
2095 			slotinfo = &pci_businfo[bus]->slotinfo[slot];
2096 			for (pin = 0; pin < 4; pin++) {
2097 				if (slotinfo->si_intpins[pin].ii_count != 0)
2098 					count++;
2099 			}
2100 		}
2101 	}
2102 	return (count);
2103 }
2104 
2105 void
pci_walk_lintr(int bus,pci_lintr_cb cb,void * arg)2106 pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg)
2107 {
2108 	struct businfo *bi;
2109 	struct slotinfo *si;
2110 	struct intxinfo *ii;
2111 	int slot, pin;
2112 
2113 	if ((bi = pci_businfo[bus]) == NULL)
2114 		return;
2115 
2116 	for (slot = 0; slot < MAXSLOTS; slot++) {
2117 		si = &bi->slotinfo[slot];
2118 		for (pin = 0; pin < 4; pin++) {
2119 			ii = &si->si_intpins[pin];
2120 			if (ii->ii_count != 0)
2121 				cb(bus, slot, pin + 1, &ii->ii_irq, arg);
2122 		}
2123 	}
2124 }
2125 
2126 /*
2127  * Return 1 if the emulated device in 'slot' is a multi-function device.
2128  * Return 0 otherwise.
2129  */
2130 static int
pci_emul_is_mfdev(int bus,int slot)2131 pci_emul_is_mfdev(int bus, int slot)
2132 {
2133 	struct businfo *bi;
2134 	struct slotinfo *si;
2135 	int f, numfuncs;
2136 
2137 	numfuncs = 0;
2138 	if ((bi = pci_businfo[bus]) != NULL) {
2139 		si = &bi->slotinfo[slot];
2140 		for (f = 0; f < MAXFUNCS; f++) {
2141 			if (si->si_funcs[f].fi_devi != NULL) {
2142 				numfuncs++;
2143 			}
2144 		}
2145 	}
2146 	return (numfuncs > 1);
2147 }
2148 
2149 /*
2150  * Ensure that the PCIM_MFDEV bit is properly set (or unset) depending on
2151  * whether or not is a multi-function being emulated in the pci 'slot'.
2152  */
2153 static void
pci_emul_hdrtype_fixup(int bus,int slot,int off,int bytes,uint32_t * rv)2154 pci_emul_hdrtype_fixup(int bus, int slot, int off, int bytes, uint32_t *rv)
2155 {
2156 	int mfdev;
2157 
2158 	if (off <= PCIR_HDRTYPE && off + bytes > PCIR_HDRTYPE) {
2159 		mfdev = pci_emul_is_mfdev(bus, slot);
2160 		switch (bytes) {
2161 		case 1:
2162 		case 2:
2163 			*rv &= ~PCIM_MFDEV;
2164 			if (mfdev) {
2165 				*rv |= PCIM_MFDEV;
2166 			}
2167 			break;
2168 		case 4:
2169 			*rv &= ~(PCIM_MFDEV << 16);
2170 			if (mfdev) {
2171 				*rv |= (PCIM_MFDEV << 16);
2172 			}
2173 			break;
2174 		}
2175 	}
2176 }
2177 
2178 /*
2179  * Update device state in response to changes to the PCI command
2180  * register.
2181  */
2182 void
pci_emul_cmd_changed(struct pci_devinst * pi,uint16_t old)2183 pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old)
2184 {
2185 	int i;
2186 	uint16_t changed, new;
2187 
2188 	new = pci_get_cfgdata16(pi, PCIR_COMMAND);
2189 	changed = old ^ new;
2190 
2191 	/*
2192 	 * If the MMIO or I/O address space decoding has changed then
2193 	 * register/unregister all BARs that decode that address space.
2194 	 */
2195 	for (i = 0; i <= PCI_BARMAX_WITH_ROM; i++) {
2196 		switch (pi->pi_bar[i].type) {
2197 			case PCIBAR_NONE:
2198 			case PCIBAR_MEMHI64:
2199 				break;
2200 			case PCIBAR_IO:
2201 				/* I/O address space decoding changed? */
2202 				if (changed & PCIM_CMD_PORTEN) {
2203 					if (new & PCIM_CMD_PORTEN)
2204 						register_bar(pi, i);
2205 					else
2206 						unregister_bar(pi, i);
2207 				}
2208 				break;
2209 			case PCIBAR_ROM:
2210 				/* skip (un-)register of ROM if it disabled */
2211 				if (!romen(pi))
2212 					break;
2213 				/* fallthrough */
2214 			case PCIBAR_MEM32:
2215 			case PCIBAR_MEM64:
2216 				/* MMIO address space decoding changed? */
2217 				if (changed & PCIM_CMD_MEMEN) {
2218 					if (new & PCIM_CMD_MEMEN)
2219 						register_bar(pi, i);
2220 					else
2221 						unregister_bar(pi, i);
2222 				}
2223 				break;
2224 			default:
2225 				assert(0);
2226 		}
2227 	}
2228 
2229 	/*
2230 	 * If INTx has been unmasked and is pending, assert the
2231 	 * interrupt.
2232 	 */
2233 	pci_lintr_update(pi);
2234 }
2235 
2236 static void
pci_emul_cmdsts_write(struct pci_devinst * pi,int coff,uint32_t new,int bytes)2237 pci_emul_cmdsts_write(struct pci_devinst *pi, int coff, uint32_t new, int bytes)
2238 {
2239 	int rshift;
2240 	uint32_t cmd, old, readonly;
2241 
2242 	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);	/* stash old value */
2243 
2244 	/*
2245 	 * From PCI Local Bus Specification 3.0 sections 6.2.2 and 6.2.3.
2246 	 *
2247 	 * XXX Bits 8, 11, 12, 13, 14 and 15 in the status register are
2248 	 * 'write 1 to clear'. However these bits are not set to '1' by
2249 	 * any device emulation so it is simpler to treat them as readonly.
2250 	 */
2251 	rshift = (coff & 0x3) * 8;
2252 	readonly = 0xFFFFF880 >> rshift;
2253 
2254 	old = CFGREAD(pi, coff, bytes);
2255 	new &= ~readonly;
2256 	new |= (old & readonly);
2257 	CFGWRITE(pi, coff, new, bytes);			/* update config */
2258 
2259 	pci_emul_cmd_changed(pi, cmd);
2260 }
2261 
2262 static void
pci_cfgrw(int in,int bus,int slot,int func,int coff,int bytes,uint32_t * valp)2263 pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes,
2264     uint32_t *valp)
2265 {
2266 	struct businfo *bi;
2267 	struct slotinfo *si;
2268 	struct pci_devinst *pi;
2269 	struct pci_devemu *pe;
2270 	int idx, needcfg;
2271 	uint64_t addr, bar, mask;
2272 
2273 	if ((bi = pci_businfo[bus]) != NULL) {
2274 		si = &bi->slotinfo[slot];
2275 		pi = si->si_funcs[func].fi_devi;
2276 	} else
2277 		pi = NULL;
2278 
2279 	/*
2280 	 * Just return if there is no device at this slot:func or if the
2281 	 * guest is doing an un-aligned access.
2282 	 */
2283 	if (pi == NULL || (bytes != 1 && bytes != 2 && bytes != 4) ||
2284 	    (coff & (bytes - 1)) != 0) {
2285 		if (in)
2286 			*valp = 0xffffffff;
2287 		return;
2288 	}
2289 
2290 	/*
2291 	 * Ignore all writes beyond the standard config space and return all
2292 	 * ones on reads.
2293 	 */
2294 	if (coff >= PCI_REGMAX + 1) {
2295 		if (in) {
2296 			*valp = 0xffffffff;
2297 			/*
2298 			 * Extended capabilities begin at offset 256 in config
2299 			 * space. Absence of extended capabilities is signaled
2300 			 * with all 0s in the extended capability header at
2301 			 * offset 256.
2302 			 */
2303 			if (coff <= PCI_REGMAX + 4)
2304 				*valp = 0x00000000;
2305 		}
2306 		return;
2307 	}
2308 
2309 	pe = pi->pi_d;
2310 
2311 	/*
2312 	 * Config read
2313 	 */
2314 	if (in) {
2315 		/* Let the device emulation override the default handler */
2316 		if (pe->pe_cfgread != NULL) {
2317 			needcfg = pe->pe_cfgread(pi, coff, bytes, valp);
2318 		} else {
2319 			needcfg = 1;
2320 		}
2321 
2322 		if (needcfg)
2323 			*valp = CFGREAD(pi, coff, bytes);
2324 
2325 		pci_emul_hdrtype_fixup(bus, slot, coff, bytes, valp);
2326 	} else {
2327 		/* Let the device emulation override the default handler */
2328 		if (pe->pe_cfgwrite != NULL &&
2329 		    (*pe->pe_cfgwrite)(pi, coff, bytes, *valp) == 0)
2330 			return;
2331 
2332 		/*
2333 		 * Special handling for write to BAR and ROM registers
2334 		 */
2335 		if (is_pcir_bar(coff) || is_pcir_bios(coff)) {
2336 			/*
2337 			 * Ignore writes to BAR registers that are not
2338 			 * 4-byte aligned.
2339 			 */
2340 			if (bytes != 4 || (coff & 0x3) != 0)
2341 				return;
2342 
2343 			if (is_pcir_bar(coff)) {
2344 				idx = (coff - PCIR_BAR(0)) / 4;
2345 			} else if (is_pcir_bios(coff)) {
2346 				idx = PCI_ROM_IDX;
2347 			} else {
2348 				errx(4, "%s: invalid BAR offset %d", __func__,
2349 				    coff);
2350 			}
2351 
2352 			mask = ~(pi->pi_bar[idx].size - 1);
2353 			switch (pi->pi_bar[idx].type) {
2354 			case PCIBAR_NONE:
2355 				pi->pi_bar[idx].addr = bar = 0;
2356 				break;
2357 			case PCIBAR_IO:
2358 				addr = *valp & mask;
2359 #if defined(PCI_EMUL_IOMASK)
2360 				addr &= PCI_EMUL_IOMASK;
2361 #endif
2362 				bar = addr | pi->pi_bar[idx].lobits;
2363 				/*
2364 				 * Register the new BAR value for interception
2365 				 */
2366 				if (addr != pi->pi_bar[idx].addr) {
2367 					update_bar_address(pi, addr, idx,
2368 							   PCIBAR_IO);
2369 				}
2370 				break;
2371 			case PCIBAR_MEM32:
2372 				addr = bar = *valp & mask;
2373 				bar |= pi->pi_bar[idx].lobits;
2374 				if (addr != pi->pi_bar[idx].addr) {
2375 					update_bar_address(pi, addr, idx,
2376 							   PCIBAR_MEM32);
2377 				}
2378 				break;
2379 			case PCIBAR_MEM64:
2380 				addr = bar = *valp & mask;
2381 				bar |= pi->pi_bar[idx].lobits;
2382 				if (addr != (uint32_t)pi->pi_bar[idx].addr) {
2383 					update_bar_address(pi, addr, idx,
2384 							   PCIBAR_MEM64);
2385 				}
2386 				break;
2387 			case PCIBAR_MEMHI64:
2388 				mask = ~(pi->pi_bar[idx - 1].size - 1);
2389 				addr = ((uint64_t)*valp << 32) & mask;
2390 				bar = addr >> 32;
2391 				if (bar != pi->pi_bar[idx - 1].addr >> 32) {
2392 					update_bar_address(pi, addr, idx - 1,
2393 							   PCIBAR_MEMHI64);
2394 				}
2395 				break;
2396 			case PCIBAR_ROM:
2397 				addr = bar = *valp & mask;
2398 				if (memen(pi) && romen(pi)) {
2399 					unregister_bar(pi, idx);
2400 				}
2401 				pi->pi_bar[idx].addr = addr;
2402 				pi->pi_bar[idx].lobits = *valp &
2403 				    PCIM_BIOS_ENABLE;
2404 				/* romen could have changed it value */
2405 				if (memen(pi) && romen(pi)) {
2406 					register_bar(pi, idx);
2407 				}
2408 				bar |= pi->pi_bar[idx].lobits;
2409 				break;
2410 			default:
2411 				assert(0);
2412 			}
2413 			pci_set_cfgdata32(pi, coff, bar);
2414 
2415 		} else if (pci_emul_iscap(pi, coff)) {
2416 			pci_emul_capwrite(pi, coff, bytes, *valp, 0, 0);
2417 		} else if (coff >= PCIR_COMMAND && coff < PCIR_REVID) {
2418 			pci_emul_cmdsts_write(pi, coff, *valp, bytes);
2419 		} else {
2420 			CFGWRITE(pi, coff, *valp, bytes);
2421 		}
2422 	}
2423 }
2424 
2425 #ifdef __amd64__
2426 static int cfgenable, cfgbus, cfgslot, cfgfunc, cfgoff;
2427 
2428 static int
pci_emul_cfgaddr(struct vmctx * ctx __unused,int in,int port __unused,int bytes,uint32_t * eax,void * arg __unused)2429 pci_emul_cfgaddr(struct vmctx *ctx __unused, int in,
2430     int port __unused, int bytes, uint32_t *eax, void *arg __unused)
2431 {
2432 	uint32_t x;
2433 
2434 	if (bytes != 4) {
2435 		if (in)
2436 			*eax = (bytes == 2) ? 0xffff : 0xff;
2437 		return (0);
2438 	}
2439 
2440 	if (in) {
2441 		x = (cfgbus << 16) | (cfgslot << 11) | (cfgfunc << 8) | cfgoff;
2442 		if (cfgenable)
2443 			x |= CONF1_ENABLE;
2444 		*eax = x;
2445 	} else {
2446 		x = *eax;
2447 		cfgenable = (x & CONF1_ENABLE) == CONF1_ENABLE;
2448 		cfgoff = (x & PCI_REGMAX) & ~0x03;
2449 		cfgfunc = (x >> 8) & PCI_FUNCMAX;
2450 		cfgslot = (x >> 11) & PCI_SLOTMAX;
2451 		cfgbus = (x >> 16) & PCI_BUSMAX;
2452 	}
2453 
2454 	return (0);
2455 }
2456 INOUT_PORT(pci_cfgaddr, CONF1_ADDR_PORT, IOPORT_F_INOUT, pci_emul_cfgaddr);
2457 
2458 static int
pci_emul_cfgdata(struct vmctx * ctx __unused,int in,int port,int bytes,uint32_t * eax,void * arg __unused)2459 pci_emul_cfgdata(struct vmctx *ctx __unused, int in, int port,
2460     int bytes, uint32_t *eax, void *arg __unused)
2461 {
2462 	int coff;
2463 
2464 	assert(bytes == 1 || bytes == 2 || bytes == 4);
2465 
2466 	coff = cfgoff + (port - CONF1_DATA_PORT);
2467 	if (cfgenable) {
2468 		pci_cfgrw(in, cfgbus, cfgslot, cfgfunc, coff, bytes, eax);
2469 	} else {
2470 		/* Ignore accesses to cfgdata if not enabled by cfgaddr */
2471 		if (in)
2472 			*eax = 0xffffffff;
2473 	}
2474 	return (0);
2475 }
2476 
2477 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+0, IOPORT_F_INOUT, pci_emul_cfgdata);
2478 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+1, IOPORT_F_INOUT, pci_emul_cfgdata);
2479 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+2, IOPORT_F_INOUT, pci_emul_cfgdata);
2480 INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+3, IOPORT_F_INOUT, pci_emul_cfgdata);
2481 #endif
2482 
2483 #ifdef BHYVE_SNAPSHOT
2484 /*
2485  * Saves/restores PCI device emulated state. Returns 0 on success.
2486  */
2487 static int
pci_snapshot_pci_dev(struct vm_snapshot_meta * meta)2488 pci_snapshot_pci_dev(struct vm_snapshot_meta *meta)
2489 {
2490 	struct pci_devinst *pi;
2491 	int i;
2492 	int ret;
2493 
2494 	pi = meta->dev_data;
2495 
2496 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msi.enabled, meta, ret, done);
2497 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msi.addr, meta, ret, done);
2498 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msi.msg_data, meta, ret, done);
2499 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msi.maxmsgnum, meta, ret, done);
2500 
2501 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.enabled, meta, ret, done);
2502 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table_bar, meta, ret, done);
2503 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.pba_bar, meta, ret, done);
2504 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table_offset, meta, ret, done);
2505 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table_count, meta, ret, done);
2506 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.pba_offset, meta, ret, done);
2507 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.pba_size, meta, ret, done);
2508 	SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.function_mask, meta, ret, done);
2509 
2510 	SNAPSHOT_BUF_OR_LEAVE(pi->pi_cfgdata, sizeof(pi->pi_cfgdata),
2511 			      meta, ret, done);
2512 
2513 	for (i = 0; i < (int)nitems(pi->pi_bar); i++) {
2514 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_bar[i].type, meta, ret, done);
2515 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_bar[i].size, meta, ret, done);
2516 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_bar[i].addr, meta, ret, done);
2517 	}
2518 
2519 	/* Restore MSI-X table. */
2520 	for (i = 0; i < pi->pi_msix.table_count; i++) {
2521 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table[i].addr,
2522 				      meta, ret, done);
2523 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table[i].msg_data,
2524 				      meta, ret, done);
2525 		SNAPSHOT_VAR_OR_LEAVE(pi->pi_msix.table[i].vector_control,
2526 				      meta, ret, done);
2527 	}
2528 
2529 done:
2530 	return (ret);
2531 }
2532 
2533 int
pci_snapshot(struct vm_snapshot_meta * meta)2534 pci_snapshot(struct vm_snapshot_meta *meta)
2535 {
2536 	struct pci_devemu *pde;
2537 	struct pci_devinst *pdi;
2538 	int ret;
2539 
2540 	assert(meta->dev_name != NULL);
2541 
2542 	pdi = meta->dev_data;
2543 	pde = pdi->pi_d;
2544 
2545 	if (pde->pe_snapshot == NULL)
2546 		return (ENOTSUP);
2547 
2548 	ret = pci_snapshot_pci_dev(meta);
2549 	if (ret == 0)
2550 		ret = (*pde->pe_snapshot)(meta);
2551 
2552 	return (ret);
2553 }
2554 
2555 int
pci_pause(struct pci_devinst * pdi)2556 pci_pause(struct pci_devinst *pdi)
2557 {
2558 	struct pci_devemu *pde = pdi->pi_d;
2559 
2560 	if (pde->pe_pause == NULL) {
2561 		/* The pause/resume functionality is optional. */
2562 		return (0);
2563 	}
2564 
2565 	return (*pde->pe_pause)(pdi);
2566 }
2567 
2568 int
pci_resume(struct pci_devinst * pdi)2569 pci_resume(struct pci_devinst *pdi)
2570 {
2571 	struct pci_devemu *pde = pdi->pi_d;
2572 
2573 	if (pde->pe_resume == NULL) {
2574 		/* The pause/resume functionality is optional. */
2575 		return (0);
2576 	}
2577 
2578 	return (*pde->pe_resume)(pdi);
2579 }
2580 #endif
2581 
2582 #define PCI_EMUL_TEST
2583 #ifdef PCI_EMUL_TEST
2584 /*
2585  * Define a dummy test device
2586  */
2587 #define DIOSZ	8
2588 #define DMEMSZ	4096
2589 struct pci_emul_dsoftc {
2590 	uint8_t   ioregs[DIOSZ];
2591 	uint8_t	  memregs[2][DMEMSZ];
2592 };
2593 
2594 #define	PCI_EMUL_MSI_MSGS	 4
2595 #define	PCI_EMUL_MSIX_MSGS	16
2596 
2597 static int
pci_emul_dinit(struct pci_devinst * pi,nvlist_t * nvl __unused)2598 pci_emul_dinit(struct pci_devinst *pi, nvlist_t *nvl __unused)
2599 {
2600 	int error;
2601 	struct pci_emul_dsoftc *sc;
2602 
2603 	sc = calloc(1, sizeof(struct pci_emul_dsoftc));
2604 
2605 	pi->pi_arg = sc;
2606 
2607 	pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0001);
2608 	pci_set_cfgdata16(pi, PCIR_VENDOR, 0x10DD);
2609 	pci_set_cfgdata8(pi, PCIR_CLASS, 0x02);
2610 
2611 	error = pci_emul_add_msicap(pi, PCI_EMUL_MSI_MSGS);
2612 	assert(error == 0);
2613 
2614 	error = pci_emul_alloc_bar(pi, 0, PCIBAR_IO, DIOSZ);
2615 	assert(error == 0);
2616 
2617 	error = pci_emul_alloc_bar(pi, 1, PCIBAR_MEM32, DMEMSZ);
2618 	assert(error == 0);
2619 
2620 	error = pci_emul_alloc_bar(pi, 2, PCIBAR_MEM32, DMEMSZ);
2621 	assert(error == 0);
2622 
2623 	return (0);
2624 }
2625 
2626 static void
pci_emul_diow(struct pci_devinst * pi,int baridx,uint64_t offset,int size,uint64_t value)2627 pci_emul_diow(struct pci_devinst *pi, int baridx, uint64_t offset, int size,
2628     uint64_t value)
2629 {
2630 	int i;
2631 	struct pci_emul_dsoftc *sc = pi->pi_arg;
2632 
2633 	if (baridx == 0) {
2634 		if (offset + size > DIOSZ) {
2635 			printf("diow: iow too large, offset %ld size %d\n",
2636 			       offset, size);
2637 			return;
2638 		}
2639 
2640 		if (size == 1) {
2641 			sc->ioregs[offset] = value & 0xff;
2642 		} else if (size == 2) {
2643 			*(uint16_t *)&sc->ioregs[offset] = value & 0xffff;
2644 		} else if (size == 4) {
2645 			*(uint32_t *)&sc->ioregs[offset] = value;
2646 		} else {
2647 			printf("diow: iow unknown size %d\n", size);
2648 		}
2649 
2650 		/*
2651 		 * Special magic value to generate an interrupt
2652 		 */
2653 		if (offset == 4 && size == 4 && pci_msi_enabled(pi))
2654 			pci_generate_msi(pi, value % pci_msi_maxmsgnum(pi));
2655 
2656 		if (value == 0xabcdef) {
2657 			for (i = 0; i < pci_msi_maxmsgnum(pi); i++)
2658 				pci_generate_msi(pi, i);
2659 		}
2660 	}
2661 
2662 	if (baridx == 1 || baridx == 2) {
2663 		if (offset + size > DMEMSZ) {
2664 			printf("diow: memw too large, offset %ld size %d\n",
2665 			       offset, size);
2666 			return;
2667 		}
2668 
2669 		i = baridx - 1;		/* 'memregs' index */
2670 
2671 		if (size == 1) {
2672 			sc->memregs[i][offset] = value;
2673 		} else if (size == 2) {
2674 			*(uint16_t *)&sc->memregs[i][offset] = value;
2675 		} else if (size == 4) {
2676 			*(uint32_t *)&sc->memregs[i][offset] = value;
2677 		} else if (size == 8) {
2678 			*(uint64_t *)&sc->memregs[i][offset] = value;
2679 		} else {
2680 			printf("diow: memw unknown size %d\n", size);
2681 		}
2682 
2683 		/*
2684 		 * magic interrupt ??
2685 		 */
2686 	}
2687 
2688 	if (baridx > 2 || baridx < 0) {
2689 		printf("diow: unknown bar idx %d\n", baridx);
2690 	}
2691 }
2692 
2693 static uint64_t
pci_emul_dior(struct pci_devinst * pi,int baridx,uint64_t offset,int size)2694 pci_emul_dior(struct pci_devinst *pi, int baridx, uint64_t offset, int size)
2695 {
2696 	struct pci_emul_dsoftc *sc = pi->pi_arg;
2697 	uint32_t value;
2698 	int i;
2699 
2700 	if (baridx == 0) {
2701 		if (offset + size > DIOSZ) {
2702 			printf("dior: ior too large, offset %ld size %d\n",
2703 			       offset, size);
2704 			return (0);
2705 		}
2706 
2707 		value = 0;
2708 		if (size == 1) {
2709 			value = sc->ioregs[offset];
2710 		} else if (size == 2) {
2711 			value = *(uint16_t *) &sc->ioregs[offset];
2712 		} else if (size == 4) {
2713 			value = *(uint32_t *) &sc->ioregs[offset];
2714 		} else {
2715 			printf("dior: ior unknown size %d\n", size);
2716 		}
2717 	}
2718 
2719 	if (baridx == 1 || baridx == 2) {
2720 		if (offset + size > DMEMSZ) {
2721 			printf("dior: memr too large, offset %ld size %d\n",
2722 			       offset, size);
2723 			return (0);
2724 		}
2725 
2726 		i = baridx - 1;		/* 'memregs' index */
2727 
2728 		if (size == 1) {
2729 			value = sc->memregs[i][offset];
2730 		} else if (size == 2) {
2731 			value = *(uint16_t *) &sc->memregs[i][offset];
2732 		} else if (size == 4) {
2733 			value = *(uint32_t *) &sc->memregs[i][offset];
2734 		} else if (size == 8) {
2735 			value = *(uint64_t *) &sc->memregs[i][offset];
2736 		} else {
2737 			printf("dior: ior unknown size %d\n", size);
2738 		}
2739 	}
2740 
2741 
2742 	if (baridx > 2 || baridx < 0) {
2743 		printf("dior: unknown bar idx %d\n", baridx);
2744 		return (0);
2745 	}
2746 
2747 	return (value);
2748 }
2749 
2750 #ifdef BHYVE_SNAPSHOT
2751 struct pci_devinst *
pci_next(const struct pci_devinst * cursor)2752 pci_next(const struct pci_devinst *cursor)
2753 {
2754 	unsigned bus = 0, slot = 0, func = 0;
2755 	struct businfo *bi;
2756 	struct slotinfo *si;
2757 	struct funcinfo *fi;
2758 
2759 	bus = cursor ? cursor->pi_bus : 0;
2760 	slot = cursor ? cursor->pi_slot : 0;
2761 	func = cursor ? (cursor->pi_func + 1) : 0;
2762 
2763 	for (; bus < MAXBUSES; bus++) {
2764 		if ((bi = pci_businfo[bus]) == NULL)
2765 			continue;
2766 
2767 		if (slot >= MAXSLOTS)
2768 			slot = 0;
2769 
2770 		for (; slot < MAXSLOTS; slot++) {
2771 			si = &bi->slotinfo[slot];
2772 			if (func >= MAXFUNCS)
2773 				func = 0;
2774 			for (; func < MAXFUNCS; func++) {
2775 				fi = &si->si_funcs[func];
2776 				if (fi->fi_devi == NULL)
2777 					continue;
2778 
2779 				return (fi->fi_devi);
2780 			}
2781 		}
2782 	}
2783 
2784 	return (NULL);
2785 }
2786 
2787 static int
pci_emul_snapshot(struct vm_snapshot_meta * meta __unused)2788 pci_emul_snapshot(struct vm_snapshot_meta *meta __unused)
2789 {
2790 	return (0);
2791 }
2792 #endif
2793 
2794 static const struct pci_devemu pci_dummy = {
2795 	.pe_emu = "dummy",
2796 	.pe_init = pci_emul_dinit,
2797 	.pe_barwrite = pci_emul_diow,
2798 	.pe_barread = pci_emul_dior,
2799 #ifdef BHYVE_SNAPSHOT
2800 	.pe_snapshot = pci_emul_snapshot,
2801 #endif
2802 };
2803 PCI_EMUL_SET(pci_dummy);
2804 
2805 #endif /* PCI_EMUL_TEST */
2806