xref: /freebsd/sys/dev/vmd/vmd.c (revision 2008043f386721d58158e37e0d7e50df8095942d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2021 Alexander Motin <mav@FreeBSD.org>
5  * Copyright 2019 Cisco Systems, Inc.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/kernel.h>
36 #include <sys/limits.h>
37 #include <sys/module.h>
38 #include <sys/sysctl.h>
39 #include <sys/systm.h>
40 #include <sys/malloc.h>
41 
42 #include <machine/bus.h>
43 #include <machine/resource.h>
44 #include <machine/intr_machdep.h>
45 #include <sys/rman.h>
46 #include <sys/lock.h>
47 #include <sys/mutex.h>
48 
49 #include <sys/pciio.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pci_private.h>
53 #include <dev/pci/pcib_private.h>
54 
55 #include <dev/vmd/vmd.h>
56 
57 #include "pcib_if.h"
58 
59 struct vmd_type {
60 	u_int16_t	vmd_vid;
61 	u_int16_t	vmd_did;
62 	char		*vmd_name;
63 	int		flags;
64 #define BUS_RESTRICT	1
65 #define VECTOR_OFFSET	2
66 #define CAN_BYPASS_MSI	4
67 };
68 
69 #define VMD_CAP		0x40
70 #define VMD_BUS_RESTRICT	0x1
71 
72 #define VMD_CONFIG	0x44
73 #define VMD_BYPASS_MSI		0x2
74 #define VMD_BUS_START(x)	((x >> 8) & 0x3)
75 
76 #define VMD_LOCK	0x70
77 
78 SYSCTL_NODE(_hw, OID_AUTO, vmd, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
79     "Intel Volume Management Device tuning parameters");
80 
81 /*
82  * By default all VMD devices remap children MSI/MSI-X interrupts into their
83  * own.  It creates additional isolation, but also complicates things due to
84  * sharing, etc.  Fortunately some VMD devices can bypass the remapping.
85  */
86 static int vmd_bypass_msi = 1;
87 SYSCTL_INT(_hw_vmd, OID_AUTO, bypass_msi, CTLFLAG_RWTUN, &vmd_bypass_msi, 0,
88     "Bypass MSI remapping on capable hardware");
89 
90 /*
91  * All MSIs within a group share address, so VMD can't distinguish them.
92  * It makes no sense to use more than one per device, only if required by
93  * some specific device drivers.
94  */
95 static int vmd_max_msi = 1;
96 SYSCTL_INT(_hw_vmd, OID_AUTO, max_msi, CTLFLAG_RWTUN, &vmd_max_msi, 0,
97     "Maximum number of MSI vectors per device");
98 
99 /*
100  * MSI-X can use different addresses, but we have limited number of MSI-X
101  * we can route to, so use conservative default to try to avoid sharing.
102  */
103 static int vmd_max_msix = 3;
104 SYSCTL_INT(_hw_vmd, OID_AUTO, max_msix, CTLFLAG_RWTUN, &vmd_max_msix, 0,
105     "Maximum number of MSI-X vectors per device");
106 
107 static struct vmd_type vmd_devs[] = {
108         { 0x8086, 0x201d, "Intel Volume Management Device", 0 },
109         { 0x8086, 0x28c0, "Intel Volume Management Device", BUS_RESTRICT | CAN_BYPASS_MSI },
110         { 0x8086, 0x467f, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
111         { 0x8086, 0x4c3d, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
112         { 0x8086, 0x7d0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
113         { 0x8086, 0x9a0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
114         { 0x8086, 0xa77f, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
115         { 0x8086, 0xad0b, "Intel Volume Management Device", BUS_RESTRICT | VECTOR_OFFSET },
116         { 0, 0, NULL, 0 }
117 };
118 
119 static int
120 vmd_probe(device_t dev)
121 {
122 	struct vmd_type *t;
123 	uint16_t vid, did;
124 
125 	vid = pci_get_vendor(dev);
126 	did = pci_get_device(dev);
127 	for (t = vmd_devs; t->vmd_name != NULL; t++) {
128 		if (vid == t->vmd_vid && did == t->vmd_did) {
129 			device_set_desc(dev, t->vmd_name);
130 			return (BUS_PROBE_DEFAULT);
131 		}
132 	}
133 	return (ENXIO);
134 }
135 
136 static void
137 vmd_free(struct vmd_softc *sc)
138 {
139 	struct vmd_irq *vi;
140 	struct vmd_irq_user *u;
141 	int i;
142 
143 	if (sc->psc.bus.rman.rm_end != 0)
144 		rman_fini(&sc->psc.bus.rman);
145 	if (sc->psc.mem.rman.rm_end != 0)
146 		rman_fini(&sc->psc.mem.rman);
147 	while ((u = LIST_FIRST(&sc->vmd_users)) != NULL) {
148 		LIST_REMOVE(u, viu_link);
149 		free(u, M_DEVBUF);
150 	}
151 	if (sc->vmd_irq != NULL) {
152 		for (i = 0; i < sc->vmd_msix_count; i++) {
153 			vi = &sc->vmd_irq[i];
154 			if (vi->vi_res == NULL)
155 				continue;
156 			bus_teardown_intr(sc->psc.dev, vi->vi_res,
157 			    vi->vi_handle);
158 			bus_release_resource(sc->psc.dev, SYS_RES_IRQ,
159 			    vi->vi_rid, vi->vi_res);
160 		}
161 	}
162 	free(sc->vmd_irq, M_DEVBUF);
163 	sc->vmd_irq = NULL;
164 	pci_release_msi(sc->psc.dev);
165 	for (i = 0; i < VMD_MAX_BAR; i++) {
166 		if (sc->vmd_regs_res[i] != NULL)
167 			bus_release_resource(sc->psc.dev, SYS_RES_MEMORY,
168 			    sc->vmd_regs_rid[i], sc->vmd_regs_res[i]);
169 	}
170 }
171 
172 /* Hidden PCI Roots are hidden in BAR(0). */
173 
174 static uint32_t
175 vmd_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width)
176 {
177 	struct vmd_softc *sc;
178 	bus_addr_t offset;
179 
180 	sc = device_get_softc(dev);
181 	if (b < sc->vmd_bus_start || b > sc->vmd_bus_end)
182 		return (0xffffffff);
183 
184 	offset = ((b - sc->vmd_bus_start) << 20) + (s << 15) + (f << 12) + reg;
185 
186 	switch (width) {
187 	case 4:
188 		return (bus_space_read_4(sc->vmd_btag, sc->vmd_bhandle,
189 		    offset));
190 	case 2:
191 		return (bus_space_read_2(sc->vmd_btag, sc->vmd_bhandle,
192 		    offset));
193 	case 1:
194 		return (bus_space_read_1(sc->vmd_btag, sc->vmd_bhandle,
195 		    offset));
196 	default:
197 		__assert_unreachable();
198 		return (0xffffffff);
199 	}
200 }
201 
202 static void
203 vmd_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg,
204     uint32_t val, int width)
205 {
206 	struct vmd_softc *sc;
207 	bus_addr_t offset;
208 
209 	sc = device_get_softc(dev);
210 	if (b < sc->vmd_bus_start || b > sc->vmd_bus_end)
211 		return;
212 
213 	offset = ((b - sc->vmd_bus_start) << 20) + (s << 15) + (f << 12) + reg;
214 
215 	switch (width) {
216 	case 4:
217 		return (bus_space_write_4(sc->vmd_btag, sc->vmd_bhandle,
218 		    offset, val));
219 	case 2:
220 		return (bus_space_write_2(sc->vmd_btag, sc->vmd_bhandle,
221 		    offset, val));
222 	case 1:
223 		return (bus_space_write_1(sc->vmd_btag, sc->vmd_bhandle,
224 		    offset, val));
225 	default:
226 		__assert_unreachable();
227 	}
228 }
229 
230 static void
231 vmd_set_msi_bypass(device_t dev, bool enable)
232 {
233 	uint16_t val;
234 
235 	val = pci_read_config(dev, VMD_CONFIG, 2);
236 	if (enable)
237 		val |= VMD_BYPASS_MSI;
238 	else
239 		val &= ~VMD_BYPASS_MSI;
240 	pci_write_config(dev, VMD_CONFIG, val, 2);
241 }
242 
243 static int
244 vmd_intr(void *arg)
245 {
246 	/*
247 	 * We have nothing to do here, but we have to register some interrupt
248 	 * handler to make PCI code setup and enable the MSI-X vector.
249 	 */
250 	return (FILTER_STRAY);
251 }
252 
253 static int
254 vmd_attach(device_t dev)
255 {
256 	struct vmd_softc *sc;
257 	struct pcib_secbus *bus;
258 	struct pcib_window *w;
259 	struct vmd_type *t;
260 	struct vmd_irq *vi;
261 	uint16_t vid, did;
262 	uint32_t bar;
263 	int i, j, error;
264 	char buf[64];
265 
266 	sc = device_get_softc(dev);
267 	bzero(sc, sizeof(*sc));
268 	sc->psc.dev = dev;
269 	sc->psc.domain = PCI_DOMAINMAX - device_get_unit(dev);
270 
271 	pci_enable_busmaster(dev);
272 
273 	for (i = 0, j = 0; i < VMD_MAX_BAR; i++, j++) {
274 		sc->vmd_regs_rid[i] = PCIR_BAR(j);
275 		bar = pci_read_config(dev, PCIR_BAR(0), 4);
276 		if (PCI_BAR_MEM(bar) && (bar & PCIM_BAR_MEM_TYPE) ==
277 		    PCIM_BAR_MEM_64)
278 			j++;
279 		if ((sc->vmd_regs_res[i] = bus_alloc_resource_any(dev,
280 		    SYS_RES_MEMORY, &sc->vmd_regs_rid[i], RF_ACTIVE)) == NULL) {
281 			device_printf(dev, "Cannot allocate resources\n");
282 			goto fail;
283 		}
284 	}
285 
286 	sc->vmd_btag = rman_get_bustag(sc->vmd_regs_res[0]);
287 	sc->vmd_bhandle = rman_get_bushandle(sc->vmd_regs_res[0]);
288 
289 	vid = pci_get_vendor(dev);
290 	did = pci_get_device(dev);
291 	for (t = vmd_devs; t->vmd_name != NULL; t++) {
292 		if (vid == t->vmd_vid && did == t->vmd_did)
293 			break;
294 	}
295 
296 	sc->vmd_bus_start = 0;
297 	if ((t->flags & BUS_RESTRICT) &&
298 	    (pci_read_config(dev, VMD_CAP, 2) & VMD_BUS_RESTRICT)) {
299 		switch (VMD_BUS_START(pci_read_config(dev, VMD_CONFIG, 2))) {
300 		case 0:
301 			sc->vmd_bus_start = 0;
302 			break;
303 		case 1:
304 			sc->vmd_bus_start = 128;
305 			break;
306 		case 2:
307 			sc->vmd_bus_start = 224;
308 			break;
309 		default:
310 			device_printf(dev, "Unknown bus offset\n");
311 			goto fail;
312 		}
313 	}
314 	sc->vmd_bus_end = MIN(PCI_BUSMAX, sc->vmd_bus_start +
315 	    (rman_get_size(sc->vmd_regs_res[0]) >> 20) - 1);
316 
317 	bus = &sc->psc.bus;
318 	bus->sec = sc->vmd_bus_start;
319 	bus->sub = sc->vmd_bus_end;
320 	bus->dev = dev;
321 	bus->rman.rm_start = 0;
322 	bus->rman.rm_end = PCI_BUSMAX;
323 	bus->rman.rm_type = RMAN_ARRAY;
324 	snprintf(buf, sizeof(buf), "%s bus numbers", device_get_nameunit(dev));
325 	bus->rman.rm_descr = strdup(buf, M_DEVBUF);
326 	error = rman_init(&bus->rman);
327 	if (error) {
328 		device_printf(dev, "Failed to initialize bus rman\n");
329 		bus->rman.rm_end = 0;
330 		goto fail;
331 	}
332 	error = rman_manage_region(&bus->rman, sc->vmd_bus_start,
333 	    sc->vmd_bus_end);
334 	if (error) {
335 		device_printf(dev, "Failed to add resource to bus rman\n");
336 		goto fail;
337 	}
338 
339 	w = &sc->psc.mem;
340 	w->rman.rm_type = RMAN_ARRAY;
341 	snprintf(buf, sizeof(buf), "%s memory window", device_get_nameunit(dev));
342 	w->rman.rm_descr = strdup(buf, M_DEVBUF);
343 	error = rman_init(&w->rman);
344 	if (error) {
345 		device_printf(dev, "Failed to initialize memory rman\n");
346 		w->rman.rm_end = 0;
347 		goto fail;
348 	}
349 	error = rman_manage_region(&w->rman,
350 	    rman_get_start(sc->vmd_regs_res[1]),
351 	    rman_get_end(sc->vmd_regs_res[1]));
352 	if (error) {
353 		device_printf(dev, "Failed to add resource to memory rman\n");
354 		goto fail;
355 	}
356 	error = rman_manage_region(&w->rman,
357 	    rman_get_start(sc->vmd_regs_res[2]) + 0x2000,
358 	    rman_get_end(sc->vmd_regs_res[2]));
359 	if (error) {
360 		device_printf(dev, "Failed to add resource to memory rman\n");
361 		goto fail;
362 	}
363 
364 	LIST_INIT(&sc->vmd_users);
365 	sc->vmd_fist_vector = (t->flags & VECTOR_OFFSET) ? 1 : 0;
366 	sc->vmd_msix_count = pci_msix_count(dev);
367 	if (vmd_bypass_msi && (t->flags & CAN_BYPASS_MSI)) {
368 		sc->vmd_msix_count = 0;
369 		vmd_set_msi_bypass(dev, true);
370 	} else if (pci_alloc_msix(dev, &sc->vmd_msix_count) == 0) {
371 		sc->vmd_irq = malloc(sizeof(struct vmd_irq) *
372 		    sc->vmd_msix_count, M_DEVBUF, M_WAITOK | M_ZERO);
373 		for (i = 0; i < sc->vmd_msix_count; i++) {
374 			vi = &sc->vmd_irq[i];
375 			vi->vi_rid = i + 1;
376 			vi->vi_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
377 			    &vi->vi_rid, RF_ACTIVE | RF_SHAREABLE);
378 			if (vi->vi_res == NULL) {
379 				device_printf(dev, "Failed to allocate irq\n");
380 				goto fail;
381 			}
382 			vi->vi_irq = rman_get_start(vi->vi_res);
383 			if (bus_setup_intr(dev, vi->vi_res, INTR_TYPE_MISC |
384 			    INTR_MPSAFE, vmd_intr, NULL, vi, &vi->vi_handle)) {
385 				device_printf(dev, "Can't set up interrupt\n");
386 				bus_release_resource(dev, SYS_RES_IRQ,
387 				    vi->vi_rid, vi->vi_res);
388 				vi->vi_res = NULL;
389 				goto fail;
390 			}
391 		}
392 		vmd_set_msi_bypass(dev, false);
393 	}
394 
395 	sc->vmd_dma_tag = bus_get_dma_tag(dev);
396 
397 	sc->psc.child = device_add_child(dev, "pci", -1);
398 	return (bus_generic_attach(dev));
399 
400 fail:
401 	vmd_free(sc);
402 	return (ENXIO);
403 }
404 
405 static int
406 vmd_detach(device_t dev)
407 {
408 	struct vmd_softc *sc = device_get_softc(dev);
409 	int error;
410 
411 	error = bus_generic_detach(dev);
412 	if (error)
413 		return (error);
414 	error = device_delete_children(dev);
415 	if (error)
416 		return (error);
417 	if (sc->vmd_msix_count == 0)
418 		vmd_set_msi_bypass(dev, false);
419 	vmd_free(sc);
420 	return (0);
421 }
422 
423 static bus_dma_tag_t
424 vmd_get_dma_tag(device_t dev, device_t child)
425 {
426 	struct vmd_softc *sc = device_get_softc(dev);
427 
428 	return (sc->vmd_dma_tag);
429 }
430 
431 static struct resource *
432 vmd_alloc_resource(device_t dev, device_t child, int type, int *rid,
433     rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
434 {
435 	struct vmd_softc *sc = device_get_softc(dev);
436 	struct resource *res;
437 
438 	switch (type) {
439 	case SYS_RES_IRQ:
440 		/* VMD harwdare does not support legacy interrupts. */
441 		if (*rid == 0)
442 			return (NULL);
443 		return (bus_generic_alloc_resource(dev, child, type, rid,
444 		    start, end, count, flags | RF_SHAREABLE));
445 	case SYS_RES_MEMORY:
446 		res = rman_reserve_resource(&sc->psc.mem.rman, start, end,
447 		    count, flags, child);
448 		if (res == NULL)
449 			return (NULL);
450 		if (bootverbose)
451 			device_printf(dev,
452 			    "allocated memory range (%#jx-%#jx) for rid %d of %s\n",
453 			    rman_get_start(res), rman_get_end(res), *rid,
454 			    pcib_child_name(child));
455 		break;
456 	case PCI_RES_BUS:
457 		res = rman_reserve_resource(&sc->psc.bus.rman, start, end,
458 		    count, flags, child);
459 		if (res == NULL)
460 			return (NULL);
461 		if (bootverbose)
462 			device_printf(dev,
463 			    "allocated bus range (%ju-%ju) for rid %d of %s\n",
464 			    rman_get_start(res), rman_get_end(res), *rid,
465 			    pcib_child_name(child));
466 		break;
467 	default:
468 		/* VMD harwdare does not support I/O ports. */
469 		return (NULL);
470 	}
471 	rman_set_rid(res, *rid);
472 	return (res);
473 }
474 
475 static int
476 vmd_adjust_resource(device_t dev, device_t child, int type,
477     struct resource *r, rman_res_t start, rman_res_t end)
478 {
479 
480 	if (type == SYS_RES_IRQ) {
481 		return (bus_generic_adjust_resource(dev, child, type, r,
482 		    start, end));
483 	}
484 	return (rman_adjust_resource(r, start, end));
485 }
486 
487 static int
488 vmd_release_resource(device_t dev, device_t child, int type, int rid,
489     struct resource *r)
490 {
491 
492 	if (type == SYS_RES_IRQ) {
493 		return (bus_generic_release_resource(dev, child, type, rid,
494 		    r));
495 	}
496 	return (rman_release_resource(r));
497 }
498 
499 static int
500 vmd_route_interrupt(device_t dev, device_t child, int pin)
501 {
502 
503 	/* VMD harwdare does not support legacy interrupts. */
504 	return (PCI_INVALID_IRQ);
505 }
506 
507 static int
508 vmd_alloc_msi(device_t dev, device_t child, int count, int maxcount,
509     int *irqs)
510 {
511 	struct vmd_softc *sc = device_get_softc(dev);
512 	struct vmd_irq_user *u;
513 	int i, ibest = 0, best = INT_MAX;
514 
515 	if (sc->vmd_msix_count == 0) {
516 		return (PCIB_ALLOC_MSI(device_get_parent(device_get_parent(dev)),
517 		    child, count, maxcount, irqs));
518 	}
519 
520 	if (count > vmd_max_msi)
521 		return (ENOSPC);
522 	LIST_FOREACH(u, &sc->vmd_users, viu_link) {
523 		if (u->viu_child == child)
524 			return (EBUSY);
525 	}
526 
527 	for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) {
528 		if (best > sc->vmd_irq[i].vi_nusers) {
529 			best = sc->vmd_irq[i].vi_nusers;
530 			ibest = i;
531 		}
532 	}
533 
534 	u = malloc(sizeof(*u), M_DEVBUF, M_WAITOK | M_ZERO);
535 	u->viu_child = child;
536 	u->viu_vector = ibest;
537 	LIST_INSERT_HEAD(&sc->vmd_users, u, viu_link);
538 	sc->vmd_irq[ibest].vi_nusers += count;
539 
540 	for (i = 0; i < count; i++)
541 		irqs[i] = sc->vmd_irq[ibest].vi_irq;
542 	return (0);
543 }
544 
545 static int
546 vmd_release_msi(device_t dev, device_t child, int count, int *irqs)
547 {
548 	struct vmd_softc *sc = device_get_softc(dev);
549 	struct vmd_irq_user *u;
550 
551 	if (sc->vmd_msix_count == 0) {
552 		return (PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)),
553 		    child, count, irqs));
554 	}
555 
556 	LIST_FOREACH(u, &sc->vmd_users, viu_link) {
557 		if (u->viu_child == child) {
558 			sc->vmd_irq[u->viu_vector].vi_nusers -= count;
559 			LIST_REMOVE(u, viu_link);
560 			free(u, M_DEVBUF);
561 			return (0);
562 		}
563 	}
564 	return (EINVAL);
565 }
566 
567 static int
568 vmd_alloc_msix(device_t dev, device_t child, int *irq)
569 {
570 	struct vmd_softc *sc = device_get_softc(dev);
571 	struct vmd_irq_user *u;
572 	int i, ibest = 0, best = INT_MAX;
573 
574 	if (sc->vmd_msix_count == 0) {
575 		return (PCIB_ALLOC_MSIX(device_get_parent(device_get_parent(dev)),
576 		    child, irq));
577 	}
578 
579 	i = 0;
580 	LIST_FOREACH(u, &sc->vmd_users, viu_link) {
581 		if (u->viu_child == child)
582 			i++;
583 	}
584 	if (i >= vmd_max_msix)
585 		return (ENOSPC);
586 
587 	for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) {
588 		if (best > sc->vmd_irq[i].vi_nusers) {
589 			best = sc->vmd_irq[i].vi_nusers;
590 			ibest = i;
591 		}
592 	}
593 
594 	u = malloc(sizeof(*u), M_DEVBUF, M_WAITOK | M_ZERO);
595 	u->viu_child = child;
596 	u->viu_vector = ibest;
597 	LIST_INSERT_HEAD(&sc->vmd_users, u, viu_link);
598 	sc->vmd_irq[ibest].vi_nusers++;
599 
600 	*irq = sc->vmd_irq[ibest].vi_irq;
601 	return (0);
602 }
603 
604 static int
605 vmd_release_msix(device_t dev, device_t child, int irq)
606 {
607 	struct vmd_softc *sc = device_get_softc(dev);
608 	struct vmd_irq_user *u;
609 
610 	if (sc->vmd_msix_count == 0) {
611 		return (PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)),
612 		    child, irq));
613 	}
614 
615 	LIST_FOREACH(u, &sc->vmd_users, viu_link) {
616 		if (u->viu_child == child &&
617 		    sc->vmd_irq[u->viu_vector].vi_irq == irq) {
618 			sc->vmd_irq[u->viu_vector].vi_nusers--;
619 			LIST_REMOVE(u, viu_link);
620 			free(u, M_DEVBUF);
621 			return (0);
622 		}
623 	}
624 	return (EINVAL);
625 }
626 
627 static int
628 vmd_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, uint32_t *data)
629 {
630 	struct vmd_softc *sc = device_get_softc(dev);
631 	int i;
632 
633 	if (sc->vmd_msix_count == 0) {
634 		return (PCIB_MAP_MSI(device_get_parent(device_get_parent(dev)),
635 		    child, irq, addr, data));
636 	}
637 
638 	for (i = sc->vmd_fist_vector; i < sc->vmd_msix_count; i++) {
639 		if (sc->vmd_irq[i].vi_irq == irq)
640 			break;
641 	}
642 	if (i >= sc->vmd_msix_count)
643 		return (EINVAL);
644 	*addr = MSI_INTEL_ADDR_BASE | (i << 12);
645 	*data = 0;
646 	return (0);
647 }
648 
649 static device_method_t vmd_pci_methods[] = {
650 	/* Device interface */
651 	DEVMETHOD(device_probe,			vmd_probe),
652 	DEVMETHOD(device_attach,		vmd_attach),
653 	DEVMETHOD(device_detach,		vmd_detach),
654 	DEVMETHOD(device_suspend,		bus_generic_suspend),
655 	DEVMETHOD(device_resume,		bus_generic_resume),
656 	DEVMETHOD(device_shutdown,		bus_generic_shutdown),
657 
658 	/* Bus interface */
659 	DEVMETHOD(bus_get_dma_tag,		vmd_get_dma_tag),
660 	DEVMETHOD(bus_read_ivar,		pcib_read_ivar),
661 	DEVMETHOD(bus_write_ivar,		pcib_write_ivar),
662 	DEVMETHOD(bus_alloc_resource,		vmd_alloc_resource),
663 	DEVMETHOD(bus_adjust_resource,		vmd_adjust_resource),
664 	DEVMETHOD(bus_release_resource,		vmd_release_resource),
665 	DEVMETHOD(bus_activate_resource,	bus_generic_activate_resource),
666 	DEVMETHOD(bus_deactivate_resource,	bus_generic_deactivate_resource),
667 	DEVMETHOD(bus_setup_intr,		bus_generic_setup_intr),
668 	DEVMETHOD(bus_teardown_intr,		bus_generic_teardown_intr),
669 
670 	/* pcib interface */
671 	DEVMETHOD(pcib_maxslots,		pcib_maxslots),
672 	DEVMETHOD(pcib_read_config,		vmd_read_config),
673 	DEVMETHOD(pcib_write_config,		vmd_write_config),
674 	DEVMETHOD(pcib_route_interrupt,		vmd_route_interrupt),
675 	DEVMETHOD(pcib_alloc_msi,		vmd_alloc_msi),
676 	DEVMETHOD(pcib_release_msi,		vmd_release_msi),
677 	DEVMETHOD(pcib_alloc_msix,		vmd_alloc_msix),
678 	DEVMETHOD(pcib_release_msix,		vmd_release_msix),
679 	DEVMETHOD(pcib_map_msi,			vmd_map_msi),
680 	DEVMETHOD(pcib_request_feature,		pcib_request_feature_allow),
681 
682 	DEVMETHOD_END
683 };
684 
685 DEFINE_CLASS_0(pcib, vmd_pci_driver, vmd_pci_methods, sizeof(struct vmd_softc));
686 DRIVER_MODULE(vmd, pci, vmd_pci_driver, NULL, NULL);
687 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmd,
688     vmd_devs, nitems(vmd_devs) - 1);
689