xref: /freebsd/sys/dev/vmm/vmm_dev.c (revision c76c2a19ae3763d17aa6a60a5831ed24cbc16e83)
1b9ef152bSMark Johnston /*-
2b9ef152bSMark Johnston  * SPDX-License-Identifier: BSD-2-Clause
3b9ef152bSMark Johnston  *
4b9ef152bSMark Johnston  * Copyright (c) 2011 NetApp, Inc.
5b9ef152bSMark Johnston  * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
6b9ef152bSMark Johnston  * All rights reserved.
7b9ef152bSMark Johnston  */
8b9ef152bSMark Johnston 
9b9ef152bSMark Johnston #include <sys/param.h>
10b9ef152bSMark Johnston #include <sys/conf.h>
11a97f683fSMark Johnston #include <sys/fcntl.h>
12b9ef152bSMark Johnston #include <sys/ioccom.h>
13b9ef152bSMark Johnston #include <sys/jail.h>
14b9ef152bSMark Johnston #include <sys/kernel.h>
15b9ef152bSMark Johnston #include <sys/malloc.h>
16b9ef152bSMark Johnston #include <sys/mman.h>
17b9ef152bSMark Johnston #include <sys/proc.h>
18b9ef152bSMark Johnston #include <sys/queue.h>
19887c0877SMark Johnston #include <sys/sx.h>
20b9ef152bSMark Johnston #include <sys/sysctl.h>
21b9ef152bSMark Johnston #include <sys/ucred.h>
22b9ef152bSMark Johnston #include <sys/uio.h>
23b9ef152bSMark Johnston 
24b9ef152bSMark Johnston #include <machine/vmm.h>
25b9ef152bSMark Johnston 
26b9ef152bSMark Johnston #include <vm/vm.h>
27b9ef152bSMark Johnston #include <vm/vm_object.h>
28b9ef152bSMark Johnston 
29b9ef152bSMark Johnston #include <dev/vmm/vmm_dev.h>
30*c76c2a19SMark Johnston #include <dev/vmm/vmm_mem.h>
31b9ef152bSMark Johnston #include <dev/vmm/vmm_stat.h>
32b9ef152bSMark Johnston 
33e12b6aafSMark Johnston #if defined(__amd64__) && defined(COMPAT_FREEBSD12)
34a852dc58SMark Johnston struct vm_memseg_12 {
35e12b6aafSMark Johnston 	int		segid;
36e12b6aafSMark Johnston 	size_t		len;
37e12b6aafSMark Johnston 	char		name[64];
38e12b6aafSMark Johnston };
39a852dc58SMark Johnston _Static_assert(sizeof(struct vm_memseg_12) == 80, "COMPAT_FREEBSD12 ABI");
40e12b6aafSMark Johnston 
41a852dc58SMark Johnston #define	VM_ALLOC_MEMSEG_12	\
42a852dc58SMark Johnston 	_IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg_12)
43a852dc58SMark Johnston #define	VM_GET_MEMSEG_12	\
44a852dc58SMark Johnston 	_IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg_12)
45e12b6aafSMark Johnston #endif
46e12b6aafSMark Johnston 
47b9ef152bSMark Johnston struct devmem_softc {
48b9ef152bSMark Johnston 	int	segid;
49b9ef152bSMark Johnston 	char	*name;
50b9ef152bSMark Johnston 	struct cdev *cdev;
51b9ef152bSMark Johnston 	struct vmmdev_softc *sc;
52b9ef152bSMark Johnston 	SLIST_ENTRY(devmem_softc) link;
53b9ef152bSMark Johnston };
54b9ef152bSMark Johnston 
55b9ef152bSMark Johnston struct vmmdev_softc {
56b9ef152bSMark Johnston 	struct vm	*vm;		/* vm instance cookie */
57b9ef152bSMark Johnston 	struct cdev	*cdev;
58b9ef152bSMark Johnston 	struct ucred	*ucred;
59b9ef152bSMark Johnston 	SLIST_ENTRY(vmmdev_softc) link;
60b9ef152bSMark Johnston 	SLIST_HEAD(, devmem_softc) devmem;
61b9ef152bSMark Johnston 	int		flags;
62b9ef152bSMark Johnston };
63b9ef152bSMark Johnston 
64b9ef152bSMark Johnston static SLIST_HEAD(, vmmdev_softc) head;
65b9ef152bSMark Johnston 
66b9ef152bSMark Johnston static unsigned pr_allow_flag;
67887c0877SMark Johnston static struct sx vmmdev_mtx;
68887c0877SMark Johnston SX_SYSINIT(vmmdev_mtx, &vmmdev_mtx, "vmm device mutex");
69b9ef152bSMark Johnston 
70b9ef152bSMark Johnston static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev");
71b9ef152bSMark Johnston 
72b9ef152bSMark Johnston SYSCTL_DECL(_hw_vmm);
73b9ef152bSMark Johnston 
74b9ef152bSMark Johnston static void devmem_destroy(void *arg);
75f4002135SMark Johnston static int devmem_create_cdev(struct vmmdev_softc *sc, int id, char *devmem);
76b9ef152bSMark Johnston 
77b9ef152bSMark Johnston static int
78b9ef152bSMark Johnston vmm_priv_check(struct ucred *ucred)
79b9ef152bSMark Johnston {
80b9ef152bSMark Johnston 	if (jailed(ucred) &&
81b9ef152bSMark Johnston 	    !(ucred->cr_prison->pr_allow & pr_allow_flag))
82b9ef152bSMark Johnston 		return (EPERM);
83b9ef152bSMark Johnston 
84b9ef152bSMark Johnston 	return (0);
85b9ef152bSMark Johnston }
86b9ef152bSMark Johnston 
87b9ef152bSMark Johnston static int
88b9ef152bSMark Johnston vcpu_lock_one(struct vcpu *vcpu)
89b9ef152bSMark Johnston {
90b9ef152bSMark Johnston 	return (vcpu_set_state(vcpu, VCPU_FROZEN, true));
91b9ef152bSMark Johnston }
92b9ef152bSMark Johnston 
93b9ef152bSMark Johnston static void
94b9ef152bSMark Johnston vcpu_unlock_one(struct vcpu *vcpu)
95b9ef152bSMark Johnston {
96b9ef152bSMark Johnston 	enum vcpu_state state;
97b9ef152bSMark Johnston 
98b9ef152bSMark Johnston 	state = vcpu_get_state(vcpu, NULL);
99b9ef152bSMark Johnston 	if (state != VCPU_FROZEN) {
100b9ef152bSMark Johnston 		panic("vcpu %s(%d) has invalid state %d",
101b9ef152bSMark Johnston 		    vm_name(vcpu_vm(vcpu)), vcpu_vcpuid(vcpu), state);
102b9ef152bSMark Johnston 	}
103b9ef152bSMark Johnston 
104b9ef152bSMark Johnston 	vcpu_set_state(vcpu, VCPU_IDLE, false);
105b9ef152bSMark Johnston }
106b9ef152bSMark Johnston 
107b9ef152bSMark Johnston static int
108b9ef152bSMark Johnston vcpu_lock_all(struct vmmdev_softc *sc)
109b9ef152bSMark Johnston {
110b9ef152bSMark Johnston 	struct vcpu *vcpu;
111b9ef152bSMark Johnston 	int error;
112b9ef152bSMark Johnston 	uint16_t i, j, maxcpus;
113b9ef152bSMark Johnston 
114b9ef152bSMark Johnston 	error = 0;
115b9ef152bSMark Johnston 	vm_slock_vcpus(sc->vm);
116b9ef152bSMark Johnston 	maxcpus = vm_get_maxcpus(sc->vm);
117b9ef152bSMark Johnston 	for (i = 0; i < maxcpus; i++) {
118b9ef152bSMark Johnston 		vcpu = vm_vcpu(sc->vm, i);
119b9ef152bSMark Johnston 		if (vcpu == NULL)
120b9ef152bSMark Johnston 			continue;
121b9ef152bSMark Johnston 		error = vcpu_lock_one(vcpu);
122b9ef152bSMark Johnston 		if (error)
123b9ef152bSMark Johnston 			break;
124b9ef152bSMark Johnston 	}
125b9ef152bSMark Johnston 
126b9ef152bSMark Johnston 	if (error) {
127b9ef152bSMark Johnston 		for (j = 0; j < i; j++) {
128b9ef152bSMark Johnston 			vcpu = vm_vcpu(sc->vm, j);
129b9ef152bSMark Johnston 			if (vcpu == NULL)
130b9ef152bSMark Johnston 				continue;
131b9ef152bSMark Johnston 			vcpu_unlock_one(vcpu);
132b9ef152bSMark Johnston 		}
133b9ef152bSMark Johnston 		vm_unlock_vcpus(sc->vm);
134b9ef152bSMark Johnston 	}
135b9ef152bSMark Johnston 
136b9ef152bSMark Johnston 	return (error);
137b9ef152bSMark Johnston }
138b9ef152bSMark Johnston 
139b9ef152bSMark Johnston static void
140b9ef152bSMark Johnston vcpu_unlock_all(struct vmmdev_softc *sc)
141b9ef152bSMark Johnston {
142b9ef152bSMark Johnston 	struct vcpu *vcpu;
143b9ef152bSMark Johnston 	uint16_t i, maxcpus;
144b9ef152bSMark Johnston 
145b9ef152bSMark Johnston 	maxcpus = vm_get_maxcpus(sc->vm);
146b9ef152bSMark Johnston 	for (i = 0; i < maxcpus; i++) {
147b9ef152bSMark Johnston 		vcpu = vm_vcpu(sc->vm, i);
148b9ef152bSMark Johnston 		if (vcpu == NULL)
149b9ef152bSMark Johnston 			continue;
150b9ef152bSMark Johnston 		vcpu_unlock_one(vcpu);
151b9ef152bSMark Johnston 	}
152b9ef152bSMark Johnston 	vm_unlock_vcpus(sc->vm);
153b9ef152bSMark Johnston }
154b9ef152bSMark Johnston 
155b9ef152bSMark Johnston static struct vmmdev_softc *
156c23da668SMark Johnston vmmdev_lookup(const char *name, struct ucred *cred)
157b9ef152bSMark Johnston {
158b9ef152bSMark Johnston 	struct vmmdev_softc *sc;
159b9ef152bSMark Johnston 
160887c0877SMark Johnston 	sx_assert(&vmmdev_mtx, SA_XLOCKED);
161b9ef152bSMark Johnston 
162b9ef152bSMark Johnston 	SLIST_FOREACH(sc, &head, link) {
163b9ef152bSMark Johnston 		if (strcmp(name, vm_name(sc->vm)) == 0)
164b9ef152bSMark Johnston 			break;
165b9ef152bSMark Johnston 	}
166b9ef152bSMark Johnston 
167b9ef152bSMark Johnston 	if (sc == NULL)
168b9ef152bSMark Johnston 		return (NULL);
169b9ef152bSMark Johnston 
170c23da668SMark Johnston 	if (cr_cansee(cred, sc->ucred))
171b9ef152bSMark Johnston 		return (NULL);
172b9ef152bSMark Johnston 
173b9ef152bSMark Johnston 	return (sc);
174b9ef152bSMark Johnston }
175b9ef152bSMark Johnston 
176b9ef152bSMark Johnston static struct vmmdev_softc *
177b9ef152bSMark Johnston vmmdev_lookup2(struct cdev *cdev)
178b9ef152bSMark Johnston {
179b9ef152bSMark Johnston 	return (cdev->si_drv1);
180b9ef152bSMark Johnston }
181b9ef152bSMark Johnston 
182b9ef152bSMark Johnston static int
183b9ef152bSMark Johnston vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags)
184b9ef152bSMark Johnston {
185b9ef152bSMark Johnston 	int error, off, c, prot;
186b9ef152bSMark Johnston 	vm_paddr_t gpa, maxaddr;
187b9ef152bSMark Johnston 	void *hpa, *cookie;
188b9ef152bSMark Johnston 	struct vmmdev_softc *sc;
189b9ef152bSMark Johnston 
190b9ef152bSMark Johnston 	sc = vmmdev_lookup2(cdev);
191b9ef152bSMark Johnston 	if (sc == NULL)
192b9ef152bSMark Johnston 		return (ENXIO);
193b9ef152bSMark Johnston 
194b9ef152bSMark Johnston 	/*
195b9ef152bSMark Johnston 	 * Get a read lock on the guest memory map.
196b9ef152bSMark Johnston 	 */
197b9ef152bSMark Johnston 	vm_slock_memsegs(sc->vm);
198b9ef152bSMark Johnston 
1997c89253bSJohn Baldwin 	error = 0;
200b9ef152bSMark Johnston 	prot = (uio->uio_rw == UIO_WRITE ? VM_PROT_WRITE : VM_PROT_READ);
201b9ef152bSMark Johnston 	maxaddr = vmm_sysmem_maxaddr(sc->vm);
202b9ef152bSMark Johnston 	while (uio->uio_resid > 0 && error == 0) {
203b9ef152bSMark Johnston 		gpa = uio->uio_offset;
204b9ef152bSMark Johnston 		off = gpa & PAGE_MASK;
205b9ef152bSMark Johnston 		c = min(uio->uio_resid, PAGE_SIZE - off);
206b9ef152bSMark Johnston 
207b9ef152bSMark Johnston 		/*
208b9ef152bSMark Johnston 		 * The VM has a hole in its physical memory map. If we want to
209b9ef152bSMark Johnston 		 * use 'dd' to inspect memory beyond the hole we need to
210b9ef152bSMark Johnston 		 * provide bogus data for memory that lies in the hole.
211b9ef152bSMark Johnston 		 *
212b9ef152bSMark Johnston 		 * Since this device does not support lseek(2), dd(1) will
213b9ef152bSMark Johnston 		 * read(2) blocks of data to simulate the lseek(2).
214b9ef152bSMark Johnston 		 */
215b9ef152bSMark Johnston 		hpa = vm_gpa_hold_global(sc->vm, gpa, c, prot, &cookie);
216b9ef152bSMark Johnston 		if (hpa == NULL) {
217b9ef152bSMark Johnston 			if (uio->uio_rw == UIO_READ && gpa < maxaddr)
218b9ef152bSMark Johnston 				error = uiomove(__DECONST(void *, zero_region),
219b9ef152bSMark Johnston 				    c, uio);
220b9ef152bSMark Johnston 			else
221b9ef152bSMark Johnston 				error = EFAULT;
222b9ef152bSMark Johnston 		} else {
223b9ef152bSMark Johnston 			error = uiomove(hpa, c, uio);
224b9ef152bSMark Johnston 			vm_gpa_release(cookie);
225b9ef152bSMark Johnston 		}
226b9ef152bSMark Johnston 	}
227b9ef152bSMark Johnston 	vm_unlock_memsegs(sc->vm);
228b9ef152bSMark Johnston 	return (error);
229b9ef152bSMark Johnston }
230b9ef152bSMark Johnston 
231b9ef152bSMark Johnston CTASSERT(sizeof(((struct vm_memseg *)0)->name) >= VM_MAX_SUFFIXLEN + 1);
232b9ef152bSMark Johnston 
233b9ef152bSMark Johnston static int
234b9ef152bSMark Johnston get_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len)
235b9ef152bSMark Johnston {
236b9ef152bSMark Johnston 	struct devmem_softc *dsc;
237b9ef152bSMark Johnston 	int error;
238b9ef152bSMark Johnston 	bool sysmem;
239b9ef152bSMark Johnston 
240b9ef152bSMark Johnston 	error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL);
241b9ef152bSMark Johnston 	if (error || mseg->len == 0)
242b9ef152bSMark Johnston 		return (error);
243b9ef152bSMark Johnston 
244b9ef152bSMark Johnston 	if (!sysmem) {
245b9ef152bSMark Johnston 		SLIST_FOREACH(dsc, &sc->devmem, link) {
246b9ef152bSMark Johnston 			if (dsc->segid == mseg->segid)
247b9ef152bSMark Johnston 				break;
248b9ef152bSMark Johnston 		}
249b9ef152bSMark Johnston 		KASSERT(dsc != NULL, ("%s: devmem segment %d not found",
250b9ef152bSMark Johnston 		    __func__, mseg->segid));
251b9ef152bSMark Johnston 		error = copystr(dsc->name, mseg->name, len, NULL);
252b9ef152bSMark Johnston 	} else {
253b9ef152bSMark Johnston 		bzero(mseg->name, len);
254b9ef152bSMark Johnston 	}
255b9ef152bSMark Johnston 
256b9ef152bSMark Johnston 	return (error);
257b9ef152bSMark Johnston }
258b9ef152bSMark Johnston 
259b9ef152bSMark Johnston static int
260b9ef152bSMark Johnston alloc_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len)
261b9ef152bSMark Johnston {
262b9ef152bSMark Johnston 	char *name;
263b9ef152bSMark Johnston 	int error;
264b9ef152bSMark Johnston 	bool sysmem;
265b9ef152bSMark Johnston 
266b9ef152bSMark Johnston 	error = 0;
267b9ef152bSMark Johnston 	name = NULL;
268b9ef152bSMark Johnston 	sysmem = true;
269b9ef152bSMark Johnston 
270b9ef152bSMark Johnston 	/*
271b9ef152bSMark Johnston 	 * The allocation is lengthened by 1 to hold a terminating NUL.  It'll
272b9ef152bSMark Johnston 	 * by stripped off when devfs processes the full string.
273b9ef152bSMark Johnston 	 */
274b9ef152bSMark Johnston 	if (VM_MEMSEG_NAME(mseg)) {
275b9ef152bSMark Johnston 		sysmem = false;
276b9ef152bSMark Johnston 		name = malloc(len, M_VMMDEV, M_WAITOK);
277b9ef152bSMark Johnston 		error = copystr(mseg->name, name, len, NULL);
278b9ef152bSMark Johnston 		if (error)
279b9ef152bSMark Johnston 			goto done;
280b9ef152bSMark Johnston 	}
281b9ef152bSMark Johnston 
282b9ef152bSMark Johnston 	error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem);
283b9ef152bSMark Johnston 	if (error)
284b9ef152bSMark Johnston 		goto done;
285b9ef152bSMark Johnston 
286b9ef152bSMark Johnston 	if (VM_MEMSEG_NAME(mseg)) {
287f4002135SMark Johnston 		error = devmem_create_cdev(sc, mseg->segid, name);
288b9ef152bSMark Johnston 		if (error)
289b9ef152bSMark Johnston 			vm_free_memseg(sc->vm, mseg->segid);
290b9ef152bSMark Johnston 		else
291b9ef152bSMark Johnston 			name = NULL;	/* freed when 'cdev' is destroyed */
292b9ef152bSMark Johnston 	}
293b9ef152bSMark Johnston done:
294b9ef152bSMark Johnston 	free(name, M_VMMDEV);
295b9ef152bSMark Johnston 	return (error);
296b9ef152bSMark Johnston }
297b9ef152bSMark Johnston 
298b9ef152bSMark Johnston static int
299b9ef152bSMark Johnston vm_get_register_set(struct vcpu *vcpu, unsigned int count, int *regnum,
300b9ef152bSMark Johnston     uint64_t *regval)
301b9ef152bSMark Johnston {
302b9ef152bSMark Johnston 	int error, i;
303b9ef152bSMark Johnston 
304b9ef152bSMark Johnston 	error = 0;
305b9ef152bSMark Johnston 	for (i = 0; i < count; i++) {
306b9ef152bSMark Johnston 		error = vm_get_register(vcpu, regnum[i], &regval[i]);
307b9ef152bSMark Johnston 		if (error)
308b9ef152bSMark Johnston 			break;
309b9ef152bSMark Johnston 	}
310b9ef152bSMark Johnston 	return (error);
311b9ef152bSMark Johnston }
312b9ef152bSMark Johnston 
313b9ef152bSMark Johnston static int
314b9ef152bSMark Johnston vm_set_register_set(struct vcpu *vcpu, unsigned int count, int *regnum,
315b9ef152bSMark Johnston     uint64_t *regval)
316b9ef152bSMark Johnston {
317b9ef152bSMark Johnston 	int error, i;
318b9ef152bSMark Johnston 
319b9ef152bSMark Johnston 	error = 0;
320b9ef152bSMark Johnston 	for (i = 0; i < count; i++) {
321b9ef152bSMark Johnston 		error = vm_set_register(vcpu, regnum[i], regval[i]);
322b9ef152bSMark Johnston 		if (error)
323b9ef152bSMark Johnston 			break;
324b9ef152bSMark Johnston 	}
325b9ef152bSMark Johnston 	return (error);
326b9ef152bSMark Johnston }
327b9ef152bSMark Johnston 
32840087581SMark Johnston static int
32940087581SMark Johnston vmmdev_open(struct cdev *dev, int flags, int fmt, struct thread *td)
33040087581SMark Johnston {
33140087581SMark Johnston 	int error;
33240087581SMark Johnston 
33340087581SMark Johnston 	/*
33440087581SMark Johnston 	 * A jail without vmm access shouldn't be able to access vmm device
33540087581SMark Johnston 	 * files at all, but check here just to be thorough.
33640087581SMark Johnston 	 */
33740087581SMark Johnston 	error = vmm_priv_check(td->td_ucred);
33840087581SMark Johnston 	if (error != 0)
33940087581SMark Johnston 		return (error);
34040087581SMark Johnston 
34140087581SMark Johnston 	return (0);
34240087581SMark Johnston }
34340087581SMark Johnston 
344b9ef152bSMark Johnston static const struct vmmdev_ioctl vmmdev_ioctls[] = {
345b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_REGISTER, VMMDEV_IOCTL_LOCK_ONE_VCPU),
346b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SET_REGISTER, VMMDEV_IOCTL_LOCK_ONE_VCPU),
347b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_REGISTER_SET, VMMDEV_IOCTL_LOCK_ONE_VCPU),
348b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SET_REGISTER_SET, VMMDEV_IOCTL_LOCK_ONE_VCPU),
349b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_CAPABILITY, VMMDEV_IOCTL_LOCK_ONE_VCPU),
350b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SET_CAPABILITY, VMMDEV_IOCTL_LOCK_ONE_VCPU),
351b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_ACTIVATE_CPU, VMMDEV_IOCTL_LOCK_ONE_VCPU),
352b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_INJECT_EXCEPTION, VMMDEV_IOCTL_LOCK_ONE_VCPU),
353b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_STATS, VMMDEV_IOCTL_LOCK_ONE_VCPU),
354b9ef152bSMark Johnston 
355b9ef152bSMark Johnston #if defined(__amd64__) && defined(COMPAT_FREEBSD12)
356a852dc58SMark Johnston 	VMMDEV_IOCTL(VM_ALLOC_MEMSEG_12,
357b9ef152bSMark Johnston 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS),
358b9ef152bSMark Johnston #endif
359b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_ALLOC_MEMSEG,
360b9ef152bSMark Johnston 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS),
361b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_MMAP_MEMSEG,
362b9ef152bSMark Johnston 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS),
363b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_MUNMAP_MEMSEG,
364b9ef152bSMark Johnston 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS),
365b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_REINIT,
366b9ef152bSMark Johnston 	    VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS),
367b9ef152bSMark Johnston 
368b9ef152bSMark Johnston #if defined(__amd64__) && defined(COMPAT_FREEBSD12)
369a852dc58SMark Johnston 	VMMDEV_IOCTL(VM_GET_MEMSEG_12, VMMDEV_IOCTL_SLOCK_MEMSEGS),
370b9ef152bSMark Johnston #endif
371b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_MEMSEG, VMMDEV_IOCTL_SLOCK_MEMSEGS),
372b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_MMAP_GETNEXT, VMMDEV_IOCTL_SLOCK_MEMSEGS),
373b9ef152bSMark Johnston 
374b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SUSPEND_CPU, VMMDEV_IOCTL_MAYBE_ALLOC_VCPU),
375b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_RESUME_CPU, VMMDEV_IOCTL_MAYBE_ALLOC_VCPU),
376b9ef152bSMark Johnston 
377b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SUSPEND, 0),
378b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_CPUS, 0),
379b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_GET_TOPOLOGY, 0),
380b9ef152bSMark Johnston 	VMMDEV_IOCTL(VM_SET_TOPOLOGY, 0),
381b9ef152bSMark Johnston };
382b9ef152bSMark Johnston 
383b9ef152bSMark Johnston static int
384b9ef152bSMark Johnston vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
385b9ef152bSMark Johnston     struct thread *td)
386b9ef152bSMark Johnston {
387b9ef152bSMark Johnston 	struct vmmdev_softc *sc;
388b9ef152bSMark Johnston 	struct vcpu *vcpu;
389b9ef152bSMark Johnston 	const struct vmmdev_ioctl *ioctl;
390b9ef152bSMark Johnston 	int error, vcpuid;
391b9ef152bSMark Johnston 
392b9ef152bSMark Johnston 	sc = vmmdev_lookup2(cdev);
393b9ef152bSMark Johnston 	if (sc == NULL)
394b9ef152bSMark Johnston 		return (ENXIO);
395b9ef152bSMark Johnston 
396b9ef152bSMark Johnston 	ioctl = NULL;
397b9ef152bSMark Johnston 	for (size_t i = 0; i < nitems(vmmdev_ioctls); i++) {
398b9ef152bSMark Johnston 		if (vmmdev_ioctls[i].cmd == cmd) {
399b9ef152bSMark Johnston 			ioctl = &vmmdev_ioctls[i];
400b9ef152bSMark Johnston 			break;
401b9ef152bSMark Johnston 		}
402b9ef152bSMark Johnston 	}
403b9ef152bSMark Johnston 	if (ioctl == NULL) {
404b9ef152bSMark Johnston 		for (size_t i = 0; i < vmmdev_machdep_ioctl_count; i++) {
405b9ef152bSMark Johnston 			if (vmmdev_machdep_ioctls[i].cmd == cmd) {
406b9ef152bSMark Johnston 				ioctl = &vmmdev_machdep_ioctls[i];
407b9ef152bSMark Johnston 				break;
408b9ef152bSMark Johnston 			}
409b9ef152bSMark Johnston 		}
410b9ef152bSMark Johnston 	}
411b9ef152bSMark Johnston 	if (ioctl == NULL)
412b9ef152bSMark Johnston 		return (ENOTTY);
413b9ef152bSMark Johnston 
414b9ef152bSMark Johnston 	if ((ioctl->flags & VMMDEV_IOCTL_XLOCK_MEMSEGS) != 0)
415b9ef152bSMark Johnston 		vm_xlock_memsegs(sc->vm);
416b9ef152bSMark Johnston 	else if ((ioctl->flags & VMMDEV_IOCTL_SLOCK_MEMSEGS) != 0)
417b9ef152bSMark Johnston 		vm_slock_memsegs(sc->vm);
418b9ef152bSMark Johnston 
419b9ef152bSMark Johnston 	vcpu = NULL;
420b9ef152bSMark Johnston 	vcpuid = -1;
421b9ef152bSMark Johnston 	if ((ioctl->flags & (VMMDEV_IOCTL_LOCK_ONE_VCPU |
422b9ef152bSMark Johnston 	    VMMDEV_IOCTL_ALLOC_VCPU | VMMDEV_IOCTL_MAYBE_ALLOC_VCPU)) != 0) {
423b9ef152bSMark Johnston 		vcpuid = *(int *)data;
424b9ef152bSMark Johnston 		if (vcpuid == -1) {
425b9ef152bSMark Johnston 			if ((ioctl->flags &
426b9ef152bSMark Johnston 			    VMMDEV_IOCTL_MAYBE_ALLOC_VCPU) == 0) {
427b9ef152bSMark Johnston 				error = EINVAL;
428b9ef152bSMark Johnston 				goto lockfail;
429b9ef152bSMark Johnston 			}
430b9ef152bSMark Johnston 		} else {
431b9ef152bSMark Johnston 			vcpu = vm_alloc_vcpu(sc->vm, vcpuid);
432b9ef152bSMark Johnston 			if (vcpu == NULL) {
433b9ef152bSMark Johnston 				error = EINVAL;
434b9ef152bSMark Johnston 				goto lockfail;
435b9ef152bSMark Johnston 			}
436b9ef152bSMark Johnston 			if ((ioctl->flags & VMMDEV_IOCTL_LOCK_ONE_VCPU) != 0) {
437b9ef152bSMark Johnston 				error = vcpu_lock_one(vcpu);
438b9ef152bSMark Johnston 				if (error)
439b9ef152bSMark Johnston 					goto lockfail;
440b9ef152bSMark Johnston 			}
441b9ef152bSMark Johnston 		}
442b9ef152bSMark Johnston 	}
443b9ef152bSMark Johnston 	if ((ioctl->flags & VMMDEV_IOCTL_LOCK_ALL_VCPUS) != 0) {
444b9ef152bSMark Johnston 		error = vcpu_lock_all(sc);
445b9ef152bSMark Johnston 		if (error)
446b9ef152bSMark Johnston 			goto lockfail;
447b9ef152bSMark Johnston 	}
448b9ef152bSMark Johnston 
449b9ef152bSMark Johnston 	switch (cmd) {
450b9ef152bSMark Johnston 	case VM_SUSPEND: {
451b9ef152bSMark Johnston 		struct vm_suspend *vmsuspend;
452b9ef152bSMark Johnston 
453b9ef152bSMark Johnston 		vmsuspend = (struct vm_suspend *)data;
454b9ef152bSMark Johnston 		error = vm_suspend(sc->vm, vmsuspend->how);
455b9ef152bSMark Johnston 		break;
456b9ef152bSMark Johnston 	}
457b9ef152bSMark Johnston 	case VM_REINIT:
458b9ef152bSMark Johnston 		error = vm_reinit(sc->vm);
459b9ef152bSMark Johnston 		break;
460b9ef152bSMark Johnston 	case VM_STAT_DESC: {
461b9ef152bSMark Johnston 		struct vm_stat_desc *statdesc;
462b9ef152bSMark Johnston 
463b9ef152bSMark Johnston 		statdesc = (struct vm_stat_desc *)data;
464b9ef152bSMark Johnston 		error = vmm_stat_desc_copy(statdesc->index, statdesc->desc,
465b9ef152bSMark Johnston 		    sizeof(statdesc->desc));
466b9ef152bSMark Johnston 		break;
467b9ef152bSMark Johnston 	}
468b9ef152bSMark Johnston 	case VM_STATS: {
469b9ef152bSMark Johnston 		struct vm_stats *vmstats;
470b9ef152bSMark Johnston 
471b9ef152bSMark Johnston 		vmstats = (struct vm_stats *)data;
472b9ef152bSMark Johnston 		getmicrotime(&vmstats->tv);
473b9ef152bSMark Johnston 		error = vmm_stat_copy(vcpu, vmstats->index,
474b9ef152bSMark Johnston 		    nitems(vmstats->statbuf), &vmstats->num_entries,
475b9ef152bSMark Johnston 		    vmstats->statbuf);
476b9ef152bSMark Johnston 		break;
477b9ef152bSMark Johnston 	}
478b9ef152bSMark Johnston 	case VM_MMAP_GETNEXT: {
479b9ef152bSMark Johnston 		struct vm_memmap *mm;
480b9ef152bSMark Johnston 
481b9ef152bSMark Johnston 		mm = (struct vm_memmap *)data;
482b9ef152bSMark Johnston 		error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid,
483b9ef152bSMark Johnston 		    &mm->segoff, &mm->len, &mm->prot, &mm->flags);
484b9ef152bSMark Johnston 		break;
485b9ef152bSMark Johnston 	}
486b9ef152bSMark Johnston 	case VM_MMAP_MEMSEG: {
487b9ef152bSMark Johnston 		struct vm_memmap *mm;
488b9ef152bSMark Johnston 
489b9ef152bSMark Johnston 		mm = (struct vm_memmap *)data;
490b9ef152bSMark Johnston 		error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff,
491b9ef152bSMark Johnston 		    mm->len, mm->prot, mm->flags);
492b9ef152bSMark Johnston 		break;
493b9ef152bSMark Johnston 	}
494b9ef152bSMark Johnston 	case VM_MUNMAP_MEMSEG: {
495b9ef152bSMark Johnston 		struct vm_munmap *mu;
496b9ef152bSMark Johnston 
497b9ef152bSMark Johnston 		mu = (struct vm_munmap *)data;
498b9ef152bSMark Johnston 		error = vm_munmap_memseg(sc->vm, mu->gpa, mu->len);
499b9ef152bSMark Johnston 		break;
500b9ef152bSMark Johnston 	}
501b9ef152bSMark Johnston #if defined(__amd64__) && defined(COMPAT_FREEBSD12)
502a852dc58SMark Johnston 	case VM_ALLOC_MEMSEG_12:
503b9ef152bSMark Johnston 		error = alloc_memseg(sc, (struct vm_memseg *)data,
504a852dc58SMark Johnston 		    sizeof(((struct vm_memseg_12 *)0)->name));
505b9ef152bSMark Johnston 		break;
506a852dc58SMark Johnston 	case VM_GET_MEMSEG_12:
507b9ef152bSMark Johnston 		error = get_memseg(sc, (struct vm_memseg *)data,
508a852dc58SMark Johnston 		    sizeof(((struct vm_memseg_12 *)0)->name));
509b9ef152bSMark Johnston 		break;
510b9ef152bSMark Johnston #endif
511b9ef152bSMark Johnston 	case VM_ALLOC_MEMSEG:
512b9ef152bSMark Johnston 		error = alloc_memseg(sc, (struct vm_memseg *)data,
513b9ef152bSMark Johnston 		    sizeof(((struct vm_memseg *)0)->name));
514b9ef152bSMark Johnston 		break;
515b9ef152bSMark Johnston 	case VM_GET_MEMSEG:
516b9ef152bSMark Johnston 		error = get_memseg(sc, (struct vm_memseg *)data,
517b9ef152bSMark Johnston 		    sizeof(((struct vm_memseg *)0)->name));
518b9ef152bSMark Johnston 		break;
519b9ef152bSMark Johnston 	case VM_GET_REGISTER: {
520b9ef152bSMark Johnston 		struct vm_register *vmreg;
521b9ef152bSMark Johnston 
522b9ef152bSMark Johnston 		vmreg = (struct vm_register *)data;
523b9ef152bSMark Johnston 		error = vm_get_register(vcpu, vmreg->regnum, &vmreg->regval);
524b9ef152bSMark Johnston 		break;
525b9ef152bSMark Johnston 	}
526b9ef152bSMark Johnston 	case VM_SET_REGISTER: {
527b9ef152bSMark Johnston 		struct vm_register *vmreg;
528b9ef152bSMark Johnston 
529b9ef152bSMark Johnston 		vmreg = (struct vm_register *)data;
530b9ef152bSMark Johnston 		error = vm_set_register(vcpu, vmreg->regnum, vmreg->regval);
531b9ef152bSMark Johnston 		break;
532b9ef152bSMark Johnston 	}
533b9ef152bSMark Johnston 	case VM_GET_REGISTER_SET: {
534b9ef152bSMark Johnston 		struct vm_register_set *vmregset;
535b9ef152bSMark Johnston 		uint64_t *regvals;
536b9ef152bSMark Johnston 		int *regnums;
537b9ef152bSMark Johnston 
538b9ef152bSMark Johnston 		vmregset = (struct vm_register_set *)data;
539b9ef152bSMark Johnston 		if (vmregset->count > VM_REG_LAST) {
540b9ef152bSMark Johnston 			error = EINVAL;
541b9ef152bSMark Johnston 			break;
542b9ef152bSMark Johnston 		}
543b9ef152bSMark Johnston 		regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
544b9ef152bSMark Johnston 		    M_WAITOK);
545b9ef152bSMark Johnston 		regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
546b9ef152bSMark Johnston 		    M_WAITOK);
547b9ef152bSMark Johnston 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
548b9ef152bSMark Johnston 		    vmregset->count);
549b9ef152bSMark Johnston 		if (error == 0)
550b9ef152bSMark Johnston 			error = vm_get_register_set(vcpu,
551b9ef152bSMark Johnston 			    vmregset->count, regnums, regvals);
552b9ef152bSMark Johnston 		if (error == 0)
553b9ef152bSMark Johnston 			error = copyout(regvals, vmregset->regvals,
554b9ef152bSMark Johnston 			    sizeof(regvals[0]) * vmregset->count);
555b9ef152bSMark Johnston 		free(regvals, M_VMMDEV);
556b9ef152bSMark Johnston 		free(regnums, M_VMMDEV);
557b9ef152bSMark Johnston 		break;
558b9ef152bSMark Johnston 	}
559b9ef152bSMark Johnston 	case VM_SET_REGISTER_SET: {
560b9ef152bSMark Johnston 		struct vm_register_set *vmregset;
561b9ef152bSMark Johnston 		uint64_t *regvals;
562b9ef152bSMark Johnston 		int *regnums;
563b9ef152bSMark Johnston 
564b9ef152bSMark Johnston 		vmregset = (struct vm_register_set *)data;
565b9ef152bSMark Johnston 		if (vmregset->count > VM_REG_LAST) {
566b9ef152bSMark Johnston 			error = EINVAL;
567b9ef152bSMark Johnston 			break;
568b9ef152bSMark Johnston 		}
569b9ef152bSMark Johnston 		regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
570b9ef152bSMark Johnston 		    M_WAITOK);
571b9ef152bSMark Johnston 		regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
572b9ef152bSMark Johnston 		    M_WAITOK);
573b9ef152bSMark Johnston 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
574b9ef152bSMark Johnston 		    vmregset->count);
575b9ef152bSMark Johnston 		if (error == 0)
576b9ef152bSMark Johnston 			error = copyin(vmregset->regvals, regvals,
577b9ef152bSMark Johnston 			    sizeof(regvals[0]) * vmregset->count);
578b9ef152bSMark Johnston 		if (error == 0)
579b9ef152bSMark Johnston 			error = vm_set_register_set(vcpu,
580b9ef152bSMark Johnston 			    vmregset->count, regnums, regvals);
581b9ef152bSMark Johnston 		free(regvals, M_VMMDEV);
582b9ef152bSMark Johnston 		free(regnums, M_VMMDEV);
583b9ef152bSMark Johnston 		break;
584b9ef152bSMark Johnston 	}
585b9ef152bSMark Johnston 	case VM_GET_CAPABILITY: {
586b9ef152bSMark Johnston 		struct vm_capability *vmcap;
587b9ef152bSMark Johnston 
588b9ef152bSMark Johnston 		vmcap = (struct vm_capability *)data;
589b9ef152bSMark Johnston 		error = vm_get_capability(vcpu, vmcap->captype, &vmcap->capval);
590b9ef152bSMark Johnston 		break;
591b9ef152bSMark Johnston 	}
592b9ef152bSMark Johnston 	case VM_SET_CAPABILITY: {
593b9ef152bSMark Johnston 		struct vm_capability *vmcap;
594b9ef152bSMark Johnston 
595b9ef152bSMark Johnston 		vmcap = (struct vm_capability *)data;
596b9ef152bSMark Johnston 		error = vm_set_capability(vcpu, vmcap->captype, vmcap->capval);
597b9ef152bSMark Johnston 		break;
598b9ef152bSMark Johnston 	}
599b9ef152bSMark Johnston 	case VM_ACTIVATE_CPU:
600b9ef152bSMark Johnston 		error = vm_activate_cpu(vcpu);
601b9ef152bSMark Johnston 		break;
602b9ef152bSMark Johnston 	case VM_GET_CPUS: {
603b9ef152bSMark Johnston 		struct vm_cpuset *vm_cpuset;
604b9ef152bSMark Johnston 		cpuset_t *cpuset;
605b9ef152bSMark Johnston 		int size;
606b9ef152bSMark Johnston 
607b9ef152bSMark Johnston 		error = 0;
608b9ef152bSMark Johnston 		vm_cpuset = (struct vm_cpuset *)data;
609b9ef152bSMark Johnston 		size = vm_cpuset->cpusetsize;
610b9ef152bSMark Johnston 		if (size < 1 || size > CPU_MAXSIZE / NBBY) {
611b9ef152bSMark Johnston 			error = ERANGE;
612b9ef152bSMark Johnston 			break;
613b9ef152bSMark Johnston 		}
614b9ef152bSMark Johnston 		cpuset = malloc(max(size, sizeof(cpuset_t)), M_TEMP,
615b9ef152bSMark Johnston 		    M_WAITOK | M_ZERO);
616b9ef152bSMark Johnston 		if (vm_cpuset->which == VM_ACTIVE_CPUS)
617b9ef152bSMark Johnston 			*cpuset = vm_active_cpus(sc->vm);
618b9ef152bSMark Johnston 		else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
619b9ef152bSMark Johnston 			*cpuset = vm_suspended_cpus(sc->vm);
620b9ef152bSMark Johnston 		else if (vm_cpuset->which == VM_DEBUG_CPUS)
621b9ef152bSMark Johnston 			*cpuset = vm_debug_cpus(sc->vm);
622b9ef152bSMark Johnston 		else
623b9ef152bSMark Johnston 			error = EINVAL;
624b9ef152bSMark Johnston 		if (error == 0 && size < howmany(CPU_FLS(cpuset), NBBY))
625b9ef152bSMark Johnston 			error = ERANGE;
626b9ef152bSMark Johnston 		if (error == 0)
627b9ef152bSMark Johnston 			error = copyout(cpuset, vm_cpuset->cpus, size);
628b9ef152bSMark Johnston 		free(cpuset, M_TEMP);
629b9ef152bSMark Johnston 		break;
630b9ef152bSMark Johnston 	}
631b9ef152bSMark Johnston 	case VM_SUSPEND_CPU:
632b9ef152bSMark Johnston 		error = vm_suspend_cpu(sc->vm, vcpu);
633b9ef152bSMark Johnston 		break;
634b9ef152bSMark Johnston 	case VM_RESUME_CPU:
635b9ef152bSMark Johnston 		error = vm_resume_cpu(sc->vm, vcpu);
636b9ef152bSMark Johnston 		break;
637b9ef152bSMark Johnston 	case VM_SET_TOPOLOGY: {
638b9ef152bSMark Johnston 		struct vm_cpu_topology *topology;
639b9ef152bSMark Johnston 
640b9ef152bSMark Johnston 		topology = (struct vm_cpu_topology *)data;
641b9ef152bSMark Johnston 		error = vm_set_topology(sc->vm, topology->sockets,
642b9ef152bSMark Johnston 		    topology->cores, topology->threads, topology->maxcpus);
643b9ef152bSMark Johnston 		break;
644b9ef152bSMark Johnston 	}
645b9ef152bSMark Johnston 	case VM_GET_TOPOLOGY: {
646b9ef152bSMark Johnston 		struct vm_cpu_topology *topology;
647b9ef152bSMark Johnston 
648b9ef152bSMark Johnston 		topology = (struct vm_cpu_topology *)data;
649b9ef152bSMark Johnston 		vm_get_topology(sc->vm, &topology->sockets, &topology->cores,
650b9ef152bSMark Johnston 		    &topology->threads, &topology->maxcpus);
651b9ef152bSMark Johnston 		error = 0;
652b9ef152bSMark Johnston 		break;
653b9ef152bSMark Johnston 	}
654b9ef152bSMark Johnston 	default:
655b9ef152bSMark Johnston 		error = vmmdev_machdep_ioctl(sc->vm, vcpu, cmd, data, fflag,
656b9ef152bSMark Johnston 		    td);
657b9ef152bSMark Johnston 		break;
658b9ef152bSMark Johnston 	}
659b9ef152bSMark Johnston 
660b9ef152bSMark Johnston 	if ((ioctl->flags &
661b9ef152bSMark Johnston 	    (VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_SLOCK_MEMSEGS)) != 0)
662b9ef152bSMark Johnston 		vm_unlock_memsegs(sc->vm);
663b9ef152bSMark Johnston 	if ((ioctl->flags & VMMDEV_IOCTL_LOCK_ALL_VCPUS) != 0)
664b9ef152bSMark Johnston 		vcpu_unlock_all(sc);
665b9ef152bSMark Johnston 	else if ((ioctl->flags & VMMDEV_IOCTL_LOCK_ONE_VCPU) != 0)
666b9ef152bSMark Johnston 		vcpu_unlock_one(vcpu);
667b9ef152bSMark Johnston 
668b9ef152bSMark Johnston 	/*
669b9ef152bSMark Johnston 	 * Make sure that no handler returns a kernel-internal
670b9ef152bSMark Johnston 	 * error value to userspace.
671b9ef152bSMark Johnston 	 */
672b9ef152bSMark Johnston 	KASSERT(error == ERESTART || error >= 0,
673b9ef152bSMark Johnston 	    ("vmmdev_ioctl: invalid error return %d", error));
674b9ef152bSMark Johnston 	return (error);
675b9ef152bSMark Johnston 
676b9ef152bSMark Johnston lockfail:
677b9ef152bSMark Johnston 	if ((ioctl->flags &
678b9ef152bSMark Johnston 	    (VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_SLOCK_MEMSEGS)) != 0)
679b9ef152bSMark Johnston 		vm_unlock_memsegs(sc->vm);
680b9ef152bSMark Johnston 	return (error);
681b9ef152bSMark Johnston }
682b9ef152bSMark Johnston 
683b9ef152bSMark Johnston static int
684b9ef152bSMark Johnston vmmdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize,
685b9ef152bSMark Johnston     struct vm_object **objp, int nprot)
686b9ef152bSMark Johnston {
687b9ef152bSMark Johnston 	struct vmmdev_softc *sc;
688b9ef152bSMark Johnston 	vm_paddr_t gpa;
689b9ef152bSMark Johnston 	size_t len;
690b9ef152bSMark Johnston 	vm_ooffset_t segoff, first, last;
691b9ef152bSMark Johnston 	int error, found, segid;
692b9ef152bSMark Johnston 	bool sysmem;
693b9ef152bSMark Johnston 
694b9ef152bSMark Johnston 	first = *offset;
695b9ef152bSMark Johnston 	last = first + mapsize;
696b9ef152bSMark Johnston 	if ((nprot & PROT_EXEC) || first < 0 || first >= last)
697b9ef152bSMark Johnston 		return (EINVAL);
698b9ef152bSMark Johnston 
699b9ef152bSMark Johnston 	sc = vmmdev_lookup2(cdev);
700b9ef152bSMark Johnston 	if (sc == NULL) {
701b9ef152bSMark Johnston 		/* virtual machine is in the process of being created */
702b9ef152bSMark Johnston 		return (EINVAL);
703b9ef152bSMark Johnston 	}
704b9ef152bSMark Johnston 
705b9ef152bSMark Johnston 	/*
706b9ef152bSMark Johnston 	 * Get a read lock on the guest memory map.
707b9ef152bSMark Johnston 	 */
708b9ef152bSMark Johnston 	vm_slock_memsegs(sc->vm);
709b9ef152bSMark Johnston 
710b9ef152bSMark Johnston 	gpa = 0;
711b9ef152bSMark Johnston 	found = 0;
712b9ef152bSMark Johnston 	while (!found) {
713b9ef152bSMark Johnston 		error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len,
714b9ef152bSMark Johnston 		    NULL, NULL);
715b9ef152bSMark Johnston 		if (error)
716b9ef152bSMark Johnston 			break;
717b9ef152bSMark Johnston 
718b9ef152bSMark Johnston 		if (first >= gpa && last <= gpa + len)
719b9ef152bSMark Johnston 			found = 1;
720b9ef152bSMark Johnston 		else
721b9ef152bSMark Johnston 			gpa += len;
722b9ef152bSMark Johnston 	}
723b9ef152bSMark Johnston 
724b9ef152bSMark Johnston 	if (found) {
725b9ef152bSMark Johnston 		error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp);
726b9ef152bSMark Johnston 		KASSERT(error == 0 && *objp != NULL,
727b9ef152bSMark Johnston 		    ("%s: invalid memory segment %d", __func__, segid));
728b9ef152bSMark Johnston 		if (sysmem) {
729b9ef152bSMark Johnston 			vm_object_reference(*objp);
730b9ef152bSMark Johnston 			*offset = segoff + (first - gpa);
731b9ef152bSMark Johnston 		} else {
732b9ef152bSMark Johnston 			error = EINVAL;
733b9ef152bSMark Johnston 		}
734b9ef152bSMark Johnston 	}
735b9ef152bSMark Johnston 	vm_unlock_memsegs(sc->vm);
736b9ef152bSMark Johnston 	return (error);
737b9ef152bSMark Johnston }
738b9ef152bSMark Johnston 
739b9ef152bSMark Johnston static void
740063a8bd9SMark Johnston vmmdev_destroy(struct vmmdev_softc *sc)
741b9ef152bSMark Johnston {
742b9ef152bSMark Johnston 	struct devmem_softc *dsc;
743b9ef152bSMark Johnston 	int error __diagused;
744b9ef152bSMark Johnston 
745cef5f43fSMark Johnston 	KASSERT(sc->cdev == NULL, ("%s: cdev not free", __func__));
746cef5f43fSMark Johnston 
747063a8bd9SMark Johnston 	/*
748063a8bd9SMark Johnston 	 * Destroy all cdevs:
749063a8bd9SMark Johnston 	 *
750063a8bd9SMark Johnston 	 * - any new operations on the 'cdev' will return an error (ENXIO).
751063a8bd9SMark Johnston 	 *
752063a8bd9SMark Johnston 	 * - the 'devmem' cdevs are destroyed before the virtual machine 'cdev'
753063a8bd9SMark Johnston 	 */
754063a8bd9SMark Johnston 	SLIST_FOREACH(dsc, &sc->devmem, link) {
755063a8bd9SMark Johnston 		KASSERT(dsc->cdev != NULL, ("devmem cdev already destroyed"));
756063a8bd9SMark Johnston 		devmem_destroy(dsc);
757063a8bd9SMark Johnston 	}
758063a8bd9SMark Johnston 
759b9ef152bSMark Johnston 	vm_disable_vcpu_creation(sc->vm);
760b9ef152bSMark Johnston 	error = vcpu_lock_all(sc);
761b9ef152bSMark Johnston 	KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error));
762b9ef152bSMark Johnston 	vm_unlock_vcpus(sc->vm);
763b9ef152bSMark Johnston 
764b9ef152bSMark Johnston 	while ((dsc = SLIST_FIRST(&sc->devmem)) != NULL) {
765b9ef152bSMark Johnston 		KASSERT(dsc->cdev == NULL, ("%s: devmem not free", __func__));
766b9ef152bSMark Johnston 		SLIST_REMOVE_HEAD(&sc->devmem, link);
767b9ef152bSMark Johnston 		free(dsc->name, M_VMMDEV);
768b9ef152bSMark Johnston 		free(dsc, M_VMMDEV);
769b9ef152bSMark Johnston 	}
770b9ef152bSMark Johnston 
771b9ef152bSMark Johnston 	if (sc->vm != NULL)
772b9ef152bSMark Johnston 		vm_destroy(sc->vm);
773b9ef152bSMark Johnston 
774b9ef152bSMark Johnston 	if (sc->ucred != NULL)
775b9ef152bSMark Johnston 		crfree(sc->ucred);
776b9ef152bSMark Johnston 
777887c0877SMark Johnston 	sx_xlock(&vmmdev_mtx);
778b9ef152bSMark Johnston 	SLIST_REMOVE(&head, sc, vmmdev_softc, link);
779887c0877SMark Johnston 	sx_xunlock(&vmmdev_mtx);
780b9ef152bSMark Johnston 	free(sc, M_VMMDEV);
781b9ef152bSMark Johnston }
782b9ef152bSMark Johnston 
783b9ef152bSMark Johnston static int
784063a8bd9SMark Johnston vmmdev_lookup_and_destroy(const char *name, struct ucred *cred)
785063a8bd9SMark Johnston {
786063a8bd9SMark Johnston 	struct cdev *cdev;
787063a8bd9SMark Johnston 	struct vmmdev_softc *sc;
788063a8bd9SMark Johnston 
789887c0877SMark Johnston 	sx_xlock(&vmmdev_mtx);
790c23da668SMark Johnston 	sc = vmmdev_lookup(name, cred);
791063a8bd9SMark Johnston 	if (sc == NULL || sc->cdev == NULL) {
792887c0877SMark Johnston 		sx_xunlock(&vmmdev_mtx);
793063a8bd9SMark Johnston 		return (EINVAL);
794063a8bd9SMark Johnston 	}
795063a8bd9SMark Johnston 
796063a8bd9SMark Johnston 	/*
797063a8bd9SMark Johnston 	 * Setting 'sc->cdev' to NULL is used to indicate that the VM
798063a8bd9SMark Johnston 	 * is scheduled for destruction.
799063a8bd9SMark Johnston 	 */
800063a8bd9SMark Johnston 	cdev = sc->cdev;
801063a8bd9SMark Johnston 	sc->cdev = NULL;
802887c0877SMark Johnston 	sx_xunlock(&vmmdev_mtx);
803063a8bd9SMark Johnston 
804063a8bd9SMark Johnston 	destroy_dev(cdev);
805063a8bd9SMark Johnston 	vmmdev_destroy(sc);
806063a8bd9SMark Johnston 
807063a8bd9SMark Johnston 	return (0);
808063a8bd9SMark Johnston }
809063a8bd9SMark Johnston 
810063a8bd9SMark Johnston static int
811b9ef152bSMark Johnston sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS)
812b9ef152bSMark Johnston {
813b9ef152bSMark Johnston 	char *buf;
814b9ef152bSMark Johnston 	int error, buflen;
815b9ef152bSMark Johnston 
816b9ef152bSMark Johnston 	error = vmm_priv_check(req->td->td_ucred);
817b9ef152bSMark Johnston 	if (error)
818b9ef152bSMark Johnston 		return (error);
819b9ef152bSMark Johnston 
820b9ef152bSMark Johnston 	buflen = VM_MAX_NAMELEN + 1;
821b9ef152bSMark Johnston 	buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
822b9ef152bSMark Johnston 	strlcpy(buf, "beavis", buflen);
823b9ef152bSMark Johnston 	error = sysctl_handle_string(oidp, buf, buflen, req);
824063a8bd9SMark Johnston 	if (error == 0 && req->newptr != NULL)
825063a8bd9SMark Johnston 		error = vmmdev_lookup_and_destroy(buf, req->td->td_ucred);
826b9ef152bSMark Johnston 	free(buf, M_VMMDEV);
827b9ef152bSMark Johnston 	return (error);
828b9ef152bSMark Johnston }
829b9ef152bSMark Johnston SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy,
830b9ef152bSMark Johnston     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
831b9ef152bSMark Johnston     NULL, 0, sysctl_vmm_destroy, "A",
832b9ef152bSMark Johnston     NULL);
833b9ef152bSMark Johnston 
834b9ef152bSMark Johnston static struct cdevsw vmmdevsw = {
835b9ef152bSMark Johnston 	.d_name		= "vmmdev",
836b9ef152bSMark Johnston 	.d_version	= D_VERSION,
83740087581SMark Johnston 	.d_open		= vmmdev_open,
838b9ef152bSMark Johnston 	.d_ioctl	= vmmdev_ioctl,
839b9ef152bSMark Johnston 	.d_mmap_single	= vmmdev_mmap_single,
840b9ef152bSMark Johnston 	.d_read		= vmmdev_rw,
841b9ef152bSMark Johnston 	.d_write	= vmmdev_rw,
842b9ef152bSMark Johnston };
843b9ef152bSMark Johnston 
844d5819709SMark Johnston static struct vmmdev_softc *
845d5819709SMark Johnston vmmdev_alloc(struct vm *vm, struct ucred *cred)
846b9ef152bSMark Johnston {
847d5819709SMark Johnston 	struct vmmdev_softc *sc;
848b9ef152bSMark Johnston 
849d5819709SMark Johnston 	sc = malloc(sizeof(*sc), M_VMMDEV, M_WAITOK | M_ZERO);
850d5819709SMark Johnston 	SLIST_INIT(&sc->devmem);
851d5819709SMark Johnston 	sc->vm = vm;
852d5819709SMark Johnston 	sc->ucred = crhold(cred);
853d5819709SMark Johnston 	return (sc);
854b9ef152bSMark Johnston }
855b9ef152bSMark Johnston 
856d5819709SMark Johnston static int
857d5819709SMark Johnston vmmdev_create(const char *name, struct ucred *cred)
858d5819709SMark Johnston {
859cef5f43fSMark Johnston 	struct make_dev_args mda;
860d5819709SMark Johnston 	struct cdev *cdev;
861cef5f43fSMark Johnston 	struct vmmdev_softc *sc;
862d5819709SMark Johnston 	struct vm *vm;
863d5819709SMark Johnston 	int error;
864b9ef152bSMark Johnston 
865887c0877SMark Johnston 	sx_xlock(&vmmdev_mtx);
866c23da668SMark Johnston 	sc = vmmdev_lookup(name, cred);
867cef5f43fSMark Johnston 	if (sc != NULL) {
868887c0877SMark Johnston 		sx_xunlock(&vmmdev_mtx);
869d5819709SMark Johnston 		return (EEXIST);
870cef5f43fSMark Johnston 	}
871d5819709SMark Johnston 
872d5819709SMark Johnston 	error = vm_create(name, &vm);
873b9ef152bSMark Johnston 	if (error != 0) {
874cef5f43fSMark Johnston 		sx_xunlock(&vmmdev_mtx);
875cef5f43fSMark Johnston 		return (error);
876cef5f43fSMark Johnston 	}
877cef5f43fSMark Johnston 	sc = vmmdev_alloc(vm, cred);
878cef5f43fSMark Johnston 	SLIST_INSERT_HEAD(&head, sc, link);
879cef5f43fSMark Johnston 
880cef5f43fSMark Johnston 	make_dev_args_init(&mda);
881cef5f43fSMark Johnston 	mda.mda_devsw = &vmmdevsw;
882cef5f43fSMark Johnston 	mda.mda_cr = sc->ucred;
883cef5f43fSMark Johnston 	mda.mda_uid = UID_ROOT;
884cef5f43fSMark Johnston 	mda.mda_gid = GID_WHEEL;
885cef5f43fSMark Johnston 	mda.mda_mode = 0600;
886cef5f43fSMark Johnston 	mda.mda_si_drv1 = sc;
887cef5f43fSMark Johnston 	mda.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
888cef5f43fSMark Johnston 	error = make_dev_s(&mda, &cdev, "vmm/%s", name);
889cef5f43fSMark Johnston 	if (error != 0) {
890cef5f43fSMark Johnston 		sx_xunlock(&vmmdev_mtx);
891b9ef152bSMark Johnston 		vmmdev_destroy(sc);
892d5819709SMark Johnston 		return (error);
893b9ef152bSMark Johnston 	}
894b9ef152bSMark Johnston 	sc->cdev = cdev;
895887c0877SMark Johnston 	sx_xunlock(&vmmdev_mtx);
896d5819709SMark Johnston 	return (0);
897d5819709SMark Johnston }
898d5819709SMark Johnston 
899d5819709SMark Johnston static int
900d5819709SMark Johnston sysctl_vmm_create(SYSCTL_HANDLER_ARGS)
901d5819709SMark Johnston {
902d5819709SMark Johnston 	char *buf;
903d5819709SMark Johnston 	int error, buflen;
904d5819709SMark Johnston 
905d5819709SMark Johnston 	error = vmm_priv_check(req->td->td_ucred);
906d5819709SMark Johnston 	if (error != 0)
907d5819709SMark Johnston 		return (error);
908d5819709SMark Johnston 
909d5819709SMark Johnston 	buflen = VM_MAX_NAMELEN + 1;
910d5819709SMark Johnston 	buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
911d5819709SMark Johnston 	strlcpy(buf, "beavis", buflen);
912d5819709SMark Johnston 	error = sysctl_handle_string(oidp, buf, buflen, req);
913d5819709SMark Johnston 	if (error == 0 && req->newptr != NULL)
914d5819709SMark Johnston 		error = vmmdev_create(buf, req->td->td_ucred);
915b9ef152bSMark Johnston 	free(buf, M_VMMDEV);
916b9ef152bSMark Johnston 	return (error);
917b9ef152bSMark Johnston }
918b9ef152bSMark Johnston SYSCTL_PROC(_hw_vmm, OID_AUTO, create,
919b9ef152bSMark Johnston     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
920b9ef152bSMark Johnston     NULL, 0, sysctl_vmm_create, "A",
921b9ef152bSMark Johnston     NULL);
922b9ef152bSMark Johnston 
923a97f683fSMark Johnston static int
924a97f683fSMark Johnston vmmctl_open(struct cdev *cdev, int flags, int fmt, struct thread *td)
925a97f683fSMark Johnston {
926a97f683fSMark Johnston 	int error;
927a97f683fSMark Johnston 
928a97f683fSMark Johnston 	error = vmm_priv_check(td->td_ucred);
929a97f683fSMark Johnston 	if (error != 0)
930a97f683fSMark Johnston 		return (error);
931a97f683fSMark Johnston 
932a97f683fSMark Johnston 	if ((flags & FWRITE) == 0)
933a97f683fSMark Johnston 		return (EPERM);
934a97f683fSMark Johnston 
935a97f683fSMark Johnston 	return (0);
936a97f683fSMark Johnston }
937a97f683fSMark Johnston 
938a97f683fSMark Johnston static int
939a97f683fSMark Johnston vmmctl_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
940a97f683fSMark Johnston     struct thread *td)
941a97f683fSMark Johnston {
942a97f683fSMark Johnston 	int error;
943a97f683fSMark Johnston 
944a97f683fSMark Johnston 	switch (cmd) {
945a97f683fSMark Johnston 	case VMMCTL_VM_CREATE: {
946a97f683fSMark Johnston 		struct vmmctl_vm_create *vmc;
947a97f683fSMark Johnston 
948a97f683fSMark Johnston 		vmc = (struct vmmctl_vm_create *)data;
949a97f683fSMark Johnston 		vmc->name[VM_MAX_NAMELEN] = '\0';
950a97f683fSMark Johnston 		for (size_t i = 0; i < nitems(vmc->reserved); i++) {
951a97f683fSMark Johnston 			if (vmc->reserved[i] != 0) {
952a97f683fSMark Johnston 				error = EINVAL;
953a97f683fSMark Johnston 				return (error);
954a97f683fSMark Johnston 			}
955a97f683fSMark Johnston 		}
956a97f683fSMark Johnston 
957a97f683fSMark Johnston 		error = vmmdev_create(vmc->name, td->td_ucred);
958a97f683fSMark Johnston 		break;
959a97f683fSMark Johnston 	}
960a97f683fSMark Johnston 	case VMMCTL_VM_DESTROY: {
961a97f683fSMark Johnston 		struct vmmctl_vm_destroy *vmd;
962a97f683fSMark Johnston 
963a97f683fSMark Johnston 		vmd = (struct vmmctl_vm_destroy *)data;
964a97f683fSMark Johnston 		vmd->name[VM_MAX_NAMELEN] = '\0';
965a97f683fSMark Johnston 		for (size_t i = 0; i < nitems(vmd->reserved); i++) {
966a97f683fSMark Johnston 			if (vmd->reserved[i] != 0) {
967a97f683fSMark Johnston 				error = EINVAL;
968a97f683fSMark Johnston 				return (error);
969a97f683fSMark Johnston 			}
970a97f683fSMark Johnston 		}
971a97f683fSMark Johnston 
972a97f683fSMark Johnston 		error = vmmdev_lookup_and_destroy(vmd->name, td->td_ucred);
973a97f683fSMark Johnston 		break;
974a97f683fSMark Johnston 	}
975a97f683fSMark Johnston 	default:
976a97f683fSMark Johnston 		error = ENOTTY;
977a97f683fSMark Johnston 		break;
978a97f683fSMark Johnston 	}
979a97f683fSMark Johnston 
980a97f683fSMark Johnston 	return (error);
981a97f683fSMark Johnston }
982a97f683fSMark Johnston 
9834a46ece6SMark Johnston static struct cdev *vmmctl_cdev;
984a97f683fSMark Johnston static struct cdevsw vmmctlsw = {
985a97f683fSMark Johnston 	.d_name		= "vmmctl",
986a97f683fSMark Johnston 	.d_version	= D_VERSION,
987a97f683fSMark Johnston 	.d_open		= vmmctl_open,
988a97f683fSMark Johnston 	.d_ioctl	= vmmctl_ioctl,
989a97f683fSMark Johnston };
990a97f683fSMark Johnston 
991a97f683fSMark Johnston int
992b9ef152bSMark Johnston vmmdev_init(void)
993b9ef152bSMark Johnston {
994a97f683fSMark Johnston 	int error;
995a97f683fSMark Johnston 
9964a46ece6SMark Johnston 	sx_xlock(&vmmdev_mtx);
9974a46ece6SMark Johnston 	error = make_dev_p(MAKEDEV_CHECKNAME, &vmmctl_cdev, &vmmctlsw, NULL,
998a97f683fSMark Johnston 	    UID_ROOT, GID_WHEEL, 0600, "vmmctl");
9994a46ece6SMark Johnston 	if (error == 0)
1000b9ef152bSMark Johnston 		pr_allow_flag = prison_add_allow(NULL, "vmm", NULL,
1001b9ef152bSMark Johnston 		    "Allow use of vmm in a jail.");
10024a46ece6SMark Johnston 	sx_xunlock(&vmmdev_mtx);
1003a97f683fSMark Johnston 
10044a46ece6SMark Johnston 	return (error);
1005b9ef152bSMark Johnston }
1006b9ef152bSMark Johnston 
1007b9ef152bSMark Johnston int
1008b9ef152bSMark Johnston vmmdev_cleanup(void)
1009b9ef152bSMark Johnston {
10104a46ece6SMark Johnston 	sx_xlock(&vmmdev_mtx);
10114a46ece6SMark Johnston 	if (!SLIST_EMPTY(&head)) {
10124a46ece6SMark Johnston 		sx_xunlock(&vmmdev_mtx);
10134a46ece6SMark Johnston 		return (EBUSY);
10144a46ece6SMark Johnston 	}
10154a46ece6SMark Johnston 	if (vmmctl_cdev != NULL) {
10164a46ece6SMark Johnston 		destroy_dev(vmmctl_cdev);
10174a46ece6SMark Johnston 		vmmctl_cdev = NULL;
10184a46ece6SMark Johnston 	}
10194a46ece6SMark Johnston 	sx_xunlock(&vmmdev_mtx);
1020b9ef152bSMark Johnston 
10214a46ece6SMark Johnston 	return (0);
1022b9ef152bSMark Johnston }
1023b9ef152bSMark Johnston 
1024b9ef152bSMark Johnston static int
1025b9ef152bSMark Johnston devmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t len,
1026b9ef152bSMark Johnston     struct vm_object **objp, int nprot)
1027b9ef152bSMark Johnston {
1028b9ef152bSMark Johnston 	struct devmem_softc *dsc;
1029b9ef152bSMark Johnston 	vm_ooffset_t first, last;
1030b9ef152bSMark Johnston 	size_t seglen;
1031b9ef152bSMark Johnston 	int error;
1032b9ef152bSMark Johnston 	bool sysmem;
1033b9ef152bSMark Johnston 
1034b9ef152bSMark Johnston 	dsc = cdev->si_drv1;
1035b9ef152bSMark Johnston 	if (dsc == NULL) {
1036b9ef152bSMark Johnston 		/* 'cdev' has been created but is not ready for use */
1037b9ef152bSMark Johnston 		return (ENXIO);
1038b9ef152bSMark Johnston 	}
1039b9ef152bSMark Johnston 
1040b9ef152bSMark Johnston 	first = *offset;
1041b9ef152bSMark Johnston 	last = *offset + len;
1042b9ef152bSMark Johnston 	if ((nprot & PROT_EXEC) || first < 0 || first >= last)
1043b9ef152bSMark Johnston 		return (EINVAL);
1044b9ef152bSMark Johnston 
1045b9ef152bSMark Johnston 	vm_slock_memsegs(dsc->sc->vm);
1046b9ef152bSMark Johnston 
1047b9ef152bSMark Johnston 	error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp);
1048b9ef152bSMark Johnston 	KASSERT(error == 0 && !sysmem && *objp != NULL,
1049b9ef152bSMark Johnston 	    ("%s: invalid devmem segment %d", __func__, dsc->segid));
1050b9ef152bSMark Johnston 
1051b9ef152bSMark Johnston 	if (seglen >= last)
1052b9ef152bSMark Johnston 		vm_object_reference(*objp);
1053b9ef152bSMark Johnston 	else
1054b9ef152bSMark Johnston 		error = EINVAL;
1055b9ef152bSMark Johnston 
1056b9ef152bSMark Johnston 	vm_unlock_memsegs(dsc->sc->vm);
1057b9ef152bSMark Johnston 	return (error);
1058b9ef152bSMark Johnston }
1059b9ef152bSMark Johnston 
1060b9ef152bSMark Johnston static struct cdevsw devmemsw = {
1061b9ef152bSMark Johnston 	.d_name		= "devmem",
1062b9ef152bSMark Johnston 	.d_version	= D_VERSION,
1063b9ef152bSMark Johnston 	.d_mmap_single	= devmem_mmap_single,
1064b9ef152bSMark Johnston };
1065b9ef152bSMark Johnston 
1066b9ef152bSMark Johnston static int
1067f4002135SMark Johnston devmem_create_cdev(struct vmmdev_softc *sc, int segid, char *devname)
1068b9ef152bSMark Johnston {
1069cef5f43fSMark Johnston 	struct make_dev_args mda;
1070b9ef152bSMark Johnston 	struct devmem_softc *dsc;
1071b9ef152bSMark Johnston 	int error;
1072b9ef152bSMark Johnston 
1073cef5f43fSMark Johnston 	sx_xlock(&vmmdev_mtx);
1074b9ef152bSMark Johnston 
1075b9ef152bSMark Johnston 	dsc = malloc(sizeof(struct devmem_softc), M_VMMDEV, M_WAITOK | M_ZERO);
1076b9ef152bSMark Johnston 	dsc->segid = segid;
1077b9ef152bSMark Johnston 	dsc->name = devname;
1078b9ef152bSMark Johnston 	dsc->sc = sc;
1079b9ef152bSMark Johnston 	SLIST_INSERT_HEAD(&sc->devmem, dsc, link);
1080cef5f43fSMark Johnston 
1081cef5f43fSMark Johnston 	make_dev_args_init(&mda);
1082cef5f43fSMark Johnston 	mda.mda_devsw = &devmemsw;
1083cef5f43fSMark Johnston 	mda.mda_cr = sc->ucred;
1084cef5f43fSMark Johnston 	mda.mda_uid = UID_ROOT;
1085cef5f43fSMark Johnston 	mda.mda_gid = GID_WHEEL;
1086cef5f43fSMark Johnston 	mda.mda_mode = 0600;
1087cef5f43fSMark Johnston 	mda.mda_si_drv1 = dsc;
1088cef5f43fSMark Johnston 	mda.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
1089cef5f43fSMark Johnston 	error = make_dev_s(&mda, &dsc->cdev, "vmm.io/%s.%s", vm_name(sc->vm),
1090cef5f43fSMark Johnston 	    devname);
1091cef5f43fSMark Johnston 	if (error != 0) {
1092cef5f43fSMark Johnston 		SLIST_REMOVE(&sc->devmem, dsc, devmem_softc, link);
1093cef5f43fSMark Johnston 		free(dsc->name, M_VMMDEV);
1094cef5f43fSMark Johnston 		free(dsc, M_VMMDEV);
1095cef5f43fSMark Johnston 	}
1096cef5f43fSMark Johnston 
1097887c0877SMark Johnston 	sx_xunlock(&vmmdev_mtx);
1098b9ef152bSMark Johnston 
1099cef5f43fSMark Johnston 	return (error);
1100b9ef152bSMark Johnston }
1101b9ef152bSMark Johnston 
1102b9ef152bSMark Johnston static void
1103b9ef152bSMark Johnston devmem_destroy(void *arg)
1104b9ef152bSMark Johnston {
1105b9ef152bSMark Johnston 	struct devmem_softc *dsc = arg;
1106b9ef152bSMark Johnston 
1107cef5f43fSMark Johnston 	destroy_dev(dsc->cdev);
1108b9ef152bSMark Johnston 	dsc->cdev = NULL;
1109b9ef152bSMark Johnston 	dsc->sc = NULL;
1110b9ef152bSMark Johnston }
1111