xref: /titanic_41/usr/src/uts/i86pc/io/xsvc/xsvc.c (revision de5d74c22760a6d2cefd94d0e7f0fd87214fb71f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  *
26  * Copyright 2016 Joyent, Inc.
27  */
28 
29 #include <sys/errno.h>
30 #include <sys/types.h>
31 #include <sys/conf.h>
32 #include <sys/kmem.h>
33 #include <sys/ddi.h>
34 #include <sys/stat.h>
35 #include <sys/sunddi.h>
36 #include <sys/file.h>
37 #include <sys/open.h>
38 #include <sys/modctl.h>
39 #include <sys/ddi_impldefs.h>
40 #include <vm/seg_kmem.h>
41 #include <sys/vmsystm.h>
42 #include <sys/sysmacros.h>
43 #include <sys/ddidevmap.h>
44 #include <sys/avl.h>
45 #ifdef __xpv
46 #include <sys/hypervisor.h>
47 #endif
48 
49 #include <sys/xsvc.h>
50 
51 /* total max memory which can be alloced with ioctl interface */
52 uint64_t xsvc_max_memory = 10 * 1024 * 1024;
53 
54 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
55 
56 
57 static int xsvc_open(dev_t *devp, int flag, int otyp, cred_t *cred);
58 static int xsvc_close(dev_t devp, int flag, int otyp, cred_t *cred);
59 static int xsvc_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
60     int *rval);
61 static int xsvc_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
62     size_t *maplen, uint_t model);
63 static int xsvc_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
64 static int xsvc_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
65 static int xsvc_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
66     void **result);
67 
68 static 	struct cb_ops xsvc_cb_ops = {
69 	xsvc_open,		/* cb_open */
70 	xsvc_close,		/* cb_close */
71 	nodev,			/* cb_strategy */
72 	nodev,			/* cb_print */
73 	nodev,			/* cb_dump */
74 	nodev,			/* cb_read */
75 	nodev,			/* cb_write */
76 	xsvc_ioctl,		/* cb_ioctl */
77 	xsvc_devmap,		/* cb_devmap */
78 	NULL,			/* cb_mmap */
79 	NULL,			/* cb_segmap */
80 	nochpoll,		/* cb_chpoll */
81 	ddi_prop_op,		/* cb_prop_op */
82 	NULL,			/* cb_stream */
83 	D_NEW | D_MP | D_64BIT | D_DEVMAP,	/* cb_flag */
84 	CB_REV
85 };
86 
87 static struct dev_ops xsvc_dev_ops = {
88 	DEVO_REV,		/* devo_rev */
89 	0,			/* devo_refcnt */
90 	xsvc_getinfo,		/* devo_getinfo */
91 	nulldev,		/* devo_identify */
92 	nulldev,		/* devo_probe */
93 	xsvc_attach,		/* devo_attach */
94 	xsvc_detach,		/* devo_detach */
95 	nodev,			/* devo_reset */
96 	&xsvc_cb_ops,		/* devo_cb_ops */
97 	NULL,			/* devo_bus_ops */
98 	NULL,			/* power */
99 	ddi_quiesce_not_needed,		/* quiesce */
100 };
101 
102 static struct modldrv xsvc_modldrv = {
103 	&mod_driverops,		/* Type of module.  This one is a driver */
104 	"xsvc driver",		/* Name of the module. */
105 	&xsvc_dev_ops,		/* driver ops */
106 };
107 
108 static struct modlinkage xsvc_modlinkage = {
109 	MODREV_1,
110 	(void *) &xsvc_modldrv,
111 	NULL
112 };
113 
114 
115 static int xsvc_ioctl_alloc_memory(xsvc_state_t *state, void *arg, int mode);
116 static int xsvc_ioctl_flush_memory(xsvc_state_t *state, void *arg, int mode);
117 static int xsvc_ioctl_free_memory(xsvc_state_t *state, void *arg, int mode);
118 static int xsvc_mem_alloc(xsvc_state_t *state, uint64_t key,
119     xsvc_mem_t **mp);
120 static void xsvc_mem_free(xsvc_state_t *state, xsvc_mem_t *mp);
121 static xsvc_mem_t *xsvc_mem_lookup(xsvc_state_t *state,
122     uint64_t key);
123 static int xsvc_mnode_key_compare(const void *q, const void *e);
124 static int xsvc_umem_cookie_alloc(caddr_t kva, size_t size, int flags,
125     ddi_umem_cookie_t *cookiep);
126 static void xsvc_umem_cookie_free(ddi_umem_cookie_t *cookiep);
127 
128 
129 void *xsvc_statep;
130 
131 static ddi_device_acc_attr_t xsvc_device_attr = {
132 	DDI_DEVICE_ATTR_V0,
133 	DDI_NEVERSWAP_ACC,
134 	DDI_STRICTORDER_ACC
135 };
136 
137 static int xsvc_devmap_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
138     offset_t off, size_t len, void **pvtp);
139 static int xsvc_devmap_dup(devmap_cookie_t dhp, void *pvtp,
140     devmap_cookie_t new_dhp, void **new_pvtp);
141 static void xsvc_devmap_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
142     size_t len, devmap_cookie_t new_dhp1, void **new_pvtp1,
143     devmap_cookie_t new_dhp2, void **new_pvtp2);
144 
145 
146 static struct devmap_callback_ctl xsvc_callbk = {
147 	DEVMAP_OPS_REV,
148 	xsvc_devmap_map,
149 	NULL,
150 	xsvc_devmap_dup,
151 	xsvc_devmap_unmap
152 };
153 
154 
155 /*
156  * _init()
157  *
158  */
159 int
_init(void)160 _init(void)
161 {
162 	int err;
163 
164 	err = ddi_soft_state_init(&xsvc_statep, sizeof (xsvc_state_t), 1);
165 	if (err != 0) {
166 		return (err);
167 	}
168 
169 	err = mod_install(&xsvc_modlinkage);
170 	if (err != 0) {
171 		ddi_soft_state_fini(&xsvc_statep);
172 		return (err);
173 	}
174 
175 	return (0);
176 }
177 
178 /*
179  * _info()
180  *
181  */
182 int
_info(struct modinfo * modinfop)183 _info(struct modinfo *modinfop)
184 {
185 	return (mod_info(&xsvc_modlinkage, modinfop));
186 }
187 
188 /*
189  * _fini()
190  *
191  */
192 int
_fini(void)193 _fini(void)
194 {
195 	int err;
196 
197 	err = mod_remove(&xsvc_modlinkage);
198 	if (err != 0) {
199 		return (err);
200 	}
201 
202 	ddi_soft_state_fini(&xsvc_statep);
203 
204 	return (0);
205 }
206 
207 /*
208  * xsvc_attach()
209  *
210  */
211 static int
xsvc_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)212 xsvc_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
213 {
214 	xsvc_state_t *state;
215 	int maxallocmem;
216 	int instance;
217 	int err;
218 
219 
220 	switch (cmd) {
221 	case DDI_ATTACH:
222 		break;
223 
224 	case DDI_RESUME:
225 		return (DDI_SUCCESS);
226 
227 	default:
228 		return (DDI_FAILURE);
229 	}
230 
231 	instance = ddi_get_instance(dip);
232 	err = ddi_soft_state_zalloc(xsvc_statep, instance);
233 	if (err != DDI_SUCCESS) {
234 		return (DDI_FAILURE);
235 	}
236 	state = ddi_get_soft_state(xsvc_statep, instance);
237 	if (state == NULL) {
238 		goto attachfail_get_soft_state;
239 	}
240 
241 	state->xs_dip = dip;
242 	state->xs_instance = instance;
243 
244 	/* Initialize allocation count */
245 	mutex_init(&state->xs_mutex, NULL, MUTEX_DRIVER, NULL);
246 	state->xs_currently_alloced = 0;
247 
248 	mutex_init(&state->xs_cookie_mutex, NULL, MUTEX_DRIVER, NULL);
249 
250 	/* create the minor node (for the ioctl) */
251 	err = ddi_create_minor_node(dip, "xsvc", S_IFCHR, instance, DDI_PSEUDO,
252 	    0);
253 	if (err != DDI_SUCCESS) {
254 		goto attachfail_minor_node;
255 	}
256 
257 	/*
258 	 * the maxallocmem property will override the default (xsvc_max_memory).
259 	 * This is the maximum total memory the ioctl will allow to be alloced.
260 	 */
261 	maxallocmem = ddi_prop_get_int(DDI_DEV_T_ANY, state->xs_dip,
262 	    DDI_PROP_DONTPASS, "maxallocmem", -1);
263 	if (maxallocmem >= 0) {
264 		xsvc_max_memory = maxallocmem * 1024;
265 	}
266 
267 	/* Initialize list of memory allocs */
268 	mutex_init(&state->xs_mlist.ml_mutex, NULL, MUTEX_DRIVER, NULL);
269 	avl_create(&state->xs_mlist.ml_avl, xsvc_mnode_key_compare,
270 	    sizeof (xsvc_mnode_t), offsetof(xsvc_mnode_t, mn_link));
271 
272 	/* Report that driver was loaded */
273 	ddi_report_dev(dip);
274 
275 	return (DDI_SUCCESS);
276 
277 attachfail_minor_node:
278 	mutex_destroy(&state->xs_cookie_mutex);
279 	mutex_destroy(&state->xs_mutex);
280 attachfail_get_soft_state:
281 	(void) ddi_soft_state_free(xsvc_statep, instance);
282 
283 	return (err);
284 }
285 
286 /*
287  * xsvc_detach()
288  *
289  */
290 static int
xsvc_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)291 xsvc_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
292 {
293 	xsvc_state_t *state;
294 	xsvc_mnode_t *mnode;
295 	xsvc_mem_t *mp;
296 	int instance;
297 
298 
299 	instance = ddi_get_instance(dip);
300 	state = ddi_get_soft_state(xsvc_statep, instance);
301 	if (state == NULL) {
302 		return (DDI_FAILURE);
303 	}
304 
305 	switch (cmd) {
306 	case DDI_DETACH:
307 		break;
308 
309 	case DDI_SUSPEND:
310 		return (DDI_SUCCESS);
311 
312 	default:
313 		return (DDI_FAILURE);
314 	}
315 
316 	ddi_remove_minor_node(dip, NULL);
317 
318 	/* Free any memory on list */
319 	while ((mnode = avl_first(&state->xs_mlist.ml_avl)) != NULL) {
320 		mp = mnode->mn_home;
321 		xsvc_mem_free(state, mp);
322 	}
323 
324 	/* remove list */
325 	avl_destroy(&state->xs_mlist.ml_avl);
326 	mutex_destroy(&state->xs_mlist.ml_mutex);
327 
328 	mutex_destroy(&state->xs_cookie_mutex);
329 	mutex_destroy(&state->xs_mutex);
330 	(void) ddi_soft_state_free(xsvc_statep, state->xs_instance);
331 	return (DDI_SUCCESS);
332 }
333 
334 /*
335  * xsvc_getinfo()
336  *
337  */
338 /*ARGSUSED*/
339 static int
xsvc_getinfo(dev_info_t * dip,ddi_info_cmd_t cmd,void * arg,void ** result)340 xsvc_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
341 {
342 	xsvc_state_t *state;
343 	int instance;
344 	dev_t dev;
345 	int err;
346 
347 
348 	dev = (dev_t)arg;
349 	instance = getminor(dev);
350 
351 	switch (cmd) {
352 	case DDI_INFO_DEVT2DEVINFO:
353 		state = ddi_get_soft_state(xsvc_statep, instance);
354 		if (state == NULL) {
355 			return (DDI_FAILURE);
356 		}
357 		*result = (void *)state->xs_dip;
358 		err = DDI_SUCCESS;
359 		break;
360 
361 	case DDI_INFO_DEVT2INSTANCE:
362 		*result = (void *)(uintptr_t)instance;
363 		err = DDI_SUCCESS;
364 		break;
365 
366 	default:
367 		err = DDI_FAILURE;
368 		break;
369 	}
370 
371 	return (err);
372 }
373 
374 
375 /*
376  * xsvc_open()
377  *
378  */
379 /*ARGSUSED*/
380 static int
xsvc_open(dev_t * devp,int flag,int otyp,cred_t * cred)381 xsvc_open(dev_t *devp, int flag, int otyp, cred_t *cred)
382 {
383 	xsvc_state_t *state;
384 	int instance;
385 
386 	instance = getminor(*devp);
387 	state = ddi_get_soft_state(xsvc_statep, instance);
388 	if (state == NULL) {
389 		return (ENXIO);
390 	}
391 
392 	return (0);
393 }
394 
395 /*
396  * xsvc_close()
397  *
398  */
399 /*ARGSUSED*/
400 static int
xsvc_close(dev_t devp,int flag,int otyp,cred_t * cred)401 xsvc_close(dev_t devp, int flag, int otyp, cred_t *cred)
402 {
403 	return (0);
404 }
405 
406 /*
407  * xsvc_ioctl()
408  *
409  */
410 /*ARGSUSED*/
411 static int
xsvc_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * cred,int * rval)412 xsvc_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred, int *rval)
413 {
414 	xsvc_state_t *state;
415 	int instance;
416 	int err;
417 
418 
419 	err = drv_priv(cred);
420 	if (err != 0) {
421 		return (EPERM);
422 	}
423 	instance = getminor(dev);
424 	if (instance == -1) {
425 		return (EBADF);
426 	}
427 	state = ddi_get_soft_state(xsvc_statep, instance);
428 	if (state == NULL) {
429 		return (EBADF);
430 	}
431 
432 	switch (cmd) {
433 	case XSVC_ALLOC_MEM:
434 		err = xsvc_ioctl_alloc_memory(state, (void *)arg, mode);
435 		break;
436 
437 	case XSVC_FREE_MEM:
438 		err = xsvc_ioctl_free_memory(state, (void *)arg, mode);
439 		break;
440 
441 	case XSVC_FLUSH_MEM:
442 		err = xsvc_ioctl_flush_memory(state, (void *)arg, mode);
443 		break;
444 
445 	default:
446 		err = ENXIO;
447 	}
448 
449 	return (err);
450 }
451 
452 /*
453  * xsvc_ioctl_alloc_memory()
454  *
455  */
456 static int
xsvc_ioctl_alloc_memory(xsvc_state_t * state,void * arg,int mode)457 xsvc_ioctl_alloc_memory(xsvc_state_t *state, void *arg, int mode)
458 {
459 	xsvc_mem_req_32 params32;
460 	xsvc_mloc_32 *usgl32;
461 	xsvc_mem_req params;
462 	xsvc_mloc_32 sgl32;
463 	xsvc_mloc *usgl;
464 	xsvc_mem_t *mp;
465 	xsvc_mloc sgl;
466 	uint64_t key;
467 	size_t size;
468 	int err;
469 	int i;
470 
471 
472 	/* Copy in the params, then get the size and key */
473 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
474 		err = ddi_copyin(arg, &params32, sizeof (xsvc_mem_req_32),
475 		    mode);
476 		if (err != 0) {
477 			return (EFAULT);
478 		}
479 
480 		key = (uint64_t)params32.xsvc_mem_reqid;
481 		size = P2ROUNDUP((size_t)params32.xsvc_mem_size, PAGESIZE);
482 	} else {
483 		err = ddi_copyin(arg, &params, sizeof (xsvc_mem_req), mode);
484 		if (err != 0) {
485 			return (EFAULT);
486 		}
487 		key = (uint64_t)params.xsvc_mem_reqid;
488 		size = P2ROUNDUP(params.xsvc_mem_size, PAGESIZE);
489 	}
490 
491 	/*
492 	 * make sure this doesn't put us over the maximum allowed to be
493 	 * allocated
494 	 */
495 	mutex_enter(&state->xs_mutex);
496 	if ((state->xs_currently_alloced + size) > xsvc_max_memory) {
497 		mutex_exit(&state->xs_mutex);
498 		return (EAGAIN);
499 	}
500 	state->xs_currently_alloced += size;
501 	mutex_exit(&state->xs_mutex);
502 
503 	/* get state to track this memory */
504 	err = xsvc_mem_alloc(state, key, &mp);
505 	if (err != 0) {
506 		return (err);
507 	}
508 	mp->xm_size = size;
509 
510 	/* allocate and bind the memory */
511 	mp->xm_dma_attr.dma_attr_version = DMA_ATTR_V0;
512 	mp->xm_dma_attr.dma_attr_count_max = (uint64_t)0xFFFFFFFF;
513 	mp->xm_dma_attr.dma_attr_burstsizes = 1;
514 	mp->xm_dma_attr.dma_attr_minxfer = 1;
515 	mp->xm_dma_attr.dma_attr_maxxfer = (uint64_t)0xFFFFFFFF;
516 	mp->xm_dma_attr.dma_attr_seg = (uint64_t)0xFFFFFFFF;
517 	mp->xm_dma_attr.dma_attr_granular = 1;
518 	mp->xm_dma_attr.dma_attr_flags = 0;
519 
520 	/* Finish converting params */
521 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
522 		mp->xm_dma_attr.dma_attr_addr_lo = params32.xsvc_mem_addr_lo;
523 		mp->xm_dma_attr.dma_attr_addr_hi = params32.xsvc_mem_addr_hi;
524 		mp->xm_dma_attr.dma_attr_sgllen = params32.xsvc_mem_sgllen;
525 		usgl32 = (xsvc_mloc_32 *)(uintptr_t)params32.xsvc_sg_list;
526 		mp->xm_dma_attr.dma_attr_align = P2ROUNDUP(
527 		    params32.xsvc_mem_align, PAGESIZE);
528 	} else {
529 		mp->xm_dma_attr.dma_attr_addr_lo = params.xsvc_mem_addr_lo;
530 		mp->xm_dma_attr.dma_attr_addr_hi = params.xsvc_mem_addr_hi;
531 		mp->xm_dma_attr.dma_attr_sgllen = params.xsvc_mem_sgllen;
532 		usgl = (xsvc_mloc *)(uintptr_t)params.xsvc_sg_list;
533 		mp->xm_dma_attr.dma_attr_align = P2ROUNDUP(
534 		    params.xsvc_mem_align, PAGESIZE);
535 	}
536 
537 	mp->xm_device_attr = xsvc_device_attr;
538 
539 	err = ddi_dma_alloc_handle(state->xs_dip, &mp->xm_dma_attr,
540 	    DDI_DMA_SLEEP, NULL, &mp->xm_dma_handle);
541 	if (err != DDI_SUCCESS) {
542 		err = EINVAL;
543 		goto allocfail_alloc_handle;
544 	}
545 
546 	/* don't sleep here so we don't get stuck in contig alloc */
547 	err = ddi_dma_mem_alloc(mp->xm_dma_handle, mp->xm_size,
548 	    &mp->xm_device_attr, DDI_DMA_CONSISTENT, DDI_DMA_DONTWAIT, NULL,
549 	    &mp->xm_addr, &mp->xm_real_length, &mp->xm_mem_handle);
550 	if (err != DDI_SUCCESS) {
551 		err = EINVAL;
552 		goto allocfail_alloc_mem;
553 	}
554 
555 	err = ddi_dma_addr_bind_handle(mp->xm_dma_handle, NULL, mp->xm_addr,
556 	    mp->xm_size, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
557 	    NULL, &mp->xm_cookie, &mp->xm_cookie_count);
558 	if (err != DDI_DMA_MAPPED) {
559 		err = EFAULT;
560 		goto allocfail_bind;
561 	}
562 
563 	/* return sgl */
564 	for (i = 0; i < mp->xm_cookie_count; i++) {
565 		if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
566 			sgl32.mloc_addr = mp->xm_cookie.dmac_laddress;
567 			sgl32.mloc_size = mp->xm_cookie.dmac_size;
568 			err = ddi_copyout(&sgl32, &usgl32[i],
569 			    sizeof (xsvc_mloc_32), mode);
570 			if (err != 0) {
571 				err = EFAULT;
572 				goto allocfail_copyout;
573 			}
574 		} else {
575 			sgl.mloc_addr = mp->xm_cookie.dmac_laddress;
576 			sgl.mloc_size = mp->xm_cookie.dmac_size;
577 			err = ddi_copyout(&sgl, &usgl[i], sizeof (xsvc_mloc),
578 			    mode);
579 			if (err != 0) {
580 				err = EFAULT;
581 				goto allocfail_copyout;
582 			}
583 		}
584 		ddi_dma_nextcookie(mp->xm_dma_handle, &mp->xm_cookie);
585 	}
586 
587 	/* set the last sgl entry to 0 to indicate cookie count */
588 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
589 		sgl32.mloc_addr = 0;
590 		sgl32.mloc_size = 0;
591 		err = ddi_copyout(&sgl32, &usgl32[i], sizeof (xsvc_mloc_32),
592 		    mode);
593 		if (err != 0) {
594 			err = EFAULT;
595 			goto allocfail_copyout;
596 		}
597 	} else {
598 		sgl.mloc_addr = 0;
599 		sgl.mloc_size = 0;
600 		err = ddi_copyout(&sgl, &usgl[i], sizeof (xsvc_mloc), mode);
601 		if (err != 0) {
602 			err = EFAULT;
603 			goto allocfail_copyout;
604 		}
605 	}
606 
607 	return (0);
608 
609 allocfail_copyout:
610 	(void) ddi_dma_unbind_handle(mp->xm_dma_handle);
611 allocfail_bind:
612 	ddi_dma_mem_free(&mp->xm_mem_handle);
613 allocfail_alloc_mem:
614 	ddi_dma_free_handle(&mp->xm_dma_handle);
615 allocfail_alloc_handle:
616 	mp->xm_dma_handle = NULL;
617 	xsvc_mem_free(state, mp);
618 
619 	mutex_enter(&state->xs_mutex);
620 	state->xs_currently_alloced = state->xs_currently_alloced - size;
621 	mutex_exit(&state->xs_mutex);
622 
623 	return (err);
624 }
625 
626 /*
627  * xsvc_ioctl_flush_memory()
628  *
629  */
630 static int
xsvc_ioctl_flush_memory(xsvc_state_t * state,void * arg,int mode)631 xsvc_ioctl_flush_memory(xsvc_state_t *state, void *arg, int mode)
632 {
633 	xsvc_mem_req_32 params32;
634 	xsvc_mem_req params;
635 	xsvc_mem_t *mp;
636 	uint64_t key;
637 	int err;
638 
639 
640 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
641 		err = ddi_copyin(arg, &params32, sizeof (xsvc_mem_req_32),
642 		    mode);
643 		if (err != 0) {
644 			return (EFAULT);
645 		}
646 		key = (uint64_t)params32.xsvc_mem_reqid;
647 	} else {
648 		err = ddi_copyin(arg, &params, sizeof (xsvc_mem_req), mode);
649 		if (err != 0) {
650 			return (EFAULT);
651 		}
652 		key = (uint64_t)params.xsvc_mem_reqid;
653 	}
654 
655 	/* find the memory */
656 	mp = xsvc_mem_lookup(state, key);
657 	if (mp == NULL) {
658 		return (EINVAL);
659 	}
660 
661 	(void) ddi_dma_sync(mp->xm_dma_handle, 0, 0, DDI_DMA_SYNC_FORCPU);
662 
663 	return (0);
664 }
665 
666 
667 /*
668  * xsvc_ioctl_free_memory()
669  *
670  */
671 static int
xsvc_ioctl_free_memory(xsvc_state_t * state,void * arg,int mode)672 xsvc_ioctl_free_memory(xsvc_state_t *state, void *arg, int mode)
673 {
674 	xsvc_mem_req_32 params32;
675 	xsvc_mem_req params;
676 	xsvc_mem_t *mp;
677 	uint64_t key;
678 	int err;
679 
680 
681 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
682 		err = ddi_copyin(arg, &params32, sizeof (xsvc_mem_req_32),
683 		    mode);
684 		if (err != 0) {
685 			return (EFAULT);
686 		}
687 		key = (uint64_t)params32.xsvc_mem_reqid;
688 	} else {
689 		err = ddi_copyin(arg, &params, sizeof (xsvc_mem_req), mode);
690 		if (err != 0) {
691 			return (EFAULT);
692 		}
693 		key = (uint64_t)params.xsvc_mem_reqid;
694 	}
695 
696 	/* find the memory */
697 	mp = xsvc_mem_lookup(state, key);
698 	if (mp == NULL) {
699 		return (EINVAL);
700 	}
701 
702 	xsvc_mem_free(state, mp);
703 
704 	return (0);
705 }
706 
707 /*
708  * xsvc_mem_alloc()
709  *
710  */
711 static int
xsvc_mem_alloc(xsvc_state_t * state,uint64_t key,xsvc_mem_t ** mp)712 xsvc_mem_alloc(xsvc_state_t *state, uint64_t key, xsvc_mem_t **mp)
713 {
714 	xsvc_mem_t *mem;
715 
716 	mem = xsvc_mem_lookup(state, key);
717 	if (mem != NULL) {
718 		xsvc_mem_free(state, mem);
719 	}
720 
721 	*mp = kmem_alloc(sizeof (xsvc_mem_t), KM_SLEEP);
722 	(*mp)->xm_mnode.mn_home = *mp;
723 	(*mp)->xm_mnode.mn_key = key;
724 
725 	mutex_enter(&state->xs_mlist.ml_mutex);
726 	avl_add(&state->xs_mlist.ml_avl, &(*mp)->xm_mnode);
727 	mutex_exit(&state->xs_mlist.ml_mutex);
728 
729 	return (0);
730 }
731 
732 /*
733  * xsvc_mem_free()
734  *
735  */
736 static void
xsvc_mem_free(xsvc_state_t * state,xsvc_mem_t * mp)737 xsvc_mem_free(xsvc_state_t *state, xsvc_mem_t *mp)
738 {
739 	if (mp->xm_dma_handle != NULL) {
740 		(void) ddi_dma_unbind_handle(mp->xm_dma_handle);
741 		ddi_dma_mem_free(&mp->xm_mem_handle);
742 		ddi_dma_free_handle(&mp->xm_dma_handle);
743 
744 		mutex_enter(&state->xs_mutex);
745 		state->xs_currently_alloced = state->xs_currently_alloced -
746 		    mp->xm_size;
747 		mutex_exit(&state->xs_mutex);
748 	}
749 
750 	mutex_enter(&state->xs_mlist.ml_mutex);
751 	avl_remove(&state->xs_mlist.ml_avl, &mp->xm_mnode);
752 	mutex_exit(&state->xs_mlist.ml_mutex);
753 
754 	kmem_free(mp, sizeof (*mp));
755 }
756 
757 /*
758  * xsvc_mem_lookup()
759  *
760  */
761 static xsvc_mem_t *
xsvc_mem_lookup(xsvc_state_t * state,uint64_t key)762 xsvc_mem_lookup(xsvc_state_t *state, uint64_t key)
763 {
764 	xsvc_mnode_t mnode;
765 	xsvc_mnode_t *mnp;
766 	avl_index_t where;
767 	xsvc_mem_t *mp;
768 
769 	mnode.mn_key = key;
770 	mutex_enter(&state->xs_mlist.ml_mutex);
771 	mnp = avl_find(&state->xs_mlist.ml_avl, &mnode, &where);
772 	mutex_exit(&state->xs_mlist.ml_mutex);
773 
774 	if (mnp != NULL) {
775 		mp = mnp->mn_home;
776 	} else {
777 		mp = NULL;
778 	}
779 
780 	return (mp);
781 }
782 
783 /*
784  * xsvc_mnode_key_compare()
785  *
786  */
787 static int
xsvc_mnode_key_compare(const void * q,const void * e)788 xsvc_mnode_key_compare(const void *q, const void *e)
789 {
790 	xsvc_mnode_t *n1;
791 	xsvc_mnode_t *n2;
792 
793 	n1 = (xsvc_mnode_t *)q;
794 	n2 = (xsvc_mnode_t *)e;
795 
796 	if (n1->mn_key < n2->mn_key) {
797 		return (-1);
798 	} else if (n1->mn_key > n2->mn_key) {
799 		return (1);
800 	} else {
801 		return (0);
802 	}
803 }
804 
805 /*
806  * xsvc_devmap()
807  *
808  */
809 /*ARGSUSED*/
810 static int
xsvc_devmap(dev_t dev,devmap_cookie_t dhp,offset_t off,size_t len,size_t * maplen,uint_t model)811 xsvc_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
812     size_t *maplen, uint_t model)
813 {
814 	ddi_umem_cookie_t cookie;
815 	xsvc_state_t *state;
816 	offset_t off_align;
817 	size_t npages;
818 	caddr_t kvai;
819 	size_t psize;
820 	int instance;
821 	caddr_t kva;
822 	pfn_t pfn;
823 	int err;
824 	int i;
825 
826 
827 	instance = getminor(dev);
828 	state = ddi_get_soft_state(xsvc_statep, instance);
829 	if (state == NULL) {
830 		return (ENXIO);
831 	}
832 
833 	/*
834 	 * On 64-bit kernels, if we have a 32-bit application doing a mmap(),
835 	 * smmap32 will sign extend the offset. We need to undo that since
836 	 * we are passed a physical address in off, not a offset.
837 	 */
838 #if defined(__amd64)
839 	if (((model & DDI_MODEL_MASK) == DDI_MODEL_ILP32) &&
840 	    ((off & ~0xFFFFFFFFll) == ~0xFFFFFFFFll)) {
841 		off = off & 0xFFFFFFFF;
842 	}
843 #endif
844 
845 #ifdef __xpv
846 	/*
847 	 * we won't allow guest OSes to devmap mfn/pfns. Maybe we'll relax
848 	 * this some later when there is a good reason.
849 	 */
850 	if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
851 		return (-1);
852 	}
853 
854 	/* we will always treat this as a foreign MFN */
855 	pfn = xen_assign_pfn(btop(off));
856 #else
857 	pfn = btop(off);
858 #endif
859 	/* always work with whole pages */
860 
861 	off_align = P2ALIGN(off, PAGESIZE);
862 	psize = P2ROUNDUP(off + len, PAGESIZE) - off_align;
863 
864 	/*
865 	 * if this is memory we're trying to map into user space, we first
866 	 * need to map the PFNs into KVA, then build up a umem cookie, and
867 	 * finally do a umem_setup to map it in.
868 	 */
869 	if (pf_is_memory(pfn)) {
870 		npages = btop(psize);
871 
872 		kva = vmem_alloc(heap_arena, psize, VM_SLEEP);
873 		if (kva == NULL) {
874 			return (-1);
875 		}
876 
877 		kvai = kva;
878 		for (i = 0; i < npages; i++) {
879 			page_t *pp = page_numtopp_nolock(pfn);
880 
881 			/*
882 			 * Preemptively check for panic conditions from
883 			 * hat_devload and error out instead.
884 			 */
885 			if (pp != NULL && (PP_ISFREE(pp) ||
886 			    (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)))) {
887 				err = DDI_FAILURE;
888 				npages = i;
889 				goto devmapfail_cookie_alloc;
890 			}
891 
892 			hat_devload(kas.a_hat, kvai, PAGESIZE, pfn,
893 			    PROT_READ | PROT_WRITE, HAT_LOAD_LOCK);
894 			pfn++;
895 			kvai = (caddr_t)((uintptr_t)kvai + PAGESIZE);
896 		}
897 
898 		err = xsvc_umem_cookie_alloc(kva, psize, KM_SLEEP, &cookie);
899 		if (err != 0) {
900 			goto devmapfail_cookie_alloc;
901 		}
902 
903 		if ((err = devmap_umem_setup(dhp, state->xs_dip, &xsvc_callbk,
904 		    cookie, 0, psize, PROT_ALL, 0, &xsvc_device_attr)) < 0) {
905 			goto devmapfail_umem_setup;
906 		}
907 		*maplen = psize;
908 
909 	/*
910 	 * If this is not memory (or a foreign MFN in i86xpv), go through
911 	 * devmem_setup.
912 	 */
913 	} else {
914 		if ((err = devmap_devmem_setup(dhp, state->xs_dip, NULL, 0,
915 		    off_align, psize, PROT_ALL, 0, &xsvc_device_attr)) < 0) {
916 			return (err);
917 		}
918 		*maplen = psize;
919 	}
920 
921 	return (0);
922 
923 devmapfail_umem_setup:
924 	xsvc_umem_cookie_free(&cookie);
925 
926 devmapfail_cookie_alloc:
927 	kvai = kva;
928 	for (i = 0; i < npages; i++) {
929 		hat_unload(kas.a_hat, kvai, PAGESIZE,
930 		    HAT_UNLOAD_UNLOCK);
931 		kvai = (caddr_t)((uintptr_t)kvai + PAGESIZE);
932 	}
933 	vmem_free(heap_arena, kva, psize);
934 
935 	return (err);
936 }
937 
938 /*
939  * xsvc_umem_cookie_alloc()
940  *
941  *   allocate a umem cookie to be used in devmap_umem_setup using KVA already
942  *   allocated.
943  */
944 int
xsvc_umem_cookie_alloc(caddr_t kva,size_t size,int flags,ddi_umem_cookie_t * cookiep)945 xsvc_umem_cookie_alloc(caddr_t kva, size_t size, int flags,
946     ddi_umem_cookie_t *cookiep)
947 {
948 	struct ddi_umem_cookie *umem_cookiep;
949 
950 	umem_cookiep = kmem_zalloc(sizeof (struct ddi_umem_cookie), flags);
951 	if (umem_cookiep == NULL) {
952 		*cookiep = NULL;
953 		return (-1);
954 	}
955 
956 	umem_cookiep->cvaddr = kva;
957 	umem_cookiep->type = KMEM_NON_PAGEABLE;
958 	umem_cookiep->size = size;
959 	*cookiep = (ddi_umem_cookie_t *)umem_cookiep;
960 
961 	return (0);
962 }
963 
964 /*
965  * xsvc_umem_cookie_free()
966  *
967  */
968 static void
xsvc_umem_cookie_free(ddi_umem_cookie_t * cookiep)969 xsvc_umem_cookie_free(ddi_umem_cookie_t *cookiep)
970 {
971 	kmem_free(*cookiep, sizeof (struct ddi_umem_cookie));
972 	*cookiep = NULL;
973 }
974 
975 
976 /*
977  * xsvc_devmap_map()
978  *
979  */
980 /*ARGSUSED*/
981 static int
xsvc_devmap_map(devmap_cookie_t dhc,dev_t dev,uint_t flags,offset_t off,size_t len,void ** pvtp)982 xsvc_devmap_map(devmap_cookie_t dhc, dev_t dev, uint_t flags, offset_t off,
983     size_t len, void **pvtp)
984 {
985 	struct ddi_umem_cookie *cp;
986 	devmap_handle_t *dhp;
987 	xsvc_state_t *state;
988 	int instance;
989 
990 
991 	instance = getminor(dev);
992 	state = ddi_get_soft_state(xsvc_statep, instance);
993 	if (state == NULL) {
994 		return (ENXIO);
995 	}
996 
997 	dhp = (devmap_handle_t *)dhc;
998 	/* This driver only supports MAP_SHARED, not MAP_PRIVATE */
999 	if (flags & MAP_PRIVATE) {
1000 		cmn_err(CE_WARN, "!xsvc driver doesn't support MAP_PRIVATE");
1001 		return (EINVAL);
1002 	}
1003 
1004 	cp = (struct ddi_umem_cookie *)dhp->dh_cookie;
1005 	cp->cook_refcnt = 1;
1006 
1007 	*pvtp = state;
1008 	return (0);
1009 }
1010 
1011 
1012 /*
1013  * xsvc_devmap_dup()
1014  *
1015  *   keep a reference count for forks so we don't unmap if we have multiple
1016  *   mappings.
1017  */
1018 /*ARGSUSED*/
1019 static int
xsvc_devmap_dup(devmap_cookie_t dhc,void * pvtp,devmap_cookie_t new_dhp,void ** new_pvtp)1020 xsvc_devmap_dup(devmap_cookie_t dhc, void *pvtp, devmap_cookie_t new_dhp,
1021     void **new_pvtp)
1022 {
1023 	struct ddi_umem_cookie *cp;
1024 	devmap_handle_t *dhp;
1025 	xsvc_state_t *state;
1026 
1027 
1028 	state = (xsvc_state_t *)pvtp;
1029 	dhp = (devmap_handle_t *)dhc;
1030 
1031 	mutex_enter(&state->xs_cookie_mutex);
1032 	cp = (struct ddi_umem_cookie *)dhp->dh_cookie;
1033 	if (cp == NULL) {
1034 		mutex_exit(&state->xs_cookie_mutex);
1035 		return (ENOMEM);
1036 	}
1037 
1038 	cp->cook_refcnt++;
1039 	mutex_exit(&state->xs_cookie_mutex);
1040 
1041 	*new_pvtp = state;
1042 	return (0);
1043 }
1044 
1045 
1046 /*
1047  * xsvc_devmap_unmap()
1048  *
1049  *   This routine is only call if we were mapping in memory in xsvc_devmap().
1050  *   i.e. we only pass in xsvc_callbk to devmap_umem_setup if pf_is_memory()
1051  *   was true. It would have been nice if devmap_callback_ctl had an args param.
1052  *   We wouldn't have had to look into the devmap_handle and into the umem
1053  *   cookie.
1054  */
1055 /*ARGSUSED*/
1056 static void
xsvc_devmap_unmap(devmap_cookie_t dhc,void * pvtp,offset_t off,size_t len,devmap_cookie_t new_dhp1,void ** new_pvtp1,devmap_cookie_t new_dhp2,void ** new_pvtp2)1057 xsvc_devmap_unmap(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
1058     devmap_cookie_t new_dhp1, void **new_pvtp1, devmap_cookie_t new_dhp2,
1059     void **new_pvtp2)
1060 {
1061 	struct ddi_umem_cookie *ncp;
1062 	struct ddi_umem_cookie *cp;
1063 	devmap_handle_t *ndhp;
1064 	devmap_handle_t *dhp;
1065 	xsvc_state_t *state;
1066 	size_t npages;
1067 	caddr_t kvai;
1068 	caddr_t kva;
1069 	size_t size;
1070 	int i;
1071 
1072 
1073 	state = (xsvc_state_t *)pvtp;
1074 	mutex_enter(&state->xs_cookie_mutex);
1075 
1076 	/* peek into the umem cookie to figure out what we need to free up */
1077 	dhp = (devmap_handle_t *)dhc;
1078 	cp = (struct ddi_umem_cookie *)dhp->dh_cookie;
1079 	ASSERT(cp != NULL);
1080 
1081 	if (new_dhp1 != NULL) {
1082 		ndhp = (devmap_handle_t *)new_dhp1;
1083 		ncp = (struct ddi_umem_cookie *)ndhp->dh_cookie;
1084 		ncp->cook_refcnt++;
1085 		*new_pvtp1 = state;
1086 	}
1087 	if (new_dhp2 != NULL) {
1088 		ndhp = (devmap_handle_t *)new_dhp2;
1089 		ncp = (struct ddi_umem_cookie *)ndhp->dh_cookie;
1090 		ncp->cook_refcnt++;
1091 		*new_pvtp2 = state;
1092 	}
1093 
1094 	cp->cook_refcnt--;
1095 	if (cp->cook_refcnt == 0) {
1096 		kva = cp->cvaddr;
1097 		size = cp->size;
1098 
1099 		/*
1100 		 * free up the umem cookie, then unmap all the pages what we
1101 		 * mapped in during devmap, then free up the kva space.
1102 		 */
1103 		npages = btop(size);
1104 		xsvc_umem_cookie_free(&dhp->dh_cookie);
1105 		kvai = kva;
1106 		for (i = 0; i < npages; i++) {
1107 			hat_unload(kas.a_hat, kvai, PAGESIZE,
1108 			    HAT_UNLOAD_UNLOCK);
1109 			kvai = (caddr_t)((uintptr_t)kvai + PAGESIZE);
1110 		}
1111 		vmem_free(heap_arena, kva, size);
1112 	}
1113 
1114 	mutex_exit(&state->xs_cookie_mutex);
1115 }
1116