xref: /titanic_41/usr/src/uts/common/vm/seg_dev.c (revision 93fb2a5ff9019dc98ff5e9836d0c2c7b5c5ecd7f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 /*
31  * University Copyright- Copyright (c) 1982, 1986, 1988
32  * The Regents of the University of California
33  * All Rights Reserved
34  *
35  * University Acknowledgment- Portions of this document are derived from
36  * software developed by the University of California, Berkeley, and its
37  * contributors.
38  */
39 
40 /*
41  * VM - segment of a mapped device.
42  *
43  * This segment driver is used when mapping character special devices.
44  */
45 
46 #include <sys/types.h>
47 #include <sys/t_lock.h>
48 #include <sys/sysmacros.h>
49 #include <sys/vtrace.h>
50 #include <sys/systm.h>
51 #include <sys/vmsystm.h>
52 #include <sys/mman.h>
53 #include <sys/errno.h>
54 #include <sys/kmem.h>
55 #include <sys/cmn_err.h>
56 #include <sys/vnode.h>
57 #include <sys/proc.h>
58 #include <sys/conf.h>
59 #include <sys/debug.h>
60 #include <sys/ddidevmap.h>
61 #include <sys/ddi_implfuncs.h>
62 #include <sys/lgrp.h>
63 
64 #include <vm/page.h>
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_dev.h>
69 #include <vm/seg_kp.h>
70 #include <vm/seg_kmem.h>
71 #include <vm/vpage.h>
72 
73 #include <sys/sunddi.h>
74 #include <sys/esunddi.h>
75 #include <sys/fs/snode.h>
76 
77 
78 #if DEBUG
79 int segdev_debug;
80 #define	DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 #else
82 #define	DEBUGF(level, args)
83 #endif
84 
85 /* Default timeout for devmap context management */
86 #define	CTX_TIMEOUT_VALUE 0
87 
88 #define	HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 			{ mutex_enter(&dhp->dh_lock); }
90 
91 #define	RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 			{ mutex_exit(&dhp->dh_lock); }
93 
94 #define	round_down_p2(a, s)	((a) & ~((s) - 1))
95 #define	round_up_p2(a, s)	(((a) + (s) - 1) & ~((s) - 1))
96 
97 /*
98  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100  */
101 #define	VA_PA_ALIGNED(uvaddr, paddr, pgsize)		\
102 	(((uvaddr | paddr) & (pgsize - 1)) == 0)
103 #define	VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)	\
104 	(((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105 
106 #define	vpgtob(n)	((n) * sizeof (struct vpage))	/* For brevity */
107 
108 #define	VTOCVP(vp)	(VTOS(vp)->s_commonvp)	/* we "know" it's an snode */
109 
110 static struct devmap_ctx *devmapctx_list = NULL;
111 static struct devmap_softlock *devmap_slist = NULL;
112 
113 /*
114  * mutex, vnode and page for the page of zeros we use for the trash mappings.
115  * One trash page is allocated on the first ddi_umem_setup call that uses it
116  * XXX Eventually, we may want to combine this with what segnf does when all
117  * hat layers implement HAT_NOFAULT.
118  *
119  * The trash page is used when the backing store for a userland mapping is
120  * removed but the application semantics do not take kindly to a SIGBUS.
121  * In that scenario, the applications pages are mapped to some dummy page
122  * which returns garbage on read and writes go into a common place.
123  * (Perfect for NO_FAULT semantics)
124  * The device driver is responsible to communicating to the app with some
125  * other mechanism that such remapping has happened and the app should take
126  * corrective action.
127  * We can also use an anonymous memory page as there is no requirement to
128  * keep the page locked, however this complicates the fault code. RFE.
129  */
130 static struct vnode trashvp;
131 static struct page *trashpp;
132 
133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 static vmem_t *umem_np_arena;
135 
136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 #define	DEVMAP_DEVMEM_COOKIE	((ddi_umem_cookie_t)0x1)
138 
139 /*
140  * Macros to check if type of devmap handle
141  */
142 #define	cookie_is_devmem(c)	\
143 	((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144 
145 #define	cookie_is_pmem(c)	\
146 	((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147 
148 #define	cookie_is_kpmem(c)	(!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 	((c)->type == KMEM_PAGEABLE))
150 
151 #define	dhp_is_devmem(dhp)	\
152 	(cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153 
154 #define	dhp_is_pmem(dhp)	\
155 	(cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156 
157 #define	dhp_is_kpmem(dhp)	\
158 	(cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159 
160 /*
161  * Private seg op routines.
162  */
163 static int	segdev_dup(struct seg *, struct seg *);
164 static int	segdev_unmap(struct seg *, caddr_t, size_t);
165 static void	segdev_free(struct seg *);
166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 		    enum fault_type, enum seg_rw);
168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 static int	segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 static int	segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 static void	segdev_badop(void);
172 static int	segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 static size_t	segdev_incore(struct seg *, caddr_t, size_t, char *);
174 static int	segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 		    ulong_t *, size_t);
176 static int	segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 static u_offset_t	segdev_getoffset(struct seg *, caddr_t);
178 static int	segdev_gettype(struct seg *, caddr_t);
179 static int	segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 static int	segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 static void	segdev_dump(struct seg *);
182 static int	segdev_pagelock(struct seg *, caddr_t, size_t,
183 		    struct page ***, enum lock_type, enum seg_rw);
184 static int	segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 static int	segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 static lgrp_mem_policy_info_t	*segdev_getpolicy(struct seg *, caddr_t);
187 static int	segdev_capable(struct seg *, segcapability_t);
188 
189 /*
190  * XXX	this struct is used by rootnex_map_fault to identify
191  *	the segment it has been passed. So if you make it
192  *	"static" you'll need to fix rootnex_map_fault.
193  */
194 struct seg_ops segdev_ops = {
195 	segdev_dup,
196 	segdev_unmap,
197 	segdev_free,
198 	segdev_fault,
199 	segdev_faulta,
200 	segdev_setprot,
201 	segdev_checkprot,
202 	(int (*)())segdev_badop,	/* kluster */
203 	(size_t (*)(struct seg *))NULL,	/* swapout */
204 	segdev_sync,			/* sync */
205 	segdev_incore,
206 	segdev_lockop,			/* lockop */
207 	segdev_getprot,
208 	segdev_getoffset,
209 	segdev_gettype,
210 	segdev_getvp,
211 	segdev_advise,
212 	segdev_dump,
213 	segdev_pagelock,
214 	segdev_setpagesize,
215 	segdev_getmemid,
216 	segdev_getpolicy,
217 	segdev_capable,
218 	seg_inherit_notsup
219 };
220 
221 /*
222  * Private segdev support routines
223  */
224 static struct segdev_data *sdp_alloc(void);
225 
226 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
227     size_t, enum seg_rw);
228 
229 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
230     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
231 
232 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
233     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
234 
235 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
236 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
237 static void devmap_softlock_rele(devmap_handle_t *);
238 static void devmap_ctx_rele(devmap_handle_t *);
239 
240 static void devmap_ctxto(void *);
241 
242 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
243     caddr_t addr);
244 
245 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
246     ulong_t *opfn, ulong_t *pagesize);
247 
248 static void free_devmap_handle(devmap_handle_t *dhp);
249 
250 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
251     struct seg *newseg);
252 
253 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
254 
255 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
256 
257 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
258 
259 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
260     offset_t off, size_t len, uint_t flags);
261 
262 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
263     caddr_t addr, size_t *llen, caddr_t *laddr);
264 
265 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
266 
267 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
268 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
269 
270 static void *devmap_umem_alloc_np(size_t size, size_t flags);
271 static void devmap_umem_free_np(void *addr, size_t size);
272 
273 /*
274  * routines to lock and unlock underlying segkp segment for
275  * KMEM_PAGEABLE type cookies.
276  */
277 static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
278 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
279 
280 /*
281  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
282  * drivers with devmap_access callbacks
283  */
284 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
285 	enum fault_type);
286 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
287 	enum fault_type);
288 
289 static kmutex_t devmapctx_lock;
290 
291 static kmutex_t devmap_slock;
292 
293 /*
294  * Initialize the thread callbacks and thread private data.
295  */
296 static struct devmap_ctx *
devmap_ctxinit(dev_t dev,ulong_t id)297 devmap_ctxinit(dev_t dev, ulong_t id)
298 {
299 	struct devmap_ctx	*devctx;
300 	struct devmap_ctx	*tmp;
301 	dev_info_t		*dip;
302 
303 	tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
304 
305 	mutex_enter(&devmapctx_lock);
306 
307 	dip = e_ddi_hold_devi_by_dev(dev, 0);
308 	ASSERT(dip != NULL);
309 	ddi_release_devi(dip);
310 
311 	for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
312 		if ((devctx->dip == dip) && (devctx->id == id))
313 			break;
314 
315 	if (devctx == NULL) {
316 		devctx = tmp;
317 		devctx->dip = dip;
318 		devctx->id = id;
319 		mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
320 		cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
321 		devctx->next = devmapctx_list;
322 		devmapctx_list = devctx;
323 	} else
324 		kmem_free(tmp, sizeof (struct devmap_ctx));
325 
326 	mutex_enter(&devctx->lock);
327 	devctx->refcnt++;
328 	mutex_exit(&devctx->lock);
329 	mutex_exit(&devmapctx_lock);
330 
331 	return (devctx);
332 }
333 
334 /*
335  * Timeout callback called if a CPU has not given up the device context
336  * within dhp->dh_timeout_length ticks
337  */
338 static void
devmap_ctxto(void * data)339 devmap_ctxto(void *data)
340 {
341 	struct devmap_ctx *devctx = data;
342 
343 	TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
344 	    "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
345 	mutex_enter(&devctx->lock);
346 	/*
347 	 * Set oncpu = 0 so the next mapping trying to get the device context
348 	 * can.
349 	 */
350 	devctx->oncpu = 0;
351 	devctx->timeout = 0;
352 	cv_signal(&devctx->cv);
353 	mutex_exit(&devctx->lock);
354 }
355 
356 /*
357  * Create a device segment.
358  */
359 int
segdev_create(struct seg * seg,void * argsp)360 segdev_create(struct seg *seg, void *argsp)
361 {
362 	struct segdev_data *sdp;
363 	struct segdev_crargs *a = (struct segdev_crargs *)argsp;
364 	devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
365 	int error;
366 
367 	/*
368 	 * Since the address space is "write" locked, we
369 	 * don't need the segment lock to protect "segdev" data.
370 	 */
371 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
372 
373 	hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
374 
375 	sdp = sdp_alloc();
376 
377 	sdp->mapfunc = a->mapfunc;
378 	sdp->offset = a->offset;
379 	sdp->prot = a->prot;
380 	sdp->maxprot = a->maxprot;
381 	sdp->type = a->type;
382 	sdp->pageprot = 0;
383 	sdp->softlockcnt = 0;
384 	sdp->vpage = NULL;
385 
386 	if (sdp->mapfunc == NULL)
387 		sdp->devmap_data = dhp;
388 	else
389 		sdp->devmap_data = dhp = NULL;
390 
391 	sdp->hat_flags = a->hat_flags;
392 	sdp->hat_attr = a->hat_attr;
393 
394 	/*
395 	 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
396 	 */
397 	ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
398 
399 	/*
400 	 * Hold shadow vnode -- segdev only deals with
401 	 * character (VCHR) devices. We use the common
402 	 * vp to hang pages on.
403 	 */
404 	sdp->vp = specfind(a->dev, VCHR);
405 	ASSERT(sdp->vp != NULL);
406 
407 	seg->s_ops = &segdev_ops;
408 	seg->s_data = sdp;
409 
410 	while (dhp != NULL) {
411 		dhp->dh_seg = seg;
412 		dhp = dhp->dh_next;
413 	}
414 
415 	/*
416 	 * Inform the vnode of the new mapping.
417 	 */
418 	/*
419 	 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
420 	 * dhp specific maxprot because spec_addmap does not use maxprot.
421 	 */
422 	error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
423 	    seg->s_as, seg->s_base, seg->s_size,
424 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
425 
426 	if (error != 0) {
427 		sdp->devmap_data = NULL;
428 		hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
429 		    HAT_UNLOAD_UNMAP);
430 	} else {
431 		/*
432 		 * Mappings of /dev/null don't count towards the VSZ of a
433 		 * process.  Mappings of /dev/null have no mapping type.
434 		 */
435 		if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
436 		    MAP_PRIVATE)) == 0) {
437 			seg->s_as->a_resvsize -= seg->s_size;
438 		}
439 	}
440 
441 	return (error);
442 }
443 
444 static struct segdev_data *
sdp_alloc(void)445 sdp_alloc(void)
446 {
447 	struct segdev_data *sdp;
448 
449 	sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
450 	rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
451 
452 	return (sdp);
453 }
454 
455 /*
456  * Duplicate seg and return new segment in newseg.
457  */
458 static int
segdev_dup(struct seg * seg,struct seg * newseg)459 segdev_dup(struct seg *seg, struct seg *newseg)
460 {
461 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
462 	struct segdev_data *newsdp;
463 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
464 	size_t npages;
465 	int ret;
466 
467 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
468 	    "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
469 
470 	DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
471 	    (void *)dhp, (void *)seg));
472 
473 	/*
474 	 * Since the address space is "write" locked, we
475 	 * don't need the segment lock to protect "segdev" data.
476 	 */
477 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
478 
479 	newsdp = sdp_alloc();
480 
481 	newseg->s_ops = seg->s_ops;
482 	newseg->s_data = (void *)newsdp;
483 
484 	VN_HOLD(sdp->vp);
485 	newsdp->vp 	= sdp->vp;
486 	newsdp->mapfunc = sdp->mapfunc;
487 	newsdp->offset	= sdp->offset;
488 	newsdp->pageprot = sdp->pageprot;
489 	newsdp->prot	= sdp->prot;
490 	newsdp->maxprot = sdp->maxprot;
491 	newsdp->type = sdp->type;
492 	newsdp->hat_attr = sdp->hat_attr;
493 	newsdp->hat_flags = sdp->hat_flags;
494 	newsdp->softlockcnt = 0;
495 
496 	/*
497 	 * Initialize per page data if the segment we are
498 	 * dup'ing has per page information.
499 	 */
500 	npages = seg_pages(newseg);
501 
502 	if (sdp->vpage != NULL) {
503 		size_t nbytes = vpgtob(npages);
504 
505 		newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
506 		bcopy(sdp->vpage, newsdp->vpage, nbytes);
507 	} else
508 		newsdp->vpage = NULL;
509 
510 	/*
511 	 * duplicate devmap handles
512 	 */
513 	if (dhp != NULL) {
514 		ret = devmap_handle_dup(dhp,
515 		    (devmap_handle_t **)&newsdp->devmap_data, newseg);
516 		if (ret != 0) {
517 			TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
518 			    "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
519 			    ret, (void *)dhp, (void *)seg);
520 			DEBUGF(1, (CE_CONT,
521 			    "segdev_dup: ret %x dhp %p seg %p\n",
522 			    ret, (void *)dhp, (void *)seg));
523 			return (ret);
524 		}
525 	}
526 
527 	/*
528 	 * Inform the common vnode of the new mapping.
529 	 */
530 	return (VOP_ADDMAP(VTOCVP(newsdp->vp),
531 	    newsdp->offset, newseg->s_as,
532 	    newseg->s_base, newseg->s_size, newsdp->prot,
533 	    newsdp->maxprot, sdp->type, CRED(), NULL));
534 }
535 
536 /*
537  * duplicate devmap handles
538  */
539 static int
devmap_handle_dup(devmap_handle_t * dhp,devmap_handle_t ** new_dhp,struct seg * newseg)540 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
541     struct seg *newseg)
542 {
543 	devmap_handle_t *newdhp_save = NULL;
544 	devmap_handle_t *newdhp = NULL;
545 	struct devmap_callback_ctl *callbackops;
546 
547 	while (dhp != NULL) {
548 		newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
549 
550 		/* Need to lock the original dhp while copying if REMAP */
551 		HOLD_DHP_LOCK(dhp);
552 		bcopy(dhp, newdhp, sizeof (devmap_handle_t));
553 		RELE_DHP_LOCK(dhp);
554 		newdhp->dh_seg = newseg;
555 		newdhp->dh_next = NULL;
556 		if (newdhp_save != NULL)
557 			newdhp_save->dh_next = newdhp;
558 		else
559 			*new_dhp = newdhp;
560 		newdhp_save = newdhp;
561 
562 		callbackops = &newdhp->dh_callbackops;
563 
564 		if (dhp->dh_softlock != NULL)
565 			newdhp->dh_softlock = devmap_softlock_init(
566 			    newdhp->dh_dev,
567 			    (ulong_t)callbackops->devmap_access);
568 		if (dhp->dh_ctx != NULL)
569 			newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
570 			    (ulong_t)callbackops->devmap_access);
571 
572 		/*
573 		 * Initialize dh_lock if we want to do remap.
574 		 */
575 		if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
576 			mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
577 			newdhp->dh_flags |= DEVMAP_LOCK_INITED;
578 		}
579 
580 		if (callbackops->devmap_dup != NULL) {
581 			int ret;
582 
583 			/*
584 			 * Call the dup callback so that the driver can
585 			 * duplicate its private data.
586 			 */
587 			ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
588 			    (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
589 
590 			if (ret != 0) {
591 				/*
592 				 * We want to free up this segment as the driver
593 				 * has indicated that we can't dup it.  But we
594 				 * don't want to call the drivers, devmap_unmap,
595 				 * callback function as the driver does not
596 				 * think this segment exists. The caller of
597 				 * devmap_dup will call seg_free on newseg
598 				 * as it was the caller that allocated the
599 				 * segment.
600 				 */
601 				DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
602 				    "newdhp %p dhp %p\n", (void *)newdhp,
603 				    (void *)dhp));
604 				callbackops->devmap_unmap = NULL;
605 				return (ret);
606 			}
607 		}
608 
609 		dhp = dhp->dh_next;
610 	}
611 
612 	return (0);
613 }
614 
615 /*
616  * Split a segment at addr for length len.
617  */
618 /*ARGSUSED*/
619 static int
segdev_unmap(struct seg * seg,caddr_t addr,size_t len)620 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
621 {
622 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
623 	register struct segdev_data *nsdp;
624 	register struct seg *nseg;
625 	register size_t	opages;		/* old segment size in pages */
626 	register size_t	npages;		/* new segment size in pages */
627 	register size_t	dpages;		/* pages being deleted (unmapped) */
628 	register size_t	nbytes;
629 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
630 	devmap_handle_t *dhpp;
631 	devmap_handle_t *newdhp;
632 	struct devmap_callback_ctl *callbackops;
633 	caddr_t nbase;
634 	offset_t off;
635 	ulong_t nsize;
636 	size_t mlen, sz;
637 
638 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
639 	    "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
640 	    (void *)dhp, (void *)seg, (void *)addr, len);
641 
642 	DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
643 	    (void *)dhp, (void *)seg, (void *)addr, len));
644 
645 	/*
646 	 * Since the address space is "write" locked, we
647 	 * don't need the segment lock to protect "segdev" data.
648 	 */
649 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
650 
651 	if ((sz = sdp->softlockcnt) > 0) {
652 		/*
653 		 * Fail the unmap if pages are SOFTLOCKed through this mapping.
654 		 * softlockcnt is protected from change by the as write lock.
655 		 */
656 		TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
657 		    "segdev_unmap:error softlockcnt = %ld", sz);
658 		DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
659 		return (EAGAIN);
660 	}
661 
662 	/*
663 	 * Check for bad sizes
664 	 */
665 	if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
666 	    (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
667 		panic("segdev_unmap");
668 
669 	if (dhp != NULL) {
670 		devmap_handle_t *tdhp;
671 		/*
672 		 * If large page size was used in hat_devload(),
673 		 * the same page size must be used in hat_unload().
674 		 */
675 		dhpp = tdhp = devmap_find_handle(dhp, addr);
676 		while (tdhp != NULL) {
677 			if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
678 				break;
679 			}
680 			tdhp = tdhp->dh_next;
681 		}
682 		if (tdhp != NULL) {	/* found a dhp using large pages */
683 			size_t slen = len;
684 			size_t mlen;
685 			size_t soff;
686 
687 			soff = (ulong_t)(addr - dhpp->dh_uvaddr);
688 			while (slen != 0) {
689 				mlen = MIN(slen, (dhpp->dh_len - soff));
690 				hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
691 				    dhpp->dh_len, HAT_UNLOAD_UNMAP);
692 				dhpp = dhpp->dh_next;
693 				ASSERT(slen >= mlen);
694 				slen -= mlen;
695 				soff = 0;
696 			}
697 		} else
698 			hat_unload(seg->s_as->a_hat, addr, len,
699 			    HAT_UNLOAD_UNMAP);
700 	} else {
701 		/*
702 		 * Unload any hardware translations in the range
703 		 * to be taken out.
704 		 */
705 		hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
706 	}
707 
708 	/*
709 	 * get the user offset which will used in the driver callbacks
710 	 */
711 	off = sdp->offset + (offset_t)(addr - seg->s_base);
712 
713 	/*
714 	 * Inform the vnode of the unmapping.
715 	 */
716 	ASSERT(sdp->vp != NULL);
717 	(void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
718 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
719 
720 	/*
721 	 * Check for entire segment
722 	 */
723 	if (addr == seg->s_base && len == seg->s_size) {
724 		seg_free(seg);
725 		return (0);
726 	}
727 
728 	opages = seg_pages(seg);
729 	dpages = btop(len);
730 	npages = opages - dpages;
731 
732 	/*
733 	 * Check for beginning of segment
734 	 */
735 	if (addr == seg->s_base) {
736 		if (sdp->vpage != NULL) {
737 			register struct vpage *ovpage;
738 
739 			ovpage = sdp->vpage;	/* keep pointer to vpage */
740 
741 			nbytes = vpgtob(npages);
742 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
743 			bcopy(&ovpage[dpages], sdp->vpage, nbytes);
744 
745 			/* free up old vpage */
746 			kmem_free(ovpage, vpgtob(opages));
747 		}
748 
749 		/*
750 		 * free devmap handles from the beginning of the mapping.
751 		 */
752 		if (dhp != NULL)
753 			devmap_handle_unmap_head(dhp, len);
754 
755 		sdp->offset += (offset_t)len;
756 
757 		seg->s_base += len;
758 		seg->s_size -= len;
759 
760 		return (0);
761 	}
762 
763 	/*
764 	 * Check for end of segment
765 	 */
766 	if (addr + len == seg->s_base + seg->s_size) {
767 		if (sdp->vpage != NULL) {
768 			register struct vpage *ovpage;
769 
770 			ovpage = sdp->vpage;	/* keep pointer to vpage */
771 
772 			nbytes = vpgtob(npages);
773 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
774 			bcopy(ovpage, sdp->vpage, nbytes);
775 
776 			/* free up old vpage */
777 			kmem_free(ovpage, vpgtob(opages));
778 		}
779 		seg->s_size -= len;
780 
781 		/*
782 		 * free devmap handles from addr to the end of the mapping.
783 		 */
784 		if (dhp != NULL)
785 			devmap_handle_unmap_tail(dhp, addr);
786 
787 		return (0);
788 	}
789 
790 	/*
791 	 * The section to go is in the middle of the segment,
792 	 * have to make it into two segments.  nseg is made for
793 	 * the high end while seg is cut down at the low end.
794 	 */
795 	nbase = addr + len;				/* new seg base */
796 	nsize = (seg->s_base + seg->s_size) - nbase;	/* new seg size */
797 	seg->s_size = addr - seg->s_base;		/* shrink old seg */
798 	nseg = seg_alloc(seg->s_as, nbase, nsize);
799 	if (nseg == NULL)
800 		panic("segdev_unmap seg_alloc");
801 
802 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
803 	    "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
804 	DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
805 	    (void *)seg, (void *)nseg));
806 	nsdp = sdp_alloc();
807 
808 	nseg->s_ops = seg->s_ops;
809 	nseg->s_data = (void *)nsdp;
810 
811 	VN_HOLD(sdp->vp);
812 	nsdp->mapfunc = sdp->mapfunc;
813 	nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
814 	nsdp->vp 	= sdp->vp;
815 	nsdp->pageprot = sdp->pageprot;
816 	nsdp->prot	= sdp->prot;
817 	nsdp->maxprot = sdp->maxprot;
818 	nsdp->type = sdp->type;
819 	nsdp->hat_attr = sdp->hat_attr;
820 	nsdp->hat_flags = sdp->hat_flags;
821 	nsdp->softlockcnt = 0;
822 
823 	/*
824 	 * Initialize per page data if the segment we are
825 	 * dup'ing has per page information.
826 	 */
827 	if (sdp->vpage != NULL) {
828 		/* need to split vpage into two arrays */
829 		register size_t nnbytes;
830 		register size_t nnpages;
831 		register struct vpage *ovpage;
832 
833 		ovpage = sdp->vpage;		/* keep pointer to vpage */
834 
835 		npages = seg_pages(seg);	/* seg has shrunk */
836 		nbytes = vpgtob(npages);
837 		nnpages = seg_pages(nseg);
838 		nnbytes = vpgtob(nnpages);
839 
840 		sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
841 		bcopy(ovpage, sdp->vpage, nbytes);
842 
843 		nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
844 		bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
845 
846 		/* free up old vpage */
847 		kmem_free(ovpage, vpgtob(opages));
848 	} else
849 		nsdp->vpage = NULL;
850 
851 	/*
852 	 * unmap dhps.
853 	 */
854 	if (dhp == NULL) {
855 		nsdp->devmap_data = NULL;
856 		return (0);
857 	}
858 	while (dhp != NULL) {
859 		callbackops = &dhp->dh_callbackops;
860 		TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
861 		    "segdev_unmap: dhp=%p addr=%p", dhp, addr);
862 		DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
863 		    (void *)dhp, (void *)addr,
864 		    (void *)dhp->dh_uvaddr, dhp->dh_len));
865 
866 		if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
867 			dhpp = dhp->dh_next;
868 			dhp->dh_next = NULL;
869 			dhp = dhpp;
870 		} else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
871 			dhp = dhp->dh_next;
872 		} else if (addr > dhp->dh_uvaddr &&
873 		    (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
874 			/*
875 			 * <addr, addr+len> is enclosed by dhp.
876 			 * create a newdhp that begins at addr+len and
877 			 * ends at dhp->dh_uvaddr+dhp->dh_len.
878 			 */
879 			newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
880 			HOLD_DHP_LOCK(dhp);
881 			bcopy(dhp, newdhp, sizeof (devmap_handle_t));
882 			RELE_DHP_LOCK(dhp);
883 			newdhp->dh_seg = nseg;
884 			newdhp->dh_next = dhp->dh_next;
885 			if (dhp->dh_softlock != NULL)
886 				newdhp->dh_softlock = devmap_softlock_init(
887 				    newdhp->dh_dev,
888 				    (ulong_t)callbackops->devmap_access);
889 			if (dhp->dh_ctx != NULL)
890 				newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
891 				    (ulong_t)callbackops->devmap_access);
892 			if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
893 				mutex_init(&newdhp->dh_lock,
894 				    NULL, MUTEX_DEFAULT, NULL);
895 			}
896 			if (callbackops->devmap_unmap != NULL)
897 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
898 				    off, len, dhp, &dhp->dh_pvtp,
899 				    newdhp, &newdhp->dh_pvtp);
900 			mlen = len + (addr - dhp->dh_uvaddr);
901 			devmap_handle_reduce_len(newdhp, mlen);
902 			nsdp->devmap_data = newdhp;
903 			/* XX Changing len should recalculate LARGE flag */
904 			dhp->dh_len = addr - dhp->dh_uvaddr;
905 			dhpp = dhp->dh_next;
906 			dhp->dh_next = NULL;
907 			dhp = dhpp;
908 		} else if ((addr > dhp->dh_uvaddr) &&
909 		    ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
910 			mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
911 			/*
912 			 * <addr, addr+len> spans over dhps.
913 			 */
914 			if (callbackops->devmap_unmap != NULL)
915 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
916 				    off, mlen, (devmap_cookie_t *)dhp,
917 				    &dhp->dh_pvtp, NULL, NULL);
918 			/* XX Changing len should recalculate LARGE flag */
919 			dhp->dh_len = addr - dhp->dh_uvaddr;
920 			dhpp = dhp->dh_next;
921 			dhp->dh_next = NULL;
922 			dhp = dhpp;
923 			nsdp->devmap_data = dhp;
924 		} else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
925 			/*
926 			 * dhp is enclosed by <addr, addr+len>.
927 			 */
928 			dhp->dh_seg = nseg;
929 			nsdp->devmap_data = dhp;
930 			dhp = devmap_handle_unmap(dhp);
931 			nsdp->devmap_data = dhp; /* XX redundant? */
932 		} else if (((addr + len) > dhp->dh_uvaddr) &&
933 		    ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
934 			mlen = addr + len - dhp->dh_uvaddr;
935 			if (callbackops->devmap_unmap != NULL)
936 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
937 				    dhp->dh_uoff, mlen, NULL,
938 				    NULL, dhp, &dhp->dh_pvtp);
939 			devmap_handle_reduce_len(dhp, mlen);
940 			nsdp->devmap_data = dhp;
941 			dhp->dh_seg = nseg;
942 			dhp = dhp->dh_next;
943 		} else {
944 			dhp->dh_seg = nseg;
945 			dhp = dhp->dh_next;
946 		}
947 	}
948 	return (0);
949 }
950 
951 /*
952  * Utility function handles reducing the length of a devmap handle during unmap
953  * Note that is only used for unmapping the front portion of the handler,
954  * i.e., we are bumping up the offset/pfn etc up by len
955  * Do not use if reducing length at the tail.
956  */
957 static void
devmap_handle_reduce_len(devmap_handle_t * dhp,size_t len)958 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
959 {
960 	struct ddi_umem_cookie *cp;
961 	struct devmap_pmem_cookie *pcp;
962 	/*
963 	 * adjust devmap handle fields
964 	 */
965 	ASSERT(len < dhp->dh_len);
966 
967 	/* Make sure only page-aligned changes are done */
968 	ASSERT((len & PAGEOFFSET) == 0);
969 
970 	dhp->dh_len -= len;
971 	dhp->dh_uoff += (offset_t)len;
972 	dhp->dh_roff += (offset_t)len;
973 	dhp->dh_uvaddr += len;
974 	/* Need to grab dhp lock if REMAP */
975 	HOLD_DHP_LOCK(dhp);
976 	cp = dhp->dh_cookie;
977 	if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
978 		if (cookie_is_devmem(cp)) {
979 			dhp->dh_pfn += btop(len);
980 		} else if (cookie_is_pmem(cp)) {
981 			pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
982 			ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
983 			    dhp->dh_roff < ptob(pcp->dp_npages));
984 		} else {
985 			ASSERT(dhp->dh_roff < cp->size);
986 			ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
987 			    dhp->dh_cvaddr < (cp->cvaddr + cp->size));
988 			ASSERT((dhp->dh_cvaddr + len) <=
989 			    (cp->cvaddr + cp->size));
990 
991 			dhp->dh_cvaddr += len;
992 		}
993 	}
994 	/* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
995 	RELE_DHP_LOCK(dhp);
996 }
997 
998 /*
999  * Free devmap handle, dhp.
1000  * Return the next devmap handle on the linked list.
1001  */
1002 static devmap_handle_t *
devmap_handle_unmap(devmap_handle_t * dhp)1003 devmap_handle_unmap(devmap_handle_t *dhp)
1004 {
1005 	struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1006 	struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1007 	devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1008 
1009 	ASSERT(dhp != NULL);
1010 
1011 	/*
1012 	 * before we free up dhp, call the driver's devmap_unmap entry point
1013 	 * to free resources allocated for this dhp.
1014 	 */
1015 	if (callbackops->devmap_unmap != NULL) {
1016 		(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1017 		    dhp->dh_len, NULL, NULL, NULL, NULL);
1018 	}
1019 
1020 	if (dhpp == dhp) {	/* releasing first dhp, change sdp data */
1021 		sdp->devmap_data = dhp->dh_next;
1022 	} else {
1023 		while (dhpp->dh_next != dhp) {
1024 			dhpp = dhpp->dh_next;
1025 		}
1026 		dhpp->dh_next = dhp->dh_next;
1027 	}
1028 	dhpp = dhp->dh_next;	/* return value is next dhp in chain */
1029 
1030 	if (dhp->dh_softlock != NULL)
1031 		devmap_softlock_rele(dhp);
1032 
1033 	if (dhp->dh_ctx != NULL)
1034 		devmap_ctx_rele(dhp);
1035 
1036 	if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1037 		mutex_destroy(&dhp->dh_lock);
1038 	}
1039 	kmem_free(dhp, sizeof (devmap_handle_t));
1040 
1041 	return (dhpp);
1042 }
1043 
1044 /*
1045  * Free complete devmap handles from dhp for len bytes
1046  * dhp can be either the first handle or a subsequent handle
1047  */
1048 static void
devmap_handle_unmap_head(devmap_handle_t * dhp,size_t len)1049 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1050 {
1051 	struct devmap_callback_ctl *callbackops;
1052 
1053 	/*
1054 	 * free the devmap handles covered by len.
1055 	 */
1056 	while (len >= dhp->dh_len) {
1057 		len -= dhp->dh_len;
1058 		dhp = devmap_handle_unmap(dhp);
1059 	}
1060 	if (len != 0) {	/* partial unmap at head of first remaining dhp */
1061 		callbackops = &dhp->dh_callbackops;
1062 
1063 		/*
1064 		 * Call the unmap callback so the drivers can make
1065 		 * adjustment on its private data.
1066 		 */
1067 		if (callbackops->devmap_unmap != NULL)
1068 			(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1069 			    dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1070 		devmap_handle_reduce_len(dhp, len);
1071 	}
1072 }
1073 
1074 /*
1075  * Free devmap handles to truncate  the mapping after addr
1076  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1077  *	Also could then use the routine in middle unmap case too
1078  */
1079 static void
devmap_handle_unmap_tail(devmap_handle_t * dhp,caddr_t addr)1080 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1081 {
1082 	register struct seg *seg = dhp->dh_seg;
1083 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1084 	register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1085 	struct devmap_callback_ctl *callbackops;
1086 	register devmap_handle_t *dhpp;
1087 	size_t maplen;
1088 	ulong_t off;
1089 	size_t len;
1090 
1091 	maplen = (size_t)(addr - dhp->dh_uvaddr);
1092 	dhph = devmap_find_handle(dhph, addr);
1093 
1094 	while (dhph != NULL) {
1095 		if (maplen == 0) {
1096 			dhph =  devmap_handle_unmap(dhph);
1097 		} else {
1098 			callbackops = &dhph->dh_callbackops;
1099 			len = dhph->dh_len - maplen;
1100 			off = (ulong_t)sdp->offset + (addr - seg->s_base);
1101 			/*
1102 			 * Call the unmap callback so the driver
1103 			 * can make adjustments on its private data.
1104 			 */
1105 			if (callbackops->devmap_unmap != NULL)
1106 				(*callbackops->devmap_unmap)(dhph,
1107 				    dhph->dh_pvtp, off, len,
1108 				    (devmap_cookie_t *)dhph,
1109 				    &dhph->dh_pvtp, NULL, NULL);
1110 			/* XXX Reducing len needs to recalculate LARGE flag */
1111 			dhph->dh_len = maplen;
1112 			maplen = 0;
1113 			dhpp = dhph->dh_next;
1114 			dhph->dh_next = NULL;
1115 			dhph = dhpp;
1116 		}
1117 	} /* end while */
1118 }
1119 
1120 /*
1121  * Free a segment.
1122  */
1123 static void
segdev_free(struct seg * seg)1124 segdev_free(struct seg *seg)
1125 {
1126 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1127 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1128 
1129 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1130 	    "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1131 	DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1132 	    (void *)dhp, (void *)seg));
1133 
1134 	/*
1135 	 * Since the address space is "write" locked, we
1136 	 * don't need the segment lock to protect "segdev" data.
1137 	 */
1138 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1139 
1140 	while (dhp != NULL)
1141 		dhp = devmap_handle_unmap(dhp);
1142 
1143 	VN_RELE(sdp->vp);
1144 	if (sdp->vpage != NULL)
1145 		kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1146 
1147 	rw_destroy(&sdp->lock);
1148 	kmem_free(sdp, sizeof (*sdp));
1149 }
1150 
1151 static void
free_devmap_handle(devmap_handle_t * dhp)1152 free_devmap_handle(devmap_handle_t *dhp)
1153 {
1154 	register devmap_handle_t *dhpp;
1155 
1156 	/*
1157 	 * free up devmap handle
1158 	 */
1159 	while (dhp != NULL) {
1160 		dhpp = dhp->dh_next;
1161 		if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1162 			mutex_destroy(&dhp->dh_lock);
1163 		}
1164 
1165 		if (dhp->dh_softlock != NULL)
1166 			devmap_softlock_rele(dhp);
1167 
1168 		if (dhp->dh_ctx != NULL)
1169 			devmap_ctx_rele(dhp);
1170 
1171 		kmem_free(dhp, sizeof (devmap_handle_t));
1172 		dhp = dhpp;
1173 	}
1174 }
1175 
1176 /*
1177  * routines to lock and unlock underlying segkp segment for
1178  * KMEM_PAGEABLE type cookies.
1179  * segkp only allows a single pending F_SOFTLOCK
1180  * we keep track of number of locks in the cookie so we can
1181  * have multiple pending faults and manage the calls to segkp.
1182  * RFE: if segkp supports either pagelock or can support multiple
1183  * calls to F_SOFTLOCK, then these routines can go away.
1184  *	If pagelock, segdev_faultpage can fault on a page by page basis
1185  *		and simplifies the code quite a bit.
1186  *	if multiple calls allowed but not partial ranges, then need for
1187  *	cookie->lock and locked count goes away, code can call as_fault directly
1188  */
1189 static faultcode_t
acquire_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1190 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1191 {
1192 	int err = 0;
1193 	ASSERT(cookie_is_kpmem(cookie));
1194 	/*
1195 	 * Fault in pages in segkp with F_SOFTLOCK.
1196 	 * We want to hold the lock until all pages have been loaded.
1197 	 * segkp only allows single caller to hold SOFTLOCK, so cookie
1198 	 * holds a count so we dont call into segkp multiple times
1199 	 */
1200 	mutex_enter(&cookie->lock);
1201 
1202 	/*
1203 	 * Check for overflow in locked field
1204 	 */
1205 	if ((UINT32_MAX - cookie->locked) < npages) {
1206 		err = FC_MAKE_ERR(ENOMEM);
1207 	} else if (cookie->locked == 0) {
1208 		/* First time locking */
1209 		err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1210 		    cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1211 	}
1212 	if (!err) {
1213 		cookie->locked += npages;
1214 	}
1215 	mutex_exit(&cookie->lock);
1216 	return (err);
1217 }
1218 
1219 static void
release_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1220 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1221 {
1222 	mutex_enter(&cookie->lock);
1223 	ASSERT(cookie_is_kpmem(cookie));
1224 	ASSERT(cookie->locked >= npages);
1225 	cookie->locked -= (uint_t)npages;
1226 	if (cookie->locked == 0) {
1227 		/* Last unlock */
1228 		if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1229 		    cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1230 			panic("segdev releasing kpmem lock %p", (void *)cookie);
1231 	}
1232 	mutex_exit(&cookie->lock);
1233 }
1234 
1235 /*
1236  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1237  * drivers with devmap_access callbacks
1238  * slock->softlocked basically works like a rw lock
1239  *	-ve counts => F_SOFTLOCK in progress
1240  *	+ve counts => F_INVAL/F_PROT in progress
1241  * We allow only one F_SOFTLOCK at a time
1242  * but can have multiple pending F_INVAL/F_PROT calls
1243  *
1244  * This routine waits using cv_wait_sig so killing processes is more graceful
1245  * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1246  */
devmap_softlock_enter(struct devmap_softlock * slock,size_t npages,enum fault_type type)1247 static int devmap_softlock_enter(
1248 	struct devmap_softlock *slock,
1249 	size_t npages,
1250 	enum fault_type type)
1251 {
1252 	if (npages == 0)
1253 		return (0);
1254 	mutex_enter(&(slock->lock));
1255 	switch (type) {
1256 	case F_SOFTLOCK :
1257 		while (slock->softlocked) {
1258 			if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1259 				/* signalled */
1260 				mutex_exit(&(slock->lock));
1261 				return (EINTR);
1262 			}
1263 		}
1264 		slock->softlocked -= npages; /* -ve count => locked */
1265 		break;
1266 	case F_INVAL :
1267 	case F_PROT :
1268 		while (slock->softlocked < 0)
1269 			if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1270 				/* signalled */
1271 				mutex_exit(&(slock->lock));
1272 				return (EINTR);
1273 			}
1274 		slock->softlocked += npages; /* +ve count => f_invals */
1275 		break;
1276 	default:
1277 		ASSERT(0);
1278 	}
1279 	mutex_exit(&(slock->lock));
1280 	return (0);
1281 }
1282 
devmap_softlock_exit(struct devmap_softlock * slock,size_t npages,enum fault_type type)1283 static void devmap_softlock_exit(
1284 	struct devmap_softlock *slock,
1285 	size_t npages,
1286 	enum fault_type type)
1287 {
1288 	if (slock == NULL)
1289 		return;
1290 	mutex_enter(&(slock->lock));
1291 	switch (type) {
1292 	case F_SOFTLOCK :
1293 		ASSERT(-slock->softlocked >= npages);
1294 		slock->softlocked += npages;	/* -ve count is softlocked */
1295 		if (slock->softlocked == 0)
1296 			cv_signal(&slock->cv);
1297 		break;
1298 	case F_INVAL :
1299 	case F_PROT:
1300 		ASSERT(slock->softlocked >= npages);
1301 		slock->softlocked -= npages;
1302 		if (slock->softlocked == 0)
1303 			cv_signal(&slock->cv);
1304 		break;
1305 	default:
1306 		ASSERT(0);
1307 	}
1308 	mutex_exit(&(slock->lock));
1309 }
1310 
1311 /*
1312  * Do a F_SOFTUNLOCK call over the range requested.
1313  * The range must have already been F_SOFTLOCK'ed.
1314  * The segment lock should be held, (but not the segment private lock?)
1315  *  The softunlock code below does not adjust for large page sizes
1316  *	assumes the caller already did any addr/len adjustments for
1317  *	pagesize mappings before calling.
1318  */
1319 /*ARGSUSED*/
1320 static void
segdev_softunlock(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum seg_rw rw)1321 segdev_softunlock(
1322 	struct hat *hat,		/* the hat */
1323 	struct seg *seg,		/* seg_dev of interest */
1324 	caddr_t addr,			/* base address of range */
1325 	size_t len,			/* number of bytes */
1326 	enum seg_rw rw)			/* type of access at fault */
1327 {
1328 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1329 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1330 
1331 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1332 	    "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1333 	    dhp_head, sdp, addr, len);
1334 	DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1335 	    "addr %p len %lx\n",
1336 	    (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1337 
1338 	hat_unlock(hat, addr, len);
1339 
1340 	if (dhp_head != NULL) {
1341 		devmap_handle_t *dhp;
1342 		size_t mlen;
1343 		size_t tlen = len;
1344 		ulong_t off;
1345 
1346 		dhp = devmap_find_handle(dhp_head, addr);
1347 		ASSERT(dhp != NULL);
1348 
1349 		off = (ulong_t)(addr - dhp->dh_uvaddr);
1350 		while (tlen != 0) {
1351 			mlen = MIN(tlen, (dhp->dh_len - off));
1352 
1353 			/*
1354 			 * unlock segkp memory, locked during F_SOFTLOCK
1355 			 */
1356 			if (dhp_is_kpmem(dhp)) {
1357 				release_kpmem_lock(
1358 				    (struct ddi_umem_cookie *)dhp->dh_cookie,
1359 				    btopr(mlen));
1360 			}
1361 
1362 			/*
1363 			 * Do the softlock accounting for devmap_access
1364 			 */
1365 			if (dhp->dh_callbackops.devmap_access != NULL) {
1366 				devmap_softlock_exit(dhp->dh_softlock,
1367 				    btopr(mlen), F_SOFTLOCK);
1368 			}
1369 
1370 			tlen -= mlen;
1371 			dhp = dhp->dh_next;
1372 			off = 0;
1373 		}
1374 	}
1375 
1376 	mutex_enter(&freemem_lock);
1377 	ASSERT(sdp->softlockcnt >= btopr(len));
1378 	sdp->softlockcnt -= btopr(len);
1379 	mutex_exit(&freemem_lock);
1380 	if (sdp->softlockcnt == 0) {
1381 		/*
1382 		 * All SOFTLOCKS are gone. Wakeup any waiting
1383 		 * unmappers so they can try again to unmap.
1384 		 * Check for waiters first without the mutex
1385 		 * held so we don't always grab the mutex on
1386 		 * softunlocks.
1387 		 */
1388 		if (AS_ISUNMAPWAIT(seg->s_as)) {
1389 			mutex_enter(&seg->s_as->a_contents);
1390 			if (AS_ISUNMAPWAIT(seg->s_as)) {
1391 				AS_CLRUNMAPWAIT(seg->s_as);
1392 				cv_broadcast(&seg->s_as->a_cv);
1393 			}
1394 			mutex_exit(&seg->s_as->a_contents);
1395 		}
1396 	}
1397 
1398 }
1399 
1400 /*
1401  * Handle fault for a single page.
1402  * Done in a separate routine so we can handle errors more easily.
1403  * This routine is called only from segdev_faultpages()
1404  * when looping over the range of addresses requested. The segment lock is held.
1405  */
1406 static faultcode_t
segdev_faultpage(struct hat * hat,struct seg * seg,caddr_t addr,struct vpage * vpage,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1407 segdev_faultpage(
1408 	struct hat *hat,		/* the hat */
1409 	struct seg *seg,		/* seg_dev of interest */
1410 	caddr_t addr,			/* address in as */
1411 	struct vpage *vpage,		/* pointer to vpage for seg, addr */
1412 	enum fault_type type,		/* type of fault */
1413 	enum seg_rw rw,			/* type of access at fault */
1414 	devmap_handle_t *dhp)		/* devmap handle if any for this page */
1415 {
1416 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1417 	uint_t prot;
1418 	pfn_t pfnum = PFN_INVALID;
1419 	u_offset_t offset;
1420 	uint_t hat_flags;
1421 	dev_info_t *dip;
1422 
1423 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1424 	    "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1425 	DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1426 	    (void *)dhp, (void *)seg, (void *)addr));
1427 
1428 	/*
1429 	 * Initialize protection value for this page.
1430 	 * If we have per page protection values check it now.
1431 	 */
1432 	if (sdp->pageprot) {
1433 		uint_t protchk;
1434 
1435 		switch (rw) {
1436 		case S_READ:
1437 			protchk = PROT_READ;
1438 			break;
1439 		case S_WRITE:
1440 			protchk = PROT_WRITE;
1441 			break;
1442 		case S_EXEC:
1443 			protchk = PROT_EXEC;
1444 			break;
1445 		case S_OTHER:
1446 		default:
1447 			protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1448 			break;
1449 		}
1450 
1451 		prot = VPP_PROT(vpage);
1452 		if ((prot & protchk) == 0)
1453 			return (FC_PROT);	/* illegal access type */
1454 	} else {
1455 		prot = sdp->prot;
1456 		/* caller has already done segment level protection check */
1457 	}
1458 
1459 	if (type == F_SOFTLOCK) {
1460 		mutex_enter(&freemem_lock);
1461 		sdp->softlockcnt++;
1462 		mutex_exit(&freemem_lock);
1463 	}
1464 
1465 	hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1466 	offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1467 	/*
1468 	 * In the devmap framework, sdp->mapfunc is set to NULL.  we can get
1469 	 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1470 	 * seg->s_base.
1471 	 */
1472 	if (dhp == NULL) {
1473 		/* If segment has devmap_data, then dhp should be non-NULL */
1474 		ASSERT(sdp->devmap_data == NULL);
1475 		pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1476 		    (off_t)offset, prot);
1477 		prot |= sdp->hat_attr;
1478 	} else {
1479 		ulong_t off;
1480 		struct ddi_umem_cookie *cp;
1481 		struct devmap_pmem_cookie *pcp;
1482 
1483 		/* ensure the dhp passed in contains addr. */
1484 		ASSERT(dhp == devmap_find_handle(
1485 		    (devmap_handle_t *)sdp->devmap_data, addr));
1486 
1487 		off = addr - dhp->dh_uvaddr;
1488 
1489 		/*
1490 		 * This routine assumes that the caller makes sure that the
1491 		 * fields in dhp used below are unchanged due to remap during
1492 		 * this call. Caller does HOLD_DHP_LOCK if neeed
1493 		 */
1494 		cp = dhp->dh_cookie;
1495 		if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1496 			pfnum = PFN_INVALID;
1497 		} else if (cookie_is_devmem(cp)) {
1498 			pfnum = dhp->dh_pfn + btop(off);
1499 		} else if (cookie_is_pmem(cp)) {
1500 			pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1501 			ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1502 			    dhp->dh_roff < ptob(pcp->dp_npages));
1503 			pfnum = page_pptonum(
1504 			    pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1505 		} else {
1506 			ASSERT(dhp->dh_roff < cp->size);
1507 			ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1508 			    dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1509 			ASSERT((dhp->dh_cvaddr + off) <=
1510 			    (cp->cvaddr + cp->size));
1511 			ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1512 			    (cp->cvaddr + cp->size));
1513 
1514 			switch (cp->type) {
1515 			case UMEM_LOCKED :
1516 				if (cp->pparray != NULL) {
1517 					ASSERT((dhp->dh_roff &
1518 					    PAGEOFFSET) == 0);
1519 					pfnum = page_pptonum(
1520 					    cp->pparray[btop(off +
1521 					    dhp->dh_roff)]);
1522 				} else {
1523 					pfnum = hat_getpfnum(
1524 					    ((proc_t *)cp->procp)->p_as->a_hat,
1525 					    cp->cvaddr + off);
1526 				}
1527 			break;
1528 			case UMEM_TRASH :
1529 				pfnum = page_pptonum(trashpp);
1530 				/*
1531 				 * We should set hat_flags to HAT_NOFAULT also
1532 				 * However, not all hat layers implement this
1533 				 */
1534 				break;
1535 			case KMEM_PAGEABLE:
1536 			case KMEM_NON_PAGEABLE:
1537 				pfnum = hat_getpfnum(kas.a_hat,
1538 				    dhp->dh_cvaddr + off);
1539 				break;
1540 			default :
1541 				pfnum = PFN_INVALID;
1542 				break;
1543 			}
1544 		}
1545 		prot |= dhp->dh_hat_attr;
1546 	}
1547 	if (pfnum == PFN_INVALID) {
1548 		return (FC_MAKE_ERR(EFAULT));
1549 	}
1550 	/* prot should already be OR'ed in with hat_attributes if needed */
1551 
1552 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1553 	    "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1554 	    pfnum, pf_is_memory(pfnum), prot, hat_flags);
1555 	DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1556 	    "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1557 
1558 	if (pf_is_memory(pfnum) || (dhp != NULL)) {
1559 		/*
1560 		 * It's not _really_ required here to pass sdp->hat_flags
1561 		 * to hat_devload even though we do it.
1562 		 * This is because hat figures it out DEVMEM mappings
1563 		 * are non-consistent, anyway.
1564 		 */
1565 		hat_devload(hat, addr, PAGESIZE, pfnum,
1566 		    prot, hat_flags | sdp->hat_flags);
1567 		return (0);
1568 	}
1569 
1570 	/*
1571 	 * Fall through to the case where devmap is not used and need to call
1572 	 * up the device tree to set up the mapping
1573 	 */
1574 
1575 	dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1576 	ASSERT(dip);
1577 
1578 	/*
1579 	 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1580 	 * This is because this calls drivers which may not expect
1581 	 * prot to have any other values than PROT_ALL
1582 	 * The root nexus driver has a hack to peek into the segment
1583 	 * structure and then OR in sdp->hat_attr.
1584 	 * XX In case the bus_ops interfaces are ever revisited
1585 	 * we need to fix this. prot should include other hat attributes
1586 	 */
1587 	if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1588 	    (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1589 		return (FC_MAKE_ERR(EFAULT));
1590 	}
1591 	return (0);
1592 }
1593 
1594 static faultcode_t
segdev_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)1595 segdev_fault(
1596 	struct hat *hat,		/* the hat */
1597 	struct seg *seg,		/* the seg_dev of interest */
1598 	caddr_t addr,			/* the address of the fault */
1599 	size_t len,			/* the length of the range */
1600 	enum fault_type type,		/* type of fault */
1601 	enum seg_rw rw)			/* type of access at fault */
1602 {
1603 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1604 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1605 	devmap_handle_t *dhp;
1606 	struct devmap_softlock *slock = NULL;
1607 	ulong_t slpage = 0;
1608 	ulong_t off;
1609 	caddr_t maddr = addr;
1610 	int err;
1611 	int err_is_faultcode = 0;
1612 
1613 	TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1614 	    "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1615 	    (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1616 	DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1617 	    "addr %p len %lx type %x\n",
1618 	    (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1619 
1620 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1621 
1622 	/* Handle non-devmap case */
1623 	if (dhp_head == NULL)
1624 		return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1625 
1626 	/* Find devmap handle */
1627 	if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1628 		return (FC_NOMAP);
1629 
1630 	/*
1631 	 * The seg_dev driver does not implement copy-on-write,
1632 	 * and always loads translations with maximal allowed permissions
1633 	 * but we got an fault trying to access the device.
1634 	 * Servicing the fault is not going to result in any better result
1635 	 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1636 	 *	faults, then the code below is written for that
1637 	 *	Pending resolution of the following:
1638 	 *	- determine if the F_INVAL/F_SOFTLOCK syncing
1639 	 *	is needed for F_PROT also or not. The code below assumes it does
1640 	 *	- If driver sees F_PROT and calls devmap_load with same type,
1641 	 *	then segdev_faultpages will fail with FC_PROT anyway, need to
1642 	 *	change that so calls from devmap_load to segdev_faultpages for
1643 	 *	F_PROT type are retagged to F_INVAL.
1644 	 * RFE: Today we dont have drivers that use devmap and want to handle
1645 	 *	F_PROT calls. The code in segdev_fault* is written to allow
1646 	 *	this case but is not tested. A driver that needs this capability
1647 	 *	should be able to remove the short-circuit case; resolve the
1648 	 *	above issues and "should" work.
1649 	 */
1650 	if (type == F_PROT) {
1651 		return (FC_PROT);
1652 	}
1653 
1654 	/*
1655 	 * Loop through dhp list calling devmap_access or segdev_faultpages for
1656 	 * each devmap handle.
1657 	 * drivers which implement devmap_access can interpose on faults and do
1658 	 * device-appropriate special actions before calling devmap_load.
1659 	 */
1660 
1661 	/*
1662 	 * Unfortunately, this simple loop has turned out to expose a variety
1663 	 * of complex problems which results in the following convoluted code.
1664 	 *
1665 	 * First, a desire to handle a serialization of F_SOFTLOCK calls
1666 	 * to the driver within the framework.
1667 	 *	This results in a dh_softlock structure that is on a per device
1668 	 *	(or device instance) basis and serializes devmap_access calls.
1669 	 *	Ideally we would need to do this for underlying
1670 	 *	memory/device regions that are being faulted on
1671 	 *	but that is hard to identify and with REMAP, harder
1672 	 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1673 	 * 	to F_SOFTLOCK calls to the driver.
1674 	 * These serializations are to simplify the driver programmer model.
1675 	 * To support these two features, the code first goes through the
1676 	 *	devmap handles and counts the pages (slpage) that are covered
1677 	 *	by devmap_access callbacks.
1678 	 * This part ends with a devmap_softlock_enter call
1679 	 *	which allows only one F_SOFTLOCK active on a device instance,
1680 	 *	but multiple F_INVAL/F_PROTs can be active except when a
1681 	 *	F_SOFTLOCK is active
1682 	 *
1683 	 * Next, we dont short-circuit the fault code upfront to call
1684 	 *	segdev_softunlock for F_SOFTUNLOCK, because we must use
1685 	 *	the same length when we softlock and softunlock.
1686 	 *
1687 	 *	-Hat layers may not support softunlocking lengths less than the
1688 	 *	original length when there is large page support.
1689 	 *	-kpmem locking is dependent on keeping the lengths same.
1690 	 *	-if drivers handled F_SOFTLOCK, they probably also expect to
1691 	 *		see an F_SOFTUNLOCK of the same length
1692 	 *	Hence, if extending lengths during softlock,
1693 	 *	softunlock has to make the same adjustments and goes through
1694 	 *	the same loop calling segdev_faultpages/segdev_softunlock
1695 	 *	But some of the synchronization and error handling is different
1696 	 */
1697 
1698 	if (type != F_SOFTUNLOCK) {
1699 		devmap_handle_t *dhpp = dhp;
1700 		size_t slen = len;
1701 
1702 		/*
1703 		 * Calculate count of pages that are :
1704 		 * a) within the (potentially extended) fault region
1705 		 * b) AND covered by devmap handle with devmap_access
1706 		 */
1707 		off = (ulong_t)(addr - dhpp->dh_uvaddr);
1708 		while (slen != 0) {
1709 			size_t mlen;
1710 
1711 			/*
1712 			 * Softlocking on a region that allows remap is
1713 			 * unsupported due to unresolved locking issues
1714 			 * XXX: unclear what these are?
1715 			 *	One potential is that if there is a pending
1716 			 *	softlock, then a remap should not be allowed
1717 			 *	until the unlock is done. This is easily
1718 			 *	fixed by returning error in devmap*remap on
1719 			 *	checking the dh->dh_softlock->softlocked value
1720 			 */
1721 			if ((type == F_SOFTLOCK) &&
1722 			    (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1723 				return (FC_NOSUPPORT);
1724 			}
1725 
1726 			mlen = MIN(slen, (dhpp->dh_len - off));
1727 			if (dhpp->dh_callbackops.devmap_access) {
1728 				size_t llen;
1729 				caddr_t laddr;
1730 				/*
1731 				 * use extended length for large page mappings
1732 				 */
1733 				HOLD_DHP_LOCK(dhpp);
1734 				if ((sdp->pageprot == 0) &&
1735 				    (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1736 					devmap_get_large_pgsize(dhpp,
1737 					    mlen, maddr, &llen, &laddr);
1738 				} else {
1739 					llen = mlen;
1740 				}
1741 				RELE_DHP_LOCK(dhpp);
1742 
1743 				slpage += btopr(llen);
1744 				slock = dhpp->dh_softlock;
1745 			}
1746 			maddr += mlen;
1747 			ASSERT(slen >= mlen);
1748 			slen -= mlen;
1749 			dhpp = dhpp->dh_next;
1750 			off = 0;
1751 		}
1752 		/*
1753 		 * synchonize with other faulting threads and wait till safe
1754 		 * devmap_softlock_enter might return due to signal in cv_wait
1755 		 *
1756 		 * devmap_softlock_enter has to be called outside of while loop
1757 		 * to prevent a deadlock if len spans over multiple dhps.
1758 		 * dh_softlock is based on device instance and if multiple dhps
1759 		 * use the same device instance, the second dhp's LOCK call
1760 		 * will hang waiting on the first to complete.
1761 		 * devmap_setup verifies that slocks in a dhp_chain are same.
1762 		 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1763 		 * 	F_INVAL/F_PROT, since we now allow multiple in parallel,
1764 		 *	we could have done the softlock_enter inside the loop
1765 		 *	and supported multi-dhp mappings with dissimilar devices
1766 		 */
1767 		if (err = devmap_softlock_enter(slock, slpage, type))
1768 			return (FC_MAKE_ERR(err));
1769 	}
1770 
1771 	/* reset 'maddr' to the start addr of the range of fault. */
1772 	maddr = addr;
1773 
1774 	/* calculate the offset corresponds to 'addr' in the first dhp. */
1775 	off = (ulong_t)(addr - dhp->dh_uvaddr);
1776 
1777 	/*
1778 	 * The fault length may span over multiple dhps.
1779 	 * Loop until the total length is satisfied.
1780 	 */
1781 	while (len != 0) {
1782 		size_t llen;
1783 		size_t mlen;
1784 		caddr_t laddr;
1785 
1786 		/*
1787 		 * mlen is the smaller of 'len' and the length
1788 		 * from addr to the end of mapping defined by dhp.
1789 		 */
1790 		mlen = MIN(len, (dhp->dh_len - off));
1791 
1792 		HOLD_DHP_LOCK(dhp);
1793 		/*
1794 		 * Pass the extended length and address to devmap_access
1795 		 * if large pagesize is used for loading address translations.
1796 		 */
1797 		if ((sdp->pageprot == 0) &&
1798 		    (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1799 			devmap_get_large_pgsize(dhp, mlen, maddr,
1800 			    &llen, &laddr);
1801 			ASSERT(maddr == addr || laddr == maddr);
1802 		} else {
1803 			llen = mlen;
1804 			laddr = maddr;
1805 		}
1806 
1807 		if (dhp->dh_callbackops.devmap_access != NULL) {
1808 			offset_t aoff;
1809 
1810 			aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1811 
1812 			/*
1813 			 * call driver's devmap_access entry point which will
1814 			 * call devmap_load/contextmgmt to load the translations
1815 			 *
1816 			 * We drop the dhp_lock before calling access so
1817 			 * drivers can call devmap_*_remap within access
1818 			 */
1819 			RELE_DHP_LOCK(dhp);
1820 
1821 			err = (*dhp->dh_callbackops.devmap_access)(
1822 			    dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1823 		} else {
1824 			/*
1825 			 * If no devmap_access entry point, then load mappings
1826 			 * hold dhp_lock across faultpages if REMAP
1827 			 */
1828 			err = segdev_faultpages(hat, seg, laddr, llen,
1829 			    type, rw, dhp);
1830 			err_is_faultcode = 1;
1831 			RELE_DHP_LOCK(dhp);
1832 		}
1833 
1834 		if (err) {
1835 			if ((type == F_SOFTLOCK) && (maddr > addr)) {
1836 				/*
1837 				 * If not first dhp, use
1838 				 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1839 				 * While this is recursion, it is incorrect to
1840 				 * call just segdev_softunlock
1841 				 * if we are using either large pages
1842 				 * or devmap_access. It will be more right
1843 				 * to go through the same loop as above
1844 				 * rather than call segdev_softunlock directly
1845 				 * It will use the right lenghths as well as
1846 				 * call into the driver devmap_access routines.
1847 				 */
1848 				size_t done = (size_t)(maddr - addr);
1849 				(void) segdev_fault(hat, seg, addr, done,
1850 				    F_SOFTUNLOCK, S_OTHER);
1851 				/*
1852 				 * reduce slpage by number of pages
1853 				 * released by segdev_softunlock
1854 				 */
1855 				ASSERT(slpage >= btopr(done));
1856 				devmap_softlock_exit(slock,
1857 				    slpage - btopr(done), type);
1858 			} else {
1859 				devmap_softlock_exit(slock, slpage, type);
1860 			}
1861 
1862 
1863 			/*
1864 			 * Segdev_faultpages() already returns a faultcode,
1865 			 * hence, result from segdev_faultpages() should be
1866 			 * returned directly.
1867 			 */
1868 			if (err_is_faultcode)
1869 				return (err);
1870 			return (FC_MAKE_ERR(err));
1871 		}
1872 
1873 		maddr += mlen;
1874 		ASSERT(len >= mlen);
1875 		len -= mlen;
1876 		dhp = dhp->dh_next;
1877 		off = 0;
1878 
1879 		ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1880 	}
1881 	/*
1882 	 * release the softlock count at end of fault
1883 	 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1884 	 */
1885 	if ((type == F_INVAL) || (type == F_PROT))
1886 		devmap_softlock_exit(slock, slpage, type);
1887 	return (0);
1888 }
1889 
1890 /*
1891  * segdev_faultpages
1892  *
1893  * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1894  * This routine assumes that the callers makes sure that the fields
1895  * in dhp used below are not changed due to remap during this call.
1896  * Caller does HOLD_DHP_LOCK if neeed
1897  * This routine returns a faultcode_t as a return value for segdev_fault.
1898  */
1899 static faultcode_t
segdev_faultpages(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1900 segdev_faultpages(
1901 	struct hat *hat,		/* the hat */
1902 	struct seg *seg,		/* the seg_dev of interest */
1903 	caddr_t addr,			/* the address of the fault */
1904 	size_t len,			/* the length of the range */
1905 	enum fault_type type,		/* type of fault */
1906 	enum seg_rw rw,			/* type of access at fault */
1907 	devmap_handle_t *dhp)		/* devmap handle */
1908 {
1909 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1910 	register caddr_t a;
1911 	struct vpage *vpage;
1912 	struct ddi_umem_cookie *kpmem_cookie = NULL;
1913 	int err;
1914 
1915 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1916 	    "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1917 	    (void *)dhp, (void *)seg, (void *)addr, len);
1918 	DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1919 	    "dhp %p seg %p addr %p len %lx\n",
1920 	    (void *)dhp, (void *)seg, (void *)addr, len));
1921 
1922 	/*
1923 	 * The seg_dev driver does not implement copy-on-write,
1924 	 * and always loads translations with maximal allowed permissions
1925 	 * but we got an fault trying to access the device.
1926 	 * Servicing the fault is not going to result in any better result
1927 	 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1928 	 * This code should be removed and let the normal fault handling
1929 	 * take care of finding the error
1930 	 */
1931 	if (type == F_PROT) {
1932 		return (FC_PROT);
1933 	}
1934 
1935 	if (type == F_SOFTUNLOCK) {
1936 		segdev_softunlock(hat, seg, addr, len, rw);
1937 		return (0);
1938 	}
1939 
1940 	/*
1941 	 * For kernel pageable memory, fault/lock segkp pages
1942 	 * We hold this until the completion of this
1943 	 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1944 	 */
1945 	if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1946 		kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1947 		if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1948 			return (err);
1949 	}
1950 
1951 	/*
1952 	 * If we have the same protections for the entire segment,
1953 	 * insure that the access being attempted is legitimate.
1954 	 */
1955 	rw_enter(&sdp->lock, RW_READER);
1956 	if (sdp->pageprot == 0) {
1957 		uint_t protchk;
1958 
1959 		switch (rw) {
1960 		case S_READ:
1961 			protchk = PROT_READ;
1962 			break;
1963 		case S_WRITE:
1964 			protchk = PROT_WRITE;
1965 			break;
1966 		case S_EXEC:
1967 			protchk = PROT_EXEC;
1968 			break;
1969 		case S_OTHER:
1970 		default:
1971 			protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1972 			break;
1973 		}
1974 
1975 		if ((sdp->prot & protchk) == 0) {
1976 			rw_exit(&sdp->lock);
1977 			/* undo kpmem locking */
1978 			if (kpmem_cookie != NULL) {
1979 				release_kpmem_lock(kpmem_cookie, btopr(len));
1980 			}
1981 			return (FC_PROT);	/* illegal access type */
1982 		}
1983 	}
1984 
1985 	/*
1986 	 * we do a single hat_devload for the range if
1987 	 *   - devmap framework (dhp is not NULL),
1988 	 *   - pageprot == 0, i.e., no per-page protection set and
1989 	 *   - is device pages, irrespective of whether we are using large pages
1990 	 */
1991 	if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1992 		pfn_t pfnum;
1993 		uint_t hat_flags;
1994 
1995 		if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1996 			rw_exit(&sdp->lock);
1997 			return (FC_NOMAP);
1998 		}
1999 
2000 		if (type == F_SOFTLOCK) {
2001 			mutex_enter(&freemem_lock);
2002 			sdp->softlockcnt += btopr(len);
2003 			mutex_exit(&freemem_lock);
2004 		}
2005 
2006 		hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2007 		pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2008 		ASSERT(!pf_is_memory(pfnum));
2009 
2010 		hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2011 		    hat_flags | sdp->hat_flags);
2012 		rw_exit(&sdp->lock);
2013 		return (0);
2014 	}
2015 
2016 	/* Handle cases where we have to loop through fault handling per-page */
2017 
2018 	if (sdp->vpage == NULL)
2019 		vpage = NULL;
2020 	else
2021 		vpage = &sdp->vpage[seg_page(seg, addr)];
2022 
2023 	/* loop over the address range handling each fault */
2024 	for (a = addr; a < addr + len; a += PAGESIZE) {
2025 		if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2026 			break;
2027 		}
2028 		if (vpage != NULL)
2029 			vpage++;
2030 	}
2031 	rw_exit(&sdp->lock);
2032 	if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2033 		size_t done = (size_t)(a - addr); /* pages fault successfully */
2034 		if (done > 0) {
2035 			/* use softunlock for those pages */
2036 			segdev_softunlock(hat, seg, addr, done, S_OTHER);
2037 		}
2038 		if (kpmem_cookie != NULL) {
2039 			/* release kpmem lock for rest of pages */
2040 			ASSERT(len >= done);
2041 			release_kpmem_lock(kpmem_cookie, btopr(len - done));
2042 		}
2043 	} else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2044 		/* for non-SOFTLOCK cases, release kpmem */
2045 		release_kpmem_lock(kpmem_cookie, btopr(len));
2046 	}
2047 	return (err);
2048 }
2049 
2050 /*
2051  * Asynchronous page fault.  We simply do nothing since this
2052  * entry point is not supposed to load up the translation.
2053  */
2054 /*ARGSUSED*/
2055 static faultcode_t
segdev_faulta(struct seg * seg,caddr_t addr)2056 segdev_faulta(struct seg *seg, caddr_t addr)
2057 {
2058 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2059 	    "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2060 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2061 
2062 	return (0);
2063 }
2064 
2065 static int
segdev_setprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2066 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2067 {
2068 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2069 	register devmap_handle_t *dhp;
2070 	register struct vpage *vp, *evp;
2071 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2072 	ulong_t off;
2073 	size_t mlen, sz;
2074 
2075 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2076 	    "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2077 	    (void *)seg, (void *)addr, len, prot);
2078 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2079 
2080 	if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2081 		/*
2082 		 * Fail the setprot if pages are SOFTLOCKed through this
2083 		 * mapping.
2084 		 * Softlockcnt is protected from change by the as read lock.
2085 		 */
2086 		TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2087 		    "segdev_setprot:error softlockcnt=%lx", sz);
2088 		DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2089 		return (EAGAIN);
2090 	}
2091 
2092 	if (dhp_head != NULL) {
2093 		if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2094 			return (EINVAL);
2095 
2096 		/*
2097 		 * check if violate maxprot.
2098 		 */
2099 		off = (ulong_t)(addr - dhp->dh_uvaddr);
2100 		mlen  = len;
2101 		while (dhp) {
2102 			if ((dhp->dh_maxprot & prot) != prot)
2103 				return (EACCES);	/* violated maxprot */
2104 
2105 			if (mlen > (dhp->dh_len - off)) {
2106 				mlen -= dhp->dh_len - off;
2107 				dhp = dhp->dh_next;
2108 				off = 0;
2109 			} else
2110 				break;
2111 		}
2112 	} else {
2113 		if ((sdp->maxprot & prot) != prot)
2114 			return (EACCES);
2115 	}
2116 
2117 	rw_enter(&sdp->lock, RW_WRITER);
2118 	if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2119 		if (sdp->prot == prot) {
2120 			rw_exit(&sdp->lock);
2121 			return (0);			/* all done */
2122 		}
2123 		sdp->prot = (uchar_t)prot;
2124 	} else {
2125 		sdp->pageprot = 1;
2126 		if (sdp->vpage == NULL) {
2127 			/*
2128 			 * First time through setting per page permissions,
2129 			 * initialize all the vpage structures to prot
2130 			 */
2131 			sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2132 			    KM_SLEEP);
2133 			evp = &sdp->vpage[seg_pages(seg)];
2134 			for (vp = sdp->vpage; vp < evp; vp++)
2135 				VPP_SETPROT(vp, sdp->prot);
2136 		}
2137 		/*
2138 		 * Now go change the needed vpages protections.
2139 		 */
2140 		evp = &sdp->vpage[seg_page(seg, addr + len)];
2141 		for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2142 			VPP_SETPROT(vp, prot);
2143 	}
2144 	rw_exit(&sdp->lock);
2145 
2146 	if (dhp_head != NULL) {
2147 		devmap_handle_t *tdhp;
2148 		/*
2149 		 * If large page size was used in hat_devload(),
2150 		 * the same page size must be used in hat_unload().
2151 		 */
2152 		dhp = tdhp = devmap_find_handle(dhp_head, addr);
2153 		while (tdhp != NULL) {
2154 			if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2155 				break;
2156 			}
2157 			tdhp = tdhp->dh_next;
2158 		}
2159 		if (tdhp) {
2160 			size_t slen = len;
2161 			size_t mlen;
2162 			size_t soff;
2163 
2164 			soff = (ulong_t)(addr - dhp->dh_uvaddr);
2165 			while (slen != 0) {
2166 				mlen = MIN(slen, (dhp->dh_len - soff));
2167 				hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2168 				    dhp->dh_len, HAT_UNLOAD);
2169 				dhp = dhp->dh_next;
2170 				ASSERT(slen >= mlen);
2171 				slen -= mlen;
2172 				soff = 0;
2173 			}
2174 			return (0);
2175 		}
2176 	}
2177 
2178 	if ((prot & ~PROT_USER) == PROT_NONE) {
2179 		hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2180 	} else {
2181 		/*
2182 		 * RFE: the segment should keep track of all attributes
2183 		 * allowing us to remove the deprecated hat_chgprot
2184 		 * and use hat_chgattr.
2185 		 */
2186 		hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2187 	}
2188 
2189 	return (0);
2190 }
2191 
2192 static int
segdev_checkprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2193 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2194 {
2195 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2196 	struct vpage *vp, *evp;
2197 
2198 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2199 	    "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2200 	    (void *)seg, (void *)addr, len, prot);
2201 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2202 
2203 	/*
2204 	 * If segment protection can be used, simply check against them
2205 	 */
2206 	rw_enter(&sdp->lock, RW_READER);
2207 	if (sdp->pageprot == 0) {
2208 		register int err;
2209 
2210 		err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2211 		rw_exit(&sdp->lock);
2212 		return (err);
2213 	}
2214 
2215 	/*
2216 	 * Have to check down to the vpage level
2217 	 */
2218 	evp = &sdp->vpage[seg_page(seg, addr + len)];
2219 	for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2220 		if ((VPP_PROT(vp) & prot) != prot) {
2221 			rw_exit(&sdp->lock);
2222 			return (EACCES);
2223 		}
2224 	}
2225 	rw_exit(&sdp->lock);
2226 	return (0);
2227 }
2228 
2229 static int
segdev_getprot(struct seg * seg,caddr_t addr,size_t len,uint_t * protv)2230 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2231 {
2232 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2233 	size_t pgno;
2234 
2235 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2236 	    "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2237 	    (void *)seg, (void *)addr, len, (void *)protv);
2238 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2239 
2240 	pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2241 	if (pgno != 0) {
2242 		rw_enter(&sdp->lock, RW_READER);
2243 		if (sdp->pageprot == 0) {
2244 			do {
2245 				protv[--pgno] = sdp->prot;
2246 			} while (pgno != 0);
2247 		} else {
2248 			size_t pgoff = seg_page(seg, addr);
2249 
2250 			do {
2251 				pgno--;
2252 				protv[pgno] =
2253 				    VPP_PROT(&sdp->vpage[pgno + pgoff]);
2254 			} while (pgno != 0);
2255 		}
2256 		rw_exit(&sdp->lock);
2257 	}
2258 	return (0);
2259 }
2260 
2261 static u_offset_t
segdev_getoffset(register struct seg * seg,caddr_t addr)2262 segdev_getoffset(register struct seg *seg, caddr_t addr)
2263 {
2264 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2265 
2266 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2267 	    "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2268 
2269 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2270 
2271 	return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2272 }
2273 
2274 /*ARGSUSED*/
2275 static int
segdev_gettype(register struct seg * seg,caddr_t addr)2276 segdev_gettype(register struct seg *seg, caddr_t addr)
2277 {
2278 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2279 
2280 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2281 	    "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2282 
2283 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2284 
2285 	return (sdp->type);
2286 }
2287 
2288 
2289 /*ARGSUSED*/
2290 static int
segdev_getvp(register struct seg * seg,caddr_t addr,struct vnode ** vpp)2291 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2292 {
2293 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2294 
2295 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2296 	    "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2297 
2298 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2299 
2300 	/*
2301 	 * Note that this vp is the common_vp of the device, where the
2302 	 * pages are hung ..
2303 	 */
2304 	*vpp = VTOCVP(sdp->vp);
2305 
2306 	return (0);
2307 }
2308 
2309 static void
segdev_badop(void)2310 segdev_badop(void)
2311 {
2312 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2313 	    "segdev_badop:start");
2314 	panic("segdev_badop");
2315 	/*NOTREACHED*/
2316 }
2317 
2318 /*
2319  * segdev pages are not in the cache, and thus can't really be controlled.
2320  * Hence, syncs are simply always successful.
2321  */
2322 /*ARGSUSED*/
2323 static int
segdev_sync(struct seg * seg,caddr_t addr,size_t len,int attr,uint_t flags)2324 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2325 {
2326 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2327 
2328 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2329 
2330 	return (0);
2331 }
2332 
2333 /*
2334  * segdev pages are always "in core".
2335  */
2336 /*ARGSUSED*/
2337 static size_t
segdev_incore(struct seg * seg,caddr_t addr,size_t len,char * vec)2338 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2339 {
2340 	size_t v = 0;
2341 
2342 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2343 
2344 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2345 
2346 	for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2347 	    v += PAGESIZE)
2348 		*vec++ = 1;
2349 	return (v);
2350 }
2351 
2352 /*
2353  * segdev pages are not in the cache, and thus can't really be controlled.
2354  * Hence, locks are simply always successful.
2355  */
2356 /*ARGSUSED*/
2357 static int
segdev_lockop(struct seg * seg,caddr_t addr,size_t len,int attr,int op,ulong_t * lockmap,size_t pos)2358 segdev_lockop(struct seg *seg, caddr_t addr,
2359     size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2360 {
2361 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2362 
2363 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2364 
2365 	return (0);
2366 }
2367 
2368 /*
2369  * segdev pages are not in the cache, and thus can't really be controlled.
2370  * Hence, advise is simply always successful.
2371  */
2372 /*ARGSUSED*/
2373 static int
segdev_advise(struct seg * seg,caddr_t addr,size_t len,uint_t behav)2374 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2375 {
2376 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2377 
2378 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2379 
2380 	return (0);
2381 }
2382 
2383 /*
2384  * segdev pages are not dumped, so we just return
2385  */
2386 /*ARGSUSED*/
2387 static void
segdev_dump(struct seg * seg)2388 segdev_dump(struct seg *seg)
2389 {}
2390 
2391 /*
2392  * ddi_segmap_setup:	Used by drivers who wish specify mapping attributes
2393  *			for a segment.	Called from a drivers segmap(9E)
2394  *			routine.
2395  */
2396 /*ARGSUSED*/
2397 int
ddi_segmap_setup(dev_t dev,off_t offset,struct as * as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,cred_t * cred,ddi_device_acc_attr_t * accattrp,uint_t rnumber)2398 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2399     off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2400     ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2401 {
2402 	struct segdev_crargs dev_a;
2403 	int (*mapfunc)(dev_t dev, off_t off, int prot);
2404 	uint_t hat_attr;
2405 	pfn_t pfn;
2406 	int	error, i;
2407 
2408 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2409 	    "ddi_segmap_setup:start");
2410 
2411 	if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2412 		return (ENODEV);
2413 
2414 	/*
2415 	 * Character devices that support the d_mmap
2416 	 * interface can only be mmap'ed shared.
2417 	 */
2418 	if ((flags & MAP_TYPE) != MAP_SHARED)
2419 		return (EINVAL);
2420 
2421 	/*
2422 	 * Check that this region is indeed mappable on this platform.
2423 	 * Use the mapping function.
2424 	 */
2425 	if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2426 		return (ENXIO);
2427 
2428 	/*
2429 	 * Check to ensure that the entire range is
2430 	 * legal and we are not trying to map in
2431 	 * more than the device will let us.
2432 	 */
2433 	for (i = 0; i < len; i += PAGESIZE) {
2434 		if (i == 0) {
2435 			/*
2436 			 * Save the pfn at offset here. This pfn will be
2437 			 * used later to get user address.
2438 			 */
2439 			if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2440 			    maxprot)) == PFN_INVALID)
2441 				return (ENXIO);
2442 		} else {
2443 			if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2444 			    PFN_INVALID)
2445 				return (ENXIO);
2446 		}
2447 	}
2448 
2449 	as_rangelock(as);
2450 	/* Pick an address w/o worrying about any vac alignment constraints. */
2451 	error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2452 	if (error != 0) {
2453 		as_rangeunlock(as);
2454 		return (error);
2455 	}
2456 
2457 	dev_a.mapfunc = mapfunc;
2458 	dev_a.dev = dev;
2459 	dev_a.offset = (offset_t)offset;
2460 	dev_a.type = flags & MAP_TYPE;
2461 	dev_a.prot = (uchar_t)prot;
2462 	dev_a.maxprot = (uchar_t)maxprot;
2463 	dev_a.hat_attr = hat_attr;
2464 	dev_a.hat_flags = 0;
2465 	dev_a.devmap_data = NULL;
2466 
2467 	error = as_map(as, *addrp, len, segdev_create, &dev_a);
2468 	as_rangeunlock(as);
2469 	return (error);
2470 
2471 }
2472 
2473 /*ARGSUSED*/
2474 static int
segdev_pagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)2475 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2476     struct page ***ppp, enum lock_type type, enum seg_rw rw)
2477 {
2478 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2479 	    "segdev_pagelock:start");
2480 	return (ENOTSUP);
2481 }
2482 
2483 /*ARGSUSED*/
2484 static int
segdev_setpagesize(struct seg * seg,caddr_t addr,size_t len,uint_t szc)2485 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2486     uint_t szc)
2487 {
2488 	return (ENOTSUP);
2489 }
2490 
2491 /*
2492  * devmap_device: Used by devmap framework to establish mapping
2493  *                called by devmap_seup(9F) during map setup time.
2494  */
2495 /*ARGSUSED*/
2496 static int
devmap_device(devmap_handle_t * dhp,struct as * as,caddr_t * addr,offset_t off,size_t len,uint_t flags)2497 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2498     offset_t off, size_t len, uint_t flags)
2499 {
2500 	devmap_handle_t *rdhp, *maxdhp;
2501 	struct segdev_crargs dev_a;
2502 	int	err;
2503 	uint_t maxprot = PROT_ALL;
2504 	offset_t offset = 0;
2505 	pfn_t pfn;
2506 	struct devmap_pmem_cookie *pcp;
2507 
2508 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2509 	    "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2510 	    (void *)dhp, (void *)addr, off, len);
2511 
2512 	DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2513 	    (void *)dhp, (void *)addr, off, len));
2514 
2515 	as_rangelock(as);
2516 	if ((flags & MAP_FIXED) == 0) {
2517 		offset_t aligned_off;
2518 
2519 		rdhp = maxdhp = dhp;
2520 		while (rdhp != NULL) {
2521 			maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2522 			    maxdhp : rdhp;
2523 			rdhp = rdhp->dh_next;
2524 			maxprot |= dhp->dh_maxprot;
2525 		}
2526 		offset = maxdhp->dh_uoff - dhp->dh_uoff;
2527 
2528 		/*
2529 		 * Use the dhp that has the
2530 		 * largest len to get user address.
2531 		 */
2532 		/*
2533 		 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2534 		 * use 0 which is as good as any other.
2535 		 */
2536 		if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2537 			aligned_off = (offset_t)0;
2538 		} else if (dhp_is_devmem(maxdhp)) {
2539 			aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2540 		} else if (dhp_is_pmem(maxdhp)) {
2541 			pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2542 			pfn = page_pptonum(
2543 			    pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2544 			aligned_off = (offset_t)ptob(pfn) - offset;
2545 		} else {
2546 			aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2547 			    offset;
2548 		}
2549 
2550 		/*
2551 		 * Pick an address aligned to dh_cookie.
2552 		 * for kernel memory/user memory, cookie is cvaddr.
2553 		 * for device memory, cookie is physical address.
2554 		 */
2555 		map_addr(addr, len, aligned_off, 1, flags);
2556 		if (*addr == NULL) {
2557 			as_rangeunlock(as);
2558 			return (ENOMEM);
2559 		}
2560 	} else {
2561 		/*
2562 		 * User-specified address; blow away any previous mappings.
2563 		 */
2564 		(void) as_unmap(as, *addr, len);
2565 	}
2566 
2567 	dev_a.mapfunc = NULL;
2568 	dev_a.dev = dhp->dh_dev;
2569 	dev_a.type = flags & MAP_TYPE;
2570 	dev_a.offset = off;
2571 	/*
2572 	 * sdp->maxprot has the least restrict protection of all dhps.
2573 	 */
2574 	dev_a.maxprot = maxprot;
2575 	dev_a.prot = dhp->dh_prot;
2576 	/*
2577 	 * devmap uses dhp->dh_hat_attr for hat.
2578 	 */
2579 	dev_a.hat_flags = 0;
2580 	dev_a.hat_attr = 0;
2581 	dev_a.devmap_data = (void *)dhp;
2582 
2583 	err = as_map(as, *addr, len, segdev_create, &dev_a);
2584 	as_rangeunlock(as);
2585 	return (err);
2586 }
2587 
2588 int
devmap_do_ctxmgt(devmap_cookie_t dhc,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw,int (* ctxmgt)(devmap_cookie_t,void *,offset_t,size_t,uint_t,uint_t))2589 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2590     uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2591     size_t, uint_t, uint_t))
2592 {
2593 	register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2594 	struct devmap_ctx *devctx;
2595 	int do_timeout = 0;
2596 	int ret;
2597 
2598 #ifdef lint
2599 	pvtp = pvtp;
2600 #endif
2601 
2602 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2603 	    "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2604 	    (void *)dhp, off, len);
2605 	DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2606 	    (void *)dhp, off, len));
2607 
2608 	if (ctxmgt == NULL)
2609 		return (FC_HWERR);
2610 
2611 	devctx = dhp->dh_ctx;
2612 
2613 	/*
2614 	 * If we are on an MP system with more than one cpu running
2615 	 * and if a thread on some CPU already has the context, wait
2616 	 * for it to finish if there is a hysteresis timeout.
2617 	 *
2618 	 * We call cv_wait() instead of cv_wait_sig() because
2619 	 * it does not matter much if it returned due to a signal
2620 	 * or due to a cv_signal() or cv_broadcast().  In either event
2621 	 * we need to complete the mapping otherwise the processes
2622 	 * will die with a SEGV.
2623 	 */
2624 	if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2625 		TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2626 		    "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2627 		    devctx, dhp);
2628 		do_timeout = 1;
2629 		mutex_enter(&devctx->lock);
2630 		while (devctx->oncpu)
2631 			cv_wait(&devctx->cv, &devctx->lock);
2632 		devctx->oncpu = 1;
2633 		mutex_exit(&devctx->lock);
2634 	}
2635 
2636 	/*
2637 	 * Call the contextmgt callback so that the driver can handle
2638 	 * the fault.
2639 	 */
2640 	ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2641 
2642 	/*
2643 	 * If devmap_access() returned -1, then there was a hardware
2644 	 * error so we need to convert the return value to something
2645 	 * that trap() will understand.  Otherwise, the return value
2646 	 * is already a fault code generated by devmap_unload()
2647 	 * or devmap_load().
2648 	 */
2649 	if (ret) {
2650 		TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2651 		    "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2652 		    ret, dhp, devctx);
2653 		DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2654 		    ret, (void *)dhp));
2655 		if (devctx->oncpu) {
2656 			mutex_enter(&devctx->lock);
2657 			devctx->oncpu = 0;
2658 			cv_signal(&devctx->cv);
2659 			mutex_exit(&devctx->lock);
2660 		}
2661 		return (FC_HWERR);
2662 	}
2663 
2664 	/*
2665 	 * Setup the timeout if we need to
2666 	 */
2667 	if (do_timeout) {
2668 		mutex_enter(&devctx->lock);
2669 		if (dhp->dh_timeout_length > 0) {
2670 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2671 			    "devmap_do_ctxmgt:timeout set");
2672 			devctx->timeout = timeout(devmap_ctxto,
2673 			    devctx, dhp->dh_timeout_length);
2674 		} else {
2675 			/*
2676 			 * We don't want to wait so set oncpu to
2677 			 * 0 and wake up anyone waiting.
2678 			 */
2679 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2680 			    "devmap_do_ctxmgt:timeout not set");
2681 			devctx->oncpu = 0;
2682 			cv_signal(&devctx->cv);
2683 		}
2684 		mutex_exit(&devctx->lock);
2685 	}
2686 
2687 	return (DDI_SUCCESS);
2688 }
2689 
2690 /*
2691  *                                       end of mapping
2692  *                    poff   fault_offset         |
2693  *            base     |        |                 |
2694  *              |      |        |                 |
2695  *              V      V        V                 V
2696  *  +-----------+---------------+-------+---------+-------+
2697  *              ^               ^       ^         ^
2698  *              |<--- offset--->|<-len->|         |
2699  *              |<--- dh_len(size of mapping) --->|
2700  *                     |<--  pg -->|
2701  *                              -->|rlen|<--
2702  */
2703 static ulong_t
devmap_roundup(devmap_handle_t * dhp,ulong_t offset,size_t len,ulong_t * opfn,ulong_t * pagesize)2704 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2705     ulong_t *opfn, ulong_t *pagesize)
2706 {
2707 	register int level;
2708 	ulong_t pg;
2709 	ulong_t poff;
2710 	ulong_t base;
2711 	caddr_t uvaddr;
2712 	long rlen;
2713 
2714 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2715 	    "devmap_roundup:start dhp=%p off=%lx len=%lx",
2716 	    (void *)dhp, offset, len);
2717 	DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2718 	    (void *)dhp, offset, len));
2719 
2720 	/*
2721 	 * get the max. pagesize that is aligned within the range
2722 	 * <dh_pfn, dh_pfn+offset>.
2723 	 *
2724 	 * The calculations below use physical address to ddetermine
2725 	 * the page size to use. The same calculations can use the
2726 	 * virtual address to determine the page size.
2727 	 */
2728 	base = (ulong_t)ptob(dhp->dh_pfn);
2729 	for (level = dhp->dh_mmulevel; level >= 0; level--) {
2730 		pg = page_get_pagesize(level);
2731 		poff = ((base + offset) & ~(pg - 1));
2732 		uvaddr = dhp->dh_uvaddr + (poff - base);
2733 		if ((poff >= base) &&
2734 		    ((poff + pg) <= (base + dhp->dh_len)) &&
2735 		    VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2736 			break;
2737 	}
2738 
2739 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2740 	    "devmap_roundup: base=%lx poff=%lx dhp=%p",
2741 	    base, poff, dhp);
2742 	DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2743 	    base, poff, dhp->dh_pfn));
2744 
2745 	ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2746 	ASSERT(level >= 0);
2747 
2748 	*pagesize = pg;
2749 	*opfn = dhp->dh_pfn + btop(poff - base);
2750 
2751 	rlen = len + offset - (poff - base + pg);
2752 
2753 	ASSERT(rlen < (long)len);
2754 
2755 	TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2756 	    "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2757 	    (void *)dhp, level, rlen, pagesize, opfn);
2758 	DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2759 	    "level %x rlen %lx psize %lx opfn %lx\n",
2760 	    (void *)dhp, level, rlen, *pagesize, *opfn));
2761 
2762 	return ((ulong_t)((rlen > 0) ? rlen : 0));
2763 }
2764 
2765 /*
2766  * find the dhp that contains addr.
2767  */
2768 static devmap_handle_t *
devmap_find_handle(devmap_handle_t * dhp_head,caddr_t addr)2769 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2770 {
2771 	devmap_handle_t *dhp;
2772 
2773 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2774 	    "devmap_find_handle:start");
2775 
2776 	dhp = dhp_head;
2777 	while (dhp) {
2778 		if (addr >= dhp->dh_uvaddr &&
2779 		    addr < (dhp->dh_uvaddr + dhp->dh_len))
2780 			return (dhp);
2781 		dhp = dhp->dh_next;
2782 	}
2783 
2784 	return ((devmap_handle_t *)NULL);
2785 }
2786 
2787 /*
2788  * devmap_unload:
2789  *			Marks a segdev segment or pages if offset->offset+len
2790  *			is not the entire segment as intercept and unloads the
2791  *			pages in the range offset -> offset+len.
2792  */
2793 int
devmap_unload(devmap_cookie_t dhc,offset_t offset,size_t len)2794 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2795 {
2796 	register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2797 	caddr_t	addr;
2798 	ulong_t	size;
2799 	ssize_t	soff;
2800 
2801 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2802 	    "devmap_unload:start dhp=%p offset=%llx len=%lx",
2803 	    (void *)dhp, offset, len);
2804 	DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2805 	    (void *)dhp, offset, len));
2806 
2807 	soff = (ssize_t)(offset - dhp->dh_uoff);
2808 	soff = round_down_p2(soff, PAGESIZE);
2809 	if (soff < 0 || soff >= dhp->dh_len)
2810 		return (FC_MAKE_ERR(EINVAL));
2811 
2812 	/*
2813 	 * Address and size must be page aligned.  Len is set to the
2814 	 * number of bytes in the number of pages that are required to
2815 	 * support len.  Offset is set to the byte offset of the first byte
2816 	 * of the page that contains offset.
2817 	 */
2818 	len = round_up_p2(len, PAGESIZE);
2819 
2820 	/*
2821 	 * If len is == 0, then calculate the size by getting
2822 	 * the number of bytes from offset to the end of the segment.
2823 	 */
2824 	if (len == 0)
2825 		size = dhp->dh_len - soff;
2826 	else {
2827 		size = len;
2828 		if ((soff + size) > dhp->dh_len)
2829 			return (FC_MAKE_ERR(EINVAL));
2830 	}
2831 
2832 	/*
2833 	 * The address is offset bytes from the base address of
2834 	 * the dhp.
2835 	 */
2836 	addr = (caddr_t)(soff + dhp->dh_uvaddr);
2837 
2838 	/*
2839 	 * If large page size was used in hat_devload(),
2840 	 * the same page size must be used in hat_unload().
2841 	 */
2842 	if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2843 		hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2844 		    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2845 	} else {
2846 		hat_unload(dhp->dh_seg->s_as->a_hat,  addr, size,
2847 		    HAT_UNLOAD|HAT_UNLOAD_OTHER);
2848 	}
2849 
2850 	return (0);
2851 }
2852 
2853 /*
2854  * calculates the optimal page size that will be used for hat_devload().
2855  */
2856 static void
devmap_get_large_pgsize(devmap_handle_t * dhp,size_t len,caddr_t addr,size_t * llen,caddr_t * laddr)2857 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2858     size_t *llen, caddr_t *laddr)
2859 {
2860 	ulong_t off;
2861 	ulong_t pfn;
2862 	ulong_t pgsize;
2863 	uint_t first = 1;
2864 
2865 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2866 	    "devmap_get_large_pgsize:start");
2867 
2868 	/*
2869 	 * RFE - Code only supports large page mappings for devmem
2870 	 * This code could be changed in future if we want to support
2871 	 * large page mappings for kernel exported memory.
2872 	 */
2873 	ASSERT(dhp_is_devmem(dhp));
2874 	ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2875 
2876 	*llen = 0;
2877 	off = (ulong_t)(addr - dhp->dh_uvaddr);
2878 	while ((long)len > 0) {
2879 		/*
2880 		 * get the optimal pfn to minimize address translations.
2881 		 * devmap_roundup() returns residue bytes for next round
2882 		 * calculations.
2883 		 */
2884 		len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2885 
2886 		if (first) {
2887 			*laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2888 			first = 0;
2889 		}
2890 
2891 		*llen += pgsize;
2892 		off = ptob(pfn - dhp->dh_pfn) + pgsize;
2893 	}
2894 	/* Large page mapping len/addr cover more range than original fault */
2895 	ASSERT(*llen >= len && *laddr <= addr);
2896 	ASSERT((*laddr + *llen) >= (addr + len));
2897 }
2898 
2899 /*
2900  * Initialize the devmap_softlock structure.
2901  */
2902 static struct devmap_softlock *
devmap_softlock_init(dev_t dev,ulong_t id)2903 devmap_softlock_init(dev_t dev, ulong_t id)
2904 {
2905 	struct devmap_softlock *slock;
2906 	struct devmap_softlock *tmp;
2907 
2908 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2909 	    "devmap_softlock_init:start");
2910 
2911 	tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2912 	mutex_enter(&devmap_slock);
2913 
2914 	for (slock = devmap_slist; slock != NULL; slock = slock->next)
2915 		if ((slock->dev == dev) && (slock->id == id))
2916 			break;
2917 
2918 	if (slock == NULL) {
2919 		slock = tmp;
2920 		slock->dev = dev;
2921 		slock->id = id;
2922 		mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2923 		cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2924 		slock->next = devmap_slist;
2925 		devmap_slist = slock;
2926 	} else
2927 		kmem_free(tmp, sizeof (struct devmap_softlock));
2928 
2929 	mutex_enter(&slock->lock);
2930 	slock->refcnt++;
2931 	mutex_exit(&slock->lock);
2932 	mutex_exit(&devmap_slock);
2933 
2934 	return (slock);
2935 }
2936 
2937 /*
2938  * Wake up processes that sleep on softlocked.
2939  * Free dh_softlock if refcnt is 0.
2940  */
2941 static void
devmap_softlock_rele(devmap_handle_t * dhp)2942 devmap_softlock_rele(devmap_handle_t *dhp)
2943 {
2944 	struct devmap_softlock *slock = dhp->dh_softlock;
2945 	struct devmap_softlock *tmp;
2946 	struct devmap_softlock *parent;
2947 
2948 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2949 	    "devmap_softlock_rele:start");
2950 
2951 	mutex_enter(&devmap_slock);
2952 	mutex_enter(&slock->lock);
2953 
2954 	ASSERT(slock->refcnt > 0);
2955 
2956 	slock->refcnt--;
2957 
2958 	/*
2959 	 * If no one is using the device, free up the slock data.
2960 	 */
2961 	if (slock->refcnt == 0) {
2962 		slock->softlocked = 0;
2963 		cv_signal(&slock->cv);
2964 
2965 		if (devmap_slist == slock)
2966 			devmap_slist = slock->next;
2967 		else {
2968 			parent = devmap_slist;
2969 			for (tmp = devmap_slist->next; tmp != NULL;
2970 			    tmp = tmp->next) {
2971 				if (tmp == slock) {
2972 					parent->next = tmp->next;
2973 					break;
2974 				}
2975 				parent = tmp;
2976 			}
2977 		}
2978 		mutex_exit(&slock->lock);
2979 		mutex_destroy(&slock->lock);
2980 		cv_destroy(&slock->cv);
2981 		kmem_free(slock, sizeof (struct devmap_softlock));
2982 	} else
2983 		mutex_exit(&slock->lock);
2984 
2985 	mutex_exit(&devmap_slock);
2986 }
2987 
2988 /*
2989  * Wake up processes that sleep on dh_ctx->locked.
2990  * Free dh_ctx if refcnt is 0.
2991  */
2992 static void
devmap_ctx_rele(devmap_handle_t * dhp)2993 devmap_ctx_rele(devmap_handle_t *dhp)
2994 {
2995 	struct devmap_ctx *devctx = dhp->dh_ctx;
2996 	struct devmap_ctx *tmp;
2997 	struct devmap_ctx *parent;
2998 	timeout_id_t tid;
2999 
3000 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3001 	    "devmap_ctx_rele:start");
3002 
3003 	mutex_enter(&devmapctx_lock);
3004 	mutex_enter(&devctx->lock);
3005 
3006 	ASSERT(devctx->refcnt > 0);
3007 
3008 	devctx->refcnt--;
3009 
3010 	/*
3011 	 * If no one is using the device, free up the devctx data.
3012 	 */
3013 	if (devctx->refcnt == 0) {
3014 		/*
3015 		 * Untimeout any threads using this mapping as they are about
3016 		 * to go away.
3017 		 */
3018 		if (devctx->timeout != 0) {
3019 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3020 			    "devmap_ctx_rele:untimeout ctx->timeout");
3021 
3022 			tid = devctx->timeout;
3023 			mutex_exit(&devctx->lock);
3024 			(void) untimeout(tid);
3025 			mutex_enter(&devctx->lock);
3026 		}
3027 
3028 		devctx->oncpu = 0;
3029 		cv_signal(&devctx->cv);
3030 
3031 		if (devmapctx_list == devctx)
3032 			devmapctx_list = devctx->next;
3033 		else {
3034 			parent = devmapctx_list;
3035 			for (tmp = devmapctx_list->next; tmp != NULL;
3036 			    tmp = tmp->next) {
3037 				if (tmp == devctx) {
3038 					parent->next = tmp->next;
3039 					break;
3040 				}
3041 				parent = tmp;
3042 			}
3043 		}
3044 		mutex_exit(&devctx->lock);
3045 		mutex_destroy(&devctx->lock);
3046 		cv_destroy(&devctx->cv);
3047 		kmem_free(devctx, sizeof (struct devmap_ctx));
3048 	} else
3049 		mutex_exit(&devctx->lock);
3050 
3051 	mutex_exit(&devmapctx_lock);
3052 }
3053 
3054 /*
3055  * devmap_load:
3056  *			Marks a segdev segment or pages if offset->offset+len
3057  *			is not the entire segment as nointercept and faults in
3058  *			the pages in the range offset -> offset+len.
3059  */
3060 int
devmap_load(devmap_cookie_t dhc,offset_t offset,size_t len,uint_t type,uint_t rw)3061 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3062     uint_t rw)
3063 {
3064 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3065 	struct as *asp = dhp->dh_seg->s_as;
3066 	caddr_t	addr;
3067 	ulong_t	size;
3068 	ssize_t	soff;	/* offset from the beginning of the segment */
3069 	int rc;
3070 
3071 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3072 	    "devmap_load:start dhp=%p offset=%llx len=%lx",
3073 	    (void *)dhp, offset, len);
3074 
3075 	DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3076 	    (void *)dhp, offset, len));
3077 
3078 	/*
3079 	 *	Hat layer only supports devload to process' context for which
3080 	 *	the as lock is held. Verify here and return error if drivers
3081 	 *	inadvertently call devmap_load on a wrong devmap handle.
3082 	 */
3083 	if ((asp != &kas) && !AS_LOCK_HELD(asp))
3084 		return (FC_MAKE_ERR(EINVAL));
3085 
3086 	soff = (ssize_t)(offset - dhp->dh_uoff);
3087 	soff = round_down_p2(soff, PAGESIZE);
3088 	if (soff < 0 || soff >= dhp->dh_len)
3089 		return (FC_MAKE_ERR(EINVAL));
3090 
3091 	/*
3092 	 * Address and size must be page aligned.  Len is set to the
3093 	 * number of bytes in the number of pages that are required to
3094 	 * support len.  Offset is set to the byte offset of the first byte
3095 	 * of the page that contains offset.
3096 	 */
3097 	len = round_up_p2(len, PAGESIZE);
3098 
3099 	/*
3100 	 * If len == 0, then calculate the size by getting
3101 	 * the number of bytes from offset to the end of the segment.
3102 	 */
3103 	if (len == 0)
3104 		size = dhp->dh_len - soff;
3105 	else {
3106 		size = len;
3107 		if ((soff + size) > dhp->dh_len)
3108 			return (FC_MAKE_ERR(EINVAL));
3109 	}
3110 
3111 	/*
3112 	 * The address is offset bytes from the base address of
3113 	 * the segment.
3114 	 */
3115 	addr = (caddr_t)(soff + dhp->dh_uvaddr);
3116 
3117 	HOLD_DHP_LOCK(dhp);
3118 	rc = segdev_faultpages(asp->a_hat,
3119 	    dhp->dh_seg, addr, size, type, rw, dhp);
3120 	RELE_DHP_LOCK(dhp);
3121 	return (rc);
3122 }
3123 
3124 int
devmap_setup(dev_t dev,offset_t off,struct as * as,caddr_t * addrp,size_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3125 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3126     size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3127 {
3128 	register devmap_handle_t *dhp;
3129 	int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3130 	    size_t *, uint_t);
3131 	int (*mmap)(dev_t, off_t, int);
3132 	struct devmap_callback_ctl *callbackops;
3133 	devmap_handle_t *dhp_head = NULL;
3134 	devmap_handle_t *dhp_prev = NULL;
3135 	devmap_handle_t *dhp_curr;
3136 	caddr_t addr;
3137 	int map_flag;
3138 	int ret;
3139 	ulong_t total_len;
3140 	size_t map_len;
3141 	size_t resid_len = len;
3142 	offset_t map_off = off;
3143 	struct devmap_softlock *slock = NULL;
3144 
3145 #ifdef lint
3146 	cred = cred;
3147 #endif
3148 
3149 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3150 	    "devmap_setup:start off=%llx len=%lx", off, len);
3151 	DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3152 	    off, len));
3153 
3154 	devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3155 	mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3156 
3157 	/*
3158 	 * driver must provide devmap(9E) entry point in cb_ops to use the
3159 	 * devmap framework.
3160 	 */
3161 	if (devmap == NULL || devmap == nulldev || devmap == nodev)
3162 		return (EINVAL);
3163 
3164 	/*
3165 	 * To protect from an inadvertent entry because the devmap entry point
3166 	 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3167 	 * mmap is NULL.
3168 	 */
3169 	map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3170 	if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3171 		return (EINVAL);
3172 
3173 	/*
3174 	 * devmap allows mmap(2) to map multiple registers.
3175 	 * one devmap_handle is created for each register mapped.
3176 	 */
3177 	for (total_len = 0; total_len < len; total_len += map_len) {
3178 		dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3179 
3180 		if (dhp_prev != NULL)
3181 			dhp_prev->dh_next = dhp;
3182 		else
3183 			dhp_head = dhp;
3184 		dhp_prev = dhp;
3185 
3186 		dhp->dh_prot = prot;
3187 		dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3188 		dhp->dh_dev = dev;
3189 		dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3190 		dhp->dh_uoff = map_off;
3191 
3192 		/*
3193 		 * Get mapping specific info from
3194 		 * the driver, such as rnumber, roff, len, callbackops,
3195 		 * accattrp and, if the mapping is for kernel memory,
3196 		 * ddi_umem_cookie.
3197 		 */
3198 		if ((ret = cdev_devmap(dev, dhp, map_off,
3199 		    resid_len, &map_len, get_udatamodel())) != 0) {
3200 			free_devmap_handle(dhp_head);
3201 			return (ENXIO);
3202 		}
3203 
3204 		if (map_len & PAGEOFFSET) {
3205 			free_devmap_handle(dhp_head);
3206 			return (EINVAL);
3207 		}
3208 
3209 		callbackops = &dhp->dh_callbackops;
3210 
3211 		if ((callbackops->devmap_access == NULL) ||
3212 		    (callbackops->devmap_access == nulldev) ||
3213 		    (callbackops->devmap_access == nodev)) {
3214 			/*
3215 			 * Normally devmap does not support MAP_PRIVATE unless
3216 			 * the drivers provide a valid devmap_access routine.
3217 			 */
3218 			if ((flags & MAP_PRIVATE) != 0) {
3219 				free_devmap_handle(dhp_head);
3220 				return (EINVAL);
3221 			}
3222 		} else {
3223 			/*
3224 			 * Initialize dhp_softlock and dh_ctx if the drivers
3225 			 * provide devmap_access.
3226 			 */
3227 			dhp->dh_softlock = devmap_softlock_init(dev,
3228 			    (ulong_t)callbackops->devmap_access);
3229 			dhp->dh_ctx = devmap_ctxinit(dev,
3230 			    (ulong_t)callbackops->devmap_access);
3231 
3232 			/*
3233 			 * segdev_fault can only work when all
3234 			 * dh_softlock in a multi-dhp mapping
3235 			 * are same. see comments in segdev_fault
3236 			 * This code keeps track of the first
3237 			 * dh_softlock allocated in slock and
3238 			 * compares all later allocations and if
3239 			 * not similar, returns an error.
3240 			 */
3241 			if (slock == NULL)
3242 				slock = dhp->dh_softlock;
3243 			if (slock != dhp->dh_softlock) {
3244 				free_devmap_handle(dhp_head);
3245 				return (ENOTSUP);
3246 			}
3247 		}
3248 
3249 		map_off += map_len;
3250 		resid_len -= map_len;
3251 	}
3252 
3253 	/*
3254 	 * get the user virtual address and establish the mapping between
3255 	 * uvaddr and device physical address.
3256 	 */
3257 	if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3258 	    != 0) {
3259 		/*
3260 		 * free devmap handles if error during the mapping.
3261 		 */
3262 		free_devmap_handle(dhp_head);
3263 
3264 		return (ret);
3265 	}
3266 
3267 	/*
3268 	 * call the driver's devmap_map callback to do more after the mapping,
3269 	 * such as to allocate driver private data for context management.
3270 	 */
3271 	dhp = dhp_head;
3272 	map_off = off;
3273 	addr = *addrp;
3274 	while (dhp != NULL) {
3275 		callbackops = &dhp->dh_callbackops;
3276 		dhp->dh_uvaddr = addr;
3277 		dhp_curr = dhp;
3278 		if (callbackops->devmap_map != NULL) {
3279 			ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3280 			    dev, flags, map_off,
3281 			    dhp->dh_len, &dhp->dh_pvtp);
3282 			if (ret != 0) {
3283 				struct segdev_data *sdp;
3284 
3285 				/*
3286 				 * call driver's devmap_unmap entry point
3287 				 * to free driver resources.
3288 				 */
3289 				dhp = dhp_head;
3290 				map_off = off;
3291 				while (dhp != dhp_curr) {
3292 					callbackops = &dhp->dh_callbackops;
3293 					if (callbackops->devmap_unmap != NULL) {
3294 						(*callbackops->devmap_unmap)(
3295 						    dhp, dhp->dh_pvtp,
3296 						    map_off, dhp->dh_len,
3297 						    NULL, NULL, NULL, NULL);
3298 					}
3299 					map_off += dhp->dh_len;
3300 					dhp = dhp->dh_next;
3301 				}
3302 				sdp = dhp_head->dh_seg->s_data;
3303 				sdp->devmap_data = NULL;
3304 				free_devmap_handle(dhp_head);
3305 				return (ENXIO);
3306 			}
3307 		}
3308 		map_off += dhp->dh_len;
3309 		addr += dhp->dh_len;
3310 		dhp = dhp->dh_next;
3311 	}
3312 
3313 	return (0);
3314 }
3315 
3316 int
ddi_devmap_segmap(dev_t dev,off_t off,ddi_as_handle_t as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3317 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3318     off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3319 {
3320 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3321 	    "devmap_segmap:start");
3322 	return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3323 	    (size_t)len, prot, maxprot, flags, cred));
3324 }
3325 
3326 /*
3327  * Called from devmap_devmem_setup/remap to see if can use large pages for
3328  * this device mapping.
3329  * Also calculate the max. page size for this mapping.
3330  * this page size will be used in fault routine for
3331  * optimal page size calculations.
3332  */
3333 static void
devmap_devmem_large_page_setup(devmap_handle_t * dhp)3334 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3335 {
3336 	ASSERT(dhp_is_devmem(dhp));
3337 	dhp->dh_mmulevel = 0;
3338 
3339 	/*
3340 	 * use large page size only if:
3341 	 *  1. device memory.
3342 	 *  2. mmu supports multiple page sizes,
3343 	 *  3. Driver did not disallow it
3344 	 *  4. dhp length is at least as big as the large pagesize
3345 	 *  5. the uvaddr and pfn are large pagesize aligned
3346 	 */
3347 	if (page_num_pagesizes() > 1 &&
3348 	    !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3349 		ulong_t base;
3350 		int level;
3351 
3352 		base = (ulong_t)ptob(dhp->dh_pfn);
3353 		for (level = 1; level < page_num_pagesizes(); level++) {
3354 			size_t pgsize = page_get_pagesize(level);
3355 			if ((dhp->dh_len < pgsize) ||
3356 			    (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3357 			    base, pgsize))) {
3358 				break;
3359 			}
3360 		}
3361 		dhp->dh_mmulevel = level - 1;
3362 	}
3363 	if (dhp->dh_mmulevel > 0) {
3364 		dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3365 	} else {
3366 		dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3367 	}
3368 }
3369 
3370 /*
3371  * Called by driver devmap routine to pass device specific info to
3372  * the framework.    used for device memory mapping only.
3373  */
3374 int
devmap_devmem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3375 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3376     struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3377     size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3378 {
3379 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3380 	ddi_acc_handle_t handle;
3381 	ddi_map_req_t mr;
3382 	ddi_acc_hdl_t *hp;
3383 	int err;
3384 
3385 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3386 	    "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3387 	    (void *)dhp, roff, rnumber, (uint_t)len);
3388 	DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3389 	    "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3390 
3391 	/*
3392 	 * First to check if this function has been called for this dhp.
3393 	 */
3394 	if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3395 		return (DDI_FAILURE);
3396 
3397 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3398 		return (DDI_FAILURE);
3399 
3400 	if (flags & DEVMAP_MAPPING_INVALID) {
3401 		/*
3402 		 * Don't go up the tree to get pfn if the driver specifies
3403 		 * DEVMAP_MAPPING_INVALID in flags.
3404 		 *
3405 		 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3406 		 * remap permission.
3407 		 */
3408 		if (!(flags & DEVMAP_ALLOW_REMAP)) {
3409 			return (DDI_FAILURE);
3410 		}
3411 		dhp->dh_pfn = PFN_INVALID;
3412 	} else {
3413 		handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3414 		if (handle == NULL)
3415 			return (DDI_FAILURE);
3416 
3417 		hp = impl_acc_hdl_get(handle);
3418 		hp->ah_vers = VERS_ACCHDL;
3419 		hp->ah_dip = dip;
3420 		hp->ah_rnumber = rnumber;
3421 		hp->ah_offset = roff;
3422 		hp->ah_len = len;
3423 		if (accattrp != NULL)
3424 			hp->ah_acc = *accattrp;
3425 
3426 		mr.map_op = DDI_MO_MAP_LOCKED;
3427 		mr.map_type = DDI_MT_RNUMBER;
3428 		mr.map_obj.rnumber = rnumber;
3429 		mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3430 		mr.map_flags = DDI_MF_DEVICE_MAPPING;
3431 		mr.map_handlep = hp;
3432 		mr.map_vers = DDI_MAP_VERSION;
3433 
3434 		/*
3435 		 * up the device tree to get pfn.
3436 		 * The rootnex_map_regspec() routine in nexus drivers has been
3437 		 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3438 		 */
3439 		err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3440 		dhp->dh_hat_attr = hp->ah_hat_flags;
3441 		impl_acc_hdl_free(handle);
3442 
3443 		if (err)
3444 			return (DDI_FAILURE);
3445 	}
3446 	/* Should not be using devmem setup for memory pages */
3447 	ASSERT(!pf_is_memory(dhp->dh_pfn));
3448 
3449 	/* Only some of the flags bits are settable by the driver */
3450 	dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3451 	dhp->dh_len = ptob(btopr(len));
3452 
3453 	dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3454 	dhp->dh_roff = ptob(btop(roff));
3455 
3456 	/* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3457 	devmap_devmem_large_page_setup(dhp);
3458 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3459 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3460 
3461 
3462 	if (callbackops != NULL) {
3463 		bcopy(callbackops, &dhp->dh_callbackops,
3464 		    sizeof (struct devmap_callback_ctl));
3465 	}
3466 
3467 	/*
3468 	 * Initialize dh_lock if we want to do remap.
3469 	 */
3470 	if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3471 		mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3472 		dhp->dh_flags |= DEVMAP_LOCK_INITED;
3473 	}
3474 
3475 	dhp->dh_flags |= DEVMAP_SETUP_DONE;
3476 
3477 	return (DDI_SUCCESS);
3478 }
3479 
3480 int
devmap_devmem_remap(devmap_cookie_t dhc,dev_info_t * dip,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3481 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3482     uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3483     uint_t flags, ddi_device_acc_attr_t *accattrp)
3484 {
3485 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3486 	ddi_acc_handle_t handle;
3487 	ddi_map_req_t mr;
3488 	ddi_acc_hdl_t *hp;
3489 	pfn_t	pfn;
3490 	uint_t	hat_flags;
3491 	int	err;
3492 
3493 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3494 	    "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3495 	    (void *)dhp, roff, rnumber, (uint_t)len);
3496 	DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3497 	    "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3498 
3499 	/*
3500 	 * Return failure if setup has not been done or no remap permission
3501 	 * has been granted during the setup.
3502 	 */
3503 	if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3504 	    (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3505 		return (DDI_FAILURE);
3506 
3507 	/* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3508 	if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3509 		return (DDI_FAILURE);
3510 
3511 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3512 		return (DDI_FAILURE);
3513 
3514 	if (!(flags & DEVMAP_MAPPING_INVALID)) {
3515 		handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3516 		if (handle == NULL)
3517 			return (DDI_FAILURE);
3518 	}
3519 
3520 	HOLD_DHP_LOCK(dhp);
3521 
3522 	/*
3523 	 * Unload the old mapping, so next fault will setup the new mappings
3524 	 * Do this while holding the dhp lock so other faults dont reestablish
3525 	 * the mappings
3526 	 */
3527 	hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3528 	    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3529 
3530 	if (flags & DEVMAP_MAPPING_INVALID) {
3531 		dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3532 		dhp->dh_pfn = PFN_INVALID;
3533 	} else {
3534 		/* clear any prior DEVMAP_MAPPING_INVALID flag */
3535 		dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3536 		hp = impl_acc_hdl_get(handle);
3537 		hp->ah_vers = VERS_ACCHDL;
3538 		hp->ah_dip = dip;
3539 		hp->ah_rnumber = rnumber;
3540 		hp->ah_offset = roff;
3541 		hp->ah_len = len;
3542 		if (accattrp != NULL)
3543 			hp->ah_acc = *accattrp;
3544 
3545 		mr.map_op = DDI_MO_MAP_LOCKED;
3546 		mr.map_type = DDI_MT_RNUMBER;
3547 		mr.map_obj.rnumber = rnumber;
3548 		mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3549 		mr.map_flags = DDI_MF_DEVICE_MAPPING;
3550 		mr.map_handlep = hp;
3551 		mr.map_vers = DDI_MAP_VERSION;
3552 
3553 		/*
3554 		 * up the device tree to get pfn.
3555 		 * The rootnex_map_regspec() routine in nexus drivers has been
3556 		 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3557 		 */
3558 		err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3559 		hat_flags = hp->ah_hat_flags;
3560 		impl_acc_hdl_free(handle);
3561 		if (err) {
3562 			RELE_DHP_LOCK(dhp);
3563 			return (DDI_FAILURE);
3564 		}
3565 		/*
3566 		 * Store result of ddi_map first in local variables, as we do
3567 		 * not want to overwrite the existing dhp with wrong data.
3568 		 */
3569 		dhp->dh_pfn = pfn;
3570 		dhp->dh_hat_attr = hat_flags;
3571 	}
3572 
3573 	/* clear the large page size flag */
3574 	dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3575 
3576 	dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3577 	dhp->dh_roff = ptob(btop(roff));
3578 
3579 	/* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3580 	devmap_devmem_large_page_setup(dhp);
3581 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3582 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3583 
3584 	RELE_DHP_LOCK(dhp);
3585 	return (DDI_SUCCESS);
3586 }
3587 
3588 /*
3589  * called by driver devmap routine to pass kernel virtual address  mapping
3590  * info to the framework.    used only for kernel memory
3591  * allocated from ddi_umem_alloc().
3592  */
3593 int
devmap_umem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3594 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3595     struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3596     offset_t off, size_t len, uint_t maxprot, uint_t flags,
3597     ddi_device_acc_attr_t *accattrp)
3598 {
3599 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3600 	struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3601 
3602 #ifdef lint
3603 	dip = dip;
3604 #endif
3605 
3606 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3607 	    "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3608 	    (void *)dhp, off, cookie, len);
3609 	DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3610 	    "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3611 
3612 	if (cookie == NULL)
3613 		return (DDI_FAILURE);
3614 
3615 	/* For UMEM_TRASH, this restriction is not needed */
3616 	if ((off + len) > cp->size)
3617 		return (DDI_FAILURE);
3618 
3619 	/* check if the cache attributes are supported */
3620 	if (i_ddi_check_cache_attr(flags) == B_FALSE)
3621 		return (DDI_FAILURE);
3622 
3623 	/*
3624 	 * First to check if this function has been called for this dhp.
3625 	 */
3626 	if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3627 		return (DDI_FAILURE);
3628 
3629 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3630 		return (DDI_FAILURE);
3631 
3632 	if (flags & DEVMAP_MAPPING_INVALID) {
3633 		/*
3634 		 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3635 		 * remap permission.
3636 		 */
3637 		if (!(flags & DEVMAP_ALLOW_REMAP)) {
3638 			return (DDI_FAILURE);
3639 		}
3640 	} else {
3641 		dhp->dh_cookie = cookie;
3642 		dhp->dh_roff = ptob(btop(off));
3643 		dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3644 		/* set HAT cache attributes */
3645 		i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3646 		/* set HAT endianess attributes */
3647 		i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3648 	}
3649 
3650 	/*
3651 	 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3652 	 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3653 	 * create consistent mappings but our intention was to create
3654 	 * non-consistent mappings.
3655 	 *
3656 	 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3657 	 * mappings.
3658 	 *
3659 	 * kernel exported memory: hat figures it out it's memory and always
3660 	 * creates consistent mappings.
3661 	 *
3662 	 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3663 	 *
3664 	 * /dev/kmem: consistent mappings are created unless they are
3665 	 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3666 	 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3667 	 * mappings of /dev/kmem. See common/io/mem.c
3668 	 */
3669 
3670 	/* Only some of the flags bits are settable by the driver */
3671 	dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3672 
3673 	dhp->dh_len = ptob(btopr(len));
3674 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3675 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3676 
3677 	if (callbackops != NULL) {
3678 		bcopy(callbackops, &dhp->dh_callbackops,
3679 		    sizeof (struct devmap_callback_ctl));
3680 	}
3681 	/*
3682 	 * Initialize dh_lock if we want to do remap.
3683 	 */
3684 	if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3685 		mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3686 		dhp->dh_flags |= DEVMAP_LOCK_INITED;
3687 	}
3688 
3689 	dhp->dh_flags |= DEVMAP_SETUP_DONE;
3690 
3691 	return (DDI_SUCCESS);
3692 }
3693 
3694 int
devmap_umem_remap(devmap_cookie_t dhc,dev_info_t * dip,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3695 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3696     ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3697     uint_t flags, ddi_device_acc_attr_t *accattrp)
3698 {
3699 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3700 	struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3701 
3702 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3703 	    "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3704 	    (void *)dhp, off, cookie, len);
3705 	DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3706 	    "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3707 
3708 #ifdef lint
3709 	dip = dip;
3710 	accattrp = accattrp;
3711 #endif
3712 	/*
3713 	 * Reture failure if setup has not been done or no remap permission
3714 	 * has been granted during the setup.
3715 	 */
3716 	if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3717 	    (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3718 		return (DDI_FAILURE);
3719 
3720 	/* No flags supported for remap yet */
3721 	if (flags != 0)
3722 		return (DDI_FAILURE);
3723 
3724 	/* check if the cache attributes are supported */
3725 	if (i_ddi_check_cache_attr(flags) == B_FALSE)
3726 		return (DDI_FAILURE);
3727 
3728 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3729 		return (DDI_FAILURE);
3730 
3731 	/* For UMEM_TRASH, this restriction is not needed */
3732 	if ((off + len) > cp->size)
3733 		return (DDI_FAILURE);
3734 
3735 	HOLD_DHP_LOCK(dhp);
3736 	/*
3737 	 * Unload the old mapping, so next fault will setup the new mappings
3738 	 * Do this while holding the dhp lock so other faults dont reestablish
3739 	 * the mappings
3740 	 */
3741 	hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3742 	    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3743 
3744 	dhp->dh_cookie = cookie;
3745 	dhp->dh_roff = ptob(btop(off));
3746 	dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3747 	/* set HAT cache attributes */
3748 	i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3749 	/* set HAT endianess attributes */
3750 	i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3751 
3752 	/* clear the large page size flag */
3753 	dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3754 
3755 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3756 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3757 	RELE_DHP_LOCK(dhp);
3758 	return (DDI_SUCCESS);
3759 }
3760 
3761 /*
3762  * to set timeout value for the driver's context management callback, e.g.
3763  * devmap_access().
3764  */
3765 void
devmap_set_ctx_timeout(devmap_cookie_t dhc,clock_t ticks)3766 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3767 {
3768 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3769 
3770 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3771 	    "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3772 	    (void *)dhp, ticks);
3773 	dhp->dh_timeout_length = ticks;
3774 }
3775 
3776 int
devmap_default_access(devmap_cookie_t dhp,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw)3777 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3778     size_t len, uint_t type, uint_t rw)
3779 {
3780 #ifdef lint
3781 	pvtp = pvtp;
3782 #endif
3783 
3784 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3785 	    "devmap_default_access:start");
3786 	return (devmap_load(dhp, off, len, type, rw));
3787 }
3788 
3789 /*
3790  * segkmem_alloc() wrapper to allocate memory which is both
3791  * non-relocatable (for DR) and sharelocked, since the rest
3792  * of this segment driver requires it.
3793  */
3794 static void *
devmap_alloc_pages(vmem_t * vmp,size_t size,int vmflag)3795 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3796 {
3797 	ASSERT(vmp != NULL);
3798 	ASSERT(kvseg.s_base != NULL);
3799 	vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3800 	return (segkmem_alloc(vmp, size, vmflag));
3801 }
3802 
3803 /*
3804  * This is where things are a bit incestuous with seg_kmem: unlike
3805  * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3806  * we need to do a bit of a dance around that to prevent duplication of
3807  * code until we decide to bite the bullet and implement a new kernel
3808  * segment for driver-allocated memory that is exported to user space.
3809  */
3810 static void
devmap_free_pages(vmem_t * vmp,void * inaddr,size_t size)3811 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3812 {
3813 	page_t *pp;
3814 	caddr_t addr = inaddr;
3815 	caddr_t eaddr;
3816 	pgcnt_t npages = btopr(size);
3817 
3818 	ASSERT(vmp != NULL);
3819 	ASSERT(kvseg.s_base != NULL);
3820 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3821 
3822 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3823 
3824 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3825 		/*
3826 		 * Use page_find() instead of page_lookup() to find the page
3827 		 * since we know that it is hashed and has a shared lock.
3828 		 */
3829 		pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3830 
3831 		if (pp == NULL)
3832 			panic("devmap_free_pages: page not found");
3833 		if (!page_tryupgrade(pp)) {
3834 			page_unlock(pp);
3835 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3836 			    SE_EXCL);
3837 			if (pp == NULL)
3838 				panic("devmap_free_pages: page already freed");
3839 		}
3840 		/* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3841 		pp->p_lckcnt = 0;
3842 		page_destroy(pp, 0);
3843 	}
3844 	page_unresv(npages);
3845 
3846 	if (vmp != NULL)
3847 		vmem_free(vmp, inaddr, size);
3848 }
3849 
3850 /*
3851  * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3852  * allocating non-pageable kmem in response to a ddi_umem_alloc()
3853  * default request. For now we allocate our own pages and we keep
3854  * them long-term sharelocked, since: A) the fault routines expect the
3855  * memory to already be locked; B) pageable umem is already long-term
3856  * locked; C) it's a lot of work to make it otherwise, particularly
3857  * since the nexus layer expects the pages to never fault. An RFE is to
3858  * not keep the pages long-term locked, but instead to be able to
3859  * take faults on them and simply look them up in kvp in case we
3860  * fault on them. Even then, we must take care not to let pageout
3861  * steal them from us since the data must remain resident; if we
3862  * do this we must come up with some way to pin the pages to prevent
3863  * faults while a driver is doing DMA to/from them.
3864  */
3865 static void *
devmap_umem_alloc_np(size_t size,size_t flags)3866 devmap_umem_alloc_np(size_t size, size_t flags)
3867 {
3868 	void *buf;
3869 	int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3870 
3871 	buf = vmem_alloc(umem_np_arena, size, vmflags);
3872 	if (buf != NULL)
3873 		bzero(buf, size);
3874 	return (buf);
3875 }
3876 
3877 static void
devmap_umem_free_np(void * addr,size_t size)3878 devmap_umem_free_np(void *addr, size_t size)
3879 {
3880 	vmem_free(umem_np_arena, addr, size);
3881 }
3882 
3883 /*
3884  * allocate page aligned kernel memory for exporting to user land.
3885  * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3886  * to find a user virtual address that is in same color as the address
3887  * allocated here.
3888  */
3889 void *
ddi_umem_alloc(size_t size,int flags,ddi_umem_cookie_t * cookie)3890 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3891 {
3892 	register size_t len = ptob(btopr(size));
3893 	void *buf = NULL;
3894 	struct ddi_umem_cookie *cp;
3895 	int iflags = 0;
3896 
3897 	*cookie = NULL;
3898 
3899 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3900 	    "devmap_umem_alloc:start");
3901 	if (len == 0)
3902 		return ((void *)NULL);
3903 
3904 	/*
3905 	 * allocate cookie
3906 	 */
3907 	if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3908 	    flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3909 		ASSERT(flags & DDI_UMEM_NOSLEEP);
3910 		return ((void *)NULL);
3911 	}
3912 
3913 	if (flags & DDI_UMEM_PAGEABLE) {
3914 		/* Only one of the flags is allowed */
3915 		ASSERT(!(flags & DDI_UMEM_TRASH));
3916 		/* initialize resource with 0 */
3917 		iflags = KPD_ZERO;
3918 
3919 		/*
3920 		 * to allocate unlocked pageable memory, use segkp_get() to
3921 		 * create a segkp segment.  Since segkp can only service kas,
3922 		 * other segment drivers such as segdev have to do
3923 		 * as_fault(segkp, SOFTLOCK) in its fault routine,
3924 		 */
3925 		if (flags & DDI_UMEM_NOSLEEP)
3926 			iflags |= KPD_NOWAIT;
3927 
3928 		if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3929 			kmem_free(cp, sizeof (struct ddi_umem_cookie));
3930 			return ((void *)NULL);
3931 		}
3932 		cp->type = KMEM_PAGEABLE;
3933 		mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3934 		cp->locked = 0;
3935 	} else if (flags & DDI_UMEM_TRASH) {
3936 		/* Only one of the flags is allowed */
3937 		ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3938 		cp->type = UMEM_TRASH;
3939 		buf = NULL;
3940 	} else {
3941 		if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3942 			kmem_free(cp, sizeof (struct ddi_umem_cookie));
3943 			return ((void *)NULL);
3944 		}
3945 
3946 		cp->type = KMEM_NON_PAGEABLE;
3947 	}
3948 
3949 	/*
3950 	 * need to save size here.  size will be used when
3951 	 * we do kmem_free.
3952 	 */
3953 	cp->size = len;
3954 	cp->cvaddr = (caddr_t)buf;
3955 
3956 	*cookie =  (void *)cp;
3957 	return (buf);
3958 }
3959 
3960 void
ddi_umem_free(ddi_umem_cookie_t cookie)3961 ddi_umem_free(ddi_umem_cookie_t cookie)
3962 {
3963 	struct ddi_umem_cookie *cp;
3964 
3965 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3966 	    "devmap_umem_free:start");
3967 
3968 	/*
3969 	 * if cookie is NULL, no effects on the system
3970 	 */
3971 	if (cookie == NULL)
3972 		return;
3973 
3974 	cp = (struct ddi_umem_cookie *)cookie;
3975 
3976 	switch (cp->type) {
3977 	case KMEM_PAGEABLE :
3978 		ASSERT(cp->cvaddr != NULL && cp->size != 0);
3979 		/*
3980 		 * Check if there are still any pending faults on the cookie
3981 		 * while the driver is deleting it,
3982 		 * XXX - could change to an ASSERT but wont catch errant drivers
3983 		 */
3984 		mutex_enter(&cp->lock);
3985 		if (cp->locked) {
3986 			mutex_exit(&cp->lock);
3987 			panic("ddi_umem_free for cookie with pending faults %p",
3988 			    (void *)cp);
3989 			return;
3990 		}
3991 
3992 		segkp_release(segkp, cp->cvaddr);
3993 
3994 		/*
3995 		 * release mutex associated with this cookie.
3996 		 */
3997 		mutex_destroy(&cp->lock);
3998 		break;
3999 	case KMEM_NON_PAGEABLE :
4000 		ASSERT(cp->cvaddr != NULL && cp->size != 0);
4001 		devmap_umem_free_np(cp->cvaddr, cp->size);
4002 		break;
4003 	case UMEM_TRASH :
4004 		break;
4005 	case UMEM_LOCKED :
4006 		/* Callers should use ddi_umem_unlock for this type */
4007 		ddi_umem_unlock(cookie);
4008 		/* Frees the cookie too */
4009 		return;
4010 	default:
4011 		/* panic so we can diagnose the underlying cause */
4012 		panic("ddi_umem_free: illegal cookie type 0x%x\n",
4013 		    cp->type);
4014 	}
4015 
4016 	kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4017 }
4018 
4019 
4020 static int
segdev_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)4021 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4022 {
4023 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4024 
4025 	/*
4026 	 * It looks as if it is always mapped shared
4027 	 */
4028 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4029 	    "segdev_getmemid:start");
4030 	memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4031 	memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4032 	return (0);
4033 }
4034 
4035 /*ARGSUSED*/
4036 static lgrp_mem_policy_info_t *
segdev_getpolicy(struct seg * seg,caddr_t addr)4037 segdev_getpolicy(struct seg *seg, caddr_t addr)
4038 {
4039 	return (NULL);
4040 }
4041 
4042 /*ARGSUSED*/
4043 static int
segdev_capable(struct seg * seg,segcapability_t capability)4044 segdev_capable(struct seg *seg, segcapability_t capability)
4045 {
4046 	return (0);
4047 }
4048 
4049 /*
4050  * ddi_umem_alloc() non-pageable quantum cache max size.
4051  * This is just a SWAG.
4052  */
4053 #define	DEVMAP_UMEM_QUANTUM	(8*PAGESIZE)
4054 
4055 /*
4056  * Initialize seg_dev from boot. This routine sets up the trash page
4057  * and creates the umem_np_arena used to back non-pageable memory
4058  * requests.
4059  */
4060 void
segdev_init(void)4061 segdev_init(void)
4062 {
4063 	struct seg kseg;
4064 
4065 	umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4066 	    devmap_alloc_pages, devmap_free_pages, heap_arena,
4067 	    DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4068 
4069 	kseg.s_as = &kas;
4070 	trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4071 	    PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4072 	if (trashpp == NULL)
4073 		panic("segdev_init: failed to create trash page");
4074 	pagezero(trashpp, 0, PAGESIZE);
4075 	page_downgrade(trashpp);
4076 }
4077 
4078 /*
4079  * Invoke platform-dependent support routines so that /proc can have
4080  * the platform code deal with curious hardware.
4081  */
4082 int
segdev_copyfrom(struct seg * seg,caddr_t uaddr,const void * devaddr,void * kaddr,size_t len)4083 segdev_copyfrom(struct seg *seg,
4084     caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4085 {
4086 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4087 	struct snode *sp = VTOS(VTOCVP(sdp->vp));
4088 
4089 	return (e_ddi_copyfromdev(sp->s_dip,
4090 	    (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4091 }
4092 
4093 int
segdev_copyto(struct seg * seg,caddr_t uaddr,const void * kaddr,void * devaddr,size_t len)4094 segdev_copyto(struct seg *seg,
4095     caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4096 {
4097 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4098 	struct snode *sp = VTOS(VTOCVP(sdp->vp));
4099 
4100 	return (e_ddi_copytodev(sp->s_dip,
4101 	    (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4102 }
4103