xref: /titanic_41/usr/src/uts/common/vm/xhat.c (revision 7c478bd95313f5f23a4c958a745db2134aa03244)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 
30 #include <sys/types.h>
31 #include <sys/cmn_err.h>
32 #include <sys/mman.h>
33 #include <sys/systm.h>
34 #include <vm/xhat.h>
35 #include <vm/page.h>
36 #include <vm/as.h>
37 
38 int xhat_debug = 0;
39 
40 krwlock_t xhat_provider_rwlock;
41 xhat_provider_t *xhat_provider = NULL;
42 
43 void
xhat_init()44 xhat_init()
45 {
46 	rw_init(&xhat_provider_rwlock, NULL, RW_DEFAULT, NULL);
47 }
48 
49 
50 
51 int
xhat_provider_register(xhat_provider_t * provider)52 xhat_provider_register(xhat_provider_t *provider)
53 {
54 	/* strlen("_cache") = 7 */
55 	char	cache_name[XHAT_CACHE_NAMELEN + 7];
56 
57 
58 	if (provider->xhat_provider_version != XHAT_PROVIDER_VERSION) {
59 		cmn_err(CE_WARN, "XHAT provider version mismatch");
60 		return (-1);
61 	}
62 
63 	if ((XHAT_POPS(provider)->xhat_alloc == NULL) ||
64 	    (XHAT_POPS(provider)->xhat_free == NULL)) {
65 		cmn_err(CE_WARN, "Malformed XHAT provider");
66 		return (-1);
67 	}
68 
69 	/* Allocate kmem_cache which will manage xhat blocks */
70 	provider->xblkcache->free_blks = NULL;
71 	(void) strncpy(cache_name, provider->xhat_provider_name,
72 	    XHAT_CACHE_NAMELEN);
73 	(void) strcat(cache_name, "_cache");
74 	provider->xblkcache->cache = kmem_cache_create(cache_name,
75 	    provider->xhat_provider_blk_size, 0, NULL, NULL,
76 	    provider->xblkcache->reclaim,
77 	    (void *)provider, NULL, 0);
78 	if (provider->xblkcache->cache == NULL) {
79 		cmn_err(CE_WARN, "Failed to allocate cache for %s",
80 		    provider->xhat_provider_name);
81 		return (-1);
82 	}
83 
84 	mutex_init(&provider->xblkcache->lock, NULL, MUTEX_DEFAULT, NULL);
85 
86 
87 	/* Insert provider in the global list */
88 	rw_enter(&xhat_provider_rwlock, RW_WRITER);
89 	provider->next = xhat_provider;
90 	provider->prev = NULL;
91 	if (xhat_provider)
92 		xhat_provider->prev = provider;
93 	xhat_provider = provider;
94 	xhat_provider->xhat_provider_refcnt = 0;
95 	rw_exit(&xhat_provider_rwlock);
96 	return (0);
97 }
98 
99 
100 
101 int
xhat_provider_unregister(xhat_provider_t * provider)102 xhat_provider_unregister(xhat_provider_t *provider)
103 {
104 	if (provider->xhat_provider_version != XHAT_PROVIDER_VERSION)
105 		return (-1);
106 
107 	rw_enter(&xhat_provider_rwlock, RW_WRITER);
108 
109 	if (provider->xhat_provider_refcnt) {
110 		rw_exit(&xhat_provider_rwlock);
111 		return (-1);
112 	}
113 
114 	if (provider->next)
115 		provider->next->prev = provider->prev;
116 	if (provider->prev)
117 		provider->prev->next = provider->next;
118 	else
119 		xhat_provider = provider->next;
120 	provider->prev = NULL;
121 	provider->next = NULL;
122 	rw_exit(&xhat_provider_rwlock);
123 
124 	/* Free all xblks that are sitting on free_blks list */
125 	provider->xblkcache->reclaim(provider);
126 
127 	kmem_cache_destroy(provider->xblkcache->cache);
128 
129 	return (0);
130 }
131 
132 
133 
134 /* Attaches an XHAT to the address space */
135 int
xhat_attach_xhat(xhat_provider_t * provider,struct as * as,struct xhat ** xhatp,void * arg)136 xhat_attach_xhat(xhat_provider_t *provider, struct as *as,
137     struct xhat **xhatp, void *arg)
138 {
139 	struct xhat *xh;
140 
141 
142 
143 	xh = XHAT_POPS(provider)->xhat_alloc(arg);
144 	if (xh == NULL) {
145 		*xhatp = NULL;
146 		return (XH_PRVDR);
147 	}
148 
149 	mutex_init(&xh->xhat_lock, NULL, MUTEX_DEFAULT, NULL);
150 	xh->xhat_provider = provider;
151 
152 	rw_enter(&xhat_provider_rwlock, RW_WRITER);
153 	provider->xhat_provider_refcnt++;
154 	rw_exit(&xhat_provider_rwlock);
155 
156 	mutex_enter(&as->a_contents);
157 
158 	/* Is address space busy (being freed, dup'd or swapped)? */
159 	if (AS_ISBUSY(as)) {
160 		mutex_exit(&as->a_contents);
161 		XHAT_POPS(provider)->xhat_free(xh);
162 
163 		rw_enter(&xhat_provider_rwlock, RW_WRITER);
164 		provider->xhat_provider_refcnt--;
165 		rw_exit(&xhat_provider_rwlock);
166 
167 		*xhatp = NULL;
168 		return (XH_ASBUSY);
169 	}
170 
171 	xh->xhat_as = as;
172 	xh->xhat_refcnt = 0;
173 	xh->holder = NULL;
174 	xh->arg = arg;
175 	xh->next = (struct xhat *)as->a_xhat;
176 	if (xh->next)
177 		xh->next->prev = xh;
178 	as->a_xhat = xh;
179 	mutex_exit(&as->a_contents);
180 	*xhatp = xh;
181 	return (0);
182 }
183 
184 
185 int
xhat_detach_xhat(xhat_provider_t * provider,struct as * as)186 xhat_detach_xhat(xhat_provider_t *provider, struct as *as)
187 {
188 	struct xhat *xh;
189 
190 
191 	mutex_enter(&as->a_contents);
192 
193 	for (xh = (struct xhat *)as->a_xhat; xh != NULL; xh = xh->next)
194 		if (xh->xhat_provider == provider) {
195 
196 
197 			if (xh->holder != NULL) {
198 				/*
199 				 * The address space is being freed,
200 				 * dup'd or swapped out.
201 				 * If we are the thread which doing one
202 				 * of those operations, we can go ahead
203 				 * and free up the XHAT.
204 				 * Otherwise, return.
205 				 */
206 				if (xh->holder != curthread) {
207 					mutex_exit(&as->a_contents);
208 					return (XH_ASBUSY);
209 				} else
210 					xhat_hat_rele(xh);
211 			}
212 
213 			if (xh->xhat_refcnt > 0) {
214 				/*
215 				 * There are still "users" of the XHAT.
216 				 * This may be either because the caller
217 				 * forgot to free something up (which is a bug)
218 				 * or because xhat_op_all() is in progress.
219 				 * Since we are not allowing any of
220 				 * xhat_op_all's ops to call xhat_detach_xhat(),
221 				 * This can only be some other thread. It
222 				 * may want to wait a bit and retry.
223 				 */
224 
225 
226 				/* Restore the hold on the XHAT */
227 				if (xh->holder == curthread)
228 					xhat_hat_hold(xh);
229 
230 				mutex_exit(&as->a_contents);
231 				return (XH_XHHELD);
232 			}
233 
234 			rw_enter(&xhat_provider_rwlock, RW_WRITER);
235 			provider->xhat_provider_refcnt--;
236 			rw_exit(&xhat_provider_rwlock);
237 
238 			if (xh->next)
239 				xh->next->prev = xh->prev;
240 			if (xh->prev)
241 				xh->prev->next = xh->next;
242 			else
243 				as->a_xhat = (void *) xh->next;
244 			mutex_exit(&as->a_contents);
245 
246 			XHAT_POPS(provider)->xhat_free(xh);
247 
248 			return (0);
249 		}
250 	mutex_exit(&as->a_contents);
251 	return (XH_NOTATTCHD);
252 }
253 
254 void
xhat_hat_hold(struct xhat * xhat)255 xhat_hat_hold(struct xhat *xhat)
256 {
257 	mutex_enter(&xhat->xhat_lock);
258 	xhat->xhat_refcnt++;
259 	mutex_exit(&xhat->xhat_lock);
260 }
261 
262 void
xhat_hat_rele(struct xhat * xhat)263 xhat_hat_rele(struct xhat *xhat)
264 {
265 	mutex_enter(&xhat->xhat_lock);
266 	xhat->xhat_refcnt--;
267 	ASSERT(xhat->xhat_refcnt >= 0);
268 	mutex_exit(&xhat->xhat_lock);
269 }
270 
271 
272 int
xhat_hat_holders(struct xhat * xhat)273 xhat_hat_holders(struct xhat *xhat)
274 {
275 	return (xhat->xhat_refcnt);
276 }
277 
278 
279 /*
280  * Assumes that address space is already locked
281  * and that AS_FREE is set for as->a_flags.
282  */
283 void
xhat_free_start_all(struct as * as)284 xhat_free_start_all(struct as *as)
285 {
286 	struct xhat *xh, *xh_nxt;
287 
288 
289 	ASSERT(AS_ISBUSY(as));
290 
291 	mutex_enter(&as->a_contents);
292 	xh = (struct xhat *)as->a_xhat;
293 
294 	/*
295 	 * Simply calling xhat_hat_hold() won't work because we will
296 	 * not be able to succeed in xhat_detach_xhat(), which may
297 	 * get called from here. We need to know _who_ the holder is.
298 	 */
299 	if (xh != NULL) {
300 		xhat_hat_hold(xh);
301 		ASSERT(xh->holder == NULL);
302 		xh->holder = curthread;
303 	}
304 
305 	while (xh != NULL) {
306 
307 		xh_nxt = xh->next;
308 		if (xh_nxt != NULL) {
309 			ASSERT(xh_nxt->holder == NULL);
310 			xhat_hat_hold(xh_nxt);
311 			xh_nxt->holder = curthread;
312 		}
313 
314 		mutex_exit(&as->a_contents);
315 
316 		XHAT_FREE_START(xh);
317 
318 		mutex_enter(&as->a_contents);
319 
320 		xh = xh_nxt;
321 	}
322 
323 	mutex_exit(&as->a_contents);
324 }
325 
326 
327 
328 /*
329  * Assumes that address space is already locked.
330  * Since xhat_free_start_all() must have been called
331  * earlier, for all XHATs holder is set to curthread.
332  * Also, since AS_BUSY is set for as->a_flags, no new
333  * XHATs could have been added.
334  */
335 void
xhat_free_end_all(struct as * as)336 xhat_free_end_all(struct as *as)
337 {
338 
339 	struct xhat *xh, *xh_nxt;
340 
341 	ASSERT(AS_ISBUSY(as));
342 
343 	mutex_enter(&as->a_contents);
344 	xh = (struct xhat *)as->a_xhat;
345 
346 
347 	while (xh != NULL) {
348 
349 		ASSERT(xh->holder == curthread);
350 
351 		xh_nxt = xh->next;
352 
353 		mutex_exit(&as->a_contents);
354 
355 		XHAT_FREE_END(xh);
356 
357 		mutex_enter(&as->a_contents);
358 
359 		xh = xh_nxt;
360 	}
361 
362 	mutex_exit(&as->a_contents);
363 }
364 
365 
366 /* Assumes that address space is already locked */
367 
368 /* ARGSUSED */
369 int
xhat_dup_all(struct as * as,struct as * newas,caddr_t addr,size_t len,uint_t flag)370 xhat_dup_all(struct as *as, struct as *newas, caddr_t addr, size_t len,
371     uint_t flag)
372 {
373 	/* This is not supported. Should we return some sort of error? */
374 
375 	ASSERT(AS_ISBUSY(as));
376 
377 	return (0);
378 }
379 
380 
381 /* Assumes that address space is already locked */
382 void
xhat_swapout_all(struct as * as)383 xhat_swapout_all(struct as *as)
384 {
385 	struct xhat *xh, *xh_nxt;
386 
387 
388 	ASSERT(AS_ISBUSY(as));
389 
390 	mutex_enter(&as->a_contents);
391 	xh = (struct xhat *)as->a_xhat;
392 
393 	if (xh != NULL) {
394 		xhat_hat_hold(xh);
395 		ASSERT(xh->holder == NULL);
396 		xh->holder = curthread;
397 	}
398 
399 
400 	while (xh != NULL) {
401 
402 		xh_nxt = xh->next;
403 		if (xh_nxt != NULL) {
404 			ASSERT(xh_nxt->holder == NULL);
405 			xhat_hat_hold(xh_nxt);
406 			xh_nxt->holder = curthread;
407 		}
408 
409 		mutex_exit(&as->a_contents);
410 
411 		XHAT_SWAPOUT(xh);
412 
413 		mutex_enter(&as->a_contents);
414 
415 		/*
416 		 * If the xh is still there (i.e. swapout did not
417 		 * destroy it), clear the holder field.
418 		 * xh_nxt->prev couldn't have been changed in xhat_attach_xhat()
419 		 * because AS_BUSY is set. xhat_detach_xhat() also couldn't
420 		 * have modified it because (holder != NULL).
421 		 * If there is only one XHAT, just see if a_xhat still
422 		 * points to us.
423 		 */
424 		if (((xh_nxt != NULL) && (xh_nxt->prev == xh)) ||
425 		    ((as->a_xhat != NULL) && (as->a_xhat == xh))) {
426 			xhat_hat_rele(xh);
427 			xh->holder = NULL;
428 		}
429 
430 		xh = xh_nxt;
431 	}
432 
433 	mutex_exit(&as->a_contents);
434 }
435 
436 
437 
438 
439 /*
440  * In the following routines, the appropriate xhat_op
441  * should never attempt to call xhat_detach_xhat(): it will
442  * never succeed since the XHAT is held.
443  */
444 
445 
446 #define	XHAT_UNLOAD_CALLBACK_OP	(0)
447 #define	XHAT_SETATTR_OP		(1)
448 #define	XHAT_CLRATTR_OP		(2)
449 #define	XHAT_CHGATTR_OP		(3)
450 #define	XHAT_CHGPROT_OP		(4)
451 #define	XHAT_UNSHARE_OP		(5)
452 
453 
454 static void
xhat_op_all(int op,struct as * as,caddr_t addr,size_t len,uint_t flags,void * ptr)455 xhat_op_all(int op, struct as *as, caddr_t addr,
456     size_t len, uint_t flags, void *ptr)
457 {
458 	struct xhat *xh, *xh_nxt;
459 
460 	mutex_enter(&as->a_contents);
461 	xh = (struct xhat *)as->a_xhat;
462 
463 	while (xh != NULL) {
464 
465 		xhat_hat_hold(xh);
466 
467 		xh_nxt = xh->next;
468 		if (xh_nxt != NULL)
469 			xhat_hat_hold(xh_nxt);
470 
471 		mutex_exit(&as->a_contents);
472 
473 		switch (op) {
474 		case XHAT_UNLOAD_CALLBACK_OP:
475 			XHAT_UNLOAD_CALLBACK(xh, addr,
476 			    len, flags, (hat_callback_t *)ptr);
477 			break;
478 		case XHAT_SETATTR_OP:
479 			XHAT_SETATTR(xh, addr, len, flags);
480 			break;
481 		case XHAT_CLRATTR_OP:
482 			XHAT_CLRATTR(xh, addr, len, flags);
483 			break;
484 		case XHAT_CHGATTR_OP:
485 			XHAT_CHGATTR(xh, addr, len, flags);
486 			break;
487 		case XHAT_CHGPROT_OP:
488 			XHAT_CHGPROT(xh, addr, len, flags);
489 			break;
490 		case XHAT_UNSHARE_OP:
491 			XHAT_UNSHARE(xh, addr, len);
492 			break;
493 		default:
494 			panic("Unknown op %d in xhat_op_all", op);
495 		}
496 
497 		mutex_enter(&as->a_contents);
498 
499 		/*
500 		 * Both pointers are still valid because both
501 		 * XHATs are held.
502 		 */
503 		xhat_hat_rele(xh);
504 		if (xh_nxt != NULL)
505 			xhat_hat_rele(xh_nxt);
506 		xh = xh_nxt;
507 	}
508 
509 	mutex_exit(&as->a_contents);
510 }
511 
512 
513 
514 void
xhat_unload_callback_all(struct as * as,caddr_t addr,size_t len,uint_t flags,hat_callback_t * callback)515 xhat_unload_callback_all(struct as *as, caddr_t addr, size_t len, uint_t flags,
516     hat_callback_t *callback)
517 {
518 	xhat_op_all(XHAT_UNLOAD_CALLBACK_OP, as, addr, len, flags, callback);
519 }
520 
521 
522 void
xhat_setattr_all(struct as * as,caddr_t addr,size_t len,uint_t attr)523 xhat_setattr_all(struct as *as, caddr_t addr, size_t len, uint_t attr)
524 {
525 	xhat_op_all(XHAT_SETATTR_OP, as, addr, len, attr, NULL);
526 }
527 
528 
529 
530 void
xhat_clrattr_all(struct as * as,caddr_t addr,size_t len,uint_t attr)531 xhat_clrattr_all(struct as *as, caddr_t addr, size_t len, uint_t attr)
532 {
533 	xhat_op_all(XHAT_CLRATTR_OP, as, addr, len, attr, NULL);
534 }
535 
536 
537 void
xhat_chgattr_all(struct as * as,caddr_t addr,size_t len,uint_t attr)538 xhat_chgattr_all(struct as *as, caddr_t addr, size_t len, uint_t attr)
539 {
540 	xhat_op_all(XHAT_CHGATTR_OP, as, addr, len, attr, NULL);
541 }
542 
543 
544 void
xhat_chgprot_all(struct as * as,caddr_t addr,size_t len,uint_t prot)545 xhat_chgprot_all(struct as *as, caddr_t addr, size_t len, uint_t prot)
546 {
547 	xhat_op_all(XHAT_CHGPROT_OP, as, addr, len, prot, NULL);
548 }
549 
550 
551 void
xhat_unshare_all(struct as * as,caddr_t addr,size_t len)552 xhat_unshare_all(struct as *as, caddr_t addr, size_t len)
553 {
554 	xhat_op_all(XHAT_UNSHARE_OP, as, addr, len, 0, NULL);
555 }
556