1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #pragma ident "%Z%%M% %I% %E% SMI" 40 41 /* 42 * VM - address spaces. 43 */ 44 45 #include <sys/types.h> 46 #include <sys/t_lock.h> 47 #include <sys/param.h> 48 #include <sys/errno.h> 49 #include <sys/systm.h> 50 #include <sys/mman.h> 51 #include <sys/sysmacros.h> 52 #include <sys/cpuvar.h> 53 #include <sys/sysinfo.h> 54 #include <sys/kmem.h> 55 #include <sys/vnode.h> 56 #include <sys/vmsystm.h> 57 #include <sys/cmn_err.h> 58 #include <sys/debug.h> 59 #include <sys/tnf_probe.h> 60 #include <sys/vtrace.h> 61 62 #include <vm/hat.h> 63 #include <vm/xhat.h> 64 #include <vm/as.h> 65 #include <vm/seg.h> 66 #include <vm/seg_vn.h> 67 #include <vm/seg_dev.h> 68 #include <vm/seg_kmem.h> 69 #include <vm/seg_map.h> 70 #include <vm/seg_spt.h> 71 #include <vm/page.h> 72 73 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */ 74 75 static struct kmem_cache *as_cache; 76 77 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t); 78 static void as_clearwatchprot(struct as *, caddr_t, size_t); 79 int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *); 80 81 82 /* 83 * Verifying the segment lists is very time-consuming; it may not be 84 * desirable always to define VERIFY_SEGLIST when DEBUG is set. 85 */ 86 #ifdef DEBUG 87 #define VERIFY_SEGLIST 88 int do_as_verify = 0; 89 #endif 90 91 /* 92 * Allocate a new callback data structure entry and fill in the events of 93 * interest, the address range of interest, and the callback argument. 94 * Link the entry on the as->a_callbacks list. A callback entry for the 95 * entire address space may be specified with vaddr = 0 and size = -1. 96 * 97 * CALLERS RESPONSIBILITY: If not calling from within the process context for 98 * the specified as, the caller must guarantee persistence of the specified as 99 * for the duration of this function (eg. pages being locked within the as 100 * will guarantee persistence). 101 */ 102 int 103 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events, 104 caddr_t vaddr, size_t size, int sleepflag) 105 { 106 struct as_callback *current_head, *cb; 107 caddr_t saddr; 108 size_t rsize; 109 110 /* callback function and an event are mandatory */ 111 if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0)) 112 return (EINVAL); 113 114 /* Adding a callback after as_free has been called is not allowed */ 115 if (as == &kas) 116 return (ENOMEM); 117 118 /* 119 * vaddr = 0 and size = -1 is used to indicate that the callback range 120 * is the entire address space so no rounding is done in that case. 121 */ 122 if (size != -1) { 123 saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 124 rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) - 125 (size_t)saddr; 126 /* check for wraparound */ 127 if (saddr + rsize < saddr) 128 return (ENOMEM); 129 } else { 130 if (vaddr != 0) 131 return (EINVAL); 132 saddr = vaddr; 133 rsize = size; 134 } 135 136 /* Allocate and initialize a callback entry */ 137 cb = kmem_zalloc(sizeof (struct as_callback), sleepflag); 138 if (cb == NULL) 139 return (EAGAIN); 140 141 cb->ascb_func = cb_func; 142 cb->ascb_arg = arg; 143 cb->ascb_events = events; 144 cb->ascb_saddr = saddr; 145 cb->ascb_len = rsize; 146 147 /* Add the entry to the list */ 148 mutex_enter(&as->a_contents); 149 current_head = as->a_callbacks; 150 as->a_callbacks = cb; 151 cb->ascb_next = current_head; 152 153 /* 154 * The call to this function may lose in a race with 155 * a pertinent event - eg. a thread does long term memory locking 156 * but before the callback is added another thread executes as_unmap. 157 * A broadcast here resolves that. 158 */ 159 if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) { 160 AS_CLRUNMAPWAIT(as); 161 cv_broadcast(&as->a_cv); 162 } 163 164 mutex_exit(&as->a_contents); 165 return (0); 166 } 167 168 /* 169 * Search the callback list for an entry which pertains to arg. 170 * 171 * This is called from within the client upon completion of the callback. 172 * RETURN VALUES: 173 * AS_CALLBACK_DELETED (callback entry found and deleted) 174 * AS_CALLBACK_NOTFOUND (no callback entry found - this is ok) 175 * AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this 176 * entry will be made in as_do_callbacks) 177 * 178 * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED 179 * set, it indicates that as_do_callbacks is processing this entry. The 180 * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made 181 * to unblock as_do_callbacks, in case it is blocked. 182 * 183 * CALLERS RESPONSIBILITY: If not calling from within the process context for 184 * the specified as, the caller must guarantee persistence of the specified as 185 * for the duration of this function (eg. pages being locked within the as 186 * will guarantee persistence). 187 */ 188 uint_t 189 as_delete_callback(struct as *as, void *arg) 190 { 191 struct as_callback **prevcb = &as->a_callbacks; 192 struct as_callback *cb; 193 uint_t rc = AS_CALLBACK_NOTFOUND; 194 195 mutex_enter(&as->a_contents); 196 for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) { 197 if (cb->ascb_arg != arg) 198 continue; 199 200 /* 201 * If the events indicate AS_CALLBACK_CALLED, just clear 202 * AS_ALL_EVENT in the events field and wakeup the thread 203 * that may be waiting in as_do_callbacks. as_do_callbacks 204 * will take care of removing this entry from the list. In 205 * that case, return AS_CALLBACK_DELETE_DEFERRED. Otherwise 206 * (AS_CALLBACK_CALLED not set), just remove it from the 207 * list, return the memory and return AS_CALLBACK_DELETED. 208 */ 209 if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) { 210 /* leave AS_CALLBACK_CALLED */ 211 cb->ascb_events &= ~AS_ALL_EVENT; 212 rc = AS_CALLBACK_DELETE_DEFERRED; 213 cv_broadcast(&as->a_cv); 214 } else { 215 *prevcb = cb->ascb_next; 216 kmem_free(cb, sizeof (struct as_callback)); 217 rc = AS_CALLBACK_DELETED; 218 } 219 break; 220 } 221 mutex_exit(&as->a_contents); 222 return (rc); 223 } 224 225 /* 226 * Searches the as callback list for a matching entry. 227 * Returns a pointer to the first matching callback, or NULL if 228 * nothing is found. 229 * This function never sleeps so it is ok to call it with more 230 * locks held but the (required) a_contents mutex. 231 * 232 * See also comment on as_do_callbacks below. 233 */ 234 static struct as_callback * 235 as_find_callback(struct as *as, uint_t events, caddr_t event_addr, 236 size_t event_len) 237 { 238 struct as_callback *cb; 239 240 ASSERT(MUTEX_HELD(&as->a_contents)); 241 for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) { 242 /* 243 * If the callback has not already been called, then 244 * check if events or address range pertains. An event_len 245 * of zero means do an unconditional callback. 246 */ 247 if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) || 248 ((event_len != 0) && (((cb->ascb_events & events) == 0) || 249 (event_addr + event_len < cb->ascb_saddr) || 250 (event_addr > (cb->ascb_saddr + cb->ascb_len))))) { 251 continue; 252 } 253 break; 254 } 255 return (cb); 256 } 257 258 /* 259 * Executes a given callback and removes it from the callback list for 260 * this address space. 261 * This function may sleep so the caller must drop all locks except 262 * a_contents before calling this func. 263 * 264 * See also comments on as_do_callbacks below. 265 */ 266 static void 267 as_execute_callback(struct as *as, struct as_callback *cb, 268 uint_t events) 269 { 270 struct as_callback **prevcb; 271 void *cb_arg; 272 273 ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events)); 274 cb->ascb_events |= AS_CALLBACK_CALLED; 275 mutex_exit(&as->a_contents); 276 (*cb->ascb_func)(as, cb->ascb_arg, events); 277 mutex_enter(&as->a_contents); 278 /* 279 * the callback function is required to delete the callback 280 * when the callback function determines it is OK for 281 * this thread to continue. as_delete_callback will clear 282 * the AS_ALL_EVENT in the events field when it is deleted. 283 * If the callback function called as_delete_callback, 284 * events will already be cleared and there will be no blocking. 285 */ 286 while ((cb->ascb_events & events) != 0) { 287 cv_wait(&as->a_cv, &as->a_contents); 288 } 289 /* 290 * This entry needs to be taken off the list. Normally, the 291 * callback func itself does that, but unfortunately the list 292 * may have changed while the callback was running because the 293 * a_contents mutex was dropped and someone else other than the 294 * callback func itself could have called as_delete_callback, 295 * so we have to search to find this entry again. The entry 296 * must have AS_CALLBACK_CALLED, and have the same 'arg'. 297 */ 298 cb_arg = cb->ascb_arg; 299 prevcb = &as->a_callbacks; 300 for (cb = as->a_callbacks; cb != NULL; 301 prevcb = &cb->ascb_next, cb = *prevcb) { 302 if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) || 303 (cb_arg != cb->ascb_arg)) { 304 continue; 305 } 306 *prevcb = cb->ascb_next; 307 kmem_free(cb, sizeof (struct as_callback)); 308 break; 309 } 310 } 311 312 /* 313 * Check the callback list for a matching event and intersection of 314 * address range. If there is a match invoke the callback. Skip an entry if: 315 * - a callback is already in progress for this entry (AS_CALLBACK_CALLED) 316 * - not event of interest 317 * - not address range of interest 318 * 319 * An event_len of zero indicates a request for an unconditional callback 320 * (regardless of event), only the AS_CALLBACK_CALLED is checked. The 321 * a_contents lock must be dropped before a callback, so only one callback 322 * can be done before returning. Return -1 (true) if a callback was 323 * executed and removed from the list, else return 0 (false). 324 * 325 * The logically separate parts, i.e. finding a matching callback and 326 * executing a given callback have been separated into two functions 327 * so that they can be called with different sets of locks held beyond 328 * the always-required a_contents. as_find_callback does not sleep so 329 * it is ok to call it if more locks than a_contents (i.e. the a_lock 330 * rwlock) are held. as_execute_callback on the other hand may sleep 331 * so all locks beyond a_contents must be dropped by the caller if one 332 * does not want to end comatose. 333 */ 334 static int 335 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr, 336 size_t event_len) 337 { 338 struct as_callback *cb; 339 340 if ((cb = as_find_callback(as, events, event_addr, event_len))) { 341 as_execute_callback(as, cb, events); 342 return (-1); 343 } 344 return (0); 345 } 346 347 /* 348 * Search for the segment containing addr. If a segment containing addr 349 * exists, that segment is returned. If no such segment exists, and 350 * the list spans addresses greater than addr, then the first segment 351 * whose base is greater than addr is returned; otherwise, NULL is 352 * returned unless tail is true, in which case the last element of the 353 * list is returned. 354 * 355 * a_seglast is used to cache the last found segment for repeated 356 * searches to the same addr (which happens frequently). 357 */ 358 struct seg * 359 as_findseg(struct as *as, caddr_t addr, int tail) 360 { 361 struct seg *seg = as->a_seglast; 362 avl_index_t where; 363 364 ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 365 366 if (seg != NULL && 367 seg->s_base <= addr && 368 addr < seg->s_base + seg->s_size) 369 return (seg); 370 371 seg = avl_find(&as->a_segtree, &addr, &where); 372 if (seg != NULL) 373 return (as->a_seglast = seg); 374 375 seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 376 if (seg == NULL && tail) 377 seg = avl_last(&as->a_segtree); 378 return (as->a_seglast = seg); 379 } 380 381 #ifdef VERIFY_SEGLIST 382 /* 383 * verify that the linked list is coherent 384 */ 385 static void 386 as_verify(struct as *as) 387 { 388 struct seg *seg, *seglast, *p, *n; 389 uint_t nsegs = 0; 390 391 if (do_as_verify == 0) 392 return; 393 394 seglast = as->a_seglast; 395 396 for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 397 ASSERT(seg->s_as == as); 398 p = AS_SEGPREV(as, seg); 399 n = AS_SEGNEXT(as, seg); 400 ASSERT(p == NULL || p->s_as == as); 401 ASSERT(p == NULL || p->s_base < seg->s_base); 402 ASSERT(n == NULL || n->s_base > seg->s_base); 403 ASSERT(n != NULL || seg == avl_last(&as->a_segtree)); 404 if (seg == seglast) 405 seglast = NULL; 406 nsegs++; 407 } 408 ASSERT(seglast == NULL); 409 ASSERT(avl_numnodes(&as->a_segtree) == nsegs); 410 } 411 #endif /* VERIFY_SEGLIST */ 412 413 /* 414 * Add a new segment to the address space. The avl_find() 415 * may be expensive so we attempt to use last segment accessed 416 * in as_gap() as an insertion point. 417 */ 418 int 419 as_addseg(struct as *as, struct seg *newseg) 420 { 421 struct seg *seg; 422 caddr_t addr; 423 caddr_t eaddr; 424 avl_index_t where; 425 426 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 427 428 as->a_updatedir = 1; /* inform /proc */ 429 gethrestime(&as->a_updatetime); 430 431 if (as->a_lastgaphl != NULL) { 432 struct seg *hseg = NULL; 433 struct seg *lseg = NULL; 434 435 if (as->a_lastgaphl->s_base > newseg->s_base) { 436 hseg = as->a_lastgaphl; 437 lseg = AVL_PREV(&as->a_segtree, hseg); 438 } else { 439 lseg = as->a_lastgaphl; 440 hseg = AVL_NEXT(&as->a_segtree, lseg); 441 } 442 443 if (hseg && lseg && lseg->s_base < newseg->s_base && 444 hseg->s_base > newseg->s_base) { 445 avl_insert_here(&as->a_segtree, newseg, lseg, 446 AVL_AFTER); 447 as->a_lastgaphl = NULL; 448 as->a_seglast = newseg; 449 return (0); 450 } 451 as->a_lastgaphl = NULL; 452 } 453 454 addr = newseg->s_base; 455 eaddr = addr + newseg->s_size; 456 again: 457 458 seg = avl_find(&as->a_segtree, &addr, &where); 459 460 if (seg == NULL) 461 seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 462 463 if (seg == NULL) 464 seg = avl_last(&as->a_segtree); 465 466 if (seg != NULL) { 467 caddr_t base = seg->s_base; 468 469 /* 470 * If top of seg is below the requested address, then 471 * the insertion point is at the end of the linked list, 472 * and seg points to the tail of the list. Otherwise, 473 * the insertion point is immediately before seg. 474 */ 475 if (base + seg->s_size > addr) { 476 if (addr >= base || eaddr > base) { 477 #ifdef __sparc 478 extern struct seg_ops segnf_ops; 479 480 /* 481 * no-fault segs must disappear if overlaid. 482 * XXX need new segment type so 483 * we don't have to check s_ops 484 */ 485 if (seg->s_ops == &segnf_ops) { 486 seg_unmap(seg); 487 goto again; 488 } 489 #endif 490 return (-1); /* overlapping segment */ 491 } 492 } 493 } 494 as->a_seglast = newseg; 495 avl_insert(&as->a_segtree, newseg, where); 496 497 #ifdef VERIFY_SEGLIST 498 as_verify(as); 499 #endif 500 return (0); 501 } 502 503 struct seg * 504 as_removeseg(struct as *as, struct seg *seg) 505 { 506 avl_tree_t *t; 507 508 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 509 510 as->a_updatedir = 1; /* inform /proc */ 511 gethrestime(&as->a_updatetime); 512 513 if (seg == NULL) 514 return (NULL); 515 516 t = &as->a_segtree; 517 if (as->a_seglast == seg) 518 as->a_seglast = NULL; 519 as->a_lastgaphl = NULL; 520 521 /* 522 * if this segment is at an address higher than 523 * a_lastgap, set a_lastgap to the next segment (NULL if last segment) 524 */ 525 if (as->a_lastgap && 526 (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base)) 527 as->a_lastgap = AVL_NEXT(t, seg); 528 529 /* 530 * remove the segment from the seg tree 531 */ 532 avl_remove(t, seg); 533 534 #ifdef VERIFY_SEGLIST 535 as_verify(as); 536 #endif 537 return (seg); 538 } 539 540 /* 541 * Find a segment containing addr. 542 */ 543 struct seg * 544 as_segat(struct as *as, caddr_t addr) 545 { 546 struct seg *seg = as->a_seglast; 547 548 ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 549 550 if (seg != NULL && seg->s_base <= addr && 551 addr < seg->s_base + seg->s_size) 552 return (seg); 553 554 seg = avl_find(&as->a_segtree, &addr, NULL); 555 return (seg); 556 } 557 558 /* 559 * Serialize all searches for holes in an address space to 560 * prevent two or more threads from allocating the same virtual 561 * address range. The address space must not be "read/write" 562 * locked by the caller since we may block. 563 */ 564 void 565 as_rangelock(struct as *as) 566 { 567 mutex_enter(&as->a_contents); 568 while (AS_ISCLAIMGAP(as)) 569 cv_wait(&as->a_cv, &as->a_contents); 570 AS_SETCLAIMGAP(as); 571 mutex_exit(&as->a_contents); 572 } 573 574 /* 575 * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads. 576 */ 577 void 578 as_rangeunlock(struct as *as) 579 { 580 mutex_enter(&as->a_contents); 581 AS_CLRCLAIMGAP(as); 582 cv_signal(&as->a_cv); 583 mutex_exit(&as->a_contents); 584 } 585 586 /* 587 * compar segments (or just an address) by segment address range 588 */ 589 static int 590 as_segcompar(const void *x, const void *y) 591 { 592 struct seg *a = (struct seg *)x; 593 struct seg *b = (struct seg *)y; 594 595 if (a->s_base < b->s_base) 596 return (-1); 597 if (a->s_base >= b->s_base + b->s_size) 598 return (1); 599 return (0); 600 } 601 602 603 void 604 as_avlinit(struct as *as) 605 { 606 avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg), 607 offsetof(struct seg, s_tree)); 608 avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page), 609 offsetof(struct watched_page, wp_link)); 610 } 611 612 /*ARGSUSED*/ 613 static int 614 as_constructor(void *buf, void *cdrarg, int kmflags) 615 { 616 struct as *as = buf; 617 618 mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL); 619 cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL); 620 rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL); 621 as_avlinit(as); 622 return (0); 623 } 624 625 /*ARGSUSED1*/ 626 static void 627 as_destructor(void *buf, void *cdrarg) 628 { 629 struct as *as = buf; 630 631 avl_destroy(&as->a_segtree); 632 mutex_destroy(&as->a_contents); 633 cv_destroy(&as->a_cv); 634 rw_destroy(&as->a_lock); 635 } 636 637 void 638 as_init(void) 639 { 640 as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0, 641 as_constructor, as_destructor, NULL, NULL, NULL, 0); 642 } 643 644 /* 645 * Allocate and initialize an address space data structure. 646 * We call hat_alloc to allow any machine dependent 647 * information in the hat structure to be initialized. 648 */ 649 struct as * 650 as_alloc(void) 651 { 652 struct as *as; 653 654 as = kmem_cache_alloc(as_cache, KM_SLEEP); 655 656 as->a_flags = 0; 657 as->a_vbits = 0; 658 as->a_hrm = NULL; 659 as->a_seglast = NULL; 660 as->a_size = 0; 661 as->a_updatedir = 0; 662 gethrestime(&as->a_updatetime); 663 as->a_objectdir = NULL; 664 as->a_sizedir = 0; 665 as->a_userlimit = (caddr_t)USERLIMIT; 666 as->a_lastgap = NULL; 667 as->a_lastgaphl = NULL; 668 as->a_callbacks = NULL; 669 670 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 671 as->a_hat = hat_alloc(as); /* create hat for default system mmu */ 672 AS_LOCK_EXIT(as, &as->a_lock); 673 674 as->a_xhat = NULL; 675 676 return (as); 677 } 678 679 /* 680 * Free an address space data structure. 681 * Need to free the hat first and then 682 * all the segments on this as and finally 683 * the space for the as struct itself. 684 */ 685 void 686 as_free(struct as *as) 687 { 688 struct hat *hat = as->a_hat; 689 struct seg *seg, *next; 690 int called = 0; 691 692 top: 693 /* 694 * Invoke ALL callbacks. as_do_callbacks will do one callback 695 * per call, and not return (-1) until the callback has completed. 696 * When as_do_callbacks returns zero, all callbacks have completed. 697 */ 698 mutex_enter(&as->a_contents); 699 while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0)); 700 701 /* This will prevent new XHATs from attaching to as */ 702 if (!called) 703 AS_SETBUSY(as); 704 mutex_exit(&as->a_contents); 705 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 706 707 if (!called) { 708 called = 1; 709 hat_free_start(hat); 710 if (as->a_xhat != NULL) 711 xhat_free_start_all(as); 712 } 713 for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) { 714 int err; 715 716 next = AS_SEGNEXT(as, seg); 717 err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 718 if (err == EAGAIN) { 719 mutex_enter(&as->a_contents); 720 if (as->a_callbacks) { 721 AS_LOCK_EXIT(as, &as->a_lock); 722 } else { 723 /* 724 * Memory is currently locked. Wait for a 725 * cv_signal that it has been unlocked, then 726 * try the operation again. 727 */ 728 if (AS_ISUNMAPWAIT(as) == 0) 729 cv_broadcast(&as->a_cv); 730 AS_SETUNMAPWAIT(as); 731 AS_LOCK_EXIT(as, &as->a_lock); 732 while (AS_ISUNMAPWAIT(as)) 733 cv_wait(&as->a_cv, &as->a_contents); 734 } 735 mutex_exit(&as->a_contents); 736 goto top; 737 } else { 738 /* 739 * We do not expect any other error return at this 740 * time. This is similar to an ASSERT in seg_unmap() 741 */ 742 ASSERT(err == 0); 743 } 744 } 745 hat_free_end(hat); 746 if (as->a_xhat != NULL) 747 xhat_free_end_all(as); 748 AS_LOCK_EXIT(as, &as->a_lock); 749 750 /* /proc stuff */ 751 ASSERT(avl_numnodes(&as->a_wpage) == 0); 752 if (as->a_objectdir) { 753 kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *)); 754 as->a_objectdir = NULL; 755 as->a_sizedir = 0; 756 } 757 758 /* 759 * Free the struct as back to kmem. Assert it has no segments. 760 */ 761 ASSERT(avl_numnodes(&as->a_segtree) == 0); 762 kmem_cache_free(as_cache, as); 763 } 764 765 int 766 as_dup(struct as *as, struct as **outas) 767 { 768 struct as *newas; 769 struct seg *seg, *newseg; 770 int error; 771 772 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 773 as_clearwatch(as); 774 newas = as_alloc(); 775 newas->a_userlimit = as->a_userlimit; 776 AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER); 777 778 /* This will prevent new XHATs from attaching */ 779 mutex_enter(&as->a_contents); 780 AS_SETBUSY(as); 781 mutex_exit(&as->a_contents); 782 mutex_enter(&newas->a_contents); 783 AS_SETBUSY(newas); 784 mutex_exit(&newas->a_contents); 785 786 787 for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 788 789 if (seg->s_flags & S_PURGE) 790 continue; 791 792 newseg = seg_alloc(newas, seg->s_base, seg->s_size); 793 if (newseg == NULL) { 794 AS_LOCK_EXIT(newas, &newas->a_lock); 795 as_setwatch(as); 796 mutex_enter(&as->a_contents); 797 AS_CLRBUSY(as); 798 mutex_exit(&as->a_contents); 799 AS_LOCK_EXIT(as, &as->a_lock); 800 as_free(newas); 801 return (-1); 802 } 803 if ((error = SEGOP_DUP(seg, newseg)) != 0) { 804 /* 805 * We call seg_free() on the new seg 806 * because the segment is not set up 807 * completely; i.e. it has no ops. 808 */ 809 as_setwatch(as); 810 mutex_enter(&as->a_contents); 811 AS_CLRBUSY(as); 812 mutex_exit(&as->a_contents); 813 AS_LOCK_EXIT(as, &as->a_lock); 814 seg_free(newseg); 815 AS_LOCK_EXIT(newas, &newas->a_lock); 816 as_free(newas); 817 return (error); 818 } 819 newas->a_size += seg->s_size; 820 } 821 822 error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL); 823 if (as->a_xhat != NULL) 824 error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL); 825 826 mutex_enter(&newas->a_contents); 827 AS_CLRBUSY(newas); 828 mutex_exit(&newas->a_contents); 829 AS_LOCK_EXIT(newas, &newas->a_lock); 830 831 as_setwatch(as); 832 mutex_enter(&as->a_contents); 833 AS_CLRBUSY(as); 834 mutex_exit(&as->a_contents); 835 AS_LOCK_EXIT(as, &as->a_lock); 836 if (error != 0) { 837 as_free(newas); 838 return (error); 839 } 840 *outas = newas; 841 return (0); 842 } 843 844 /* 845 * Handle a ``fault'' at addr for size bytes. 846 */ 847 faultcode_t 848 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size, 849 enum fault_type type, enum seg_rw rw) 850 { 851 struct seg *seg; 852 caddr_t raddr; /* rounded down addr */ 853 size_t rsize; /* rounded up size */ 854 size_t ssize; 855 faultcode_t res = 0; 856 caddr_t addrsav; 857 struct seg *segsav; 858 int as_lock_held; 859 klwp_t *lwp = ttolwp(curthread); 860 int is_xhat = 0; 861 int holding_wpage = 0; 862 extern struct seg_ops segdev_ops; 863 864 865 866 if (as->a_hat != hat) { 867 /* This must be an XHAT then */ 868 is_xhat = 1; 869 870 if ((type != F_INVAL) || (as == &kas)) 871 return (FC_NOSUPPORT); 872 } 873 874 retry: 875 if (!is_xhat) { 876 /* 877 * Indicate that the lwp is not to be stopped while waiting 878 * for a pagefault. This is to avoid deadlock while debugging 879 * a process via /proc over NFS (in particular). 880 */ 881 if (lwp != NULL) { 882 lwp->lwp_nostop++; 883 lwp->lwp_nostop_r++; 884 } 885 886 /* 887 * same length must be used when we softlock and softunlock. 888 * We don't support softunlocking lengths less than 889 * the original length when there is largepage support. 890 * See seg_dev.c for more comments. 891 */ 892 switch (type) { 893 894 case F_SOFTLOCK: 895 CPU_STATS_ADD_K(vm, softlock, 1); 896 break; 897 898 case F_SOFTUNLOCK: 899 break; 900 901 case F_PROT: 902 CPU_STATS_ADD_K(vm, prot_fault, 1); 903 break; 904 905 case F_INVAL: 906 CPU_STATS_ENTER_K(); 907 CPU_STATS_ADDQ(CPU, vm, as_fault, 1); 908 if (as == &kas) 909 CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1); 910 CPU_STATS_EXIT_K(); 911 break; 912 } 913 } 914 915 /* Kernel probe */ 916 TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */, 917 tnf_opaque, address, addr, 918 tnf_fault_type, fault_type, type, 919 tnf_seg_access, access, rw); 920 921 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 922 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 923 (size_t)raddr; 924 925 /* 926 * XXX -- Don't grab the as lock for segkmap. We should grab it for 927 * correctness, but then we could be stuck holding this lock for 928 * a LONG time if the fault needs to be resolved on a slow 929 * filesystem, and then no-one will be able to exec new commands, 930 * as exec'ing requires the write lock on the as. 931 */ 932 if (as == &kas && segkmap && segkmap->s_base <= raddr && 933 raddr + size < segkmap->s_base + segkmap->s_size) { 934 /* 935 * if (as==&kas), this can't be XHAT: we've already returned 936 * FC_NOSUPPORT. 937 */ 938 seg = segkmap; 939 as_lock_held = 0; 940 } else { 941 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 942 if (is_xhat && avl_numnodes(&as->a_wpage) != 0) { 943 /* 944 * Grab and hold the writers' lock on the as 945 * if the fault is to a watched page. 946 * This will keep CPUs from "peeking" at the 947 * address range while we're temporarily boosting 948 * the permissions for the XHAT device to 949 * resolve the fault in the segment layer. 950 * 951 * We could check whether faulted address 952 * is within a watched page and only then grab 953 * the writer lock, but this is simpler. 954 */ 955 AS_LOCK_EXIT(as, &as->a_lock); 956 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 957 } 958 959 seg = as_segat(as, raddr); 960 if (seg == NULL) { 961 AS_LOCK_EXIT(as, &as->a_lock); 962 if ((lwp != NULL) && (!is_xhat)) { 963 lwp->lwp_nostop--; 964 lwp->lwp_nostop_r--; 965 } 966 return (FC_NOMAP); 967 } 968 969 as_lock_held = 1; 970 } 971 972 addrsav = raddr; 973 segsav = seg; 974 975 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 976 if (raddr >= seg->s_base + seg->s_size) { 977 seg = AS_SEGNEXT(as, seg); 978 if (seg == NULL || raddr != seg->s_base) { 979 res = FC_NOMAP; 980 break; 981 } 982 } 983 if (raddr + rsize > seg->s_base + seg->s_size) 984 ssize = seg->s_base + seg->s_size - raddr; 985 else 986 ssize = rsize; 987 988 if (!is_xhat || (seg->s_ops != &segdev_ops)) { 989 990 if (is_xhat && avl_numnodes(&as->a_wpage) != 0 && 991 pr_is_watchpage_as(raddr, rw, as)) { 992 /* 993 * Handle watch pages. If we're faulting on a 994 * watched page from an X-hat, we have to 995 * restore the original permissions while we 996 * handle the fault. 997 */ 998 as_clearwatch(as); 999 holding_wpage = 1; 1000 } 1001 1002 res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw); 1003 1004 /* Restore watchpoints */ 1005 if (holding_wpage) { 1006 as_setwatch(as); 1007 holding_wpage = 0; 1008 } 1009 1010 if (res != 0) 1011 break; 1012 } else { 1013 /* XHAT does not support seg_dev */ 1014 res = FC_NOSUPPORT; 1015 break; 1016 } 1017 } 1018 1019 /* 1020 * If we were SOFTLOCKing and encountered a failure, 1021 * we must SOFTUNLOCK the range we already did. (Maybe we 1022 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing 1023 * right here...) 1024 */ 1025 if (res != 0 && type == F_SOFTLOCK) { 1026 for (seg = segsav; addrsav < raddr; addrsav += ssize) { 1027 if (addrsav >= seg->s_base + seg->s_size) 1028 seg = AS_SEGNEXT(as, seg); 1029 ASSERT(seg != NULL); 1030 /* 1031 * Now call the fault routine again to perform the 1032 * unlock using S_OTHER instead of the rw variable 1033 * since we never got a chance to touch the pages. 1034 */ 1035 if (raddr > seg->s_base + seg->s_size) 1036 ssize = seg->s_base + seg->s_size - addrsav; 1037 else 1038 ssize = raddr - addrsav; 1039 (void) SEGOP_FAULT(hat, seg, addrsav, ssize, 1040 F_SOFTUNLOCK, S_OTHER); 1041 } 1042 } 1043 if (as_lock_held) 1044 AS_LOCK_EXIT(as, &as->a_lock); 1045 if ((lwp != NULL) && (!is_xhat)) { 1046 lwp->lwp_nostop--; 1047 lwp->lwp_nostop_r--; 1048 } 1049 /* 1050 * If the lower levels returned EDEADLK for a fault, 1051 * It means that we should retry the fault. Let's wait 1052 * a bit also to let the deadlock causing condition clear. 1053 * This is part of a gross hack to work around a design flaw 1054 * in the ufs/sds logging code and should go away when the 1055 * logging code is re-designed to fix the problem. See bug 1056 * 4125102 for details of the problem. 1057 */ 1058 if (FC_ERRNO(res) == EDEADLK) { 1059 delay(deadlk_wait); 1060 res = 0; 1061 goto retry; 1062 } 1063 return (res); 1064 } 1065 1066 1067 1068 /* 1069 * Asynchronous ``fault'' at addr for size bytes. 1070 */ 1071 faultcode_t 1072 as_faulta(struct as *as, caddr_t addr, size_t size) 1073 { 1074 struct seg *seg; 1075 caddr_t raddr; /* rounded down addr */ 1076 size_t rsize; /* rounded up size */ 1077 faultcode_t res = 0; 1078 klwp_t *lwp = ttolwp(curthread); 1079 1080 retry: 1081 /* 1082 * Indicate that the lwp is not to be stopped while waiting 1083 * for a pagefault. This is to avoid deadlock while debugging 1084 * a process via /proc over NFS (in particular). 1085 */ 1086 if (lwp != NULL) { 1087 lwp->lwp_nostop++; 1088 lwp->lwp_nostop_r++; 1089 } 1090 1091 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 1092 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1093 (size_t)raddr; 1094 1095 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1096 seg = as_segat(as, raddr); 1097 if (seg == NULL) { 1098 AS_LOCK_EXIT(as, &as->a_lock); 1099 if (lwp != NULL) { 1100 lwp->lwp_nostop--; 1101 lwp->lwp_nostop_r--; 1102 } 1103 return (FC_NOMAP); 1104 } 1105 1106 for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) { 1107 if (raddr >= seg->s_base + seg->s_size) { 1108 seg = AS_SEGNEXT(as, seg); 1109 if (seg == NULL || raddr != seg->s_base) { 1110 res = FC_NOMAP; 1111 break; 1112 } 1113 } 1114 res = SEGOP_FAULTA(seg, raddr); 1115 if (res != 0) 1116 break; 1117 } 1118 AS_LOCK_EXIT(as, &as->a_lock); 1119 if (lwp != NULL) { 1120 lwp->lwp_nostop--; 1121 lwp->lwp_nostop_r--; 1122 } 1123 /* 1124 * If the lower levels returned EDEADLK for a fault, 1125 * It means that we should retry the fault. Let's wait 1126 * a bit also to let the deadlock causing condition clear. 1127 * This is part of a gross hack to work around a design flaw 1128 * in the ufs/sds logging code and should go away when the 1129 * logging code is re-designed to fix the problem. See bug 1130 * 4125102 for details of the problem. 1131 */ 1132 if (FC_ERRNO(res) == EDEADLK) { 1133 delay(deadlk_wait); 1134 res = 0; 1135 goto retry; 1136 } 1137 return (res); 1138 } 1139 1140 /* 1141 * Set the virtual mapping for the interval from [addr : addr + size) 1142 * in address space `as' to have the specified protection. 1143 * It is ok for the range to cross over several segments, 1144 * as long as they are contiguous. 1145 */ 1146 int 1147 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 1148 { 1149 struct seg *seg; 1150 struct as_callback *cb; 1151 size_t ssize; 1152 caddr_t raddr; /* rounded down addr */ 1153 size_t rsize; /* rounded up size */ 1154 int error = 0, writer = 0; 1155 caddr_t saveraddr; 1156 size_t saversize; 1157 1158 setprot_top: 1159 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 1160 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1161 (size_t)raddr; 1162 1163 if (raddr + rsize < raddr) /* check for wraparound */ 1164 return (ENOMEM); 1165 1166 saveraddr = raddr; 1167 saversize = rsize; 1168 1169 /* 1170 * Normally we only lock the as as a reader. But 1171 * if due to setprot the segment driver needs to split 1172 * a segment it will return IE_RETRY. Therefore we re-aquire 1173 * the as lock as a writer so the segment driver can change 1174 * the seg list. Also the segment driver will return IE_RETRY 1175 * after it has changed the segment list so we therefore keep 1176 * locking as a writer. Since these opeartions should be rare 1177 * want to only lock as a writer when necessary. 1178 */ 1179 if (writer || avl_numnodes(&as->a_wpage) != 0) { 1180 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1181 } else { 1182 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1183 } 1184 1185 as_clearwatchprot(as, raddr, rsize); 1186 seg = as_segat(as, raddr); 1187 if (seg == NULL) { 1188 as_setwatch(as); 1189 AS_LOCK_EXIT(as, &as->a_lock); 1190 return (ENOMEM); 1191 } 1192 1193 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 1194 if (raddr >= seg->s_base + seg->s_size) { 1195 seg = AS_SEGNEXT(as, seg); 1196 if (seg == NULL || raddr != seg->s_base) { 1197 error = ENOMEM; 1198 break; 1199 } 1200 } 1201 if ((raddr + rsize) > (seg->s_base + seg->s_size)) 1202 ssize = seg->s_base + seg->s_size - raddr; 1203 else 1204 ssize = rsize; 1205 error = SEGOP_SETPROT(seg, raddr, ssize, prot); 1206 1207 if (error == IE_NOMEM) { 1208 error = EAGAIN; 1209 break; 1210 } 1211 1212 if (error == IE_RETRY) { 1213 AS_LOCK_EXIT(as, &as->a_lock); 1214 writer = 1; 1215 goto setprot_top; 1216 } 1217 1218 if (error == EAGAIN) { 1219 /* 1220 * Make sure we have a_lock as writer. 1221 */ 1222 if (writer == 0) { 1223 AS_LOCK_EXIT(as, &as->a_lock); 1224 writer = 1; 1225 goto setprot_top; 1226 } 1227 1228 /* 1229 * Memory is currently locked. It must be unlocked 1230 * before this operation can succeed through a retry. 1231 * The possible reasons for locked memory and 1232 * corresponding strategies for unlocking are: 1233 * (1) Normal I/O 1234 * wait for a signal that the I/O operation 1235 * has completed and the memory is unlocked. 1236 * (2) Asynchronous I/O 1237 * The aio subsystem does not unlock pages when 1238 * the I/O is completed. Those pages are unlocked 1239 * when the application calls aiowait/aioerror. 1240 * So, to prevent blocking forever, cv_broadcast() 1241 * is done to wake up aio_cleanup_thread. 1242 * Subsequently, segvn_reclaim will be called, and 1243 * that will do AS_CLRUNMAPWAIT() and wake us up. 1244 * (3) Long term page locking: 1245 * Drivers intending to have pages locked for a 1246 * period considerably longer than for normal I/O 1247 * (essentially forever) may have registered for a 1248 * callback so they may unlock these pages on 1249 * request. This is needed to allow this operation 1250 * to succeed. Each entry on the callback list is 1251 * examined. If the event or address range pertains 1252 * the callback is invoked (unless it already is in 1253 * progress). The a_contents lock must be dropped 1254 * before the callback, so only one callback can 1255 * be done at a time. Go to the top and do more 1256 * until zero is returned. If zero is returned, 1257 * either there were no callbacks for this event 1258 * or they were already in progress. 1259 */ 1260 mutex_enter(&as->a_contents); 1261 if (as->a_callbacks && 1262 (cb = as_find_callback(as, AS_SETPROT_EVENT, 1263 seg->s_base, seg->s_size))) { 1264 AS_LOCK_EXIT(as, &as->a_lock); 1265 as_execute_callback(as, cb, AS_SETPROT_EVENT); 1266 } else { 1267 if (AS_ISUNMAPWAIT(as) == 0) 1268 cv_broadcast(&as->a_cv); 1269 AS_SETUNMAPWAIT(as); 1270 AS_LOCK_EXIT(as, &as->a_lock); 1271 while (AS_ISUNMAPWAIT(as)) 1272 cv_wait(&as->a_cv, &as->a_contents); 1273 } 1274 mutex_exit(&as->a_contents); 1275 goto setprot_top; 1276 } else if (error != 0) 1277 break; 1278 } 1279 if (error != 0) { 1280 as_setwatch(as); 1281 } else { 1282 as_setwatchprot(as, saveraddr, saversize, prot); 1283 } 1284 AS_LOCK_EXIT(as, &as->a_lock); 1285 return (error); 1286 } 1287 1288 /* 1289 * Check to make sure that the interval [addr, addr + size) 1290 * in address space `as' has at least the specified protection. 1291 * It is ok for the range to cross over several segments, as long 1292 * as they are contiguous. 1293 */ 1294 int 1295 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 1296 { 1297 struct seg *seg; 1298 size_t ssize; 1299 caddr_t raddr; /* rounded down addr */ 1300 size_t rsize; /* rounded up size */ 1301 int error = 0; 1302 1303 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 1304 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1305 (size_t)raddr; 1306 1307 if (raddr + rsize < raddr) /* check for wraparound */ 1308 return (ENOMEM); 1309 1310 /* 1311 * This is ugly as sin... 1312 * Normally, we only acquire the address space readers lock. 1313 * However, if the address space has watchpoints present, 1314 * we must acquire the writer lock on the address space for 1315 * the benefit of as_clearwatchprot() and as_setwatchprot(). 1316 */ 1317 if (avl_numnodes(&as->a_wpage) != 0) 1318 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1319 else 1320 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1321 as_clearwatchprot(as, raddr, rsize); 1322 seg = as_segat(as, raddr); 1323 if (seg == NULL) { 1324 as_setwatch(as); 1325 AS_LOCK_EXIT(as, &as->a_lock); 1326 return (ENOMEM); 1327 } 1328 1329 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 1330 if (raddr >= seg->s_base + seg->s_size) { 1331 seg = AS_SEGNEXT(as, seg); 1332 if (seg == NULL || raddr != seg->s_base) { 1333 error = ENOMEM; 1334 break; 1335 } 1336 } 1337 if ((raddr + rsize) > (seg->s_base + seg->s_size)) 1338 ssize = seg->s_base + seg->s_size - raddr; 1339 else 1340 ssize = rsize; 1341 1342 error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); 1343 if (error != 0) 1344 break; 1345 } 1346 as_setwatch(as); 1347 AS_LOCK_EXIT(as, &as->a_lock); 1348 return (error); 1349 } 1350 1351 int 1352 as_unmap(struct as *as, caddr_t addr, size_t size) 1353 { 1354 struct seg *seg, *seg_next; 1355 struct as_callback *cb; 1356 caddr_t raddr, eaddr; 1357 size_t ssize; 1358 int err; 1359 1360 top: 1361 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 1362 eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & 1363 (uintptr_t)PAGEMASK); 1364 1365 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1366 1367 as->a_updatedir = 1; /* inform /proc */ 1368 gethrestime(&as->a_updatetime); 1369 1370 /* 1371 * Use as_findseg to find the first segment in the range, then 1372 * step through the segments in order, following s_next. 1373 */ 1374 as_clearwatchprot(as, raddr, eaddr - raddr); 1375 1376 for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) { 1377 if (eaddr <= seg->s_base) 1378 break; /* eaddr was in a gap; all done */ 1379 1380 /* this is implied by the test above */ 1381 ASSERT(raddr < eaddr); 1382 1383 if (raddr < seg->s_base) 1384 raddr = seg->s_base; /* raddr was in a gap */ 1385 1386 if (eaddr > (seg->s_base + seg->s_size)) 1387 ssize = seg->s_base + seg->s_size - raddr; 1388 else 1389 ssize = eaddr - raddr; 1390 1391 /* 1392 * Save next segment pointer since seg can be 1393 * destroyed during the segment unmap operation. 1394 */ 1395 seg_next = AS_SEGNEXT(as, seg); 1396 1397 err = SEGOP_UNMAP(seg, raddr, ssize); 1398 if (err == EAGAIN) { 1399 /* 1400 * Memory is currently locked. It must be unlocked 1401 * before this operation can succeed through a retry. 1402 * The possible reasons for locked memory and 1403 * corresponding strategies for unlocking are: 1404 * (1) Normal I/O 1405 * wait for a signal that the I/O operation 1406 * has completed and the memory is unlocked. 1407 * (2) Asynchronous I/O 1408 * The aio subsystem does not unlock pages when 1409 * the I/O is completed. Those pages are unlocked 1410 * when the application calls aiowait/aioerror. 1411 * So, to prevent blocking forever, cv_broadcast() 1412 * is done to wake up aio_cleanup_thread. 1413 * Subsequently, segvn_reclaim will be called, and 1414 * that will do AS_CLRUNMAPWAIT() and wake us up. 1415 * (3) Long term page locking: 1416 * Drivers intending to have pages locked for a 1417 * period considerably longer than for normal I/O 1418 * (essentially forever) may have registered for a 1419 * callback so they may unlock these pages on 1420 * request. This is needed to allow this operation 1421 * to succeed. Each entry on the callback list is 1422 * examined. If the event or address range pertains 1423 * the callback is invoked (unless it already is in 1424 * progress). The a_contents lock must be dropped 1425 * before the callback, so only one callback can 1426 * be done at a time. Go to the top and do more 1427 * until zero is returned. If zero is returned, 1428 * either there were no callbacks for this event 1429 * or they were already in progress. 1430 */ 1431 as_setwatch(as); 1432 mutex_enter(&as->a_contents); 1433 if (as->a_callbacks && 1434 (cb = as_find_callback(as, AS_UNMAP_EVENT, 1435 seg->s_base, seg->s_size))) { 1436 AS_LOCK_EXIT(as, &as->a_lock); 1437 as_execute_callback(as, cb, AS_UNMAP_EVENT); 1438 } else { 1439 if (AS_ISUNMAPWAIT(as) == 0) 1440 cv_broadcast(&as->a_cv); 1441 AS_SETUNMAPWAIT(as); 1442 AS_LOCK_EXIT(as, &as->a_lock); 1443 while (AS_ISUNMAPWAIT(as)) 1444 cv_wait(&as->a_cv, &as->a_contents); 1445 } 1446 mutex_exit(&as->a_contents); 1447 goto top; 1448 } else if (err == IE_RETRY) { 1449 as_setwatch(as); 1450 AS_LOCK_EXIT(as, &as->a_lock); 1451 goto top; 1452 } else if (err) { 1453 as_setwatch(as); 1454 AS_LOCK_EXIT(as, &as->a_lock); 1455 return (-1); 1456 } 1457 1458 as->a_size -= ssize; 1459 raddr += ssize; 1460 } 1461 AS_LOCK_EXIT(as, &as->a_lock); 1462 return (0); 1463 } 1464 1465 static int 1466 as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec, 1467 int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 1468 { 1469 uint_t szc; 1470 uint_t nszc; 1471 int error; 1472 caddr_t a; 1473 caddr_t eaddr; 1474 size_t segsize; 1475 struct seg *seg; 1476 size_t pgsz; 1477 int do_off = (vn_a->vp != NULL || vn_a->amp != NULL); 1478 uint_t save_szcvec; 1479 1480 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1481 ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 1482 ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 1483 ASSERT(vn_a->vp == NULL || vn_a->amp == NULL); 1484 if (!do_off) { 1485 vn_a->offset = 0; 1486 } 1487 1488 if (szcvec <= 1) { 1489 seg = seg_alloc(as, addr, size); 1490 if (seg == NULL) { 1491 return (ENOMEM); 1492 } 1493 vn_a->szc = 0; 1494 error = (*crfp)(seg, vn_a); 1495 if (error != 0) { 1496 seg_free(seg); 1497 } 1498 return (error); 1499 } 1500 1501 eaddr = addr + size; 1502 save_szcvec = szcvec; 1503 szcvec >>= 1; 1504 szc = 0; 1505 nszc = 0; 1506 while (szcvec) { 1507 if ((szcvec & 0x1) == 0) { 1508 nszc++; 1509 szcvec >>= 1; 1510 continue; 1511 } 1512 nszc++; 1513 pgsz = page_get_pagesize(nszc); 1514 a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 1515 if (a != addr) { 1516 ASSERT(a < eaddr); 1517 segsize = a - addr; 1518 seg = seg_alloc(as, addr, segsize); 1519 if (seg == NULL) { 1520 return (ENOMEM); 1521 } 1522 vn_a->szc = szc; 1523 error = (*crfp)(seg, vn_a); 1524 if (error != 0) { 1525 seg_free(seg); 1526 return (error); 1527 } 1528 *segcreated = 1; 1529 if (do_off) { 1530 vn_a->offset += segsize; 1531 } 1532 addr = a; 1533 } 1534 szc = nszc; 1535 szcvec >>= 1; 1536 } 1537 1538 ASSERT(addr < eaddr); 1539 szcvec = save_szcvec | 1; /* add 8K pages */ 1540 while (szcvec) { 1541 a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 1542 ASSERT(a >= addr); 1543 if (a != addr) { 1544 segsize = a - addr; 1545 seg = seg_alloc(as, addr, segsize); 1546 if (seg == NULL) { 1547 return (ENOMEM); 1548 } 1549 vn_a->szc = szc; 1550 error = (*crfp)(seg, vn_a); 1551 if (error != 0) { 1552 seg_free(seg); 1553 return (error); 1554 } 1555 *segcreated = 1; 1556 if (do_off) { 1557 vn_a->offset += segsize; 1558 } 1559 addr = a; 1560 } 1561 szcvec &= ~(1 << szc); 1562 if (szcvec) { 1563 szc = highbit(szcvec) - 1; 1564 pgsz = page_get_pagesize(szc); 1565 } 1566 } 1567 ASSERT(addr == eaddr); 1568 1569 return (0); 1570 } 1571 1572 static int 1573 as_map_vnsegs(struct as *as, caddr_t addr, size_t size, 1574 int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 1575 { 1576 uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA); 1577 int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 1578 uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 1579 type, 0); 1580 int error; 1581 struct seg *seg; 1582 struct vattr va; 1583 u_offset_t eoff; 1584 size_t save_size = 0; 1585 1586 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1587 ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 1588 ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 1589 ASSERT(vn_a->vp != NULL); 1590 ASSERT(vn_a->amp == NULL); 1591 1592 again: 1593 if (szcvec <= 1) { 1594 seg = seg_alloc(as, addr, size); 1595 if (seg == NULL) { 1596 return (ENOMEM); 1597 } 1598 vn_a->szc = 0; 1599 error = (*crfp)(seg, vn_a); 1600 if (error != 0) { 1601 seg_free(seg); 1602 } 1603 return (error); 1604 } 1605 1606 va.va_mask = AT_SIZE; 1607 if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred) != 0) { 1608 szcvec = 0; 1609 goto again; 1610 } 1611 eoff = vn_a->offset & PAGEMASK; 1612 if (eoff >= va.va_size) { 1613 szcvec = 0; 1614 goto again; 1615 } 1616 eoff += size; 1617 if (btopr(va.va_size) < btopr(eoff)) { 1618 save_size = size; 1619 size = va.va_size - (vn_a->offset & PAGEMASK); 1620 size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t); 1621 szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 1622 type, 0); 1623 if (szcvec <= 1) { 1624 size = save_size; 1625 goto again; 1626 } 1627 } 1628 1629 error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a, 1630 segcreated); 1631 if (error != 0) { 1632 return (error); 1633 } 1634 if (save_size) { 1635 addr += size; 1636 size = save_size - size; 1637 szcvec = 0; 1638 goto again; 1639 } 1640 return (0); 1641 } 1642 1643 /* 1644 * as_map_ansegs: shared or private anonymous memory. Note that the flags 1645 * passed to map_pgszvec cannot be MAP_INITDATA, for anon. 1646 */ 1647 static int 1648 as_map_ansegs(struct as *as, caddr_t addr, size_t size, 1649 int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 1650 { 1651 uint_t szcvec; 1652 uchar_t type; 1653 1654 ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE); 1655 if (vn_a->type == MAP_SHARED) { 1656 type = MAPPGSZC_SHM; 1657 } else if (vn_a->type == MAP_PRIVATE) { 1658 if (vn_a->szc == AS_MAP_HEAP) { 1659 type = MAPPGSZC_HEAP; 1660 } else if (vn_a->szc == AS_MAP_STACK) { 1661 type = MAPPGSZC_STACK; 1662 } else { 1663 type = MAPPGSZC_PRIVM; 1664 } 1665 } 1666 szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ? 1667 (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE), 1668 (vn_a->flags & MAP_TEXT), type, 0); 1669 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 1670 ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 1671 ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 1672 ASSERT(vn_a->vp == NULL); 1673 1674 return (as_map_segvn_segs(as, addr, size, szcvec, 1675 crfp, vn_a, segcreated)); 1676 } 1677 1678 int 1679 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp) 1680 { 1681 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1682 return (as_map_locked(as, addr, size, crfp, argsp)); 1683 } 1684 1685 int 1686 as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(), 1687 void *argsp) 1688 { 1689 struct seg *seg = NULL; 1690 caddr_t raddr; /* rounded down addr */ 1691 size_t rsize; /* rounded up size */ 1692 int error; 1693 int unmap = 0; 1694 struct proc *p = curproc; 1695 struct segvn_crargs crargs; 1696 1697 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 1698 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1699 (size_t)raddr; 1700 1701 /* 1702 * check for wrap around 1703 */ 1704 if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { 1705 AS_LOCK_EXIT(as, &as->a_lock); 1706 return (ENOMEM); 1707 } 1708 1709 as->a_updatedir = 1; /* inform /proc */ 1710 gethrestime(&as->a_updatetime); 1711 1712 if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) { 1713 AS_LOCK_EXIT(as, &as->a_lock); 1714 1715 (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p, 1716 RCA_UNSAFE_ALL); 1717 1718 return (ENOMEM); 1719 } 1720 1721 if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) { 1722 crargs = *(struct segvn_crargs *)argsp; 1723 error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap); 1724 if (error != 0) { 1725 AS_LOCK_EXIT(as, &as->a_lock); 1726 if (unmap) { 1727 (void) as_unmap(as, addr, size); 1728 } 1729 return (error); 1730 } 1731 } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) { 1732 crargs = *(struct segvn_crargs *)argsp; 1733 error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap); 1734 if (error != 0) { 1735 AS_LOCK_EXIT(as, &as->a_lock); 1736 if (unmap) { 1737 (void) as_unmap(as, addr, size); 1738 } 1739 return (error); 1740 } 1741 } else { 1742 seg = seg_alloc(as, addr, size); 1743 if (seg == NULL) { 1744 AS_LOCK_EXIT(as, &as->a_lock); 1745 return (ENOMEM); 1746 } 1747 1748 error = (*crfp)(seg, argsp); 1749 if (error != 0) { 1750 seg_free(seg); 1751 AS_LOCK_EXIT(as, &as->a_lock); 1752 return (error); 1753 } 1754 } 1755 1756 /* 1757 * Add size now so as_unmap will work if as_ctl fails. 1758 */ 1759 as->a_size += rsize; 1760 1761 as_setwatch(as); 1762 1763 /* 1764 * If the address space is locked, 1765 * establish memory locks for the new segment. 1766 */ 1767 mutex_enter(&as->a_contents); 1768 if (AS_ISPGLCK(as)) { 1769 mutex_exit(&as->a_contents); 1770 AS_LOCK_EXIT(as, &as->a_lock); 1771 error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); 1772 if (error != 0) 1773 (void) as_unmap(as, addr, size); 1774 } else { 1775 mutex_exit(&as->a_contents); 1776 AS_LOCK_EXIT(as, &as->a_lock); 1777 } 1778 return (error); 1779 } 1780 1781 1782 /* 1783 * Delete all segments in the address space marked with S_PURGE. 1784 * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c). 1785 * These segments are deleted as a first step before calls to as_gap(), so 1786 * that they don't affect mmap() or shmat(). 1787 */ 1788 void 1789 as_purge(struct as *as) 1790 { 1791 struct seg *seg; 1792 struct seg *next_seg; 1793 1794 /* 1795 * the setting of NEEDSPURGE is protect by as_rangelock(), so 1796 * no need to grab a_contents mutex for this check 1797 */ 1798 if ((as->a_flags & AS_NEEDSPURGE) == 0) 1799 return; 1800 1801 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1802 next_seg = NULL; 1803 seg = AS_SEGFIRST(as); 1804 while (seg != NULL) { 1805 next_seg = AS_SEGNEXT(as, seg); 1806 if (seg->s_flags & S_PURGE) 1807 SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 1808 seg = next_seg; 1809 } 1810 AS_LOCK_EXIT(as, &as->a_lock); 1811 1812 mutex_enter(&as->a_contents); 1813 as->a_flags &= ~AS_NEEDSPURGE; 1814 mutex_exit(&as->a_contents); 1815 } 1816 1817 /* 1818 * Find a hole of at least size minlen within [base, base + len). 1819 * 1820 * If flags specifies AH_HI, the hole will have the highest possible address 1821 * in the range. We use the as->a_lastgap field to figure out where to 1822 * start looking for a gap. 1823 * 1824 * Otherwise, the gap will have the lowest possible address. 1825 * 1826 * If flags specifies AH_CONTAIN, the hole will contain the address addr. 1827 * 1828 * If an adequate hole is found, base and len are set to reflect the part of 1829 * the hole that is within range, and 0 is returned, otherwise, 1830 * -1 is returned. 1831 * 1832 * NOTE: This routine is not correct when base+len overflows caddr_t. 1833 */ 1834 int 1835 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags, 1836 caddr_t addr) 1837 { 1838 caddr_t lobound = *basep; 1839 caddr_t hibound = lobound + *lenp; 1840 struct seg *lseg, *hseg; 1841 caddr_t lo, hi; 1842 int forward; 1843 caddr_t save_base; 1844 size_t save_len; 1845 1846 save_base = *basep; 1847 save_len = *lenp; 1848 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1849 if (AS_SEGFIRST(as) == NULL) { 1850 if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) { 1851 AS_LOCK_EXIT(as, &as->a_lock); 1852 return (0); 1853 } else { 1854 AS_LOCK_EXIT(as, &as->a_lock); 1855 *basep = save_base; 1856 *lenp = save_len; 1857 return (-1); 1858 } 1859 } 1860 1861 /* 1862 * Set up to iterate over all the inter-segment holes in the given 1863 * direction. lseg is NULL for the lowest-addressed hole and hseg is 1864 * NULL for the highest-addressed hole. If moving backwards, we reset 1865 * sseg to denote the highest-addressed segment. 1866 */ 1867 forward = (flags & AH_DIR) == AH_LO; 1868 if (forward) { 1869 hseg = as_findseg(as, lobound, 1); 1870 lseg = AS_SEGPREV(as, hseg); 1871 } else { 1872 1873 /* 1874 * If allocating at least as much as the last allocation, 1875 * use a_lastgap's base as a better estimate of hibound. 1876 */ 1877 if (as->a_lastgap && 1878 minlen >= as->a_lastgap->s_size && 1879 hibound >= as->a_lastgap->s_base) 1880 hibound = as->a_lastgap->s_base; 1881 1882 hseg = as_findseg(as, hibound, 1); 1883 if (hseg->s_base + hseg->s_size < hibound) { 1884 lseg = hseg; 1885 hseg = NULL; 1886 } else { 1887 lseg = AS_SEGPREV(as, hseg); 1888 } 1889 } 1890 1891 for (;;) { 1892 /* 1893 * Set lo and hi to the hole's boundaries. (We should really 1894 * use MAXADDR in place of hibound in the expression below, 1895 * but can't express it easily; using hibound in its place is 1896 * harmless.) 1897 */ 1898 lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size; 1899 hi = (hseg == NULL) ? hibound : hseg->s_base; 1900 /* 1901 * If the iteration has moved past the interval from lobound 1902 * to hibound it's pointless to continue. 1903 */ 1904 if ((forward && lo > hibound) || (!forward && hi < lobound)) 1905 break; 1906 else if (lo > hibound || hi < lobound) 1907 goto cont; 1908 /* 1909 * Candidate hole lies at least partially within the allowable 1910 * range. Restrict it to fall completely within that range, 1911 * i.e., to [max(lo, lobound), min(hi, hibound)]. 1912 */ 1913 if (lo < lobound) 1914 lo = lobound; 1915 if (hi > hibound) 1916 hi = hibound; 1917 /* 1918 * Verify that the candidate hole is big enough and meets 1919 * hardware constraints. 1920 */ 1921 *basep = lo; 1922 *lenp = hi - lo; 1923 if (valid_va_range(basep, lenp, minlen, 1924 forward ? AH_LO : AH_HI) && 1925 ((flags & AH_CONTAIN) == 0 || 1926 (*basep <= addr && *basep + *lenp > addr))) { 1927 if (!forward) 1928 as->a_lastgap = hseg; 1929 if (hseg != NULL) 1930 as->a_lastgaphl = hseg; 1931 else 1932 as->a_lastgaphl = lseg; 1933 AS_LOCK_EXIT(as, &as->a_lock); 1934 return (0); 1935 } 1936 cont: 1937 /* 1938 * Move to the next hole. 1939 */ 1940 if (forward) { 1941 lseg = hseg; 1942 if (lseg == NULL) 1943 break; 1944 hseg = AS_SEGNEXT(as, hseg); 1945 } else { 1946 hseg = lseg; 1947 if (hseg == NULL) 1948 break; 1949 lseg = AS_SEGPREV(as, lseg); 1950 } 1951 } 1952 *basep = save_base; 1953 *lenp = save_len; 1954 AS_LOCK_EXIT(as, &as->a_lock); 1955 return (-1); 1956 } 1957 1958 /* 1959 * Return the next range within [base, base + len) that is backed 1960 * with "real memory". Skip holes and non-seg_vn segments. 1961 * We're lazy and only return one segment at a time. 1962 */ 1963 int 1964 as_memory(struct as *as, caddr_t *basep, size_t *lenp) 1965 { 1966 extern struct seg_ops segspt_shmops; /* needs a header file */ 1967 struct seg *seg; 1968 caddr_t addr, eaddr; 1969 caddr_t segend; 1970 1971 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1972 1973 addr = *basep; 1974 eaddr = addr + *lenp; 1975 1976 seg = as_findseg(as, addr, 0); 1977 if (seg != NULL) 1978 addr = MAX(seg->s_base, addr); 1979 1980 for (;;) { 1981 if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { 1982 AS_LOCK_EXIT(as, &as->a_lock); 1983 return (EINVAL); 1984 } 1985 1986 if (seg->s_ops == &segvn_ops) { 1987 segend = seg->s_base + seg->s_size; 1988 break; 1989 } 1990 1991 /* 1992 * We do ISM by looking into the private data 1993 * to determine the real size of the segment. 1994 */ 1995 if (seg->s_ops == &segspt_shmops) { 1996 segend = seg->s_base + spt_realsize(seg); 1997 if (addr < segend) 1998 break; 1999 } 2000 2001 seg = AS_SEGNEXT(as, seg); 2002 2003 if (seg != NULL) 2004 addr = seg->s_base; 2005 } 2006 2007 *basep = addr; 2008 2009 if (segend > eaddr) 2010 *lenp = eaddr - addr; 2011 else 2012 *lenp = segend - addr; 2013 2014 AS_LOCK_EXIT(as, &as->a_lock); 2015 return (0); 2016 } 2017 2018 /* 2019 * Swap the pages associated with the address space as out to 2020 * secondary storage, returning the number of bytes actually 2021 * swapped. 2022 * 2023 * The value returned is intended to correlate well with the process's 2024 * memory requirements. Its usefulness for this purpose depends on 2025 * how well the segment-level routines do at returning accurate 2026 * information. 2027 */ 2028 size_t 2029 as_swapout(struct as *as) 2030 { 2031 struct seg *seg; 2032 size_t swpcnt = 0; 2033 2034 /* 2035 * Kernel-only processes have given up their address 2036 * spaces. Of course, we shouldn't be attempting to 2037 * swap out such processes in the first place... 2038 */ 2039 if (as == NULL) 2040 return (0); 2041 2042 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 2043 2044 /* Prevent XHATs from attaching */ 2045 mutex_enter(&as->a_contents); 2046 AS_SETBUSY(as); 2047 mutex_exit(&as->a_contents); 2048 2049 2050 /* 2051 * Free all mapping resources associated with the address 2052 * space. The segment-level swapout routines capitalize 2053 * on this unmapping by scavanging pages that have become 2054 * unmapped here. 2055 */ 2056 hat_swapout(as->a_hat); 2057 if (as->a_xhat != NULL) 2058 xhat_swapout_all(as); 2059 2060 mutex_enter(&as->a_contents); 2061 AS_CLRBUSY(as); 2062 mutex_exit(&as->a_contents); 2063 2064 /* 2065 * Call the swapout routines of all segments in the address 2066 * space to do the actual work, accumulating the amount of 2067 * space reclaimed. 2068 */ 2069 for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 2070 struct seg_ops *ov = seg->s_ops; 2071 2072 /* 2073 * We have to check to see if the seg has 2074 * an ops vector because the seg may have 2075 * been in the middle of being set up when 2076 * the process was picked for swapout. 2077 */ 2078 if ((ov != NULL) && (ov->swapout != NULL)) 2079 swpcnt += SEGOP_SWAPOUT(seg); 2080 } 2081 AS_LOCK_EXIT(as, &as->a_lock); 2082 return (swpcnt); 2083 } 2084 2085 /* 2086 * Determine whether data from the mappings in interval [addr, addr + size) 2087 * are in the primary memory (core) cache. 2088 */ 2089 int 2090 as_incore(struct as *as, caddr_t addr, 2091 size_t size, char *vec, size_t *sizep) 2092 { 2093 struct seg *seg; 2094 size_t ssize; 2095 caddr_t raddr; /* rounded down addr */ 2096 size_t rsize; /* rounded up size */ 2097 size_t isize; /* iteration size */ 2098 int error = 0; /* result, assume success */ 2099 2100 *sizep = 0; 2101 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2102 rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) - 2103 (size_t)raddr; 2104 2105 if (raddr + rsize < raddr) /* check for wraparound */ 2106 return (ENOMEM); 2107 2108 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 2109 seg = as_segat(as, raddr); 2110 if (seg == NULL) { 2111 AS_LOCK_EXIT(as, &as->a_lock); 2112 return (-1); 2113 } 2114 2115 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 2116 if (raddr >= seg->s_base + seg->s_size) { 2117 seg = AS_SEGNEXT(as, seg); 2118 if (seg == NULL || raddr != seg->s_base) { 2119 error = -1; 2120 break; 2121 } 2122 } 2123 if ((raddr + rsize) > (seg->s_base + seg->s_size)) 2124 ssize = seg->s_base + seg->s_size - raddr; 2125 else 2126 ssize = rsize; 2127 *sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec); 2128 if (isize != ssize) { 2129 error = -1; 2130 break; 2131 } 2132 vec += btopr(ssize); 2133 } 2134 AS_LOCK_EXIT(as, &as->a_lock); 2135 return (error); 2136 } 2137 2138 static void 2139 as_segunlock(struct seg *seg, caddr_t addr, int attr, 2140 ulong_t *bitmap, size_t position, size_t npages) 2141 { 2142 caddr_t range_start; 2143 size_t pos1 = position; 2144 size_t pos2; 2145 size_t size; 2146 size_t end_pos = npages + position; 2147 2148 while (bt_range(bitmap, &pos1, &pos2, end_pos)) { 2149 size = ptob((pos2 - pos1)); 2150 range_start = (caddr_t)((uintptr_t)addr + 2151 ptob(pos1 - position)); 2152 2153 (void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK, 2154 (ulong_t *)NULL, (size_t)NULL); 2155 pos1 = pos2; 2156 } 2157 } 2158 2159 static void 2160 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map, 2161 caddr_t raddr, size_t rsize) 2162 { 2163 struct seg *seg = as_segat(as, raddr); 2164 size_t ssize; 2165 2166 while (rsize != 0) { 2167 if (raddr >= seg->s_base + seg->s_size) 2168 seg = AS_SEGNEXT(as, seg); 2169 2170 if ((raddr + rsize) > (seg->s_base + seg->s_size)) 2171 ssize = seg->s_base + seg->s_size - raddr; 2172 else 2173 ssize = rsize; 2174 2175 as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize)); 2176 2177 rsize -= ssize; 2178 raddr += ssize; 2179 } 2180 } 2181 2182 /* 2183 * Cache control operations over the interval [addr, addr + size) in 2184 * address space "as". 2185 */ 2186 /*ARGSUSED*/ 2187 int 2188 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr, 2189 uintptr_t arg, ulong_t *lock_map, size_t pos) 2190 { 2191 struct seg *seg; /* working segment */ 2192 caddr_t raddr; /* rounded down addr */ 2193 caddr_t initraddr; /* saved initial rounded down addr */ 2194 size_t rsize; /* rounded up size */ 2195 size_t initrsize; /* saved initial rounded up size */ 2196 size_t ssize; /* size of seg */ 2197 int error = 0; /* result */ 2198 size_t mlock_size; /* size of bitmap */ 2199 ulong_t *mlock_map; /* pointer to bitmap used */ 2200 /* to represent the locked */ 2201 /* pages. */ 2202 retry: 2203 if (error == IE_RETRY) 2204 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 2205 else 2206 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 2207 2208 /* 2209 * If these are address space lock/unlock operations, loop over 2210 * all segments in the address space, as appropriate. 2211 */ 2212 if (func == MC_LOCKAS) { 2213 size_t npages, idx; 2214 size_t rlen = 0; /* rounded as length */ 2215 2216 idx = pos; 2217 2218 if (arg & MCL_FUTURE) { 2219 mutex_enter(&as->a_contents); 2220 AS_SETPGLCK(as); 2221 mutex_exit(&as->a_contents); 2222 } 2223 if ((arg & MCL_CURRENT) == 0) { 2224 AS_LOCK_EXIT(as, &as->a_lock); 2225 return (0); 2226 } 2227 2228 seg = AS_SEGFIRST(as); 2229 if (seg == NULL) { 2230 AS_LOCK_EXIT(as, &as->a_lock); 2231 return (0); 2232 } 2233 2234 do { 2235 raddr = (caddr_t)((uintptr_t)seg->s_base & 2236 (uintptr_t)PAGEMASK); 2237 rlen += (((uintptr_t)(seg->s_base + seg->s_size) + 2238 PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr; 2239 } while ((seg = AS_SEGNEXT(as, seg)) != NULL); 2240 2241 mlock_size = BT_BITOUL(btopr(rlen)); 2242 if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 2243 sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 2244 AS_LOCK_EXIT(as, &as->a_lock); 2245 return (EAGAIN); 2246 } 2247 2248 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 2249 error = SEGOP_LOCKOP(seg, seg->s_base, 2250 seg->s_size, attr, MC_LOCK, mlock_map, pos); 2251 if (error != 0) 2252 break; 2253 pos += seg_pages(seg); 2254 } 2255 2256 if (error) { 2257 for (seg = AS_SEGFIRST(as); seg != NULL; 2258 seg = AS_SEGNEXT(as, seg)) { 2259 2260 raddr = (caddr_t)((uintptr_t)seg->s_base & 2261 (uintptr_t)PAGEMASK); 2262 npages = seg_pages(seg); 2263 as_segunlock(seg, raddr, attr, mlock_map, 2264 idx, npages); 2265 idx += npages; 2266 } 2267 } 2268 2269 kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 2270 AS_LOCK_EXIT(as, &as->a_lock); 2271 goto lockerr; 2272 } else if (func == MC_UNLOCKAS) { 2273 mutex_enter(&as->a_contents); 2274 AS_CLRPGLCK(as); 2275 mutex_exit(&as->a_contents); 2276 2277 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 2278 error = SEGOP_LOCKOP(seg, seg->s_base, 2279 seg->s_size, attr, MC_UNLOCK, NULL, 0); 2280 if (error != 0) 2281 break; 2282 } 2283 2284 AS_LOCK_EXIT(as, &as->a_lock); 2285 goto lockerr; 2286 } 2287 2288 /* 2289 * Normalize addresses and sizes. 2290 */ 2291 initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2292 initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2293 (size_t)raddr; 2294 2295 if (raddr + rsize < raddr) { /* check for wraparound */ 2296 AS_LOCK_EXIT(as, &as->a_lock); 2297 return (ENOMEM); 2298 } 2299 2300 /* 2301 * Get initial segment. 2302 */ 2303 if ((seg = as_segat(as, raddr)) == NULL) { 2304 AS_LOCK_EXIT(as, &as->a_lock); 2305 return (ENOMEM); 2306 } 2307 2308 if (func == MC_LOCK) { 2309 mlock_size = BT_BITOUL(btopr(rsize)); 2310 if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 2311 sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 2312 AS_LOCK_EXIT(as, &as->a_lock); 2313 return (EAGAIN); 2314 } 2315 } 2316 2317 /* 2318 * Loop over all segments. If a hole in the address range is 2319 * discovered, then fail. For each segment, perform the appropriate 2320 * control operation. 2321 */ 2322 while (rsize != 0) { 2323 2324 /* 2325 * Make sure there's no hole, calculate the portion 2326 * of the next segment to be operated over. 2327 */ 2328 if (raddr >= seg->s_base + seg->s_size) { 2329 seg = AS_SEGNEXT(as, seg); 2330 if (seg == NULL || raddr != seg->s_base) { 2331 if (func == MC_LOCK) { 2332 as_unlockerr(as, attr, mlock_map, 2333 initraddr, initrsize - rsize); 2334 kmem_free(mlock_map, 2335 mlock_size * sizeof (ulong_t)); 2336 } 2337 AS_LOCK_EXIT(as, &as->a_lock); 2338 return (ENOMEM); 2339 } 2340 } 2341 if ((raddr + rsize) > (seg->s_base + seg->s_size)) 2342 ssize = seg->s_base + seg->s_size - raddr; 2343 else 2344 ssize = rsize; 2345 2346 /* 2347 * Dispatch on specific function. 2348 */ 2349 switch (func) { 2350 2351 /* 2352 * Synchronize cached data from mappings with backing 2353 * objects. 2354 */ 2355 case MC_SYNC: 2356 if (error = SEGOP_SYNC(seg, raddr, ssize, 2357 attr, (uint_t)arg)) { 2358 AS_LOCK_EXIT(as, &as->a_lock); 2359 return (error); 2360 } 2361 break; 2362 2363 /* 2364 * Lock pages in memory. 2365 */ 2366 case MC_LOCK: 2367 if (error = SEGOP_LOCKOP(seg, raddr, ssize, 2368 attr, func, mlock_map, pos)) { 2369 as_unlockerr(as, attr, mlock_map, initraddr, 2370 initrsize - rsize + ssize); 2371 kmem_free(mlock_map, mlock_size * 2372 sizeof (ulong_t)); 2373 AS_LOCK_EXIT(as, &as->a_lock); 2374 goto lockerr; 2375 } 2376 break; 2377 2378 /* 2379 * Unlock mapped pages. 2380 */ 2381 case MC_UNLOCK: 2382 (void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func, 2383 (ulong_t *)NULL, (size_t)NULL); 2384 break; 2385 2386 /* 2387 * Store VM advise for mapped pages in segment layer. 2388 */ 2389 case MC_ADVISE: 2390 error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg); 2391 2392 /* 2393 * Check for regular errors and special retry error 2394 */ 2395 if (error) { 2396 if (error == IE_RETRY) { 2397 /* 2398 * Need to acquire writers lock, so 2399 * have to drop readers lock and start 2400 * all over again 2401 */ 2402 AS_LOCK_EXIT(as, &as->a_lock); 2403 goto retry; 2404 } else if (error == IE_REATTACH) { 2405 /* 2406 * Find segment for current address 2407 * because current segment just got 2408 * split or concatenated 2409 */ 2410 seg = as_segat(as, raddr); 2411 if (seg == NULL) { 2412 AS_LOCK_EXIT(as, &as->a_lock); 2413 return (ENOMEM); 2414 } 2415 } else { 2416 /* 2417 * Regular error 2418 */ 2419 AS_LOCK_EXIT(as, &as->a_lock); 2420 return (error); 2421 } 2422 } 2423 break; 2424 2425 /* 2426 * Can't happen. 2427 */ 2428 default: 2429 panic("as_ctl: bad operation %d", func); 2430 /*NOTREACHED*/ 2431 } 2432 2433 rsize -= ssize; 2434 raddr += ssize; 2435 } 2436 2437 if (func == MC_LOCK) 2438 kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 2439 AS_LOCK_EXIT(as, &as->a_lock); 2440 return (0); 2441 lockerr: 2442 2443 /* 2444 * If the lower levels returned EDEADLK for a segment lockop, 2445 * it means that we should retry the operation. Let's wait 2446 * a bit also to let the deadlock causing condition clear. 2447 * This is part of a gross hack to work around a design flaw 2448 * in the ufs/sds logging code and should go away when the 2449 * logging code is re-designed to fix the problem. See bug 2450 * 4125102 for details of the problem. 2451 */ 2452 if (error == EDEADLK) { 2453 delay(deadlk_wait); 2454 error = 0; 2455 goto retry; 2456 } 2457 return (error); 2458 } 2459 2460 /* 2461 * Special code for exec to move the stack segment from its interim 2462 * place in the old address to the right place in the new address space. 2463 */ 2464 /*ARGSUSED*/ 2465 int 2466 as_exec(struct as *oas, caddr_t ostka, size_t stksz, 2467 struct as *nas, caddr_t nstka, uint_t hatflag) 2468 { 2469 struct seg *stkseg; 2470 2471 AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER); 2472 stkseg = as_segat(oas, ostka); 2473 stkseg = as_removeseg(oas, stkseg); 2474 ASSERT(stkseg != NULL); 2475 ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz); 2476 stkseg->s_as = nas; 2477 stkseg->s_base = nstka; 2478 2479 /* 2480 * It's ok to lock the address space we are about to exec to. 2481 */ 2482 AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER); 2483 ASSERT(avl_numnodes(&nas->a_wpage) == 0); 2484 nas->a_size += stkseg->s_size; 2485 oas->a_size -= stkseg->s_size; 2486 (void) as_addseg(nas, stkseg); 2487 AS_LOCK_EXIT(nas, &nas->a_lock); 2488 AS_LOCK_EXIT(oas, &oas->a_lock); 2489 return (0); 2490 } 2491 2492 static int 2493 f_decode(faultcode_t fault_err) 2494 { 2495 int error = 0; 2496 2497 switch (FC_CODE(fault_err)) { 2498 case FC_OBJERR: 2499 error = FC_ERRNO(fault_err); 2500 break; 2501 case FC_PROT: 2502 error = EACCES; 2503 break; 2504 default: 2505 error = EFAULT; 2506 break; 2507 } 2508 return (error); 2509 } 2510 2511 /* 2512 * lock pages in a given address space. Return shadow list. If 2513 * the list is NULL, the MMU mapping is also locked. 2514 */ 2515 int 2516 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr, 2517 size_t size, enum seg_rw rw) 2518 { 2519 size_t rsize; 2520 caddr_t base; 2521 caddr_t raddr; 2522 faultcode_t fault_err; 2523 struct seg *seg; 2524 int res; 2525 int prefaulted = 0; 2526 2527 TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START, 2528 "as_pagelock_start: addr %p size %ld", addr, size); 2529 2530 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2531 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2532 (size_t)raddr; 2533 top: 2534 /* 2535 * if the request crosses two segments let 2536 * as_fault handle it. 2537 */ 2538 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 2539 seg = as_findseg(as, addr, 0); 2540 if ((seg == NULL) || ((base = seg->s_base) > addr) || 2541 (addr + size) > base + seg->s_size) { 2542 AS_LOCK_EXIT(as, &as->a_lock); 2543 goto slow; 2544 } 2545 2546 TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, 2547 "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize); 2548 2549 /* 2550 * try to lock pages and pass back shadow list 2551 */ 2552 res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); 2553 2554 TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); 2555 AS_LOCK_EXIT(as, &as->a_lock); 2556 if (res == 0) { 2557 return (0); 2558 } else if (res == ENOTSUP || prefaulted) { 2559 /* 2560 * (1) segment driver doesn't support PAGELOCK fastpath, or 2561 * (2) we've already tried fast path unsuccessfully after 2562 * faulting in the addr range below; system might be 2563 * thrashing or there may not be enough availrmem. 2564 */ 2565 goto slow; 2566 } 2567 2568 TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START, 2569 "as_fault_start: addr %p size %ld", addr, size); 2570 2571 /* 2572 * we might get here because of some COW fault or non 2573 * existing page. Let as_fault deal with it. Just load 2574 * the page, don't lock the MMU mapping. 2575 */ 2576 fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw); 2577 if (fault_err != 0) { 2578 return (f_decode(fault_err)); 2579 } 2580 2581 prefaulted = 1; 2582 2583 /* 2584 * try fast path again; since we've dropped a_lock, 2585 * we need to try the dance from the start to see if 2586 * the addr range is still valid. 2587 */ 2588 goto top; 2589 slow: 2590 /* 2591 * load the page and lock the MMU mapping. 2592 */ 2593 fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw); 2594 if (fault_err != 0) { 2595 return (f_decode(fault_err)); 2596 } 2597 *ppp = NULL; 2598 2599 TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end"); 2600 return (0); 2601 } 2602 2603 /* 2604 * unlock pages in a given address range 2605 */ 2606 void 2607 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size, 2608 enum seg_rw rw) 2609 { 2610 struct seg *seg; 2611 size_t rsize; 2612 caddr_t raddr; 2613 2614 TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START, 2615 "as_pageunlock_start: addr %p size %ld", addr, size); 2616 2617 /* 2618 * if the shadow list is NULL, as_pagelock was 2619 * falling back to as_fault 2620 */ 2621 if (pp == NULL) { 2622 (void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw); 2623 return; 2624 } 2625 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2626 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2627 (size_t)raddr; 2628 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 2629 seg = as_findseg(as, addr, 0); 2630 ASSERT(seg); 2631 TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, 2632 "seg_unlock_start: raddr %p rsize %ld", raddr, rsize); 2633 SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); 2634 AS_LOCK_EXIT(as, &as->a_lock); 2635 TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); 2636 } 2637 2638 /* 2639 * reclaim cached pages in a given address range 2640 */ 2641 void 2642 as_pagereclaim(struct as *as, struct page **pp, caddr_t addr, 2643 size_t size, enum seg_rw rw) 2644 { 2645 struct seg *seg; 2646 size_t rsize; 2647 caddr_t raddr; 2648 2649 ASSERT(AS_READ_HELD(as, &as->a_lock)); 2650 ASSERT(pp != NULL); 2651 2652 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2653 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2654 (size_t)raddr; 2655 seg = as_findseg(as, addr, 0); 2656 ASSERT(seg); 2657 SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw); 2658 } 2659 2660 #define MAXPAGEFLIP 4 2661 #define MAXPAGEFLIPSIZ MAXPAGEFLIP*PAGESIZE 2662 2663 int 2664 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc, 2665 boolean_t wait) 2666 { 2667 struct seg *seg; 2668 size_t ssize; 2669 caddr_t raddr; /* rounded down addr */ 2670 size_t rsize; /* rounded up size */ 2671 int error = 0; 2672 size_t pgsz = page_get_pagesize(szc); 2673 2674 setpgsz_top: 2675 if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) { 2676 return (EINVAL); 2677 } 2678 2679 raddr = addr; 2680 rsize = size; 2681 2682 if (raddr + rsize < raddr) /* check for wraparound */ 2683 return (ENOMEM); 2684 2685 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 2686 as_clearwatchprot(as, raddr, rsize); 2687 seg = as_segat(as, raddr); 2688 if (seg == NULL) { 2689 as_setwatch(as); 2690 AS_LOCK_EXIT(as, &as->a_lock); 2691 return (ENOMEM); 2692 } 2693 2694 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 2695 if (raddr >= seg->s_base + seg->s_size) { 2696 seg = AS_SEGNEXT(as, seg); 2697 if (seg == NULL || raddr != seg->s_base) { 2698 error = ENOMEM; 2699 break; 2700 } 2701 } 2702 if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 2703 ssize = seg->s_base + seg->s_size - raddr; 2704 } else { 2705 ssize = rsize; 2706 } 2707 2708 error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 2709 2710 if (error == IE_NOMEM) { 2711 error = EAGAIN; 2712 break; 2713 } 2714 2715 if (error == IE_RETRY) { 2716 AS_LOCK_EXIT(as, &as->a_lock); 2717 goto setpgsz_top; 2718 } 2719 2720 if (error == ENOTSUP) { 2721 error = EINVAL; 2722 break; 2723 } 2724 2725 if (wait && (error == EAGAIN)) { 2726 /* 2727 * Memory is currently locked. It must be unlocked 2728 * before this operation can succeed through a retry. 2729 * The possible reasons for locked memory and 2730 * corresponding strategies for unlocking are: 2731 * (1) Normal I/O 2732 * wait for a signal that the I/O operation 2733 * has completed and the memory is unlocked. 2734 * (2) Asynchronous I/O 2735 * The aio subsystem does not unlock pages when 2736 * the I/O is completed. Those pages are unlocked 2737 * when the application calls aiowait/aioerror. 2738 * So, to prevent blocking forever, cv_broadcast() 2739 * is done to wake up aio_cleanup_thread. 2740 * Subsequently, segvn_reclaim will be called, and 2741 * that will do AS_CLRUNMAPWAIT() and wake us up. 2742 * (3) Long term page locking: 2743 * This is not relevant for as_setpagesize() 2744 * because we cannot change the page size for 2745 * driver memory. The attempt to do so will 2746 * fail with a different error than EAGAIN so 2747 * there's no need to trigger as callbacks like 2748 * as_unmap, as_setprot or as_free would do. 2749 */ 2750 mutex_enter(&as->a_contents); 2751 if (AS_ISUNMAPWAIT(as) == 0) { 2752 cv_broadcast(&as->a_cv); 2753 } 2754 AS_SETUNMAPWAIT(as); 2755 AS_LOCK_EXIT(as, &as->a_lock); 2756 while (AS_ISUNMAPWAIT(as)) { 2757 cv_wait(&as->a_cv, &as->a_contents); 2758 } 2759 mutex_exit(&as->a_contents); 2760 goto setpgsz_top; 2761 } else if (error != 0) { 2762 break; 2763 } 2764 } 2765 as_setwatch(as); 2766 AS_LOCK_EXIT(as, &as->a_lock); 2767 return (error); 2768 } 2769 2770 /* 2771 * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments 2772 * in its chunk where s_szc is less than the szc we want to set. 2773 */ 2774 static int 2775 as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 2776 int *retry) 2777 { 2778 struct seg *seg; 2779 size_t ssize; 2780 int error; 2781 2782 seg = as_segat(as, raddr); 2783 if (seg == NULL) { 2784 panic("as_iset3_default_lpsize: no seg"); 2785 } 2786 2787 for (; rsize != 0; rsize -= ssize, raddr += ssize) { 2788 if (raddr >= seg->s_base + seg->s_size) { 2789 seg = AS_SEGNEXT(as, seg); 2790 if (seg == NULL || raddr != seg->s_base) { 2791 panic("as_iset3_default_lpsize: as changed"); 2792 } 2793 } 2794 if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 2795 ssize = seg->s_base + seg->s_size - raddr; 2796 } else { 2797 ssize = rsize; 2798 } 2799 2800 if (szc > seg->s_szc) { 2801 error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 2802 /* Only retry on EINVAL segments that have no vnode. */ 2803 if (error == EINVAL) { 2804 vnode_t *vp = NULL; 2805 if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) && 2806 (SEGOP_GETVP(seg, raddr, &vp) != 0 || 2807 vp == NULL)) { 2808 *retry = 1; 2809 } else { 2810 *retry = 0; 2811 } 2812 } 2813 if (error) { 2814 return (error); 2815 } 2816 } 2817 } 2818 return (0); 2819 } 2820 2821 /* 2822 * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the 2823 * pagesize on each segment in its range, but if any fails with EINVAL, 2824 * then it reduces the pagesizes to the next size in the bitmap and 2825 * retries as_iset3_default_lpsize(). The reason why the code retries 2826 * smaller allowed sizes on EINVAL is because (a) the anon offset may not 2827 * match the bigger sizes, and (b) it's hard to get this offset (to begin 2828 * with) to pass to map_pgszcvec(). 2829 */ 2830 static int 2831 as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc, 2832 uint_t szcvec) 2833 { 2834 int error; 2835 int retry; 2836 2837 for (;;) { 2838 error = as_iset3_default_lpsize(as, addr, size, szc, &retry); 2839 if (error == EINVAL && retry) { 2840 szcvec &= ~(1 << szc); 2841 if (szcvec <= 1) { 2842 return (EINVAL); 2843 } 2844 szc = highbit(szcvec) - 1; 2845 } else { 2846 return (error); 2847 } 2848 } 2849 } 2850 2851 /* 2852 * as_iset1_default_lpsize() breaks its chunk into areas where existing 2853 * segments have a smaller szc than we want to set. For each such area, 2854 * it calls as_iset2_default_lpsize() 2855 */ 2856 static int 2857 as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 2858 uint_t szcvec) 2859 { 2860 struct seg *seg; 2861 size_t ssize; 2862 caddr_t setaddr = raddr; 2863 size_t setsize = 0; 2864 int set; 2865 int error; 2866 2867 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 2868 2869 seg = as_segat(as, raddr); 2870 if (seg == NULL) { 2871 panic("as_iset1_default_lpsize: no seg"); 2872 } 2873 if (seg->s_szc < szc) { 2874 set = 1; 2875 } else { 2876 set = 0; 2877 } 2878 2879 for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 2880 if (raddr >= seg->s_base + seg->s_size) { 2881 seg = AS_SEGNEXT(as, seg); 2882 if (seg == NULL || raddr != seg->s_base) { 2883 panic("as_iset1_default_lpsize: as changed"); 2884 } 2885 if (seg->s_szc >= szc && set) { 2886 ASSERT(setsize != 0); 2887 error = as_iset2_default_lpsize(as, 2888 setaddr, setsize, szc, szcvec); 2889 if (error) { 2890 return (error); 2891 } 2892 set = 0; 2893 } else if (seg->s_szc < szc && !set) { 2894 setaddr = raddr; 2895 setsize = 0; 2896 set = 1; 2897 } 2898 } 2899 if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 2900 ssize = seg->s_base + seg->s_size - raddr; 2901 } else { 2902 ssize = rsize; 2903 } 2904 } 2905 error = 0; 2906 if (set) { 2907 ASSERT(setsize != 0); 2908 error = as_iset2_default_lpsize(as, setaddr, setsize, 2909 szc, szcvec); 2910 } 2911 return (error); 2912 } 2913 2914 /* 2915 * as_iset_default_lpsize() breaks its chunk according to the size code bitmap 2916 * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each 2917 * chunk to as_iset1_default_lpsize(). 2918 */ 2919 static int 2920 as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags, 2921 int type) 2922 { 2923 int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 2924 uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, 2925 flags, rtype, 1); 2926 uint_t szc; 2927 uint_t nszc; 2928 int error; 2929 caddr_t a; 2930 caddr_t eaddr; 2931 size_t segsize; 2932 size_t pgsz; 2933 uint_t save_szcvec; 2934 2935 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 2936 ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 2937 ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 2938 2939 szcvec &= ~1; 2940 if (szcvec <= 1) { /* skip if base page size */ 2941 return (0); 2942 } 2943 2944 /* Get the pagesize of the first larger page size. */ 2945 szc = lowbit(szcvec) - 1; 2946 pgsz = page_get_pagesize(szc); 2947 eaddr = addr + size; 2948 addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 2949 eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 2950 2951 save_szcvec = szcvec; 2952 szcvec >>= (szc + 1); 2953 nszc = szc; 2954 while (szcvec) { 2955 if ((szcvec & 0x1) == 0) { 2956 nszc++; 2957 szcvec >>= 1; 2958 continue; 2959 } 2960 nszc++; 2961 pgsz = page_get_pagesize(nszc); 2962 a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 2963 if (a != addr) { 2964 ASSERT(szc > 0); 2965 ASSERT(a < eaddr); 2966 segsize = a - addr; 2967 error = as_iset1_default_lpsize(as, addr, segsize, szc, 2968 save_szcvec); 2969 if (error) { 2970 return (error); 2971 } 2972 addr = a; 2973 } 2974 szc = nszc; 2975 szcvec >>= 1; 2976 } 2977 2978 ASSERT(addr < eaddr); 2979 szcvec = save_szcvec; 2980 while (szcvec) { 2981 a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 2982 ASSERT(a >= addr); 2983 if (a != addr) { 2984 ASSERT(szc > 0); 2985 segsize = a - addr; 2986 error = as_iset1_default_lpsize(as, addr, segsize, szc, 2987 save_szcvec); 2988 if (error) { 2989 return (error); 2990 } 2991 addr = a; 2992 } 2993 szcvec &= ~(1 << szc); 2994 if (szcvec) { 2995 szc = highbit(szcvec) - 1; 2996 pgsz = page_get_pagesize(szc); 2997 } 2998 } 2999 ASSERT(addr == eaddr); 3000 3001 return (0); 3002 } 3003 3004 /* 3005 * Set the default large page size for the range. Called via memcntl with 3006 * page size set to 0. as_set_default_lpsize breaks the range down into 3007 * chunks with the same type/flags, ignores-non segvn segments, and passes 3008 * each chunk to as_iset_default_lpsize(). 3009 */ 3010 int 3011 as_set_default_lpsize(struct as *as, caddr_t addr, size_t size) 3012 { 3013 struct seg *seg; 3014 caddr_t raddr; 3015 size_t rsize; 3016 size_t ssize; 3017 int rtype, rflags; 3018 int stype, sflags; 3019 int error; 3020 caddr_t setaddr; 3021 size_t setsize; 3022 int segvn; 3023 3024 if (size == 0) 3025 return (0); 3026 3027 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 3028 again: 3029 error = 0; 3030 3031 raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3032 rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 3033 (size_t)raddr; 3034 3035 if (raddr + rsize < raddr) { /* check for wraparound */ 3036 AS_LOCK_EXIT(as, &as->a_lock); 3037 return (ENOMEM); 3038 } 3039 as_clearwatchprot(as, raddr, rsize); 3040 seg = as_segat(as, raddr); 3041 if (seg == NULL) { 3042 as_setwatch(as); 3043 AS_LOCK_EXIT(as, &as->a_lock); 3044 return (ENOMEM); 3045 } 3046 if (seg->s_ops == &segvn_ops) { 3047 rtype = SEGOP_GETTYPE(seg, addr); 3048 rflags = rtype & (MAP_TEXT | MAP_INITDATA); 3049 rtype = rtype & (MAP_SHARED | MAP_PRIVATE); 3050 segvn = 1; 3051 } else { 3052 segvn = 0; 3053 } 3054 setaddr = raddr; 3055 setsize = 0; 3056 3057 for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 3058 if (raddr >= (seg->s_base + seg->s_size)) { 3059 seg = AS_SEGNEXT(as, seg); 3060 if (seg == NULL || raddr != seg->s_base) { 3061 error = ENOMEM; 3062 break; 3063 } 3064 if (seg->s_ops == &segvn_ops) { 3065 stype = SEGOP_GETTYPE(seg, raddr); 3066 sflags = stype & (MAP_TEXT | MAP_INITDATA); 3067 stype &= (MAP_SHARED | MAP_PRIVATE); 3068 if (segvn && (rflags != sflags || 3069 rtype != stype)) { 3070 /* 3071 * The next segment is also segvn but 3072 * has different flags and/or type. 3073 */ 3074 ASSERT(setsize != 0); 3075 error = as_iset_default_lpsize(as, 3076 setaddr, setsize, rflags, rtype); 3077 if (error) { 3078 break; 3079 } 3080 rflags = sflags; 3081 rtype = stype; 3082 setaddr = raddr; 3083 setsize = 0; 3084 } else if (!segvn) { 3085 rflags = sflags; 3086 rtype = stype; 3087 setaddr = raddr; 3088 setsize = 0; 3089 segvn = 1; 3090 } 3091 } else if (segvn) { 3092 /* The next segment is not segvn. */ 3093 ASSERT(setsize != 0); 3094 error = as_iset_default_lpsize(as, 3095 setaddr, setsize, rflags, rtype); 3096 if (error) { 3097 break; 3098 } 3099 segvn = 0; 3100 } 3101 } 3102 if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 3103 ssize = seg->s_base + seg->s_size - raddr; 3104 } else { 3105 ssize = rsize; 3106 } 3107 } 3108 if (error == 0 && segvn) { 3109 /* The last chunk when rsize == 0. */ 3110 ASSERT(setsize != 0); 3111 error = as_iset_default_lpsize(as, setaddr, setsize, 3112 rflags, rtype); 3113 } 3114 3115 if (error == IE_RETRY) { 3116 goto again; 3117 } else if (error == IE_NOMEM) { 3118 error = EAGAIN; 3119 } else if (error == ENOTSUP) { 3120 error = EINVAL; 3121 } else if (error == EAGAIN) { 3122 mutex_enter(&as->a_contents); 3123 if (AS_ISUNMAPWAIT(as) == 0) { 3124 cv_broadcast(&as->a_cv); 3125 } 3126 AS_SETUNMAPWAIT(as); 3127 AS_LOCK_EXIT(as, &as->a_lock); 3128 while (AS_ISUNMAPWAIT(as)) { 3129 cv_wait(&as->a_cv, &as->a_contents); 3130 } 3131 mutex_exit(&as->a_contents); 3132 AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 3133 goto again; 3134 } 3135 3136 as_setwatch(as); 3137 AS_LOCK_EXIT(as, &as->a_lock); 3138 return (error); 3139 } 3140 3141 /* 3142 * Setup all of the uninitialized watched pages that we can. 3143 */ 3144 void 3145 as_setwatch(struct as *as) 3146 { 3147 struct watched_page *pwp; 3148 struct seg *seg; 3149 caddr_t vaddr; 3150 uint_t prot; 3151 int err, retrycnt; 3152 3153 if (avl_numnodes(&as->a_wpage) == 0) 3154 return; 3155 3156 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 3157 3158 for (pwp = avl_first(&as->a_wpage); pwp != NULL; 3159 pwp = AVL_NEXT(&as->a_wpage, pwp)) { 3160 retrycnt = 0; 3161 retry: 3162 vaddr = pwp->wp_vaddr; 3163 if (pwp->wp_oprot != 0 || /* already set up */ 3164 (seg = as_segat(as, vaddr)) == NULL || 3165 SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0) 3166 continue; 3167 3168 pwp->wp_oprot = prot; 3169 if (pwp->wp_read) 3170 prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 3171 if (pwp->wp_write) 3172 prot &= ~PROT_WRITE; 3173 if (pwp->wp_exec) 3174 prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 3175 if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) { 3176 err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 3177 if (err == IE_RETRY) { 3178 pwp->wp_oprot = 0; 3179 ASSERT(retrycnt == 0); 3180 retrycnt++; 3181 goto retry; 3182 } 3183 } 3184 pwp->wp_prot = prot; 3185 } 3186 } 3187 3188 /* 3189 * Clear all of the watched pages in the address space. 3190 */ 3191 void 3192 as_clearwatch(struct as *as) 3193 { 3194 struct watched_page *pwp; 3195 struct seg *seg; 3196 caddr_t vaddr; 3197 uint_t prot; 3198 int err, retrycnt; 3199 3200 if (avl_numnodes(&as->a_wpage) == 0) 3201 return; 3202 3203 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 3204 3205 for (pwp = avl_first(&as->a_wpage); pwp != NULL; 3206 pwp = AVL_NEXT(&as->a_wpage, pwp)) { 3207 retrycnt = 0; 3208 retry: 3209 vaddr = pwp->wp_vaddr; 3210 if (pwp->wp_oprot == 0 || /* not set up */ 3211 (seg = as_segat(as, vaddr)) == NULL) 3212 continue; 3213 3214 if ((prot = pwp->wp_oprot) != pwp->wp_prot) { 3215 err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 3216 if (err == IE_RETRY) { 3217 ASSERT(retrycnt == 0); 3218 retrycnt++; 3219 goto retry; 3220 } 3221 } 3222 pwp->wp_oprot = 0; 3223 pwp->wp_prot = 0; 3224 } 3225 } 3226 3227 /* 3228 * Force a new setup for all the watched pages in the range. 3229 */ 3230 static void 3231 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 3232 { 3233 struct watched_page *pwp; 3234 struct watched_page tpw; 3235 caddr_t eaddr = addr + size; 3236 caddr_t vaddr; 3237 struct seg *seg; 3238 int err, retrycnt; 3239 uint_t wprot; 3240 avl_index_t where; 3241 3242 if (avl_numnodes(&as->a_wpage) == 0) 3243 return; 3244 3245 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 3246 3247 tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3248 if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 3249 pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 3250 3251 while (pwp != NULL && pwp->wp_vaddr < eaddr) { 3252 retrycnt = 0; 3253 vaddr = pwp->wp_vaddr; 3254 3255 wprot = prot; 3256 if (pwp->wp_read) 3257 wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 3258 if (pwp->wp_write) 3259 wprot &= ~PROT_WRITE; 3260 if (pwp->wp_exec) 3261 wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 3262 if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) { 3263 retry: 3264 seg = as_segat(as, vaddr); 3265 if (seg == NULL) { 3266 panic("as_setwatchprot: no seg"); 3267 /*NOTREACHED*/ 3268 } 3269 err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot); 3270 if (err == IE_RETRY) { 3271 ASSERT(retrycnt == 0); 3272 retrycnt++; 3273 goto retry; 3274 } 3275 } 3276 pwp->wp_oprot = prot; 3277 pwp->wp_prot = wprot; 3278 3279 pwp = AVL_NEXT(&as->a_wpage, pwp); 3280 } 3281 } 3282 3283 /* 3284 * Clear all of the watched pages in the range. 3285 */ 3286 static void 3287 as_clearwatchprot(struct as *as, caddr_t addr, size_t size) 3288 { 3289 caddr_t eaddr = addr + size; 3290 struct watched_page *pwp; 3291 struct watched_page tpw; 3292 uint_t prot; 3293 struct seg *seg; 3294 int err, retrycnt; 3295 avl_index_t where; 3296 3297 if (avl_numnodes(&as->a_wpage) == 0) 3298 return; 3299 3300 tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3301 if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 3302 pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 3303 3304 ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 3305 3306 while (pwp != NULL && pwp->wp_vaddr < eaddr) { 3307 ASSERT(addr >= pwp->wp_vaddr); 3308 3309 if ((prot = pwp->wp_oprot) != 0) { 3310 retrycnt = 0; 3311 3312 if (prot != pwp->wp_prot) { 3313 retry: 3314 seg = as_segat(as, pwp->wp_vaddr); 3315 if (seg == NULL) 3316 continue; 3317 err = SEGOP_SETPROT(seg, pwp->wp_vaddr, 3318 PAGESIZE, prot); 3319 if (err == IE_RETRY) { 3320 ASSERT(retrycnt == 0); 3321 retrycnt++; 3322 goto retry; 3323 3324 } 3325 } 3326 pwp->wp_oprot = 0; 3327 pwp->wp_prot = 0; 3328 } 3329 3330 pwp = AVL_NEXT(&as->a_wpage, pwp); 3331 } 3332 } 3333 3334 void 3335 as_signal_proc(struct as *as, k_siginfo_t *siginfo) 3336 { 3337 struct proc *p; 3338 3339 mutex_enter(&pidlock); 3340 for (p = practive; p; p = p->p_next) { 3341 if (p->p_as == as) { 3342 mutex_enter(&p->p_lock); 3343 if (p->p_as == as) 3344 sigaddq(p, NULL, siginfo, KM_NOSLEEP); 3345 mutex_exit(&p->p_lock); 3346 } 3347 } 3348 mutex_exit(&pidlock); 3349 } 3350 3351 /* 3352 * return memory object ID 3353 */ 3354 int 3355 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) 3356 { 3357 struct seg *seg; 3358 int sts; 3359 3360 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 3361 seg = as_segat(as, addr); 3362 if (seg == NULL) { 3363 AS_LOCK_EXIT(as, &as->a_lock); 3364 return (EFAULT); 3365 } 3366 /* 3367 * catch old drivers which may not support getmemid 3368 */ 3369 if (seg->s_ops->getmemid == NULL) { 3370 AS_LOCK_EXIT(as, &as->a_lock); 3371 return (ENODEV); 3372 } 3373 3374 sts = SEGOP_GETMEMID(seg, addr, memidp); 3375 3376 AS_LOCK_EXIT(as, &as->a_lock); 3377 return (sts); 3378 } 3379