1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * Portions of this source code were derived from Berkeley 4.3 BSD 31 * under license from the Regents of the University of California. 32 */ 33 34 /* 35 * VM - segment for non-faulting loads. 36 */ 37 38 #include <sys/types.h> 39 #include <sys/t_lock.h> 40 #include <sys/param.h> 41 #include <sys/mman.h> 42 #include <sys/errno.h> 43 #include <sys/kmem.h> 44 #include <sys/cmn_err.h> 45 #include <sys/vnode.h> 46 #include <sys/proc.h> 47 #include <sys/conf.h> 48 #include <sys/debug.h> 49 #include <sys/archsystm.h> 50 #include <sys/lgrp.h> 51 52 #include <vm/page.h> 53 #include <vm/hat.h> 54 #include <vm/as.h> 55 #include <vm/seg.h> 56 #include <vm/vpage.h> 57 58 /* 59 * Private seg op routines. 60 */ 61 static int segnf_dup(struct seg *seg, struct seg *newseg); 62 static int segnf_unmap(struct seg *seg, caddr_t addr, size_t len); 63 static void segnf_free(struct seg *seg); 64 static faultcode_t segnf_nomap(void); 65 static int segnf_setprot(struct seg *seg, caddr_t addr, 66 size_t len, uint_t prot); 67 static int segnf_checkprot(struct seg *seg, caddr_t addr, 68 size_t len, uint_t prot); 69 static void segnf_badop(void); 70 static int segnf_nop(void); 71 static int segnf_getprot(struct seg *seg, caddr_t addr, 72 size_t len, uint_t *protv); 73 static u_offset_t segnf_getoffset(struct seg *seg, caddr_t addr); 74 static int segnf_gettype(struct seg *seg, caddr_t addr); 75 static int segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 76 static void segnf_dump(struct seg *seg); 77 static int segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 78 struct page ***ppp, enum lock_type type, enum seg_rw rw); 79 static int segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 80 uint_t szc); 81 static int segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 82 static lgrp_mem_policy_info_t *segnf_getpolicy(struct seg *seg, 83 caddr_t addr); 84 85 86 struct seg_ops segnf_ops = { 87 segnf_dup, 88 segnf_unmap, 89 segnf_free, 90 (faultcode_t (*)(struct hat *, struct seg *, caddr_t, size_t, 91 enum fault_type, enum seg_rw)) 92 segnf_nomap, /* fault */ 93 (faultcode_t (*)(struct seg *, caddr_t)) 94 segnf_nomap, /* faulta */ 95 segnf_setprot, 96 segnf_checkprot, 97 (int (*)())segnf_badop, /* kluster */ 98 (size_t (*)(struct seg *))NULL, /* swapout */ 99 (int (*)(struct seg *, caddr_t, size_t, int, uint_t)) 100 segnf_nop, /* sync */ 101 (size_t (*)(struct seg *, caddr_t, size_t, char *)) 102 segnf_nop, /* incore */ 103 (int (*)(struct seg *, caddr_t, size_t, int, int, ulong_t *, size_t)) 104 segnf_nop, /* lockop */ 105 segnf_getprot, 106 segnf_getoffset, 107 segnf_gettype, 108 segnf_getvp, 109 (int (*)(struct seg *, caddr_t, size_t, uint_t)) 110 segnf_nop, /* advise */ 111 segnf_dump, 112 segnf_pagelock, 113 segnf_setpagesize, 114 segnf_getmemid, 115 segnf_getpolicy, 116 }; 117 118 /* 119 * vnode and page for the page of zeros we use for the nf mappings. 120 */ 121 static kmutex_t segnf_lock; 122 static struct vnode nfvp; 123 static struct page **nfpp; 124 125 #define addr_to_vcolor(addr) \ 126 (shm_alignment) ? \ 127 ((int)(((uintptr_t)(addr) & (shm_alignment - 1)) >> PAGESHIFT)) : 0 128 129 /* 130 * We try to limit the number of Non-fault segments created. 131 * Non fault segments are created to optimize sparc V9 code which uses 132 * the sparc nonfaulting load ASI (ASI_PRIMARY_NOFAULT). 133 * 134 * There are several reasons why creating too many non-fault segments 135 * could cause problems. 136 * 137 * First, excessive allocation of kernel resources for the seg 138 * structures and the HAT data to map the zero pages. 139 * 140 * Secondly, creating nofault segments actually uses up user virtual 141 * address space. This makes it unavailable for subsequent mmap(0, ...) 142 * calls which use as_gap() to find empty va regions. Creation of too 143 * many nofault segments could thus interfere with the ability of the 144 * runtime linker to load a shared object. 145 */ 146 #define MAXSEGFORNF (10000) 147 #define MAXNFSEARCH (5) 148 149 150 /* 151 * Must be called from startup() 152 */ 153 void 154 segnf_init() 155 { 156 mutex_init(&segnf_lock, NULL, MUTEX_DEFAULT, NULL); 157 } 158 159 160 /* 161 * Create a no-fault segment. 162 * 163 * The no-fault segment is not technically necessary, as the code in 164 * nfload() in trap.c will emulate the SPARC instruction and load 165 * a value of zero in the destination register. 166 * 167 * However, this code tries to put a page of zero's at the nofault address 168 * so that subsequent non-faulting loads to the same page will not 169 * trap with a tlb miss. 170 * 171 * In order to help limit the number of segments we merge adjacent nofault 172 * segments into a single segment. If we get a large number of segments 173 * we'll also try to delete a random other nf segment. 174 */ 175 /* ARGSUSED */ 176 int 177 segnf_create(struct seg *seg, void *argsp) 178 { 179 uint_t prot; 180 pgcnt_t vacpgs; 181 u_offset_t off = 0; 182 caddr_t vaddr = NULL; 183 int i, color; 184 struct seg *s1; 185 struct seg *s2; 186 size_t size; 187 struct as *as = seg->s_as; 188 189 ASSERT(as && AS_WRITE_HELD(as)); 190 191 /* 192 * Need a page per virtual color or just 1 if no vac. 193 */ 194 mutex_enter(&segnf_lock); 195 if (nfpp == NULL) { 196 struct seg kseg; 197 198 vacpgs = 1; 199 if (shm_alignment > PAGESIZE) { 200 vacpgs = shm_alignment >> PAGESHIFT; 201 } 202 203 nfpp = kmem_alloc(sizeof (*nfpp) * vacpgs, KM_SLEEP); 204 205 kseg.s_as = &kas; 206 for (i = 0; i < vacpgs; i++, off += PAGESIZE, 207 vaddr += PAGESIZE) { 208 nfpp[i] = page_create_va(&nfvp, off, PAGESIZE, 209 PG_WAIT | PG_NORELOC, &kseg, vaddr); 210 page_io_unlock(nfpp[i]); 211 page_downgrade(nfpp[i]); 212 pagezero(nfpp[i], 0, PAGESIZE); 213 } 214 } 215 mutex_exit(&segnf_lock); 216 217 hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP); 218 219 /* 220 * s_data can't be NULL because of ASSERTS in the common vm code. 221 */ 222 seg->s_ops = &segnf_ops; 223 seg->s_data = seg; 224 seg->s_flags |= S_PURGE; 225 226 mutex_enter(&as->a_contents); 227 as->a_flags |= AS_NEEDSPURGE; 228 mutex_exit(&as->a_contents); 229 230 prot = PROT_READ; 231 color = addr_to_vcolor(seg->s_base); 232 if (as != &kas) 233 prot |= PROT_USER; 234 hat_memload(as->a_hat, seg->s_base, nfpp[color], 235 prot | HAT_NOFAULT, HAT_LOAD); 236 237 /* 238 * At this point see if we can concatenate a segment to 239 * a non-fault segment immediately before and/or after it. 240 */ 241 if ((s1 = AS_SEGPREV(as, seg)) != NULL && 242 s1->s_ops == &segnf_ops && 243 s1->s_base + s1->s_size == seg->s_base) { 244 size = s1->s_size; 245 seg_free(s1); 246 seg->s_base -= size; 247 seg->s_size += size; 248 } 249 250 if ((s2 = AS_SEGNEXT(as, seg)) != NULL && 251 s2->s_ops == &segnf_ops && 252 seg->s_base + seg->s_size == s2->s_base) { 253 size = s2->s_size; 254 seg_free(s2); 255 seg->s_size += size; 256 } 257 258 /* 259 * if we already have a lot of segments, try to delete some other 260 * nofault segment to reduce the probability of uncontrolled segment 261 * creation. 262 * 263 * the code looks around quickly (no more than MAXNFSEARCH segments 264 * each way) for another NF segment and then deletes it. 265 */ 266 if (avl_numnodes(&as->a_segtree) > MAXSEGFORNF) { 267 size = 0; 268 s2 = NULL; 269 s1 = AS_SEGPREV(as, seg); 270 while (size++ < MAXNFSEARCH && s1 != NULL) { 271 if (s1->s_ops == &segnf_ops) 272 s2 = s1; 273 s1 = AS_SEGPREV(s1->s_as, seg); 274 } 275 if (s2 == NULL) { 276 s1 = AS_SEGNEXT(as, seg); 277 while (size-- > 0 && s1 != NULL) { 278 if (s1->s_ops == &segnf_ops) 279 s2 = s1; 280 s1 = AS_SEGNEXT(as, seg); 281 } 282 } 283 if (s2 != NULL) 284 seg_unmap(s2); 285 } 286 287 return (0); 288 } 289 290 /* 291 * Never really need "No fault" segments, so they aren't dup'd. 292 */ 293 /* ARGSUSED */ 294 static int 295 segnf_dup(struct seg *seg, struct seg *newseg) 296 { 297 panic("segnf_dup"); 298 return (0); 299 } 300 301 /* 302 * Split a segment at addr for length len. 303 */ 304 static int 305 segnf_unmap(struct seg *seg, caddr_t addr, size_t len) 306 { 307 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 308 309 /* 310 * Check for bad sizes. 311 */ 312 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size || 313 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) { 314 cmn_err(CE_PANIC, "segnf_unmap: bad unmap size"); 315 } 316 317 /* 318 * Unload any hardware translations in the range to be taken out. 319 */ 320 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); 321 322 if (addr == seg->s_base && len == seg->s_size) { 323 /* 324 * Freeing entire segment. 325 */ 326 seg_free(seg); 327 } else if (addr == seg->s_base) { 328 /* 329 * Freeing the beginning of the segment. 330 */ 331 seg->s_base += len; 332 seg->s_size -= len; 333 } else if (addr + len == seg->s_base + seg->s_size) { 334 /* 335 * Freeing the end of the segment. 336 */ 337 seg->s_size -= len; 338 } else { 339 /* 340 * The section to go is in the middle of the segment, so we 341 * have to cut it into two segments. We shrink the existing 342 * "seg" at the low end, and create "nseg" for the high end. 343 */ 344 caddr_t nbase = addr + len; 345 size_t nsize = (seg->s_base + seg->s_size) - nbase; 346 struct seg *nseg; 347 348 /* 349 * Trim down "seg" before trying to stick "nseg" into the as. 350 */ 351 seg->s_size = addr - seg->s_base; 352 nseg = seg_alloc(seg->s_as, nbase, nsize); 353 if (nseg == NULL) 354 cmn_err(CE_PANIC, "segnf_unmap: seg_alloc failed"); 355 356 /* 357 * s_data can't be NULL because of ASSERTs in common VM code. 358 */ 359 nseg->s_ops = seg->s_ops; 360 nseg->s_data = nseg; 361 nseg->s_flags |= S_PURGE; 362 mutex_enter(&seg->s_as->a_contents); 363 seg->s_as->a_flags |= AS_NEEDSPURGE; 364 mutex_exit(&seg->s_as->a_contents); 365 } 366 367 return (0); 368 } 369 370 /* 371 * Free a segment. 372 */ 373 static void 374 segnf_free(struct seg *seg) 375 { 376 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 377 } 378 379 /* 380 * No faults allowed on segnf. 381 */ 382 static faultcode_t 383 segnf_nomap(void) 384 { 385 return (FC_NOMAP); 386 } 387 388 /* ARGSUSED */ 389 static int 390 segnf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 391 { 392 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 393 return (EACCES); 394 } 395 396 /* ARGSUSED */ 397 static int 398 segnf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 399 { 400 uint_t sprot; 401 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 402 403 sprot = seg->s_as == &kas ? PROT_READ : PROT_READ|PROT_USER; 404 return ((prot & sprot) == prot ? 0 : EACCES); 405 } 406 407 static void 408 segnf_badop(void) 409 { 410 panic("segnf_badop"); 411 /*NOTREACHED*/ 412 } 413 414 static int 415 segnf_nop(void) 416 { 417 return (0); 418 } 419 420 static int 421 segnf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 422 { 423 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1; 424 size_t p; 425 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 426 427 for (p = 0; p < pgno; ++p) 428 protv[p] = PROT_READ; 429 return (0); 430 } 431 432 /* ARGSUSED */ 433 static u_offset_t 434 segnf_getoffset(struct seg *seg, caddr_t addr) 435 { 436 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 437 438 return ((u_offset_t)0); 439 } 440 441 /* ARGSUSED */ 442 static int 443 segnf_gettype(struct seg *seg, caddr_t addr) 444 { 445 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 446 447 return (MAP_SHARED); 448 } 449 450 /* ARGSUSED */ 451 static int 452 segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 453 { 454 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 455 456 *vpp = &nfvp; 457 return (0); 458 } 459 460 /* 461 * segnf pages are not dumped, so we just return 462 */ 463 /* ARGSUSED */ 464 static void 465 segnf_dump(struct seg *seg) 466 {} 467 468 /*ARGSUSED*/ 469 static int 470 segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 471 struct page ***ppp, enum lock_type type, enum seg_rw rw) 472 { 473 return (ENOTSUP); 474 } 475 476 /*ARGSUSED*/ 477 static int 478 segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 479 uint_t szc) 480 { 481 return (ENOTSUP); 482 } 483 484 /*ARGSUSED*/ 485 static int 486 segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 487 { 488 return (ENODEV); 489 } 490 491 /*ARGSUSED*/ 492 static lgrp_mem_policy_info_t * 493 segnf_getpolicy(struct seg *seg, caddr_t addr) 494 { 495 return (NULL); 496 } 497