1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 /* 29 * University Copyright- Copyright (c) 1982, 1986, 1988 30 * The Regents of the University of California 31 * All Rights Reserved 32 * 33 * University Acknowledgment- Portions of this document are derived from 34 * software developed by the University of California, Berkeley, and its 35 * contributors. 36 */ 37 38 #ifndef _VM_ANON_H 39 #define _VM_ANON_H 40 41 #include <sys/cred.h> 42 #include <sys/zone.h> 43 #include <vm/seg.h> 44 #include <vm/vpage.h> 45 46 #ifdef __cplusplus 47 extern "C" { 48 #endif 49 50 /* 51 * VM - Anonymous pages. 52 */ 53 54 typedef unsigned long anoff_t; /* anon offsets */ 55 56 /* 57 * Each anonymous page, either in memory or in swap, has an anon structure. 58 * The structure (slot) provides a level of indirection between anonymous pages 59 * and their backing store. 60 * 61 * (an_vp, an_off) names the vnode of the anonymous page for this slot. 62 * 63 * (an_pvp, an_poff) names the location of the physical backing store 64 * for the page this slot represents. If the name is null there is no 65 * associated physical store. The physical backing store location can 66 * change while the slot is in use. 67 * 68 * an_hash is a hash list of anon slots. The list is hashed by 69 * (an_vp, an_off) of the associated anonymous page and provides a 70 * method of going from the name of an anonymous page to its 71 * associated anon slot. 72 * 73 * an_refcnt holds a reference count which is the number of separate 74 * copies that will need to be created in case of copy-on-write. 75 * A refcnt > 0 protects the existence of the slot. The refcnt is 76 * initialized to 1 when the anon slot is created in anon_alloc(). 77 * If a client obtains an anon slot and allows multiple threads to 78 * share it, then it is the client's responsibility to insure that 79 * it does not allow one thread to try to reference the slot at the 80 * same time as another is trying to decrement the last count and 81 * destroy the anon slot. E.g., the seg_vn segment type protects 82 * against this with higher level locks. 83 */ 84 85 struct anon { 86 struct vnode *an_vp; /* vnode of anon page */ 87 struct vnode *an_pvp; /* vnode of physical backing store */ 88 anoff_t an_off; /* offset of anon page */ 89 anoff_t an_poff; /* offset in vnode */ 90 struct anon *an_hash; /* hash table of anon slots */ 91 int an_refcnt; /* # of people sharing slot */ 92 }; 93 94 #define AN_CACHE_ALIGN_LOG2 4 /* log2(AN_CACHE_ALIGN) */ 95 #define AN_CACHE_ALIGN (1U << AN_CACHE_ALIGN_LOG2) /* anon address aligned */ 96 /* 16 bytes */ 97 98 99 #ifdef _KERNEL 100 /* 101 * The swapinfo_lock protects: 102 * swapinfo list 103 * individual swapinfo structures 104 * 105 * The anoninfo_lock protects: 106 * anoninfo counters 107 * 108 * The anonhash_lock protects: 109 * anon hash lists 110 * anon slot fields 111 * 112 * Fields in the anon slot which are read-only for the life of the slot 113 * (an_vp, an_off) do not require the anonhash_lock be held to access them. 114 * If you access a field without the anonhash_lock held you must be holding 115 * the slot with an_refcnt to make sure it isn't destroyed. 116 * To write (an_pvp, an_poff) in a given slot you must also hold the 117 * p_iolock of the anonymous page for slot. 118 */ 119 extern kmutex_t anoninfo_lock; 120 extern kmutex_t swapinfo_lock; 121 extern pad_mutex_t *anonhash_lock; 122 extern pad_mutex_t anon_array_lock[]; 123 extern kcondvar_t anon_array_cv[]; 124 125 /* 126 * Global hash table to provide a function from (vp, off) -> ap 127 */ 128 extern size_t anon_hash_size; 129 extern unsigned int anon_hash_shift; 130 extern struct anon **anon_hash; 131 #define ANON_HASH_SIZE anon_hash_size 132 #define ANON_HASHAVELEN 4 133 /* 134 * Try to use as many bits of randomness from both vp and off as we can. 135 * This should help spreading evenly for a variety of workloads. See comments 136 * for PAGE_HASH_FUNC for more explanation. 137 */ 138 #define ANON_HASH(vp, off) \ 139 (((((uintptr_t)(off) >> PAGESHIFT) ^ \ 140 ((uintptr_t)(off) >> (PAGESHIFT + anon_hash_shift))) ^ \ 141 (((uintptr_t)(vp) >> 3) ^ \ 142 ((uintptr_t)(vp) >> (3 + anon_hash_shift)) ^ \ 143 ((uintptr_t)(vp) >> (3 + 2 * anon_hash_shift)) ^ \ 144 ((uintptr_t)(vp) << \ 145 (anon_hash_shift - AN_VPSHIFT - VNODE_ALIGN_LOG2)))) & \ 146 (anon_hash_size - 1)) 147 148 #define AH_LOCK_SIZE (2 << NCPU_LOG2) 149 150 #define AH_MUTEX(vp, off) \ 151 (&anonhash_lock[(ANON_HASH((vp), (off)) & \ 152 (AH_LOCK_SIZE - 1))].pad_mutex) 153 154 #endif /* _KERNEL */ 155 156 /* 157 * Declaration for the Global counters to accurately 158 * track the kernel foot print in memory. 159 */ 160 extern pgcnt_t pages_locked; 161 extern pgcnt_t pages_claimed; 162 extern pgcnt_t pages_useclaim; 163 extern pgcnt_t obp_pages; 164 165 /* 166 * Anonymous backing store accounting structure for swapctl. 167 * 168 * ani_max = maximum amount of swap space 169 * (including potentially available physical memory) 170 * ani_free = amount of unallocated anonymous memory 171 * (some of which might be reserved and including 172 * potentially available physical memory) 173 * ani_resv = amount of claimed (reserved) anonymous memory 174 * 175 * The swap data can be aquired more efficiently through the 176 * kstats interface. 177 * Total slots currently available for reservation = 178 * MAX(ani_max - ani_resv, 0) + (availrmem - swapfs_minfree) 179 */ 180 struct anoninfo { 181 pgcnt_t ani_max; 182 pgcnt_t ani_free; 183 pgcnt_t ani_resv; 184 }; 185 186 #ifdef _SYSCALL32 187 struct anoninfo32 { 188 size32_t ani_max; 189 size32_t ani_free; 190 size32_t ani_resv; 191 }; 192 #endif /* _SYSCALL32 */ 193 194 /* 195 * Define the NCPU pool of the ani_free counters. Update the counter 196 * of the cpu on which the thread is running and in every clock intr 197 * sync anoninfo.ani_free with the current total off all the NCPU entries. 198 */ 199 200 typedef struct ani_free { 201 pgcnt_t ani_count; 202 uchar_t pad[64 - sizeof (pgcnt_t)]; 203 /* XXX 64 = cacheline size */ 204 } ani_free_t; 205 206 #define ANI_MAX_POOL (NCPU_P2) 207 extern ani_free_t *ani_free_pool; 208 209 /* 210 * Since each CPU has its own bucket in ani_free_pool, there should be no 211 * contention here. 212 */ 213 #define ANI_ADD(inc) { \ 214 pgcnt_t *ani_countp; \ 215 int index; \ 216 index = (CPU->cpu_seqid & (ANI_MAX_POOL - 1)); \ 217 ani_countp = &ani_free_pool[index].ani_count; \ 218 atomic_add_long(ani_countp, inc); \ 219 } 220 221 extern void set_anoninfo(void); 222 223 /* 224 * Anon array pointers are allocated in chunks. Each chunk 225 * has PAGESIZE/sizeof(u_long *) of anon pointers. 226 * There are two levels of arrays for anon array pointers larger 227 * than a chunk. The first level points to anon array chunks. 228 * The second level consists of chunks of anon pointers. 229 * 230 * If anon array is smaller than a chunk then the whole anon array 231 * is created (memory is allocated for whole anon array). 232 * If anon array is larger than a chunk only first level array is 233 * allocated. Then other arrays (chunks) are allocated only when 234 * they are initialized with anon pointers. 235 */ 236 struct anon_hdr { 237 kmutex_t serial_lock; /* serialize array chunk allocation */ 238 pgcnt_t size; /* number of pointers to (anon) pages */ 239 void **array_chunk; /* pointers to anon pointers or chunks of */ 240 /* anon pointers */ 241 int flags; /* ANON_ALLOC_FORCE force preallocation of */ 242 /* whole anon array */ 243 }; 244 245 #ifdef _LP64 246 #define ANON_PTRSHIFT 3 247 #define ANON_PTRMASK ~7 248 #else 249 #define ANON_PTRSHIFT 2 250 #define ANON_PTRMASK ~3 251 #endif 252 253 #define ANON_CHUNK_SIZE (PAGESIZE >> ANON_PTRSHIFT) 254 #define ANON_CHUNK_SHIFT (PAGESHIFT - ANON_PTRSHIFT) 255 #define ANON_CHUNK_OFF (ANON_CHUNK_SIZE - 1) 256 257 /* 258 * Anon flags. 259 */ 260 #define ANON_SLEEP 0x0 /* ok to block */ 261 #define ANON_NOSLEEP 0x1 /* non-blocking call */ 262 #define ANON_ALLOC_FORCE 0x2 /* force single level anon array */ 263 #define ANON_GROWDOWN 0x4 /* anon array should grow downward */ 264 265 struct kshmid; 266 267 /* 268 * The anon_map structure is used by various clients of the anon layer to 269 * manage anonymous memory. When anonymous memory is shared, 270 * then the different clients sharing it will point to the 271 * same anon_map structure. Also, if a segment is unmapped 272 * in the middle where an anon_map structure exists, the 273 * newly created segment will also share the anon_map structure, 274 * although the two segments will use different ranges of the 275 * anon array. When mappings are private (or shared with 276 * a reference count of 1), an unmap operation will free up 277 * a range of anon slots in the array given by the anon_map 278 * structure. Because of fragmentation due to this unmapping, 279 * we have to store the size of the anon array in the anon_map 280 * structure so that we can free everything when the referernce 281 * count goes to zero. 282 * 283 * A new rangelock scheme is introduced to make the anon layer scale. 284 * A reader/writer lock per anon_amp and an array of system-wide hash 285 * locks, anon_array_lock[] are introduced to replace serial_lock and 286 * anonmap lock. The writer lock is held when we want to singlethreaD 287 * the reference to the anon array pointers or when references to 288 * anon_map's members, whereas reader lock and anon_array_lock are 289 * held to allows multiple threads to reference different part of 290 * anon array. A global set of condition variables, anon_array_cv, 291 * are used with anon_array_lock[] to make the hold time of the locks 292 * short. 293 * 294 * szc is used to calculate the index of hash locks and cv's. We 295 * could've just used seg->s_szc if not for the possible sharing of 296 * anon_amp between SYSV shared memory and ISM, so now we introduce 297 * szc in the anon_map structure. For MAP_SHARED, the amp->szc is either 298 * 0 (base page size) or page_num_pagesizes() - 1, while MAP_PRIVATE 299 * the amp->szc could be anything in [0, page_num_pagesizes() - 1]. 300 */ 301 typedef struct anon_map { 302 krwlock_t a_rwlock; /* protect anon_map and anon array */ 303 size_t size; /* size in bytes mapped by the anon array */ 304 struct anon_hdr *ahp; /* anon array header pointer, containing */ 305 /* anon pointer array(s) */ 306 size_t swresv; /* swap space reserved for this anon_map */ 307 ulong_t refcnt; /* reference count on this structure */ 308 ushort_t a_szc; /* max szc among shared processes */ 309 void *locality; /* lgroup locality info */ 310 struct kshmid *a_sp; /* kshmid if amp backs sysV, or NULL */ 311 int a_purgewait; /* somebody waits for slocks to go away */ 312 kcondvar_t a_purgecv; /* cv for waiting for slocks to go away */ 313 kmutex_t a_purgemtx; /* mutex for anonmap_purge() */ 314 spgcnt_t a_softlockcnt; /* number of pages locked in pcache */ 315 kmutex_t a_pmtx; /* protects amp's pcache list */ 316 pcache_link_t a_phead; /* head of amp's pcache list */ 317 } amp_t; 318 319 #ifdef _KERNEL 320 321 #define ANON_BUSY 0x1 322 #define ANON_ISBUSY(slot) (*(slot) & ANON_BUSY) 323 #define ANON_SETBUSY(slot) (*(slot) |= ANON_BUSY) 324 #define ANON_CLRBUSY(slot) (*(slot) &= ~ANON_BUSY) 325 326 #define ANON_MAP_SHIFT 6 /* log2(sizeof (struct anon_map)) */ 327 #define ANON_ARRAY_SHIFT 7 /* log2(ANON_LOCKSIZE) */ 328 #define ANON_LOCKSIZE 128 329 330 #define ANON_LOCK_ENTER(lock, type) rw_enter((lock), (type)) 331 #define ANON_LOCK_EXIT(lock) rw_exit((lock)) 332 #define ANON_LOCK_HELD(lock) RW_LOCK_HELD((lock)) 333 #define ANON_READ_HELD(lock) RW_READ_HELD((lock)) 334 #define ANON_WRITE_HELD(lock) RW_WRITE_HELD((lock)) 335 336 #define ANON_ARRAY_HASH(amp, idx)\ 337 ((((idx) + ((idx) >> ANON_ARRAY_SHIFT) +\ 338 ((idx) >> (ANON_ARRAY_SHIFT << 1)) +\ 339 ((idx) >> (ANON_ARRAY_SHIFT + (ANON_ARRAY_SHIFT << 1)))) ^\ 340 ((uintptr_t)(amp) >> ANON_MAP_SHIFT)) & (ANON_LOCKSIZE - 1)) 341 342 typedef struct anon_sync_obj { 343 kmutex_t *sync_mutex; 344 kcondvar_t *sync_cv; 345 ulong_t *sync_data; 346 } anon_sync_obj_t; 347 348 /* 349 * Anonymous backing store accounting structure for kernel. 350 * ani_max = total reservable slots on physical (disk-backed) swap 351 * ani_phys_resv = total phys slots reserved for use by clients 352 * ani_mem_resv = total mem slots reserved for use by clients 353 * ani_free = # unallocated physical slots + # of reserved unallocated 354 * memory slots 355 */ 356 357 /* 358 * Initial total swap slots available for reservation 359 */ 360 #define TOTAL_AVAILABLE_SWAP \ 361 (k_anoninfo.ani_max + MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 362 363 /* 364 * Swap slots currently available for reservation 365 */ 366 #define CURRENT_TOTAL_AVAILABLE_SWAP \ 367 ((k_anoninfo.ani_max - k_anoninfo.ani_phys_resv) + \ 368 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 369 370 struct k_anoninfo { 371 pgcnt_t ani_max; /* total reservable slots on phys */ 372 /* (disk) swap */ 373 pgcnt_t ani_free; /* # of unallocated phys and mem slots */ 374 pgcnt_t ani_phys_resv; /* # of reserved phys (disk) slots */ 375 pgcnt_t ani_mem_resv; /* # of reserved mem slots */ 376 pgcnt_t ani_locked_swap; /* # of swap slots locked in reserved */ 377 /* mem swap */ 378 }; 379 380 extern struct k_anoninfo k_anoninfo; 381 382 extern void anon_init(void); 383 extern struct anon *anon_alloc(struct vnode *, anoff_t); 384 extern void anon_dup(struct anon_hdr *, ulong_t, 385 struct anon_hdr *, ulong_t, size_t); 386 extern void anon_dup_fill_holes(struct anon_hdr *, ulong_t, 387 struct anon_hdr *, ulong_t, size_t, uint_t, int); 388 extern int anon_fill_cow_holes(struct seg *, caddr_t, struct anon_hdr *, 389 ulong_t, struct vnode *, u_offset_t, size_t, uint_t, 390 uint_t, struct vpage [], struct cred *); 391 extern void anon_free(struct anon_hdr *, ulong_t, size_t); 392 extern void anon_free_pages(struct anon_hdr *, ulong_t, size_t, uint_t); 393 extern void anon_disclaim(struct anon_map *, ulong_t, size_t); 394 extern int anon_getpage(struct anon **, uint_t *, struct page **, 395 size_t, struct seg *, caddr_t, enum seg_rw, struct cred *); 396 extern int swap_getconpage(struct vnode *, u_offset_t, size_t, 397 uint_t *, page_t *[], size_t, page_t *, uint_t *, 398 spgcnt_t *, struct seg *, caddr_t, 399 enum seg_rw, struct cred *); 400 extern int anon_map_getpages(struct anon_map *, ulong_t, 401 uint_t, struct seg *, caddr_t, uint_t, 402 uint_t *, page_t *[], uint_t *, 403 struct vpage [], enum seg_rw, int, int, int, struct cred *); 404 extern int anon_map_privatepages(struct anon_map *, ulong_t, 405 uint_t, struct seg *, caddr_t, uint_t, 406 page_t *[], struct vpage [], int, int, struct cred *); 407 extern struct page *anon_private(struct anon **, struct seg *, 408 caddr_t, uint_t, struct page *, 409 int, struct cred *); 410 extern struct page *anon_zero(struct seg *, caddr_t, 411 struct anon **, struct cred *); 412 extern int anon_map_createpages(struct anon_map *, ulong_t, 413 size_t, struct page **, 414 struct seg *, caddr_t, 415 enum seg_rw, struct cred *); 416 extern int anon_map_demotepages(struct anon_map *, ulong_t, 417 struct seg *, caddr_t, uint_t, 418 struct vpage [], struct cred *); 419 extern void anon_shmap_free_pages(struct anon_map *, ulong_t, size_t); 420 extern int anon_resvmem(size_t, boolean_t, zone_t *, int); 421 extern void anon_unresvmem(size_t, zone_t *); 422 extern struct anon_map *anonmap_alloc(size_t, size_t, int); 423 extern void anonmap_free(struct anon_map *); 424 extern void anonmap_purge(struct anon_map *); 425 extern void anon_swap_free(struct anon *, struct page *); 426 extern void anon_decref(struct anon *); 427 extern int non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *); 428 extern pgcnt_t anon_pages(struct anon_hdr *, ulong_t, pgcnt_t); 429 extern int anon_swap_adjust(pgcnt_t); 430 extern void anon_swap_restore(pgcnt_t); 431 extern struct anon_hdr *anon_create(pgcnt_t, int); 432 extern void anon_release(struct anon_hdr *, pgcnt_t); 433 extern struct anon *anon_get_ptr(struct anon_hdr *, ulong_t); 434 extern ulong_t *anon_get_slot(struct anon_hdr *, ulong_t); 435 extern struct anon *anon_get_next_ptr(struct anon_hdr *, ulong_t *); 436 extern int anon_set_ptr(struct anon_hdr *, ulong_t, struct anon *, int); 437 extern int anon_copy_ptr(struct anon_hdr *, ulong_t, 438 struct anon_hdr *, ulong_t, pgcnt_t, int); 439 extern pgcnt_t anon_grow(struct anon_hdr *, ulong_t *, pgcnt_t, pgcnt_t, int); 440 extern void anon_array_enter(struct anon_map *, ulong_t, 441 anon_sync_obj_t *); 442 extern int anon_array_try_enter(struct anon_map *, ulong_t, 443 anon_sync_obj_t *); 444 extern void anon_array_exit(anon_sync_obj_t *); 445 446 /* 447 * anon_resv checks to see if there is enough swap space to fulfill a 448 * request and if so, reserves the appropriate anonymous memory resources. 449 * anon_checkspace just checks to see if there is space to fulfill the request, 450 * without taking any resources. Both return 1 if successful and 0 if not. 451 * 452 * Macros are provided as anon reservation is usually charged to the zone of 453 * the current process. In some cases (such as anon reserved by tmpfs), a 454 * zone pointer is needed to charge the appropriate zone. 455 */ 456 #define anon_unresv(size) anon_unresvmem(size, curproc->p_zone) 457 #define anon_unresv_zone(size, zone) anon_unresvmem(size, zone) 458 #define anon_resv(size) \ 459 anon_resvmem((size), 1, curproc->p_zone, 1) 460 #define anon_resv_zone(size, zone) anon_resvmem((size), 1, zone, 1) 461 #define anon_checkspace(size, zone) anon_resvmem((size), 0, zone, 0) 462 #define anon_try_resv_zone(size, zone) anon_resvmem((size), 1, zone, 0) 463 464 /* 465 * Flags to anon_private 466 */ 467 #define STEAL_PAGE 0x1 /* page can be stolen */ 468 #define LOCK_PAGE 0x2 /* page must be ``logically'' locked */ 469 470 /* 471 * SEGKP ANON pages that are locked are assumed to be LWP stack pages 472 * and thus count towards the user pages locked count. 473 * This value is protected by the same lock as availrmem. 474 */ 475 extern pgcnt_t anon_segkp_pages_locked; 476 477 extern int anon_debug; 478 479 #ifdef ANON_DEBUG 480 481 #define A_ANON 0x01 482 #define A_RESV 0x02 483 #define A_MRESV 0x04 484 485 /* vararg-like debugging macro. */ 486 #define ANON_PRINT(f, printf_args) \ 487 if (anon_debug & f) \ 488 printf printf_args 489 490 #else /* ANON_DEBUG */ 491 492 #define ANON_PRINT(f, printf_args) 493 494 #endif /* ANON_DEBUG */ 495 496 #endif /* _KERNEL */ 497 498 #ifdef __cplusplus 499 } 500 #endif 501 502 #endif /* _VM_ANON_H */ 503