1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 /* 31 * University Copyright- Copyright (c) 1982, 1986, 1988 32 * The Regents of the University of California 33 * All Rights Reserved 34 * 35 * University Acknowledgment- Portions of this document are derived from 36 * software developed by the University of California, Berkeley, and its 37 * contributors. 38 */ 39 40 #ifndef _VM_ANON_H 41 #define _VM_ANON_H 42 43 #pragma ident "%Z%%M% %I% %E% SMI" 44 45 #include <sys/cred.h> 46 #include <vm/seg.h> 47 #include <vm/vpage.h> 48 49 #ifdef __cplusplus 50 extern "C" { 51 #endif 52 53 /* 54 * VM - Anonymous pages. 55 */ 56 57 typedef unsigned long anoff_t; /* anon offsets */ 58 59 /* 60 * Each anonymous page, either in memory or in swap, has an anon structure. 61 * The structure (slot) provides a level of indirection between anonymous pages 62 * and their backing store. 63 * 64 * (an_vp, an_off) names the vnode of the anonymous page for this slot. 65 * 66 * (an_pvp, an_poff) names the location of the physical backing store 67 * for the page this slot represents. If the name is null there is no 68 * associated physical store. The physical backing store location can 69 * change while the slot is in use. 70 * 71 * an_hash is a hash list of anon slots. The list is hashed by 72 * (an_vp, an_off) of the associated anonymous page and provides a 73 * method of going from the name of an anonymous page to its 74 * associated anon slot. 75 * 76 * an_refcnt holds a reference count which is the number of separate 77 * copies that will need to be created in case of copy-on-write. 78 * A refcnt > 0 protects the existence of the slot. The refcnt is 79 * initialized to 1 when the anon slot is created in anon_alloc(). 80 * If a client obtains an anon slot and allows multiple threads to 81 * share it, then it is the client's responsibility to insure that 82 * it does not allow one thread to try to reference the slot at the 83 * same time as another is trying to decrement the last count and 84 * destroy the anon slot. E.g., the seg_vn segment type protects 85 * against this with higher level locks. 86 */ 87 88 struct anon { 89 struct vnode *an_vp; /* vnode of anon page */ 90 struct vnode *an_pvp; /* vnode of physical backing store */ 91 anoff_t an_off; /* offset of anon page */ 92 anoff_t an_poff; /* offset in vnode */ 93 struct anon *an_hash; /* hash table of anon slots */ 94 int an_refcnt; /* # of people sharing slot */ 95 }; 96 97 #ifdef _KERNEL 98 /* 99 * The swapinfo_lock protects: 100 * swapinfo list 101 * individual swapinfo structures 102 * 103 * The anoninfo_lock protects: 104 * anoninfo counters 105 * 106 * The anonhash_lock protects: 107 * anon hash lists 108 * anon slot fields 109 * 110 * Fields in the anon slot which are read-only for the life of the slot 111 * (an_vp, an_off) do not require the anonhash_lock be held to access them. 112 * If you access a field without the anonhash_lock held you must be holding 113 * the slot with an_refcnt to make sure it isn't destroyed. 114 * To write (an_pvp, an_poff) in a given slot you must also hold the 115 * p_iolock of the anonymous page for slot. 116 */ 117 extern kmutex_t anoninfo_lock; 118 extern kmutex_t swapinfo_lock; 119 extern kmutex_t anonhash_lock[]; 120 extern pad_mutex_t anon_array_lock[]; 121 extern kcondvar_t anon_array_cv[]; 122 123 /* 124 * Global hash table to provide a function from (vp, off) -> ap 125 */ 126 extern size_t anon_hash_size; 127 extern struct anon **anon_hash; 128 #define ANON_HASH_SIZE anon_hash_size 129 #define ANON_HASHAVELEN 4 130 #define ANON_HASH(VP, OFF) \ 131 ((((uintptr_t)(VP) >> 7) ^ ((OFF) >> PAGESHIFT)) & (ANON_HASH_SIZE - 1)) 132 133 #define AH_LOCK_SIZE 64 134 #define AH_LOCK(vp, off) (ANON_HASH((vp), (off)) & (AH_LOCK_SIZE -1)) 135 136 #endif /* _KERNEL */ 137 138 /* 139 * Declaration for the Global counters to accurately 140 * track the kernel foot print in memory. 141 */ 142 extern pgcnt_t segvn_pages_locked; 143 extern pgcnt_t pages_locked; 144 extern pgcnt_t pages_claimed; 145 extern pgcnt_t pages_useclaim; 146 extern pgcnt_t obp_pages; 147 148 /* 149 * Anonymous backing store accounting structure for swapctl. 150 * 151 * ani_max = maximum amount of swap space 152 * (including potentially available physical memory) 153 * ani_free = amount of unallocated anonymous memory 154 * (some of which might be reserved and including 155 * potentially available physical memory) 156 * ani_resv = amount of claimed (reserved) anonymous memory 157 * 158 * The swap data can be aquired more efficiently through the 159 * kstats interface. 160 * Total slots currently available for reservation = 161 * MAX(ani_max - ani_resv, 0) + (availrmem - swapfs_minfree) 162 */ 163 struct anoninfo { 164 pgcnt_t ani_max; 165 pgcnt_t ani_free; 166 pgcnt_t ani_resv; 167 }; 168 169 #ifdef _SYSCALL32 170 struct anoninfo32 { 171 size32_t ani_max; 172 size32_t ani_free; 173 size32_t ani_resv; 174 }; 175 #endif /* _SYSCALL32 */ 176 177 /* 178 * Define the NCPU pool of the ani_free counters. Update the counter 179 * of the cpu on which the thread is running and in every clock intr 180 * sync anoninfo.ani_free with the current total off all the NCPU entries. 181 */ 182 183 typedef struct ani_free { 184 kmutex_t ani_lock; 185 pgcnt_t ani_count; 186 uchar_t pad[64 - sizeof (kmutex_t) - sizeof (pgcnt_t)]; 187 /* XXX 64 = cacheline size */ 188 } ani_free_t; 189 190 #define ANI_MAX_POOL 128 191 extern ani_free_t ani_free_pool[]; 192 193 #define ANI_ADD(inc) { \ 194 ani_free_t *anifp; \ 195 int index; \ 196 index = (CPU->cpu_id & (ANI_MAX_POOL - 1)); \ 197 anifp = &ani_free_pool[index]; \ 198 mutex_enter(&anifp->ani_lock); \ 199 anifp->ani_count += inc; \ 200 mutex_exit(&anifp->ani_lock); \ 201 } 202 203 /* 204 * Anon array pointers are allocated in chunks. Each chunk 205 * has PAGESIZE/sizeof(u_long *) of anon pointers. 206 * There are two levels of arrays for anon array pointers larger 207 * than a chunk. The first level points to anon array chunks. 208 * The second level consists of chunks of anon pointers. 209 * 210 * If anon array is smaller than a chunk then the whole anon array 211 * is created (memory is allocated for whole anon array). 212 * If anon array is larger than a chunk only first level array is 213 * allocated. Then other arrays (chunks) are allocated only when 214 * they are initialized with anon pointers. 215 */ 216 struct anon_hdr { 217 kmutex_t serial_lock; /* serialize array chunk allocation */ 218 pgcnt_t size; /* number of pointers to (anon) pages */ 219 void **array_chunk; /* pointers to anon pointers or chunks of */ 220 /* anon pointers */ 221 int flags; /* ANON_ALLOC_FORCE force preallocation of */ 222 /* whole anon array */ 223 }; 224 225 #ifdef _LP64 226 #define ANON_PTRSHIFT 3 227 #define ANON_PTRMASK ~7 228 #else 229 #define ANON_PTRSHIFT 2 230 #define ANON_PTRMASK ~3 231 #endif 232 233 #define ANON_CHUNK_SIZE (PAGESIZE >> ANON_PTRSHIFT) 234 #define ANON_CHUNK_SHIFT (PAGESHIFT - ANON_PTRSHIFT) 235 #define ANON_CHUNK_OFF (ANON_CHUNK_SIZE - 1) 236 237 /* 238 * Anon flags. 239 */ 240 #define ANON_SLEEP 0x0 /* ok to block */ 241 #define ANON_NOSLEEP 0x1 /* non-blocking call */ 242 #define ANON_ALLOC_FORCE 0x2 /* force single level anon array */ 243 #define ANON_GROWDOWN 0x4 /* anon array should grow downward */ 244 245 /* 246 * The anon_map structure is used by various clients of the anon layer to 247 * manage anonymous memory. When anonymous memory is shared, 248 * then the different clients sharing it will point to the 249 * same anon_map structure. Also, if a segment is unmapped 250 * in the middle where an anon_map structure exists, the 251 * newly created segment will also share the anon_map structure, 252 * although the two segments will use different ranges of the 253 * anon array. When mappings are private (or shared with 254 * a reference count of 1), an unmap operation will free up 255 * a range of anon slots in the array given by the anon_map 256 * structure. Because of fragmentation due to this unmapping, 257 * we have to store the size of the anon array in the anon_map 258 * structure so that we can free everything when the referernce 259 * count goes to zero. 260 * 261 * A new rangelock scheme is introduced to make the anon layer scale. 262 * A reader/writer lock per anon_amp and an array of system-wide hash 263 * locks, anon_array_lock[] are introduced to replace serial_lock and 264 * anonmap lock. The writer lock is held when we want to singlethreaD 265 * the reference to the anon array pointers or when references to 266 * anon_map's members, whereas reader lock and anon_array_lock are 267 * held to allows multiple threads to reference different part of 268 * anon array. A global set of condition variables, anon_array_cv, 269 * are used with anon_array_lock[] to make the hold time of the locks 270 * short. 271 * 272 * szc is used to calculate the index of hash locks and cv's. We 273 * could've just used seg->s_szc if not for the possible sharing of 274 * anon_amp between SYSV shared memory and ISM, so now we introduce 275 * szc in the anon_map structure. For MAP_SHARED, the amp->szc is either 276 * 0 (base page size) or page_num_pagesizes() - 1, while MAP_PRIVATE 277 * the amp->szc could be anything in [0, page_num_pagesizes() - 1]. 278 */ 279 struct anon_map { 280 krwlock_t a_rwlock; /* protect anon_map and anon array */ 281 size_t size; /* size in bytes mapped by the anon array */ 282 struct anon_hdr *ahp; /* anon array header pointer, containing */ 283 /* anon pointer array(s) */ 284 size_t swresv; /* swap space reserved for this anon_map */ 285 uint_t refcnt; /* reference count on this structure */ 286 ushort_t a_szc; /* max szc among shared processes */ 287 void *locality; /* lgroup locality info */ 288 }; 289 290 #ifdef _KERNEL 291 292 #define ANON_BUSY 0x1 293 #define ANON_ISBUSY(slot) (*(slot) & ANON_BUSY) 294 #define ANON_SETBUSY(slot) (*(slot) |= ANON_BUSY) 295 #define ANON_CLRBUSY(slot) (*(slot) &= ~ANON_BUSY) 296 297 #define ANON_MAP_SHIFT 6 /* log2(sizeof (struct anon_map)) */ 298 #define ANON_ARRAY_SHIFT 7 /* log2(ANON_LOCKSIZE) */ 299 #define ANON_LOCKSIZE 128 300 301 #define ANON_LOCK_ENTER(lock, type) rw_enter((lock), (type)) 302 #define ANON_LOCK_EXIT(lock) rw_exit((lock)) 303 304 #define ANON_ARRAY_HASH(amp, idx)\ 305 ((((idx) + ((idx) >> ANON_ARRAY_SHIFT) +\ 306 ((idx) >> (ANON_ARRAY_SHIFT << 1)) +\ 307 ((idx) >> (ANON_ARRAY_SHIFT + (ANON_ARRAY_SHIFT << 1)))) ^\ 308 ((uintptr_t)(amp) >> ANON_MAP_SHIFT)) & (ANON_LOCKSIZE - 1)) 309 310 typedef struct anon_sync_obj { 311 kmutex_t *sync_mutex; 312 kcondvar_t *sync_cv; 313 ulong_t *sync_data; 314 } anon_sync_obj_t; 315 316 /* 317 * Anonymous backing store accounting structure for kernel. 318 * ani_max = total reservable slots on physical (disk-backed) swap 319 * ani_phys_resv = total phys slots reserved for use by clients 320 * ani_mem_resv = total mem slots reserved for use by clients 321 * ani_free = # unallocated physical slots + # of reserved unallocated 322 * memory slots 323 */ 324 325 /* 326 * Initial total swap slots available for reservation 327 */ 328 #define TOTAL_AVAILABLE_SWAP \ 329 (k_anoninfo.ani_max + MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 330 331 /* 332 * Swap slots currently available for reservation 333 */ 334 #define CURRENT_TOTAL_AVAILABLE_SWAP \ 335 ((k_anoninfo.ani_max - k_anoninfo.ani_phys_resv) + \ 336 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 337 338 struct k_anoninfo { 339 pgcnt_t ani_max; /* total reservable slots on phys */ 340 /* (disk) swap */ 341 pgcnt_t ani_free; /* # of unallocated phys and mem slots */ 342 pgcnt_t ani_phys_resv; /* # of reserved phys (disk) slots */ 343 pgcnt_t ani_mem_resv; /* # of reserved mem slots */ 344 pgcnt_t ani_locked_swap; /* # of swap slots locked in reserved */ 345 /* mem swap */ 346 }; 347 348 extern struct k_anoninfo k_anoninfo; 349 350 extern void anon_init(void); 351 extern struct anon *anon_alloc(struct vnode *, anoff_t); 352 extern void anon_dup(struct anon_hdr *, ulong_t, 353 struct anon_hdr *, ulong_t, size_t); 354 extern void anon_dup_fill_holes(struct anon_hdr *, ulong_t, 355 struct anon_hdr *, ulong_t, size_t, uint_t, int); 356 extern int anon_fill_cow_holes(struct seg *, caddr_t, struct anon_hdr *, 357 ulong_t, struct vnode *, u_offset_t, size_t, uint_t, 358 uint_t, struct vpage [], struct cred *); 359 extern void anon_free(struct anon_hdr *, ulong_t, size_t); 360 extern void anon_free_pages(struct anon_hdr *, ulong_t, size_t, uint_t); 361 extern void anon_disclaim(struct anon_map *, ulong_t, size_t, int); 362 extern int anon_getpage(struct anon **, uint_t *, struct page **, 363 size_t, struct seg *, caddr_t, enum seg_rw, struct cred *); 364 extern int swap_getconpage(struct vnode *, u_offset_t, size_t, 365 uint_t *, page_t *[], size_t, page_t *, 366 spgcnt_t *, struct seg *, caddr_t, 367 enum seg_rw, struct cred *); 368 extern int anon_map_getpages(struct anon_map *, ulong_t, 369 uint_t, struct seg *, caddr_t, uint_t, 370 uint_t *, page_t *[], uint_t *, 371 struct vpage [], enum seg_rw, int, int, struct cred *); 372 extern int anon_map_privatepages(struct anon_map *, ulong_t, 373 uint_t, struct seg *, caddr_t, uint_t, 374 page_t *[], struct vpage [], int, struct cred *); 375 extern struct page *anon_private(struct anon **, struct seg *, 376 caddr_t, uint_t, struct page *, 377 int, struct cred *); 378 extern struct page *anon_zero(struct seg *, caddr_t, 379 struct anon **, struct cred *); 380 extern int anon_map_createpages(struct anon_map *, ulong_t, 381 size_t, struct page **, 382 struct seg *, caddr_t, 383 enum seg_rw, struct cred *); 384 extern int anon_map_demotepages(struct anon_map *, ulong_t, 385 struct seg *, caddr_t, uint_t, 386 struct vpage [], struct cred *); 387 extern int anon_resvmem(size_t, uint_t); 388 extern void anon_unresv(size_t); 389 extern struct anon_map *anonmap_alloc(size_t, size_t); 390 extern void anonmap_free(struct anon_map *); 391 extern void anon_decref(struct anon *); 392 extern int non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *); 393 extern pgcnt_t anon_pages(struct anon_hdr *, ulong_t, pgcnt_t); 394 extern int anon_swap_adjust(pgcnt_t); 395 extern void anon_swap_restore(pgcnt_t); 396 extern struct anon_hdr *anon_create(pgcnt_t, int); 397 extern void anon_release(struct anon_hdr *, pgcnt_t); 398 extern struct anon *anon_get_ptr(struct anon_hdr *, ulong_t); 399 extern ulong_t *anon_get_slot(struct anon_hdr *, ulong_t); 400 extern struct anon *anon_get_next_ptr(struct anon_hdr *, ulong_t *); 401 extern int anon_set_ptr(struct anon_hdr *, ulong_t, struct anon *, int); 402 extern int anon_copy_ptr(struct anon_hdr *, ulong_t, 403 struct anon_hdr *, ulong_t, pgcnt_t, int); 404 extern pgcnt_t anon_grow(struct anon_hdr *, ulong_t *, pgcnt_t, pgcnt_t, int); 405 extern void anon_array_enter(struct anon_map *, ulong_t, 406 anon_sync_obj_t *); 407 extern int anon_array_try_enter(struct anon_map *, ulong_t, 408 anon_sync_obj_t *); 409 extern void anon_array_exit(anon_sync_obj_t *); 410 411 /* 412 * anon_resv checks to see if there is enough swap space to fulfill a 413 * request and if so, reserves the appropriate anonymous memory resources. 414 * anon_checkspace just checks to see if there is space to fulfill the request, 415 * without taking any resources. Both return 1 if successful and 0 if not. 416 */ 417 #define anon_resv(size) anon_resvmem((size), 1) 418 #define anon_checkspace(size) anon_resvmem((size), 0) 419 420 /* 421 * Flags to anon_private 422 */ 423 #define STEAL_PAGE 0x1 /* page can be stolen */ 424 #define LOCK_PAGE 0x2 /* page must be ``logically'' locked */ 425 426 /* 427 * Flags to anon_disclaim 428 */ 429 #define ANON_PGLOOKUP_BLK 0x1 /* block on locked pages */ 430 431 /* 432 * SEGKP ANON pages that are locked are assumed to be LWP stack pages 433 * and thus count towards the user pages locked count. 434 * This value is protected by the same lock as availrmem. 435 */ 436 extern pgcnt_t anon_segkp_pages_locked; 437 438 extern int anon_debug; 439 440 #ifdef ANON_DEBUG 441 442 #define A_ANON 0x01 443 #define A_RESV 0x02 444 #define A_MRESV 0x04 445 446 /* vararg-like debugging macro. */ 447 #define ANON_PRINT(f, printf_args) \ 448 if (anon_debug & f) \ 449 printf printf_args 450 451 #else /* ANON_DEBUG */ 452 453 #define ANON_PRINT(f, printf_args) 454 455 #endif /* ANON_DEBUG */ 456 457 #endif /* _KERNEL */ 458 459 #ifdef __cplusplus 460 } 461 #endif 462 463 #endif /* _VM_ANON_H */ 464