1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * DVA-based Adjustable Relpacement Cache 30 * 31 * While much of the theory of operation used here is 32 * based on the self-tuning, low overhead replacement cache 33 * presented by Megiddo and Modha at FAST 2003, there are some 34 * significant differences: 35 * 36 * 1. The Megiddo and Modha model assumes any page is evictable. 37 * Pages in its cache cannot be "locked" into memory. This makes 38 * the eviction algorithm simple: evict the last page in the list. 39 * This also make the performance characteristics easy to reason 40 * about. Our cache is not so simple. At any given moment, some 41 * subset of the blocks in the cache are un-evictable because we 42 * have handed out a reference to them. Blocks are only evictable 43 * when there are no external references active. This makes 44 * eviction far more problematic: we choose to evict the evictable 45 * blocks that are the "lowest" in the list. 46 * 47 * There are times when it is not possible to evict the requested 48 * space. In these circumstances we are unable to adjust the cache 49 * size. To prevent the cache growing unbounded at these times we 50 * implement a "cache throttle" that slowes the flow of new data 51 * into the cache until we can make space avaiable. 52 * 53 * 2. The Megiddo and Modha model assumes a fixed cache size. 54 * Pages are evicted when the cache is full and there is a cache 55 * miss. Our model has a variable sized cache. It grows with 56 * high use, but also tries to react to memory preasure from the 57 * operating system: decreasing its size when system memory is 58 * tight. 59 * 60 * 3. The Megiddo and Modha model assumes a fixed page size. All 61 * elements of the cache are therefor exactly the same size. So 62 * when adjusting the cache size following a cache miss, its simply 63 * a matter of choosing a single page to evict. In our model, we 64 * have variable sized cache blocks (rangeing from 512 bytes to 65 * 128K bytes). We therefor choose a set of blocks to evict to make 66 * space for a cache miss that approximates as closely as possible 67 * the space used by the new block. 68 * 69 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70 * by N. Megiddo & D. Modha, FAST 2003 71 */ 72 73 /* 74 * The locking model: 75 * 76 * A new reference to a cache buffer can be obtained in two 77 * ways: 1) via a hash table lookup using the DVA as a key, 78 * or 2) via one of the ARC lists. The arc_read() inerface 79 * uses method 1, while the internal arc algorithms for 80 * adjusting the cache use method 2. We therefor provide two 81 * types of locks: 1) the hash table lock array, and 2) the 82 * arc list locks. 83 * 84 * Buffers do not have their own mutexs, rather they rely on the 85 * hash table mutexs for the bulk of their protection (i.e. most 86 * fields in the arc_buf_hdr_t are protected by these mutexs). 87 * 88 * buf_hash_find() returns the appropriate mutex (held) when it 89 * locates the requested buffer in the hash table. It returns 90 * NULL for the mutex if the buffer was not in the table. 91 * 92 * buf_hash_remove() expects the appropriate hash mutex to be 93 * already held before it is invoked. 94 * 95 * Each arc state also has a mutex which is used to protect the 96 * buffer list associated with the state. When attempting to 97 * obtain a hash table lock while holding an arc list lock you 98 * must use: mutex_tryenter() to avoid deadlock. Also note that 99 * the active state mutex must be held before the ghost state mutex. 100 * 101 * Arc buffers may have an associated eviction callback function. 102 * This function will be invoked prior to removing the buffer (e.g. 103 * in arc_do_user_evicts()). Note however that the data associated 104 * with the buffer may be evicted prior to the callback. The callback 105 * must be made with *no locks held* (to prevent deadlock). Additionally, 106 * the users of callbacks must ensure that their private data is 107 * protected from simultaneous callbacks from arc_buf_evict() 108 * and arc_do_user_evicts(). 109 * 110 * Note that the majority of the performance stats are manipulated 111 * with atomic operations. 112 */ 113 114 #include <sys/spa.h> 115 #include <sys/zio.h> 116 #include <sys/zfs_context.h> 117 #include <sys/arc.h> 118 #include <sys/refcount.h> 119 #ifdef _KERNEL 120 #include <sys/vmsystm.h> 121 #include <vm/anon.h> 122 #include <sys/fs/swapnode.h> 123 #include <sys/dnlc.h> 124 #endif 125 #include <sys/callb.h> 126 127 static kmutex_t arc_reclaim_thr_lock; 128 static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 129 static uint8_t arc_thread_exit; 130 131 #define ARC_REDUCE_DNLC_PERCENT 3 132 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 133 134 typedef enum arc_reclaim_strategy { 135 ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 136 ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 137 } arc_reclaim_strategy_t; 138 139 /* number of seconds before growing cache again */ 140 static int arc_grow_retry = 60; 141 142 /* 143 * minimum lifespan of a prefetch block in clock ticks 144 * (initialized in arc_init()) 145 */ 146 static int arc_min_prefetch_lifespan; 147 148 static kmutex_t arc_reclaim_lock; 149 static int arc_dead; 150 151 /* 152 * Note that buffers can be on one of 5 states: 153 * ARC_anon - anonymous (discussed below) 154 * ARC_mru - recently used, currently cached 155 * ARC_mru_ghost - recentely used, no longer in cache 156 * ARC_mfu - frequently used, currently cached 157 * ARC_mfu_ghost - frequently used, no longer in cache 158 * When there are no active references to the buffer, they 159 * are linked onto one of the lists in arc. These are the 160 * only buffers that can be evicted or deleted. 161 * 162 * Anonymous buffers are buffers that are not associated with 163 * a DVA. These are buffers that hold dirty block copies 164 * before they are written to stable storage. By definition, 165 * they are "ref'd" and are considered part of arc_mru 166 * that cannot be freed. Generally, they will aquire a DVA 167 * as they are written and migrate onto the arc_mru list. 168 */ 169 170 typedef struct arc_state { 171 list_t list; /* linked list of evictable buffer in state */ 172 uint64_t lsize; /* total size of buffers in the linked list */ 173 uint64_t size; /* total size of all buffers in this state */ 174 uint64_t hits; 175 kmutex_t mtx; 176 } arc_state_t; 177 178 /* The 5 states: */ 179 static arc_state_t ARC_anon; 180 static arc_state_t ARC_mru; 181 static arc_state_t ARC_mru_ghost; 182 static arc_state_t ARC_mfu; 183 static arc_state_t ARC_mfu_ghost; 184 185 static struct arc { 186 arc_state_t *anon; 187 arc_state_t *mru; 188 arc_state_t *mru_ghost; 189 arc_state_t *mfu; 190 arc_state_t *mfu_ghost; 191 uint64_t size; /* Actual total arc size */ 192 uint64_t p; /* Target size (in bytes) of mru */ 193 uint64_t c; /* Target size of cache (in bytes) */ 194 uint64_t c_min; /* Minimum target cache size */ 195 uint64_t c_max; /* Maximum target cache size */ 196 197 /* performance stats */ 198 uint64_t hits; 199 uint64_t misses; 200 uint64_t deleted; 201 uint64_t recycle_miss; 202 uint64_t mutex_miss; 203 uint64_t evict_skip; 204 uint64_t hash_elements; 205 uint64_t hash_elements_max; 206 uint64_t hash_collisions; 207 uint64_t hash_chains; 208 uint32_t hash_chain_max; 209 210 int no_grow; /* Don't try to grow cache size */ 211 } arc; 212 213 static uint64_t arc_tempreserve; 214 215 typedef struct arc_callback arc_callback_t; 216 217 struct arc_callback { 218 arc_done_func_t *acb_done; 219 void *acb_private; 220 arc_byteswap_func_t *acb_byteswap; 221 arc_buf_t *acb_buf; 222 zio_t *acb_zio_dummy; 223 arc_callback_t *acb_next; 224 }; 225 226 struct arc_buf_hdr { 227 /* immutable */ 228 uint64_t b_size; 229 spa_t *b_spa; 230 231 /* protected by hash lock */ 232 dva_t b_dva; 233 uint64_t b_birth; 234 uint64_t b_cksum0; 235 236 arc_buf_hdr_t *b_hash_next; 237 arc_buf_t *b_buf; 238 uint32_t b_flags; 239 uint32_t b_datacnt; 240 241 kcondvar_t b_cv; 242 arc_callback_t *b_acb; 243 244 /* protected by arc state mutex */ 245 arc_state_t *b_state; 246 list_node_t b_arc_node; 247 248 /* updated atomically */ 249 clock_t b_arc_access; 250 251 /* self protecting */ 252 refcount_t b_refcnt; 253 }; 254 255 static arc_buf_t *arc_eviction_list; 256 static kmutex_t arc_eviction_mtx; 257 static void arc_get_data_buf(arc_buf_t *buf); 258 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 259 260 #define GHOST_STATE(state) \ 261 ((state) == arc.mru_ghost || (state) == arc.mfu_ghost) 262 263 /* 264 * Private ARC flags. These flags are private ARC only flags that will show up 265 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 266 * be passed in as arc_flags in things like arc_read. However, these flags 267 * should never be passed and should only be set by ARC code. When adding new 268 * public flags, make sure not to smash the private ones. 269 */ 270 271 #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 272 #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 273 #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 274 #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 275 #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 276 #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 277 278 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 279 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 280 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 281 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 282 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 283 284 /* 285 * Hash table routines 286 */ 287 288 #define HT_LOCK_PAD 64 289 290 struct ht_lock { 291 kmutex_t ht_lock; 292 #ifdef _KERNEL 293 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 294 #endif 295 }; 296 297 #define BUF_LOCKS 256 298 typedef struct buf_hash_table { 299 uint64_t ht_mask; 300 arc_buf_hdr_t **ht_table; 301 struct ht_lock ht_locks[BUF_LOCKS]; 302 } buf_hash_table_t; 303 304 static buf_hash_table_t buf_hash_table; 305 306 #define BUF_HASH_INDEX(spa, dva, birth) \ 307 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 308 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 309 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 310 #define HDR_LOCK(buf) \ 311 (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 312 313 uint64_t zfs_crc64_table[256]; 314 315 static uint64_t 316 buf_hash(spa_t *spa, dva_t *dva, uint64_t birth) 317 { 318 uintptr_t spav = (uintptr_t)spa; 319 uint8_t *vdva = (uint8_t *)dva; 320 uint64_t crc = -1ULL; 321 int i; 322 323 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 324 325 for (i = 0; i < sizeof (dva_t); i++) 326 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 327 328 crc ^= (spav>>8) ^ birth; 329 330 return (crc); 331 } 332 333 #define BUF_EMPTY(buf) \ 334 ((buf)->b_dva.dva_word[0] == 0 && \ 335 (buf)->b_dva.dva_word[1] == 0 && \ 336 (buf)->b_birth == 0) 337 338 #define BUF_EQUAL(spa, dva, birth, buf) \ 339 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 340 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 341 ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 342 343 static arc_buf_hdr_t * 344 buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp) 345 { 346 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 347 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 348 arc_buf_hdr_t *buf; 349 350 mutex_enter(hash_lock); 351 for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 352 buf = buf->b_hash_next) { 353 if (BUF_EQUAL(spa, dva, birth, buf)) { 354 *lockp = hash_lock; 355 return (buf); 356 } 357 } 358 mutex_exit(hash_lock); 359 *lockp = NULL; 360 return (NULL); 361 } 362 363 /* 364 * Insert an entry into the hash table. If there is already an element 365 * equal to elem in the hash table, then the already existing element 366 * will be returned and the new element will not be inserted. 367 * Otherwise returns NULL. 368 */ 369 static arc_buf_hdr_t * 370 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 371 { 372 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 373 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 374 arc_buf_hdr_t *fbuf; 375 uint32_t max, i; 376 377 ASSERT(!HDR_IN_HASH_TABLE(buf)); 378 *lockp = hash_lock; 379 mutex_enter(hash_lock); 380 for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 381 fbuf = fbuf->b_hash_next, i++) { 382 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 383 return (fbuf); 384 } 385 386 buf->b_hash_next = buf_hash_table.ht_table[idx]; 387 buf_hash_table.ht_table[idx] = buf; 388 buf->b_flags |= ARC_IN_HASH_TABLE; 389 390 /* collect some hash table performance data */ 391 if (i > 0) { 392 atomic_add_64(&arc.hash_collisions, 1); 393 if (i == 1) 394 atomic_add_64(&arc.hash_chains, 1); 395 } 396 while (i > (max = arc.hash_chain_max) && 397 max != atomic_cas_32(&arc.hash_chain_max, max, i)) { 398 continue; 399 } 400 atomic_add_64(&arc.hash_elements, 1); 401 if (arc.hash_elements > arc.hash_elements_max) 402 atomic_add_64(&arc.hash_elements_max, 1); 403 404 return (NULL); 405 } 406 407 static void 408 buf_hash_remove(arc_buf_hdr_t *buf) 409 { 410 arc_buf_hdr_t *fbuf, **bufp; 411 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 412 413 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 414 ASSERT(HDR_IN_HASH_TABLE(buf)); 415 416 bufp = &buf_hash_table.ht_table[idx]; 417 while ((fbuf = *bufp) != buf) { 418 ASSERT(fbuf != NULL); 419 bufp = &fbuf->b_hash_next; 420 } 421 *bufp = buf->b_hash_next; 422 buf->b_hash_next = NULL; 423 buf->b_flags &= ~ARC_IN_HASH_TABLE; 424 425 /* collect some hash table performance data */ 426 atomic_add_64(&arc.hash_elements, -1); 427 if (buf_hash_table.ht_table[idx] && 428 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 429 atomic_add_64(&arc.hash_chains, -1); 430 } 431 432 /* 433 * Global data structures and functions for the buf kmem cache. 434 */ 435 static kmem_cache_t *hdr_cache; 436 static kmem_cache_t *buf_cache; 437 438 static void 439 buf_fini(void) 440 { 441 int i; 442 443 kmem_free(buf_hash_table.ht_table, 444 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 445 for (i = 0; i < BUF_LOCKS; i++) 446 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 447 kmem_cache_destroy(hdr_cache); 448 kmem_cache_destroy(buf_cache); 449 } 450 451 /* 452 * Constructor callback - called when the cache is empty 453 * and a new buf is requested. 454 */ 455 /* ARGSUSED */ 456 static int 457 hdr_cons(void *vbuf, void *unused, int kmflag) 458 { 459 arc_buf_hdr_t *buf = vbuf; 460 461 bzero(buf, sizeof (arc_buf_hdr_t)); 462 refcount_create(&buf->b_refcnt); 463 cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 464 return (0); 465 } 466 467 /* 468 * Destructor callback - called when a cached buf is 469 * no longer required. 470 */ 471 /* ARGSUSED */ 472 static void 473 hdr_dest(void *vbuf, void *unused) 474 { 475 arc_buf_hdr_t *buf = vbuf; 476 477 refcount_destroy(&buf->b_refcnt); 478 cv_destroy(&buf->b_cv); 479 } 480 481 static int arc_reclaim_needed(void); 482 void arc_kmem_reclaim(void); 483 484 /* 485 * Reclaim callback -- invoked when memory is low. 486 */ 487 /* ARGSUSED */ 488 static void 489 hdr_recl(void *unused) 490 { 491 dprintf("hdr_recl called\n"); 492 if (arc_reclaim_needed()) 493 arc_kmem_reclaim(); 494 } 495 496 static void 497 buf_init(void) 498 { 499 uint64_t *ct; 500 uint64_t hsize = 1ULL << 12; 501 int i, j; 502 503 /* 504 * The hash table is big enough to fill all of physical memory 505 * with an average 64K block size. The table will take up 506 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 507 */ 508 while (hsize * 65536 < physmem * PAGESIZE) 509 hsize <<= 1; 510 retry: 511 buf_hash_table.ht_mask = hsize - 1; 512 buf_hash_table.ht_table = 513 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 514 if (buf_hash_table.ht_table == NULL) { 515 ASSERT(hsize > (1ULL << 8)); 516 hsize >>= 1; 517 goto retry; 518 } 519 520 hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 521 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 522 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 523 0, NULL, NULL, NULL, NULL, NULL, 0); 524 525 for (i = 0; i < 256; i++) 526 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 527 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 528 529 for (i = 0; i < BUF_LOCKS; i++) { 530 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 531 NULL, MUTEX_DEFAULT, NULL); 532 } 533 } 534 535 #define ARC_MINTIME (hz>>4) /* 62 ms */ 536 537 static void 538 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 539 { 540 ASSERT(MUTEX_HELD(hash_lock)); 541 542 if ((refcount_add(&ab->b_refcnt, tag) == 1) && 543 (ab->b_state != arc.anon)) { 544 int delta = ab->b_size * ab->b_datacnt; 545 546 ASSERT(!MUTEX_HELD(&ab->b_state->mtx)); 547 mutex_enter(&ab->b_state->mtx); 548 ASSERT(list_link_active(&ab->b_arc_node)); 549 list_remove(&ab->b_state->list, ab); 550 if (GHOST_STATE(ab->b_state)) { 551 ASSERT3U(ab->b_datacnt, ==, 0); 552 ASSERT3P(ab->b_buf, ==, NULL); 553 delta = ab->b_size; 554 } 555 ASSERT(delta > 0); 556 ASSERT3U(ab->b_state->lsize, >=, delta); 557 atomic_add_64(&ab->b_state->lsize, -delta); 558 mutex_exit(&ab->b_state->mtx); 559 /* remove the prefetch flag is we get a reference */ 560 if (ab->b_flags & ARC_PREFETCH) 561 ab->b_flags &= ~ARC_PREFETCH; 562 } 563 } 564 565 static int 566 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 567 { 568 int cnt; 569 570 ASSERT(ab->b_state == arc.anon || MUTEX_HELD(hash_lock)); 571 ASSERT(!GHOST_STATE(ab->b_state)); 572 573 if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 574 (ab->b_state != arc.anon)) { 575 576 ASSERT(!MUTEX_HELD(&ab->b_state->mtx)); 577 mutex_enter(&ab->b_state->mtx); 578 ASSERT(!list_link_active(&ab->b_arc_node)); 579 list_insert_head(&ab->b_state->list, ab); 580 ASSERT(ab->b_datacnt > 0); 581 atomic_add_64(&ab->b_state->lsize, ab->b_size * ab->b_datacnt); 582 ASSERT3U(ab->b_state->size, >=, ab->b_state->lsize); 583 mutex_exit(&ab->b_state->mtx); 584 } 585 return (cnt); 586 } 587 588 /* 589 * Move the supplied buffer to the indicated state. The mutex 590 * for the buffer must be held by the caller. 591 */ 592 static void 593 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 594 { 595 arc_state_t *old_state = ab->b_state; 596 int refcnt = refcount_count(&ab->b_refcnt); 597 int from_delta, to_delta; 598 599 ASSERT(MUTEX_HELD(hash_lock)); 600 ASSERT(new_state != old_state); 601 ASSERT(refcnt == 0 || ab->b_datacnt > 0); 602 ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 603 604 from_delta = to_delta = ab->b_datacnt * ab->b_size; 605 606 /* 607 * If this buffer is evictable, transfer it from the 608 * old state list to the new state list. 609 */ 610 if (refcnt == 0) { 611 if (old_state != arc.anon) { 612 int use_mutex = !MUTEX_HELD(&old_state->mtx); 613 614 if (use_mutex) 615 mutex_enter(&old_state->mtx); 616 617 ASSERT(list_link_active(&ab->b_arc_node)); 618 list_remove(&old_state->list, ab); 619 620 /* 621 * If prefetching out of the ghost cache, 622 * we will have a non-null datacnt. 623 */ 624 if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 625 /* ghost elements have a ghost size */ 626 ASSERT(ab->b_buf == NULL); 627 from_delta = ab->b_size; 628 } 629 ASSERT3U(old_state->lsize, >=, from_delta); 630 atomic_add_64(&old_state->lsize, -from_delta); 631 632 if (use_mutex) 633 mutex_exit(&old_state->mtx); 634 } 635 if (new_state != arc.anon) { 636 int use_mutex = !MUTEX_HELD(&new_state->mtx); 637 638 if (use_mutex) 639 mutex_enter(&new_state->mtx); 640 641 list_insert_head(&new_state->list, ab); 642 643 /* ghost elements have a ghost size */ 644 if (GHOST_STATE(new_state)) { 645 ASSERT(ab->b_datacnt == 0); 646 ASSERT(ab->b_buf == NULL); 647 to_delta = ab->b_size; 648 } 649 atomic_add_64(&new_state->lsize, to_delta); 650 ASSERT3U(new_state->size + to_delta, >=, 651 new_state->lsize); 652 653 if (use_mutex) 654 mutex_exit(&new_state->mtx); 655 } 656 } 657 658 ASSERT(!BUF_EMPTY(ab)); 659 if (new_state == arc.anon && old_state != arc.anon) { 660 buf_hash_remove(ab); 661 } 662 663 /* adjust state sizes */ 664 if (to_delta) 665 atomic_add_64(&new_state->size, to_delta); 666 if (from_delta) { 667 ASSERT3U(old_state->size, >=, from_delta); 668 atomic_add_64(&old_state->size, -from_delta); 669 } 670 ab->b_state = new_state; 671 } 672 673 arc_buf_t * 674 arc_buf_alloc(spa_t *spa, int size, void *tag) 675 { 676 arc_buf_hdr_t *hdr; 677 arc_buf_t *buf; 678 679 ASSERT3U(size, >, 0); 680 hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 681 ASSERT(BUF_EMPTY(hdr)); 682 hdr->b_size = size; 683 hdr->b_spa = spa; 684 hdr->b_state = arc.anon; 685 hdr->b_arc_access = 0; 686 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 687 buf->b_hdr = hdr; 688 buf->b_data = NULL; 689 buf->b_efunc = NULL; 690 buf->b_private = NULL; 691 buf->b_next = NULL; 692 hdr->b_buf = buf; 693 arc_get_data_buf(buf); 694 hdr->b_datacnt = 1; 695 hdr->b_flags = 0; 696 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 697 (void) refcount_add(&hdr->b_refcnt, tag); 698 699 return (buf); 700 } 701 702 static arc_buf_t * 703 arc_buf_clone(arc_buf_t *from) 704 { 705 arc_buf_t *buf; 706 arc_buf_hdr_t *hdr = from->b_hdr; 707 uint64_t size = hdr->b_size; 708 709 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 710 buf->b_hdr = hdr; 711 buf->b_data = NULL; 712 buf->b_efunc = NULL; 713 buf->b_private = NULL; 714 buf->b_next = hdr->b_buf; 715 hdr->b_buf = buf; 716 arc_get_data_buf(buf); 717 bcopy(from->b_data, buf->b_data, size); 718 hdr->b_datacnt += 1; 719 return (buf); 720 } 721 722 void 723 arc_buf_add_ref(arc_buf_t *buf, void* tag) 724 { 725 arc_buf_hdr_t *hdr = buf->b_hdr; 726 kmutex_t *hash_lock; 727 728 /* 729 * Check to see if this buffer is currently being evicted via 730 * arc_do_user_evicts(). We can do this without holding any 731 * locks because if we happen to obtain the header before its 732 * cleared, we will find b_data is NULL later. 733 */ 734 if (hdr == NULL) 735 return; 736 737 hash_lock = HDR_LOCK(hdr); 738 mutex_enter(hash_lock); 739 if (buf->b_data == NULL) { 740 /* 741 * This buffer is evicted. 742 */ 743 mutex_exit(hash_lock); 744 return; 745 } 746 747 ASSERT(buf->b_hdr == hdr); 748 ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 749 add_reference(hdr, hash_lock, tag); 750 arc_access(hdr, hash_lock); 751 mutex_exit(hash_lock); 752 atomic_add_64(&arc.hits, 1); 753 } 754 755 static void 756 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 757 { 758 arc_buf_t **bufp; 759 760 /* free up data associated with the buf */ 761 if (buf->b_data) { 762 arc_state_t *state = buf->b_hdr->b_state; 763 uint64_t size = buf->b_hdr->b_size; 764 765 if (!recycle) { 766 zio_buf_free(buf->b_data, size); 767 atomic_add_64(&arc.size, -size); 768 } 769 if (list_link_active(&buf->b_hdr->b_arc_node)) { 770 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 771 ASSERT(state != arc.anon); 772 ASSERT3U(state->lsize, >=, size); 773 atomic_add_64(&state->lsize, -size); 774 } 775 ASSERT3U(state->size, >=, size); 776 atomic_add_64(&state->size, -size); 777 buf->b_data = NULL; 778 ASSERT(buf->b_hdr->b_datacnt > 0); 779 buf->b_hdr->b_datacnt -= 1; 780 } 781 782 /* only remove the buf if requested */ 783 if (!all) 784 return; 785 786 /* remove the buf from the hdr list */ 787 for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 788 continue; 789 *bufp = buf->b_next; 790 791 ASSERT(buf->b_efunc == NULL); 792 793 /* clean up the buf */ 794 buf->b_hdr = NULL; 795 kmem_cache_free(buf_cache, buf); 796 } 797 798 static void 799 arc_hdr_destroy(arc_buf_hdr_t *hdr) 800 { 801 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 802 ASSERT3P(hdr->b_state, ==, arc.anon); 803 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 804 805 if (!BUF_EMPTY(hdr)) { 806 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 807 bzero(&hdr->b_dva, sizeof (dva_t)); 808 hdr->b_birth = 0; 809 hdr->b_cksum0 = 0; 810 } 811 while (hdr->b_buf) { 812 arc_buf_t *buf = hdr->b_buf; 813 814 if (buf->b_efunc) { 815 mutex_enter(&arc_eviction_mtx); 816 ASSERT(buf->b_hdr != NULL); 817 arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 818 hdr->b_buf = buf->b_next; 819 buf->b_next = arc_eviction_list; 820 arc_eviction_list = buf; 821 mutex_exit(&arc_eviction_mtx); 822 } else { 823 arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 824 } 825 } 826 827 ASSERT(!list_link_active(&hdr->b_arc_node)); 828 ASSERT3P(hdr->b_hash_next, ==, NULL); 829 ASSERT3P(hdr->b_acb, ==, NULL); 830 kmem_cache_free(hdr_cache, hdr); 831 } 832 833 void 834 arc_buf_free(arc_buf_t *buf, void *tag) 835 { 836 arc_buf_hdr_t *hdr = buf->b_hdr; 837 int hashed = hdr->b_state != arc.anon; 838 839 ASSERT(buf->b_efunc == NULL); 840 ASSERT(buf->b_data != NULL); 841 842 if (hashed) { 843 kmutex_t *hash_lock = HDR_LOCK(hdr); 844 845 mutex_enter(hash_lock); 846 (void) remove_reference(hdr, hash_lock, tag); 847 if (hdr->b_datacnt > 1) 848 arc_buf_destroy(buf, FALSE, TRUE); 849 else 850 hdr->b_flags |= ARC_BUF_AVAILABLE; 851 mutex_exit(hash_lock); 852 } else if (HDR_IO_IN_PROGRESS(hdr)) { 853 int destroy_hdr; 854 /* 855 * We are in the middle of an async write. Don't destroy 856 * this buffer unless the write completes before we finish 857 * decrementing the reference count. 858 */ 859 mutex_enter(&arc_eviction_mtx); 860 (void) remove_reference(hdr, NULL, tag); 861 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 862 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 863 mutex_exit(&arc_eviction_mtx); 864 if (destroy_hdr) 865 arc_hdr_destroy(hdr); 866 } else { 867 if (remove_reference(hdr, NULL, tag) > 0) { 868 ASSERT(HDR_IO_ERROR(hdr)); 869 arc_buf_destroy(buf, FALSE, TRUE); 870 } else { 871 arc_hdr_destroy(hdr); 872 } 873 } 874 } 875 876 int 877 arc_buf_remove_ref(arc_buf_t *buf, void* tag) 878 { 879 arc_buf_hdr_t *hdr = buf->b_hdr; 880 kmutex_t *hash_lock = HDR_LOCK(hdr); 881 int no_callback = (buf->b_efunc == NULL); 882 883 if (hdr->b_state == arc.anon) { 884 arc_buf_free(buf, tag); 885 return (no_callback); 886 } 887 888 mutex_enter(hash_lock); 889 ASSERT(hdr->b_state != arc.anon); 890 ASSERT(buf->b_data != NULL); 891 892 (void) remove_reference(hdr, hash_lock, tag); 893 if (hdr->b_datacnt > 1) { 894 if (no_callback) 895 arc_buf_destroy(buf, FALSE, TRUE); 896 } else if (no_callback) { 897 ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 898 hdr->b_flags |= ARC_BUF_AVAILABLE; 899 } 900 ASSERT(no_callback || hdr->b_datacnt > 1 || 901 refcount_is_zero(&hdr->b_refcnt)); 902 mutex_exit(hash_lock); 903 return (no_callback); 904 } 905 906 int 907 arc_buf_size(arc_buf_t *buf) 908 { 909 return (buf->b_hdr->b_size); 910 } 911 912 /* 913 * Evict buffers from list until we've removed the specified number of 914 * bytes. Move the removed buffers to the appropriate evict state. 915 * If the recycle flag is set, then attempt to "recycle" a buffer: 916 * - look for a buffer to evict that is `bytes' long. 917 * - return the data block from this buffer rather than freeing it. 918 * This flag is used by callers that are trying to make space for a 919 * new buffer in a full arc cache. 920 */ 921 static void * 922 arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle) 923 { 924 arc_state_t *evicted_state; 925 uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 926 arc_buf_hdr_t *ab, *ab_prev; 927 kmutex_t *hash_lock; 928 boolean_t have_lock; 929 void *steal = NULL; 930 931 ASSERT(state == arc.mru || state == arc.mfu); 932 933 evicted_state = (state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost; 934 935 mutex_enter(&state->mtx); 936 mutex_enter(&evicted_state->mtx); 937 938 for (ab = list_tail(&state->list); ab; ab = ab_prev) { 939 ab_prev = list_prev(&state->list, ab); 940 /* prefetch buffers have a minimum lifespan */ 941 if (HDR_IO_IN_PROGRESS(ab) || 942 (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 943 lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 944 skipped++; 945 continue; 946 } 947 if (recycle && (ab->b_size != bytes || ab->b_datacnt > 1)) 948 continue; 949 hash_lock = HDR_LOCK(ab); 950 have_lock = MUTEX_HELD(hash_lock); 951 if (have_lock || mutex_tryenter(hash_lock)) { 952 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 953 ASSERT(ab->b_datacnt > 0); 954 while (ab->b_buf) { 955 arc_buf_t *buf = ab->b_buf; 956 if (buf->b_data) { 957 bytes_evicted += ab->b_size; 958 if (recycle) 959 steal = buf->b_data; 960 } 961 if (buf->b_efunc) { 962 mutex_enter(&arc_eviction_mtx); 963 arc_buf_destroy(buf, recycle, FALSE); 964 ab->b_buf = buf->b_next; 965 buf->b_next = arc_eviction_list; 966 arc_eviction_list = buf; 967 mutex_exit(&arc_eviction_mtx); 968 } else { 969 arc_buf_destroy(buf, recycle, TRUE); 970 } 971 } 972 ASSERT(ab->b_datacnt == 0); 973 arc_change_state(evicted_state, ab, hash_lock); 974 ASSERT(HDR_IN_HASH_TABLE(ab)); 975 ab->b_flags = ARC_IN_HASH_TABLE; 976 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 977 if (!have_lock) 978 mutex_exit(hash_lock); 979 if (bytes >= 0 && bytes_evicted >= bytes) 980 break; 981 } else { 982 missed += 1; 983 } 984 } 985 mutex_exit(&evicted_state->mtx); 986 mutex_exit(&state->mtx); 987 988 if (bytes_evicted < bytes) 989 dprintf("only evicted %lld bytes from %x", 990 (longlong_t)bytes_evicted, state); 991 992 if (skipped) 993 atomic_add_64(&arc.evict_skip, skipped); 994 if (missed) 995 atomic_add_64(&arc.mutex_miss, missed); 996 return (steal); 997 } 998 999 /* 1000 * Remove buffers from list until we've removed the specified number of 1001 * bytes. Destroy the buffers that are removed. 1002 */ 1003 static void 1004 arc_evict_ghost(arc_state_t *state, int64_t bytes) 1005 { 1006 arc_buf_hdr_t *ab, *ab_prev; 1007 kmutex_t *hash_lock; 1008 uint64_t bytes_deleted = 0; 1009 uint_t bufs_skipped = 0; 1010 1011 ASSERT(GHOST_STATE(state)); 1012 top: 1013 mutex_enter(&state->mtx); 1014 for (ab = list_tail(&state->list); ab; ab = ab_prev) { 1015 ab_prev = list_prev(&state->list, ab); 1016 hash_lock = HDR_LOCK(ab); 1017 if (mutex_tryenter(hash_lock)) { 1018 ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1019 ASSERT(ab->b_buf == NULL); 1020 arc_change_state(arc.anon, ab, hash_lock); 1021 mutex_exit(hash_lock); 1022 atomic_add_64(&arc.deleted, 1); 1023 bytes_deleted += ab->b_size; 1024 arc_hdr_destroy(ab); 1025 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1026 if (bytes >= 0 && bytes_deleted >= bytes) 1027 break; 1028 } else { 1029 if (bytes < 0) { 1030 mutex_exit(&state->mtx); 1031 mutex_enter(hash_lock); 1032 mutex_exit(hash_lock); 1033 goto top; 1034 } 1035 bufs_skipped += 1; 1036 } 1037 } 1038 mutex_exit(&state->mtx); 1039 1040 if (bufs_skipped) { 1041 atomic_add_64(&arc.mutex_miss, bufs_skipped); 1042 ASSERT(bytes >= 0); 1043 } 1044 1045 if (bytes_deleted < bytes) 1046 dprintf("only deleted %lld bytes from %p", 1047 (longlong_t)bytes_deleted, state); 1048 } 1049 1050 static void 1051 arc_adjust(void) 1052 { 1053 int64_t top_sz, mru_over, arc_over; 1054 1055 top_sz = arc.anon->size + arc.mru->size; 1056 1057 if (top_sz > arc.p && arc.mru->lsize > 0) { 1058 int64_t toevict = MIN(arc.mru->lsize, top_sz-arc.p); 1059 (void) arc_evict(arc.mru, toevict, FALSE); 1060 top_sz = arc.anon->size + arc.mru->size; 1061 } 1062 1063 mru_over = top_sz + arc.mru_ghost->size - arc.c; 1064 1065 if (mru_over > 0) { 1066 if (arc.mru_ghost->lsize > 0) { 1067 int64_t todelete = MIN(arc.mru_ghost->lsize, mru_over); 1068 arc_evict_ghost(arc.mru_ghost, todelete); 1069 } 1070 } 1071 1072 if ((arc_over = arc.size - arc.c) > 0) { 1073 int64_t tbl_over; 1074 1075 if (arc.mfu->lsize > 0) { 1076 int64_t toevict = MIN(arc.mfu->lsize, arc_over); 1077 (void) arc_evict(arc.mfu, toevict, FALSE); 1078 } 1079 1080 tbl_over = arc.size + arc.mru_ghost->lsize + 1081 arc.mfu_ghost->lsize - arc.c*2; 1082 1083 if (tbl_over > 0 && arc.mfu_ghost->lsize > 0) { 1084 int64_t todelete = MIN(arc.mfu_ghost->lsize, tbl_over); 1085 arc_evict_ghost(arc.mfu_ghost, todelete); 1086 } 1087 } 1088 } 1089 1090 static void 1091 arc_do_user_evicts(void) 1092 { 1093 mutex_enter(&arc_eviction_mtx); 1094 while (arc_eviction_list != NULL) { 1095 arc_buf_t *buf = arc_eviction_list; 1096 arc_eviction_list = buf->b_next; 1097 buf->b_hdr = NULL; 1098 mutex_exit(&arc_eviction_mtx); 1099 1100 if (buf->b_efunc != NULL) 1101 VERIFY(buf->b_efunc(buf) == 0); 1102 1103 buf->b_efunc = NULL; 1104 buf->b_private = NULL; 1105 kmem_cache_free(buf_cache, buf); 1106 mutex_enter(&arc_eviction_mtx); 1107 } 1108 mutex_exit(&arc_eviction_mtx); 1109 } 1110 1111 /* 1112 * Flush all *evictable* data from the cache. 1113 * NOTE: this will not touch "active" (i.e. referenced) data. 1114 */ 1115 void 1116 arc_flush(void) 1117 { 1118 while (list_head(&arc.mru->list)) 1119 (void) arc_evict(arc.mru, -1, FALSE); 1120 while (list_head(&arc.mfu->list)) 1121 (void) arc_evict(arc.mfu, -1, FALSE); 1122 1123 arc_evict_ghost(arc.mru_ghost, -1); 1124 arc_evict_ghost(arc.mfu_ghost, -1); 1125 1126 mutex_enter(&arc_reclaim_thr_lock); 1127 arc_do_user_evicts(); 1128 mutex_exit(&arc_reclaim_thr_lock); 1129 ASSERT(arc_eviction_list == NULL); 1130 } 1131 1132 int arc_kmem_reclaim_shift = 5; /* log2(fraction of arc to reclaim) */ 1133 1134 void 1135 arc_kmem_reclaim(void) 1136 { 1137 uint64_t to_free; 1138 1139 /* 1140 * We need arc_reclaim_lock because we don't want multiple 1141 * threads trying to reclaim concurrently. 1142 */ 1143 1144 /* 1145 * umem calls the reclaim func when we destroy the buf cache, 1146 * which is after we do arc_fini(). So we set a flag to prevent 1147 * accessing the destroyed mutexes and lists. 1148 */ 1149 if (arc_dead) 1150 return; 1151 1152 if (arc.c <= arc.c_min) 1153 return; 1154 1155 mutex_enter(&arc_reclaim_lock); 1156 1157 #ifdef _KERNEL 1158 to_free = MAX(arc.c >> arc_kmem_reclaim_shift, ptob(needfree)); 1159 #else 1160 to_free = arc.c >> arc_kmem_reclaim_shift; 1161 #endif 1162 if (arc.c > to_free) 1163 atomic_add_64(&arc.c, -to_free); 1164 else 1165 arc.c = arc.c_min; 1166 1167 atomic_add_64(&arc.p, -(arc.p >> arc_kmem_reclaim_shift)); 1168 if (arc.c > arc.size) 1169 arc.c = arc.size; 1170 if (arc.c < arc.c_min) 1171 arc.c = arc.c_min; 1172 if (arc.p > arc.c) 1173 arc.p = (arc.c >> 1); 1174 ASSERT((int64_t)arc.p >= 0); 1175 1176 arc_adjust(); 1177 1178 mutex_exit(&arc_reclaim_lock); 1179 } 1180 1181 static int 1182 arc_reclaim_needed(void) 1183 { 1184 uint64_t extra; 1185 1186 #ifdef _KERNEL 1187 1188 if (needfree) 1189 return (1); 1190 1191 /* 1192 * take 'desfree' extra pages, so we reclaim sooner, rather than later 1193 */ 1194 extra = desfree; 1195 1196 /* 1197 * check that we're out of range of the pageout scanner. It starts to 1198 * schedule paging if freemem is less than lotsfree and needfree. 1199 * lotsfree is the high-water mark for pageout, and needfree is the 1200 * number of needed free pages. We add extra pages here to make sure 1201 * the scanner doesn't start up while we're freeing memory. 1202 */ 1203 if (freemem < lotsfree + needfree + extra) 1204 return (1); 1205 1206 /* 1207 * check to make sure that swapfs has enough space so that anon 1208 * reservations can still succeeed. anon_resvmem() checks that the 1209 * availrmem is greater than swapfs_minfree, and the number of reserved 1210 * swap pages. We also add a bit of extra here just to prevent 1211 * circumstances from getting really dire. 1212 */ 1213 if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1214 return (1); 1215 1216 #if defined(__i386) 1217 /* 1218 * If we're on an i386 platform, it's possible that we'll exhaust the 1219 * kernel heap space before we ever run out of available physical 1220 * memory. Most checks of the size of the heap_area compare against 1221 * tune.t_minarmem, which is the minimum available real memory that we 1222 * can have in the system. However, this is generally fixed at 25 pages 1223 * which is so low that it's useless. In this comparison, we seek to 1224 * calculate the total heap-size, and reclaim if more than 3/4ths of the 1225 * heap is allocated. (Or, in the caclulation, if less than 1/4th is 1226 * free) 1227 */ 1228 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1229 (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1230 return (1); 1231 #endif 1232 1233 #else 1234 if (spa_get_random(100) == 0) 1235 return (1); 1236 #endif 1237 return (0); 1238 } 1239 1240 static void 1241 arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1242 { 1243 size_t i; 1244 kmem_cache_t *prev_cache = NULL; 1245 extern kmem_cache_t *zio_buf_cache[]; 1246 1247 #ifdef _KERNEL 1248 /* 1249 * First purge some DNLC entries, in case the DNLC is using 1250 * up too much memory. 1251 */ 1252 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 1253 1254 #if defined(__i386) 1255 /* 1256 * Reclaim unused memory from all kmem caches. 1257 */ 1258 kmem_reap(); 1259 #endif 1260 #endif 1261 1262 /* 1263 * An agressive reclamation will shrink the cache size as well as 1264 * reap free buffers from the arc kmem caches. 1265 */ 1266 if (strat == ARC_RECLAIM_AGGR) 1267 arc_kmem_reclaim(); 1268 1269 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1270 if (zio_buf_cache[i] != prev_cache) { 1271 prev_cache = zio_buf_cache[i]; 1272 kmem_cache_reap_now(zio_buf_cache[i]); 1273 } 1274 } 1275 kmem_cache_reap_now(buf_cache); 1276 kmem_cache_reap_now(hdr_cache); 1277 } 1278 1279 static void 1280 arc_reclaim_thread(void) 1281 { 1282 clock_t growtime = 0; 1283 arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1284 callb_cpr_t cpr; 1285 1286 CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1287 1288 mutex_enter(&arc_reclaim_thr_lock); 1289 while (arc_thread_exit == 0) { 1290 if (arc_reclaim_needed()) { 1291 1292 if (arc.no_grow) { 1293 if (last_reclaim == ARC_RECLAIM_CONS) { 1294 last_reclaim = ARC_RECLAIM_AGGR; 1295 } else { 1296 last_reclaim = ARC_RECLAIM_CONS; 1297 } 1298 } else { 1299 arc.no_grow = TRUE; 1300 last_reclaim = ARC_RECLAIM_AGGR; 1301 membar_producer(); 1302 } 1303 1304 /* reset the growth delay for every reclaim */ 1305 growtime = lbolt + (arc_grow_retry * hz); 1306 ASSERT(growtime > 0); 1307 1308 arc_kmem_reap_now(last_reclaim); 1309 1310 } else if ((growtime > 0) && ((growtime - lbolt) <= 0)) { 1311 arc.no_grow = FALSE; 1312 } 1313 1314 if (arc_eviction_list != NULL) 1315 arc_do_user_evicts(); 1316 1317 /* block until needed, or one second, whichever is shorter */ 1318 CALLB_CPR_SAFE_BEGIN(&cpr); 1319 (void) cv_timedwait(&arc_reclaim_thr_cv, 1320 &arc_reclaim_thr_lock, (lbolt + hz)); 1321 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1322 } 1323 1324 arc_thread_exit = 0; 1325 cv_broadcast(&arc_reclaim_thr_cv); 1326 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1327 thread_exit(); 1328 } 1329 1330 /* 1331 * Adapt arc info given the number of bytes we are trying to add and 1332 * the state that we are comming from. This function is only called 1333 * when we are adding new content to the cache. 1334 */ 1335 static void 1336 arc_adapt(int bytes, arc_state_t *state) 1337 { 1338 int mult; 1339 1340 ASSERT(bytes > 0); 1341 /* 1342 * Adapt the target size of the MRU list: 1343 * - if we just hit in the MRU ghost list, then increase 1344 * the target size of the MRU list. 1345 * - if we just hit in the MFU ghost list, then increase 1346 * the target size of the MFU list by decreasing the 1347 * target size of the MRU list. 1348 */ 1349 if (state == arc.mru_ghost) { 1350 mult = ((arc.mru_ghost->size >= arc.mfu_ghost->size) ? 1351 1 : (arc.mfu_ghost->size/arc.mru_ghost->size)); 1352 1353 arc.p = MIN(arc.c, arc.p + bytes * mult); 1354 } else if (state == arc.mfu_ghost) { 1355 mult = ((arc.mfu_ghost->size >= arc.mru_ghost->size) ? 1356 1 : (arc.mru_ghost->size/arc.mfu_ghost->size)); 1357 1358 arc.p = MAX(0, (int64_t)arc.p - bytes * mult); 1359 } 1360 ASSERT((int64_t)arc.p >= 0); 1361 1362 if (arc_reclaim_needed()) { 1363 cv_signal(&arc_reclaim_thr_cv); 1364 return; 1365 } 1366 1367 if (arc.no_grow) 1368 return; 1369 1370 if (arc.c >= arc.c_max) 1371 return; 1372 1373 /* 1374 * If we're within (2 * maxblocksize) bytes of the target 1375 * cache size, increment the target cache size 1376 */ 1377 if (arc.size > arc.c - (2ULL << SPA_MAXBLOCKSHIFT)) { 1378 atomic_add_64(&arc.c, (int64_t)bytes); 1379 if (arc.c > arc.c_max) 1380 arc.c = arc.c_max; 1381 else if (state == arc.anon) 1382 atomic_add_64(&arc.p, (int64_t)bytes); 1383 if (arc.p > arc.c) 1384 arc.p = arc.c; 1385 } 1386 ASSERT((int64_t)arc.p >= 0); 1387 } 1388 1389 /* 1390 * Check if the cache has reached its limits and eviction is required 1391 * prior to insert. 1392 */ 1393 static int 1394 arc_evict_needed() 1395 { 1396 if (arc_reclaim_needed()) 1397 return (1); 1398 1399 return (arc.size > arc.c); 1400 } 1401 1402 /* 1403 * The buffer, supplied as the first argument, needs a data block. 1404 * So, if we are at cache max, determine which cache should be victimized. 1405 * We have the following cases: 1406 * 1407 * 1. Insert for MRU, p > sizeof(arc.anon + arc.mru) -> 1408 * In this situation if we're out of space, but the resident size of the MFU is 1409 * under the limit, victimize the MFU cache to satisfy this insertion request. 1410 * 1411 * 2. Insert for MRU, p <= sizeof(arc.anon + arc.mru) -> 1412 * Here, we've used up all of the available space for the MRU, so we need to 1413 * evict from our own cache instead. Evict from the set of resident MRU 1414 * entries. 1415 * 1416 * 3. Insert for MFU (c - p) > sizeof(arc.mfu) -> 1417 * c minus p represents the MFU space in the cache, since p is the size of the 1418 * cache that is dedicated to the MRU. In this situation there's still space on 1419 * the MFU side, so the MRU side needs to be victimized. 1420 * 1421 * 4. Insert for MFU (c - p) < sizeof(arc.mfu) -> 1422 * MFU's resident set is consuming more space than it has been allotted. In 1423 * this situation, we must victimize our own cache, the MFU, for this insertion. 1424 */ 1425 static void 1426 arc_get_data_buf(arc_buf_t *buf) 1427 { 1428 arc_state_t *state = buf->b_hdr->b_state; 1429 uint64_t size = buf->b_hdr->b_size; 1430 1431 arc_adapt(size, state); 1432 1433 /* 1434 * We have not yet reached cache maximum size, 1435 * just allocate a new buffer. 1436 */ 1437 if (!arc_evict_needed()) { 1438 buf->b_data = zio_buf_alloc(size); 1439 atomic_add_64(&arc.size, size); 1440 goto out; 1441 } 1442 1443 /* 1444 * If we are prefetching from the mfu ghost list, this buffer 1445 * will end up on the mru list; so steal space from there. 1446 */ 1447 if (state == arc.mfu_ghost) 1448 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc.mru : arc.mfu; 1449 else if (state == arc.mru_ghost) 1450 state = arc.mru; 1451 1452 if (state == arc.mru || state == arc.anon) { 1453 uint64_t mru_used = arc.anon->size + arc.mru->size; 1454 state = (arc.p > mru_used) ? arc.mfu : arc.mru; 1455 } else { 1456 /* MFU cases */ 1457 uint64_t mfu_space = arc.c - arc.p; 1458 state = (mfu_space > arc.mfu->size) ? arc.mru : arc.mfu; 1459 } 1460 if ((buf->b_data = arc_evict(state, size, TRUE)) == NULL) { 1461 (void) arc_evict(state, size, FALSE); 1462 buf->b_data = zio_buf_alloc(size); 1463 atomic_add_64(&arc.size, size); 1464 atomic_add_64(&arc.recycle_miss, 1); 1465 if (arc.size > arc.c) 1466 arc_adjust(); 1467 } 1468 ASSERT(buf->b_data != NULL); 1469 out: 1470 /* 1471 * Update the state size. Note that ghost states have a 1472 * "ghost size" and so don't need to be updated. 1473 */ 1474 if (!GHOST_STATE(buf->b_hdr->b_state)) { 1475 arc_buf_hdr_t *hdr = buf->b_hdr; 1476 1477 atomic_add_64(&hdr->b_state->size, size); 1478 if (list_link_active(&hdr->b_arc_node)) { 1479 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1480 atomic_add_64(&hdr->b_state->lsize, size); 1481 } 1482 } 1483 } 1484 1485 /* 1486 * This routine is called whenever a buffer is accessed. 1487 * NOTE: the hash lock is dropped in this function. 1488 */ 1489 static void 1490 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 1491 { 1492 ASSERT(MUTEX_HELD(hash_lock)); 1493 1494 if (buf->b_state == arc.anon) { 1495 /* 1496 * This buffer is not in the cache, and does not 1497 * appear in our "ghost" list. Add the new buffer 1498 * to the MRU state. 1499 */ 1500 1501 ASSERT(buf->b_arc_access == 0); 1502 buf->b_arc_access = lbolt; 1503 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1504 arc_change_state(arc.mru, buf, hash_lock); 1505 1506 } else if (buf->b_state == arc.mru) { 1507 /* 1508 * If this buffer is here because of a prefetch, then either: 1509 * - clear the flag if this is a "referencing" read 1510 * (any subsequent access will bump this into the MFU state). 1511 * or 1512 * - move the buffer to the head of the list if this is 1513 * another prefetch (to make it less likely to be evicted). 1514 */ 1515 if ((buf->b_flags & ARC_PREFETCH) != 0) { 1516 if (refcount_count(&buf->b_refcnt) == 0) { 1517 ASSERT(list_link_active(&buf->b_arc_node)); 1518 mutex_enter(&arc.mru->mtx); 1519 list_remove(&arc.mru->list, buf); 1520 list_insert_head(&arc.mru->list, buf); 1521 mutex_exit(&arc.mru->mtx); 1522 } else { 1523 buf->b_flags &= ~ARC_PREFETCH; 1524 atomic_add_64(&arc.mru->hits, 1); 1525 } 1526 buf->b_arc_access = lbolt; 1527 return; 1528 } 1529 1530 /* 1531 * This buffer has been "accessed" only once so far, 1532 * but it is still in the cache. Move it to the MFU 1533 * state. 1534 */ 1535 if (lbolt > buf->b_arc_access + ARC_MINTIME) { 1536 /* 1537 * More than 125ms have passed since we 1538 * instantiated this buffer. Move it to the 1539 * most frequently used state. 1540 */ 1541 buf->b_arc_access = lbolt; 1542 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1543 arc_change_state(arc.mfu, buf, hash_lock); 1544 } 1545 atomic_add_64(&arc.mru->hits, 1); 1546 } else if (buf->b_state == arc.mru_ghost) { 1547 arc_state_t *new_state; 1548 /* 1549 * This buffer has been "accessed" recently, but 1550 * was evicted from the cache. Move it to the 1551 * MFU state. 1552 */ 1553 1554 if (buf->b_flags & ARC_PREFETCH) { 1555 new_state = arc.mru; 1556 if (refcount_count(&buf->b_refcnt) > 0) 1557 buf->b_flags &= ~ARC_PREFETCH; 1558 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1559 } else { 1560 new_state = arc.mfu; 1561 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1562 } 1563 1564 buf->b_arc_access = lbolt; 1565 arc_change_state(new_state, buf, hash_lock); 1566 1567 atomic_add_64(&arc.mru_ghost->hits, 1); 1568 } else if (buf->b_state == arc.mfu) { 1569 /* 1570 * This buffer has been accessed more than once and is 1571 * still in the cache. Keep it in the MFU state. 1572 * 1573 * NOTE: an add_reference() that occurred when we did 1574 * the arc_read() will have kicked this off the list. 1575 * If it was a prefetch, we will explicitly move it to 1576 * the head of the list now. 1577 */ 1578 if ((buf->b_flags & ARC_PREFETCH) != 0) { 1579 ASSERT(refcount_count(&buf->b_refcnt) == 0); 1580 ASSERT(list_link_active(&buf->b_arc_node)); 1581 mutex_enter(&arc.mfu->mtx); 1582 list_remove(&arc.mfu->list, buf); 1583 list_insert_head(&arc.mfu->list, buf); 1584 mutex_exit(&arc.mfu->mtx); 1585 } 1586 atomic_add_64(&arc.mfu->hits, 1); 1587 buf->b_arc_access = lbolt; 1588 } else if (buf->b_state == arc.mfu_ghost) { 1589 arc_state_t *new_state = arc.mfu; 1590 /* 1591 * This buffer has been accessed more than once but has 1592 * been evicted from the cache. Move it back to the 1593 * MFU state. 1594 */ 1595 1596 if (buf->b_flags & ARC_PREFETCH) { 1597 /* 1598 * This is a prefetch access... 1599 * move this block back to the MRU state. 1600 */ 1601 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 1602 new_state = arc.mru; 1603 } 1604 1605 buf->b_arc_access = lbolt; 1606 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1607 arc_change_state(new_state, buf, hash_lock); 1608 1609 atomic_add_64(&arc.mfu_ghost->hits, 1); 1610 } else { 1611 ASSERT(!"invalid arc state"); 1612 } 1613 } 1614 1615 /* a generic arc_done_func_t which you can use */ 1616 /* ARGSUSED */ 1617 void 1618 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 1619 { 1620 bcopy(buf->b_data, arg, buf->b_hdr->b_size); 1621 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1622 } 1623 1624 /* a generic arc_done_func_t which you can use */ 1625 void 1626 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 1627 { 1628 arc_buf_t **bufp = arg; 1629 if (zio && zio->io_error) { 1630 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1631 *bufp = NULL; 1632 } else { 1633 *bufp = buf; 1634 } 1635 } 1636 1637 static void 1638 arc_read_done(zio_t *zio) 1639 { 1640 arc_buf_hdr_t *hdr, *found; 1641 arc_buf_t *buf; 1642 arc_buf_t *abuf; /* buffer we're assigning to callback */ 1643 kmutex_t *hash_lock; 1644 arc_callback_t *callback_list, *acb; 1645 int freeable = FALSE; 1646 1647 buf = zio->io_private; 1648 hdr = buf->b_hdr; 1649 1650 /* 1651 * The hdr was inserted into hash-table and removed from lists 1652 * prior to starting I/O. We should find this header, since 1653 * it's in the hash table, and it should be legit since it's 1654 * not possible to evict it during the I/O. The only possible 1655 * reason for it not to be found is if we were freed during the 1656 * read. 1657 */ 1658 found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 1659 &hash_lock); 1660 1661 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 1662 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp)))); 1663 1664 /* byteswap if necessary */ 1665 callback_list = hdr->b_acb; 1666 ASSERT(callback_list != NULL); 1667 if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap) 1668 callback_list->acb_byteswap(buf->b_data, hdr->b_size); 1669 1670 /* create copies of the data buffer for the callers */ 1671 abuf = buf; 1672 for (acb = callback_list; acb; acb = acb->acb_next) { 1673 if (acb->acb_done) { 1674 if (abuf == NULL) 1675 abuf = arc_buf_clone(buf); 1676 acb->acb_buf = abuf; 1677 abuf = NULL; 1678 } 1679 } 1680 hdr->b_acb = NULL; 1681 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 1682 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 1683 if (abuf == buf) 1684 hdr->b_flags |= ARC_BUF_AVAILABLE; 1685 1686 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 1687 1688 if (zio->io_error != 0) { 1689 hdr->b_flags |= ARC_IO_ERROR; 1690 if (hdr->b_state != arc.anon) 1691 arc_change_state(arc.anon, hdr, hash_lock); 1692 if (HDR_IN_HASH_TABLE(hdr)) 1693 buf_hash_remove(hdr); 1694 freeable = refcount_is_zero(&hdr->b_refcnt); 1695 /* convert checksum errors into IO errors */ 1696 if (zio->io_error == ECKSUM) 1697 zio->io_error = EIO; 1698 } 1699 1700 /* 1701 * Broadcast before we drop the hash_lock to avoid the possibility 1702 * that the hdr (and hence the cv) might be freed before we get to 1703 * the cv_broadcast(). 1704 */ 1705 cv_broadcast(&hdr->b_cv); 1706 1707 if (hash_lock) { 1708 /* 1709 * Only call arc_access on anonymous buffers. This is because 1710 * if we've issued an I/O for an evicted buffer, we've already 1711 * called arc_access (to prevent any simultaneous readers from 1712 * getting confused). 1713 */ 1714 if (zio->io_error == 0 && hdr->b_state == arc.anon) 1715 arc_access(hdr, hash_lock); 1716 mutex_exit(hash_lock); 1717 } else { 1718 /* 1719 * This block was freed while we waited for the read to 1720 * complete. It has been removed from the hash table and 1721 * moved to the anonymous state (so that it won't show up 1722 * in the cache). 1723 */ 1724 ASSERT3P(hdr->b_state, ==, arc.anon); 1725 freeable = refcount_is_zero(&hdr->b_refcnt); 1726 } 1727 1728 /* execute each callback and free its structure */ 1729 while ((acb = callback_list) != NULL) { 1730 if (acb->acb_done) 1731 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 1732 1733 if (acb->acb_zio_dummy != NULL) { 1734 acb->acb_zio_dummy->io_error = zio->io_error; 1735 zio_nowait(acb->acb_zio_dummy); 1736 } 1737 1738 callback_list = acb->acb_next; 1739 kmem_free(acb, sizeof (arc_callback_t)); 1740 } 1741 1742 if (freeable) 1743 arc_hdr_destroy(hdr); 1744 } 1745 1746 /* 1747 * "Read" the block block at the specified DVA (in bp) via the 1748 * cache. If the block is found in the cache, invoke the provided 1749 * callback immediately and return. Note that the `zio' parameter 1750 * in the callback will be NULL in this case, since no IO was 1751 * required. If the block is not in the cache pass the read request 1752 * on to the spa with a substitute callback function, so that the 1753 * requested block will be added to the cache. 1754 * 1755 * If a read request arrives for a block that has a read in-progress, 1756 * either wait for the in-progress read to complete (and return the 1757 * results); or, if this is a read with a "done" func, add a record 1758 * to the read to invoke the "done" func when the read completes, 1759 * and return; or just return. 1760 * 1761 * arc_read_done() will invoke all the requested "done" functions 1762 * for readers of this block. 1763 */ 1764 int 1765 arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap, 1766 arc_done_func_t *done, void *private, int priority, int flags, 1767 uint32_t *arc_flags, zbookmark_t *zb) 1768 { 1769 arc_buf_hdr_t *hdr; 1770 arc_buf_t *buf; 1771 kmutex_t *hash_lock; 1772 zio_t *rzio; 1773 1774 top: 1775 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 1776 if (hdr && hdr->b_datacnt > 0) { 1777 1778 *arc_flags |= ARC_CACHED; 1779 1780 if (HDR_IO_IN_PROGRESS(hdr)) { 1781 1782 if (*arc_flags & ARC_WAIT) { 1783 cv_wait(&hdr->b_cv, hash_lock); 1784 mutex_exit(hash_lock); 1785 goto top; 1786 } 1787 ASSERT(*arc_flags & ARC_NOWAIT); 1788 1789 if (done) { 1790 arc_callback_t *acb = NULL; 1791 1792 acb = kmem_zalloc(sizeof (arc_callback_t), 1793 KM_SLEEP); 1794 acb->acb_done = done; 1795 acb->acb_private = private; 1796 acb->acb_byteswap = swap; 1797 if (pio != NULL) 1798 acb->acb_zio_dummy = zio_null(pio, 1799 spa, NULL, NULL, flags); 1800 1801 ASSERT(acb->acb_done != NULL); 1802 acb->acb_next = hdr->b_acb; 1803 hdr->b_acb = acb; 1804 add_reference(hdr, hash_lock, private); 1805 mutex_exit(hash_lock); 1806 return (0); 1807 } 1808 mutex_exit(hash_lock); 1809 return (0); 1810 } 1811 1812 ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 1813 1814 if (done) { 1815 add_reference(hdr, hash_lock, private); 1816 /* 1817 * If this block is already in use, create a new 1818 * copy of the data so that we will be guaranteed 1819 * that arc_release() will always succeed. 1820 */ 1821 buf = hdr->b_buf; 1822 ASSERT(buf); 1823 ASSERT(buf->b_data); 1824 if (HDR_BUF_AVAILABLE(hdr)) { 1825 ASSERT(buf->b_efunc == NULL); 1826 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 1827 } else { 1828 buf = arc_buf_clone(buf); 1829 } 1830 } else if (*arc_flags & ARC_PREFETCH && 1831 refcount_count(&hdr->b_refcnt) == 0) { 1832 hdr->b_flags |= ARC_PREFETCH; 1833 } 1834 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 1835 arc_access(hdr, hash_lock); 1836 mutex_exit(hash_lock); 1837 atomic_add_64(&arc.hits, 1); 1838 if (done) 1839 done(NULL, buf, private); 1840 } else { 1841 uint64_t size = BP_GET_LSIZE(bp); 1842 arc_callback_t *acb; 1843 1844 if (hdr == NULL) { 1845 /* this block is not in the cache */ 1846 arc_buf_hdr_t *exists; 1847 1848 buf = arc_buf_alloc(spa, size, private); 1849 hdr = buf->b_hdr; 1850 hdr->b_dva = *BP_IDENTITY(bp); 1851 hdr->b_birth = bp->blk_birth; 1852 hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 1853 exists = buf_hash_insert(hdr, &hash_lock); 1854 if (exists) { 1855 /* somebody beat us to the hash insert */ 1856 mutex_exit(hash_lock); 1857 bzero(&hdr->b_dva, sizeof (dva_t)); 1858 hdr->b_birth = 0; 1859 hdr->b_cksum0 = 0; 1860 (void) arc_buf_remove_ref(buf, private); 1861 goto top; /* restart the IO request */ 1862 } 1863 /* if this is a prefetch, we don't have a reference */ 1864 if (*arc_flags & ARC_PREFETCH) { 1865 (void) remove_reference(hdr, hash_lock, 1866 private); 1867 hdr->b_flags |= ARC_PREFETCH; 1868 } 1869 if (BP_GET_LEVEL(bp) > 0) 1870 hdr->b_flags |= ARC_INDIRECT; 1871 } else { 1872 /* this block is in the ghost cache */ 1873 ASSERT(GHOST_STATE(hdr->b_state)); 1874 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1875 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 1876 ASSERT(hdr->b_buf == NULL); 1877 1878 /* if this is a prefetch, we don't have a reference */ 1879 if (*arc_flags & ARC_PREFETCH) 1880 hdr->b_flags |= ARC_PREFETCH; 1881 else 1882 add_reference(hdr, hash_lock, private); 1883 buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 1884 buf->b_hdr = hdr; 1885 buf->b_data = NULL; 1886 buf->b_efunc = NULL; 1887 buf->b_private = NULL; 1888 buf->b_next = NULL; 1889 hdr->b_buf = buf; 1890 arc_get_data_buf(buf); 1891 ASSERT(hdr->b_datacnt == 0); 1892 hdr->b_datacnt = 1; 1893 1894 } 1895 1896 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 1897 acb->acb_done = done; 1898 acb->acb_private = private; 1899 acb->acb_byteswap = swap; 1900 1901 ASSERT(hdr->b_acb == NULL); 1902 hdr->b_acb = acb; 1903 hdr->b_flags |= ARC_IO_IN_PROGRESS; 1904 1905 /* 1906 * If the buffer has been evicted, migrate it to a present state 1907 * before issuing the I/O. Once we drop the hash-table lock, 1908 * the header will be marked as I/O in progress and have an 1909 * attached buffer. At this point, anybody who finds this 1910 * buffer ought to notice that it's legit but has a pending I/O. 1911 */ 1912 1913 if (GHOST_STATE(hdr->b_state)) 1914 arc_access(hdr, hash_lock); 1915 mutex_exit(hash_lock); 1916 1917 ASSERT3U(hdr->b_size, ==, size); 1918 DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 1919 zbookmark_t *, zb); 1920 atomic_add_64(&arc.misses, 1); 1921 1922 rzio = zio_read(pio, spa, bp, buf->b_data, size, 1923 arc_read_done, buf, priority, flags, zb); 1924 1925 if (*arc_flags & ARC_WAIT) 1926 return (zio_wait(rzio)); 1927 1928 ASSERT(*arc_flags & ARC_NOWAIT); 1929 zio_nowait(rzio); 1930 } 1931 return (0); 1932 } 1933 1934 /* 1935 * arc_read() variant to support pool traversal. If the block is already 1936 * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 1937 * The idea is that we don't want pool traversal filling up memory, but 1938 * if the ARC already has the data anyway, we shouldn't pay for the I/O. 1939 */ 1940 int 1941 arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 1942 { 1943 arc_buf_hdr_t *hdr; 1944 kmutex_t *hash_mtx; 1945 int rc = 0; 1946 1947 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 1948 1949 if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 1950 arc_buf_t *buf = hdr->b_buf; 1951 1952 ASSERT(buf); 1953 while (buf->b_data == NULL) { 1954 buf = buf->b_next; 1955 ASSERT(buf); 1956 } 1957 bcopy(buf->b_data, data, hdr->b_size); 1958 } else { 1959 rc = ENOENT; 1960 } 1961 1962 if (hash_mtx) 1963 mutex_exit(hash_mtx); 1964 1965 return (rc); 1966 } 1967 1968 void 1969 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 1970 { 1971 ASSERT(buf->b_hdr != NULL); 1972 ASSERT(buf->b_hdr->b_state != arc.anon); 1973 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 1974 buf->b_efunc = func; 1975 buf->b_private = private; 1976 } 1977 1978 /* 1979 * This is used by the DMU to let the ARC know that a buffer is 1980 * being evicted, so the ARC should clean up. If this arc buf 1981 * is not yet in the evicted state, it will be put there. 1982 */ 1983 int 1984 arc_buf_evict(arc_buf_t *buf) 1985 { 1986 arc_buf_hdr_t *hdr = buf->b_hdr; 1987 kmutex_t *hash_lock; 1988 arc_buf_t **bufp; 1989 1990 if (hdr == NULL) { 1991 /* 1992 * We are in arc_do_user_evicts(). 1993 */ 1994 ASSERT(buf->b_data == NULL); 1995 return (0); 1996 } 1997 1998 hash_lock = HDR_LOCK(hdr); 1999 mutex_enter(hash_lock); 2000 2001 if (buf->b_data == NULL) { 2002 /* 2003 * We are on the eviction list. 2004 */ 2005 mutex_exit(hash_lock); 2006 mutex_enter(&arc_eviction_mtx); 2007 if (buf->b_hdr == NULL) { 2008 /* 2009 * We are already in arc_do_user_evicts(). 2010 */ 2011 mutex_exit(&arc_eviction_mtx); 2012 return (0); 2013 } else { 2014 arc_buf_t copy = *buf; /* structure assignment */ 2015 /* 2016 * Process this buffer now 2017 * but let arc_do_user_evicts() do the reaping. 2018 */ 2019 buf->b_efunc = NULL; 2020 mutex_exit(&arc_eviction_mtx); 2021 VERIFY(copy.b_efunc(©) == 0); 2022 return (1); 2023 } 2024 } 2025 2026 ASSERT(buf->b_hdr == hdr); 2027 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2028 ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 2029 2030 /* 2031 * Pull this buffer off of the hdr 2032 */ 2033 bufp = &hdr->b_buf; 2034 while (*bufp != buf) 2035 bufp = &(*bufp)->b_next; 2036 *bufp = buf->b_next; 2037 2038 ASSERT(buf->b_data != NULL); 2039 buf->b_hdr = hdr; 2040 arc_buf_destroy(buf, FALSE, FALSE); 2041 2042 if (hdr->b_datacnt == 0) { 2043 arc_state_t *old_state = hdr->b_state; 2044 arc_state_t *evicted_state; 2045 2046 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2047 2048 evicted_state = 2049 (old_state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost; 2050 2051 mutex_enter(&old_state->mtx); 2052 mutex_enter(&evicted_state->mtx); 2053 2054 arc_change_state(evicted_state, hdr, hash_lock); 2055 ASSERT(HDR_IN_HASH_TABLE(hdr)); 2056 hdr->b_flags = ARC_IN_HASH_TABLE; 2057 2058 mutex_exit(&evicted_state->mtx); 2059 mutex_exit(&old_state->mtx); 2060 } 2061 mutex_exit(hash_lock); 2062 2063 VERIFY(buf->b_efunc(buf) == 0); 2064 buf->b_efunc = NULL; 2065 buf->b_private = NULL; 2066 buf->b_hdr = NULL; 2067 kmem_cache_free(buf_cache, buf); 2068 return (1); 2069 } 2070 2071 /* 2072 * Release this buffer from the cache. This must be done 2073 * after a read and prior to modifying the buffer contents. 2074 * If the buffer has more than one reference, we must make 2075 * make a new hdr for the buffer. 2076 */ 2077 void 2078 arc_release(arc_buf_t *buf, void *tag) 2079 { 2080 arc_buf_hdr_t *hdr = buf->b_hdr; 2081 kmutex_t *hash_lock = HDR_LOCK(hdr); 2082 2083 /* this buffer is not on any list */ 2084 ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2085 2086 if (hdr->b_state == arc.anon) { 2087 /* this buffer is already released */ 2088 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2089 ASSERT(BUF_EMPTY(hdr)); 2090 ASSERT(buf->b_efunc == NULL); 2091 return; 2092 } 2093 2094 mutex_enter(hash_lock); 2095 2096 /* 2097 * Do we have more than one buf? 2098 */ 2099 if (hdr->b_buf != buf || buf->b_next != NULL) { 2100 arc_buf_hdr_t *nhdr; 2101 arc_buf_t **bufp; 2102 uint64_t blksz = hdr->b_size; 2103 spa_t *spa = hdr->b_spa; 2104 2105 ASSERT(hdr->b_datacnt > 1); 2106 /* 2107 * Pull the data off of this buf and attach it to 2108 * a new anonymous buf. 2109 */ 2110 (void) remove_reference(hdr, hash_lock, tag); 2111 bufp = &hdr->b_buf; 2112 while (*bufp != buf) 2113 bufp = &(*bufp)->b_next; 2114 *bufp = (*bufp)->b_next; 2115 2116 ASSERT3U(hdr->b_state->size, >=, hdr->b_size); 2117 atomic_add_64(&hdr->b_state->size, -hdr->b_size); 2118 if (refcount_is_zero(&hdr->b_refcnt)) { 2119 ASSERT3U(hdr->b_state->lsize, >=, hdr->b_size); 2120 atomic_add_64(&hdr->b_state->lsize, -hdr->b_size); 2121 } 2122 hdr->b_datacnt -= 1; 2123 2124 mutex_exit(hash_lock); 2125 2126 nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 2127 nhdr->b_size = blksz; 2128 nhdr->b_spa = spa; 2129 nhdr->b_buf = buf; 2130 nhdr->b_state = arc.anon; 2131 nhdr->b_arc_access = 0; 2132 nhdr->b_flags = 0; 2133 nhdr->b_datacnt = 1; 2134 buf->b_hdr = nhdr; 2135 buf->b_next = NULL; 2136 (void) refcount_add(&nhdr->b_refcnt, tag); 2137 atomic_add_64(&arc.anon->size, blksz); 2138 2139 hdr = nhdr; 2140 } else { 2141 ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2142 ASSERT(!list_link_active(&hdr->b_arc_node)); 2143 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2144 arc_change_state(arc.anon, hdr, hash_lock); 2145 hdr->b_arc_access = 0; 2146 mutex_exit(hash_lock); 2147 bzero(&hdr->b_dva, sizeof (dva_t)); 2148 hdr->b_birth = 0; 2149 hdr->b_cksum0 = 0; 2150 } 2151 buf->b_efunc = NULL; 2152 buf->b_private = NULL; 2153 } 2154 2155 int 2156 arc_released(arc_buf_t *buf) 2157 { 2158 return (buf->b_data != NULL && buf->b_hdr->b_state == arc.anon); 2159 } 2160 2161 int 2162 arc_has_callback(arc_buf_t *buf) 2163 { 2164 return (buf->b_efunc != NULL); 2165 } 2166 2167 #ifdef ZFS_DEBUG 2168 int 2169 arc_referenced(arc_buf_t *buf) 2170 { 2171 return (refcount_count(&buf->b_hdr->b_refcnt)); 2172 } 2173 #endif 2174 2175 static void 2176 arc_write_done(zio_t *zio) 2177 { 2178 arc_buf_t *buf; 2179 arc_buf_hdr_t *hdr; 2180 arc_callback_t *acb; 2181 2182 buf = zio->io_private; 2183 hdr = buf->b_hdr; 2184 acb = hdr->b_acb; 2185 hdr->b_acb = NULL; 2186 ASSERT(acb != NULL); 2187 2188 /* this buffer is on no lists and is not in the hash table */ 2189 ASSERT3P(hdr->b_state, ==, arc.anon); 2190 2191 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 2192 hdr->b_birth = zio->io_bp->blk_birth; 2193 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 2194 /* 2195 * If the block to be written was all-zero, we may have 2196 * compressed it away. In this case no write was performed 2197 * so there will be no dva/birth-date/checksum. The buffer 2198 * must therefor remain anonymous (and uncached). 2199 */ 2200 if (!BUF_EMPTY(hdr)) { 2201 arc_buf_hdr_t *exists; 2202 kmutex_t *hash_lock; 2203 2204 exists = buf_hash_insert(hdr, &hash_lock); 2205 if (exists) { 2206 /* 2207 * This can only happen if we overwrite for 2208 * sync-to-convergence, because we remove 2209 * buffers from the hash table when we arc_free(). 2210 */ 2211 ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 2212 BP_IDENTITY(zio->io_bp))); 2213 ASSERT3U(zio->io_bp_orig.blk_birth, ==, 2214 zio->io_bp->blk_birth); 2215 2216 ASSERT(refcount_is_zero(&exists->b_refcnt)); 2217 arc_change_state(arc.anon, exists, hash_lock); 2218 mutex_exit(hash_lock); 2219 arc_hdr_destroy(exists); 2220 exists = buf_hash_insert(hdr, &hash_lock); 2221 ASSERT3P(exists, ==, NULL); 2222 } 2223 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2224 arc_access(hdr, hash_lock); 2225 mutex_exit(hash_lock); 2226 } else if (acb->acb_done == NULL) { 2227 int destroy_hdr; 2228 /* 2229 * This is an anonymous buffer with no user callback, 2230 * destroy it if there are no active references. 2231 */ 2232 mutex_enter(&arc_eviction_mtx); 2233 destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 2234 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2235 mutex_exit(&arc_eviction_mtx); 2236 if (destroy_hdr) 2237 arc_hdr_destroy(hdr); 2238 } else { 2239 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2240 } 2241 2242 if (acb->acb_done) { 2243 ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 2244 acb->acb_done(zio, buf, acb->acb_private); 2245 } 2246 2247 kmem_free(acb, sizeof (arc_callback_t)); 2248 } 2249 2250 int 2251 arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 2252 uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 2253 arc_done_func_t *done, void *private, int priority, int flags, 2254 uint32_t arc_flags, zbookmark_t *zb) 2255 { 2256 arc_buf_hdr_t *hdr = buf->b_hdr; 2257 arc_callback_t *acb; 2258 zio_t *rzio; 2259 2260 /* this is a private buffer - no locking required */ 2261 ASSERT3P(hdr->b_state, ==, arc.anon); 2262 ASSERT(BUF_EMPTY(hdr)); 2263 ASSERT(!HDR_IO_ERROR(hdr)); 2264 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 2265 ASSERT(hdr->b_acb == 0); 2266 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2267 acb->acb_done = done; 2268 acb->acb_private = private; 2269 acb->acb_byteswap = (arc_byteswap_func_t *)-1; 2270 hdr->b_acb = acb; 2271 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2272 rzio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp, 2273 buf->b_data, hdr->b_size, arc_write_done, buf, priority, flags, zb); 2274 2275 if (arc_flags & ARC_WAIT) 2276 return (zio_wait(rzio)); 2277 2278 ASSERT(arc_flags & ARC_NOWAIT); 2279 zio_nowait(rzio); 2280 2281 return (0); 2282 } 2283 2284 int 2285 arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 2286 zio_done_func_t *done, void *private, uint32_t arc_flags) 2287 { 2288 arc_buf_hdr_t *ab; 2289 kmutex_t *hash_lock; 2290 zio_t *zio; 2291 2292 /* 2293 * If this buffer is in the cache, release it, so it 2294 * can be re-used. 2295 */ 2296 ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2297 if (ab != NULL) { 2298 /* 2299 * The checksum of blocks to free is not always 2300 * preserved (eg. on the deadlist). However, if it is 2301 * nonzero, it should match what we have in the cache. 2302 */ 2303 ASSERT(bp->blk_cksum.zc_word[0] == 0 || 2304 ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 2305 if (ab->b_state != arc.anon) 2306 arc_change_state(arc.anon, ab, hash_lock); 2307 if (HDR_IO_IN_PROGRESS(ab)) { 2308 /* 2309 * This should only happen when we prefetch. 2310 */ 2311 ASSERT(ab->b_flags & ARC_PREFETCH); 2312 ASSERT3U(ab->b_datacnt, ==, 1); 2313 ab->b_flags |= ARC_FREED_IN_READ; 2314 if (HDR_IN_HASH_TABLE(ab)) 2315 buf_hash_remove(ab); 2316 ab->b_arc_access = 0; 2317 bzero(&ab->b_dva, sizeof (dva_t)); 2318 ab->b_birth = 0; 2319 ab->b_cksum0 = 0; 2320 ab->b_buf->b_efunc = NULL; 2321 ab->b_buf->b_private = NULL; 2322 mutex_exit(hash_lock); 2323 } else if (refcount_is_zero(&ab->b_refcnt)) { 2324 mutex_exit(hash_lock); 2325 arc_hdr_destroy(ab); 2326 atomic_add_64(&arc.deleted, 1); 2327 } else { 2328 /* 2329 * We still have an active reference on this 2330 * buffer. This can happen, e.g., from 2331 * dbuf_unoverride(). 2332 */ 2333 ASSERT(!HDR_IN_HASH_TABLE(ab)); 2334 ab->b_arc_access = 0; 2335 bzero(&ab->b_dva, sizeof (dva_t)); 2336 ab->b_birth = 0; 2337 ab->b_cksum0 = 0; 2338 ab->b_buf->b_efunc = NULL; 2339 ab->b_buf->b_private = NULL; 2340 mutex_exit(hash_lock); 2341 } 2342 } 2343 2344 zio = zio_free(pio, spa, txg, bp, done, private); 2345 2346 if (arc_flags & ARC_WAIT) 2347 return (zio_wait(zio)); 2348 2349 ASSERT(arc_flags & ARC_NOWAIT); 2350 zio_nowait(zio); 2351 2352 return (0); 2353 } 2354 2355 void 2356 arc_tempreserve_clear(uint64_t tempreserve) 2357 { 2358 atomic_add_64(&arc_tempreserve, -tempreserve); 2359 ASSERT((int64_t)arc_tempreserve >= 0); 2360 } 2361 2362 int 2363 arc_tempreserve_space(uint64_t tempreserve) 2364 { 2365 #ifdef ZFS_DEBUG 2366 /* 2367 * Once in a while, fail for no reason. Everything should cope. 2368 */ 2369 if (spa_get_random(10000) == 0) { 2370 dprintf("forcing random failure\n"); 2371 return (ERESTART); 2372 } 2373 #endif 2374 if (tempreserve > arc.c/4 && !arc.no_grow) 2375 arc.c = MIN(arc.c_max, tempreserve * 4); 2376 if (tempreserve > arc.c) 2377 return (ENOMEM); 2378 2379 /* 2380 * Throttle writes when the amount of dirty data in the cache 2381 * gets too large. We try to keep the cache less than half full 2382 * of dirty blocks so that our sync times don't grow too large. 2383 * Note: if two requests come in concurrently, we might let them 2384 * both succeed, when one of them should fail. Not a huge deal. 2385 * 2386 * XXX The limit should be adjusted dynamically to keep the time 2387 * to sync a dataset fixed (around 1-5 seconds?). 2388 */ 2389 2390 if (tempreserve + arc_tempreserve + arc.anon->size > arc.c / 2 && 2391 arc_tempreserve + arc.anon->size > arc.c / 4) { 2392 dprintf("failing, arc_tempreserve=%lluK anon=%lluK " 2393 "tempreserve=%lluK arc.c=%lluK\n", 2394 arc_tempreserve>>10, arc.anon->lsize>>10, 2395 tempreserve>>10, arc.c>>10); 2396 return (ERESTART); 2397 } 2398 atomic_add_64(&arc_tempreserve, tempreserve); 2399 return (0); 2400 } 2401 2402 void 2403 arc_init(void) 2404 { 2405 mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL); 2406 mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 2407 cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 2408 2409 /* Convert seconds to clock ticks */ 2410 arc_min_prefetch_lifespan = 1 * hz; 2411 2412 /* Start out with 1/8 of all memory */ 2413 arc.c = physmem * PAGESIZE / 8; 2414 2415 #ifdef _KERNEL 2416 /* 2417 * On architectures where the physical memory can be larger 2418 * than the addressable space (intel in 32-bit mode), we may 2419 * need to limit the cache to 1/8 of VM size. 2420 */ 2421 arc.c = MIN(arc.c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 2422 #endif 2423 2424 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 2425 arc.c_min = MAX(arc.c / 4, 64<<20); 2426 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 2427 if (arc.c * 8 >= 1<<30) 2428 arc.c_max = (arc.c * 8) - (1<<30); 2429 else 2430 arc.c_max = arc.c_min; 2431 arc.c_max = MAX(arc.c * 6, arc.c_max); 2432 arc.c = arc.c_max; 2433 arc.p = (arc.c >> 1); 2434 2435 /* if kmem_flags are set, lets try to use less memory */ 2436 if (kmem_debugging()) 2437 arc.c = arc.c / 2; 2438 if (arc.c < arc.c_min) 2439 arc.c = arc.c_min; 2440 2441 arc.anon = &ARC_anon; 2442 arc.mru = &ARC_mru; 2443 arc.mru_ghost = &ARC_mru_ghost; 2444 arc.mfu = &ARC_mfu; 2445 arc.mfu_ghost = &ARC_mfu_ghost; 2446 arc.size = 0; 2447 2448 arc.hits = 0; 2449 arc.recycle_miss = 0; 2450 arc.evict_skip = 0; 2451 arc.mutex_miss = 0; 2452 2453 mutex_init(&arc.anon->mtx, NULL, MUTEX_DEFAULT, NULL); 2454 mutex_init(&arc.mru->mtx, NULL, MUTEX_DEFAULT, NULL); 2455 mutex_init(&arc.mru_ghost->mtx, NULL, MUTEX_DEFAULT, NULL); 2456 mutex_init(&arc.mfu->mtx, NULL, MUTEX_DEFAULT, NULL); 2457 mutex_init(&arc.mfu_ghost->mtx, NULL, MUTEX_DEFAULT, NULL); 2458 2459 list_create(&arc.mru->list, sizeof (arc_buf_hdr_t), 2460 offsetof(arc_buf_hdr_t, b_arc_node)); 2461 list_create(&arc.mru_ghost->list, sizeof (arc_buf_hdr_t), 2462 offsetof(arc_buf_hdr_t, b_arc_node)); 2463 list_create(&arc.mfu->list, sizeof (arc_buf_hdr_t), 2464 offsetof(arc_buf_hdr_t, b_arc_node)); 2465 list_create(&arc.mfu_ghost->list, sizeof (arc_buf_hdr_t), 2466 offsetof(arc_buf_hdr_t, b_arc_node)); 2467 2468 buf_init(); 2469 2470 arc_thread_exit = 0; 2471 arc_eviction_list = NULL; 2472 mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 2473 2474 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 2475 TS_RUN, minclsyspri); 2476 } 2477 2478 void 2479 arc_fini(void) 2480 { 2481 mutex_enter(&arc_reclaim_thr_lock); 2482 arc_thread_exit = 1; 2483 while (arc_thread_exit != 0) 2484 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 2485 mutex_exit(&arc_reclaim_thr_lock); 2486 2487 arc_flush(); 2488 2489 arc_dead = TRUE; 2490 2491 mutex_destroy(&arc_eviction_mtx); 2492 mutex_destroy(&arc_reclaim_lock); 2493 mutex_destroy(&arc_reclaim_thr_lock); 2494 cv_destroy(&arc_reclaim_thr_cv); 2495 2496 list_destroy(&arc.mru->list); 2497 list_destroy(&arc.mru_ghost->list); 2498 list_destroy(&arc.mfu->list); 2499 list_destroy(&arc.mfu_ghost->list); 2500 2501 mutex_destroy(&arc.anon->mtx); 2502 mutex_destroy(&arc.mru->mtx); 2503 mutex_destroy(&arc.mru_ghost->mtx); 2504 mutex_destroy(&arc.mfu->mtx); 2505 mutex_destroy(&arc.mfu_ghost->mtx); 2506 2507 buf_fini(); 2508 } 2509