1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * DVA-based Adjustable Replacement Cache 28 * 29 * While much of the theory of operation used here is 30 * based on the self-tuning, low overhead replacement cache 31 * presented by Megiddo and Modha at FAST 2003, there are some 32 * significant differences: 33 * 34 * 1. The Megiddo and Modha model assumes any page is evictable. 35 * Pages in its cache cannot be "locked" into memory. This makes 36 * the eviction algorithm simple: evict the last page in the list. 37 * This also make the performance characteristics easy to reason 38 * about. Our cache is not so simple. At any given moment, some 39 * subset of the blocks in the cache are un-evictable because we 40 * have handed out a reference to them. Blocks are only evictable 41 * when there are no external references active. This makes 42 * eviction far more problematic: we choose to evict the evictable 43 * blocks that are the "lowest" in the list. 44 * 45 * There are times when it is not possible to evict the requested 46 * space. In these circumstances we are unable to adjust the cache 47 * size. To prevent the cache growing unbounded at these times we 48 * implement a "cache throttle" that slows the flow of new data 49 * into the cache until we can make space available. 50 * 51 * 2. The Megiddo and Modha model assumes a fixed cache size. 52 * Pages are evicted when the cache is full and there is a cache 53 * miss. Our model has a variable sized cache. It grows with 54 * high use, but also tries to react to memory pressure from the 55 * operating system: decreasing its size when system memory is 56 * tight. 57 * 58 * 3. The Megiddo and Modha model assumes a fixed page size. All 59 * elements of the cache are therefor exactly the same size. So 60 * when adjusting the cache size following a cache miss, its simply 61 * a matter of choosing a single page to evict. In our model, we 62 * have variable sized cache blocks (rangeing from 512 bytes to 63 * 128K bytes). We therefor choose a set of blocks to evict to make 64 * space for a cache miss that approximates as closely as possible 65 * the space used by the new block. 66 * 67 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 68 * by N. Megiddo & D. Modha, FAST 2003 69 */ 70 71 /* 72 * The locking model: 73 * 74 * A new reference to a cache buffer can be obtained in two 75 * ways: 1) via a hash table lookup using the DVA as a key, 76 * or 2) via one of the ARC lists. The arc_read() interface 77 * uses method 1, while the internal arc algorithms for 78 * adjusting the cache use method 2. We therefor provide two 79 * types of locks: 1) the hash table lock array, and 2) the 80 * arc list locks. 81 * 82 * Buffers do not have their own mutexs, rather they rely on the 83 * hash table mutexs for the bulk of their protection (i.e. most 84 * fields in the arc_buf_hdr_t are protected by these mutexs). 85 * 86 * buf_hash_find() returns the appropriate mutex (held) when it 87 * locates the requested buffer in the hash table. It returns 88 * NULL for the mutex if the buffer was not in the table. 89 * 90 * buf_hash_remove() expects the appropriate hash mutex to be 91 * already held before it is invoked. 92 * 93 * Each arc state also has a mutex which is used to protect the 94 * buffer list associated with the state. When attempting to 95 * obtain a hash table lock while holding an arc list lock you 96 * must use: mutex_tryenter() to avoid deadlock. Also note that 97 * the active state mutex must be held before the ghost state mutex. 98 * 99 * Arc buffers may have an associated eviction callback function. 100 * This function will be invoked prior to removing the buffer (e.g. 101 * in arc_do_user_evicts()). Note however that the data associated 102 * with the buffer may be evicted prior to the callback. The callback 103 * must be made with *no locks held* (to prevent deadlock). Additionally, 104 * the users of callbacks must ensure that their private data is 105 * protected from simultaneous callbacks from arc_buf_evict() 106 * and arc_do_user_evicts(). 107 * 108 * Note that the majority of the performance stats are manipulated 109 * with atomic operations. 110 * 111 * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 112 * 113 * - L2ARC buflist creation 114 * - L2ARC buflist eviction 115 * - L2ARC write completion, which walks L2ARC buflists 116 * - ARC header destruction, as it removes from L2ARC buflists 117 * - ARC header release, as it removes from L2ARC buflists 118 */ 119 120 #include <sys/spa.h> 121 #include <sys/zio.h> 122 #include <sys/zfs_context.h> 123 #include <sys/arc.h> 124 #include <sys/refcount.h> 125 #include <sys/vdev.h> 126 #include <sys/vdev_impl.h> 127 #ifdef _KERNEL 128 #include <sys/vmsystm.h> 129 #include <vm/anon.h> 130 #include <sys/fs/swapnode.h> 131 #include <sys/dnlc.h> 132 #endif 133 #include <sys/callb.h> 134 #include <sys/kstat.h> 135 #include <zfs_fletcher.h> 136 137 static kmutex_t arc_reclaim_thr_lock; 138 static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 139 static uint8_t arc_thread_exit; 140 141 extern int zfs_write_limit_shift; 142 extern uint64_t zfs_write_limit_max; 143 extern kmutex_t zfs_write_limit_lock; 144 145 #define ARC_REDUCE_DNLC_PERCENT 3 146 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 147 148 typedef enum arc_reclaim_strategy { 149 ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 150 ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 151 } arc_reclaim_strategy_t; 152 153 /* number of seconds before growing cache again */ 154 static int arc_grow_retry = 60; 155 156 /* shift of arc_c for calculating both min and max arc_p */ 157 static int arc_p_min_shift = 4; 158 159 /* log2(fraction of arc to reclaim) */ 160 static int arc_shrink_shift = 5; 161 162 /* 163 * minimum lifespan of a prefetch block in clock ticks 164 * (initialized in arc_init()) 165 */ 166 static int arc_min_prefetch_lifespan; 167 168 static int arc_dead; 169 170 /* 171 * The arc has filled available memory and has now warmed up. 172 */ 173 static boolean_t arc_warm; 174 175 /* 176 * These tunables are for performance analysis. 177 */ 178 uint64_t zfs_arc_max; 179 uint64_t zfs_arc_min; 180 uint64_t zfs_arc_meta_limit = 0; 181 int zfs_arc_grow_retry = 0; 182 int zfs_arc_shrink_shift = 0; 183 int zfs_arc_p_min_shift = 0; 184 185 /* 186 * Note that buffers can be in one of 6 states: 187 * ARC_anon - anonymous (discussed below) 188 * ARC_mru - recently used, currently cached 189 * ARC_mru_ghost - recentely used, no longer in cache 190 * ARC_mfu - frequently used, currently cached 191 * ARC_mfu_ghost - frequently used, no longer in cache 192 * ARC_l2c_only - exists in L2ARC but not other states 193 * When there are no active references to the buffer, they are 194 * are linked onto a list in one of these arc states. These are 195 * the only buffers that can be evicted or deleted. Within each 196 * state there are multiple lists, one for meta-data and one for 197 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 198 * etc.) is tracked separately so that it can be managed more 199 * explicitly: favored over data, limited explicitly. 200 * 201 * Anonymous buffers are buffers that are not associated with 202 * a DVA. These are buffers that hold dirty block copies 203 * before they are written to stable storage. By definition, 204 * they are "ref'd" and are considered part of arc_mru 205 * that cannot be freed. Generally, they will aquire a DVA 206 * as they are written and migrate onto the arc_mru list. 207 * 208 * The ARC_l2c_only state is for buffers that are in the second 209 * level ARC but no longer in any of the ARC_m* lists. The second 210 * level ARC itself may also contain buffers that are in any of 211 * the ARC_m* states - meaning that a buffer can exist in two 212 * places. The reason for the ARC_l2c_only state is to keep the 213 * buffer header in the hash table, so that reads that hit the 214 * second level ARC benefit from these fast lookups. 215 */ 216 217 typedef struct arc_state { 218 list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 219 uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 220 uint64_t arcs_size; /* total amount of data in this state */ 221 kmutex_t arcs_mtx; 222 } arc_state_t; 223 224 /* The 6 states: */ 225 static arc_state_t ARC_anon; 226 static arc_state_t ARC_mru; 227 static arc_state_t ARC_mru_ghost; 228 static arc_state_t ARC_mfu; 229 static arc_state_t ARC_mfu_ghost; 230 static arc_state_t ARC_l2c_only; 231 232 typedef struct arc_stats { 233 kstat_named_t arcstat_hits; 234 kstat_named_t arcstat_misses; 235 kstat_named_t arcstat_demand_data_hits; 236 kstat_named_t arcstat_demand_data_misses; 237 kstat_named_t arcstat_demand_metadata_hits; 238 kstat_named_t arcstat_demand_metadata_misses; 239 kstat_named_t arcstat_prefetch_data_hits; 240 kstat_named_t arcstat_prefetch_data_misses; 241 kstat_named_t arcstat_prefetch_metadata_hits; 242 kstat_named_t arcstat_prefetch_metadata_misses; 243 kstat_named_t arcstat_mru_hits; 244 kstat_named_t arcstat_mru_ghost_hits; 245 kstat_named_t arcstat_mfu_hits; 246 kstat_named_t arcstat_mfu_ghost_hits; 247 kstat_named_t arcstat_deleted; 248 kstat_named_t arcstat_recycle_miss; 249 kstat_named_t arcstat_mutex_miss; 250 kstat_named_t arcstat_evict_skip; 251 kstat_named_t arcstat_evict_l2_cached; 252 kstat_named_t arcstat_evict_l2_eligible; 253 kstat_named_t arcstat_evict_l2_ineligible; 254 kstat_named_t arcstat_hash_elements; 255 kstat_named_t arcstat_hash_elements_max; 256 kstat_named_t arcstat_hash_collisions; 257 kstat_named_t arcstat_hash_chains; 258 kstat_named_t arcstat_hash_chain_max; 259 kstat_named_t arcstat_p; 260 kstat_named_t arcstat_c; 261 kstat_named_t arcstat_c_min; 262 kstat_named_t arcstat_c_max; 263 kstat_named_t arcstat_size; 264 kstat_named_t arcstat_hdr_size; 265 kstat_named_t arcstat_data_size; 266 kstat_named_t arcstat_other_size; 267 kstat_named_t arcstat_l2_hits; 268 kstat_named_t arcstat_l2_misses; 269 kstat_named_t arcstat_l2_feeds; 270 kstat_named_t arcstat_l2_rw_clash; 271 kstat_named_t arcstat_l2_read_bytes; 272 kstat_named_t arcstat_l2_write_bytes; 273 kstat_named_t arcstat_l2_writes_sent; 274 kstat_named_t arcstat_l2_writes_done; 275 kstat_named_t arcstat_l2_writes_error; 276 kstat_named_t arcstat_l2_writes_hdr_miss; 277 kstat_named_t arcstat_l2_evict_lock_retry; 278 kstat_named_t arcstat_l2_evict_reading; 279 kstat_named_t arcstat_l2_free_on_write; 280 kstat_named_t arcstat_l2_abort_lowmem; 281 kstat_named_t arcstat_l2_cksum_bad; 282 kstat_named_t arcstat_l2_io_error; 283 kstat_named_t arcstat_l2_size; 284 kstat_named_t arcstat_l2_hdr_size; 285 kstat_named_t arcstat_memory_throttle_count; 286 } arc_stats_t; 287 288 static arc_stats_t arc_stats = { 289 { "hits", KSTAT_DATA_UINT64 }, 290 { "misses", KSTAT_DATA_UINT64 }, 291 { "demand_data_hits", KSTAT_DATA_UINT64 }, 292 { "demand_data_misses", KSTAT_DATA_UINT64 }, 293 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 294 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 295 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 296 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 297 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 298 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 299 { "mru_hits", KSTAT_DATA_UINT64 }, 300 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 301 { "mfu_hits", KSTAT_DATA_UINT64 }, 302 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 303 { "deleted", KSTAT_DATA_UINT64 }, 304 { "recycle_miss", KSTAT_DATA_UINT64 }, 305 { "mutex_miss", KSTAT_DATA_UINT64 }, 306 { "evict_skip", KSTAT_DATA_UINT64 }, 307 { "evict_l2_cached", KSTAT_DATA_UINT64 }, 308 { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 309 { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 310 { "hash_elements", KSTAT_DATA_UINT64 }, 311 { "hash_elements_max", KSTAT_DATA_UINT64 }, 312 { "hash_collisions", KSTAT_DATA_UINT64 }, 313 { "hash_chains", KSTAT_DATA_UINT64 }, 314 { "hash_chain_max", KSTAT_DATA_UINT64 }, 315 { "p", KSTAT_DATA_UINT64 }, 316 { "c", KSTAT_DATA_UINT64 }, 317 { "c_min", KSTAT_DATA_UINT64 }, 318 { "c_max", KSTAT_DATA_UINT64 }, 319 { "size", KSTAT_DATA_UINT64 }, 320 { "hdr_size", KSTAT_DATA_UINT64 }, 321 { "data_size", KSTAT_DATA_UINT64 }, 322 { "other_size", KSTAT_DATA_UINT64 }, 323 { "l2_hits", KSTAT_DATA_UINT64 }, 324 { "l2_misses", KSTAT_DATA_UINT64 }, 325 { "l2_feeds", KSTAT_DATA_UINT64 }, 326 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 327 { "l2_read_bytes", KSTAT_DATA_UINT64 }, 328 { "l2_write_bytes", KSTAT_DATA_UINT64 }, 329 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 330 { "l2_writes_done", KSTAT_DATA_UINT64 }, 331 { "l2_writes_error", KSTAT_DATA_UINT64 }, 332 { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 333 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 334 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 335 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 336 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 337 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 338 { "l2_io_error", KSTAT_DATA_UINT64 }, 339 { "l2_size", KSTAT_DATA_UINT64 }, 340 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 341 { "memory_throttle_count", KSTAT_DATA_UINT64 } 342 }; 343 344 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 345 346 #define ARCSTAT_INCR(stat, val) \ 347 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 348 349 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 350 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 351 352 #define ARCSTAT_MAX(stat, val) { \ 353 uint64_t m; \ 354 while ((val) > (m = arc_stats.stat.value.ui64) && \ 355 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 356 continue; \ 357 } 358 359 #define ARCSTAT_MAXSTAT(stat) \ 360 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 361 362 /* 363 * We define a macro to allow ARC hits/misses to be easily broken down by 364 * two separate conditions, giving a total of four different subtypes for 365 * each of hits and misses (so eight statistics total). 366 */ 367 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 368 if (cond1) { \ 369 if (cond2) { \ 370 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 371 } else { \ 372 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 373 } \ 374 } else { \ 375 if (cond2) { \ 376 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 377 } else { \ 378 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 379 } \ 380 } 381 382 kstat_t *arc_ksp; 383 static arc_state_t *arc_anon; 384 static arc_state_t *arc_mru; 385 static arc_state_t *arc_mru_ghost; 386 static arc_state_t *arc_mfu; 387 static arc_state_t *arc_mfu_ghost; 388 static arc_state_t *arc_l2c_only; 389 390 /* 391 * There are several ARC variables that are critical to export as kstats -- 392 * but we don't want to have to grovel around in the kstat whenever we wish to 393 * manipulate them. For these variables, we therefore define them to be in 394 * terms of the statistic variable. This assures that we are not introducing 395 * the possibility of inconsistency by having shadow copies of the variables, 396 * while still allowing the code to be readable. 397 */ 398 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 399 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 400 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 401 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 402 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 403 404 static int arc_no_grow; /* Don't try to grow cache size */ 405 static uint64_t arc_tempreserve; 406 static uint64_t arc_loaned_bytes; 407 static uint64_t arc_meta_used; 408 static uint64_t arc_meta_limit; 409 static uint64_t arc_meta_max = 0; 410 411 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 412 413 typedef struct arc_callback arc_callback_t; 414 415 struct arc_callback { 416 void *acb_private; 417 arc_done_func_t *acb_done; 418 arc_buf_t *acb_buf; 419 zio_t *acb_zio_dummy; 420 arc_callback_t *acb_next; 421 }; 422 423 typedef struct arc_write_callback arc_write_callback_t; 424 425 struct arc_write_callback { 426 void *awcb_private; 427 arc_done_func_t *awcb_ready; 428 arc_done_func_t *awcb_done; 429 arc_buf_t *awcb_buf; 430 }; 431 432 struct arc_buf_hdr { 433 /* protected by hash lock */ 434 dva_t b_dva; 435 uint64_t b_birth; 436 uint64_t b_cksum0; 437 438 kmutex_t b_freeze_lock; 439 zio_cksum_t *b_freeze_cksum; 440 441 arc_buf_hdr_t *b_hash_next; 442 arc_buf_t *b_buf; 443 uint32_t b_flags; 444 uint32_t b_datacnt; 445 446 arc_callback_t *b_acb; 447 kcondvar_t b_cv; 448 449 /* immutable */ 450 arc_buf_contents_t b_type; 451 uint64_t b_size; 452 uint64_t b_spa; 453 454 /* protected by arc state mutex */ 455 arc_state_t *b_state; 456 list_node_t b_arc_node; 457 458 /* updated atomically */ 459 clock_t b_arc_access; 460 461 /* self protecting */ 462 refcount_t b_refcnt; 463 464 l2arc_buf_hdr_t *b_l2hdr; 465 list_node_t b_l2node; 466 }; 467 468 static arc_buf_t *arc_eviction_list; 469 static kmutex_t arc_eviction_mtx; 470 static arc_buf_hdr_t arc_eviction_hdr; 471 static void arc_get_data_buf(arc_buf_t *buf); 472 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 473 static int arc_evict_needed(arc_buf_contents_t type); 474 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes); 475 476 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); 477 478 #define GHOST_STATE(state) \ 479 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 480 (state) == arc_l2c_only) 481 482 /* 483 * Private ARC flags. These flags are private ARC only flags that will show up 484 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 485 * be passed in as arc_flags in things like arc_read. However, these flags 486 * should never be passed and should only be set by ARC code. When adding new 487 * public flags, make sure not to smash the private ones. 488 */ 489 490 #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 491 #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 492 #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 493 #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 494 #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 495 #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 496 #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 497 #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 498 #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 499 #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 500 501 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 502 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 503 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 504 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_PREFETCH) 505 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 506 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 507 #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 508 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 509 #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 510 (hdr)->b_l2hdr != NULL) 511 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 512 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 513 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 514 515 /* 516 * Other sizes 517 */ 518 519 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 520 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 521 522 /* 523 * Hash table routines 524 */ 525 526 #define HT_LOCK_PAD 64 527 528 struct ht_lock { 529 kmutex_t ht_lock; 530 #ifdef _KERNEL 531 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 532 #endif 533 }; 534 535 #define BUF_LOCKS 256 536 typedef struct buf_hash_table { 537 uint64_t ht_mask; 538 arc_buf_hdr_t **ht_table; 539 struct ht_lock ht_locks[BUF_LOCKS]; 540 } buf_hash_table_t; 541 542 static buf_hash_table_t buf_hash_table; 543 544 #define BUF_HASH_INDEX(spa, dva, birth) \ 545 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 546 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 547 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 548 #define HDR_LOCK(buf) \ 549 (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 550 551 uint64_t zfs_crc64_table[256]; 552 553 /* 554 * Level 2 ARC 555 */ 556 557 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 558 #define L2ARC_HEADROOM 2 /* num of writes */ 559 #define L2ARC_FEED_SECS 1 /* caching interval secs */ 560 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 561 562 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 563 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 564 565 /* 566 * L2ARC Performance Tunables 567 */ 568 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 569 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 570 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 571 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 572 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 573 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 574 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 575 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 576 577 /* 578 * L2ARC Internals 579 */ 580 typedef struct l2arc_dev { 581 vdev_t *l2ad_vdev; /* vdev */ 582 spa_t *l2ad_spa; /* spa */ 583 uint64_t l2ad_hand; /* next write location */ 584 uint64_t l2ad_write; /* desired write size, bytes */ 585 uint64_t l2ad_boost; /* warmup write boost, bytes */ 586 uint64_t l2ad_start; /* first addr on device */ 587 uint64_t l2ad_end; /* last addr on device */ 588 uint64_t l2ad_evict; /* last addr eviction reached */ 589 boolean_t l2ad_first; /* first sweep through */ 590 boolean_t l2ad_writing; /* currently writing */ 591 list_t *l2ad_buflist; /* buffer list */ 592 list_node_t l2ad_node; /* device list node */ 593 } l2arc_dev_t; 594 595 static list_t L2ARC_dev_list; /* device list */ 596 static list_t *l2arc_dev_list; /* device list pointer */ 597 static kmutex_t l2arc_dev_mtx; /* device list mutex */ 598 static l2arc_dev_t *l2arc_dev_last; /* last device used */ 599 static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 600 static list_t L2ARC_free_on_write; /* free after write buf list */ 601 static list_t *l2arc_free_on_write; /* free after write list ptr */ 602 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 603 static uint64_t l2arc_ndev; /* number of devices */ 604 605 typedef struct l2arc_read_callback { 606 arc_buf_t *l2rcb_buf; /* read buffer */ 607 spa_t *l2rcb_spa; /* spa */ 608 blkptr_t l2rcb_bp; /* original blkptr */ 609 zbookmark_t l2rcb_zb; /* original bookmark */ 610 int l2rcb_flags; /* original flags */ 611 } l2arc_read_callback_t; 612 613 typedef struct l2arc_write_callback { 614 l2arc_dev_t *l2wcb_dev; /* device info */ 615 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 616 } l2arc_write_callback_t; 617 618 struct l2arc_buf_hdr { 619 /* protected by arc_buf_hdr mutex */ 620 l2arc_dev_t *b_dev; /* L2ARC device */ 621 uint64_t b_daddr; /* disk address, offset byte */ 622 }; 623 624 typedef struct l2arc_data_free { 625 /* protected by l2arc_free_on_write_mtx */ 626 void *l2df_data; 627 size_t l2df_size; 628 void (*l2df_func)(void *, size_t); 629 list_node_t l2df_list_node; 630 } l2arc_data_free_t; 631 632 static kmutex_t l2arc_feed_thr_lock; 633 static kcondvar_t l2arc_feed_thr_cv; 634 static uint8_t l2arc_thread_exit; 635 636 static void l2arc_read_done(zio_t *zio); 637 static void l2arc_hdr_stat_add(void); 638 static void l2arc_hdr_stat_remove(void); 639 640 static uint64_t 641 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 642 { 643 uint8_t *vdva = (uint8_t *)dva; 644 uint64_t crc = -1ULL; 645 int i; 646 647 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 648 649 for (i = 0; i < sizeof (dva_t); i++) 650 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 651 652 crc ^= (spa>>8) ^ birth; 653 654 return (crc); 655 } 656 657 #define BUF_EMPTY(buf) \ 658 ((buf)->b_dva.dva_word[0] == 0 && \ 659 (buf)->b_dva.dva_word[1] == 0 && \ 660 (buf)->b_birth == 0) 661 662 #define BUF_EQUAL(spa, dva, birth, buf) \ 663 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 664 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 665 ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 666 667 static arc_buf_hdr_t * 668 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 669 { 670 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 671 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 672 arc_buf_hdr_t *buf; 673 674 mutex_enter(hash_lock); 675 for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 676 buf = buf->b_hash_next) { 677 if (BUF_EQUAL(spa, dva, birth, buf)) { 678 *lockp = hash_lock; 679 return (buf); 680 } 681 } 682 mutex_exit(hash_lock); 683 *lockp = NULL; 684 return (NULL); 685 } 686 687 /* 688 * Insert an entry into the hash table. If there is already an element 689 * equal to elem in the hash table, then the already existing element 690 * will be returned and the new element will not be inserted. 691 * Otherwise returns NULL. 692 */ 693 static arc_buf_hdr_t * 694 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 695 { 696 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 697 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 698 arc_buf_hdr_t *fbuf; 699 uint32_t i; 700 701 ASSERT(!HDR_IN_HASH_TABLE(buf)); 702 *lockp = hash_lock; 703 mutex_enter(hash_lock); 704 for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 705 fbuf = fbuf->b_hash_next, i++) { 706 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 707 return (fbuf); 708 } 709 710 buf->b_hash_next = buf_hash_table.ht_table[idx]; 711 buf_hash_table.ht_table[idx] = buf; 712 buf->b_flags |= ARC_IN_HASH_TABLE; 713 714 /* collect some hash table performance data */ 715 if (i > 0) { 716 ARCSTAT_BUMP(arcstat_hash_collisions); 717 if (i == 1) 718 ARCSTAT_BUMP(arcstat_hash_chains); 719 720 ARCSTAT_MAX(arcstat_hash_chain_max, i); 721 } 722 723 ARCSTAT_BUMP(arcstat_hash_elements); 724 ARCSTAT_MAXSTAT(arcstat_hash_elements); 725 726 return (NULL); 727 } 728 729 static void 730 buf_hash_remove(arc_buf_hdr_t *buf) 731 { 732 arc_buf_hdr_t *fbuf, **bufp; 733 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 734 735 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 736 ASSERT(HDR_IN_HASH_TABLE(buf)); 737 738 bufp = &buf_hash_table.ht_table[idx]; 739 while ((fbuf = *bufp) != buf) { 740 ASSERT(fbuf != NULL); 741 bufp = &fbuf->b_hash_next; 742 } 743 *bufp = buf->b_hash_next; 744 buf->b_hash_next = NULL; 745 buf->b_flags &= ~ARC_IN_HASH_TABLE; 746 747 /* collect some hash table performance data */ 748 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 749 750 if (buf_hash_table.ht_table[idx] && 751 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 752 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 753 } 754 755 /* 756 * Global data structures and functions for the buf kmem cache. 757 */ 758 static kmem_cache_t *hdr_cache; 759 static kmem_cache_t *buf_cache; 760 761 static void 762 buf_fini(void) 763 { 764 int i; 765 766 kmem_free(buf_hash_table.ht_table, 767 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 768 for (i = 0; i < BUF_LOCKS; i++) 769 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 770 kmem_cache_destroy(hdr_cache); 771 kmem_cache_destroy(buf_cache); 772 } 773 774 /* 775 * Constructor callback - called when the cache is empty 776 * and a new buf is requested. 777 */ 778 /* ARGSUSED */ 779 static int 780 hdr_cons(void *vbuf, void *unused, int kmflag) 781 { 782 arc_buf_hdr_t *buf = vbuf; 783 784 bzero(buf, sizeof (arc_buf_hdr_t)); 785 refcount_create(&buf->b_refcnt); 786 cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 787 mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 788 arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 789 790 return (0); 791 } 792 793 /* ARGSUSED */ 794 static int 795 buf_cons(void *vbuf, void *unused, int kmflag) 796 { 797 arc_buf_t *buf = vbuf; 798 799 bzero(buf, sizeof (arc_buf_t)); 800 rw_init(&buf->b_lock, NULL, RW_DEFAULT, NULL); 801 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 802 803 return (0); 804 } 805 806 /* 807 * Destructor callback - called when a cached buf is 808 * no longer required. 809 */ 810 /* ARGSUSED */ 811 static void 812 hdr_dest(void *vbuf, void *unused) 813 { 814 arc_buf_hdr_t *buf = vbuf; 815 816 ASSERT(BUF_EMPTY(buf)); 817 refcount_destroy(&buf->b_refcnt); 818 cv_destroy(&buf->b_cv); 819 mutex_destroy(&buf->b_freeze_lock); 820 arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS); 821 } 822 823 /* ARGSUSED */ 824 static void 825 buf_dest(void *vbuf, void *unused) 826 { 827 arc_buf_t *buf = vbuf; 828 829 rw_destroy(&buf->b_lock); 830 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 831 } 832 833 /* 834 * Reclaim callback -- invoked when memory is low. 835 */ 836 /* ARGSUSED */ 837 static void 838 hdr_recl(void *unused) 839 { 840 dprintf("hdr_recl called\n"); 841 /* 842 * umem calls the reclaim func when we destroy the buf cache, 843 * which is after we do arc_fini(). 844 */ 845 if (!arc_dead) 846 cv_signal(&arc_reclaim_thr_cv); 847 } 848 849 static void 850 buf_init(void) 851 { 852 uint64_t *ct; 853 uint64_t hsize = 1ULL << 12; 854 int i, j; 855 856 /* 857 * The hash table is big enough to fill all of physical memory 858 * with an average 64K block size. The table will take up 859 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 860 */ 861 while (hsize * 65536 < physmem * PAGESIZE) 862 hsize <<= 1; 863 retry: 864 buf_hash_table.ht_mask = hsize - 1; 865 buf_hash_table.ht_table = 866 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 867 if (buf_hash_table.ht_table == NULL) { 868 ASSERT(hsize > (1ULL << 8)); 869 hsize >>= 1; 870 goto retry; 871 } 872 873 hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 874 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 875 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 876 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 877 878 for (i = 0; i < 256; i++) 879 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 880 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 881 882 for (i = 0; i < BUF_LOCKS; i++) { 883 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 884 NULL, MUTEX_DEFAULT, NULL); 885 } 886 } 887 888 #define ARC_MINTIME (hz>>4) /* 62 ms */ 889 890 static void 891 arc_cksum_verify(arc_buf_t *buf) 892 { 893 zio_cksum_t zc; 894 895 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 896 return; 897 898 mutex_enter(&buf->b_hdr->b_freeze_lock); 899 if (buf->b_hdr->b_freeze_cksum == NULL || 900 (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 901 mutex_exit(&buf->b_hdr->b_freeze_lock); 902 return; 903 } 904 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 905 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 906 panic("buffer modified while frozen!"); 907 mutex_exit(&buf->b_hdr->b_freeze_lock); 908 } 909 910 static int 911 arc_cksum_equal(arc_buf_t *buf) 912 { 913 zio_cksum_t zc; 914 int equal; 915 916 mutex_enter(&buf->b_hdr->b_freeze_lock); 917 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 918 equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 919 mutex_exit(&buf->b_hdr->b_freeze_lock); 920 921 return (equal); 922 } 923 924 static void 925 arc_cksum_compute(arc_buf_t *buf, boolean_t force) 926 { 927 if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 928 return; 929 930 mutex_enter(&buf->b_hdr->b_freeze_lock); 931 if (buf->b_hdr->b_freeze_cksum != NULL) { 932 mutex_exit(&buf->b_hdr->b_freeze_lock); 933 return; 934 } 935 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 936 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 937 buf->b_hdr->b_freeze_cksum); 938 mutex_exit(&buf->b_hdr->b_freeze_lock); 939 } 940 941 void 942 arc_buf_thaw(arc_buf_t *buf) 943 { 944 if (zfs_flags & ZFS_DEBUG_MODIFY) { 945 if (buf->b_hdr->b_state != arc_anon) 946 panic("modifying non-anon buffer!"); 947 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 948 panic("modifying buffer while i/o in progress!"); 949 arc_cksum_verify(buf); 950 } 951 952 mutex_enter(&buf->b_hdr->b_freeze_lock); 953 if (buf->b_hdr->b_freeze_cksum != NULL) { 954 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 955 buf->b_hdr->b_freeze_cksum = NULL; 956 } 957 mutex_exit(&buf->b_hdr->b_freeze_lock); 958 } 959 960 void 961 arc_buf_freeze(arc_buf_t *buf) 962 { 963 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 964 return; 965 966 ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 967 buf->b_hdr->b_state == arc_anon); 968 arc_cksum_compute(buf, B_FALSE); 969 } 970 971 static void 972 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 973 { 974 ASSERT(MUTEX_HELD(hash_lock)); 975 976 if ((refcount_add(&ab->b_refcnt, tag) == 1) && 977 (ab->b_state != arc_anon)) { 978 uint64_t delta = ab->b_size * ab->b_datacnt; 979 list_t *list = &ab->b_state->arcs_list[ab->b_type]; 980 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 981 982 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 983 mutex_enter(&ab->b_state->arcs_mtx); 984 ASSERT(list_link_active(&ab->b_arc_node)); 985 list_remove(list, ab); 986 if (GHOST_STATE(ab->b_state)) { 987 ASSERT3U(ab->b_datacnt, ==, 0); 988 ASSERT3P(ab->b_buf, ==, NULL); 989 delta = ab->b_size; 990 } 991 ASSERT(delta > 0); 992 ASSERT3U(*size, >=, delta); 993 atomic_add_64(size, -delta); 994 mutex_exit(&ab->b_state->arcs_mtx); 995 /* remove the prefetch flag if we get a reference */ 996 if (ab->b_flags & ARC_PREFETCH) 997 ab->b_flags &= ~ARC_PREFETCH; 998 } 999 } 1000 1001 static int 1002 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 1003 { 1004 int cnt; 1005 arc_state_t *state = ab->b_state; 1006 1007 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 1008 ASSERT(!GHOST_STATE(state)); 1009 1010 if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 1011 (state != arc_anon)) { 1012 uint64_t *size = &state->arcs_lsize[ab->b_type]; 1013 1014 ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 1015 mutex_enter(&state->arcs_mtx); 1016 ASSERT(!list_link_active(&ab->b_arc_node)); 1017 list_insert_head(&state->arcs_list[ab->b_type], ab); 1018 ASSERT(ab->b_datacnt > 0); 1019 atomic_add_64(size, ab->b_size * ab->b_datacnt); 1020 mutex_exit(&state->arcs_mtx); 1021 } 1022 return (cnt); 1023 } 1024 1025 /* 1026 * Move the supplied buffer to the indicated state. The mutex 1027 * for the buffer must be held by the caller. 1028 */ 1029 static void 1030 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 1031 { 1032 arc_state_t *old_state = ab->b_state; 1033 int64_t refcnt = refcount_count(&ab->b_refcnt); 1034 uint64_t from_delta, to_delta; 1035 1036 ASSERT(MUTEX_HELD(hash_lock)); 1037 ASSERT(new_state != old_state); 1038 ASSERT(refcnt == 0 || ab->b_datacnt > 0); 1039 ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 1040 ASSERT(ab->b_datacnt <= 1 || new_state != arc_anon); 1041 ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon); 1042 1043 from_delta = to_delta = ab->b_datacnt * ab->b_size; 1044 1045 /* 1046 * If this buffer is evictable, transfer it from the 1047 * old state list to the new state list. 1048 */ 1049 if (refcnt == 0) { 1050 if (old_state != arc_anon) { 1051 int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 1052 uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1053 1054 if (use_mutex) 1055 mutex_enter(&old_state->arcs_mtx); 1056 1057 ASSERT(list_link_active(&ab->b_arc_node)); 1058 list_remove(&old_state->arcs_list[ab->b_type], ab); 1059 1060 /* 1061 * If prefetching out of the ghost cache, 1062 * we will have a non-null datacnt. 1063 */ 1064 if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 1065 /* ghost elements have a ghost size */ 1066 ASSERT(ab->b_buf == NULL); 1067 from_delta = ab->b_size; 1068 } 1069 ASSERT3U(*size, >=, from_delta); 1070 atomic_add_64(size, -from_delta); 1071 1072 if (use_mutex) 1073 mutex_exit(&old_state->arcs_mtx); 1074 } 1075 if (new_state != arc_anon) { 1076 int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 1077 uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1078 1079 if (use_mutex) 1080 mutex_enter(&new_state->arcs_mtx); 1081 1082 list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1083 1084 /* ghost elements have a ghost size */ 1085 if (GHOST_STATE(new_state)) { 1086 ASSERT(ab->b_datacnt == 0); 1087 ASSERT(ab->b_buf == NULL); 1088 to_delta = ab->b_size; 1089 } 1090 atomic_add_64(size, to_delta); 1091 1092 if (use_mutex) 1093 mutex_exit(&new_state->arcs_mtx); 1094 } 1095 } 1096 1097 ASSERT(!BUF_EMPTY(ab)); 1098 if (new_state == arc_anon) { 1099 buf_hash_remove(ab); 1100 } 1101 1102 /* adjust state sizes */ 1103 if (to_delta) 1104 atomic_add_64(&new_state->arcs_size, to_delta); 1105 if (from_delta) { 1106 ASSERT3U(old_state->arcs_size, >=, from_delta); 1107 atomic_add_64(&old_state->arcs_size, -from_delta); 1108 } 1109 ab->b_state = new_state; 1110 1111 /* adjust l2arc hdr stats */ 1112 if (new_state == arc_l2c_only) 1113 l2arc_hdr_stat_add(); 1114 else if (old_state == arc_l2c_only) 1115 l2arc_hdr_stat_remove(); 1116 } 1117 1118 void 1119 arc_space_consume(uint64_t space, arc_space_type_t type) 1120 { 1121 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1122 1123 switch (type) { 1124 case ARC_SPACE_DATA: 1125 ARCSTAT_INCR(arcstat_data_size, space); 1126 break; 1127 case ARC_SPACE_OTHER: 1128 ARCSTAT_INCR(arcstat_other_size, space); 1129 break; 1130 case ARC_SPACE_HDRS: 1131 ARCSTAT_INCR(arcstat_hdr_size, space); 1132 break; 1133 case ARC_SPACE_L2HDRS: 1134 ARCSTAT_INCR(arcstat_l2_hdr_size, space); 1135 break; 1136 } 1137 1138 atomic_add_64(&arc_meta_used, space); 1139 atomic_add_64(&arc_size, space); 1140 } 1141 1142 void 1143 arc_space_return(uint64_t space, arc_space_type_t type) 1144 { 1145 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 1146 1147 switch (type) { 1148 case ARC_SPACE_DATA: 1149 ARCSTAT_INCR(arcstat_data_size, -space); 1150 break; 1151 case ARC_SPACE_OTHER: 1152 ARCSTAT_INCR(arcstat_other_size, -space); 1153 break; 1154 case ARC_SPACE_HDRS: 1155 ARCSTAT_INCR(arcstat_hdr_size, -space); 1156 break; 1157 case ARC_SPACE_L2HDRS: 1158 ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 1159 break; 1160 } 1161 1162 ASSERT(arc_meta_used >= space); 1163 if (arc_meta_max < arc_meta_used) 1164 arc_meta_max = arc_meta_used; 1165 atomic_add_64(&arc_meta_used, -space); 1166 ASSERT(arc_size >= space); 1167 atomic_add_64(&arc_size, -space); 1168 } 1169 1170 void * 1171 arc_data_buf_alloc(uint64_t size) 1172 { 1173 if (arc_evict_needed(ARC_BUFC_DATA)) 1174 cv_signal(&arc_reclaim_thr_cv); 1175 atomic_add_64(&arc_size, size); 1176 return (zio_data_buf_alloc(size)); 1177 } 1178 1179 void 1180 arc_data_buf_free(void *buf, uint64_t size) 1181 { 1182 zio_data_buf_free(buf, size); 1183 ASSERT(arc_size >= size); 1184 atomic_add_64(&arc_size, -size); 1185 } 1186 1187 arc_buf_t * 1188 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1189 { 1190 arc_buf_hdr_t *hdr; 1191 arc_buf_t *buf; 1192 1193 ASSERT3U(size, >, 0); 1194 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1195 ASSERT(BUF_EMPTY(hdr)); 1196 hdr->b_size = size; 1197 hdr->b_type = type; 1198 hdr->b_spa = spa_guid(spa); 1199 hdr->b_state = arc_anon; 1200 hdr->b_arc_access = 0; 1201 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1202 buf->b_hdr = hdr; 1203 buf->b_data = NULL; 1204 buf->b_efunc = NULL; 1205 buf->b_private = NULL; 1206 buf->b_next = NULL; 1207 hdr->b_buf = buf; 1208 arc_get_data_buf(buf); 1209 hdr->b_datacnt = 1; 1210 hdr->b_flags = 0; 1211 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1212 (void) refcount_add(&hdr->b_refcnt, tag); 1213 1214 return (buf); 1215 } 1216 1217 static char *arc_onloan_tag = "onloan"; 1218 1219 /* 1220 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 1221 * flight data by arc_tempreserve_space() until they are "returned". Loaned 1222 * buffers must be returned to the arc before they can be used by the DMU or 1223 * freed. 1224 */ 1225 arc_buf_t * 1226 arc_loan_buf(spa_t *spa, int size) 1227 { 1228 arc_buf_t *buf; 1229 1230 buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA); 1231 1232 atomic_add_64(&arc_loaned_bytes, size); 1233 return (buf); 1234 } 1235 1236 /* 1237 * Return a loaned arc buffer to the arc. 1238 */ 1239 void 1240 arc_return_buf(arc_buf_t *buf, void *tag) 1241 { 1242 arc_buf_hdr_t *hdr = buf->b_hdr; 1243 1244 ASSERT(buf->b_data != NULL); 1245 (void) refcount_add(&hdr->b_refcnt, tag); 1246 (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag); 1247 1248 atomic_add_64(&arc_loaned_bytes, -hdr->b_size); 1249 } 1250 1251 /* Detach an arc_buf from a dbuf (tag) */ 1252 void 1253 arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 1254 { 1255 arc_buf_hdr_t *hdr; 1256 1257 rw_enter(&buf->b_lock, RW_WRITER); 1258 ASSERT(buf->b_data != NULL); 1259 hdr = buf->b_hdr; 1260 (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag); 1261 (void) refcount_remove(&hdr->b_refcnt, tag); 1262 buf->b_efunc = NULL; 1263 buf->b_private = NULL; 1264 1265 atomic_add_64(&arc_loaned_bytes, hdr->b_size); 1266 rw_exit(&buf->b_lock); 1267 } 1268 1269 static arc_buf_t * 1270 arc_buf_clone(arc_buf_t *from) 1271 { 1272 arc_buf_t *buf; 1273 arc_buf_hdr_t *hdr = from->b_hdr; 1274 uint64_t size = hdr->b_size; 1275 1276 ASSERT(hdr->b_state != arc_anon); 1277 1278 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1279 buf->b_hdr = hdr; 1280 buf->b_data = NULL; 1281 buf->b_efunc = NULL; 1282 buf->b_private = NULL; 1283 buf->b_next = hdr->b_buf; 1284 hdr->b_buf = buf; 1285 arc_get_data_buf(buf); 1286 bcopy(from->b_data, buf->b_data, size); 1287 hdr->b_datacnt += 1; 1288 return (buf); 1289 } 1290 1291 void 1292 arc_buf_add_ref(arc_buf_t *buf, void* tag) 1293 { 1294 arc_buf_hdr_t *hdr; 1295 kmutex_t *hash_lock; 1296 1297 /* 1298 * Check to see if this buffer is evicted. Callers 1299 * must verify b_data != NULL to know if the add_ref 1300 * was successful. 1301 */ 1302 rw_enter(&buf->b_lock, RW_READER); 1303 if (buf->b_data == NULL) { 1304 rw_exit(&buf->b_lock); 1305 return; 1306 } 1307 hdr = buf->b_hdr; 1308 ASSERT(hdr != NULL); 1309 hash_lock = HDR_LOCK(hdr); 1310 mutex_enter(hash_lock); 1311 rw_exit(&buf->b_lock); 1312 1313 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1314 add_reference(hdr, hash_lock, tag); 1315 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 1316 arc_access(hdr, hash_lock); 1317 mutex_exit(hash_lock); 1318 ARCSTAT_BUMP(arcstat_hits); 1319 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 1320 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 1321 data, metadata, hits); 1322 } 1323 1324 /* 1325 * Free the arc data buffer. If it is an l2arc write in progress, 1326 * the buffer is placed on l2arc_free_on_write to be freed later. 1327 */ 1328 static void 1329 arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t), 1330 void *data, size_t size) 1331 { 1332 if (HDR_L2_WRITING(hdr)) { 1333 l2arc_data_free_t *df; 1334 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1335 df->l2df_data = data; 1336 df->l2df_size = size; 1337 df->l2df_func = free_func; 1338 mutex_enter(&l2arc_free_on_write_mtx); 1339 list_insert_head(l2arc_free_on_write, df); 1340 mutex_exit(&l2arc_free_on_write_mtx); 1341 ARCSTAT_BUMP(arcstat_l2_free_on_write); 1342 } else { 1343 free_func(data, size); 1344 } 1345 } 1346 1347 static void 1348 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1349 { 1350 arc_buf_t **bufp; 1351 1352 /* free up data associated with the buf */ 1353 if (buf->b_data) { 1354 arc_state_t *state = buf->b_hdr->b_state; 1355 uint64_t size = buf->b_hdr->b_size; 1356 arc_buf_contents_t type = buf->b_hdr->b_type; 1357 1358 arc_cksum_verify(buf); 1359 1360 if (!recycle) { 1361 if (type == ARC_BUFC_METADATA) { 1362 arc_buf_data_free(buf->b_hdr, zio_buf_free, 1363 buf->b_data, size); 1364 arc_space_return(size, ARC_SPACE_DATA); 1365 } else { 1366 ASSERT(type == ARC_BUFC_DATA); 1367 arc_buf_data_free(buf->b_hdr, 1368 zio_data_buf_free, buf->b_data, size); 1369 ARCSTAT_INCR(arcstat_data_size, -size); 1370 atomic_add_64(&arc_size, -size); 1371 } 1372 } 1373 if (list_link_active(&buf->b_hdr->b_arc_node)) { 1374 uint64_t *cnt = &state->arcs_lsize[type]; 1375 1376 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 1377 ASSERT(state != arc_anon); 1378 1379 ASSERT3U(*cnt, >=, size); 1380 atomic_add_64(cnt, -size); 1381 } 1382 ASSERT3U(state->arcs_size, >=, size); 1383 atomic_add_64(&state->arcs_size, -size); 1384 buf->b_data = NULL; 1385 ASSERT(buf->b_hdr->b_datacnt > 0); 1386 buf->b_hdr->b_datacnt -= 1; 1387 } 1388 1389 /* only remove the buf if requested */ 1390 if (!all) 1391 return; 1392 1393 /* remove the buf from the hdr list */ 1394 for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1395 continue; 1396 *bufp = buf->b_next; 1397 1398 ASSERT(buf->b_efunc == NULL); 1399 1400 /* clean up the buf */ 1401 buf->b_hdr = NULL; 1402 kmem_cache_free(buf_cache, buf); 1403 } 1404 1405 static void 1406 arc_hdr_destroy(arc_buf_hdr_t *hdr) 1407 { 1408 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1409 ASSERT3P(hdr->b_state, ==, arc_anon); 1410 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1411 l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr; 1412 1413 if (l2hdr != NULL) { 1414 boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx); 1415 /* 1416 * To prevent arc_free() and l2arc_evict() from 1417 * attempting to free the same buffer at the same time, 1418 * a FREE_IN_PROGRESS flag is given to arc_free() to 1419 * give it priority. l2arc_evict() can't destroy this 1420 * header while we are waiting on l2arc_buflist_mtx. 1421 * 1422 * The hdr may be removed from l2ad_buflist before we 1423 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked. 1424 */ 1425 if (!buflist_held) { 1426 mutex_enter(&l2arc_buflist_mtx); 1427 l2hdr = hdr->b_l2hdr; 1428 } 1429 1430 if (l2hdr != NULL) { 1431 list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 1432 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1433 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 1434 if (hdr->b_state == arc_l2c_only) 1435 l2arc_hdr_stat_remove(); 1436 hdr->b_l2hdr = NULL; 1437 } 1438 1439 if (!buflist_held) 1440 mutex_exit(&l2arc_buflist_mtx); 1441 } 1442 1443 if (!BUF_EMPTY(hdr)) { 1444 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1445 bzero(&hdr->b_dva, sizeof (dva_t)); 1446 hdr->b_birth = 0; 1447 hdr->b_cksum0 = 0; 1448 } 1449 while (hdr->b_buf) { 1450 arc_buf_t *buf = hdr->b_buf; 1451 1452 if (buf->b_efunc) { 1453 mutex_enter(&arc_eviction_mtx); 1454 rw_enter(&buf->b_lock, RW_WRITER); 1455 ASSERT(buf->b_hdr != NULL); 1456 arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1457 hdr->b_buf = buf->b_next; 1458 buf->b_hdr = &arc_eviction_hdr; 1459 buf->b_next = arc_eviction_list; 1460 arc_eviction_list = buf; 1461 rw_exit(&buf->b_lock); 1462 mutex_exit(&arc_eviction_mtx); 1463 } else { 1464 arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1465 } 1466 } 1467 if (hdr->b_freeze_cksum != NULL) { 1468 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 1469 hdr->b_freeze_cksum = NULL; 1470 } 1471 1472 ASSERT(!list_link_active(&hdr->b_arc_node)); 1473 ASSERT3P(hdr->b_hash_next, ==, NULL); 1474 ASSERT3P(hdr->b_acb, ==, NULL); 1475 kmem_cache_free(hdr_cache, hdr); 1476 } 1477 1478 void 1479 arc_buf_free(arc_buf_t *buf, void *tag) 1480 { 1481 arc_buf_hdr_t *hdr = buf->b_hdr; 1482 int hashed = hdr->b_state != arc_anon; 1483 1484 ASSERT(buf->b_efunc == NULL); 1485 ASSERT(buf->b_data != NULL); 1486 1487 if (hashed) { 1488 kmutex_t *hash_lock = HDR_LOCK(hdr); 1489 1490 mutex_enter(hash_lock); 1491 (void) remove_reference(hdr, hash_lock, tag); 1492 if (hdr->b_datacnt > 1) { 1493 arc_buf_destroy(buf, FALSE, TRUE); 1494 } else { 1495 ASSERT(buf == hdr->b_buf); 1496 ASSERT(buf->b_efunc == NULL); 1497 hdr->b_flags |= ARC_BUF_AVAILABLE; 1498 } 1499 mutex_exit(hash_lock); 1500 } else if (HDR_IO_IN_PROGRESS(hdr)) { 1501 int destroy_hdr; 1502 /* 1503 * We are in the middle of an async write. Don't destroy 1504 * this buffer unless the write completes before we finish 1505 * decrementing the reference count. 1506 */ 1507 mutex_enter(&arc_eviction_mtx); 1508 (void) remove_reference(hdr, NULL, tag); 1509 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1510 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1511 mutex_exit(&arc_eviction_mtx); 1512 if (destroy_hdr) 1513 arc_hdr_destroy(hdr); 1514 } else { 1515 if (remove_reference(hdr, NULL, tag) > 0) { 1516 ASSERT(HDR_IO_ERROR(hdr)); 1517 arc_buf_destroy(buf, FALSE, TRUE); 1518 } else { 1519 arc_hdr_destroy(hdr); 1520 } 1521 } 1522 } 1523 1524 int 1525 arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1526 { 1527 arc_buf_hdr_t *hdr = buf->b_hdr; 1528 kmutex_t *hash_lock = HDR_LOCK(hdr); 1529 int no_callback = (buf->b_efunc == NULL); 1530 1531 if (hdr->b_state == arc_anon) { 1532 ASSERT(hdr->b_datacnt == 1); 1533 arc_buf_free(buf, tag); 1534 return (no_callback); 1535 } 1536 1537 mutex_enter(hash_lock); 1538 ASSERT(hdr->b_state != arc_anon); 1539 ASSERT(buf->b_data != NULL); 1540 1541 (void) remove_reference(hdr, hash_lock, tag); 1542 if (hdr->b_datacnt > 1) { 1543 if (no_callback) 1544 arc_buf_destroy(buf, FALSE, TRUE); 1545 } else if (no_callback) { 1546 ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1547 ASSERT(buf->b_efunc == NULL); 1548 hdr->b_flags |= ARC_BUF_AVAILABLE; 1549 } 1550 ASSERT(no_callback || hdr->b_datacnt > 1 || 1551 refcount_is_zero(&hdr->b_refcnt)); 1552 mutex_exit(hash_lock); 1553 return (no_callback); 1554 } 1555 1556 int 1557 arc_buf_size(arc_buf_t *buf) 1558 { 1559 return (buf->b_hdr->b_size); 1560 } 1561 1562 /* 1563 * Evict buffers from list until we've removed the specified number of 1564 * bytes. Move the removed buffers to the appropriate evict state. 1565 * If the recycle flag is set, then attempt to "recycle" a buffer: 1566 * - look for a buffer to evict that is `bytes' long. 1567 * - return the data block from this buffer rather than freeing it. 1568 * This flag is used by callers that are trying to make space for a 1569 * new buffer in a full arc cache. 1570 * 1571 * This function makes a "best effort". It skips over any buffers 1572 * it can't get a hash_lock on, and so may not catch all candidates. 1573 * It may also return without evicting as much space as requested. 1574 */ 1575 static void * 1576 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle, 1577 arc_buf_contents_t type) 1578 { 1579 arc_state_t *evicted_state; 1580 uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 1581 arc_buf_hdr_t *ab, *ab_prev = NULL; 1582 list_t *list = &state->arcs_list[type]; 1583 kmutex_t *hash_lock; 1584 boolean_t have_lock; 1585 void *stolen = NULL; 1586 1587 ASSERT(state == arc_mru || state == arc_mfu); 1588 1589 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1590 1591 mutex_enter(&state->arcs_mtx); 1592 mutex_enter(&evicted_state->arcs_mtx); 1593 1594 for (ab = list_tail(list); ab; ab = ab_prev) { 1595 ab_prev = list_prev(list, ab); 1596 /* prefetch buffers have a minimum lifespan */ 1597 if (HDR_IO_IN_PROGRESS(ab) || 1598 (spa && ab->b_spa != spa) || 1599 (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1600 ddi_get_lbolt() - ab->b_arc_access < 1601 arc_min_prefetch_lifespan)) { 1602 skipped++; 1603 continue; 1604 } 1605 /* "lookahead" for better eviction candidate */ 1606 if (recycle && ab->b_size != bytes && 1607 ab_prev && ab_prev->b_size == bytes) 1608 continue; 1609 hash_lock = HDR_LOCK(ab); 1610 have_lock = MUTEX_HELD(hash_lock); 1611 if (have_lock || mutex_tryenter(hash_lock)) { 1612 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1613 ASSERT(ab->b_datacnt > 0); 1614 while (ab->b_buf) { 1615 arc_buf_t *buf = ab->b_buf; 1616 if (!rw_tryenter(&buf->b_lock, RW_WRITER)) { 1617 missed += 1; 1618 break; 1619 } 1620 if (buf->b_data) { 1621 bytes_evicted += ab->b_size; 1622 if (recycle && ab->b_type == type && 1623 ab->b_size == bytes && 1624 !HDR_L2_WRITING(ab)) { 1625 stolen = buf->b_data; 1626 recycle = FALSE; 1627 } 1628 } 1629 if (buf->b_efunc) { 1630 mutex_enter(&arc_eviction_mtx); 1631 arc_buf_destroy(buf, 1632 buf->b_data == stolen, FALSE); 1633 ab->b_buf = buf->b_next; 1634 buf->b_hdr = &arc_eviction_hdr; 1635 buf->b_next = arc_eviction_list; 1636 arc_eviction_list = buf; 1637 mutex_exit(&arc_eviction_mtx); 1638 rw_exit(&buf->b_lock); 1639 } else { 1640 rw_exit(&buf->b_lock); 1641 arc_buf_destroy(buf, 1642 buf->b_data == stolen, TRUE); 1643 } 1644 } 1645 1646 if (ab->b_l2hdr) { 1647 ARCSTAT_INCR(arcstat_evict_l2_cached, 1648 ab->b_size); 1649 } else { 1650 if (l2arc_write_eligible(ab->b_spa, ab)) { 1651 ARCSTAT_INCR(arcstat_evict_l2_eligible, 1652 ab->b_size); 1653 } else { 1654 ARCSTAT_INCR( 1655 arcstat_evict_l2_ineligible, 1656 ab->b_size); 1657 } 1658 } 1659 1660 if (ab->b_datacnt == 0) { 1661 arc_change_state(evicted_state, ab, hash_lock); 1662 ASSERT(HDR_IN_HASH_TABLE(ab)); 1663 ab->b_flags |= ARC_IN_HASH_TABLE; 1664 ab->b_flags &= ~ARC_BUF_AVAILABLE; 1665 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 1666 } 1667 if (!have_lock) 1668 mutex_exit(hash_lock); 1669 if (bytes >= 0 && bytes_evicted >= bytes) 1670 break; 1671 } else { 1672 missed += 1; 1673 } 1674 } 1675 1676 mutex_exit(&evicted_state->arcs_mtx); 1677 mutex_exit(&state->arcs_mtx); 1678 1679 if (bytes_evicted < bytes) 1680 dprintf("only evicted %lld bytes from %x", 1681 (longlong_t)bytes_evicted, state); 1682 1683 if (skipped) 1684 ARCSTAT_INCR(arcstat_evict_skip, skipped); 1685 1686 if (missed) 1687 ARCSTAT_INCR(arcstat_mutex_miss, missed); 1688 1689 /* 1690 * We have just evicted some date into the ghost state, make 1691 * sure we also adjust the ghost state size if necessary. 1692 */ 1693 if (arc_no_grow && 1694 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1695 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1696 arc_mru_ghost->arcs_size - arc_c; 1697 1698 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1699 int64_t todelete = 1700 MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1701 arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1702 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1703 int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1704 arc_mru_ghost->arcs_size + 1705 arc_mfu_ghost->arcs_size - arc_c); 1706 arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1707 } 1708 } 1709 1710 return (stolen); 1711 } 1712 1713 /* 1714 * Remove buffers from list until we've removed the specified number of 1715 * bytes. Destroy the buffers that are removed. 1716 */ 1717 static void 1718 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes) 1719 { 1720 arc_buf_hdr_t *ab, *ab_prev; 1721 list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1722 kmutex_t *hash_lock; 1723 uint64_t bytes_deleted = 0; 1724 uint64_t bufs_skipped = 0; 1725 1726 ASSERT(GHOST_STATE(state)); 1727 top: 1728 mutex_enter(&state->arcs_mtx); 1729 for (ab = list_tail(list); ab; ab = ab_prev) { 1730 ab_prev = list_prev(list, ab); 1731 if (spa && ab->b_spa != spa) 1732 continue; 1733 hash_lock = HDR_LOCK(ab); 1734 /* caller may be trying to modify this buffer, skip it */ 1735 if (MUTEX_HELD(hash_lock)) 1736 continue; 1737 if (mutex_tryenter(hash_lock)) { 1738 ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1739 ASSERT(ab->b_buf == NULL); 1740 ARCSTAT_BUMP(arcstat_deleted); 1741 bytes_deleted += ab->b_size; 1742 1743 if (ab->b_l2hdr != NULL) { 1744 /* 1745 * This buffer is cached on the 2nd Level ARC; 1746 * don't destroy the header. 1747 */ 1748 arc_change_state(arc_l2c_only, ab, hash_lock); 1749 mutex_exit(hash_lock); 1750 } else { 1751 arc_change_state(arc_anon, ab, hash_lock); 1752 mutex_exit(hash_lock); 1753 arc_hdr_destroy(ab); 1754 } 1755 1756 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1757 if (bytes >= 0 && bytes_deleted >= bytes) 1758 break; 1759 } else { 1760 if (bytes < 0) { 1761 mutex_exit(&state->arcs_mtx); 1762 mutex_enter(hash_lock); 1763 mutex_exit(hash_lock); 1764 goto top; 1765 } 1766 bufs_skipped += 1; 1767 } 1768 } 1769 mutex_exit(&state->arcs_mtx); 1770 1771 if (list == &state->arcs_list[ARC_BUFC_DATA] && 1772 (bytes < 0 || bytes_deleted < bytes)) { 1773 list = &state->arcs_list[ARC_BUFC_METADATA]; 1774 goto top; 1775 } 1776 1777 if (bufs_skipped) { 1778 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1779 ASSERT(bytes >= 0); 1780 } 1781 1782 if (bytes_deleted < bytes) 1783 dprintf("only deleted %lld bytes from %p", 1784 (longlong_t)bytes_deleted, state); 1785 } 1786 1787 static void 1788 arc_adjust(void) 1789 { 1790 int64_t adjustment, delta; 1791 1792 /* 1793 * Adjust MRU size 1794 */ 1795 1796 adjustment = MIN(arc_size - arc_c, 1797 arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - arc_p); 1798 1799 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 1800 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment); 1801 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA); 1802 adjustment -= delta; 1803 } 1804 1805 if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1806 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment); 1807 (void) arc_evict(arc_mru, NULL, delta, FALSE, 1808 ARC_BUFC_METADATA); 1809 } 1810 1811 /* 1812 * Adjust MFU size 1813 */ 1814 1815 adjustment = arc_size - arc_c; 1816 1817 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 1818 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]); 1819 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA); 1820 adjustment -= delta; 1821 } 1822 1823 if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1824 int64_t delta = MIN(adjustment, 1825 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]); 1826 (void) arc_evict(arc_mfu, NULL, delta, FALSE, 1827 ARC_BUFC_METADATA); 1828 } 1829 1830 /* 1831 * Adjust ghost lists 1832 */ 1833 1834 adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c; 1835 1836 if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) { 1837 delta = MIN(arc_mru_ghost->arcs_size, adjustment); 1838 arc_evict_ghost(arc_mru_ghost, NULL, delta); 1839 } 1840 1841 adjustment = 1842 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c; 1843 1844 if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) { 1845 delta = MIN(arc_mfu_ghost->arcs_size, adjustment); 1846 arc_evict_ghost(arc_mfu_ghost, NULL, delta); 1847 } 1848 } 1849 1850 static void 1851 arc_do_user_evicts(void) 1852 { 1853 mutex_enter(&arc_eviction_mtx); 1854 while (arc_eviction_list != NULL) { 1855 arc_buf_t *buf = arc_eviction_list; 1856 arc_eviction_list = buf->b_next; 1857 rw_enter(&buf->b_lock, RW_WRITER); 1858 buf->b_hdr = NULL; 1859 rw_exit(&buf->b_lock); 1860 mutex_exit(&arc_eviction_mtx); 1861 1862 if (buf->b_efunc != NULL) 1863 VERIFY(buf->b_efunc(buf) == 0); 1864 1865 buf->b_efunc = NULL; 1866 buf->b_private = NULL; 1867 kmem_cache_free(buf_cache, buf); 1868 mutex_enter(&arc_eviction_mtx); 1869 } 1870 mutex_exit(&arc_eviction_mtx); 1871 } 1872 1873 /* 1874 * Flush all *evictable* data from the cache for the given spa. 1875 * NOTE: this will not touch "active" (i.e. referenced) data. 1876 */ 1877 void 1878 arc_flush(spa_t *spa) 1879 { 1880 uint64_t guid = 0; 1881 1882 if (spa) 1883 guid = spa_guid(spa); 1884 1885 while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 1886 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA); 1887 if (spa) 1888 break; 1889 } 1890 while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 1891 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA); 1892 if (spa) 1893 break; 1894 } 1895 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 1896 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA); 1897 if (spa) 1898 break; 1899 } 1900 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 1901 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA); 1902 if (spa) 1903 break; 1904 } 1905 1906 arc_evict_ghost(arc_mru_ghost, guid, -1); 1907 arc_evict_ghost(arc_mfu_ghost, guid, -1); 1908 1909 mutex_enter(&arc_reclaim_thr_lock); 1910 arc_do_user_evicts(); 1911 mutex_exit(&arc_reclaim_thr_lock); 1912 ASSERT(spa || arc_eviction_list == NULL); 1913 } 1914 1915 void 1916 arc_shrink(void) 1917 { 1918 if (arc_c > arc_c_min) { 1919 uint64_t to_free; 1920 1921 #ifdef _KERNEL 1922 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 1923 #else 1924 to_free = arc_c >> arc_shrink_shift; 1925 #endif 1926 if (arc_c > arc_c_min + to_free) 1927 atomic_add_64(&arc_c, -to_free); 1928 else 1929 arc_c = arc_c_min; 1930 1931 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 1932 if (arc_c > arc_size) 1933 arc_c = MAX(arc_size, arc_c_min); 1934 if (arc_p > arc_c) 1935 arc_p = (arc_c >> 1); 1936 ASSERT(arc_c >= arc_c_min); 1937 ASSERT((int64_t)arc_p >= 0); 1938 } 1939 1940 if (arc_size > arc_c) 1941 arc_adjust(); 1942 } 1943 1944 static int 1945 arc_reclaim_needed(void) 1946 { 1947 uint64_t extra; 1948 1949 #ifdef _KERNEL 1950 1951 if (needfree) 1952 return (1); 1953 1954 /* 1955 * take 'desfree' extra pages, so we reclaim sooner, rather than later 1956 */ 1957 extra = desfree; 1958 1959 /* 1960 * check that we're out of range of the pageout scanner. It starts to 1961 * schedule paging if freemem is less than lotsfree and needfree. 1962 * lotsfree is the high-water mark for pageout, and needfree is the 1963 * number of needed free pages. We add extra pages here to make sure 1964 * the scanner doesn't start up while we're freeing memory. 1965 */ 1966 if (freemem < lotsfree + needfree + extra) 1967 return (1); 1968 1969 /* 1970 * check to make sure that swapfs has enough space so that anon 1971 * reservations can still succeed. anon_resvmem() checks that the 1972 * availrmem is greater than swapfs_minfree, and the number of reserved 1973 * swap pages. We also add a bit of extra here just to prevent 1974 * circumstances from getting really dire. 1975 */ 1976 if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1977 return (1); 1978 1979 #if defined(__i386) 1980 /* 1981 * If we're on an i386 platform, it's possible that we'll exhaust the 1982 * kernel heap space before we ever run out of available physical 1983 * memory. Most checks of the size of the heap_area compare against 1984 * tune.t_minarmem, which is the minimum available real memory that we 1985 * can have in the system. However, this is generally fixed at 25 pages 1986 * which is so low that it's useless. In this comparison, we seek to 1987 * calculate the total heap-size, and reclaim if more than 3/4ths of the 1988 * heap is allocated. (Or, in the calculation, if less than 1/4th is 1989 * free) 1990 */ 1991 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1992 (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1993 return (1); 1994 #endif 1995 1996 #else 1997 if (spa_get_random(100) == 0) 1998 return (1); 1999 #endif 2000 return (0); 2001 } 2002 2003 static void 2004 arc_kmem_reap_now(arc_reclaim_strategy_t strat) 2005 { 2006 size_t i; 2007 kmem_cache_t *prev_cache = NULL; 2008 kmem_cache_t *prev_data_cache = NULL; 2009 extern kmem_cache_t *zio_buf_cache[]; 2010 extern kmem_cache_t *zio_data_buf_cache[]; 2011 2012 #ifdef _KERNEL 2013 if (arc_meta_used >= arc_meta_limit) { 2014 /* 2015 * We are exceeding our meta-data cache limit. 2016 * Purge some DNLC entries to release holds on meta-data. 2017 */ 2018 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 2019 } 2020 #if defined(__i386) 2021 /* 2022 * Reclaim unused memory from all kmem caches. 2023 */ 2024 kmem_reap(); 2025 #endif 2026 #endif 2027 2028 /* 2029 * An aggressive reclamation will shrink the cache size as well as 2030 * reap free buffers from the arc kmem caches. 2031 */ 2032 if (strat == ARC_RECLAIM_AGGR) 2033 arc_shrink(); 2034 2035 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 2036 if (zio_buf_cache[i] != prev_cache) { 2037 prev_cache = zio_buf_cache[i]; 2038 kmem_cache_reap_now(zio_buf_cache[i]); 2039 } 2040 if (zio_data_buf_cache[i] != prev_data_cache) { 2041 prev_data_cache = zio_data_buf_cache[i]; 2042 kmem_cache_reap_now(zio_data_buf_cache[i]); 2043 } 2044 } 2045 kmem_cache_reap_now(buf_cache); 2046 kmem_cache_reap_now(hdr_cache); 2047 } 2048 2049 static void 2050 arc_reclaim_thread(void) 2051 { 2052 clock_t growtime = 0; 2053 arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 2054 callb_cpr_t cpr; 2055 2056 CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 2057 2058 mutex_enter(&arc_reclaim_thr_lock); 2059 while (arc_thread_exit == 0) { 2060 if (arc_reclaim_needed()) { 2061 2062 if (arc_no_grow) { 2063 if (last_reclaim == ARC_RECLAIM_CONS) { 2064 last_reclaim = ARC_RECLAIM_AGGR; 2065 } else { 2066 last_reclaim = ARC_RECLAIM_CONS; 2067 } 2068 } else { 2069 arc_no_grow = TRUE; 2070 last_reclaim = ARC_RECLAIM_AGGR; 2071 membar_producer(); 2072 } 2073 2074 /* reset the growth delay for every reclaim */ 2075 growtime = ddi_get_lbolt() + (arc_grow_retry * hz); 2076 2077 arc_kmem_reap_now(last_reclaim); 2078 arc_warm = B_TRUE; 2079 2080 } else if (arc_no_grow && ddi_get_lbolt() >= growtime) { 2081 arc_no_grow = FALSE; 2082 } 2083 2084 if (2 * arc_c < arc_size + 2085 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 2086 arc_adjust(); 2087 2088 if (arc_eviction_list != NULL) 2089 arc_do_user_evicts(); 2090 2091 /* block until needed, or one second, whichever is shorter */ 2092 CALLB_CPR_SAFE_BEGIN(&cpr); 2093 (void) cv_timedwait(&arc_reclaim_thr_cv, 2094 &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz)); 2095 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 2096 } 2097 2098 arc_thread_exit = 0; 2099 cv_broadcast(&arc_reclaim_thr_cv); 2100 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 2101 thread_exit(); 2102 } 2103 2104 /* 2105 * Adapt arc info given the number of bytes we are trying to add and 2106 * the state that we are comming from. This function is only called 2107 * when we are adding new content to the cache. 2108 */ 2109 static void 2110 arc_adapt(int bytes, arc_state_t *state) 2111 { 2112 int mult; 2113 uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 2114 2115 if (state == arc_l2c_only) 2116 return; 2117 2118 ASSERT(bytes > 0); 2119 /* 2120 * Adapt the target size of the MRU list: 2121 * - if we just hit in the MRU ghost list, then increase 2122 * the target size of the MRU list. 2123 * - if we just hit in the MFU ghost list, then increase 2124 * the target size of the MFU list by decreasing the 2125 * target size of the MRU list. 2126 */ 2127 if (state == arc_mru_ghost) { 2128 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 2129 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 2130 2131 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 2132 } else if (state == arc_mfu_ghost) { 2133 uint64_t delta; 2134 2135 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 2136 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 2137 2138 delta = MIN(bytes * mult, arc_p); 2139 arc_p = MAX(arc_p_min, arc_p - delta); 2140 } 2141 ASSERT((int64_t)arc_p >= 0); 2142 2143 if (arc_reclaim_needed()) { 2144 cv_signal(&arc_reclaim_thr_cv); 2145 return; 2146 } 2147 2148 if (arc_no_grow) 2149 return; 2150 2151 if (arc_c >= arc_c_max) 2152 return; 2153 2154 /* 2155 * If we're within (2 * maxblocksize) bytes of the target 2156 * cache size, increment the target cache size 2157 */ 2158 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 2159 atomic_add_64(&arc_c, (int64_t)bytes); 2160 if (arc_c > arc_c_max) 2161 arc_c = arc_c_max; 2162 else if (state == arc_anon) 2163 atomic_add_64(&arc_p, (int64_t)bytes); 2164 if (arc_p > arc_c) 2165 arc_p = arc_c; 2166 } 2167 ASSERT((int64_t)arc_p >= 0); 2168 } 2169 2170 /* 2171 * Check if the cache has reached its limits and eviction is required 2172 * prior to insert. 2173 */ 2174 static int 2175 arc_evict_needed(arc_buf_contents_t type) 2176 { 2177 if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 2178 return (1); 2179 2180 #ifdef _KERNEL 2181 /* 2182 * If zio data pages are being allocated out of a separate heap segment, 2183 * then enforce that the size of available vmem for this area remains 2184 * above about 1/32nd free. 2185 */ 2186 if (type == ARC_BUFC_DATA && zio_arena != NULL && 2187 vmem_size(zio_arena, VMEM_FREE) < 2188 (vmem_size(zio_arena, VMEM_ALLOC) >> 5)) 2189 return (1); 2190 #endif 2191 2192 if (arc_reclaim_needed()) 2193 return (1); 2194 2195 return (arc_size > arc_c); 2196 } 2197 2198 /* 2199 * The buffer, supplied as the first argument, needs a data block. 2200 * So, if we are at cache max, determine which cache should be victimized. 2201 * We have the following cases: 2202 * 2203 * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2204 * In this situation if we're out of space, but the resident size of the MFU is 2205 * under the limit, victimize the MFU cache to satisfy this insertion request. 2206 * 2207 * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2208 * Here, we've used up all of the available space for the MRU, so we need to 2209 * evict from our own cache instead. Evict from the set of resident MRU 2210 * entries. 2211 * 2212 * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2213 * c minus p represents the MFU space in the cache, since p is the size of the 2214 * cache that is dedicated to the MRU. In this situation there's still space on 2215 * the MFU side, so the MRU side needs to be victimized. 2216 * 2217 * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2218 * MFU's resident set is consuming more space than it has been allotted. In 2219 * this situation, we must victimize our own cache, the MFU, for this insertion. 2220 */ 2221 static void 2222 arc_get_data_buf(arc_buf_t *buf) 2223 { 2224 arc_state_t *state = buf->b_hdr->b_state; 2225 uint64_t size = buf->b_hdr->b_size; 2226 arc_buf_contents_t type = buf->b_hdr->b_type; 2227 2228 arc_adapt(size, state); 2229 2230 /* 2231 * We have not yet reached cache maximum size, 2232 * just allocate a new buffer. 2233 */ 2234 if (!arc_evict_needed(type)) { 2235 if (type == ARC_BUFC_METADATA) { 2236 buf->b_data = zio_buf_alloc(size); 2237 arc_space_consume(size, ARC_SPACE_DATA); 2238 } else { 2239 ASSERT(type == ARC_BUFC_DATA); 2240 buf->b_data = zio_data_buf_alloc(size); 2241 ARCSTAT_INCR(arcstat_data_size, size); 2242 atomic_add_64(&arc_size, size); 2243 } 2244 goto out; 2245 } 2246 2247 /* 2248 * If we are prefetching from the mfu ghost list, this buffer 2249 * will end up on the mru list; so steal space from there. 2250 */ 2251 if (state == arc_mfu_ghost) 2252 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 2253 else if (state == arc_mru_ghost) 2254 state = arc_mru; 2255 2256 if (state == arc_mru || state == arc_anon) { 2257 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 2258 state = (arc_mfu->arcs_lsize[type] >= size && 2259 arc_p > mru_used) ? arc_mfu : arc_mru; 2260 } else { 2261 /* MFU cases */ 2262 uint64_t mfu_space = arc_c - arc_p; 2263 state = (arc_mru->arcs_lsize[type] >= size && 2264 mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 2265 } 2266 if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2267 if (type == ARC_BUFC_METADATA) { 2268 buf->b_data = zio_buf_alloc(size); 2269 arc_space_consume(size, ARC_SPACE_DATA); 2270 } else { 2271 ASSERT(type == ARC_BUFC_DATA); 2272 buf->b_data = zio_data_buf_alloc(size); 2273 ARCSTAT_INCR(arcstat_data_size, size); 2274 atomic_add_64(&arc_size, size); 2275 } 2276 ARCSTAT_BUMP(arcstat_recycle_miss); 2277 } 2278 ASSERT(buf->b_data != NULL); 2279 out: 2280 /* 2281 * Update the state size. Note that ghost states have a 2282 * "ghost size" and so don't need to be updated. 2283 */ 2284 if (!GHOST_STATE(buf->b_hdr->b_state)) { 2285 arc_buf_hdr_t *hdr = buf->b_hdr; 2286 2287 atomic_add_64(&hdr->b_state->arcs_size, size); 2288 if (list_link_active(&hdr->b_arc_node)) { 2289 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2290 atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2291 } 2292 /* 2293 * If we are growing the cache, and we are adding anonymous 2294 * data, and we have outgrown arc_p, update arc_p 2295 */ 2296 if (arc_size < arc_c && hdr->b_state == arc_anon && 2297 arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 2298 arc_p = MIN(arc_c, arc_p + size); 2299 } 2300 } 2301 2302 /* 2303 * This routine is called whenever a buffer is accessed. 2304 * NOTE: the hash lock is dropped in this function. 2305 */ 2306 static void 2307 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2308 { 2309 clock_t now; 2310 2311 ASSERT(MUTEX_HELD(hash_lock)); 2312 2313 if (buf->b_state == arc_anon) { 2314 /* 2315 * This buffer is not in the cache, and does not 2316 * appear in our "ghost" list. Add the new buffer 2317 * to the MRU state. 2318 */ 2319 2320 ASSERT(buf->b_arc_access == 0); 2321 buf->b_arc_access = ddi_get_lbolt(); 2322 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2323 arc_change_state(arc_mru, buf, hash_lock); 2324 2325 } else if (buf->b_state == arc_mru) { 2326 now = ddi_get_lbolt(); 2327 2328 /* 2329 * If this buffer is here because of a prefetch, then either: 2330 * - clear the flag if this is a "referencing" read 2331 * (any subsequent access will bump this into the MFU state). 2332 * or 2333 * - move the buffer to the head of the list if this is 2334 * another prefetch (to make it less likely to be evicted). 2335 */ 2336 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2337 if (refcount_count(&buf->b_refcnt) == 0) { 2338 ASSERT(list_link_active(&buf->b_arc_node)); 2339 } else { 2340 buf->b_flags &= ~ARC_PREFETCH; 2341 ARCSTAT_BUMP(arcstat_mru_hits); 2342 } 2343 buf->b_arc_access = now; 2344 return; 2345 } 2346 2347 /* 2348 * This buffer has been "accessed" only once so far, 2349 * but it is still in the cache. Move it to the MFU 2350 * state. 2351 */ 2352 if (now > buf->b_arc_access + ARC_MINTIME) { 2353 /* 2354 * More than 125ms have passed since we 2355 * instantiated this buffer. Move it to the 2356 * most frequently used state. 2357 */ 2358 buf->b_arc_access = now; 2359 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2360 arc_change_state(arc_mfu, buf, hash_lock); 2361 } 2362 ARCSTAT_BUMP(arcstat_mru_hits); 2363 } else if (buf->b_state == arc_mru_ghost) { 2364 arc_state_t *new_state; 2365 /* 2366 * This buffer has been "accessed" recently, but 2367 * was evicted from the cache. Move it to the 2368 * MFU state. 2369 */ 2370 2371 if (buf->b_flags & ARC_PREFETCH) { 2372 new_state = arc_mru; 2373 if (refcount_count(&buf->b_refcnt) > 0) 2374 buf->b_flags &= ~ARC_PREFETCH; 2375 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2376 } else { 2377 new_state = arc_mfu; 2378 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2379 } 2380 2381 buf->b_arc_access = ddi_get_lbolt(); 2382 arc_change_state(new_state, buf, hash_lock); 2383 2384 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 2385 } else if (buf->b_state == arc_mfu) { 2386 /* 2387 * This buffer has been accessed more than once and is 2388 * still in the cache. Keep it in the MFU state. 2389 * 2390 * NOTE: an add_reference() that occurred when we did 2391 * the arc_read() will have kicked this off the list. 2392 * If it was a prefetch, we will explicitly move it to 2393 * the head of the list now. 2394 */ 2395 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2396 ASSERT(refcount_count(&buf->b_refcnt) == 0); 2397 ASSERT(list_link_active(&buf->b_arc_node)); 2398 } 2399 ARCSTAT_BUMP(arcstat_mfu_hits); 2400 buf->b_arc_access = ddi_get_lbolt(); 2401 } else if (buf->b_state == arc_mfu_ghost) { 2402 arc_state_t *new_state = arc_mfu; 2403 /* 2404 * This buffer has been accessed more than once but has 2405 * been evicted from the cache. Move it back to the 2406 * MFU state. 2407 */ 2408 2409 if (buf->b_flags & ARC_PREFETCH) { 2410 /* 2411 * This is a prefetch access... 2412 * move this block back to the MRU state. 2413 */ 2414 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 2415 new_state = arc_mru; 2416 } 2417 2418 buf->b_arc_access = ddi_get_lbolt(); 2419 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2420 arc_change_state(new_state, buf, hash_lock); 2421 2422 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2423 } else if (buf->b_state == arc_l2c_only) { 2424 /* 2425 * This buffer is on the 2nd Level ARC. 2426 */ 2427 2428 buf->b_arc_access = ddi_get_lbolt(); 2429 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2430 arc_change_state(arc_mfu, buf, hash_lock); 2431 } else { 2432 ASSERT(!"invalid arc state"); 2433 } 2434 } 2435 2436 /* a generic arc_done_func_t which you can use */ 2437 /* ARGSUSED */ 2438 void 2439 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2440 { 2441 bcopy(buf->b_data, arg, buf->b_hdr->b_size); 2442 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2443 } 2444 2445 /* a generic arc_done_func_t */ 2446 void 2447 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2448 { 2449 arc_buf_t **bufp = arg; 2450 if (zio && zio->io_error) { 2451 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2452 *bufp = NULL; 2453 } else { 2454 *bufp = buf; 2455 } 2456 } 2457 2458 static void 2459 arc_read_done(zio_t *zio) 2460 { 2461 arc_buf_hdr_t *hdr, *found; 2462 arc_buf_t *buf; 2463 arc_buf_t *abuf; /* buffer we're assigning to callback */ 2464 kmutex_t *hash_lock; 2465 arc_callback_t *callback_list, *acb; 2466 int freeable = FALSE; 2467 2468 buf = zio->io_private; 2469 hdr = buf->b_hdr; 2470 2471 /* 2472 * The hdr was inserted into hash-table and removed from lists 2473 * prior to starting I/O. We should find this header, since 2474 * it's in the hash table, and it should be legit since it's 2475 * not possible to evict it during the I/O. The only possible 2476 * reason for it not to be found is if we were freed during the 2477 * read. 2478 */ 2479 found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth, 2480 &hash_lock); 2481 2482 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2483 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2484 (found == hdr && HDR_L2_READING(hdr))); 2485 2486 hdr->b_flags &= ~ARC_L2_EVICTED; 2487 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 2488 hdr->b_flags &= ~ARC_L2CACHE; 2489 2490 /* byteswap if necessary */ 2491 callback_list = hdr->b_acb; 2492 ASSERT(callback_list != NULL); 2493 if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) { 2494 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2495 byteswap_uint64_array : 2496 dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap; 2497 func(buf->b_data, hdr->b_size); 2498 } 2499 2500 arc_cksum_compute(buf, B_FALSE); 2501 2502 if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) { 2503 /* 2504 * Only call arc_access on anonymous buffers. This is because 2505 * if we've issued an I/O for an evicted buffer, we've already 2506 * called arc_access (to prevent any simultaneous readers from 2507 * getting confused). 2508 */ 2509 arc_access(hdr, hash_lock); 2510 } 2511 2512 /* create copies of the data buffer for the callers */ 2513 abuf = buf; 2514 for (acb = callback_list; acb; acb = acb->acb_next) { 2515 if (acb->acb_done) { 2516 if (abuf == NULL) 2517 abuf = arc_buf_clone(buf); 2518 acb->acb_buf = abuf; 2519 abuf = NULL; 2520 } 2521 } 2522 hdr->b_acb = NULL; 2523 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2524 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2525 if (abuf == buf) { 2526 ASSERT(buf->b_efunc == NULL); 2527 ASSERT(hdr->b_datacnt == 1); 2528 hdr->b_flags |= ARC_BUF_AVAILABLE; 2529 } 2530 2531 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2532 2533 if (zio->io_error != 0) { 2534 hdr->b_flags |= ARC_IO_ERROR; 2535 if (hdr->b_state != arc_anon) 2536 arc_change_state(arc_anon, hdr, hash_lock); 2537 if (HDR_IN_HASH_TABLE(hdr)) 2538 buf_hash_remove(hdr); 2539 freeable = refcount_is_zero(&hdr->b_refcnt); 2540 } 2541 2542 /* 2543 * Broadcast before we drop the hash_lock to avoid the possibility 2544 * that the hdr (and hence the cv) might be freed before we get to 2545 * the cv_broadcast(). 2546 */ 2547 cv_broadcast(&hdr->b_cv); 2548 2549 if (hash_lock) { 2550 mutex_exit(hash_lock); 2551 } else { 2552 /* 2553 * This block was freed while we waited for the read to 2554 * complete. It has been removed from the hash table and 2555 * moved to the anonymous state (so that it won't show up 2556 * in the cache). 2557 */ 2558 ASSERT3P(hdr->b_state, ==, arc_anon); 2559 freeable = refcount_is_zero(&hdr->b_refcnt); 2560 } 2561 2562 /* execute each callback and free its structure */ 2563 while ((acb = callback_list) != NULL) { 2564 if (acb->acb_done) 2565 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2566 2567 if (acb->acb_zio_dummy != NULL) { 2568 acb->acb_zio_dummy->io_error = zio->io_error; 2569 zio_nowait(acb->acb_zio_dummy); 2570 } 2571 2572 callback_list = acb->acb_next; 2573 kmem_free(acb, sizeof (arc_callback_t)); 2574 } 2575 2576 if (freeable) 2577 arc_hdr_destroy(hdr); 2578 } 2579 2580 /* 2581 * "Read" the block block at the specified DVA (in bp) via the 2582 * cache. If the block is found in the cache, invoke the provided 2583 * callback immediately and return. Note that the `zio' parameter 2584 * in the callback will be NULL in this case, since no IO was 2585 * required. If the block is not in the cache pass the read request 2586 * on to the spa with a substitute callback function, so that the 2587 * requested block will be added to the cache. 2588 * 2589 * If a read request arrives for a block that has a read in-progress, 2590 * either wait for the in-progress read to complete (and return the 2591 * results); or, if this is a read with a "done" func, add a record 2592 * to the read to invoke the "done" func when the read completes, 2593 * and return; or just return. 2594 * 2595 * arc_read_done() will invoke all the requested "done" functions 2596 * for readers of this block. 2597 * 2598 * Normal callers should use arc_read and pass the arc buffer and offset 2599 * for the bp. But if you know you don't need locking, you can use 2600 * arc_read_bp. 2601 */ 2602 int 2603 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_buf_t *pbuf, 2604 arc_done_func_t *done, void *private, int priority, int zio_flags, 2605 uint32_t *arc_flags, const zbookmark_t *zb) 2606 { 2607 int err; 2608 2609 ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt)); 2610 ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size); 2611 rw_enter(&pbuf->b_lock, RW_READER); 2612 2613 err = arc_read_nolock(pio, spa, bp, done, private, priority, 2614 zio_flags, arc_flags, zb); 2615 rw_exit(&pbuf->b_lock); 2616 2617 return (err); 2618 } 2619 2620 int 2621 arc_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bp, 2622 arc_done_func_t *done, void *private, int priority, int zio_flags, 2623 uint32_t *arc_flags, const zbookmark_t *zb) 2624 { 2625 arc_buf_hdr_t *hdr; 2626 arc_buf_t *buf; 2627 kmutex_t *hash_lock; 2628 zio_t *rzio; 2629 uint64_t guid = spa_guid(spa); 2630 2631 top: 2632 hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 2633 &hash_lock); 2634 if (hdr && hdr->b_datacnt > 0) { 2635 2636 *arc_flags |= ARC_CACHED; 2637 2638 if (HDR_IO_IN_PROGRESS(hdr)) { 2639 2640 if (*arc_flags & ARC_WAIT) { 2641 cv_wait(&hdr->b_cv, hash_lock); 2642 mutex_exit(hash_lock); 2643 goto top; 2644 } 2645 ASSERT(*arc_flags & ARC_NOWAIT); 2646 2647 if (done) { 2648 arc_callback_t *acb = NULL; 2649 2650 acb = kmem_zalloc(sizeof (arc_callback_t), 2651 KM_SLEEP); 2652 acb->acb_done = done; 2653 acb->acb_private = private; 2654 if (pio != NULL) 2655 acb->acb_zio_dummy = zio_null(pio, 2656 spa, NULL, NULL, NULL, zio_flags); 2657 2658 ASSERT(acb->acb_done != NULL); 2659 acb->acb_next = hdr->b_acb; 2660 hdr->b_acb = acb; 2661 add_reference(hdr, hash_lock, private); 2662 mutex_exit(hash_lock); 2663 return (0); 2664 } 2665 mutex_exit(hash_lock); 2666 return (0); 2667 } 2668 2669 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2670 2671 if (done) { 2672 add_reference(hdr, hash_lock, private); 2673 /* 2674 * If this block is already in use, create a new 2675 * copy of the data so that we will be guaranteed 2676 * that arc_release() will always succeed. 2677 */ 2678 buf = hdr->b_buf; 2679 ASSERT(buf); 2680 ASSERT(buf->b_data); 2681 if (HDR_BUF_AVAILABLE(hdr)) { 2682 ASSERT(buf->b_efunc == NULL); 2683 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2684 } else { 2685 buf = arc_buf_clone(buf); 2686 } 2687 2688 } else if (*arc_flags & ARC_PREFETCH && 2689 refcount_count(&hdr->b_refcnt) == 0) { 2690 hdr->b_flags |= ARC_PREFETCH; 2691 } 2692 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 2693 arc_access(hdr, hash_lock); 2694 if (*arc_flags & ARC_L2CACHE) 2695 hdr->b_flags |= ARC_L2CACHE; 2696 mutex_exit(hash_lock); 2697 ARCSTAT_BUMP(arcstat_hits); 2698 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2699 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2700 data, metadata, hits); 2701 2702 if (done) 2703 done(NULL, buf, private); 2704 } else { 2705 uint64_t size = BP_GET_LSIZE(bp); 2706 arc_callback_t *acb; 2707 vdev_t *vd = NULL; 2708 uint64_t addr; 2709 boolean_t devw = B_FALSE; 2710 2711 if (hdr == NULL) { 2712 /* this block is not in the cache */ 2713 arc_buf_hdr_t *exists; 2714 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2715 buf = arc_buf_alloc(spa, size, private, type); 2716 hdr = buf->b_hdr; 2717 hdr->b_dva = *BP_IDENTITY(bp); 2718 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 2719 hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2720 exists = buf_hash_insert(hdr, &hash_lock); 2721 if (exists) { 2722 /* somebody beat us to the hash insert */ 2723 mutex_exit(hash_lock); 2724 bzero(&hdr->b_dva, sizeof (dva_t)); 2725 hdr->b_birth = 0; 2726 hdr->b_cksum0 = 0; 2727 (void) arc_buf_remove_ref(buf, private); 2728 goto top; /* restart the IO request */ 2729 } 2730 /* if this is a prefetch, we don't have a reference */ 2731 if (*arc_flags & ARC_PREFETCH) { 2732 (void) remove_reference(hdr, hash_lock, 2733 private); 2734 hdr->b_flags |= ARC_PREFETCH; 2735 } 2736 if (*arc_flags & ARC_L2CACHE) 2737 hdr->b_flags |= ARC_L2CACHE; 2738 if (BP_GET_LEVEL(bp) > 0) 2739 hdr->b_flags |= ARC_INDIRECT; 2740 } else { 2741 /* this block is in the ghost cache */ 2742 ASSERT(GHOST_STATE(hdr->b_state)); 2743 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2744 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2745 ASSERT(hdr->b_buf == NULL); 2746 2747 /* if this is a prefetch, we don't have a reference */ 2748 if (*arc_flags & ARC_PREFETCH) 2749 hdr->b_flags |= ARC_PREFETCH; 2750 else 2751 add_reference(hdr, hash_lock, private); 2752 if (*arc_flags & ARC_L2CACHE) 2753 hdr->b_flags |= ARC_L2CACHE; 2754 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2755 buf->b_hdr = hdr; 2756 buf->b_data = NULL; 2757 buf->b_efunc = NULL; 2758 buf->b_private = NULL; 2759 buf->b_next = NULL; 2760 hdr->b_buf = buf; 2761 ASSERT(hdr->b_datacnt == 0); 2762 hdr->b_datacnt = 1; 2763 arc_get_data_buf(buf); 2764 arc_access(hdr, hash_lock); 2765 } 2766 2767 ASSERT(!GHOST_STATE(hdr->b_state)); 2768 2769 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2770 acb->acb_done = done; 2771 acb->acb_private = private; 2772 2773 ASSERT(hdr->b_acb == NULL); 2774 hdr->b_acb = acb; 2775 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2776 2777 if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL && 2778 (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) { 2779 devw = hdr->b_l2hdr->b_dev->l2ad_writing; 2780 addr = hdr->b_l2hdr->b_daddr; 2781 /* 2782 * Lock out device removal. 2783 */ 2784 if (vdev_is_dead(vd) || 2785 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 2786 vd = NULL; 2787 } 2788 2789 mutex_exit(hash_lock); 2790 2791 ASSERT3U(hdr->b_size, ==, size); 2792 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 2793 uint64_t, size, zbookmark_t *, zb); 2794 ARCSTAT_BUMP(arcstat_misses); 2795 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2796 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2797 data, metadata, misses); 2798 2799 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 2800 /* 2801 * Read from the L2ARC if the following are true: 2802 * 1. The L2ARC vdev was previously cached. 2803 * 2. This buffer still has L2ARC metadata. 2804 * 3. This buffer isn't currently writing to the L2ARC. 2805 * 4. The L2ARC entry wasn't evicted, which may 2806 * also have invalidated the vdev. 2807 * 5. This isn't prefetch and l2arc_noprefetch is set. 2808 */ 2809 if (hdr->b_l2hdr != NULL && 2810 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 2811 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 2812 l2arc_read_callback_t *cb; 2813 2814 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 2815 ARCSTAT_BUMP(arcstat_l2_hits); 2816 2817 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 2818 KM_SLEEP); 2819 cb->l2rcb_buf = buf; 2820 cb->l2rcb_spa = spa; 2821 cb->l2rcb_bp = *bp; 2822 cb->l2rcb_zb = *zb; 2823 cb->l2rcb_flags = zio_flags; 2824 2825 /* 2826 * l2arc read. The SCL_L2ARC lock will be 2827 * released by l2arc_read_done(). 2828 */ 2829 rzio = zio_read_phys(pio, vd, addr, size, 2830 buf->b_data, ZIO_CHECKSUM_OFF, 2831 l2arc_read_done, cb, priority, zio_flags | 2832 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | 2833 ZIO_FLAG_DONT_PROPAGATE | 2834 ZIO_FLAG_DONT_RETRY, B_FALSE); 2835 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 2836 zio_t *, rzio); 2837 ARCSTAT_INCR(arcstat_l2_read_bytes, size); 2838 2839 if (*arc_flags & ARC_NOWAIT) { 2840 zio_nowait(rzio); 2841 return (0); 2842 } 2843 2844 ASSERT(*arc_flags & ARC_WAIT); 2845 if (zio_wait(rzio) == 0) 2846 return (0); 2847 2848 /* l2arc read error; goto zio_read() */ 2849 } else { 2850 DTRACE_PROBE1(l2arc__miss, 2851 arc_buf_hdr_t *, hdr); 2852 ARCSTAT_BUMP(arcstat_l2_misses); 2853 if (HDR_L2_WRITING(hdr)) 2854 ARCSTAT_BUMP(arcstat_l2_rw_clash); 2855 spa_config_exit(spa, SCL_L2ARC, vd); 2856 } 2857 } else { 2858 if (vd != NULL) 2859 spa_config_exit(spa, SCL_L2ARC, vd); 2860 if (l2arc_ndev != 0) { 2861 DTRACE_PROBE1(l2arc__miss, 2862 arc_buf_hdr_t *, hdr); 2863 ARCSTAT_BUMP(arcstat_l2_misses); 2864 } 2865 } 2866 2867 rzio = zio_read(pio, spa, bp, buf->b_data, size, 2868 arc_read_done, buf, priority, zio_flags, zb); 2869 2870 if (*arc_flags & ARC_WAIT) 2871 return (zio_wait(rzio)); 2872 2873 ASSERT(*arc_flags & ARC_NOWAIT); 2874 zio_nowait(rzio); 2875 } 2876 return (0); 2877 } 2878 2879 void 2880 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2881 { 2882 ASSERT(buf->b_hdr != NULL); 2883 ASSERT(buf->b_hdr->b_state != arc_anon); 2884 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2885 ASSERT(buf->b_efunc == NULL); 2886 ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr)); 2887 2888 buf->b_efunc = func; 2889 buf->b_private = private; 2890 } 2891 2892 /* 2893 * This is used by the DMU to let the ARC know that a buffer is 2894 * being evicted, so the ARC should clean up. If this arc buf 2895 * is not yet in the evicted state, it will be put there. 2896 */ 2897 int 2898 arc_buf_evict(arc_buf_t *buf) 2899 { 2900 arc_buf_hdr_t *hdr; 2901 kmutex_t *hash_lock; 2902 arc_buf_t **bufp; 2903 2904 rw_enter(&buf->b_lock, RW_WRITER); 2905 hdr = buf->b_hdr; 2906 if (hdr == NULL) { 2907 /* 2908 * We are in arc_do_user_evicts(). 2909 */ 2910 ASSERT(buf->b_data == NULL); 2911 rw_exit(&buf->b_lock); 2912 return (0); 2913 } else if (buf->b_data == NULL) { 2914 arc_buf_t copy = *buf; /* structure assignment */ 2915 /* 2916 * We are on the eviction list; process this buffer now 2917 * but let arc_do_user_evicts() do the reaping. 2918 */ 2919 buf->b_efunc = NULL; 2920 rw_exit(&buf->b_lock); 2921 VERIFY(copy.b_efunc(©) == 0); 2922 return (1); 2923 } 2924 hash_lock = HDR_LOCK(hdr); 2925 mutex_enter(hash_lock); 2926 2927 ASSERT(buf->b_hdr == hdr); 2928 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2929 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2930 2931 /* 2932 * Pull this buffer off of the hdr 2933 */ 2934 bufp = &hdr->b_buf; 2935 while (*bufp != buf) 2936 bufp = &(*bufp)->b_next; 2937 *bufp = buf->b_next; 2938 2939 ASSERT(buf->b_data != NULL); 2940 arc_buf_destroy(buf, FALSE, FALSE); 2941 2942 if (hdr->b_datacnt == 0) { 2943 arc_state_t *old_state = hdr->b_state; 2944 arc_state_t *evicted_state; 2945 2946 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2947 2948 evicted_state = 2949 (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2950 2951 mutex_enter(&old_state->arcs_mtx); 2952 mutex_enter(&evicted_state->arcs_mtx); 2953 2954 arc_change_state(evicted_state, hdr, hash_lock); 2955 ASSERT(HDR_IN_HASH_TABLE(hdr)); 2956 hdr->b_flags |= ARC_IN_HASH_TABLE; 2957 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2958 2959 mutex_exit(&evicted_state->arcs_mtx); 2960 mutex_exit(&old_state->arcs_mtx); 2961 } 2962 mutex_exit(hash_lock); 2963 rw_exit(&buf->b_lock); 2964 2965 VERIFY(buf->b_efunc(buf) == 0); 2966 buf->b_efunc = NULL; 2967 buf->b_private = NULL; 2968 buf->b_hdr = NULL; 2969 kmem_cache_free(buf_cache, buf); 2970 return (1); 2971 } 2972 2973 /* 2974 * Release this buffer from the cache. This must be done 2975 * after a read and prior to modifying the buffer contents. 2976 * If the buffer has more than one reference, we must make 2977 * a new hdr for the buffer. 2978 */ 2979 void 2980 arc_release(arc_buf_t *buf, void *tag) 2981 { 2982 arc_buf_hdr_t *hdr; 2983 kmutex_t *hash_lock; 2984 l2arc_buf_hdr_t *l2hdr; 2985 uint64_t buf_size; 2986 boolean_t released = B_FALSE; 2987 2988 rw_enter(&buf->b_lock, RW_WRITER); 2989 hdr = buf->b_hdr; 2990 2991 /* this buffer is not on any list */ 2992 ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2993 2994 if (hdr->b_state == arc_anon) { 2995 /* this buffer is already released */ 2996 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2997 ASSERT(BUF_EMPTY(hdr)); 2998 ASSERT(buf->b_efunc == NULL); 2999 arc_buf_thaw(buf); 3000 rw_exit(&buf->b_lock); 3001 released = B_TRUE; 3002 } else { 3003 hash_lock = HDR_LOCK(hdr); 3004 mutex_enter(hash_lock); 3005 } 3006 3007 l2hdr = hdr->b_l2hdr; 3008 if (l2hdr) { 3009 mutex_enter(&l2arc_buflist_mtx); 3010 hdr->b_l2hdr = NULL; 3011 buf_size = hdr->b_size; 3012 } 3013 3014 if (released) 3015 goto out; 3016 3017 /* 3018 * Do we have more than one buf? 3019 */ 3020 if (hdr->b_datacnt > 1) { 3021 arc_buf_hdr_t *nhdr; 3022 arc_buf_t **bufp; 3023 uint64_t blksz = hdr->b_size; 3024 uint64_t spa = hdr->b_spa; 3025 arc_buf_contents_t type = hdr->b_type; 3026 uint32_t flags = hdr->b_flags; 3027 3028 ASSERT(hdr->b_buf != buf || buf->b_next != NULL); 3029 /* 3030 * Pull the data off of this buf and attach it to 3031 * a new anonymous buf. 3032 */ 3033 (void) remove_reference(hdr, hash_lock, tag); 3034 bufp = &hdr->b_buf; 3035 while (*bufp != buf) 3036 bufp = &(*bufp)->b_next; 3037 *bufp = (*bufp)->b_next; 3038 buf->b_next = NULL; 3039 3040 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 3041 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 3042 if (refcount_is_zero(&hdr->b_refcnt)) { 3043 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 3044 ASSERT3U(*size, >=, hdr->b_size); 3045 atomic_add_64(size, -hdr->b_size); 3046 } 3047 hdr->b_datacnt -= 1; 3048 arc_cksum_verify(buf); 3049 3050 mutex_exit(hash_lock); 3051 3052 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 3053 nhdr->b_size = blksz; 3054 nhdr->b_spa = spa; 3055 nhdr->b_type = type; 3056 nhdr->b_buf = buf; 3057 nhdr->b_state = arc_anon; 3058 nhdr->b_arc_access = 0; 3059 nhdr->b_flags = flags & ARC_L2_WRITING; 3060 nhdr->b_l2hdr = NULL; 3061 nhdr->b_datacnt = 1; 3062 nhdr->b_freeze_cksum = NULL; 3063 (void) refcount_add(&nhdr->b_refcnt, tag); 3064 buf->b_hdr = nhdr; 3065 rw_exit(&buf->b_lock); 3066 atomic_add_64(&arc_anon->arcs_size, blksz); 3067 } else { 3068 rw_exit(&buf->b_lock); 3069 ASSERT(refcount_count(&hdr->b_refcnt) == 1); 3070 ASSERT(!list_link_active(&hdr->b_arc_node)); 3071 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3072 arc_change_state(arc_anon, hdr, hash_lock); 3073 hdr->b_arc_access = 0; 3074 mutex_exit(hash_lock); 3075 3076 bzero(&hdr->b_dva, sizeof (dva_t)); 3077 hdr->b_birth = 0; 3078 hdr->b_cksum0 = 0; 3079 arc_buf_thaw(buf); 3080 } 3081 buf->b_efunc = NULL; 3082 buf->b_private = NULL; 3083 3084 out: 3085 if (l2hdr) { 3086 list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 3087 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 3088 ARCSTAT_INCR(arcstat_l2_size, -buf_size); 3089 mutex_exit(&l2arc_buflist_mtx); 3090 } 3091 } 3092 3093 int 3094 arc_released(arc_buf_t *buf) 3095 { 3096 int released; 3097 3098 rw_enter(&buf->b_lock, RW_READER); 3099 released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 3100 rw_exit(&buf->b_lock); 3101 return (released); 3102 } 3103 3104 int 3105 arc_has_callback(arc_buf_t *buf) 3106 { 3107 int callback; 3108 3109 rw_enter(&buf->b_lock, RW_READER); 3110 callback = (buf->b_efunc != NULL); 3111 rw_exit(&buf->b_lock); 3112 return (callback); 3113 } 3114 3115 #ifdef ZFS_DEBUG 3116 int 3117 arc_referenced(arc_buf_t *buf) 3118 { 3119 int referenced; 3120 3121 rw_enter(&buf->b_lock, RW_READER); 3122 referenced = (refcount_count(&buf->b_hdr->b_refcnt)); 3123 rw_exit(&buf->b_lock); 3124 return (referenced); 3125 } 3126 #endif 3127 3128 static void 3129 arc_write_ready(zio_t *zio) 3130 { 3131 arc_write_callback_t *callback = zio->io_private; 3132 arc_buf_t *buf = callback->awcb_buf; 3133 arc_buf_hdr_t *hdr = buf->b_hdr; 3134 3135 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 3136 callback->awcb_ready(zio, buf, callback->awcb_private); 3137 3138 /* 3139 * If the IO is already in progress, then this is a re-write 3140 * attempt, so we need to thaw and re-compute the cksum. 3141 * It is the responsibility of the callback to handle the 3142 * accounting for any re-write attempt. 3143 */ 3144 if (HDR_IO_IN_PROGRESS(hdr)) { 3145 mutex_enter(&hdr->b_freeze_lock); 3146 if (hdr->b_freeze_cksum != NULL) { 3147 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 3148 hdr->b_freeze_cksum = NULL; 3149 } 3150 mutex_exit(&hdr->b_freeze_lock); 3151 } 3152 arc_cksum_compute(buf, B_FALSE); 3153 hdr->b_flags |= ARC_IO_IN_PROGRESS; 3154 } 3155 3156 static void 3157 arc_write_done(zio_t *zio) 3158 { 3159 arc_write_callback_t *callback = zio->io_private; 3160 arc_buf_t *buf = callback->awcb_buf; 3161 arc_buf_hdr_t *hdr = buf->b_hdr; 3162 3163 ASSERT(hdr->b_acb == NULL); 3164 3165 if (zio->io_error == 0) { 3166 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3167 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 3168 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3169 } else { 3170 ASSERT(BUF_EMPTY(hdr)); 3171 } 3172 3173 /* 3174 * If the block to be written was all-zero, we may have 3175 * compressed it away. In this case no write was performed 3176 * so there will be no dva/birth-date/checksum. The buffer 3177 * must therefor remain anonymous (and uncached). 3178 */ 3179 if (!BUF_EMPTY(hdr)) { 3180 arc_buf_hdr_t *exists; 3181 kmutex_t *hash_lock; 3182 3183 ASSERT(zio->io_error == 0); 3184 3185 arc_cksum_verify(buf); 3186 3187 exists = buf_hash_insert(hdr, &hash_lock); 3188 if (exists) { 3189 /* 3190 * This can only happen if we overwrite for 3191 * sync-to-convergence, because we remove 3192 * buffers from the hash table when we arc_free(). 3193 */ 3194 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 3195 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 3196 panic("bad overwrite, hdr=%p exists=%p", 3197 (void *)hdr, (void *)exists); 3198 ASSERT(refcount_is_zero(&exists->b_refcnt)); 3199 arc_change_state(arc_anon, exists, hash_lock); 3200 mutex_exit(hash_lock); 3201 arc_hdr_destroy(exists); 3202 exists = buf_hash_insert(hdr, &hash_lock); 3203 ASSERT3P(exists, ==, NULL); 3204 } else { 3205 /* Dedup */ 3206 ASSERT(hdr->b_datacnt == 1); 3207 ASSERT(hdr->b_state == arc_anon); 3208 ASSERT(BP_GET_DEDUP(zio->io_bp)); 3209 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 3210 } 3211 } 3212 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3213 /* if it's not anon, we are doing a scrub */ 3214 if (!exists && hdr->b_state == arc_anon) 3215 arc_access(hdr, hash_lock); 3216 mutex_exit(hash_lock); 3217 } else { 3218 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3219 } 3220 3221 ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3222 callback->awcb_done(zio, buf, callback->awcb_private); 3223 3224 kmem_free(callback, sizeof (arc_write_callback_t)); 3225 } 3226 3227 zio_t * 3228 arc_write(zio_t *pio, spa_t *spa, uint64_t txg, 3229 blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp, 3230 arc_done_func_t *ready, arc_done_func_t *done, void *private, 3231 int priority, int zio_flags, const zbookmark_t *zb) 3232 { 3233 arc_buf_hdr_t *hdr = buf->b_hdr; 3234 arc_write_callback_t *callback; 3235 zio_t *zio; 3236 3237 ASSERT(ready != NULL); 3238 ASSERT(done != NULL); 3239 ASSERT(!HDR_IO_ERROR(hdr)); 3240 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3241 ASSERT(hdr->b_acb == NULL); 3242 if (l2arc) 3243 hdr->b_flags |= ARC_L2CACHE; 3244 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3245 callback->awcb_ready = ready; 3246 callback->awcb_done = done; 3247 callback->awcb_private = private; 3248 callback->awcb_buf = buf; 3249 3250 zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, 3251 arc_write_ready, arc_write_done, callback, priority, zio_flags, zb); 3252 3253 return (zio); 3254 } 3255 3256 void 3257 arc_free(spa_t *spa, const blkptr_t *bp) 3258 { 3259 arc_buf_hdr_t *ab; 3260 kmutex_t *hash_lock; 3261 uint64_t guid = spa_guid(spa); 3262 3263 /* 3264 * If this buffer is in the cache, release it, so it can be re-used. 3265 */ 3266 ab = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), 3267 &hash_lock); 3268 if (ab != NULL) { 3269 if (ab->b_state != arc_anon) 3270 arc_change_state(arc_anon, ab, hash_lock); 3271 if (HDR_IO_IN_PROGRESS(ab)) { 3272 /* 3273 * This should only happen when we prefetch. 3274 */ 3275 ASSERT(ab->b_flags & ARC_PREFETCH); 3276 ASSERT3U(ab->b_datacnt, ==, 1); 3277 ab->b_flags |= ARC_FREED_IN_READ; 3278 if (HDR_IN_HASH_TABLE(ab)) 3279 buf_hash_remove(ab); 3280 ab->b_arc_access = 0; 3281 bzero(&ab->b_dva, sizeof (dva_t)); 3282 ab->b_birth = 0; 3283 ab->b_cksum0 = 0; 3284 ab->b_buf->b_efunc = NULL; 3285 ab->b_buf->b_private = NULL; 3286 mutex_exit(hash_lock); 3287 } else { 3288 ASSERT(refcount_is_zero(&ab->b_refcnt)); 3289 ab->b_flags |= ARC_FREE_IN_PROGRESS; 3290 mutex_exit(hash_lock); 3291 arc_hdr_destroy(ab); 3292 ARCSTAT_BUMP(arcstat_deleted); 3293 } 3294 } 3295 } 3296 3297 static int 3298 arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg) 3299 { 3300 #ifdef _KERNEL 3301 uint64_t available_memory = ptob(freemem); 3302 static uint64_t page_load = 0; 3303 static uint64_t last_txg = 0; 3304 3305 #if defined(__i386) 3306 available_memory = 3307 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 3308 #endif 3309 if (available_memory >= zfs_write_limit_max) 3310 return (0); 3311 3312 if (txg > last_txg) { 3313 last_txg = txg; 3314 page_load = 0; 3315 } 3316 /* 3317 * If we are in pageout, we know that memory is already tight, 3318 * the arc is already going to be evicting, so we just want to 3319 * continue to let page writes occur as quickly as possible. 3320 */ 3321 if (curproc == proc_pageout) { 3322 if (page_load > MAX(ptob(minfree), available_memory) / 4) 3323 return (ERESTART); 3324 /* Note: reserve is inflated, so we deflate */ 3325 page_load += reserve / 8; 3326 return (0); 3327 } else if (page_load > 0 && arc_reclaim_needed()) { 3328 /* memory is low, delay before restarting */ 3329 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3330 return (EAGAIN); 3331 } 3332 page_load = 0; 3333 3334 if (arc_size > arc_c_min) { 3335 uint64_t evictable_memory = 3336 arc_mru->arcs_lsize[ARC_BUFC_DATA] + 3337 arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 3338 arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 3339 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 3340 available_memory += MIN(evictable_memory, arc_size - arc_c_min); 3341 } 3342 3343 if (inflight_data > available_memory / 4) { 3344 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3345 return (ERESTART); 3346 } 3347 #endif 3348 return (0); 3349 } 3350 3351 void 3352 arc_tempreserve_clear(uint64_t reserve) 3353 { 3354 atomic_add_64(&arc_tempreserve, -reserve); 3355 ASSERT((int64_t)arc_tempreserve >= 0); 3356 } 3357 3358 int 3359 arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3360 { 3361 int error; 3362 uint64_t anon_size; 3363 3364 #ifdef ZFS_DEBUG 3365 /* 3366 * Once in a while, fail for no reason. Everything should cope. 3367 */ 3368 if (spa_get_random(10000) == 0) { 3369 dprintf("forcing random failure\n"); 3370 return (ERESTART); 3371 } 3372 #endif 3373 if (reserve > arc_c/4 && !arc_no_grow) 3374 arc_c = MIN(arc_c_max, reserve * 4); 3375 if (reserve > arc_c) 3376 return (ENOMEM); 3377 3378 /* 3379 * Don't count loaned bufs as in flight dirty data to prevent long 3380 * network delays from blocking transactions that are ready to be 3381 * assigned to a txg. 3382 */ 3383 anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0); 3384 3385 /* 3386 * Writes will, almost always, require additional memory allocations 3387 * in order to compress/encrypt/etc the data. We therefor need to 3388 * make sure that there is sufficient available memory for this. 3389 */ 3390 if (error = arc_memory_throttle(reserve, anon_size, txg)) 3391 return (error); 3392 3393 /* 3394 * Throttle writes when the amount of dirty data in the cache 3395 * gets too large. We try to keep the cache less than half full 3396 * of dirty blocks so that our sync times don't grow too large. 3397 * Note: if two requests come in concurrently, we might let them 3398 * both succeed, when one of them should fail. Not a huge deal. 3399 */ 3400 3401 if (reserve + arc_tempreserve + anon_size > arc_c / 2 && 3402 anon_size > arc_c / 4) { 3403 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 3404 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 3405 arc_tempreserve>>10, 3406 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 3407 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 3408 reserve>>10, arc_c>>10); 3409 return (ERESTART); 3410 } 3411 atomic_add_64(&arc_tempreserve, reserve); 3412 return (0); 3413 } 3414 3415 void 3416 arc_init(void) 3417 { 3418 mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3419 cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3420 3421 /* Convert seconds to clock ticks */ 3422 arc_min_prefetch_lifespan = 1 * hz; 3423 3424 /* Start out with 1/8 of all memory */ 3425 arc_c = physmem * PAGESIZE / 8; 3426 3427 #ifdef _KERNEL 3428 /* 3429 * On architectures where the physical memory can be larger 3430 * than the addressable space (intel in 32-bit mode), we may 3431 * need to limit the cache to 1/8 of VM size. 3432 */ 3433 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3434 #endif 3435 3436 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 3437 arc_c_min = MAX(arc_c / 4, 64<<20); 3438 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 3439 if (arc_c * 8 >= 1<<30) 3440 arc_c_max = (arc_c * 8) - (1<<30); 3441 else 3442 arc_c_max = arc_c_min; 3443 arc_c_max = MAX(arc_c * 6, arc_c_max); 3444 3445 /* 3446 * Allow the tunables to override our calculations if they are 3447 * reasonable (ie. over 64MB) 3448 */ 3449 if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 3450 arc_c_max = zfs_arc_max; 3451 if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 3452 arc_c_min = zfs_arc_min; 3453 3454 arc_c = arc_c_max; 3455 arc_p = (arc_c >> 1); 3456 3457 /* limit meta-data to 1/4 of the arc capacity */ 3458 arc_meta_limit = arc_c_max / 4; 3459 3460 /* Allow the tunable to override if it is reasonable */ 3461 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 3462 arc_meta_limit = zfs_arc_meta_limit; 3463 3464 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 3465 arc_c_min = arc_meta_limit / 2; 3466 3467 if (zfs_arc_grow_retry > 0) 3468 arc_grow_retry = zfs_arc_grow_retry; 3469 3470 if (zfs_arc_shrink_shift > 0) 3471 arc_shrink_shift = zfs_arc_shrink_shift; 3472 3473 if (zfs_arc_p_min_shift > 0) 3474 arc_p_min_shift = zfs_arc_p_min_shift; 3475 3476 /* if kmem_flags are set, lets try to use less memory */ 3477 if (kmem_debugging()) 3478 arc_c = arc_c / 2; 3479 if (arc_c < arc_c_min) 3480 arc_c = arc_c_min; 3481 3482 arc_anon = &ARC_anon; 3483 arc_mru = &ARC_mru; 3484 arc_mru_ghost = &ARC_mru_ghost; 3485 arc_mfu = &ARC_mfu; 3486 arc_mfu_ghost = &ARC_mfu_ghost; 3487 arc_l2c_only = &ARC_l2c_only; 3488 arc_size = 0; 3489 3490 mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3491 mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3492 mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3493 mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3494 mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3495 mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3496 3497 list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 3498 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3499 list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 3500 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3501 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 3502 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3503 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 3504 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3505 list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 3506 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3507 list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 3508 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3509 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 3510 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3511 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 3512 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3513 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3514 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3515 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3516 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3517 3518 buf_init(); 3519 3520 arc_thread_exit = 0; 3521 arc_eviction_list = NULL; 3522 mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 3523 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3524 3525 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 3526 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 3527 3528 if (arc_ksp != NULL) { 3529 arc_ksp->ks_data = &arc_stats; 3530 kstat_install(arc_ksp); 3531 } 3532 3533 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3534 TS_RUN, minclsyspri); 3535 3536 arc_dead = FALSE; 3537 arc_warm = B_FALSE; 3538 3539 if (zfs_write_limit_max == 0) 3540 zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift; 3541 else 3542 zfs_write_limit_shift = 0; 3543 mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL); 3544 } 3545 3546 void 3547 arc_fini(void) 3548 { 3549 mutex_enter(&arc_reclaim_thr_lock); 3550 arc_thread_exit = 1; 3551 while (arc_thread_exit != 0) 3552 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3553 mutex_exit(&arc_reclaim_thr_lock); 3554 3555 arc_flush(NULL); 3556 3557 arc_dead = TRUE; 3558 3559 if (arc_ksp != NULL) { 3560 kstat_delete(arc_ksp); 3561 arc_ksp = NULL; 3562 } 3563 3564 mutex_destroy(&arc_eviction_mtx); 3565 mutex_destroy(&arc_reclaim_thr_lock); 3566 cv_destroy(&arc_reclaim_thr_cv); 3567 3568 list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 3569 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 3570 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 3571 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 3572 list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 3573 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 3574 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 3575 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3576 3577 mutex_destroy(&arc_anon->arcs_mtx); 3578 mutex_destroy(&arc_mru->arcs_mtx); 3579 mutex_destroy(&arc_mru_ghost->arcs_mtx); 3580 mutex_destroy(&arc_mfu->arcs_mtx); 3581 mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3582 mutex_destroy(&arc_l2c_only->arcs_mtx); 3583 3584 mutex_destroy(&zfs_write_limit_lock); 3585 3586 buf_fini(); 3587 3588 ASSERT(arc_loaned_bytes == 0); 3589 } 3590 3591 /* 3592 * Level 2 ARC 3593 * 3594 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3595 * It uses dedicated storage devices to hold cached data, which are populated 3596 * using large infrequent writes. The main role of this cache is to boost 3597 * the performance of random read workloads. The intended L2ARC devices 3598 * include short-stroked disks, solid state disks, and other media with 3599 * substantially faster read latency than disk. 3600 * 3601 * +-----------------------+ 3602 * | ARC | 3603 * +-----------------------+ 3604 * | ^ ^ 3605 * | | | 3606 * l2arc_feed_thread() arc_read() 3607 * | | | 3608 * | l2arc read | 3609 * V | | 3610 * +---------------+ | 3611 * | L2ARC | | 3612 * +---------------+ | 3613 * | ^ | 3614 * l2arc_write() | | 3615 * | | | 3616 * V | | 3617 * +-------+ +-------+ 3618 * | vdev | | vdev | 3619 * | cache | | cache | 3620 * +-------+ +-------+ 3621 * +=========+ .-----. 3622 * : L2ARC : |-_____-| 3623 * : devices : | Disks | 3624 * +=========+ `-_____-' 3625 * 3626 * Read requests are satisfied from the following sources, in order: 3627 * 3628 * 1) ARC 3629 * 2) vdev cache of L2ARC devices 3630 * 3) L2ARC devices 3631 * 4) vdev cache of disks 3632 * 5) disks 3633 * 3634 * Some L2ARC device types exhibit extremely slow write performance. 3635 * To accommodate for this there are some significant differences between 3636 * the L2ARC and traditional cache design: 3637 * 3638 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3639 * the ARC behave as usual, freeing buffers and placing headers on ghost 3640 * lists. The ARC does not send buffers to the L2ARC during eviction as 3641 * this would add inflated write latencies for all ARC memory pressure. 3642 * 3643 * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3644 * It does this by periodically scanning buffers from the eviction-end of 3645 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3646 * not already there. It scans until a headroom of buffers is satisfied, 3647 * which itself is a buffer for ARC eviction. The thread that does this is 3648 * l2arc_feed_thread(), illustrated below; example sizes are included to 3649 * provide a better sense of ratio than this diagram: 3650 * 3651 * head --> tail 3652 * +---------------------+----------+ 3653 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3654 * +---------------------+----------+ | o L2ARC eligible 3655 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3656 * +---------------------+----------+ | 3657 * 15.9 Gbytes ^ 32 Mbytes | 3658 * headroom | 3659 * l2arc_feed_thread() 3660 * | 3661 * l2arc write hand <--[oooo]--' 3662 * | 8 Mbyte 3663 * | write max 3664 * V 3665 * +==============================+ 3666 * L2ARC dev |####|#|###|###| |####| ... | 3667 * +==============================+ 3668 * 32 Gbytes 3669 * 3670 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3671 * evicted, then the L2ARC has cached a buffer much sooner than it probably 3672 * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3673 * safe to say that this is an uncommon case, since buffers at the end of 3674 * the ARC lists have moved there due to inactivity. 3675 * 3676 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3677 * then the L2ARC simply misses copying some buffers. This serves as a 3678 * pressure valve to prevent heavy read workloads from both stalling the ARC 3679 * with waits and clogging the L2ARC with writes. This also helps prevent 3680 * the potential for the L2ARC to churn if it attempts to cache content too 3681 * quickly, such as during backups of the entire pool. 3682 * 3683 * 5. After system boot and before the ARC has filled main memory, there are 3684 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 3685 * lists can remain mostly static. Instead of searching from tail of these 3686 * lists as pictured, the l2arc_feed_thread() will search from the list heads 3687 * for eligible buffers, greatly increasing its chance of finding them. 3688 * 3689 * The L2ARC device write speed is also boosted during this time so that 3690 * the L2ARC warms up faster. Since there have been no ARC evictions yet, 3691 * there are no L2ARC reads, and no fear of degrading read performance 3692 * through increased writes. 3693 * 3694 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3695 * the vdev queue can aggregate them into larger and fewer writes. Each 3696 * device is written to in a rotor fashion, sweeping writes through 3697 * available space then repeating. 3698 * 3699 * 7. The L2ARC does not store dirty content. It never needs to flush 3700 * write buffers back to disk based storage. 3701 * 3702 * 8. If an ARC buffer is written (and dirtied) which also exists in the 3703 * L2ARC, the now stale L2ARC buffer is immediately dropped. 3704 * 3705 * The performance of the L2ARC can be tweaked by a number of tunables, which 3706 * may be necessary for different workloads: 3707 * 3708 * l2arc_write_max max write bytes per interval 3709 * l2arc_write_boost extra write bytes during device warmup 3710 * l2arc_noprefetch skip caching prefetched buffers 3711 * l2arc_headroom number of max device writes to precache 3712 * l2arc_feed_secs seconds between L2ARC writing 3713 * 3714 * Tunables may be removed or added as future performance improvements are 3715 * integrated, and also may become zpool properties. 3716 * 3717 * There are three key functions that control how the L2ARC warms up: 3718 * 3719 * l2arc_write_eligible() check if a buffer is eligible to cache 3720 * l2arc_write_size() calculate how much to write 3721 * l2arc_write_interval() calculate sleep delay between writes 3722 * 3723 * These three functions determine what to write, how much, and how quickly 3724 * to send writes. 3725 */ 3726 3727 static boolean_t 3728 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab) 3729 { 3730 /* 3731 * A buffer is *not* eligible for the L2ARC if it: 3732 * 1. belongs to a different spa. 3733 * 2. is already cached on the L2ARC. 3734 * 3. has an I/O in progress (it may be an incomplete read). 3735 * 4. is flagged not eligible (zfs property). 3736 */ 3737 if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL || 3738 HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) 3739 return (B_FALSE); 3740 3741 return (B_TRUE); 3742 } 3743 3744 static uint64_t 3745 l2arc_write_size(l2arc_dev_t *dev) 3746 { 3747 uint64_t size; 3748 3749 size = dev->l2ad_write; 3750 3751 if (arc_warm == B_FALSE) 3752 size += dev->l2ad_boost; 3753 3754 return (size); 3755 3756 } 3757 3758 static clock_t 3759 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 3760 { 3761 clock_t interval, next, now; 3762 3763 /* 3764 * If the ARC lists are busy, increase our write rate; if the 3765 * lists are stale, idle back. This is achieved by checking 3766 * how much we previously wrote - if it was more than half of 3767 * what we wanted, schedule the next write much sooner. 3768 */ 3769 if (l2arc_feed_again && wrote > (wanted / 2)) 3770 interval = (hz * l2arc_feed_min_ms) / 1000; 3771 else 3772 interval = hz * l2arc_feed_secs; 3773 3774 now = ddi_get_lbolt(); 3775 next = MAX(now, MIN(now + interval, began + interval)); 3776 3777 return (next); 3778 } 3779 3780 static void 3781 l2arc_hdr_stat_add(void) 3782 { 3783 ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 3784 ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 3785 } 3786 3787 static void 3788 l2arc_hdr_stat_remove(void) 3789 { 3790 ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 3791 ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 3792 } 3793 3794 /* 3795 * Cycle through L2ARC devices. This is how L2ARC load balances. 3796 * If a device is returned, this also returns holding the spa config lock. 3797 */ 3798 static l2arc_dev_t * 3799 l2arc_dev_get_next(void) 3800 { 3801 l2arc_dev_t *first, *next = NULL; 3802 3803 /* 3804 * Lock out the removal of spas (spa_namespace_lock), then removal 3805 * of cache devices (l2arc_dev_mtx). Once a device has been selected, 3806 * both locks will be dropped and a spa config lock held instead. 3807 */ 3808 mutex_enter(&spa_namespace_lock); 3809 mutex_enter(&l2arc_dev_mtx); 3810 3811 /* if there are no vdevs, there is nothing to do */ 3812 if (l2arc_ndev == 0) 3813 goto out; 3814 3815 first = NULL; 3816 next = l2arc_dev_last; 3817 do { 3818 /* loop around the list looking for a non-faulted vdev */ 3819 if (next == NULL) { 3820 next = list_head(l2arc_dev_list); 3821 } else { 3822 next = list_next(l2arc_dev_list, next); 3823 if (next == NULL) 3824 next = list_head(l2arc_dev_list); 3825 } 3826 3827 /* if we have come back to the start, bail out */ 3828 if (first == NULL) 3829 first = next; 3830 else if (next == first) 3831 break; 3832 3833 } while (vdev_is_dead(next->l2ad_vdev)); 3834 3835 /* if we were unable to find any usable vdevs, return NULL */ 3836 if (vdev_is_dead(next->l2ad_vdev)) 3837 next = NULL; 3838 3839 l2arc_dev_last = next; 3840 3841 out: 3842 mutex_exit(&l2arc_dev_mtx); 3843 3844 /* 3845 * Grab the config lock to prevent the 'next' device from being 3846 * removed while we are writing to it. 3847 */ 3848 if (next != NULL) 3849 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 3850 mutex_exit(&spa_namespace_lock); 3851 3852 return (next); 3853 } 3854 3855 /* 3856 * Free buffers that were tagged for destruction. 3857 */ 3858 static void 3859 l2arc_do_free_on_write() 3860 { 3861 list_t *buflist; 3862 l2arc_data_free_t *df, *df_prev; 3863 3864 mutex_enter(&l2arc_free_on_write_mtx); 3865 buflist = l2arc_free_on_write; 3866 3867 for (df = list_tail(buflist); df; df = df_prev) { 3868 df_prev = list_prev(buflist, df); 3869 ASSERT(df->l2df_data != NULL); 3870 ASSERT(df->l2df_func != NULL); 3871 df->l2df_func(df->l2df_data, df->l2df_size); 3872 list_remove(buflist, df); 3873 kmem_free(df, sizeof (l2arc_data_free_t)); 3874 } 3875 3876 mutex_exit(&l2arc_free_on_write_mtx); 3877 } 3878 3879 /* 3880 * A write to a cache device has completed. Update all headers to allow 3881 * reads from these buffers to begin. 3882 */ 3883 static void 3884 l2arc_write_done(zio_t *zio) 3885 { 3886 l2arc_write_callback_t *cb; 3887 l2arc_dev_t *dev; 3888 list_t *buflist; 3889 arc_buf_hdr_t *head, *ab, *ab_prev; 3890 l2arc_buf_hdr_t *abl2; 3891 kmutex_t *hash_lock; 3892 3893 cb = zio->io_private; 3894 ASSERT(cb != NULL); 3895 dev = cb->l2wcb_dev; 3896 ASSERT(dev != NULL); 3897 head = cb->l2wcb_head; 3898 ASSERT(head != NULL); 3899 buflist = dev->l2ad_buflist; 3900 ASSERT(buflist != NULL); 3901 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 3902 l2arc_write_callback_t *, cb); 3903 3904 if (zio->io_error != 0) 3905 ARCSTAT_BUMP(arcstat_l2_writes_error); 3906 3907 mutex_enter(&l2arc_buflist_mtx); 3908 3909 /* 3910 * All writes completed, or an error was hit. 3911 */ 3912 for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 3913 ab_prev = list_prev(buflist, ab); 3914 3915 hash_lock = HDR_LOCK(ab); 3916 if (!mutex_tryenter(hash_lock)) { 3917 /* 3918 * This buffer misses out. It may be in a stage 3919 * of eviction. Its ARC_L2_WRITING flag will be 3920 * left set, denying reads to this buffer. 3921 */ 3922 ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 3923 continue; 3924 } 3925 3926 if (zio->io_error != 0) { 3927 /* 3928 * Error - drop L2ARC entry. 3929 */ 3930 list_remove(buflist, ab); 3931 abl2 = ab->b_l2hdr; 3932 ab->b_l2hdr = NULL; 3933 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 3934 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 3935 } 3936 3937 /* 3938 * Allow ARC to begin reads to this L2ARC entry. 3939 */ 3940 ab->b_flags &= ~ARC_L2_WRITING; 3941 3942 mutex_exit(hash_lock); 3943 } 3944 3945 atomic_inc_64(&l2arc_writes_done); 3946 list_remove(buflist, head); 3947 kmem_cache_free(hdr_cache, head); 3948 mutex_exit(&l2arc_buflist_mtx); 3949 3950 l2arc_do_free_on_write(); 3951 3952 kmem_free(cb, sizeof (l2arc_write_callback_t)); 3953 } 3954 3955 /* 3956 * A read to a cache device completed. Validate buffer contents before 3957 * handing over to the regular ARC routines. 3958 */ 3959 static void 3960 l2arc_read_done(zio_t *zio) 3961 { 3962 l2arc_read_callback_t *cb; 3963 arc_buf_hdr_t *hdr; 3964 arc_buf_t *buf; 3965 kmutex_t *hash_lock; 3966 int equal; 3967 3968 ASSERT(zio->io_vd != NULL); 3969 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 3970 3971 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 3972 3973 cb = zio->io_private; 3974 ASSERT(cb != NULL); 3975 buf = cb->l2rcb_buf; 3976 ASSERT(buf != NULL); 3977 hdr = buf->b_hdr; 3978 ASSERT(hdr != NULL); 3979 3980 hash_lock = HDR_LOCK(hdr); 3981 mutex_enter(hash_lock); 3982 3983 /* 3984 * Check this survived the L2ARC journey. 3985 */ 3986 equal = arc_cksum_equal(buf); 3987 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 3988 mutex_exit(hash_lock); 3989 zio->io_private = buf; 3990 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 3991 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 3992 arc_read_done(zio); 3993 } else { 3994 mutex_exit(hash_lock); 3995 /* 3996 * Buffer didn't survive caching. Increment stats and 3997 * reissue to the original storage device. 3998 */ 3999 if (zio->io_error != 0) { 4000 ARCSTAT_BUMP(arcstat_l2_io_error); 4001 } else { 4002 zio->io_error = EIO; 4003 } 4004 if (!equal) 4005 ARCSTAT_BUMP(arcstat_l2_cksum_bad); 4006 4007 /* 4008 * If there's no waiter, issue an async i/o to the primary 4009 * storage now. If there *is* a waiter, the caller must 4010 * issue the i/o in a context where it's OK to block. 4011 */ 4012 if (zio->io_waiter == NULL) { 4013 zio_t *pio = zio_unique_parent(zio); 4014 4015 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 4016 4017 zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp, 4018 buf->b_data, zio->io_size, arc_read_done, buf, 4019 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb)); 4020 } 4021 } 4022 4023 kmem_free(cb, sizeof (l2arc_read_callback_t)); 4024 } 4025 4026 /* 4027 * This is the list priority from which the L2ARC will search for pages to 4028 * cache. This is used within loops (0..3) to cycle through lists in the 4029 * desired order. This order can have a significant effect on cache 4030 * performance. 4031 * 4032 * Currently the metadata lists are hit first, MFU then MRU, followed by 4033 * the data lists. This function returns a locked list, and also returns 4034 * the lock pointer. 4035 */ 4036 static list_t * 4037 l2arc_list_locked(int list_num, kmutex_t **lock) 4038 { 4039 list_t *list; 4040 4041 ASSERT(list_num >= 0 && list_num <= 3); 4042 4043 switch (list_num) { 4044 case 0: 4045 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 4046 *lock = &arc_mfu->arcs_mtx; 4047 break; 4048 case 1: 4049 list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 4050 *lock = &arc_mru->arcs_mtx; 4051 break; 4052 case 2: 4053 list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 4054 *lock = &arc_mfu->arcs_mtx; 4055 break; 4056 case 3: 4057 list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 4058 *lock = &arc_mru->arcs_mtx; 4059 break; 4060 } 4061 4062 ASSERT(!(MUTEX_HELD(*lock))); 4063 mutex_enter(*lock); 4064 return (list); 4065 } 4066 4067 /* 4068 * Evict buffers from the device write hand to the distance specified in 4069 * bytes. This distance may span populated buffers, it may span nothing. 4070 * This is clearing a region on the L2ARC device ready for writing. 4071 * If the 'all' boolean is set, every buffer is evicted. 4072 */ 4073 static void 4074 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 4075 { 4076 list_t *buflist; 4077 l2arc_buf_hdr_t *abl2; 4078 arc_buf_hdr_t *ab, *ab_prev; 4079 kmutex_t *hash_lock; 4080 uint64_t taddr; 4081 4082 buflist = dev->l2ad_buflist; 4083 4084 if (buflist == NULL) 4085 return; 4086 4087 if (!all && dev->l2ad_first) { 4088 /* 4089 * This is the first sweep through the device. There is 4090 * nothing to evict. 4091 */ 4092 return; 4093 } 4094 4095 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 4096 /* 4097 * When nearing the end of the device, evict to the end 4098 * before the device write hand jumps to the start. 4099 */ 4100 taddr = dev->l2ad_end; 4101 } else { 4102 taddr = dev->l2ad_hand + distance; 4103 } 4104 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 4105 uint64_t, taddr, boolean_t, all); 4106 4107 top: 4108 mutex_enter(&l2arc_buflist_mtx); 4109 for (ab = list_tail(buflist); ab; ab = ab_prev) { 4110 ab_prev = list_prev(buflist, ab); 4111 4112 hash_lock = HDR_LOCK(ab); 4113 if (!mutex_tryenter(hash_lock)) { 4114 /* 4115 * Missed the hash lock. Retry. 4116 */ 4117 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 4118 mutex_exit(&l2arc_buflist_mtx); 4119 mutex_enter(hash_lock); 4120 mutex_exit(hash_lock); 4121 goto top; 4122 } 4123 4124 if (HDR_L2_WRITE_HEAD(ab)) { 4125 /* 4126 * We hit a write head node. Leave it for 4127 * l2arc_write_done(). 4128 */ 4129 list_remove(buflist, ab); 4130 mutex_exit(hash_lock); 4131 continue; 4132 } 4133 4134 if (!all && ab->b_l2hdr != NULL && 4135 (ab->b_l2hdr->b_daddr > taddr || 4136 ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 4137 /* 4138 * We've evicted to the target address, 4139 * or the end of the device. 4140 */ 4141 mutex_exit(hash_lock); 4142 break; 4143 } 4144 4145 if (HDR_FREE_IN_PROGRESS(ab)) { 4146 /* 4147 * Already on the path to destruction. 4148 */ 4149 mutex_exit(hash_lock); 4150 continue; 4151 } 4152 4153 if (ab->b_state == arc_l2c_only) { 4154 ASSERT(!HDR_L2_READING(ab)); 4155 /* 4156 * This doesn't exist in the ARC. Destroy. 4157 * arc_hdr_destroy() will call list_remove() 4158 * and decrement arcstat_l2_size. 4159 */ 4160 arc_change_state(arc_anon, ab, hash_lock); 4161 arc_hdr_destroy(ab); 4162 } else { 4163 /* 4164 * Invalidate issued or about to be issued 4165 * reads, since we may be about to write 4166 * over this location. 4167 */ 4168 if (HDR_L2_READING(ab)) { 4169 ARCSTAT_BUMP(arcstat_l2_evict_reading); 4170 ab->b_flags |= ARC_L2_EVICTED; 4171 } 4172 4173 /* 4174 * Tell ARC this no longer exists in L2ARC. 4175 */ 4176 if (ab->b_l2hdr != NULL) { 4177 abl2 = ab->b_l2hdr; 4178 ab->b_l2hdr = NULL; 4179 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4180 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4181 } 4182 list_remove(buflist, ab); 4183 4184 /* 4185 * This may have been leftover after a 4186 * failed write. 4187 */ 4188 ab->b_flags &= ~ARC_L2_WRITING; 4189 } 4190 mutex_exit(hash_lock); 4191 } 4192 mutex_exit(&l2arc_buflist_mtx); 4193 4194 vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0); 4195 dev->l2ad_evict = taddr; 4196 } 4197 4198 /* 4199 * Find and write ARC buffers to the L2ARC device. 4200 * 4201 * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4202 * for reading until they have completed writing. 4203 */ 4204 static uint64_t 4205 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 4206 { 4207 arc_buf_hdr_t *ab, *ab_prev, *head; 4208 l2arc_buf_hdr_t *hdrl2; 4209 list_t *list; 4210 uint64_t passed_sz, write_sz, buf_sz, headroom; 4211 void *buf_data; 4212 kmutex_t *hash_lock, *list_lock; 4213 boolean_t have_lock, full; 4214 l2arc_write_callback_t *cb; 4215 zio_t *pio, *wzio; 4216 uint64_t guid = spa_guid(spa); 4217 4218 ASSERT(dev->l2ad_vdev != NULL); 4219 4220 pio = NULL; 4221 write_sz = 0; 4222 full = B_FALSE; 4223 head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4224 head->b_flags |= ARC_L2_WRITE_HEAD; 4225 4226 /* 4227 * Copy buffers for L2ARC writing. 4228 */ 4229 mutex_enter(&l2arc_buflist_mtx); 4230 for (int try = 0; try <= 3; try++) { 4231 list = l2arc_list_locked(try, &list_lock); 4232 passed_sz = 0; 4233 4234 /* 4235 * L2ARC fast warmup. 4236 * 4237 * Until the ARC is warm and starts to evict, read from the 4238 * head of the ARC lists rather than the tail. 4239 */ 4240 headroom = target_sz * l2arc_headroom; 4241 if (arc_warm == B_FALSE) 4242 ab = list_head(list); 4243 else 4244 ab = list_tail(list); 4245 4246 for (; ab; ab = ab_prev) { 4247 if (arc_warm == B_FALSE) 4248 ab_prev = list_next(list, ab); 4249 else 4250 ab_prev = list_prev(list, ab); 4251 4252 hash_lock = HDR_LOCK(ab); 4253 have_lock = MUTEX_HELD(hash_lock); 4254 if (!have_lock && !mutex_tryenter(hash_lock)) { 4255 /* 4256 * Skip this buffer rather than waiting. 4257 */ 4258 continue; 4259 } 4260 4261 passed_sz += ab->b_size; 4262 if (passed_sz > headroom) { 4263 /* 4264 * Searched too far. 4265 */ 4266 mutex_exit(hash_lock); 4267 break; 4268 } 4269 4270 if (!l2arc_write_eligible(guid, ab)) { 4271 mutex_exit(hash_lock); 4272 continue; 4273 } 4274 4275 if ((write_sz + ab->b_size) > target_sz) { 4276 full = B_TRUE; 4277 mutex_exit(hash_lock); 4278 break; 4279 } 4280 4281 if (pio == NULL) { 4282 /* 4283 * Insert a dummy header on the buflist so 4284 * l2arc_write_done() can find where the 4285 * write buffers begin without searching. 4286 */ 4287 list_insert_head(dev->l2ad_buflist, head); 4288 4289 cb = kmem_alloc( 4290 sizeof (l2arc_write_callback_t), KM_SLEEP); 4291 cb->l2wcb_dev = dev; 4292 cb->l2wcb_head = head; 4293 pio = zio_root(spa, l2arc_write_done, cb, 4294 ZIO_FLAG_CANFAIL); 4295 } 4296 4297 /* 4298 * Create and add a new L2ARC header. 4299 */ 4300 hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4301 hdrl2->b_dev = dev; 4302 hdrl2->b_daddr = dev->l2ad_hand; 4303 4304 ab->b_flags |= ARC_L2_WRITING; 4305 ab->b_l2hdr = hdrl2; 4306 list_insert_head(dev->l2ad_buflist, ab); 4307 buf_data = ab->b_buf->b_data; 4308 buf_sz = ab->b_size; 4309 4310 /* 4311 * Compute and store the buffer cksum before 4312 * writing. On debug the cksum is verified first. 4313 */ 4314 arc_cksum_verify(ab->b_buf); 4315 arc_cksum_compute(ab->b_buf, B_TRUE); 4316 4317 mutex_exit(hash_lock); 4318 4319 wzio = zio_write_phys(pio, dev->l2ad_vdev, 4320 dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4321 NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4322 ZIO_FLAG_CANFAIL, B_FALSE); 4323 4324 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4325 zio_t *, wzio); 4326 (void) zio_nowait(wzio); 4327 4328 /* 4329 * Keep the clock hand suitably device-aligned. 4330 */ 4331 buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz); 4332 4333 write_sz += buf_sz; 4334 dev->l2ad_hand += buf_sz; 4335 } 4336 4337 mutex_exit(list_lock); 4338 4339 if (full == B_TRUE) 4340 break; 4341 } 4342 mutex_exit(&l2arc_buflist_mtx); 4343 4344 if (pio == NULL) { 4345 ASSERT3U(write_sz, ==, 0); 4346 kmem_cache_free(hdr_cache, head); 4347 return (0); 4348 } 4349 4350 ASSERT3U(write_sz, <=, target_sz); 4351 ARCSTAT_BUMP(arcstat_l2_writes_sent); 4352 ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz); 4353 ARCSTAT_INCR(arcstat_l2_size, write_sz); 4354 vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0); 4355 4356 /* 4357 * Bump device hand to the device start if it is approaching the end. 4358 * l2arc_evict() will already have evicted ahead for this case. 4359 */ 4360 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4361 vdev_space_update(dev->l2ad_vdev, 4362 dev->l2ad_end - dev->l2ad_hand, 0, 0); 4363 dev->l2ad_hand = dev->l2ad_start; 4364 dev->l2ad_evict = dev->l2ad_start; 4365 dev->l2ad_first = B_FALSE; 4366 } 4367 4368 dev->l2ad_writing = B_TRUE; 4369 (void) zio_wait(pio); 4370 dev->l2ad_writing = B_FALSE; 4371 4372 return (write_sz); 4373 } 4374 4375 /* 4376 * This thread feeds the L2ARC at regular intervals. This is the beating 4377 * heart of the L2ARC. 4378 */ 4379 static void 4380 l2arc_feed_thread(void) 4381 { 4382 callb_cpr_t cpr; 4383 l2arc_dev_t *dev; 4384 spa_t *spa; 4385 uint64_t size, wrote; 4386 clock_t begin, next = ddi_get_lbolt(); 4387 4388 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4389 4390 mutex_enter(&l2arc_feed_thr_lock); 4391 4392 while (l2arc_thread_exit == 0) { 4393 CALLB_CPR_SAFE_BEGIN(&cpr); 4394 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 4395 next); 4396 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4397 next = ddi_get_lbolt() + hz; 4398 4399 /* 4400 * Quick check for L2ARC devices. 4401 */ 4402 mutex_enter(&l2arc_dev_mtx); 4403 if (l2arc_ndev == 0) { 4404 mutex_exit(&l2arc_dev_mtx); 4405 continue; 4406 } 4407 mutex_exit(&l2arc_dev_mtx); 4408 begin = ddi_get_lbolt(); 4409 4410 /* 4411 * This selects the next l2arc device to write to, and in 4412 * doing so the next spa to feed from: dev->l2ad_spa. This 4413 * will return NULL if there are now no l2arc devices or if 4414 * they are all faulted. 4415 * 4416 * If a device is returned, its spa's config lock is also 4417 * held to prevent device removal. l2arc_dev_get_next() 4418 * will grab and release l2arc_dev_mtx. 4419 */ 4420 if ((dev = l2arc_dev_get_next()) == NULL) 4421 continue; 4422 4423 spa = dev->l2ad_spa; 4424 ASSERT(spa != NULL); 4425 4426 /* 4427 * Avoid contributing to memory pressure. 4428 */ 4429 if (arc_reclaim_needed()) { 4430 ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4431 spa_config_exit(spa, SCL_L2ARC, dev); 4432 continue; 4433 } 4434 4435 ARCSTAT_BUMP(arcstat_l2_feeds); 4436 4437 size = l2arc_write_size(dev); 4438 4439 /* 4440 * Evict L2ARC buffers that will be overwritten. 4441 */ 4442 l2arc_evict(dev, size, B_FALSE); 4443 4444 /* 4445 * Write ARC buffers. 4446 */ 4447 wrote = l2arc_write_buffers(spa, dev, size); 4448 4449 /* 4450 * Calculate interval between writes. 4451 */ 4452 next = l2arc_write_interval(begin, size, wrote); 4453 spa_config_exit(spa, SCL_L2ARC, dev); 4454 } 4455 4456 l2arc_thread_exit = 0; 4457 cv_broadcast(&l2arc_feed_thr_cv); 4458 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4459 thread_exit(); 4460 } 4461 4462 boolean_t 4463 l2arc_vdev_present(vdev_t *vd) 4464 { 4465 l2arc_dev_t *dev; 4466 4467 mutex_enter(&l2arc_dev_mtx); 4468 for (dev = list_head(l2arc_dev_list); dev != NULL; 4469 dev = list_next(l2arc_dev_list, dev)) { 4470 if (dev->l2ad_vdev == vd) 4471 break; 4472 } 4473 mutex_exit(&l2arc_dev_mtx); 4474 4475 return (dev != NULL); 4476 } 4477 4478 /* 4479 * Add a vdev for use by the L2ARC. By this point the spa has already 4480 * validated the vdev and opened it. 4481 */ 4482 void 4483 l2arc_add_vdev(spa_t *spa, vdev_t *vd) 4484 { 4485 l2arc_dev_t *adddev; 4486 4487 ASSERT(!l2arc_vdev_present(vd)); 4488 4489 /* 4490 * Create a new l2arc device entry. 4491 */ 4492 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 4493 adddev->l2ad_spa = spa; 4494 adddev->l2ad_vdev = vd; 4495 adddev->l2ad_write = l2arc_write_max; 4496 adddev->l2ad_boost = l2arc_write_boost; 4497 adddev->l2ad_start = VDEV_LABEL_START_SIZE; 4498 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 4499 adddev->l2ad_hand = adddev->l2ad_start; 4500 adddev->l2ad_evict = adddev->l2ad_start; 4501 adddev->l2ad_first = B_TRUE; 4502 adddev->l2ad_writing = B_FALSE; 4503 ASSERT3U(adddev->l2ad_write, >, 0); 4504 4505 /* 4506 * This is a list of all ARC buffers that are still valid on the 4507 * device. 4508 */ 4509 adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 4510 list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 4511 offsetof(arc_buf_hdr_t, b_l2node)); 4512 4513 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 4514 4515 /* 4516 * Add device to global list 4517 */ 4518 mutex_enter(&l2arc_dev_mtx); 4519 list_insert_head(l2arc_dev_list, adddev); 4520 atomic_inc_64(&l2arc_ndev); 4521 mutex_exit(&l2arc_dev_mtx); 4522 } 4523 4524 /* 4525 * Remove a vdev from the L2ARC. 4526 */ 4527 void 4528 l2arc_remove_vdev(vdev_t *vd) 4529 { 4530 l2arc_dev_t *dev, *nextdev, *remdev = NULL; 4531 4532 /* 4533 * Find the device by vdev 4534 */ 4535 mutex_enter(&l2arc_dev_mtx); 4536 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 4537 nextdev = list_next(l2arc_dev_list, dev); 4538 if (vd == dev->l2ad_vdev) { 4539 remdev = dev; 4540 break; 4541 } 4542 } 4543 ASSERT(remdev != NULL); 4544 4545 /* 4546 * Remove device from global list 4547 */ 4548 list_remove(l2arc_dev_list, remdev); 4549 l2arc_dev_last = NULL; /* may have been invalidated */ 4550 atomic_dec_64(&l2arc_ndev); 4551 mutex_exit(&l2arc_dev_mtx); 4552 4553 /* 4554 * Clear all buflists and ARC references. L2ARC device flush. 4555 */ 4556 l2arc_evict(remdev, 0, B_TRUE); 4557 list_destroy(remdev->l2ad_buflist); 4558 kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 4559 kmem_free(remdev, sizeof (l2arc_dev_t)); 4560 } 4561 4562 void 4563 l2arc_init(void) 4564 { 4565 l2arc_thread_exit = 0; 4566 l2arc_ndev = 0; 4567 l2arc_writes_sent = 0; 4568 l2arc_writes_done = 0; 4569 4570 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 4571 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 4572 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 4573 mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 4574 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 4575 4576 l2arc_dev_list = &L2ARC_dev_list; 4577 l2arc_free_on_write = &L2ARC_free_on_write; 4578 list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 4579 offsetof(l2arc_dev_t, l2ad_node)); 4580 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 4581 offsetof(l2arc_data_free_t, l2df_list_node)); 4582 } 4583 4584 void 4585 l2arc_fini(void) 4586 { 4587 /* 4588 * This is called from dmu_fini(), which is called from spa_fini(); 4589 * Because of this, we can assume that all l2arc devices have 4590 * already been removed when the pools themselves were removed. 4591 */ 4592 4593 l2arc_do_free_on_write(); 4594 4595 mutex_destroy(&l2arc_feed_thr_lock); 4596 cv_destroy(&l2arc_feed_thr_cv); 4597 mutex_destroy(&l2arc_dev_mtx); 4598 mutex_destroy(&l2arc_buflist_mtx); 4599 mutex_destroy(&l2arc_free_on_write_mtx); 4600 4601 list_destroy(l2arc_dev_list); 4602 list_destroy(l2arc_free_on_write); 4603 } 4604 4605 void 4606 l2arc_start(void) 4607 { 4608 if (!(spa_mode_global & FWRITE)) 4609 return; 4610 4611 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 4612 TS_RUN, minclsyspri); 4613 } 4614 4615 void 4616 l2arc_stop(void) 4617 { 4618 if (!(spa_mode_global & FWRITE)) 4619 return; 4620 4621 mutex_enter(&l2arc_feed_thr_lock); 4622 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 4623 l2arc_thread_exit = 1; 4624 while (l2arc_thread_exit != 0) 4625 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 4626 mutex_exit(&l2arc_feed_thr_lock); 4627 } 4628