1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * DVA-based Adjustable Replacement Cache 30 * 31 * While much of the theory of operation used here is 32 * based on the self-tuning, low overhead replacement cache 33 * presented by Megiddo and Modha at FAST 2003, there are some 34 * significant differences: 35 * 36 * 1. The Megiddo and Modha model assumes any page is evictable. 37 * Pages in its cache cannot be "locked" into memory. This makes 38 * the eviction algorithm simple: evict the last page in the list. 39 * This also make the performance characteristics easy to reason 40 * about. Our cache is not so simple. At any given moment, some 41 * subset of the blocks in the cache are un-evictable because we 42 * have handed out a reference to them. Blocks are only evictable 43 * when there are no external references active. This makes 44 * eviction far more problematic: we choose to evict the evictable 45 * blocks that are the "lowest" in the list. 46 * 47 * There are times when it is not possible to evict the requested 48 * space. In these circumstances we are unable to adjust the cache 49 * size. To prevent the cache growing unbounded at these times we 50 * implement a "cache throttle" that slows the flow of new data 51 * into the cache until we can make space available. 52 * 53 * 2. The Megiddo and Modha model assumes a fixed cache size. 54 * Pages are evicted when the cache is full and there is a cache 55 * miss. Our model has a variable sized cache. It grows with 56 * high use, but also tries to react to memory pressure from the 57 * operating system: decreasing its size when system memory is 58 * tight. 59 * 60 * 3. The Megiddo and Modha model assumes a fixed page size. All 61 * elements of the cache are therefor exactly the same size. So 62 * when adjusting the cache size following a cache miss, its simply 63 * a matter of choosing a single page to evict. In our model, we 64 * have variable sized cache blocks (rangeing from 512 bytes to 65 * 128K bytes). We therefor choose a set of blocks to evict to make 66 * space for a cache miss that approximates as closely as possible 67 * the space used by the new block. 68 * 69 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70 * by N. Megiddo & D. Modha, FAST 2003 71 */ 72 73 /* 74 * The locking model: 75 * 76 * A new reference to a cache buffer can be obtained in two 77 * ways: 1) via a hash table lookup using the DVA as a key, 78 * or 2) via one of the ARC lists. The arc_read() interface 79 * uses method 1, while the internal arc algorithms for 80 * adjusting the cache use method 2. We therefor provide two 81 * types of locks: 1) the hash table lock array, and 2) the 82 * arc list locks. 83 * 84 * Buffers do not have their own mutexs, rather they rely on the 85 * hash table mutexs for the bulk of their protection (i.e. most 86 * fields in the arc_buf_hdr_t are protected by these mutexs). 87 * 88 * buf_hash_find() returns the appropriate mutex (held) when it 89 * locates the requested buffer in the hash table. It returns 90 * NULL for the mutex if the buffer was not in the table. 91 * 92 * buf_hash_remove() expects the appropriate hash mutex to be 93 * already held before it is invoked. 94 * 95 * Each arc state also has a mutex which is used to protect the 96 * buffer list associated with the state. When attempting to 97 * obtain a hash table lock while holding an arc list lock you 98 * must use: mutex_tryenter() to avoid deadlock. Also note that 99 * the active state mutex must be held before the ghost state mutex. 100 * 101 * Arc buffers may have an associated eviction callback function. 102 * This function will be invoked prior to removing the buffer (e.g. 103 * in arc_do_user_evicts()). Note however that the data associated 104 * with the buffer may be evicted prior to the callback. The callback 105 * must be made with *no locks held* (to prevent deadlock). Additionally, 106 * the users of callbacks must ensure that their private data is 107 * protected from simultaneous callbacks from arc_buf_evict() 108 * and arc_do_user_evicts(). 109 * 110 * Note that the majority of the performance stats are manipulated 111 * with atomic operations. 112 * 113 * The L2ARC uses the l2arc_buflist_mtx global mutex for the following: 114 * 115 * - L2ARC buflist creation 116 * - L2ARC buflist eviction 117 * - L2ARC write completion, which walks L2ARC buflists 118 * - ARC header destruction, as it removes from L2ARC buflists 119 * - ARC header release, as it removes from L2ARC buflists 120 */ 121 122 #include <sys/spa.h> 123 #include <sys/zio.h> 124 #include <sys/zio_checksum.h> 125 #include <sys/zfs_context.h> 126 #include <sys/arc.h> 127 #include <sys/refcount.h> 128 #include <sys/vdev.h> 129 #ifdef _KERNEL 130 #include <sys/vmsystm.h> 131 #include <vm/anon.h> 132 #include <sys/fs/swapnode.h> 133 #include <sys/dnlc.h> 134 #endif 135 #include <sys/callb.h> 136 #include <sys/kstat.h> 137 138 static kmutex_t arc_reclaim_thr_lock; 139 static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 140 static uint8_t arc_thread_exit; 141 142 extern int zfs_write_limit_shift; 143 extern uint64_t zfs_write_limit_max; 144 extern uint64_t zfs_write_limit_inflated; 145 146 #define ARC_REDUCE_DNLC_PERCENT 3 147 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 148 149 typedef enum arc_reclaim_strategy { 150 ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 151 ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 152 } arc_reclaim_strategy_t; 153 154 /* number of seconds before growing cache again */ 155 static int arc_grow_retry = 60; 156 157 /* 158 * minimum lifespan of a prefetch block in clock ticks 159 * (initialized in arc_init()) 160 */ 161 static int arc_min_prefetch_lifespan; 162 163 static int arc_dead; 164 165 /* 166 * The arc has filled available memory and has now warmed up. 167 */ 168 static boolean_t arc_warm; 169 170 /* 171 * These tunables are for performance analysis. 172 */ 173 uint64_t zfs_arc_max; 174 uint64_t zfs_arc_min; 175 uint64_t zfs_arc_meta_limit = 0; 176 int zfs_mdcomp_disable = 0; 177 178 /* 179 * Note that buffers can be in one of 6 states: 180 * ARC_anon - anonymous (discussed below) 181 * ARC_mru - recently used, currently cached 182 * ARC_mru_ghost - recentely used, no longer in cache 183 * ARC_mfu - frequently used, currently cached 184 * ARC_mfu_ghost - frequently used, no longer in cache 185 * ARC_l2c_only - exists in L2ARC but not other states 186 * When there are no active references to the buffer, they are 187 * are linked onto a list in one of these arc states. These are 188 * the only buffers that can be evicted or deleted. Within each 189 * state there are multiple lists, one for meta-data and one for 190 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 191 * etc.) is tracked separately so that it can be managed more 192 * explicitly: favored over data, limited explicitly. 193 * 194 * Anonymous buffers are buffers that are not associated with 195 * a DVA. These are buffers that hold dirty block copies 196 * before they are written to stable storage. By definition, 197 * they are "ref'd" and are considered part of arc_mru 198 * that cannot be freed. Generally, they will aquire a DVA 199 * as they are written and migrate onto the arc_mru list. 200 * 201 * The ARC_l2c_only state is for buffers that are in the second 202 * level ARC but no longer in any of the ARC_m* lists. The second 203 * level ARC itself may also contain buffers that are in any of 204 * the ARC_m* states - meaning that a buffer can exist in two 205 * places. The reason for the ARC_l2c_only state is to keep the 206 * buffer header in the hash table, so that reads that hit the 207 * second level ARC benefit from these fast lookups. 208 */ 209 210 typedef struct arc_state { 211 list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 212 uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 213 uint64_t arcs_size; /* total amount of data in this state */ 214 kmutex_t arcs_mtx; 215 } arc_state_t; 216 217 /* The 6 states: */ 218 static arc_state_t ARC_anon; 219 static arc_state_t ARC_mru; 220 static arc_state_t ARC_mru_ghost; 221 static arc_state_t ARC_mfu; 222 static arc_state_t ARC_mfu_ghost; 223 static arc_state_t ARC_l2c_only; 224 225 typedef struct arc_stats { 226 kstat_named_t arcstat_hits; 227 kstat_named_t arcstat_misses; 228 kstat_named_t arcstat_demand_data_hits; 229 kstat_named_t arcstat_demand_data_misses; 230 kstat_named_t arcstat_demand_metadata_hits; 231 kstat_named_t arcstat_demand_metadata_misses; 232 kstat_named_t arcstat_prefetch_data_hits; 233 kstat_named_t arcstat_prefetch_data_misses; 234 kstat_named_t arcstat_prefetch_metadata_hits; 235 kstat_named_t arcstat_prefetch_metadata_misses; 236 kstat_named_t arcstat_mru_hits; 237 kstat_named_t arcstat_mru_ghost_hits; 238 kstat_named_t arcstat_mfu_hits; 239 kstat_named_t arcstat_mfu_ghost_hits; 240 kstat_named_t arcstat_deleted; 241 kstat_named_t arcstat_recycle_miss; 242 kstat_named_t arcstat_mutex_miss; 243 kstat_named_t arcstat_evict_skip; 244 kstat_named_t arcstat_hash_elements; 245 kstat_named_t arcstat_hash_elements_max; 246 kstat_named_t arcstat_hash_collisions; 247 kstat_named_t arcstat_hash_chains; 248 kstat_named_t arcstat_hash_chain_max; 249 kstat_named_t arcstat_p; 250 kstat_named_t arcstat_c; 251 kstat_named_t arcstat_c_min; 252 kstat_named_t arcstat_c_max; 253 kstat_named_t arcstat_size; 254 kstat_named_t arcstat_hdr_size; 255 kstat_named_t arcstat_l2_hits; 256 kstat_named_t arcstat_l2_misses; 257 kstat_named_t arcstat_l2_feeds; 258 kstat_named_t arcstat_l2_rw_clash; 259 kstat_named_t arcstat_l2_writes_sent; 260 kstat_named_t arcstat_l2_writes_done; 261 kstat_named_t arcstat_l2_writes_error; 262 kstat_named_t arcstat_l2_writes_hdr_miss; 263 kstat_named_t arcstat_l2_evict_lock_retry; 264 kstat_named_t arcstat_l2_evict_reading; 265 kstat_named_t arcstat_l2_free_on_write; 266 kstat_named_t arcstat_l2_abort_lowmem; 267 kstat_named_t arcstat_l2_cksum_bad; 268 kstat_named_t arcstat_l2_io_error; 269 kstat_named_t arcstat_l2_size; 270 kstat_named_t arcstat_l2_hdr_size; 271 kstat_named_t arcstat_memory_throttle_count; 272 } arc_stats_t; 273 274 static arc_stats_t arc_stats = { 275 { "hits", KSTAT_DATA_UINT64 }, 276 { "misses", KSTAT_DATA_UINT64 }, 277 { "demand_data_hits", KSTAT_DATA_UINT64 }, 278 { "demand_data_misses", KSTAT_DATA_UINT64 }, 279 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 280 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 281 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 282 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 283 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 284 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 285 { "mru_hits", KSTAT_DATA_UINT64 }, 286 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 287 { "mfu_hits", KSTAT_DATA_UINT64 }, 288 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 289 { "deleted", KSTAT_DATA_UINT64 }, 290 { "recycle_miss", KSTAT_DATA_UINT64 }, 291 { "mutex_miss", KSTAT_DATA_UINT64 }, 292 { "evict_skip", KSTAT_DATA_UINT64 }, 293 { "hash_elements", KSTAT_DATA_UINT64 }, 294 { "hash_elements_max", KSTAT_DATA_UINT64 }, 295 { "hash_collisions", KSTAT_DATA_UINT64 }, 296 { "hash_chains", KSTAT_DATA_UINT64 }, 297 { "hash_chain_max", KSTAT_DATA_UINT64 }, 298 { "p", KSTAT_DATA_UINT64 }, 299 { "c", KSTAT_DATA_UINT64 }, 300 { "c_min", KSTAT_DATA_UINT64 }, 301 { "c_max", KSTAT_DATA_UINT64 }, 302 { "size", KSTAT_DATA_UINT64 }, 303 { "hdr_size", KSTAT_DATA_UINT64 }, 304 { "l2_hits", KSTAT_DATA_UINT64 }, 305 { "l2_misses", KSTAT_DATA_UINT64 }, 306 { "l2_feeds", KSTAT_DATA_UINT64 }, 307 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 308 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 309 { "l2_writes_done", KSTAT_DATA_UINT64 }, 310 { "l2_writes_error", KSTAT_DATA_UINT64 }, 311 { "l2_writes_hdr_miss", KSTAT_DATA_UINT64 }, 312 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 313 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 314 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 315 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 316 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 317 { "l2_io_error", KSTAT_DATA_UINT64 }, 318 { "l2_size", KSTAT_DATA_UINT64 }, 319 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 320 { "memory_throttle_count", KSTAT_DATA_UINT64 } 321 }; 322 323 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 324 325 #define ARCSTAT_INCR(stat, val) \ 326 atomic_add_64(&arc_stats.stat.value.ui64, (val)); 327 328 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 329 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 330 331 #define ARCSTAT_MAX(stat, val) { \ 332 uint64_t m; \ 333 while ((val) > (m = arc_stats.stat.value.ui64) && \ 334 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 335 continue; \ 336 } 337 338 #define ARCSTAT_MAXSTAT(stat) \ 339 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 340 341 /* 342 * We define a macro to allow ARC hits/misses to be easily broken down by 343 * two separate conditions, giving a total of four different subtypes for 344 * each of hits and misses (so eight statistics total). 345 */ 346 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 347 if (cond1) { \ 348 if (cond2) { \ 349 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 350 } else { \ 351 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 352 } \ 353 } else { \ 354 if (cond2) { \ 355 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 356 } else { \ 357 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 358 } \ 359 } 360 361 kstat_t *arc_ksp; 362 static arc_state_t *arc_anon; 363 static arc_state_t *arc_mru; 364 static arc_state_t *arc_mru_ghost; 365 static arc_state_t *arc_mfu; 366 static arc_state_t *arc_mfu_ghost; 367 static arc_state_t *arc_l2c_only; 368 369 /* 370 * There are several ARC variables that are critical to export as kstats -- 371 * but we don't want to have to grovel around in the kstat whenever we wish to 372 * manipulate them. For these variables, we therefore define them to be in 373 * terms of the statistic variable. This assures that we are not introducing 374 * the possibility of inconsistency by having shadow copies of the variables, 375 * while still allowing the code to be readable. 376 */ 377 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 378 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 379 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 380 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 381 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 382 383 static int arc_no_grow; /* Don't try to grow cache size */ 384 static uint64_t arc_tempreserve; 385 static uint64_t arc_meta_used; 386 static uint64_t arc_meta_limit; 387 static uint64_t arc_meta_max = 0; 388 389 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; 390 391 typedef struct arc_callback arc_callback_t; 392 393 struct arc_callback { 394 void *acb_private; 395 arc_done_func_t *acb_done; 396 arc_buf_t *acb_buf; 397 zio_t *acb_zio_dummy; 398 arc_callback_t *acb_next; 399 }; 400 401 typedef struct arc_write_callback arc_write_callback_t; 402 403 struct arc_write_callback { 404 void *awcb_private; 405 arc_done_func_t *awcb_ready; 406 arc_done_func_t *awcb_done; 407 arc_buf_t *awcb_buf; 408 }; 409 410 struct arc_buf_hdr { 411 /* protected by hash lock */ 412 dva_t b_dva; 413 uint64_t b_birth; 414 uint64_t b_cksum0; 415 416 kmutex_t b_freeze_lock; 417 zio_cksum_t *b_freeze_cksum; 418 419 arc_buf_hdr_t *b_hash_next; 420 arc_buf_t *b_buf; 421 uint32_t b_flags; 422 uint32_t b_datacnt; 423 424 arc_callback_t *b_acb; 425 kcondvar_t b_cv; 426 427 /* immutable */ 428 arc_buf_contents_t b_type; 429 uint64_t b_size; 430 spa_t *b_spa; 431 432 /* protected by arc state mutex */ 433 arc_state_t *b_state; 434 list_node_t b_arc_node; 435 436 /* updated atomically */ 437 clock_t b_arc_access; 438 439 /* self protecting */ 440 refcount_t b_refcnt; 441 442 l2arc_buf_hdr_t *b_l2hdr; 443 list_node_t b_l2node; 444 /* 445 * scrub code can lockout access to the buf while it changes 446 * bp's contained within it. 447 */ 448 krwlock_t b_datalock; 449 }; 450 451 static arc_buf_t *arc_eviction_list; 452 static kmutex_t arc_eviction_mtx; 453 static arc_buf_hdr_t arc_eviction_hdr; 454 static void arc_get_data_buf(arc_buf_t *buf); 455 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 456 static int arc_evict_needed(arc_buf_contents_t type); 457 static void arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes); 458 459 #define GHOST_STATE(state) \ 460 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 461 (state) == arc_l2c_only) 462 463 /* 464 * Private ARC flags. These flags are private ARC only flags that will show up 465 * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 466 * be passed in as arc_flags in things like arc_read. However, these flags 467 * should never be passed and should only be set by ARC code. When adding new 468 * public flags, make sure not to smash the private ones. 469 */ 470 471 #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 472 #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 473 #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 474 #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 475 #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 476 #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 477 #define ARC_FREE_IN_PROGRESS (1 << 15) /* hdr about to be freed */ 478 #define ARC_L2_WRITING (1 << 16) /* L2ARC write in progress */ 479 #define ARC_L2_EVICTED (1 << 17) /* evicted during I/O */ 480 #define ARC_L2_WRITE_HEAD (1 << 18) /* head of write list */ 481 #define ARC_STORED (1 << 19) /* has been store()d to */ 482 483 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 484 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 485 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 486 #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 487 #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 488 #define HDR_FREE_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FREE_IN_PROGRESS) 489 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_L2CACHE) 490 #define HDR_L2_READING(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS && \ 491 (hdr)->b_l2hdr != NULL) 492 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_L2_WRITING) 493 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_L2_EVICTED) 494 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_L2_WRITE_HEAD) 495 496 /* 497 * Other sizes 498 */ 499 500 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 501 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t)) 502 503 /* 504 * Hash table routines 505 */ 506 507 #define HT_LOCK_PAD 64 508 509 struct ht_lock { 510 kmutex_t ht_lock; 511 #ifdef _KERNEL 512 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 513 #endif 514 }; 515 516 #define BUF_LOCKS 256 517 typedef struct buf_hash_table { 518 uint64_t ht_mask; 519 arc_buf_hdr_t **ht_table; 520 struct ht_lock ht_locks[BUF_LOCKS]; 521 } buf_hash_table_t; 522 523 static buf_hash_table_t buf_hash_table; 524 525 #define BUF_HASH_INDEX(spa, dva, birth) \ 526 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 527 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 528 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 529 #define HDR_LOCK(buf) \ 530 (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 531 532 uint64_t zfs_crc64_table[256]; 533 534 /* 535 * Level 2 ARC 536 */ 537 538 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 539 #define L2ARC_HEADROOM 4 /* num of writes */ 540 #define L2ARC_FEED_SECS 1 /* caching interval */ 541 542 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 543 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 544 545 /* 546 * L2ARC Performance Tunables 547 */ 548 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 549 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 550 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 551 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 552 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 553 554 /* 555 * L2ARC Internals 556 */ 557 typedef struct l2arc_dev { 558 vdev_t *l2ad_vdev; /* vdev */ 559 spa_t *l2ad_spa; /* spa */ 560 uint64_t l2ad_hand; /* next write location */ 561 uint64_t l2ad_write; /* desired write size, bytes */ 562 uint64_t l2ad_boost; /* warmup write boost, bytes */ 563 uint64_t l2ad_start; /* first addr on device */ 564 uint64_t l2ad_end; /* last addr on device */ 565 uint64_t l2ad_evict; /* last addr eviction reached */ 566 boolean_t l2ad_first; /* first sweep through */ 567 list_t *l2ad_buflist; /* buffer list */ 568 list_node_t l2ad_node; /* device list node */ 569 } l2arc_dev_t; 570 571 static list_t L2ARC_dev_list; /* device list */ 572 static list_t *l2arc_dev_list; /* device list pointer */ 573 static kmutex_t l2arc_dev_mtx; /* device list mutex */ 574 static l2arc_dev_t *l2arc_dev_last; /* last device used */ 575 static kmutex_t l2arc_buflist_mtx; /* mutex for all buflists */ 576 static list_t L2ARC_free_on_write; /* free after write buf list */ 577 static list_t *l2arc_free_on_write; /* free after write list ptr */ 578 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 579 static uint64_t l2arc_ndev; /* number of devices */ 580 581 typedef struct l2arc_read_callback { 582 arc_buf_t *l2rcb_buf; /* read buffer */ 583 spa_t *l2rcb_spa; /* spa */ 584 blkptr_t l2rcb_bp; /* original blkptr */ 585 zbookmark_t l2rcb_zb; /* original bookmark */ 586 int l2rcb_flags; /* original flags */ 587 } l2arc_read_callback_t; 588 589 typedef struct l2arc_write_callback { 590 l2arc_dev_t *l2wcb_dev; /* device info */ 591 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 592 } l2arc_write_callback_t; 593 594 struct l2arc_buf_hdr { 595 /* protected by arc_buf_hdr mutex */ 596 l2arc_dev_t *b_dev; /* L2ARC device */ 597 daddr_t b_daddr; /* disk address, offset byte */ 598 }; 599 600 typedef struct l2arc_data_free { 601 /* protected by l2arc_free_on_write_mtx */ 602 void *l2df_data; 603 size_t l2df_size; 604 void (*l2df_func)(void *, size_t); 605 list_node_t l2df_list_node; 606 } l2arc_data_free_t; 607 608 static kmutex_t l2arc_feed_thr_lock; 609 static kcondvar_t l2arc_feed_thr_cv; 610 static uint8_t l2arc_thread_exit; 611 612 static void l2arc_read_done(zio_t *zio); 613 static void l2arc_hdr_stat_add(void); 614 static void l2arc_hdr_stat_remove(void); 615 616 static uint64_t 617 buf_hash(spa_t *spa, const dva_t *dva, uint64_t birth) 618 { 619 uintptr_t spav = (uintptr_t)spa; 620 uint8_t *vdva = (uint8_t *)dva; 621 uint64_t crc = -1ULL; 622 int i; 623 624 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 625 626 for (i = 0; i < sizeof (dva_t); i++) 627 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 628 629 crc ^= (spav>>8) ^ birth; 630 631 return (crc); 632 } 633 634 #define BUF_EMPTY(buf) \ 635 ((buf)->b_dva.dva_word[0] == 0 && \ 636 (buf)->b_dva.dva_word[1] == 0 && \ 637 (buf)->b_birth == 0) 638 639 #define BUF_EQUAL(spa, dva, birth, buf) \ 640 ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 641 ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 642 ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 643 644 static arc_buf_hdr_t * 645 buf_hash_find(spa_t *spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) 646 { 647 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 648 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 649 arc_buf_hdr_t *buf; 650 651 mutex_enter(hash_lock); 652 for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 653 buf = buf->b_hash_next) { 654 if (BUF_EQUAL(spa, dva, birth, buf)) { 655 *lockp = hash_lock; 656 return (buf); 657 } 658 } 659 mutex_exit(hash_lock); 660 *lockp = NULL; 661 return (NULL); 662 } 663 664 /* 665 * Insert an entry into the hash table. If there is already an element 666 * equal to elem in the hash table, then the already existing element 667 * will be returned and the new element will not be inserted. 668 * Otherwise returns NULL. 669 */ 670 static arc_buf_hdr_t * 671 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 672 { 673 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 674 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 675 arc_buf_hdr_t *fbuf; 676 uint32_t i; 677 678 ASSERT(!HDR_IN_HASH_TABLE(buf)); 679 *lockp = hash_lock; 680 mutex_enter(hash_lock); 681 for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 682 fbuf = fbuf->b_hash_next, i++) { 683 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 684 return (fbuf); 685 } 686 687 buf->b_hash_next = buf_hash_table.ht_table[idx]; 688 buf_hash_table.ht_table[idx] = buf; 689 buf->b_flags |= ARC_IN_HASH_TABLE; 690 691 /* collect some hash table performance data */ 692 if (i > 0) { 693 ARCSTAT_BUMP(arcstat_hash_collisions); 694 if (i == 1) 695 ARCSTAT_BUMP(arcstat_hash_chains); 696 697 ARCSTAT_MAX(arcstat_hash_chain_max, i); 698 } 699 700 ARCSTAT_BUMP(arcstat_hash_elements); 701 ARCSTAT_MAXSTAT(arcstat_hash_elements); 702 703 return (NULL); 704 } 705 706 static void 707 buf_hash_remove(arc_buf_hdr_t *buf) 708 { 709 arc_buf_hdr_t *fbuf, **bufp; 710 uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 711 712 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 713 ASSERT(HDR_IN_HASH_TABLE(buf)); 714 715 bufp = &buf_hash_table.ht_table[idx]; 716 while ((fbuf = *bufp) != buf) { 717 ASSERT(fbuf != NULL); 718 bufp = &fbuf->b_hash_next; 719 } 720 *bufp = buf->b_hash_next; 721 buf->b_hash_next = NULL; 722 buf->b_flags &= ~ARC_IN_HASH_TABLE; 723 724 /* collect some hash table performance data */ 725 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 726 727 if (buf_hash_table.ht_table[idx] && 728 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 729 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 730 } 731 732 /* 733 * Global data structures and functions for the buf kmem cache. 734 */ 735 static kmem_cache_t *hdr_cache; 736 static kmem_cache_t *buf_cache; 737 738 static void 739 buf_fini(void) 740 { 741 int i; 742 743 kmem_free(buf_hash_table.ht_table, 744 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 745 for (i = 0; i < BUF_LOCKS; i++) 746 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 747 kmem_cache_destroy(hdr_cache); 748 kmem_cache_destroy(buf_cache); 749 } 750 751 /* 752 * Constructor callback - called when the cache is empty 753 * and a new buf is requested. 754 */ 755 /* ARGSUSED */ 756 static int 757 hdr_cons(void *vbuf, void *unused, int kmflag) 758 { 759 arc_buf_hdr_t *buf = vbuf; 760 761 bzero(buf, sizeof (arc_buf_hdr_t)); 762 refcount_create(&buf->b_refcnt); 763 cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 764 mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 765 rw_init(&buf->b_datalock, NULL, RW_DEFAULT, NULL); 766 767 ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 768 return (0); 769 } 770 771 /* 772 * Destructor callback - called when a cached buf is 773 * no longer required. 774 */ 775 /* ARGSUSED */ 776 static void 777 hdr_dest(void *vbuf, void *unused) 778 { 779 arc_buf_hdr_t *buf = vbuf; 780 781 refcount_destroy(&buf->b_refcnt); 782 cv_destroy(&buf->b_cv); 783 mutex_destroy(&buf->b_freeze_lock); 784 rw_destroy(&buf->b_datalock); 785 786 ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 787 } 788 789 /* 790 * Reclaim callback -- invoked when memory is low. 791 */ 792 /* ARGSUSED */ 793 static void 794 hdr_recl(void *unused) 795 { 796 dprintf("hdr_recl called\n"); 797 /* 798 * umem calls the reclaim func when we destroy the buf cache, 799 * which is after we do arc_fini(). 800 */ 801 if (!arc_dead) 802 cv_signal(&arc_reclaim_thr_cv); 803 } 804 805 static void 806 buf_init(void) 807 { 808 uint64_t *ct; 809 uint64_t hsize = 1ULL << 12; 810 int i, j; 811 812 /* 813 * The hash table is big enough to fill all of physical memory 814 * with an average 64K block size. The table will take up 815 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 816 */ 817 while (hsize * 65536 < physmem * PAGESIZE) 818 hsize <<= 1; 819 retry: 820 buf_hash_table.ht_mask = hsize - 1; 821 buf_hash_table.ht_table = 822 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 823 if (buf_hash_table.ht_table == NULL) { 824 ASSERT(hsize > (1ULL << 8)); 825 hsize >>= 1; 826 goto retry; 827 } 828 829 hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 830 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 831 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 832 0, NULL, NULL, NULL, NULL, NULL, 0); 833 834 for (i = 0; i < 256; i++) 835 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 836 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 837 838 for (i = 0; i < BUF_LOCKS; i++) { 839 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 840 NULL, MUTEX_DEFAULT, NULL); 841 } 842 } 843 844 #define ARC_MINTIME (hz>>4) /* 62 ms */ 845 846 static void 847 arc_cksum_verify(arc_buf_t *buf) 848 { 849 zio_cksum_t zc; 850 851 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 852 return; 853 854 mutex_enter(&buf->b_hdr->b_freeze_lock); 855 if (buf->b_hdr->b_freeze_cksum == NULL || 856 (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 857 mutex_exit(&buf->b_hdr->b_freeze_lock); 858 return; 859 } 860 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 861 if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 862 panic("buffer modified while frozen!"); 863 mutex_exit(&buf->b_hdr->b_freeze_lock); 864 } 865 866 static int 867 arc_cksum_equal(arc_buf_t *buf) 868 { 869 zio_cksum_t zc; 870 int equal; 871 872 mutex_enter(&buf->b_hdr->b_freeze_lock); 873 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 874 equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc); 875 mutex_exit(&buf->b_hdr->b_freeze_lock); 876 877 return (equal); 878 } 879 880 static void 881 arc_cksum_compute(arc_buf_t *buf, boolean_t force) 882 { 883 if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY)) 884 return; 885 886 mutex_enter(&buf->b_hdr->b_freeze_lock); 887 if (buf->b_hdr->b_freeze_cksum != NULL) { 888 mutex_exit(&buf->b_hdr->b_freeze_lock); 889 return; 890 } 891 buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 892 fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 893 buf->b_hdr->b_freeze_cksum); 894 mutex_exit(&buf->b_hdr->b_freeze_lock); 895 } 896 897 void 898 arc_buf_thaw(arc_buf_t *buf) 899 { 900 if (zfs_flags & ZFS_DEBUG_MODIFY) { 901 if (buf->b_hdr->b_state != arc_anon) 902 panic("modifying non-anon buffer!"); 903 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 904 panic("modifying buffer while i/o in progress!"); 905 arc_cksum_verify(buf); 906 } 907 908 mutex_enter(&buf->b_hdr->b_freeze_lock); 909 if (buf->b_hdr->b_freeze_cksum != NULL) { 910 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 911 buf->b_hdr->b_freeze_cksum = NULL; 912 } 913 mutex_exit(&buf->b_hdr->b_freeze_lock); 914 } 915 916 void 917 arc_buf_freeze(arc_buf_t *buf) 918 { 919 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 920 return; 921 922 ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 923 buf->b_hdr->b_state == arc_anon); 924 arc_cksum_compute(buf, B_FALSE); 925 } 926 927 static void 928 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 929 { 930 ASSERT(MUTEX_HELD(hash_lock)); 931 932 if ((refcount_add(&ab->b_refcnt, tag) == 1) && 933 (ab->b_state != arc_anon)) { 934 uint64_t delta = ab->b_size * ab->b_datacnt; 935 list_t *list = &ab->b_state->arcs_list[ab->b_type]; 936 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 937 938 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 939 mutex_enter(&ab->b_state->arcs_mtx); 940 ASSERT(list_link_active(&ab->b_arc_node)); 941 list_remove(list, ab); 942 if (GHOST_STATE(ab->b_state)) { 943 ASSERT3U(ab->b_datacnt, ==, 0); 944 ASSERT3P(ab->b_buf, ==, NULL); 945 delta = ab->b_size; 946 } 947 ASSERT(delta > 0); 948 ASSERT3U(*size, >=, delta); 949 atomic_add_64(size, -delta); 950 mutex_exit(&ab->b_state->arcs_mtx); 951 /* remove the prefetch flag if we get a reference */ 952 if (ab->b_flags & ARC_PREFETCH) 953 ab->b_flags &= ~ARC_PREFETCH; 954 } 955 } 956 957 static int 958 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 959 { 960 int cnt; 961 arc_state_t *state = ab->b_state; 962 963 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 964 ASSERT(!GHOST_STATE(state)); 965 966 if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 967 (state != arc_anon)) { 968 uint64_t *size = &state->arcs_lsize[ab->b_type]; 969 970 ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 971 mutex_enter(&state->arcs_mtx); 972 ASSERT(!list_link_active(&ab->b_arc_node)); 973 list_insert_head(&state->arcs_list[ab->b_type], ab); 974 ASSERT(ab->b_datacnt > 0); 975 atomic_add_64(size, ab->b_size * ab->b_datacnt); 976 mutex_exit(&state->arcs_mtx); 977 } 978 return (cnt); 979 } 980 981 /* 982 * Move the supplied buffer to the indicated state. The mutex 983 * for the buffer must be held by the caller. 984 */ 985 static void 986 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 987 { 988 arc_state_t *old_state = ab->b_state; 989 int64_t refcnt = refcount_count(&ab->b_refcnt); 990 uint64_t from_delta, to_delta; 991 992 ASSERT(MUTEX_HELD(hash_lock)); 993 ASSERT(new_state != old_state); 994 ASSERT(refcnt == 0 || ab->b_datacnt > 0); 995 ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 996 997 from_delta = to_delta = ab->b_datacnt * ab->b_size; 998 999 /* 1000 * If this buffer is evictable, transfer it from the 1001 * old state list to the new state list. 1002 */ 1003 if (refcnt == 0) { 1004 if (old_state != arc_anon) { 1005 int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 1006 uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 1007 1008 if (use_mutex) 1009 mutex_enter(&old_state->arcs_mtx); 1010 1011 ASSERT(list_link_active(&ab->b_arc_node)); 1012 list_remove(&old_state->arcs_list[ab->b_type], ab); 1013 1014 /* 1015 * If prefetching out of the ghost cache, 1016 * we will have a non-null datacnt. 1017 */ 1018 if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 1019 /* ghost elements have a ghost size */ 1020 ASSERT(ab->b_buf == NULL); 1021 from_delta = ab->b_size; 1022 } 1023 ASSERT3U(*size, >=, from_delta); 1024 atomic_add_64(size, -from_delta); 1025 1026 if (use_mutex) 1027 mutex_exit(&old_state->arcs_mtx); 1028 } 1029 if (new_state != arc_anon) { 1030 int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 1031 uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 1032 1033 if (use_mutex) 1034 mutex_enter(&new_state->arcs_mtx); 1035 1036 list_insert_head(&new_state->arcs_list[ab->b_type], ab); 1037 1038 /* ghost elements have a ghost size */ 1039 if (GHOST_STATE(new_state)) { 1040 ASSERT(ab->b_datacnt == 0); 1041 ASSERT(ab->b_buf == NULL); 1042 to_delta = ab->b_size; 1043 } 1044 atomic_add_64(size, to_delta); 1045 1046 if (use_mutex) 1047 mutex_exit(&new_state->arcs_mtx); 1048 } 1049 } 1050 1051 ASSERT(!BUF_EMPTY(ab)); 1052 if (new_state == arc_anon) { 1053 buf_hash_remove(ab); 1054 } 1055 1056 /* adjust state sizes */ 1057 if (to_delta) 1058 atomic_add_64(&new_state->arcs_size, to_delta); 1059 if (from_delta) { 1060 ASSERT3U(old_state->arcs_size, >=, from_delta); 1061 atomic_add_64(&old_state->arcs_size, -from_delta); 1062 } 1063 ab->b_state = new_state; 1064 1065 /* adjust l2arc hdr stats */ 1066 if (new_state == arc_l2c_only) 1067 l2arc_hdr_stat_add(); 1068 else if (old_state == arc_l2c_only) 1069 l2arc_hdr_stat_remove(); 1070 } 1071 1072 void 1073 arc_space_consume(uint64_t space) 1074 { 1075 atomic_add_64(&arc_meta_used, space); 1076 atomic_add_64(&arc_size, space); 1077 } 1078 1079 void 1080 arc_space_return(uint64_t space) 1081 { 1082 ASSERT(arc_meta_used >= space); 1083 if (arc_meta_max < arc_meta_used) 1084 arc_meta_max = arc_meta_used; 1085 atomic_add_64(&arc_meta_used, -space); 1086 ASSERT(arc_size >= space); 1087 atomic_add_64(&arc_size, -space); 1088 } 1089 1090 void * 1091 arc_data_buf_alloc(uint64_t size) 1092 { 1093 if (arc_evict_needed(ARC_BUFC_DATA)) 1094 cv_signal(&arc_reclaim_thr_cv); 1095 atomic_add_64(&arc_size, size); 1096 return (zio_data_buf_alloc(size)); 1097 } 1098 1099 void 1100 arc_data_buf_free(void *buf, uint64_t size) 1101 { 1102 zio_data_buf_free(buf, size); 1103 ASSERT(arc_size >= size); 1104 atomic_add_64(&arc_size, -size); 1105 } 1106 1107 arc_buf_t * 1108 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 1109 { 1110 arc_buf_hdr_t *hdr; 1111 arc_buf_t *buf; 1112 1113 ASSERT3U(size, >, 0); 1114 hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 1115 ASSERT(BUF_EMPTY(hdr)); 1116 hdr->b_size = size; 1117 hdr->b_type = type; 1118 hdr->b_spa = spa; 1119 hdr->b_state = arc_anon; 1120 hdr->b_arc_access = 0; 1121 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1122 buf->b_hdr = hdr; 1123 buf->b_data = NULL; 1124 buf->b_efunc = NULL; 1125 buf->b_private = NULL; 1126 buf->b_next = NULL; 1127 hdr->b_buf = buf; 1128 arc_get_data_buf(buf); 1129 hdr->b_datacnt = 1; 1130 hdr->b_flags = 0; 1131 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1132 (void) refcount_add(&hdr->b_refcnt, tag); 1133 1134 return (buf); 1135 } 1136 1137 static arc_buf_t * 1138 arc_buf_clone(arc_buf_t *from) 1139 { 1140 arc_buf_t *buf; 1141 arc_buf_hdr_t *hdr = from->b_hdr; 1142 uint64_t size = hdr->b_size; 1143 1144 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 1145 buf->b_hdr = hdr; 1146 buf->b_data = NULL; 1147 buf->b_efunc = NULL; 1148 buf->b_private = NULL; 1149 buf->b_next = hdr->b_buf; 1150 hdr->b_buf = buf; 1151 arc_get_data_buf(buf); 1152 bcopy(from->b_data, buf->b_data, size); 1153 hdr->b_datacnt += 1; 1154 return (buf); 1155 } 1156 1157 void 1158 arc_buf_add_ref(arc_buf_t *buf, void* tag) 1159 { 1160 arc_buf_hdr_t *hdr; 1161 kmutex_t *hash_lock; 1162 1163 /* 1164 * Check to see if this buffer is currently being evicted via 1165 * arc_do_user_evicts(). 1166 */ 1167 mutex_enter(&arc_eviction_mtx); 1168 hdr = buf->b_hdr; 1169 if (hdr == NULL) { 1170 mutex_exit(&arc_eviction_mtx); 1171 return; 1172 } 1173 hash_lock = HDR_LOCK(hdr); 1174 mutex_exit(&arc_eviction_mtx); 1175 1176 mutex_enter(hash_lock); 1177 if (buf->b_data == NULL) { 1178 /* 1179 * This buffer is evicted. 1180 */ 1181 mutex_exit(hash_lock); 1182 return; 1183 } 1184 1185 ASSERT(buf->b_hdr == hdr); 1186 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 1187 add_reference(hdr, hash_lock, tag); 1188 arc_access(hdr, hash_lock); 1189 mutex_exit(hash_lock); 1190 ARCSTAT_BUMP(arcstat_hits); 1191 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 1192 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 1193 data, metadata, hits); 1194 } 1195 1196 /* 1197 * Free the arc data buffer. If it is an l2arc write in progress, 1198 * the buffer is placed on l2arc_free_on_write to be freed later. 1199 */ 1200 static void 1201 arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t), 1202 void *data, size_t size) 1203 { 1204 if (HDR_L2_WRITING(hdr)) { 1205 l2arc_data_free_t *df; 1206 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP); 1207 df->l2df_data = data; 1208 df->l2df_size = size; 1209 df->l2df_func = free_func; 1210 mutex_enter(&l2arc_free_on_write_mtx); 1211 list_insert_head(l2arc_free_on_write, df); 1212 mutex_exit(&l2arc_free_on_write_mtx); 1213 ARCSTAT_BUMP(arcstat_l2_free_on_write); 1214 } else { 1215 free_func(data, size); 1216 } 1217 } 1218 1219 static void 1220 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 1221 { 1222 arc_buf_t **bufp; 1223 1224 /* free up data associated with the buf */ 1225 if (buf->b_data) { 1226 arc_state_t *state = buf->b_hdr->b_state; 1227 uint64_t size = buf->b_hdr->b_size; 1228 arc_buf_contents_t type = buf->b_hdr->b_type; 1229 1230 arc_cksum_verify(buf); 1231 if (!recycle) { 1232 if (type == ARC_BUFC_METADATA) { 1233 arc_buf_data_free(buf->b_hdr, zio_buf_free, 1234 buf->b_data, size); 1235 arc_space_return(size); 1236 } else { 1237 ASSERT(type == ARC_BUFC_DATA); 1238 arc_buf_data_free(buf->b_hdr, 1239 zio_data_buf_free, buf->b_data, size); 1240 atomic_add_64(&arc_size, -size); 1241 } 1242 } 1243 if (list_link_active(&buf->b_hdr->b_arc_node)) { 1244 uint64_t *cnt = &state->arcs_lsize[type]; 1245 1246 ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 1247 ASSERT(state != arc_anon); 1248 1249 ASSERT3U(*cnt, >=, size); 1250 atomic_add_64(cnt, -size); 1251 } 1252 ASSERT3U(state->arcs_size, >=, size); 1253 atomic_add_64(&state->arcs_size, -size); 1254 buf->b_data = NULL; 1255 ASSERT(buf->b_hdr->b_datacnt > 0); 1256 buf->b_hdr->b_datacnt -= 1; 1257 } 1258 1259 /* only remove the buf if requested */ 1260 if (!all) 1261 return; 1262 1263 /* remove the buf from the hdr list */ 1264 for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 1265 continue; 1266 *bufp = buf->b_next; 1267 1268 ASSERT(buf->b_efunc == NULL); 1269 1270 /* clean up the buf */ 1271 buf->b_hdr = NULL; 1272 kmem_cache_free(buf_cache, buf); 1273 } 1274 1275 static void 1276 arc_hdr_destroy(arc_buf_hdr_t *hdr) 1277 { 1278 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1279 ASSERT3P(hdr->b_state, ==, arc_anon); 1280 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1281 ASSERT(!(hdr->b_flags & ARC_STORED)); 1282 1283 if (hdr->b_l2hdr != NULL) { 1284 if (!MUTEX_HELD(&l2arc_buflist_mtx)) { 1285 /* 1286 * To prevent arc_free() and l2arc_evict() from 1287 * attempting to free the same buffer at the same time, 1288 * a FREE_IN_PROGRESS flag is given to arc_free() to 1289 * give it priority. l2arc_evict() can't destroy this 1290 * header while we are waiting on l2arc_buflist_mtx. 1291 */ 1292 mutex_enter(&l2arc_buflist_mtx); 1293 ASSERT(hdr->b_l2hdr != NULL); 1294 1295 list_remove(hdr->b_l2hdr->b_dev->l2ad_buflist, hdr); 1296 mutex_exit(&l2arc_buflist_mtx); 1297 } else { 1298 list_remove(hdr->b_l2hdr->b_dev->l2ad_buflist, hdr); 1299 } 1300 ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size); 1301 kmem_free(hdr->b_l2hdr, sizeof (l2arc_buf_hdr_t)); 1302 if (hdr->b_state == arc_l2c_only) 1303 l2arc_hdr_stat_remove(); 1304 hdr->b_l2hdr = NULL; 1305 } 1306 1307 if (!BUF_EMPTY(hdr)) { 1308 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1309 bzero(&hdr->b_dva, sizeof (dva_t)); 1310 hdr->b_birth = 0; 1311 hdr->b_cksum0 = 0; 1312 } 1313 while (hdr->b_buf) { 1314 arc_buf_t *buf = hdr->b_buf; 1315 1316 if (buf->b_efunc) { 1317 mutex_enter(&arc_eviction_mtx); 1318 ASSERT(buf->b_hdr != NULL); 1319 arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 1320 hdr->b_buf = buf->b_next; 1321 buf->b_hdr = &arc_eviction_hdr; 1322 buf->b_next = arc_eviction_list; 1323 arc_eviction_list = buf; 1324 mutex_exit(&arc_eviction_mtx); 1325 } else { 1326 arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 1327 } 1328 } 1329 if (hdr->b_freeze_cksum != NULL) { 1330 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 1331 hdr->b_freeze_cksum = NULL; 1332 } 1333 1334 ASSERT(!list_link_active(&hdr->b_arc_node)); 1335 ASSERT3P(hdr->b_hash_next, ==, NULL); 1336 ASSERT3P(hdr->b_acb, ==, NULL); 1337 kmem_cache_free(hdr_cache, hdr); 1338 } 1339 1340 void 1341 arc_buf_free(arc_buf_t *buf, void *tag) 1342 { 1343 arc_buf_hdr_t *hdr = buf->b_hdr; 1344 int hashed = hdr->b_state != arc_anon; 1345 1346 ASSERT(buf->b_efunc == NULL); 1347 ASSERT(buf->b_data != NULL); 1348 1349 if (hashed) { 1350 kmutex_t *hash_lock = HDR_LOCK(hdr); 1351 1352 mutex_enter(hash_lock); 1353 (void) remove_reference(hdr, hash_lock, tag); 1354 if (hdr->b_datacnt > 1) 1355 arc_buf_destroy(buf, FALSE, TRUE); 1356 else 1357 hdr->b_flags |= ARC_BUF_AVAILABLE; 1358 mutex_exit(hash_lock); 1359 } else if (HDR_IO_IN_PROGRESS(hdr)) { 1360 int destroy_hdr; 1361 /* 1362 * We are in the middle of an async write. Don't destroy 1363 * this buffer unless the write completes before we finish 1364 * decrementing the reference count. 1365 */ 1366 mutex_enter(&arc_eviction_mtx); 1367 (void) remove_reference(hdr, NULL, tag); 1368 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 1369 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 1370 mutex_exit(&arc_eviction_mtx); 1371 if (destroy_hdr) 1372 arc_hdr_destroy(hdr); 1373 } else { 1374 if (remove_reference(hdr, NULL, tag) > 0) { 1375 ASSERT(HDR_IO_ERROR(hdr)); 1376 arc_buf_destroy(buf, FALSE, TRUE); 1377 } else { 1378 arc_hdr_destroy(hdr); 1379 } 1380 } 1381 } 1382 1383 int 1384 arc_buf_remove_ref(arc_buf_t *buf, void* tag) 1385 { 1386 arc_buf_hdr_t *hdr = buf->b_hdr; 1387 kmutex_t *hash_lock = HDR_LOCK(hdr); 1388 int no_callback = (buf->b_efunc == NULL); 1389 1390 if (hdr->b_state == arc_anon) { 1391 arc_buf_free(buf, tag); 1392 return (no_callback); 1393 } 1394 1395 mutex_enter(hash_lock); 1396 ASSERT(hdr->b_state != arc_anon); 1397 ASSERT(buf->b_data != NULL); 1398 1399 (void) remove_reference(hdr, hash_lock, tag); 1400 if (hdr->b_datacnt > 1) { 1401 if (no_callback) 1402 arc_buf_destroy(buf, FALSE, TRUE); 1403 } else if (no_callback) { 1404 ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 1405 hdr->b_flags |= ARC_BUF_AVAILABLE; 1406 } 1407 ASSERT(no_callback || hdr->b_datacnt > 1 || 1408 refcount_is_zero(&hdr->b_refcnt)); 1409 mutex_exit(hash_lock); 1410 return (no_callback); 1411 } 1412 1413 int 1414 arc_buf_size(arc_buf_t *buf) 1415 { 1416 return (buf->b_hdr->b_size); 1417 } 1418 1419 /* 1420 * Evict buffers from list until we've removed the specified number of 1421 * bytes. Move the removed buffers to the appropriate evict state. 1422 * If the recycle flag is set, then attempt to "recycle" a buffer: 1423 * - look for a buffer to evict that is `bytes' long. 1424 * - return the data block from this buffer rather than freeing it. 1425 * This flag is used by callers that are trying to make space for a 1426 * new buffer in a full arc cache. 1427 * 1428 * This function makes a "best effort". It skips over any buffers 1429 * it can't get a hash_lock on, and so may not catch all candidates. 1430 * It may also return without evicting as much space as requested. 1431 */ 1432 static void * 1433 arc_evict(arc_state_t *state, spa_t *spa, int64_t bytes, boolean_t recycle, 1434 arc_buf_contents_t type) 1435 { 1436 arc_state_t *evicted_state; 1437 uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 1438 arc_buf_hdr_t *ab, *ab_prev = NULL; 1439 list_t *list = &state->arcs_list[type]; 1440 kmutex_t *hash_lock; 1441 boolean_t have_lock; 1442 void *stolen = NULL; 1443 1444 ASSERT(state == arc_mru || state == arc_mfu); 1445 1446 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1447 1448 mutex_enter(&state->arcs_mtx); 1449 mutex_enter(&evicted_state->arcs_mtx); 1450 1451 for (ab = list_tail(list); ab; ab = ab_prev) { 1452 ab_prev = list_prev(list, ab); 1453 /* prefetch buffers have a minimum lifespan */ 1454 if (HDR_IO_IN_PROGRESS(ab) || 1455 (spa && ab->b_spa != spa) || 1456 (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 1457 lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 1458 skipped++; 1459 continue; 1460 } 1461 /* "lookahead" for better eviction candidate */ 1462 if (recycle && ab->b_size != bytes && 1463 ab_prev && ab_prev->b_size == bytes) 1464 continue; 1465 hash_lock = HDR_LOCK(ab); 1466 have_lock = MUTEX_HELD(hash_lock); 1467 if (have_lock || mutex_tryenter(hash_lock)) { 1468 ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 1469 ASSERT(ab->b_datacnt > 0); 1470 while (ab->b_buf) { 1471 arc_buf_t *buf = ab->b_buf; 1472 if (buf->b_data) { 1473 bytes_evicted += ab->b_size; 1474 if (recycle && ab->b_type == type && 1475 ab->b_size == bytes && 1476 !HDR_L2_WRITING(ab)) { 1477 stolen = buf->b_data; 1478 recycle = FALSE; 1479 } 1480 } 1481 if (buf->b_efunc) { 1482 mutex_enter(&arc_eviction_mtx); 1483 arc_buf_destroy(buf, 1484 buf->b_data == stolen, FALSE); 1485 ab->b_buf = buf->b_next; 1486 buf->b_hdr = &arc_eviction_hdr; 1487 buf->b_next = arc_eviction_list; 1488 arc_eviction_list = buf; 1489 mutex_exit(&arc_eviction_mtx); 1490 } else { 1491 arc_buf_destroy(buf, 1492 buf->b_data == stolen, TRUE); 1493 } 1494 } 1495 ASSERT(ab->b_datacnt == 0); 1496 arc_change_state(evicted_state, ab, hash_lock); 1497 ASSERT(HDR_IN_HASH_TABLE(ab)); 1498 ab->b_flags |= ARC_IN_HASH_TABLE; 1499 ab->b_flags &= ~ARC_BUF_AVAILABLE; 1500 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 1501 if (!have_lock) 1502 mutex_exit(hash_lock); 1503 if (bytes >= 0 && bytes_evicted >= bytes) 1504 break; 1505 } else { 1506 missed += 1; 1507 } 1508 } 1509 1510 mutex_exit(&evicted_state->arcs_mtx); 1511 mutex_exit(&state->arcs_mtx); 1512 1513 if (bytes_evicted < bytes) 1514 dprintf("only evicted %lld bytes from %x", 1515 (longlong_t)bytes_evicted, state); 1516 1517 if (skipped) 1518 ARCSTAT_INCR(arcstat_evict_skip, skipped); 1519 1520 if (missed) 1521 ARCSTAT_INCR(arcstat_mutex_miss, missed); 1522 1523 /* 1524 * We have just evicted some date into the ghost state, make 1525 * sure we also adjust the ghost state size if necessary. 1526 */ 1527 if (arc_no_grow && 1528 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1529 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1530 arc_mru_ghost->arcs_size - arc_c; 1531 1532 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1533 int64_t todelete = 1534 MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1535 arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1536 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1537 int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1538 arc_mru_ghost->arcs_size + 1539 arc_mfu_ghost->arcs_size - arc_c); 1540 arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1541 } 1542 } 1543 1544 return (stolen); 1545 } 1546 1547 /* 1548 * Remove buffers from list until we've removed the specified number of 1549 * bytes. Destroy the buffers that are removed. 1550 */ 1551 static void 1552 arc_evict_ghost(arc_state_t *state, spa_t *spa, int64_t bytes) 1553 { 1554 arc_buf_hdr_t *ab, *ab_prev; 1555 list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1556 kmutex_t *hash_lock; 1557 uint64_t bytes_deleted = 0; 1558 uint64_t bufs_skipped = 0; 1559 1560 ASSERT(GHOST_STATE(state)); 1561 top: 1562 mutex_enter(&state->arcs_mtx); 1563 for (ab = list_tail(list); ab; ab = ab_prev) { 1564 ab_prev = list_prev(list, ab); 1565 if (spa && ab->b_spa != spa) 1566 continue; 1567 hash_lock = HDR_LOCK(ab); 1568 if (mutex_tryenter(hash_lock)) { 1569 ASSERT(!HDR_IO_IN_PROGRESS(ab)); 1570 ASSERT(ab->b_buf == NULL); 1571 ARCSTAT_BUMP(arcstat_deleted); 1572 bytes_deleted += ab->b_size; 1573 1574 if (ab->b_l2hdr != NULL) { 1575 /* 1576 * This buffer is cached on the 2nd Level ARC; 1577 * don't destroy the header. 1578 */ 1579 arc_change_state(arc_l2c_only, ab, hash_lock); 1580 mutex_exit(hash_lock); 1581 } else { 1582 arc_change_state(arc_anon, ab, hash_lock); 1583 mutex_exit(hash_lock); 1584 arc_hdr_destroy(ab); 1585 } 1586 1587 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1588 if (bytes >= 0 && bytes_deleted >= bytes) 1589 break; 1590 } else { 1591 if (bytes < 0) { 1592 mutex_exit(&state->arcs_mtx); 1593 mutex_enter(hash_lock); 1594 mutex_exit(hash_lock); 1595 goto top; 1596 } 1597 bufs_skipped += 1; 1598 } 1599 } 1600 mutex_exit(&state->arcs_mtx); 1601 1602 if (list == &state->arcs_list[ARC_BUFC_DATA] && 1603 (bytes < 0 || bytes_deleted < bytes)) { 1604 list = &state->arcs_list[ARC_BUFC_METADATA]; 1605 goto top; 1606 } 1607 1608 if (bufs_skipped) { 1609 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1610 ASSERT(bytes >= 0); 1611 } 1612 1613 if (bytes_deleted < bytes) 1614 dprintf("only deleted %lld bytes from %p", 1615 (longlong_t)bytes_deleted, state); 1616 } 1617 1618 static void 1619 arc_adjust(void) 1620 { 1621 int64_t top_sz, mru_over, arc_over, todelete; 1622 1623 top_sz = arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used; 1624 1625 if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 1626 int64_t toevict = 1627 MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], top_sz - arc_p); 1628 (void) arc_evict(arc_mru, NULL, toevict, FALSE, ARC_BUFC_DATA); 1629 top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1630 } 1631 1632 if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1633 int64_t toevict = 1634 MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], top_sz - arc_p); 1635 (void) arc_evict(arc_mru, NULL, toevict, FALSE, 1636 ARC_BUFC_METADATA); 1637 top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1638 } 1639 1640 mru_over = top_sz + arc_mru_ghost->arcs_size - arc_c; 1641 1642 if (mru_over > 0) { 1643 if (arc_mru_ghost->arcs_size > 0) { 1644 todelete = MIN(arc_mru_ghost->arcs_size, mru_over); 1645 arc_evict_ghost(arc_mru_ghost, NULL, todelete); 1646 } 1647 } 1648 1649 if ((arc_over = arc_size - arc_c) > 0) { 1650 int64_t tbl_over; 1651 1652 if (arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 1653 int64_t toevict = 1654 MIN(arc_mfu->arcs_lsize[ARC_BUFC_DATA], arc_over); 1655 (void) arc_evict(arc_mfu, NULL, toevict, FALSE, 1656 ARC_BUFC_DATA); 1657 arc_over = arc_size - arc_c; 1658 } 1659 1660 if (arc_over > 0 && 1661 arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 1662 int64_t toevict = 1663 MIN(arc_mfu->arcs_lsize[ARC_BUFC_METADATA], 1664 arc_over); 1665 (void) arc_evict(arc_mfu, NULL, toevict, FALSE, 1666 ARC_BUFC_METADATA); 1667 } 1668 1669 tbl_over = arc_size + arc_mru_ghost->arcs_size + 1670 arc_mfu_ghost->arcs_size - arc_c * 2; 1671 1672 if (tbl_over > 0 && arc_mfu_ghost->arcs_size > 0) { 1673 todelete = MIN(arc_mfu_ghost->arcs_size, tbl_over); 1674 arc_evict_ghost(arc_mfu_ghost, NULL, todelete); 1675 } 1676 } 1677 } 1678 1679 static void 1680 arc_do_user_evicts(void) 1681 { 1682 mutex_enter(&arc_eviction_mtx); 1683 while (arc_eviction_list != NULL) { 1684 arc_buf_t *buf = arc_eviction_list; 1685 arc_eviction_list = buf->b_next; 1686 buf->b_hdr = NULL; 1687 mutex_exit(&arc_eviction_mtx); 1688 1689 if (buf->b_efunc != NULL) 1690 VERIFY(buf->b_efunc(buf) == 0); 1691 1692 buf->b_efunc = NULL; 1693 buf->b_private = NULL; 1694 kmem_cache_free(buf_cache, buf); 1695 mutex_enter(&arc_eviction_mtx); 1696 } 1697 mutex_exit(&arc_eviction_mtx); 1698 } 1699 1700 /* 1701 * Flush all *evictable* data from the cache for the given spa. 1702 * NOTE: this will not touch "active" (i.e. referenced) data. 1703 */ 1704 void 1705 arc_flush(spa_t *spa) 1706 { 1707 while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) { 1708 (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_DATA); 1709 if (spa) 1710 break; 1711 } 1712 while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) { 1713 (void) arc_evict(arc_mru, spa, -1, FALSE, ARC_BUFC_METADATA); 1714 if (spa) 1715 break; 1716 } 1717 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) { 1718 (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_DATA); 1719 if (spa) 1720 break; 1721 } 1722 while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) { 1723 (void) arc_evict(arc_mfu, spa, -1, FALSE, ARC_BUFC_METADATA); 1724 if (spa) 1725 break; 1726 } 1727 1728 arc_evict_ghost(arc_mru_ghost, spa, -1); 1729 arc_evict_ghost(arc_mfu_ghost, spa, -1); 1730 1731 mutex_enter(&arc_reclaim_thr_lock); 1732 arc_do_user_evicts(); 1733 mutex_exit(&arc_reclaim_thr_lock); 1734 ASSERT(spa || arc_eviction_list == NULL); 1735 } 1736 1737 int arc_shrink_shift = 5; /* log2(fraction of arc to reclaim) */ 1738 1739 void 1740 arc_shrink(void) 1741 { 1742 if (arc_c > arc_c_min) { 1743 uint64_t to_free; 1744 1745 #ifdef _KERNEL 1746 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 1747 #else 1748 to_free = arc_c >> arc_shrink_shift; 1749 #endif 1750 if (arc_c > arc_c_min + to_free) 1751 atomic_add_64(&arc_c, -to_free); 1752 else 1753 arc_c = arc_c_min; 1754 1755 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 1756 if (arc_c > arc_size) 1757 arc_c = MAX(arc_size, arc_c_min); 1758 if (arc_p > arc_c) 1759 arc_p = (arc_c >> 1); 1760 ASSERT(arc_c >= arc_c_min); 1761 ASSERT((int64_t)arc_p >= 0); 1762 } 1763 1764 if (arc_size > arc_c) 1765 arc_adjust(); 1766 } 1767 1768 static int 1769 arc_reclaim_needed(void) 1770 { 1771 uint64_t extra; 1772 1773 #ifdef _KERNEL 1774 1775 if (needfree) 1776 return (1); 1777 1778 /* 1779 * take 'desfree' extra pages, so we reclaim sooner, rather than later 1780 */ 1781 extra = desfree; 1782 1783 /* 1784 * check that we're out of range of the pageout scanner. It starts to 1785 * schedule paging if freemem is less than lotsfree and needfree. 1786 * lotsfree is the high-water mark for pageout, and needfree is the 1787 * number of needed free pages. We add extra pages here to make sure 1788 * the scanner doesn't start up while we're freeing memory. 1789 */ 1790 if (freemem < lotsfree + needfree + extra) 1791 return (1); 1792 1793 /* 1794 * check to make sure that swapfs has enough space so that anon 1795 * reservations can still succeed. anon_resvmem() checks that the 1796 * availrmem is greater than swapfs_minfree, and the number of reserved 1797 * swap pages. We also add a bit of extra here just to prevent 1798 * circumstances from getting really dire. 1799 */ 1800 if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1801 return (1); 1802 1803 #if defined(__i386) 1804 /* 1805 * If we're on an i386 platform, it's possible that we'll exhaust the 1806 * kernel heap space before we ever run out of available physical 1807 * memory. Most checks of the size of the heap_area compare against 1808 * tune.t_minarmem, which is the minimum available real memory that we 1809 * can have in the system. However, this is generally fixed at 25 pages 1810 * which is so low that it's useless. In this comparison, we seek to 1811 * calculate the total heap-size, and reclaim if more than 3/4ths of the 1812 * heap is allocated. (Or, in the calculation, if less than 1/4th is 1813 * free) 1814 */ 1815 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1816 (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1817 return (1); 1818 #endif 1819 1820 #else 1821 if (spa_get_random(100) == 0) 1822 return (1); 1823 #endif 1824 return (0); 1825 } 1826 1827 static void 1828 arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1829 { 1830 size_t i; 1831 kmem_cache_t *prev_cache = NULL; 1832 kmem_cache_t *prev_data_cache = NULL; 1833 extern kmem_cache_t *zio_buf_cache[]; 1834 extern kmem_cache_t *zio_data_buf_cache[]; 1835 1836 #ifdef _KERNEL 1837 if (arc_meta_used >= arc_meta_limit) { 1838 /* 1839 * We are exceeding our meta-data cache limit. 1840 * Purge some DNLC entries to release holds on meta-data. 1841 */ 1842 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 1843 } 1844 #if defined(__i386) 1845 /* 1846 * Reclaim unused memory from all kmem caches. 1847 */ 1848 kmem_reap(); 1849 #endif 1850 #endif 1851 1852 /* 1853 * An aggressive reclamation will shrink the cache size as well as 1854 * reap free buffers from the arc kmem caches. 1855 */ 1856 if (strat == ARC_RECLAIM_AGGR) 1857 arc_shrink(); 1858 1859 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1860 if (zio_buf_cache[i] != prev_cache) { 1861 prev_cache = zio_buf_cache[i]; 1862 kmem_cache_reap_now(zio_buf_cache[i]); 1863 } 1864 if (zio_data_buf_cache[i] != prev_data_cache) { 1865 prev_data_cache = zio_data_buf_cache[i]; 1866 kmem_cache_reap_now(zio_data_buf_cache[i]); 1867 } 1868 } 1869 kmem_cache_reap_now(buf_cache); 1870 kmem_cache_reap_now(hdr_cache); 1871 } 1872 1873 static void 1874 arc_reclaim_thread(void) 1875 { 1876 clock_t growtime = 0; 1877 arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1878 callb_cpr_t cpr; 1879 1880 CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1881 1882 mutex_enter(&arc_reclaim_thr_lock); 1883 while (arc_thread_exit == 0) { 1884 if (arc_reclaim_needed()) { 1885 1886 if (arc_no_grow) { 1887 if (last_reclaim == ARC_RECLAIM_CONS) { 1888 last_reclaim = ARC_RECLAIM_AGGR; 1889 } else { 1890 last_reclaim = ARC_RECLAIM_CONS; 1891 } 1892 } else { 1893 arc_no_grow = TRUE; 1894 last_reclaim = ARC_RECLAIM_AGGR; 1895 membar_producer(); 1896 } 1897 1898 /* reset the growth delay for every reclaim */ 1899 growtime = lbolt + (arc_grow_retry * hz); 1900 1901 arc_kmem_reap_now(last_reclaim); 1902 arc_warm = B_TRUE; 1903 1904 } else if (arc_no_grow && lbolt >= growtime) { 1905 arc_no_grow = FALSE; 1906 } 1907 1908 if (2 * arc_c < arc_size + 1909 arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 1910 arc_adjust(); 1911 1912 if (arc_eviction_list != NULL) 1913 arc_do_user_evicts(); 1914 1915 /* block until needed, or one second, whichever is shorter */ 1916 CALLB_CPR_SAFE_BEGIN(&cpr); 1917 (void) cv_timedwait(&arc_reclaim_thr_cv, 1918 &arc_reclaim_thr_lock, (lbolt + hz)); 1919 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1920 } 1921 1922 arc_thread_exit = 0; 1923 cv_broadcast(&arc_reclaim_thr_cv); 1924 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1925 thread_exit(); 1926 } 1927 1928 /* 1929 * Adapt arc info given the number of bytes we are trying to add and 1930 * the state that we are comming from. This function is only called 1931 * when we are adding new content to the cache. 1932 */ 1933 static void 1934 arc_adapt(int bytes, arc_state_t *state) 1935 { 1936 int mult; 1937 1938 if (state == arc_l2c_only) 1939 return; 1940 1941 ASSERT(bytes > 0); 1942 /* 1943 * Adapt the target size of the MRU list: 1944 * - if we just hit in the MRU ghost list, then increase 1945 * the target size of the MRU list. 1946 * - if we just hit in the MFU ghost list, then increase 1947 * the target size of the MFU list by decreasing the 1948 * target size of the MRU list. 1949 */ 1950 if (state == arc_mru_ghost) { 1951 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 1952 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 1953 1954 arc_p = MIN(arc_c, arc_p + bytes * mult); 1955 } else if (state == arc_mfu_ghost) { 1956 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 1957 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 1958 1959 arc_p = MAX(0, (int64_t)arc_p - bytes * mult); 1960 } 1961 ASSERT((int64_t)arc_p >= 0); 1962 1963 if (arc_reclaim_needed()) { 1964 cv_signal(&arc_reclaim_thr_cv); 1965 return; 1966 } 1967 1968 if (arc_no_grow) 1969 return; 1970 1971 if (arc_c >= arc_c_max) 1972 return; 1973 1974 /* 1975 * If we're within (2 * maxblocksize) bytes of the target 1976 * cache size, increment the target cache size 1977 */ 1978 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 1979 atomic_add_64(&arc_c, (int64_t)bytes); 1980 if (arc_c > arc_c_max) 1981 arc_c = arc_c_max; 1982 else if (state == arc_anon) 1983 atomic_add_64(&arc_p, (int64_t)bytes); 1984 if (arc_p > arc_c) 1985 arc_p = arc_c; 1986 } 1987 ASSERT((int64_t)arc_p >= 0); 1988 } 1989 1990 /* 1991 * Check if the cache has reached its limits and eviction is required 1992 * prior to insert. 1993 */ 1994 static int 1995 arc_evict_needed(arc_buf_contents_t type) 1996 { 1997 if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 1998 return (1); 1999 2000 #ifdef _KERNEL 2001 /* 2002 * If zio data pages are being allocated out of a separate heap segment, 2003 * then enforce that the size of available vmem for this area remains 2004 * above about 1/32nd free. 2005 */ 2006 if (type == ARC_BUFC_DATA && zio_arena != NULL && 2007 vmem_size(zio_arena, VMEM_FREE) < 2008 (vmem_size(zio_arena, VMEM_ALLOC) >> 5)) 2009 return (1); 2010 #endif 2011 2012 if (arc_reclaim_needed()) 2013 return (1); 2014 2015 return (arc_size > arc_c); 2016 } 2017 2018 /* 2019 * The buffer, supplied as the first argument, needs a data block. 2020 * So, if we are at cache max, determine which cache should be victimized. 2021 * We have the following cases: 2022 * 2023 * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 2024 * In this situation if we're out of space, but the resident size of the MFU is 2025 * under the limit, victimize the MFU cache to satisfy this insertion request. 2026 * 2027 * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 2028 * Here, we've used up all of the available space for the MRU, so we need to 2029 * evict from our own cache instead. Evict from the set of resident MRU 2030 * entries. 2031 * 2032 * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 2033 * c minus p represents the MFU space in the cache, since p is the size of the 2034 * cache that is dedicated to the MRU. In this situation there's still space on 2035 * the MFU side, so the MRU side needs to be victimized. 2036 * 2037 * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 2038 * MFU's resident set is consuming more space than it has been allotted. In 2039 * this situation, we must victimize our own cache, the MFU, for this insertion. 2040 */ 2041 static void 2042 arc_get_data_buf(arc_buf_t *buf) 2043 { 2044 arc_state_t *state = buf->b_hdr->b_state; 2045 uint64_t size = buf->b_hdr->b_size; 2046 arc_buf_contents_t type = buf->b_hdr->b_type; 2047 2048 arc_adapt(size, state); 2049 2050 /* 2051 * We have not yet reached cache maximum size, 2052 * just allocate a new buffer. 2053 */ 2054 if (!arc_evict_needed(type)) { 2055 if (type == ARC_BUFC_METADATA) { 2056 buf->b_data = zio_buf_alloc(size); 2057 arc_space_consume(size); 2058 } else { 2059 ASSERT(type == ARC_BUFC_DATA); 2060 buf->b_data = zio_data_buf_alloc(size); 2061 atomic_add_64(&arc_size, size); 2062 } 2063 goto out; 2064 } 2065 2066 /* 2067 * If we are prefetching from the mfu ghost list, this buffer 2068 * will end up on the mru list; so steal space from there. 2069 */ 2070 if (state == arc_mfu_ghost) 2071 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 2072 else if (state == arc_mru_ghost) 2073 state = arc_mru; 2074 2075 if (state == arc_mru || state == arc_anon) { 2076 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 2077 state = (arc_mfu->arcs_lsize[type] > 0 && 2078 arc_p > mru_used) ? arc_mfu : arc_mru; 2079 } else { 2080 /* MFU cases */ 2081 uint64_t mfu_space = arc_c - arc_p; 2082 state = (arc_mru->arcs_lsize[type] > 0 && 2083 mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 2084 } 2085 if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) { 2086 if (type == ARC_BUFC_METADATA) { 2087 buf->b_data = zio_buf_alloc(size); 2088 arc_space_consume(size); 2089 } else { 2090 ASSERT(type == ARC_BUFC_DATA); 2091 buf->b_data = zio_data_buf_alloc(size); 2092 atomic_add_64(&arc_size, size); 2093 } 2094 ARCSTAT_BUMP(arcstat_recycle_miss); 2095 } 2096 ASSERT(buf->b_data != NULL); 2097 out: 2098 /* 2099 * Update the state size. Note that ghost states have a 2100 * "ghost size" and so don't need to be updated. 2101 */ 2102 if (!GHOST_STATE(buf->b_hdr->b_state)) { 2103 arc_buf_hdr_t *hdr = buf->b_hdr; 2104 2105 atomic_add_64(&hdr->b_state->arcs_size, size); 2106 if (list_link_active(&hdr->b_arc_node)) { 2107 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2108 atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 2109 } 2110 /* 2111 * If we are growing the cache, and we are adding anonymous 2112 * data, and we have outgrown arc_p, update arc_p 2113 */ 2114 if (arc_size < arc_c && hdr->b_state == arc_anon && 2115 arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 2116 arc_p = MIN(arc_c, arc_p + size); 2117 } 2118 } 2119 2120 /* 2121 * This routine is called whenever a buffer is accessed. 2122 * NOTE: the hash lock is dropped in this function. 2123 */ 2124 static void 2125 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 2126 { 2127 ASSERT(MUTEX_HELD(hash_lock)); 2128 2129 if (buf->b_state == arc_anon) { 2130 /* 2131 * This buffer is not in the cache, and does not 2132 * appear in our "ghost" list. Add the new buffer 2133 * to the MRU state. 2134 */ 2135 2136 ASSERT(buf->b_arc_access == 0); 2137 buf->b_arc_access = lbolt; 2138 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2139 arc_change_state(arc_mru, buf, hash_lock); 2140 2141 } else if (buf->b_state == arc_mru) { 2142 /* 2143 * If this buffer is here because of a prefetch, then either: 2144 * - clear the flag if this is a "referencing" read 2145 * (any subsequent access will bump this into the MFU state). 2146 * or 2147 * - move the buffer to the head of the list if this is 2148 * another prefetch (to make it less likely to be evicted). 2149 */ 2150 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2151 if (refcount_count(&buf->b_refcnt) == 0) { 2152 ASSERT(list_link_active(&buf->b_arc_node)); 2153 } else { 2154 buf->b_flags &= ~ARC_PREFETCH; 2155 ARCSTAT_BUMP(arcstat_mru_hits); 2156 } 2157 buf->b_arc_access = lbolt; 2158 return; 2159 } 2160 2161 /* 2162 * This buffer has been "accessed" only once so far, 2163 * but it is still in the cache. Move it to the MFU 2164 * state. 2165 */ 2166 if (lbolt > buf->b_arc_access + ARC_MINTIME) { 2167 /* 2168 * More than 125ms have passed since we 2169 * instantiated this buffer. Move it to the 2170 * most frequently used state. 2171 */ 2172 buf->b_arc_access = lbolt; 2173 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2174 arc_change_state(arc_mfu, buf, hash_lock); 2175 } 2176 ARCSTAT_BUMP(arcstat_mru_hits); 2177 } else if (buf->b_state == arc_mru_ghost) { 2178 arc_state_t *new_state; 2179 /* 2180 * This buffer has been "accessed" recently, but 2181 * was evicted from the cache. Move it to the 2182 * MFU state. 2183 */ 2184 2185 if (buf->b_flags & ARC_PREFETCH) { 2186 new_state = arc_mru; 2187 if (refcount_count(&buf->b_refcnt) > 0) 2188 buf->b_flags &= ~ARC_PREFETCH; 2189 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 2190 } else { 2191 new_state = arc_mfu; 2192 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2193 } 2194 2195 buf->b_arc_access = lbolt; 2196 arc_change_state(new_state, buf, hash_lock); 2197 2198 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 2199 } else if (buf->b_state == arc_mfu) { 2200 /* 2201 * This buffer has been accessed more than once and is 2202 * still in the cache. Keep it in the MFU state. 2203 * 2204 * NOTE: an add_reference() that occurred when we did 2205 * the arc_read() will have kicked this off the list. 2206 * If it was a prefetch, we will explicitly move it to 2207 * the head of the list now. 2208 */ 2209 if ((buf->b_flags & ARC_PREFETCH) != 0) { 2210 ASSERT(refcount_count(&buf->b_refcnt) == 0); 2211 ASSERT(list_link_active(&buf->b_arc_node)); 2212 } 2213 ARCSTAT_BUMP(arcstat_mfu_hits); 2214 buf->b_arc_access = lbolt; 2215 } else if (buf->b_state == arc_mfu_ghost) { 2216 arc_state_t *new_state = arc_mfu; 2217 /* 2218 * This buffer has been accessed more than once but has 2219 * been evicted from the cache. Move it back to the 2220 * MFU state. 2221 */ 2222 2223 if (buf->b_flags & ARC_PREFETCH) { 2224 /* 2225 * This is a prefetch access... 2226 * move this block back to the MRU state. 2227 */ 2228 ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 2229 new_state = arc_mru; 2230 } 2231 2232 buf->b_arc_access = lbolt; 2233 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2234 arc_change_state(new_state, buf, hash_lock); 2235 2236 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 2237 } else if (buf->b_state == arc_l2c_only) { 2238 /* 2239 * This buffer is on the 2nd Level ARC. 2240 */ 2241 2242 buf->b_arc_access = lbolt; 2243 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 2244 arc_change_state(arc_mfu, buf, hash_lock); 2245 } else { 2246 ASSERT(!"invalid arc state"); 2247 } 2248 } 2249 2250 /* a generic arc_done_func_t which you can use */ 2251 /* ARGSUSED */ 2252 void 2253 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 2254 { 2255 bcopy(buf->b_data, arg, buf->b_hdr->b_size); 2256 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2257 } 2258 2259 /* a generic arc_done_func_t */ 2260 void 2261 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 2262 { 2263 arc_buf_t **bufp = arg; 2264 if (zio && zio->io_error) { 2265 VERIFY(arc_buf_remove_ref(buf, arg) == 1); 2266 *bufp = NULL; 2267 } else { 2268 *bufp = buf; 2269 } 2270 } 2271 2272 static void 2273 arc_read_done(zio_t *zio) 2274 { 2275 arc_buf_hdr_t *hdr, *found; 2276 arc_buf_t *buf; 2277 arc_buf_t *abuf; /* buffer we're assigning to callback */ 2278 kmutex_t *hash_lock; 2279 arc_callback_t *callback_list, *acb; 2280 int freeable = FALSE; 2281 2282 buf = zio->io_private; 2283 hdr = buf->b_hdr; 2284 2285 /* 2286 * The hdr was inserted into hash-table and removed from lists 2287 * prior to starting I/O. We should find this header, since 2288 * it's in the hash table, and it should be legit since it's 2289 * not possible to evict it during the I/O. The only possible 2290 * reason for it not to be found is if we were freed during the 2291 * read. 2292 */ 2293 found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 2294 &hash_lock); 2295 2296 ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 2297 (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 2298 (found == hdr && HDR_L2_READING(hdr))); 2299 2300 hdr->b_flags &= ~ARC_L2_EVICTED; 2301 if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) 2302 hdr->b_flags &= ~ARC_L2CACHE; 2303 2304 /* byteswap if necessary */ 2305 callback_list = hdr->b_acb; 2306 ASSERT(callback_list != NULL); 2307 if (BP_SHOULD_BYTESWAP(zio->io_bp)) { 2308 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ? 2309 byteswap_uint64_array : 2310 dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap; 2311 func(buf->b_data, hdr->b_size); 2312 } 2313 2314 arc_cksum_compute(buf, B_FALSE); 2315 2316 /* create copies of the data buffer for the callers */ 2317 abuf = buf; 2318 for (acb = callback_list; acb; acb = acb->acb_next) { 2319 if (acb->acb_done) { 2320 if (abuf == NULL) 2321 abuf = arc_buf_clone(buf); 2322 acb->acb_buf = abuf; 2323 abuf = NULL; 2324 } 2325 } 2326 hdr->b_acb = NULL; 2327 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2328 ASSERT(!HDR_BUF_AVAILABLE(hdr)); 2329 if (abuf == buf) 2330 hdr->b_flags |= ARC_BUF_AVAILABLE; 2331 2332 ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2333 2334 if (zio->io_error != 0) { 2335 hdr->b_flags |= ARC_IO_ERROR; 2336 if (hdr->b_state != arc_anon) 2337 arc_change_state(arc_anon, hdr, hash_lock); 2338 if (HDR_IN_HASH_TABLE(hdr)) 2339 buf_hash_remove(hdr); 2340 freeable = refcount_is_zero(&hdr->b_refcnt); 2341 /* convert checksum errors into IO errors */ 2342 if (zio->io_error == ECKSUM) 2343 zio->io_error = EIO; 2344 } 2345 2346 /* 2347 * Broadcast before we drop the hash_lock to avoid the possibility 2348 * that the hdr (and hence the cv) might be freed before we get to 2349 * the cv_broadcast(). 2350 */ 2351 cv_broadcast(&hdr->b_cv); 2352 2353 if (hash_lock) { 2354 /* 2355 * Only call arc_access on anonymous buffers. This is because 2356 * if we've issued an I/O for an evicted buffer, we've already 2357 * called arc_access (to prevent any simultaneous readers from 2358 * getting confused). 2359 */ 2360 if (zio->io_error == 0 && hdr->b_state == arc_anon) 2361 arc_access(hdr, hash_lock); 2362 mutex_exit(hash_lock); 2363 } else { 2364 /* 2365 * This block was freed while we waited for the read to 2366 * complete. It has been removed from the hash table and 2367 * moved to the anonymous state (so that it won't show up 2368 * in the cache). 2369 */ 2370 ASSERT3P(hdr->b_state, ==, arc_anon); 2371 freeable = refcount_is_zero(&hdr->b_refcnt); 2372 } 2373 2374 /* execute each callback and free its structure */ 2375 while ((acb = callback_list) != NULL) { 2376 if (acb->acb_done) 2377 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2378 2379 if (acb->acb_zio_dummy != NULL) { 2380 acb->acb_zio_dummy->io_error = zio->io_error; 2381 zio_nowait(acb->acb_zio_dummy); 2382 } 2383 2384 callback_list = acb->acb_next; 2385 kmem_free(acb, sizeof (arc_callback_t)); 2386 } 2387 2388 if (freeable) 2389 arc_hdr_destroy(hdr); 2390 } 2391 2392 /* 2393 * "Read" the block block at the specified DVA (in bp) via the 2394 * cache. If the block is found in the cache, invoke the provided 2395 * callback immediately and return. Note that the `zio' parameter 2396 * in the callback will be NULL in this case, since no IO was 2397 * required. If the block is not in the cache pass the read request 2398 * on to the spa with a substitute callback function, so that the 2399 * requested block will be added to the cache. 2400 * 2401 * If a read request arrives for a block that has a read in-progress, 2402 * either wait for the in-progress read to complete (and return the 2403 * results); or, if this is a read with a "done" func, add a record 2404 * to the read to invoke the "done" func when the read completes, 2405 * and return; or just return. 2406 * 2407 * arc_read_done() will invoke all the requested "done" functions 2408 * for readers of this block. 2409 * 2410 * Normal callers should use arc_read and pass the arc buffer and offset 2411 * for the bp. But if you know you don't need locking, you can use 2412 * arc_read_bp. 2413 */ 2414 int 2415 arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_buf_t *pbuf, 2416 arc_done_func_t *done, void *private, int priority, int zio_flags, 2417 uint32_t *arc_flags, const zbookmark_t *zb) 2418 { 2419 int err; 2420 2421 ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt)); 2422 ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size); 2423 rw_enter(&pbuf->b_hdr->b_datalock, RW_READER); 2424 2425 err = arc_read_nolock(pio, spa, bp, done, private, priority, 2426 zio_flags, arc_flags, zb); 2427 2428 rw_exit(&pbuf->b_hdr->b_datalock); 2429 return (err); 2430 } 2431 2432 int 2433 arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp, 2434 arc_done_func_t *done, void *private, int priority, int zio_flags, 2435 uint32_t *arc_flags, const zbookmark_t *zb) 2436 { 2437 arc_buf_hdr_t *hdr; 2438 arc_buf_t *buf; 2439 kmutex_t *hash_lock; 2440 zio_t *rzio; 2441 2442 top: 2443 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2444 if (hdr && hdr->b_datacnt > 0) { 2445 2446 *arc_flags |= ARC_CACHED; 2447 2448 if (HDR_IO_IN_PROGRESS(hdr)) { 2449 2450 if (*arc_flags & ARC_WAIT) { 2451 cv_wait(&hdr->b_cv, hash_lock); 2452 mutex_exit(hash_lock); 2453 goto top; 2454 } 2455 ASSERT(*arc_flags & ARC_NOWAIT); 2456 2457 if (done) { 2458 arc_callback_t *acb = NULL; 2459 2460 acb = kmem_zalloc(sizeof (arc_callback_t), 2461 KM_SLEEP); 2462 acb->acb_done = done; 2463 acb->acb_private = private; 2464 if (pio != NULL) 2465 acb->acb_zio_dummy = zio_null(pio, 2466 spa, NULL, NULL, zio_flags); 2467 2468 ASSERT(acb->acb_done != NULL); 2469 acb->acb_next = hdr->b_acb; 2470 hdr->b_acb = acb; 2471 add_reference(hdr, hash_lock, private); 2472 mutex_exit(hash_lock); 2473 return (0); 2474 } 2475 mutex_exit(hash_lock); 2476 return (0); 2477 } 2478 2479 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2480 2481 if (done) { 2482 add_reference(hdr, hash_lock, private); 2483 /* 2484 * If this block is already in use, create a new 2485 * copy of the data so that we will be guaranteed 2486 * that arc_release() will always succeed. 2487 */ 2488 buf = hdr->b_buf; 2489 ASSERT(buf); 2490 ASSERT(buf->b_data); 2491 if (HDR_BUF_AVAILABLE(hdr)) { 2492 ASSERT(buf->b_efunc == NULL); 2493 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2494 } else { 2495 buf = arc_buf_clone(buf); 2496 } 2497 } else if (*arc_flags & ARC_PREFETCH && 2498 refcount_count(&hdr->b_refcnt) == 0) { 2499 hdr->b_flags |= ARC_PREFETCH; 2500 } 2501 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 2502 arc_access(hdr, hash_lock); 2503 if (*arc_flags & ARC_L2CACHE) 2504 hdr->b_flags |= ARC_L2CACHE; 2505 mutex_exit(hash_lock); 2506 ARCSTAT_BUMP(arcstat_hits); 2507 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2508 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2509 data, metadata, hits); 2510 2511 if (done) 2512 done(NULL, buf, private); 2513 } else { 2514 uint64_t size = BP_GET_LSIZE(bp); 2515 arc_callback_t *acb; 2516 vdev_t *vd = NULL; 2517 daddr_t addr; 2518 2519 if (hdr == NULL) { 2520 /* this block is not in the cache */ 2521 arc_buf_hdr_t *exists; 2522 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 2523 buf = arc_buf_alloc(spa, size, private, type); 2524 hdr = buf->b_hdr; 2525 hdr->b_dva = *BP_IDENTITY(bp); 2526 hdr->b_birth = bp->blk_birth; 2527 hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2528 exists = buf_hash_insert(hdr, &hash_lock); 2529 if (exists) { 2530 /* somebody beat us to the hash insert */ 2531 mutex_exit(hash_lock); 2532 bzero(&hdr->b_dva, sizeof (dva_t)); 2533 hdr->b_birth = 0; 2534 hdr->b_cksum0 = 0; 2535 (void) arc_buf_remove_ref(buf, private); 2536 goto top; /* restart the IO request */ 2537 } 2538 /* if this is a prefetch, we don't have a reference */ 2539 if (*arc_flags & ARC_PREFETCH) { 2540 (void) remove_reference(hdr, hash_lock, 2541 private); 2542 hdr->b_flags |= ARC_PREFETCH; 2543 } 2544 if (*arc_flags & ARC_L2CACHE) 2545 hdr->b_flags |= ARC_L2CACHE; 2546 if (BP_GET_LEVEL(bp) > 0) 2547 hdr->b_flags |= ARC_INDIRECT; 2548 } else { 2549 /* this block is in the ghost cache */ 2550 ASSERT(GHOST_STATE(hdr->b_state)); 2551 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2552 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 2553 ASSERT(hdr->b_buf == NULL); 2554 2555 /* if this is a prefetch, we don't have a reference */ 2556 if (*arc_flags & ARC_PREFETCH) 2557 hdr->b_flags |= ARC_PREFETCH; 2558 else 2559 add_reference(hdr, hash_lock, private); 2560 if (*arc_flags & ARC_L2CACHE) 2561 hdr->b_flags |= ARC_L2CACHE; 2562 buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2563 buf->b_hdr = hdr; 2564 buf->b_data = NULL; 2565 buf->b_efunc = NULL; 2566 buf->b_private = NULL; 2567 buf->b_next = NULL; 2568 hdr->b_buf = buf; 2569 arc_get_data_buf(buf); 2570 ASSERT(hdr->b_datacnt == 0); 2571 hdr->b_datacnt = 1; 2572 2573 } 2574 2575 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2576 acb->acb_done = done; 2577 acb->acb_private = private; 2578 2579 ASSERT(hdr->b_acb == NULL); 2580 hdr->b_acb = acb; 2581 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2582 2583 /* 2584 * If the buffer has been evicted, migrate it to a present state 2585 * before issuing the I/O. Once we drop the hash-table lock, 2586 * the header will be marked as I/O in progress and have an 2587 * attached buffer. At this point, anybody who finds this 2588 * buffer ought to notice that it's legit but has a pending I/O. 2589 */ 2590 2591 if (GHOST_STATE(hdr->b_state)) 2592 arc_access(hdr, hash_lock); 2593 2594 if (hdr->b_l2hdr != NULL) { 2595 vd = hdr->b_l2hdr->b_dev->l2ad_vdev; 2596 addr = hdr->b_l2hdr->b_daddr; 2597 } 2598 2599 mutex_exit(hash_lock); 2600 2601 ASSERT3U(hdr->b_size, ==, size); 2602 DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 2603 zbookmark_t *, zb); 2604 ARCSTAT_BUMP(arcstat_misses); 2605 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 2606 demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 2607 data, metadata, misses); 2608 2609 if (l2arc_ndev != 0 && HDR_L2CACHE(hdr)) { 2610 /* 2611 * Lock out device removal. 2612 */ 2613 spa_config_enter(spa, RW_READER, FTAG); 2614 2615 /* 2616 * Read from the L2ARC if the following are true: 2617 * 1. The L2ARC vdev was previously cached. 2618 * 2. This buffer still has L2ARC metadata. 2619 * 3. This buffer isn't currently writing to the L2ARC. 2620 * 4. The L2ARC entry wasn't evicted, which may 2621 * also have invalidated the vdev. 2622 */ 2623 if (vd != NULL && hdr->b_l2hdr != NULL && 2624 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) { 2625 l2arc_read_callback_t *cb; 2626 2627 if (vdev_is_dead(vd)) 2628 goto l2skip; 2629 2630 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 2631 ARCSTAT_BUMP(arcstat_l2_hits); 2632 2633 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 2634 KM_SLEEP); 2635 cb->l2rcb_buf = buf; 2636 cb->l2rcb_spa = spa; 2637 cb->l2rcb_bp = *bp; 2638 cb->l2rcb_zb = *zb; 2639 cb->l2rcb_flags = zio_flags; 2640 2641 /* 2642 * l2arc read. 2643 */ 2644 rzio = zio_read_phys(pio, vd, addr, size, 2645 buf->b_data, ZIO_CHECKSUM_OFF, 2646 l2arc_read_done, cb, priority, zio_flags | 2647 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL, 2648 B_FALSE); 2649 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 2650 zio_t *, rzio); 2651 spa_config_exit(spa, FTAG); 2652 2653 if (*arc_flags & ARC_NOWAIT) { 2654 zio_nowait(rzio); 2655 return (0); 2656 } 2657 2658 ASSERT(*arc_flags & ARC_WAIT); 2659 if (zio_wait(rzio) == 0) 2660 return (0); 2661 2662 /* l2arc read error; goto zio_read() */ 2663 } else { 2664 DTRACE_PROBE1(l2arc__miss, 2665 arc_buf_hdr_t *, hdr); 2666 ARCSTAT_BUMP(arcstat_l2_misses); 2667 if (HDR_L2_WRITING(hdr)) 2668 ARCSTAT_BUMP(arcstat_l2_rw_clash); 2669 l2skip: 2670 spa_config_exit(spa, FTAG); 2671 } 2672 } 2673 2674 rzio = zio_read(pio, spa, bp, buf->b_data, size, 2675 arc_read_done, buf, priority, zio_flags, zb); 2676 2677 if (*arc_flags & ARC_WAIT) 2678 return (zio_wait(rzio)); 2679 2680 ASSERT(*arc_flags & ARC_NOWAIT); 2681 zio_nowait(rzio); 2682 } 2683 return (0); 2684 } 2685 2686 /* 2687 * arc_read() variant to support pool traversal. If the block is already 2688 * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 2689 * The idea is that we don't want pool traversal filling up memory, but 2690 * if the ARC already has the data anyway, we shouldn't pay for the I/O. 2691 */ 2692 int 2693 arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 2694 { 2695 arc_buf_hdr_t *hdr; 2696 kmutex_t *hash_mtx; 2697 int rc = 0; 2698 2699 hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 2700 2701 if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 2702 arc_buf_t *buf = hdr->b_buf; 2703 2704 ASSERT(buf); 2705 while (buf->b_data == NULL) { 2706 buf = buf->b_next; 2707 ASSERT(buf); 2708 } 2709 bcopy(buf->b_data, data, hdr->b_size); 2710 } else { 2711 rc = ENOENT; 2712 } 2713 2714 if (hash_mtx) 2715 mutex_exit(hash_mtx); 2716 2717 return (rc); 2718 } 2719 2720 void 2721 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 2722 { 2723 ASSERT(buf->b_hdr != NULL); 2724 ASSERT(buf->b_hdr->b_state != arc_anon); 2725 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 2726 buf->b_efunc = func; 2727 buf->b_private = private; 2728 } 2729 2730 /* 2731 * This is used by the DMU to let the ARC know that a buffer is 2732 * being evicted, so the ARC should clean up. If this arc buf 2733 * is not yet in the evicted state, it will be put there. 2734 */ 2735 int 2736 arc_buf_evict(arc_buf_t *buf) 2737 { 2738 arc_buf_hdr_t *hdr; 2739 kmutex_t *hash_lock; 2740 arc_buf_t **bufp; 2741 2742 mutex_enter(&arc_eviction_mtx); 2743 hdr = buf->b_hdr; 2744 if (hdr == NULL) { 2745 /* 2746 * We are in arc_do_user_evicts(). 2747 */ 2748 ASSERT(buf->b_data == NULL); 2749 mutex_exit(&arc_eviction_mtx); 2750 return (0); 2751 } 2752 hash_lock = HDR_LOCK(hdr); 2753 mutex_exit(&arc_eviction_mtx); 2754 2755 mutex_enter(hash_lock); 2756 2757 if (buf->b_data == NULL) { 2758 /* 2759 * We are on the eviction list. 2760 */ 2761 mutex_exit(hash_lock); 2762 mutex_enter(&arc_eviction_mtx); 2763 if (buf->b_hdr == NULL) { 2764 /* 2765 * We are already in arc_do_user_evicts(). 2766 */ 2767 mutex_exit(&arc_eviction_mtx); 2768 return (0); 2769 } else { 2770 arc_buf_t copy = *buf; /* structure assignment */ 2771 /* 2772 * Process this buffer now 2773 * but let arc_do_user_evicts() do the reaping. 2774 */ 2775 buf->b_efunc = NULL; 2776 mutex_exit(&arc_eviction_mtx); 2777 VERIFY(copy.b_efunc(©) == 0); 2778 return (1); 2779 } 2780 } 2781 2782 ASSERT(buf->b_hdr == hdr); 2783 ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 2784 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2785 2786 /* 2787 * Pull this buffer off of the hdr 2788 */ 2789 bufp = &hdr->b_buf; 2790 while (*bufp != buf) 2791 bufp = &(*bufp)->b_next; 2792 *bufp = buf->b_next; 2793 2794 ASSERT(buf->b_data != NULL); 2795 arc_buf_destroy(buf, FALSE, FALSE); 2796 2797 if (hdr->b_datacnt == 0) { 2798 arc_state_t *old_state = hdr->b_state; 2799 arc_state_t *evicted_state; 2800 2801 ASSERT(refcount_is_zero(&hdr->b_refcnt)); 2802 2803 evicted_state = 2804 (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 2805 2806 mutex_enter(&old_state->arcs_mtx); 2807 mutex_enter(&evicted_state->arcs_mtx); 2808 2809 arc_change_state(evicted_state, hdr, hash_lock); 2810 ASSERT(HDR_IN_HASH_TABLE(hdr)); 2811 hdr->b_flags |= ARC_IN_HASH_TABLE; 2812 hdr->b_flags &= ~ARC_BUF_AVAILABLE; 2813 2814 mutex_exit(&evicted_state->arcs_mtx); 2815 mutex_exit(&old_state->arcs_mtx); 2816 } 2817 mutex_exit(hash_lock); 2818 2819 VERIFY(buf->b_efunc(buf) == 0); 2820 buf->b_efunc = NULL; 2821 buf->b_private = NULL; 2822 buf->b_hdr = NULL; 2823 kmem_cache_free(buf_cache, buf); 2824 return (1); 2825 } 2826 2827 /* 2828 * Release this buffer from the cache. This must be done 2829 * after a read and prior to modifying the buffer contents. 2830 * If the buffer has more than one reference, we must make 2831 * a new hdr for the buffer. 2832 */ 2833 void 2834 arc_release(arc_buf_t *buf, void *tag) 2835 { 2836 arc_buf_hdr_t *hdr = buf->b_hdr; 2837 kmutex_t *hash_lock = HDR_LOCK(hdr); 2838 l2arc_buf_hdr_t *l2hdr = NULL; 2839 uint64_t buf_size; 2840 2841 /* this buffer is not on any list */ 2842 ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2843 ASSERT(!(hdr->b_flags & ARC_STORED)); 2844 2845 if (hdr->b_state == arc_anon) { 2846 /* this buffer is already released */ 2847 ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2848 ASSERT(BUF_EMPTY(hdr)); 2849 ASSERT(buf->b_efunc == NULL); 2850 arc_buf_thaw(buf); 2851 return; 2852 } 2853 2854 mutex_enter(hash_lock); 2855 2856 /* 2857 * Do we have more than one buf? 2858 */ 2859 if (hdr->b_buf != buf || buf->b_next != NULL) { 2860 arc_buf_hdr_t *nhdr; 2861 arc_buf_t **bufp; 2862 uint64_t blksz = hdr->b_size; 2863 spa_t *spa = hdr->b_spa; 2864 arc_buf_contents_t type = hdr->b_type; 2865 uint32_t flags = hdr->b_flags; 2866 2867 ASSERT(hdr->b_datacnt > 1); 2868 /* 2869 * Pull the data off of this buf and attach it to 2870 * a new anonymous buf. 2871 */ 2872 (void) remove_reference(hdr, hash_lock, tag); 2873 bufp = &hdr->b_buf; 2874 while (*bufp != buf) 2875 bufp = &(*bufp)->b_next; 2876 *bufp = (*bufp)->b_next; 2877 buf->b_next = NULL; 2878 2879 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 2880 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 2881 if (refcount_is_zero(&hdr->b_refcnt)) { 2882 uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 2883 ASSERT3U(*size, >=, hdr->b_size); 2884 atomic_add_64(size, -hdr->b_size); 2885 } 2886 hdr->b_datacnt -= 1; 2887 if (hdr->b_l2hdr != NULL) { 2888 mutex_enter(&l2arc_buflist_mtx); 2889 l2hdr = hdr->b_l2hdr; 2890 hdr->b_l2hdr = NULL; 2891 buf_size = hdr->b_size; 2892 } 2893 arc_cksum_verify(buf); 2894 2895 mutex_exit(hash_lock); 2896 2897 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 2898 nhdr->b_size = blksz; 2899 nhdr->b_spa = spa; 2900 nhdr->b_type = type; 2901 nhdr->b_buf = buf; 2902 nhdr->b_state = arc_anon; 2903 nhdr->b_arc_access = 0; 2904 nhdr->b_flags = flags & ARC_L2_WRITING; 2905 nhdr->b_l2hdr = NULL; 2906 nhdr->b_datacnt = 1; 2907 nhdr->b_freeze_cksum = NULL; 2908 (void) refcount_add(&nhdr->b_refcnt, tag); 2909 buf->b_hdr = nhdr; 2910 atomic_add_64(&arc_anon->arcs_size, blksz); 2911 } else { 2912 ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2913 ASSERT(!list_link_active(&hdr->b_arc_node)); 2914 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2915 arc_change_state(arc_anon, hdr, hash_lock); 2916 hdr->b_arc_access = 0; 2917 if (hdr->b_l2hdr != NULL) { 2918 mutex_enter(&l2arc_buflist_mtx); 2919 l2hdr = hdr->b_l2hdr; 2920 hdr->b_l2hdr = NULL; 2921 buf_size = hdr->b_size; 2922 } 2923 mutex_exit(hash_lock); 2924 2925 bzero(&hdr->b_dva, sizeof (dva_t)); 2926 hdr->b_birth = 0; 2927 hdr->b_cksum0 = 0; 2928 arc_buf_thaw(buf); 2929 } 2930 buf->b_efunc = NULL; 2931 buf->b_private = NULL; 2932 2933 if (l2hdr) { 2934 list_remove(l2hdr->b_dev->l2ad_buflist, hdr); 2935 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t)); 2936 ARCSTAT_INCR(arcstat_l2_size, -buf_size); 2937 } 2938 if (MUTEX_HELD(&l2arc_buflist_mtx)) 2939 mutex_exit(&l2arc_buflist_mtx); 2940 } 2941 2942 int 2943 arc_released(arc_buf_t *buf) 2944 { 2945 return (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 2946 } 2947 2948 int 2949 arc_has_callback(arc_buf_t *buf) 2950 { 2951 return (buf->b_efunc != NULL); 2952 } 2953 2954 #ifdef ZFS_DEBUG 2955 int 2956 arc_referenced(arc_buf_t *buf) 2957 { 2958 return (refcount_count(&buf->b_hdr->b_refcnt)); 2959 } 2960 #endif 2961 2962 static void 2963 arc_write_ready(zio_t *zio) 2964 { 2965 arc_write_callback_t *callback = zio->io_private; 2966 arc_buf_t *buf = callback->awcb_buf; 2967 arc_buf_hdr_t *hdr = buf->b_hdr; 2968 2969 if (zio->io_error == 0 && callback->awcb_ready) { 2970 ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 2971 callback->awcb_ready(zio, buf, callback->awcb_private); 2972 } 2973 /* 2974 * If the IO is already in progress, then this is a re-write 2975 * attempt, so we need to thaw and re-compute the cksum. It is 2976 * the responsibility of the callback to handle the freeing 2977 * and accounting for any re-write attempt. If we don't have a 2978 * callback registered then simply free the block here. 2979 */ 2980 if (HDR_IO_IN_PROGRESS(hdr)) { 2981 if (!BP_IS_HOLE(&zio->io_bp_orig) && 2982 callback->awcb_ready == NULL) { 2983 zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 2984 &zio->io_bp_orig, NULL, NULL)); 2985 } 2986 mutex_enter(&hdr->b_freeze_lock); 2987 if (hdr->b_freeze_cksum != NULL) { 2988 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 2989 hdr->b_freeze_cksum = NULL; 2990 } 2991 mutex_exit(&hdr->b_freeze_lock); 2992 } 2993 arc_cksum_compute(buf, B_FALSE); 2994 hdr->b_flags |= ARC_IO_IN_PROGRESS; 2995 } 2996 2997 static void 2998 arc_write_done(zio_t *zio) 2999 { 3000 arc_write_callback_t *callback = zio->io_private; 3001 arc_buf_t *buf = callback->awcb_buf; 3002 arc_buf_hdr_t *hdr = buf->b_hdr; 3003 3004 hdr->b_acb = NULL; 3005 3006 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 3007 hdr->b_birth = zio->io_bp->blk_birth; 3008 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 3009 /* 3010 * If the block to be written was all-zero, we may have 3011 * compressed it away. In this case no write was performed 3012 * so there will be no dva/birth-date/checksum. The buffer 3013 * must therefor remain anonymous (and uncached). 3014 */ 3015 if (!BUF_EMPTY(hdr)) { 3016 arc_buf_hdr_t *exists; 3017 kmutex_t *hash_lock; 3018 3019 arc_cksum_verify(buf); 3020 3021 exists = buf_hash_insert(hdr, &hash_lock); 3022 if (exists) { 3023 /* 3024 * This can only happen if we overwrite for 3025 * sync-to-convergence, because we remove 3026 * buffers from the hash table when we arc_free(). 3027 */ 3028 ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 3029 BP_IDENTITY(zio->io_bp))); 3030 ASSERT3U(zio->io_bp_orig.blk_birth, ==, 3031 zio->io_bp->blk_birth); 3032 3033 ASSERT(refcount_is_zero(&exists->b_refcnt)); 3034 arc_change_state(arc_anon, exists, hash_lock); 3035 mutex_exit(hash_lock); 3036 arc_hdr_destroy(exists); 3037 exists = buf_hash_insert(hdr, &hash_lock); 3038 ASSERT3P(exists, ==, NULL); 3039 } 3040 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3041 /* if it's not anon, we are doing a scrub */ 3042 if (hdr->b_state == arc_anon) 3043 arc_access(hdr, hash_lock); 3044 mutex_exit(hash_lock); 3045 } else if (callback->awcb_done == NULL) { 3046 int destroy_hdr; 3047 /* 3048 * This is an anonymous buffer with no user callback, 3049 * destroy it if there are no active references. 3050 */ 3051 mutex_enter(&arc_eviction_mtx); 3052 destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 3053 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3054 mutex_exit(&arc_eviction_mtx); 3055 if (destroy_hdr) 3056 arc_hdr_destroy(hdr); 3057 } else { 3058 hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 3059 } 3060 hdr->b_flags &= ~ARC_STORED; 3061 3062 if (callback->awcb_done) { 3063 ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 3064 callback->awcb_done(zio, buf, callback->awcb_private); 3065 } 3066 3067 kmem_free(callback, sizeof (arc_write_callback_t)); 3068 } 3069 3070 static void 3071 write_policy(spa_t *spa, const writeprops_t *wp, 3072 int *cksump, int *compp, int *copiesp) 3073 { 3074 int copies = wp->wp_copies; 3075 boolean_t ismd = (wp->wp_level > 0 || dmu_ot[wp->wp_type].ot_metadata); 3076 3077 /* Determine copies setting */ 3078 if (ismd) 3079 copies++; 3080 *copiesp = MIN(copies, spa_max_replication(spa)); 3081 3082 /* Determine checksum setting */ 3083 if (ismd) { 3084 /* 3085 * Metadata always gets checksummed. If the data 3086 * checksum is multi-bit correctable, and it's not a 3087 * ZBT-style checksum, then it's suitable for metadata 3088 * as well. Otherwise, the metadata checksum defaults 3089 * to fletcher4. 3090 */ 3091 if (zio_checksum_table[wp->wp_oschecksum].ci_correctable && 3092 !zio_checksum_table[wp->wp_oschecksum].ci_zbt) 3093 *cksump = wp->wp_oschecksum; 3094 else 3095 *cksump = ZIO_CHECKSUM_FLETCHER_4; 3096 } else { 3097 *cksump = zio_checksum_select(wp->wp_dnchecksum, 3098 wp->wp_oschecksum); 3099 } 3100 3101 /* Determine compression setting */ 3102 if (ismd) { 3103 /* 3104 * XXX -- we should design a compression algorithm 3105 * that specializes in arrays of bps. 3106 */ 3107 *compp = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY : 3108 ZIO_COMPRESS_LZJB; 3109 } else { 3110 *compp = zio_compress_select(wp->wp_dncompress, 3111 wp->wp_oscompress); 3112 } 3113 } 3114 3115 zio_t * 3116 arc_write(zio_t *pio, spa_t *spa, const writeprops_t *wp, 3117 boolean_t l2arc, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 3118 arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority, 3119 int zio_flags, const zbookmark_t *zb) 3120 { 3121 arc_buf_hdr_t *hdr = buf->b_hdr; 3122 arc_write_callback_t *callback; 3123 zio_t *zio; 3124 int cksum, comp, copies; 3125 3126 ASSERT(!HDR_IO_ERROR(hdr)); 3127 ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 3128 ASSERT(hdr->b_acb == 0); 3129 if (l2arc) 3130 hdr->b_flags |= ARC_L2CACHE; 3131 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 3132 callback->awcb_ready = ready; 3133 callback->awcb_done = done; 3134 callback->awcb_private = private; 3135 callback->awcb_buf = buf; 3136 3137 write_policy(spa, wp, &cksum, &comp, &copies); 3138 zio = zio_write(pio, spa, cksum, comp, copies, txg, bp, 3139 buf->b_data, hdr->b_size, arc_write_ready, arc_write_done, 3140 callback, priority, zio_flags, zb); 3141 3142 return (zio); 3143 } 3144 3145 int 3146 arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 3147 zio_done_func_t *done, void *private, uint32_t arc_flags) 3148 { 3149 arc_buf_hdr_t *ab; 3150 kmutex_t *hash_lock; 3151 zio_t *zio; 3152 3153 /* 3154 * If this buffer is in the cache, release it, so it 3155 * can be re-used. 3156 */ 3157 ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 3158 if (ab != NULL) { 3159 /* 3160 * The checksum of blocks to free is not always 3161 * preserved (eg. on the deadlist). However, if it is 3162 * nonzero, it should match what we have in the cache. 3163 */ 3164 ASSERT(bp->blk_cksum.zc_word[0] == 0 || 3165 ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 3166 if (ab->b_state != arc_anon) 3167 arc_change_state(arc_anon, ab, hash_lock); 3168 if (HDR_IO_IN_PROGRESS(ab)) { 3169 /* 3170 * This should only happen when we prefetch. 3171 */ 3172 ASSERT(ab->b_flags & ARC_PREFETCH); 3173 ASSERT3U(ab->b_datacnt, ==, 1); 3174 ab->b_flags |= ARC_FREED_IN_READ; 3175 if (HDR_IN_HASH_TABLE(ab)) 3176 buf_hash_remove(ab); 3177 ab->b_arc_access = 0; 3178 bzero(&ab->b_dva, sizeof (dva_t)); 3179 ab->b_birth = 0; 3180 ab->b_cksum0 = 0; 3181 ab->b_buf->b_efunc = NULL; 3182 ab->b_buf->b_private = NULL; 3183 mutex_exit(hash_lock); 3184 } else if (refcount_is_zero(&ab->b_refcnt)) { 3185 ab->b_flags |= ARC_FREE_IN_PROGRESS; 3186 mutex_exit(hash_lock); 3187 arc_hdr_destroy(ab); 3188 ARCSTAT_BUMP(arcstat_deleted); 3189 } else { 3190 /* 3191 * We still have an active reference on this 3192 * buffer. This can happen, e.g., from 3193 * dbuf_unoverride(). 3194 */ 3195 ASSERT(!HDR_IN_HASH_TABLE(ab)); 3196 ab->b_arc_access = 0; 3197 bzero(&ab->b_dva, sizeof (dva_t)); 3198 ab->b_birth = 0; 3199 ab->b_cksum0 = 0; 3200 ab->b_buf->b_efunc = NULL; 3201 ab->b_buf->b_private = NULL; 3202 mutex_exit(hash_lock); 3203 } 3204 } 3205 3206 zio = zio_free(pio, spa, txg, bp, done, private); 3207 3208 if (arc_flags & ARC_WAIT) 3209 return (zio_wait(zio)); 3210 3211 ASSERT(arc_flags & ARC_NOWAIT); 3212 zio_nowait(zio); 3213 3214 return (0); 3215 } 3216 3217 static int 3218 arc_memory_throttle(uint64_t reserve, uint64_t txg) 3219 { 3220 #ifdef _KERNEL 3221 uint64_t inflight_data = arc_anon->arcs_size; 3222 uint64_t available_memory = ptob(freemem); 3223 static uint64_t page_load = 0; 3224 static uint64_t last_txg = 0; 3225 3226 #if defined(__i386) 3227 available_memory = 3228 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 3229 #endif 3230 if (available_memory >= zfs_write_limit_max) 3231 return (0); 3232 3233 if (txg > last_txg) { 3234 last_txg = txg; 3235 page_load = 0; 3236 } 3237 /* 3238 * If we are in pageout, we know that memory is already tight, 3239 * the arc is already going to be evicting, so we just want to 3240 * continue to let page writes occur as quickly as possible. 3241 */ 3242 if (curproc == proc_pageout) { 3243 if (page_load > MAX(ptob(minfree), available_memory) / 4) 3244 return (ERESTART); 3245 /* Note: reserve is inflated, so we deflate */ 3246 page_load += reserve / 8; 3247 return (0); 3248 } else if (page_load > 0 && arc_reclaim_needed()) { 3249 /* memory is low, delay before restarting */ 3250 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3251 return (EAGAIN); 3252 } 3253 page_load = 0; 3254 3255 if (arc_size > arc_c_min) { 3256 uint64_t evictable_memory = 3257 arc_mru->arcs_lsize[ARC_BUFC_DATA] + 3258 arc_mru->arcs_lsize[ARC_BUFC_METADATA] + 3259 arc_mfu->arcs_lsize[ARC_BUFC_DATA] + 3260 arc_mfu->arcs_lsize[ARC_BUFC_METADATA]; 3261 available_memory += MIN(evictable_memory, arc_size - arc_c_min); 3262 } 3263 3264 if (inflight_data > available_memory / 4) { 3265 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 3266 return (ERESTART); 3267 } 3268 #endif 3269 return (0); 3270 } 3271 3272 void 3273 arc_tempreserve_clear(uint64_t reserve) 3274 { 3275 atomic_add_64(&arc_tempreserve, -reserve); 3276 ASSERT((int64_t)arc_tempreserve >= 0); 3277 } 3278 3279 int 3280 arc_tempreserve_space(uint64_t reserve, uint64_t txg) 3281 { 3282 int error; 3283 3284 #ifdef ZFS_DEBUG 3285 /* 3286 * Once in a while, fail for no reason. Everything should cope. 3287 */ 3288 if (spa_get_random(10000) == 0) { 3289 dprintf("forcing random failure\n"); 3290 return (ERESTART); 3291 } 3292 #endif 3293 if (reserve > arc_c/4 && !arc_no_grow) 3294 arc_c = MIN(arc_c_max, reserve * 4); 3295 if (reserve > arc_c) 3296 return (ENOMEM); 3297 3298 /* 3299 * Writes will, almost always, require additional memory allocations 3300 * in order to compress/encrypt/etc the data. We therefor need to 3301 * make sure that there is sufficient available memory for this. 3302 */ 3303 if (error = arc_memory_throttle(reserve, txg)) 3304 return (error); 3305 3306 /* 3307 * Throttle writes when the amount of dirty data in the cache 3308 * gets too large. We try to keep the cache less than half full 3309 * of dirty blocks so that our sync times don't grow too large. 3310 * Note: if two requests come in concurrently, we might let them 3311 * both succeed, when one of them should fail. Not a huge deal. 3312 */ 3313 if (reserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 && 3314 arc_anon->arcs_size > arc_c / 4) { 3315 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 3316 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 3317 arc_tempreserve>>10, 3318 arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 3319 arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 3320 reserve>>10, arc_c>>10); 3321 return (ERESTART); 3322 } 3323 atomic_add_64(&arc_tempreserve, reserve); 3324 return (0); 3325 } 3326 3327 void 3328 arc_init(void) 3329 { 3330 mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 3331 cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 3332 3333 /* Convert seconds to clock ticks */ 3334 arc_min_prefetch_lifespan = 1 * hz; 3335 3336 /* Start out with 1/8 of all memory */ 3337 arc_c = physmem * PAGESIZE / 8; 3338 3339 #ifdef _KERNEL 3340 /* 3341 * On architectures where the physical memory can be larger 3342 * than the addressable space (intel in 32-bit mode), we may 3343 * need to limit the cache to 1/8 of VM size. 3344 */ 3345 arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 3346 #endif 3347 3348 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 3349 arc_c_min = MAX(arc_c / 4, 64<<20); 3350 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 3351 if (arc_c * 8 >= 1<<30) 3352 arc_c_max = (arc_c * 8) - (1<<30); 3353 else 3354 arc_c_max = arc_c_min; 3355 arc_c_max = MAX(arc_c * 6, arc_c_max); 3356 3357 /* 3358 * Allow the tunables to override our calculations if they are 3359 * reasonable (ie. over 64MB) 3360 */ 3361 if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 3362 arc_c_max = zfs_arc_max; 3363 if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 3364 arc_c_min = zfs_arc_min; 3365 3366 arc_c = arc_c_max; 3367 arc_p = (arc_c >> 1); 3368 3369 /* limit meta-data to 1/4 of the arc capacity */ 3370 arc_meta_limit = arc_c_max / 4; 3371 3372 /* Allow the tunable to override if it is reasonable */ 3373 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 3374 arc_meta_limit = zfs_arc_meta_limit; 3375 3376 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 3377 arc_c_min = arc_meta_limit / 2; 3378 3379 /* if kmem_flags are set, lets try to use less memory */ 3380 if (kmem_debugging()) 3381 arc_c = arc_c / 2; 3382 if (arc_c < arc_c_min) 3383 arc_c = arc_c_min; 3384 3385 arc_anon = &ARC_anon; 3386 arc_mru = &ARC_mru; 3387 arc_mru_ghost = &ARC_mru_ghost; 3388 arc_mfu = &ARC_mfu; 3389 arc_mfu_ghost = &ARC_mfu_ghost; 3390 arc_l2c_only = &ARC_l2c_only; 3391 arc_size = 0; 3392 3393 mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3394 mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3395 mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3396 mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3397 mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3398 mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 3399 3400 list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 3401 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3402 list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 3403 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3404 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 3405 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3406 list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 3407 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3408 list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 3409 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3410 list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 3411 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3412 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 3413 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3414 list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 3415 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3416 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], 3417 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3418 list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], 3419 sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 3420 3421 buf_init(); 3422 3423 arc_thread_exit = 0; 3424 arc_eviction_list = NULL; 3425 mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 3426 bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 3427 3428 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 3429 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 3430 3431 if (arc_ksp != NULL) { 3432 arc_ksp->ks_data = &arc_stats; 3433 kstat_install(arc_ksp); 3434 } 3435 3436 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 3437 TS_RUN, minclsyspri); 3438 3439 arc_dead = FALSE; 3440 arc_warm = B_FALSE; 3441 3442 if (zfs_write_limit_max == 0) 3443 zfs_write_limit_max = physmem * PAGESIZE >> 3444 zfs_write_limit_shift; 3445 else 3446 zfs_write_limit_shift = 0; 3447 } 3448 3449 void 3450 arc_fini(void) 3451 { 3452 mutex_enter(&arc_reclaim_thr_lock); 3453 arc_thread_exit = 1; 3454 while (arc_thread_exit != 0) 3455 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 3456 mutex_exit(&arc_reclaim_thr_lock); 3457 3458 arc_flush(NULL); 3459 3460 arc_dead = TRUE; 3461 3462 if (arc_ksp != NULL) { 3463 kstat_delete(arc_ksp); 3464 arc_ksp = NULL; 3465 } 3466 3467 mutex_destroy(&arc_eviction_mtx); 3468 mutex_destroy(&arc_reclaim_thr_lock); 3469 cv_destroy(&arc_reclaim_thr_cv); 3470 3471 list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 3472 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 3473 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 3474 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 3475 list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 3476 list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 3477 list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 3478 list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 3479 3480 mutex_destroy(&arc_anon->arcs_mtx); 3481 mutex_destroy(&arc_mru->arcs_mtx); 3482 mutex_destroy(&arc_mru_ghost->arcs_mtx); 3483 mutex_destroy(&arc_mfu->arcs_mtx); 3484 mutex_destroy(&arc_mfu_ghost->arcs_mtx); 3485 3486 buf_fini(); 3487 } 3488 3489 /* 3490 * Level 2 ARC 3491 * 3492 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 3493 * It uses dedicated storage devices to hold cached data, which are populated 3494 * using large infrequent writes. The main role of this cache is to boost 3495 * the performance of random read workloads. The intended L2ARC devices 3496 * include short-stroked disks, solid state disks, and other media with 3497 * substantially faster read latency than disk. 3498 * 3499 * +-----------------------+ 3500 * | ARC | 3501 * +-----------------------+ 3502 * | ^ ^ 3503 * | | | 3504 * l2arc_feed_thread() arc_read() 3505 * | | | 3506 * | l2arc read | 3507 * V | | 3508 * +---------------+ | 3509 * | L2ARC | | 3510 * +---------------+ | 3511 * | ^ | 3512 * l2arc_write() | | 3513 * | | | 3514 * V | | 3515 * +-------+ +-------+ 3516 * | vdev | | vdev | 3517 * | cache | | cache | 3518 * +-------+ +-------+ 3519 * +=========+ .-----. 3520 * : L2ARC : |-_____-| 3521 * : devices : | Disks | 3522 * +=========+ `-_____-' 3523 * 3524 * Read requests are satisfied from the following sources, in order: 3525 * 3526 * 1) ARC 3527 * 2) vdev cache of L2ARC devices 3528 * 3) L2ARC devices 3529 * 4) vdev cache of disks 3530 * 5) disks 3531 * 3532 * Some L2ARC device types exhibit extremely slow write performance. 3533 * To accommodate for this there are some significant differences between 3534 * the L2ARC and traditional cache design: 3535 * 3536 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 3537 * the ARC behave as usual, freeing buffers and placing headers on ghost 3538 * lists. The ARC does not send buffers to the L2ARC during eviction as 3539 * this would add inflated write latencies for all ARC memory pressure. 3540 * 3541 * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 3542 * It does this by periodically scanning buffers from the eviction-end of 3543 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 3544 * not already there. It scans until a headroom of buffers is satisfied, 3545 * which itself is a buffer for ARC eviction. The thread that does this is 3546 * l2arc_feed_thread(), illustrated below; example sizes are included to 3547 * provide a better sense of ratio than this diagram: 3548 * 3549 * head --> tail 3550 * +---------------------+----------+ 3551 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 3552 * +---------------------+----------+ | o L2ARC eligible 3553 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 3554 * +---------------------+----------+ | 3555 * 15.9 Gbytes ^ 32 Mbytes | 3556 * headroom | 3557 * l2arc_feed_thread() 3558 * | 3559 * l2arc write hand <--[oooo]--' 3560 * | 8 Mbyte 3561 * | write max 3562 * V 3563 * +==============================+ 3564 * L2ARC dev |####|#|###|###| |####| ... | 3565 * +==============================+ 3566 * 32 Gbytes 3567 * 3568 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 3569 * evicted, then the L2ARC has cached a buffer much sooner than it probably 3570 * needed to, potentially wasting L2ARC device bandwidth and storage. It is 3571 * safe to say that this is an uncommon case, since buffers at the end of 3572 * the ARC lists have moved there due to inactivity. 3573 * 3574 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 3575 * then the L2ARC simply misses copying some buffers. This serves as a 3576 * pressure valve to prevent heavy read workloads from both stalling the ARC 3577 * with waits and clogging the L2ARC with writes. This also helps prevent 3578 * the potential for the L2ARC to churn if it attempts to cache content too 3579 * quickly, such as during backups of the entire pool. 3580 * 3581 * 5. After system boot and before the ARC has filled main memory, there are 3582 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 3583 * lists can remain mostly static. Instead of searching from tail of these 3584 * lists as pictured, the l2arc_feed_thread() will search from the list heads 3585 * for eligible buffers, greatly increasing its chance of finding them. 3586 * 3587 * The L2ARC device write speed is also boosted during this time so that 3588 * the L2ARC warms up faster. Since there have been no ARC evictions yet, 3589 * there are no L2ARC reads, and no fear of degrading read performance 3590 * through increased writes. 3591 * 3592 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 3593 * the vdev queue can aggregate them into larger and fewer writes. Each 3594 * device is written to in a rotor fashion, sweeping writes through 3595 * available space then repeating. 3596 * 3597 * 7. The L2ARC does not store dirty content. It never needs to flush 3598 * write buffers back to disk based storage. 3599 * 3600 * 8. If an ARC buffer is written (and dirtied) which also exists in the 3601 * L2ARC, the now stale L2ARC buffer is immediately dropped. 3602 * 3603 * The performance of the L2ARC can be tweaked by a number of tunables, which 3604 * may be necessary for different workloads: 3605 * 3606 * l2arc_write_max max write bytes per interval 3607 * l2arc_write_boost extra write bytes during device warmup 3608 * l2arc_noprefetch skip caching prefetched buffers 3609 * l2arc_headroom number of max device writes to precache 3610 * l2arc_feed_secs seconds between L2ARC writing 3611 * 3612 * Tunables may be removed or added as future performance improvements are 3613 * integrated, and also may become zpool properties. 3614 */ 3615 3616 static void 3617 l2arc_hdr_stat_add(void) 3618 { 3619 ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE); 3620 ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE); 3621 } 3622 3623 static void 3624 l2arc_hdr_stat_remove(void) 3625 { 3626 ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE)); 3627 ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE); 3628 } 3629 3630 /* 3631 * Cycle through L2ARC devices. This is how L2ARC load balances. 3632 * If a device is returned, this also returns holding the spa config lock. 3633 */ 3634 static l2arc_dev_t * 3635 l2arc_dev_get_next(void) 3636 { 3637 l2arc_dev_t *first, *next = NULL; 3638 3639 /* 3640 * Lock out the removal of spas (spa_namespace_lock), then removal 3641 * of cache devices (l2arc_dev_mtx). Once a device has been selected, 3642 * both locks will be dropped and a spa config lock held instead. 3643 */ 3644 mutex_enter(&spa_namespace_lock); 3645 mutex_enter(&l2arc_dev_mtx); 3646 3647 /* if there are no vdevs, there is nothing to do */ 3648 if (l2arc_ndev == 0) 3649 goto out; 3650 3651 first = NULL; 3652 next = l2arc_dev_last; 3653 do { 3654 /* loop around the list looking for a non-faulted vdev */ 3655 if (next == NULL) { 3656 next = list_head(l2arc_dev_list); 3657 } else { 3658 next = list_next(l2arc_dev_list, next); 3659 if (next == NULL) 3660 next = list_head(l2arc_dev_list); 3661 } 3662 3663 /* if we have come back to the start, bail out */ 3664 if (first == NULL) 3665 first = next; 3666 else if (next == first) 3667 break; 3668 3669 } while (vdev_is_dead(next->l2ad_vdev)); 3670 3671 /* if we were unable to find any usable vdevs, return NULL */ 3672 if (vdev_is_dead(next->l2ad_vdev)) 3673 next = NULL; 3674 3675 l2arc_dev_last = next; 3676 3677 out: 3678 mutex_exit(&l2arc_dev_mtx); 3679 3680 /* 3681 * Grab the config lock to prevent the 'next' device from being 3682 * removed while we are writing to it. 3683 */ 3684 if (next != NULL) 3685 spa_config_enter(next->l2ad_spa, RW_READER, next); 3686 mutex_exit(&spa_namespace_lock); 3687 3688 return (next); 3689 } 3690 3691 /* 3692 * Free buffers that were tagged for destruction. 3693 */ 3694 static void 3695 l2arc_do_free_on_write() 3696 { 3697 list_t *buflist; 3698 l2arc_data_free_t *df, *df_prev; 3699 3700 mutex_enter(&l2arc_free_on_write_mtx); 3701 buflist = l2arc_free_on_write; 3702 3703 for (df = list_tail(buflist); df; df = df_prev) { 3704 df_prev = list_prev(buflist, df); 3705 ASSERT(df->l2df_data != NULL); 3706 ASSERT(df->l2df_func != NULL); 3707 df->l2df_func(df->l2df_data, df->l2df_size); 3708 list_remove(buflist, df); 3709 kmem_free(df, sizeof (l2arc_data_free_t)); 3710 } 3711 3712 mutex_exit(&l2arc_free_on_write_mtx); 3713 } 3714 3715 /* 3716 * A write to a cache device has completed. Update all headers to allow 3717 * reads from these buffers to begin. 3718 */ 3719 static void 3720 l2arc_write_done(zio_t *zio) 3721 { 3722 l2arc_write_callback_t *cb; 3723 l2arc_dev_t *dev; 3724 list_t *buflist; 3725 arc_buf_hdr_t *head, *ab, *ab_prev; 3726 l2arc_buf_hdr_t *abl2; 3727 kmutex_t *hash_lock; 3728 3729 cb = zio->io_private; 3730 ASSERT(cb != NULL); 3731 dev = cb->l2wcb_dev; 3732 ASSERT(dev != NULL); 3733 head = cb->l2wcb_head; 3734 ASSERT(head != NULL); 3735 buflist = dev->l2ad_buflist; 3736 ASSERT(buflist != NULL); 3737 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 3738 l2arc_write_callback_t *, cb); 3739 3740 if (zio->io_error != 0) 3741 ARCSTAT_BUMP(arcstat_l2_writes_error); 3742 3743 mutex_enter(&l2arc_buflist_mtx); 3744 3745 /* 3746 * All writes completed, or an error was hit. 3747 */ 3748 for (ab = list_prev(buflist, head); ab; ab = ab_prev) { 3749 ab_prev = list_prev(buflist, ab); 3750 3751 hash_lock = HDR_LOCK(ab); 3752 if (!mutex_tryenter(hash_lock)) { 3753 /* 3754 * This buffer misses out. It may be in a stage 3755 * of eviction. Its ARC_L2_WRITING flag will be 3756 * left set, denying reads to this buffer. 3757 */ 3758 ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss); 3759 continue; 3760 } 3761 3762 if (zio->io_error != 0) { 3763 /* 3764 * Error - drop L2ARC entry. 3765 */ 3766 list_remove(buflist, ab); 3767 abl2 = ab->b_l2hdr; 3768 ab->b_l2hdr = NULL; 3769 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 3770 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 3771 } 3772 3773 /* 3774 * Allow ARC to begin reads to this L2ARC entry. 3775 */ 3776 ab->b_flags &= ~ARC_L2_WRITING; 3777 3778 mutex_exit(hash_lock); 3779 } 3780 3781 atomic_inc_64(&l2arc_writes_done); 3782 list_remove(buflist, head); 3783 kmem_cache_free(hdr_cache, head); 3784 mutex_exit(&l2arc_buflist_mtx); 3785 3786 l2arc_do_free_on_write(); 3787 3788 kmem_free(cb, sizeof (l2arc_write_callback_t)); 3789 } 3790 3791 /* 3792 * A read to a cache device completed. Validate buffer contents before 3793 * handing over to the regular ARC routines. 3794 */ 3795 static void 3796 l2arc_read_done(zio_t *zio) 3797 { 3798 l2arc_read_callback_t *cb; 3799 arc_buf_hdr_t *hdr; 3800 arc_buf_t *buf; 3801 zio_t *rzio; 3802 kmutex_t *hash_lock; 3803 int equal; 3804 3805 cb = zio->io_private; 3806 ASSERT(cb != NULL); 3807 buf = cb->l2rcb_buf; 3808 ASSERT(buf != NULL); 3809 hdr = buf->b_hdr; 3810 ASSERT(hdr != NULL); 3811 3812 hash_lock = HDR_LOCK(hdr); 3813 mutex_enter(hash_lock); 3814 3815 /* 3816 * Check this survived the L2ARC journey. 3817 */ 3818 equal = arc_cksum_equal(buf); 3819 if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 3820 mutex_exit(hash_lock); 3821 zio->io_private = buf; 3822 arc_read_done(zio); 3823 } else { 3824 mutex_exit(hash_lock); 3825 /* 3826 * Buffer didn't survive caching. Increment stats and 3827 * reissue to the original storage device. 3828 */ 3829 if (zio->io_error != 0) { 3830 ARCSTAT_BUMP(arcstat_l2_io_error); 3831 } else { 3832 zio->io_error = EIO; 3833 } 3834 if (!equal) 3835 ARCSTAT_BUMP(arcstat_l2_cksum_bad); 3836 3837 if (zio->io_waiter == NULL) { 3838 /* 3839 * Let the resent I/O call arc_read_done() instead. 3840 */ 3841 zio->io_done = NULL; 3842 zio->io_flags &= ~ZIO_FLAG_DONT_CACHE; 3843 3844 rzio = zio_read(NULL, cb->l2rcb_spa, &cb->l2rcb_bp, 3845 buf->b_data, zio->io_size, arc_read_done, buf, 3846 zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb); 3847 3848 (void) zio_nowait(rzio); 3849 } 3850 } 3851 3852 kmem_free(cb, sizeof (l2arc_read_callback_t)); 3853 } 3854 3855 /* 3856 * This is the list priority from which the L2ARC will search for pages to 3857 * cache. This is used within loops (0..3) to cycle through lists in the 3858 * desired order. This order can have a significant effect on cache 3859 * performance. 3860 * 3861 * Currently the metadata lists are hit first, MFU then MRU, followed by 3862 * the data lists. This function returns a locked list, and also returns 3863 * the lock pointer. 3864 */ 3865 static list_t * 3866 l2arc_list_locked(int list_num, kmutex_t **lock) 3867 { 3868 list_t *list; 3869 3870 ASSERT(list_num >= 0 && list_num <= 3); 3871 3872 switch (list_num) { 3873 case 0: 3874 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; 3875 *lock = &arc_mfu->arcs_mtx; 3876 break; 3877 case 1: 3878 list = &arc_mru->arcs_list[ARC_BUFC_METADATA]; 3879 *lock = &arc_mru->arcs_mtx; 3880 break; 3881 case 2: 3882 list = &arc_mfu->arcs_list[ARC_BUFC_DATA]; 3883 *lock = &arc_mfu->arcs_mtx; 3884 break; 3885 case 3: 3886 list = &arc_mru->arcs_list[ARC_BUFC_DATA]; 3887 *lock = &arc_mru->arcs_mtx; 3888 break; 3889 } 3890 3891 ASSERT(!(MUTEX_HELD(*lock))); 3892 mutex_enter(*lock); 3893 return (list); 3894 } 3895 3896 /* 3897 * Evict buffers from the device write hand to the distance specified in 3898 * bytes. This distance may span populated buffers, it may span nothing. 3899 * This is clearing a region on the L2ARC device ready for writing. 3900 * If the 'all' boolean is set, every buffer is evicted. 3901 */ 3902 static void 3903 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 3904 { 3905 list_t *buflist; 3906 l2arc_buf_hdr_t *abl2; 3907 arc_buf_hdr_t *ab, *ab_prev; 3908 kmutex_t *hash_lock; 3909 uint64_t taddr; 3910 3911 buflist = dev->l2ad_buflist; 3912 3913 if (buflist == NULL) 3914 return; 3915 3916 if (!all && dev->l2ad_first) { 3917 /* 3918 * This is the first sweep through the device. There is 3919 * nothing to evict. 3920 */ 3921 return; 3922 } 3923 3924 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 3925 /* 3926 * When nearing the end of the device, evict to the end 3927 * before the device write hand jumps to the start. 3928 */ 3929 taddr = dev->l2ad_end; 3930 } else { 3931 taddr = dev->l2ad_hand + distance; 3932 } 3933 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 3934 uint64_t, taddr, boolean_t, all); 3935 3936 top: 3937 mutex_enter(&l2arc_buflist_mtx); 3938 for (ab = list_tail(buflist); ab; ab = ab_prev) { 3939 ab_prev = list_prev(buflist, ab); 3940 3941 hash_lock = HDR_LOCK(ab); 3942 if (!mutex_tryenter(hash_lock)) { 3943 /* 3944 * Missed the hash lock. Retry. 3945 */ 3946 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 3947 mutex_exit(&l2arc_buflist_mtx); 3948 mutex_enter(hash_lock); 3949 mutex_exit(hash_lock); 3950 goto top; 3951 } 3952 3953 if (HDR_L2_WRITE_HEAD(ab)) { 3954 /* 3955 * We hit a write head node. Leave it for 3956 * l2arc_write_done(). 3957 */ 3958 list_remove(buflist, ab); 3959 mutex_exit(hash_lock); 3960 continue; 3961 } 3962 3963 if (!all && ab->b_l2hdr != NULL && 3964 (ab->b_l2hdr->b_daddr > taddr || 3965 ab->b_l2hdr->b_daddr < dev->l2ad_hand)) { 3966 /* 3967 * We've evicted to the target address, 3968 * or the end of the device. 3969 */ 3970 mutex_exit(hash_lock); 3971 break; 3972 } 3973 3974 if (HDR_FREE_IN_PROGRESS(ab)) { 3975 /* 3976 * Already on the path to destruction. 3977 */ 3978 mutex_exit(hash_lock); 3979 continue; 3980 } 3981 3982 if (ab->b_state == arc_l2c_only) { 3983 ASSERT(!HDR_L2_READING(ab)); 3984 /* 3985 * This doesn't exist in the ARC. Destroy. 3986 * arc_hdr_destroy() will call list_remove() 3987 * and decrement arcstat_l2_size. 3988 */ 3989 arc_change_state(arc_anon, ab, hash_lock); 3990 arc_hdr_destroy(ab); 3991 } else { 3992 /* 3993 * Invalidate issued or about to be issued 3994 * reads, since we may be about to write 3995 * over this location. 3996 */ 3997 if (HDR_L2_READING(ab)) { 3998 ARCSTAT_BUMP(arcstat_l2_evict_reading); 3999 ab->b_flags |= ARC_L2_EVICTED; 4000 } 4001 4002 /* 4003 * Tell ARC this no longer exists in L2ARC. 4004 */ 4005 if (ab->b_l2hdr != NULL) { 4006 abl2 = ab->b_l2hdr; 4007 ab->b_l2hdr = NULL; 4008 kmem_free(abl2, sizeof (l2arc_buf_hdr_t)); 4009 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size); 4010 } 4011 list_remove(buflist, ab); 4012 4013 /* 4014 * This may have been leftover after a 4015 * failed write. 4016 */ 4017 ab->b_flags &= ~ARC_L2_WRITING; 4018 } 4019 mutex_exit(hash_lock); 4020 } 4021 mutex_exit(&l2arc_buflist_mtx); 4022 4023 spa_l2cache_space_update(dev->l2ad_vdev, 0, -(taddr - dev->l2ad_evict)); 4024 dev->l2ad_evict = taddr; 4025 } 4026 4027 /* 4028 * Find and write ARC buffers to the L2ARC device. 4029 * 4030 * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid 4031 * for reading until they have completed writing. 4032 */ 4033 static void 4034 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 4035 { 4036 arc_buf_hdr_t *ab, *ab_prev, *head; 4037 l2arc_buf_hdr_t *hdrl2; 4038 list_t *list; 4039 uint64_t passed_sz, write_sz, buf_sz, headroom; 4040 void *buf_data; 4041 kmutex_t *hash_lock, *list_lock; 4042 boolean_t have_lock, full; 4043 l2arc_write_callback_t *cb; 4044 zio_t *pio, *wzio; 4045 4046 ASSERT(dev->l2ad_vdev != NULL); 4047 4048 pio = NULL; 4049 write_sz = 0; 4050 full = B_FALSE; 4051 head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE); 4052 head->b_flags |= ARC_L2_WRITE_HEAD; 4053 4054 /* 4055 * Copy buffers for L2ARC writing. 4056 */ 4057 mutex_enter(&l2arc_buflist_mtx); 4058 for (int try = 0; try <= 3; try++) { 4059 list = l2arc_list_locked(try, &list_lock); 4060 passed_sz = 0; 4061 4062 /* 4063 * L2ARC fast warmup. 4064 * 4065 * Until the ARC is warm and starts to evict, read from the 4066 * head of the ARC lists rather than the tail. 4067 */ 4068 headroom = target_sz * l2arc_headroom; 4069 if (arc_warm == B_FALSE) 4070 ab = list_head(list); 4071 else 4072 ab = list_tail(list); 4073 4074 for (; ab; ab = ab_prev) { 4075 if (arc_warm == B_FALSE) 4076 ab_prev = list_next(list, ab); 4077 else 4078 ab_prev = list_prev(list, ab); 4079 4080 hash_lock = HDR_LOCK(ab); 4081 have_lock = MUTEX_HELD(hash_lock); 4082 if (!have_lock && !mutex_tryenter(hash_lock)) { 4083 /* 4084 * Skip this buffer rather than waiting. 4085 */ 4086 continue; 4087 } 4088 4089 passed_sz += ab->b_size; 4090 if (passed_sz > headroom) { 4091 /* 4092 * Searched too far. 4093 */ 4094 mutex_exit(hash_lock); 4095 break; 4096 } 4097 4098 if (ab->b_spa != spa) { 4099 mutex_exit(hash_lock); 4100 continue; 4101 } 4102 4103 if (ab->b_l2hdr != NULL) { 4104 /* 4105 * Already in L2ARC. 4106 */ 4107 mutex_exit(hash_lock); 4108 continue; 4109 } 4110 4111 if (HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) { 4112 mutex_exit(hash_lock); 4113 continue; 4114 } 4115 4116 if ((write_sz + ab->b_size) > target_sz) { 4117 full = B_TRUE; 4118 mutex_exit(hash_lock); 4119 break; 4120 } 4121 4122 if (ab->b_buf == NULL) { 4123 DTRACE_PROBE1(l2arc__buf__null, void *, ab); 4124 mutex_exit(hash_lock); 4125 continue; 4126 } 4127 4128 if (pio == NULL) { 4129 /* 4130 * Insert a dummy header on the buflist so 4131 * l2arc_write_done() can find where the 4132 * write buffers begin without searching. 4133 */ 4134 list_insert_head(dev->l2ad_buflist, head); 4135 4136 cb = kmem_alloc( 4137 sizeof (l2arc_write_callback_t), KM_SLEEP); 4138 cb->l2wcb_dev = dev; 4139 cb->l2wcb_head = head; 4140 pio = zio_root(spa, l2arc_write_done, cb, 4141 ZIO_FLAG_CANFAIL); 4142 } 4143 4144 /* 4145 * Create and add a new L2ARC header. 4146 */ 4147 hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP); 4148 hdrl2->b_dev = dev; 4149 hdrl2->b_daddr = dev->l2ad_hand; 4150 4151 ab->b_flags |= ARC_L2_WRITING; 4152 ab->b_l2hdr = hdrl2; 4153 list_insert_head(dev->l2ad_buflist, ab); 4154 buf_data = ab->b_buf->b_data; 4155 buf_sz = ab->b_size; 4156 4157 /* 4158 * Compute and store the buffer cksum before 4159 * writing. On debug the cksum is verified first. 4160 */ 4161 arc_cksum_verify(ab->b_buf); 4162 arc_cksum_compute(ab->b_buf, B_TRUE); 4163 4164 mutex_exit(hash_lock); 4165 4166 wzio = zio_write_phys(pio, dev->l2ad_vdev, 4167 dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF, 4168 NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, 4169 ZIO_FLAG_CANFAIL, B_FALSE); 4170 4171 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 4172 zio_t *, wzio); 4173 (void) zio_nowait(wzio); 4174 4175 write_sz += buf_sz; 4176 dev->l2ad_hand += buf_sz; 4177 } 4178 4179 mutex_exit(list_lock); 4180 4181 if (full == B_TRUE) 4182 break; 4183 } 4184 mutex_exit(&l2arc_buflist_mtx); 4185 4186 if (pio == NULL) { 4187 ASSERT3U(write_sz, ==, 0); 4188 kmem_cache_free(hdr_cache, head); 4189 return; 4190 } 4191 4192 ASSERT3U(write_sz, <=, target_sz); 4193 ARCSTAT_BUMP(arcstat_l2_writes_sent); 4194 ARCSTAT_INCR(arcstat_l2_size, write_sz); 4195 spa_l2cache_space_update(dev->l2ad_vdev, 0, write_sz); 4196 4197 /* 4198 * Bump device hand to the device start if it is approaching the end. 4199 * l2arc_evict() will already have evicted ahead for this case. 4200 */ 4201 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 4202 spa_l2cache_space_update(dev->l2ad_vdev, 0, 4203 dev->l2ad_end - dev->l2ad_hand); 4204 dev->l2ad_hand = dev->l2ad_start; 4205 dev->l2ad_evict = dev->l2ad_start; 4206 dev->l2ad_first = B_FALSE; 4207 } 4208 4209 (void) zio_wait(pio); 4210 } 4211 4212 /* 4213 * This thread feeds the L2ARC at regular intervals. This is the beating 4214 * heart of the L2ARC. 4215 */ 4216 static void 4217 l2arc_feed_thread(void) 4218 { 4219 callb_cpr_t cpr; 4220 l2arc_dev_t *dev; 4221 spa_t *spa; 4222 uint64_t size; 4223 4224 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 4225 4226 mutex_enter(&l2arc_feed_thr_lock); 4227 4228 while (l2arc_thread_exit == 0) { 4229 /* 4230 * Pause for l2arc_feed_secs seconds between writes. 4231 */ 4232 CALLB_CPR_SAFE_BEGIN(&cpr); 4233 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 4234 lbolt + (hz * l2arc_feed_secs)); 4235 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 4236 4237 /* 4238 * Quick check for L2ARC devices. 4239 */ 4240 mutex_enter(&l2arc_dev_mtx); 4241 if (l2arc_ndev == 0) { 4242 mutex_exit(&l2arc_dev_mtx); 4243 continue; 4244 } 4245 mutex_exit(&l2arc_dev_mtx); 4246 4247 /* 4248 * This selects the next l2arc device to write to, and in 4249 * doing so the next spa to feed from: dev->l2ad_spa. This 4250 * will return NULL if there are now no l2arc devices or if 4251 * they are all faulted. 4252 * 4253 * If a device is returned, its spa's config lock is also 4254 * held to prevent device removal. l2arc_dev_get_next() 4255 * will grab and release l2arc_dev_mtx. 4256 */ 4257 if ((dev = l2arc_dev_get_next()) == NULL) 4258 continue; 4259 4260 spa = dev->l2ad_spa; 4261 ASSERT(spa != NULL); 4262 4263 /* 4264 * Avoid contributing to memory pressure. 4265 */ 4266 if (arc_reclaim_needed()) { 4267 ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 4268 spa_config_exit(spa, dev); 4269 continue; 4270 } 4271 4272 ARCSTAT_BUMP(arcstat_l2_feeds); 4273 4274 size = dev->l2ad_write; 4275 if (arc_warm == B_FALSE) 4276 size += dev->l2ad_boost; 4277 4278 /* 4279 * Evict L2ARC buffers that will be overwritten. 4280 */ 4281 l2arc_evict(dev, size, B_FALSE); 4282 4283 /* 4284 * Write ARC buffers. 4285 */ 4286 l2arc_write_buffers(spa, dev, size); 4287 spa_config_exit(spa, dev); 4288 } 4289 4290 l2arc_thread_exit = 0; 4291 cv_broadcast(&l2arc_feed_thr_cv); 4292 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 4293 thread_exit(); 4294 } 4295 4296 boolean_t 4297 l2arc_vdev_present(vdev_t *vd) 4298 { 4299 l2arc_dev_t *dev; 4300 4301 mutex_enter(&l2arc_dev_mtx); 4302 for (dev = list_head(l2arc_dev_list); dev != NULL; 4303 dev = list_next(l2arc_dev_list, dev)) { 4304 if (dev->l2ad_vdev == vd) 4305 break; 4306 } 4307 mutex_exit(&l2arc_dev_mtx); 4308 4309 return (dev != NULL); 4310 } 4311 4312 /* 4313 * Add a vdev for use by the L2ARC. By this point the spa has already 4314 * validated the vdev and opened it. 4315 */ 4316 void 4317 l2arc_add_vdev(spa_t *spa, vdev_t *vd, uint64_t start, uint64_t end) 4318 { 4319 l2arc_dev_t *adddev; 4320 4321 ASSERT(!l2arc_vdev_present(vd)); 4322 4323 /* 4324 * Create a new l2arc device entry. 4325 */ 4326 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 4327 adddev->l2ad_spa = spa; 4328 adddev->l2ad_vdev = vd; 4329 adddev->l2ad_write = l2arc_write_max; 4330 adddev->l2ad_boost = l2arc_write_boost; 4331 adddev->l2ad_start = start; 4332 adddev->l2ad_end = end; 4333 adddev->l2ad_hand = adddev->l2ad_start; 4334 adddev->l2ad_evict = adddev->l2ad_start; 4335 adddev->l2ad_first = B_TRUE; 4336 ASSERT3U(adddev->l2ad_write, >, 0); 4337 4338 /* 4339 * This is a list of all ARC buffers that are still valid on the 4340 * device. 4341 */ 4342 adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP); 4343 list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 4344 offsetof(arc_buf_hdr_t, b_l2node)); 4345 4346 spa_l2cache_space_update(vd, adddev->l2ad_end - adddev->l2ad_hand, 0); 4347 4348 /* 4349 * Add device to global list 4350 */ 4351 mutex_enter(&l2arc_dev_mtx); 4352 list_insert_head(l2arc_dev_list, adddev); 4353 atomic_inc_64(&l2arc_ndev); 4354 mutex_exit(&l2arc_dev_mtx); 4355 } 4356 4357 /* 4358 * Remove a vdev from the L2ARC. 4359 */ 4360 void 4361 l2arc_remove_vdev(vdev_t *vd) 4362 { 4363 l2arc_dev_t *dev, *nextdev, *remdev = NULL; 4364 4365 /* 4366 * Find the device by vdev 4367 */ 4368 mutex_enter(&l2arc_dev_mtx); 4369 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 4370 nextdev = list_next(l2arc_dev_list, dev); 4371 if (vd == dev->l2ad_vdev) { 4372 remdev = dev; 4373 break; 4374 } 4375 } 4376 ASSERT(remdev != NULL); 4377 4378 /* 4379 * Remove device from global list 4380 */ 4381 list_remove(l2arc_dev_list, remdev); 4382 l2arc_dev_last = NULL; /* may have been invalidated */ 4383 atomic_dec_64(&l2arc_ndev); 4384 mutex_exit(&l2arc_dev_mtx); 4385 4386 /* 4387 * Clear all buflists and ARC references. L2ARC device flush. 4388 */ 4389 l2arc_evict(remdev, 0, B_TRUE); 4390 list_destroy(remdev->l2ad_buflist); 4391 kmem_free(remdev->l2ad_buflist, sizeof (list_t)); 4392 kmem_free(remdev, sizeof (l2arc_dev_t)); 4393 } 4394 4395 void 4396 l2arc_init() 4397 { 4398 l2arc_thread_exit = 0; 4399 l2arc_ndev = 0; 4400 l2arc_writes_sent = 0; 4401 l2arc_writes_done = 0; 4402 4403 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 4404 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 4405 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 4406 mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL); 4407 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 4408 4409 l2arc_dev_list = &L2ARC_dev_list; 4410 l2arc_free_on_write = &L2ARC_free_on_write; 4411 list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 4412 offsetof(l2arc_dev_t, l2ad_node)); 4413 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 4414 offsetof(l2arc_data_free_t, l2df_list_node)); 4415 4416 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 4417 TS_RUN, minclsyspri); 4418 } 4419 4420 void 4421 l2arc_fini() 4422 { 4423 /* 4424 * This is called from dmu_fini(), which is called from spa_fini(); 4425 * Because of this, we can assume that all l2arc devices have 4426 * already been removed when the pools themselves were removed. 4427 */ 4428 4429 mutex_enter(&l2arc_feed_thr_lock); 4430 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 4431 l2arc_thread_exit = 1; 4432 while (l2arc_thread_exit != 0) 4433 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 4434 mutex_exit(&l2arc_feed_thr_lock); 4435 4436 l2arc_do_free_on_write(); 4437 4438 mutex_destroy(&l2arc_feed_thr_lock); 4439 cv_destroy(&l2arc_feed_thr_cv); 4440 mutex_destroy(&l2arc_dev_mtx); 4441 mutex_destroy(&l2arc_buflist_mtx); 4442 mutex_destroy(&l2arc_free_on_write_mtx); 4443 4444 list_destroy(l2arc_dev_list); 4445 list_destroy(l2arc_free_on_write); 4446 } 4447