1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012, Joyent, Inc. All rights reserved. 24 * Copyright (c) 2011, 2017 by Delphix. All rights reserved. 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. 26 * Copyright 2015 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 /* 30 * DVA-based Adjustable Replacement Cache 31 * 32 * While much of the theory of operation used here is 33 * based on the self-tuning, low overhead replacement cache 34 * presented by Megiddo and Modha at FAST 2003, there are some 35 * significant differences: 36 * 37 * 1. The Megiddo and Modha model assumes any page is evictable. 38 * Pages in its cache cannot be "locked" into memory. This makes 39 * the eviction algorithm simple: evict the last page in the list. 40 * This also make the performance characteristics easy to reason 41 * about. Our cache is not so simple. At any given moment, some 42 * subset of the blocks in the cache are un-evictable because we 43 * have handed out a reference to them. Blocks are only evictable 44 * when there are no external references active. This makes 45 * eviction far more problematic: we choose to evict the evictable 46 * blocks that are the "lowest" in the list. 47 * 48 * There are times when it is not possible to evict the requested 49 * space. In these circumstances we are unable to adjust the cache 50 * size. To prevent the cache growing unbounded at these times we 51 * implement a "cache throttle" that slows the flow of new data 52 * into the cache until we can make space available. 53 * 54 * 2. The Megiddo and Modha model assumes a fixed cache size. 55 * Pages are evicted when the cache is full and there is a cache 56 * miss. Our model has a variable sized cache. It grows with 57 * high use, but also tries to react to memory pressure from the 58 * operating system: decreasing its size when system memory is 59 * tight. 60 * 61 * 3. The Megiddo and Modha model assumes a fixed page size. All 62 * elements of the cache are therefore exactly the same size. So 63 * when adjusting the cache size following a cache miss, its simply 64 * a matter of choosing a single page to evict. In our model, we 65 * have variable sized cache blocks (rangeing from 512 bytes to 66 * 128K bytes). We therefore choose a set of blocks to evict to make 67 * space for a cache miss that approximates as closely as possible 68 * the space used by the new block. 69 * 70 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 71 * by N. Megiddo & D. Modha, FAST 2003 72 */ 73 74 /* 75 * The locking model: 76 * 77 * A new reference to a cache buffer can be obtained in two 78 * ways: 1) via a hash table lookup using the DVA as a key, 79 * or 2) via one of the ARC lists. The arc_read() interface 80 * uses method 1, while the internal ARC algorithms for 81 * adjusting the cache use method 2. We therefore provide two 82 * types of locks: 1) the hash table lock array, and 2) the 83 * ARC list locks. 84 * 85 * Buffers do not have their own mutexes, rather they rely on the 86 * hash table mutexes for the bulk of their protection (i.e. most 87 * fields in the arc_buf_hdr_t are protected by these mutexes). 88 * 89 * buf_hash_find() returns the appropriate mutex (held) when it 90 * locates the requested buffer in the hash table. It returns 91 * NULL for the mutex if the buffer was not in the table. 92 * 93 * buf_hash_remove() expects the appropriate hash mutex to be 94 * already held before it is invoked. 95 * 96 * Each ARC state also has a mutex which is used to protect the 97 * buffer list associated with the state. When attempting to 98 * obtain a hash table lock while holding an ARC list lock you 99 * must use: mutex_tryenter() to avoid deadlock. Also note that 100 * the active state mutex must be held before the ghost state mutex. 101 * 102 * Note that the majority of the performance stats are manipulated 103 * with atomic operations. 104 * 105 * The L2ARC uses the l2ad_mtx on each vdev for the following: 106 * 107 * - L2ARC buflist creation 108 * - L2ARC buflist eviction 109 * - L2ARC write completion, which walks L2ARC buflists 110 * - ARC header destruction, as it removes from L2ARC buflists 111 * - ARC header release, as it removes from L2ARC buflists 112 */ 113 114 /* 115 * ARC operation: 116 * 117 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure. 118 * This structure can point either to a block that is still in the cache or to 119 * one that is only accessible in an L2 ARC device, or it can provide 120 * information about a block that was recently evicted. If a block is 121 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough 122 * information to retrieve it from the L2ARC device. This information is 123 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block 124 * that is in this state cannot access the data directly. 125 * 126 * Blocks that are actively being referenced or have not been evicted 127 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within 128 * the arc_buf_hdr_t that will point to the data block in memory. A block can 129 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC 130 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and 131 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd). 132 * 133 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the 134 * ability to store the physical data (b_pabd) associated with the DVA of the 135 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block, 136 * it will match its on-disk compression characteristics. This behavior can be 137 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the 138 * compressed ARC functionality is disabled, the b_pabd will point to an 139 * uncompressed version of the on-disk data. 140 * 141 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each 142 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it. 143 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC 144 * consumer. The ARC will provide references to this data and will keep it 145 * cached until it is no longer in use. The ARC caches only the L1ARC's physical 146 * data block and will evict any arc_buf_t that is no longer referenced. The 147 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the 148 * "overhead_size" kstat. 149 * 150 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or 151 * compressed form. The typical case is that consumers will want uncompressed 152 * data, and when that happens a new data buffer is allocated where the data is 153 * decompressed for them to use. Currently the only consumer who wants 154 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it 155 * exists on disk. When this happens, the arc_buf_t's data buffer is shared 156 * with the arc_buf_hdr_t. 157 * 158 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The 159 * first one is owned by a compressed send consumer (and therefore references 160 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be 161 * used by any other consumer (and has its own uncompressed copy of the data 162 * buffer). 163 * 164 * arc_buf_hdr_t 165 * +-----------+ 166 * | fields | 167 * | common to | 168 * | L1- and | 169 * | L2ARC | 170 * +-----------+ 171 * | l2arc_buf_hdr_t 172 * | | 173 * +-----------+ 174 * | l1arc_buf_hdr_t 175 * | | arc_buf_t 176 * | b_buf +------------>+-----------+ arc_buf_t 177 * | b_pabd +-+ |b_next +---->+-----------+ 178 * +-----------+ | |-----------| |b_next +-->NULL 179 * | |b_comp = T | +-----------+ 180 * | |b_data +-+ |b_comp = F | 181 * | +-----------+ | |b_data +-+ 182 * +->+------+ | +-----------+ | 183 * compressed | | | | 184 * data | |<--------------+ | uncompressed 185 * +------+ compressed, | data 186 * shared +-->+------+ 187 * data | | 188 * | | 189 * +------+ 190 * 191 * When a consumer reads a block, the ARC must first look to see if the 192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new 193 * arc_buf_t and either copies uncompressed data into a new data buffer from an 194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a 195 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the 196 * hdr is compressed and the desired compression characteristics of the 197 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the 198 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be 199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can 200 * be anywhere in the hdr's list. 201 * 202 * The diagram below shows an example of an uncompressed ARC hdr that is 203 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is 204 * the last element in the buf list): 205 * 206 * arc_buf_hdr_t 207 * +-----------+ 208 * | | 209 * | | 210 * | | 211 * +-----------+ 212 * l2arc_buf_hdr_t| | 213 * | | 214 * +-----------+ 215 * l1arc_buf_hdr_t| | 216 * | | arc_buf_t (shared) 217 * | b_buf +------------>+---------+ arc_buf_t 218 * | | |b_next +---->+---------+ 219 * | b_pabd +-+ |---------| |b_next +-->NULL 220 * +-----------+ | | | +---------+ 221 * | |b_data +-+ | | 222 * | +---------+ | |b_data +-+ 223 * +->+------+ | +---------+ | 224 * | | | | 225 * uncompressed | | | | 226 * data +------+ | | 227 * ^ +->+------+ | 228 * | uncompressed | | | 229 * | data | | | 230 * | +------+ | 231 * +---------------------------------+ 232 * 233 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd 234 * since the physical block is about to be rewritten. The new data contents 235 * will be contained in the arc_buf_t. As the I/O pipeline performs the write, 236 * it may compress the data before writing it to disk. The ARC will be called 237 * with the transformed data and will bcopy the transformed on-disk block into 238 * a newly allocated b_pabd. Writes are always done into buffers which have 239 * either been loaned (and hence are new and don't have other readers) or 240 * buffers which have been released (and hence have their own hdr, if there 241 * were originally other readers of the buf's original hdr). This ensures that 242 * the ARC only needs to update a single buf and its hdr after a write occurs. 243 * 244 * When the L2ARC is in use, it will also take advantage of the b_pabd. The 245 * L2ARC will always write the contents of b_pabd to the L2ARC. This means 246 * that when compressed ARC is enabled that the L2ARC blocks are identical 247 * to the on-disk block in the main data pool. This provides a significant 248 * advantage since the ARC can leverage the bp's checksum when reading from the 249 * L2ARC to determine if the contents are valid. However, if the compressed 250 * ARC is disabled, then the L2ARC's block must be transformed to look 251 * like the physical block in the main data pool before comparing the 252 * checksum and determining its validity. 253 */ 254 255 #include <sys/spa.h> 256 #include <sys/zio.h> 257 #include <sys/spa_impl.h> 258 #include <sys/zio_compress.h> 259 #include <sys/zio_checksum.h> 260 #include <sys/zfs_context.h> 261 #include <sys/arc.h> 262 #include <sys/refcount.h> 263 #include <sys/vdev.h> 264 #include <sys/vdev_impl.h> 265 #include <sys/dsl_pool.h> 266 #include <sys/zio_checksum.h> 267 #include <sys/multilist.h> 268 #include <sys/abd.h> 269 #ifdef _KERNEL 270 #include <sys/vmsystm.h> 271 #include <vm/anon.h> 272 #include <sys/fs/swapnode.h> 273 #include <sys/dnlc.h> 274 #endif 275 #include <sys/callb.h> 276 #include <sys/kstat.h> 277 #include <zfs_fletcher.h> 278 279 #ifndef _KERNEL 280 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */ 281 boolean_t arc_watch = B_FALSE; 282 int arc_procfd; 283 #endif 284 285 static kmutex_t arc_reclaim_lock; 286 static kcondvar_t arc_reclaim_thread_cv; 287 static boolean_t arc_reclaim_thread_exit; 288 static kcondvar_t arc_reclaim_waiters_cv; 289 290 uint_t arc_reduce_dnlc_percent = 3; 291 292 /* 293 * The number of headers to evict in arc_evict_state_impl() before 294 * dropping the sublist lock and evicting from another sublist. A lower 295 * value means we're more likely to evict the "correct" header (i.e. the 296 * oldest header in the arc state), but comes with higher overhead 297 * (i.e. more invocations of arc_evict_state_impl()). 298 */ 299 int zfs_arc_evict_batch_limit = 10; 300 301 /* number of seconds before growing cache again */ 302 static int arc_grow_retry = 60; 303 304 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */ 305 int zfs_arc_overflow_shift = 8; 306 307 /* shift of arc_c for calculating both min and max arc_p */ 308 static int arc_p_min_shift = 4; 309 310 /* log2(fraction of arc to reclaim) */ 311 static int arc_shrink_shift = 7; 312 313 /* 314 * log2(fraction of ARC which must be free to allow growing). 315 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory, 316 * when reading a new block into the ARC, we will evict an equal-sized block 317 * from the ARC. 318 * 319 * This must be less than arc_shrink_shift, so that when we shrink the ARC, 320 * we will still not allow it to grow. 321 */ 322 int arc_no_grow_shift = 5; 323 324 325 /* 326 * minimum lifespan of a prefetch block in clock ticks 327 * (initialized in arc_init()) 328 */ 329 static int arc_min_prefetch_lifespan; 330 331 /* 332 * If this percent of memory is free, don't throttle. 333 */ 334 int arc_lotsfree_percent = 10; 335 336 static int arc_dead; 337 338 /* 339 * The arc has filled available memory and has now warmed up. 340 */ 341 static boolean_t arc_warm; 342 343 /* 344 * log2 fraction of the zio arena to keep free. 345 */ 346 int arc_zio_arena_free_shift = 2; 347 348 /* 349 * These tunables are for performance analysis. 350 */ 351 uint64_t zfs_arc_max; 352 uint64_t zfs_arc_min; 353 uint64_t zfs_arc_meta_limit = 0; 354 uint64_t zfs_arc_meta_min = 0; 355 int zfs_arc_grow_retry = 0; 356 int zfs_arc_shrink_shift = 0; 357 int zfs_arc_p_min_shift = 0; 358 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ 359 360 boolean_t zfs_compressed_arc_enabled = B_TRUE; 361 362 /* 363 * Note that buffers can be in one of 6 states: 364 * ARC_anon - anonymous (discussed below) 365 * ARC_mru - recently used, currently cached 366 * ARC_mru_ghost - recentely used, no longer in cache 367 * ARC_mfu - frequently used, currently cached 368 * ARC_mfu_ghost - frequently used, no longer in cache 369 * ARC_l2c_only - exists in L2ARC but not other states 370 * When there are no active references to the buffer, they are 371 * are linked onto a list in one of these arc states. These are 372 * the only buffers that can be evicted or deleted. Within each 373 * state there are multiple lists, one for meta-data and one for 374 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 375 * etc.) is tracked separately so that it can be managed more 376 * explicitly: favored over data, limited explicitly. 377 * 378 * Anonymous buffers are buffers that are not associated with 379 * a DVA. These are buffers that hold dirty block copies 380 * before they are written to stable storage. By definition, 381 * they are "ref'd" and are considered part of arc_mru 382 * that cannot be freed. Generally, they will aquire a DVA 383 * as they are written and migrate onto the arc_mru list. 384 * 385 * The ARC_l2c_only state is for buffers that are in the second 386 * level ARC but no longer in any of the ARC_m* lists. The second 387 * level ARC itself may also contain buffers that are in any of 388 * the ARC_m* states - meaning that a buffer can exist in two 389 * places. The reason for the ARC_l2c_only state is to keep the 390 * buffer header in the hash table, so that reads that hit the 391 * second level ARC benefit from these fast lookups. 392 */ 393 394 typedef struct arc_state { 395 /* 396 * list of evictable buffers 397 */ 398 multilist_t *arcs_list[ARC_BUFC_NUMTYPES]; 399 /* 400 * total amount of evictable data in this state 401 */ 402 refcount_t arcs_esize[ARC_BUFC_NUMTYPES]; 403 /* 404 * total amount of data in this state; this includes: evictable, 405 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA. 406 */ 407 refcount_t arcs_size; 408 } arc_state_t; 409 410 /* The 6 states: */ 411 static arc_state_t ARC_anon; 412 static arc_state_t ARC_mru; 413 static arc_state_t ARC_mru_ghost; 414 static arc_state_t ARC_mfu; 415 static arc_state_t ARC_mfu_ghost; 416 static arc_state_t ARC_l2c_only; 417 418 typedef struct arc_stats { 419 kstat_named_t arcstat_hits; 420 kstat_named_t arcstat_misses; 421 kstat_named_t arcstat_demand_data_hits; 422 kstat_named_t arcstat_demand_data_misses; 423 kstat_named_t arcstat_demand_metadata_hits; 424 kstat_named_t arcstat_demand_metadata_misses; 425 kstat_named_t arcstat_prefetch_data_hits; 426 kstat_named_t arcstat_prefetch_data_misses; 427 kstat_named_t arcstat_prefetch_metadata_hits; 428 kstat_named_t arcstat_prefetch_metadata_misses; 429 kstat_named_t arcstat_mru_hits; 430 kstat_named_t arcstat_mru_ghost_hits; 431 kstat_named_t arcstat_mfu_hits; 432 kstat_named_t arcstat_mfu_ghost_hits; 433 kstat_named_t arcstat_deleted; 434 /* 435 * Number of buffers that could not be evicted because the hash lock 436 * was held by another thread. The lock may not necessarily be held 437 * by something using the same buffer, since hash locks are shared 438 * by multiple buffers. 439 */ 440 kstat_named_t arcstat_mutex_miss; 441 /* 442 * Number of buffers skipped because they have I/O in progress, are 443 * indrect prefetch buffers that have not lived long enough, or are 444 * not from the spa we're trying to evict from. 445 */ 446 kstat_named_t arcstat_evict_skip; 447 /* 448 * Number of times arc_evict_state() was unable to evict enough 449 * buffers to reach it's target amount. 450 */ 451 kstat_named_t arcstat_evict_not_enough; 452 kstat_named_t arcstat_evict_l2_cached; 453 kstat_named_t arcstat_evict_l2_eligible; 454 kstat_named_t arcstat_evict_l2_ineligible; 455 kstat_named_t arcstat_evict_l2_skip; 456 kstat_named_t arcstat_hash_elements; 457 kstat_named_t arcstat_hash_elements_max; 458 kstat_named_t arcstat_hash_collisions; 459 kstat_named_t arcstat_hash_chains; 460 kstat_named_t arcstat_hash_chain_max; 461 kstat_named_t arcstat_p; 462 kstat_named_t arcstat_c; 463 kstat_named_t arcstat_c_min; 464 kstat_named_t arcstat_c_max; 465 kstat_named_t arcstat_size; 466 /* 467 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd. 468 * Note that the compressed bytes may match the uncompressed bytes 469 * if the block is either not compressed or compressed arc is disabled. 470 */ 471 kstat_named_t arcstat_compressed_size; 472 /* 473 * Uncompressed size of the data stored in b_pabd. If compressed 474 * arc is disabled then this value will be identical to the stat 475 * above. 476 */ 477 kstat_named_t arcstat_uncompressed_size; 478 /* 479 * Number of bytes stored in all the arc_buf_t's. This is classified 480 * as "overhead" since this data is typically short-lived and will 481 * be evicted from the arc when it becomes unreferenced unless the 482 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level 483 * values have been set (see comment in dbuf.c for more information). 484 */ 485 kstat_named_t arcstat_overhead_size; 486 /* 487 * Number of bytes consumed by internal ARC structures necessary 488 * for tracking purposes; these structures are not actually 489 * backed by ARC buffers. This includes arc_buf_hdr_t structures 490 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only 491 * caches), and arc_buf_t structures (allocated via arc_buf_t 492 * cache). 493 */ 494 kstat_named_t arcstat_hdr_size; 495 /* 496 * Number of bytes consumed by ARC buffers of type equal to 497 * ARC_BUFC_DATA. This is generally consumed by buffers backing 498 * on disk user data (e.g. plain file contents). 499 */ 500 kstat_named_t arcstat_data_size; 501 /* 502 * Number of bytes consumed by ARC buffers of type equal to 503 * ARC_BUFC_METADATA. This is generally consumed by buffers 504 * backing on disk data that is used for internal ZFS 505 * structures (e.g. ZAP, dnode, indirect blocks, etc). 506 */ 507 kstat_named_t arcstat_metadata_size; 508 /* 509 * Number of bytes consumed by various buffers and structures 510 * not actually backed with ARC buffers. This includes bonus 511 * buffers (allocated directly via zio_buf_* functions), 512 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t 513 * cache), and dnode_t structures (allocated via dnode_t cache). 514 */ 515 kstat_named_t arcstat_other_size; 516 /* 517 * Total number of bytes consumed by ARC buffers residing in the 518 * arc_anon state. This includes *all* buffers in the arc_anon 519 * state; e.g. data, metadata, evictable, and unevictable buffers 520 * are all included in this value. 521 */ 522 kstat_named_t arcstat_anon_size; 523 /* 524 * Number of bytes consumed by ARC buffers that meet the 525 * following criteria: backing buffers of type ARC_BUFC_DATA, 526 * residing in the arc_anon state, and are eligible for eviction 527 * (e.g. have no outstanding holds on the buffer). 528 */ 529 kstat_named_t arcstat_anon_evictable_data; 530 /* 531 * Number of bytes consumed by ARC buffers that meet the 532 * following criteria: backing buffers of type ARC_BUFC_METADATA, 533 * residing in the arc_anon state, and are eligible for eviction 534 * (e.g. have no outstanding holds on the buffer). 535 */ 536 kstat_named_t arcstat_anon_evictable_metadata; 537 /* 538 * Total number of bytes consumed by ARC buffers residing in the 539 * arc_mru state. This includes *all* buffers in the arc_mru 540 * state; e.g. data, metadata, evictable, and unevictable buffers 541 * are all included in this value. 542 */ 543 kstat_named_t arcstat_mru_size; 544 /* 545 * Number of bytes consumed by ARC buffers that meet the 546 * following criteria: backing buffers of type ARC_BUFC_DATA, 547 * residing in the arc_mru state, and are eligible for eviction 548 * (e.g. have no outstanding holds on the buffer). 549 */ 550 kstat_named_t arcstat_mru_evictable_data; 551 /* 552 * Number of bytes consumed by ARC buffers that meet the 553 * following criteria: backing buffers of type ARC_BUFC_METADATA, 554 * residing in the arc_mru state, and are eligible for eviction 555 * (e.g. have no outstanding holds on the buffer). 556 */ 557 kstat_named_t arcstat_mru_evictable_metadata; 558 /* 559 * Total number of bytes that *would have been* consumed by ARC 560 * buffers in the arc_mru_ghost state. The key thing to note 561 * here, is the fact that this size doesn't actually indicate 562 * RAM consumption. The ghost lists only consist of headers and 563 * don't actually have ARC buffers linked off of these headers. 564 * Thus, *if* the headers had associated ARC buffers, these 565 * buffers *would have* consumed this number of bytes. 566 */ 567 kstat_named_t arcstat_mru_ghost_size; 568 /* 569 * Number of bytes that *would have been* consumed by ARC 570 * buffers that are eligible for eviction, of type 571 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state. 572 */ 573 kstat_named_t arcstat_mru_ghost_evictable_data; 574 /* 575 * Number of bytes that *would have been* consumed by ARC 576 * buffers that are eligible for eviction, of type 577 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 578 */ 579 kstat_named_t arcstat_mru_ghost_evictable_metadata; 580 /* 581 * Total number of bytes consumed by ARC buffers residing in the 582 * arc_mfu state. This includes *all* buffers in the arc_mfu 583 * state; e.g. data, metadata, evictable, and unevictable buffers 584 * are all included in this value. 585 */ 586 kstat_named_t arcstat_mfu_size; 587 /* 588 * Number of bytes consumed by ARC buffers that are eligible for 589 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu 590 * state. 591 */ 592 kstat_named_t arcstat_mfu_evictable_data; 593 /* 594 * Number of bytes consumed by ARC buffers that are eligible for 595 * eviction, of type ARC_BUFC_METADATA, and reside in the 596 * arc_mfu state. 597 */ 598 kstat_named_t arcstat_mfu_evictable_metadata; 599 /* 600 * Total number of bytes that *would have been* consumed by ARC 601 * buffers in the arc_mfu_ghost state. See the comment above 602 * arcstat_mru_ghost_size for more details. 603 */ 604 kstat_named_t arcstat_mfu_ghost_size; 605 /* 606 * Number of bytes that *would have been* consumed by ARC 607 * buffers that are eligible for eviction, of type 608 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state. 609 */ 610 kstat_named_t arcstat_mfu_ghost_evictable_data; 611 /* 612 * Number of bytes that *would have been* consumed by ARC 613 * buffers that are eligible for eviction, of type 614 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 615 */ 616 kstat_named_t arcstat_mfu_ghost_evictable_metadata; 617 kstat_named_t arcstat_l2_hits; 618 kstat_named_t arcstat_l2_misses; 619 kstat_named_t arcstat_l2_feeds; 620 kstat_named_t arcstat_l2_rw_clash; 621 kstat_named_t arcstat_l2_read_bytes; 622 kstat_named_t arcstat_l2_write_bytes; 623 kstat_named_t arcstat_l2_writes_sent; 624 kstat_named_t arcstat_l2_writes_done; 625 kstat_named_t arcstat_l2_writes_error; 626 kstat_named_t arcstat_l2_writes_lock_retry; 627 kstat_named_t arcstat_l2_evict_lock_retry; 628 kstat_named_t arcstat_l2_evict_reading; 629 kstat_named_t arcstat_l2_evict_l1cached; 630 kstat_named_t arcstat_l2_free_on_write; 631 kstat_named_t arcstat_l2_abort_lowmem; 632 kstat_named_t arcstat_l2_cksum_bad; 633 kstat_named_t arcstat_l2_io_error; 634 kstat_named_t arcstat_l2_lsize; 635 kstat_named_t arcstat_l2_psize; 636 kstat_named_t arcstat_l2_hdr_size; 637 kstat_named_t arcstat_memory_throttle_count; 638 kstat_named_t arcstat_meta_used; 639 kstat_named_t arcstat_meta_limit; 640 kstat_named_t arcstat_meta_max; 641 kstat_named_t arcstat_meta_min; 642 kstat_named_t arcstat_sync_wait_for_async; 643 kstat_named_t arcstat_demand_hit_predictive_prefetch; 644 } arc_stats_t; 645 646 static arc_stats_t arc_stats = { 647 { "hits", KSTAT_DATA_UINT64 }, 648 { "misses", KSTAT_DATA_UINT64 }, 649 { "demand_data_hits", KSTAT_DATA_UINT64 }, 650 { "demand_data_misses", KSTAT_DATA_UINT64 }, 651 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 652 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 653 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 654 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 655 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 656 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 657 { "mru_hits", KSTAT_DATA_UINT64 }, 658 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 659 { "mfu_hits", KSTAT_DATA_UINT64 }, 660 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 661 { "deleted", KSTAT_DATA_UINT64 }, 662 { "mutex_miss", KSTAT_DATA_UINT64 }, 663 { "evict_skip", KSTAT_DATA_UINT64 }, 664 { "evict_not_enough", KSTAT_DATA_UINT64 }, 665 { "evict_l2_cached", KSTAT_DATA_UINT64 }, 666 { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 667 { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 668 { "evict_l2_skip", KSTAT_DATA_UINT64 }, 669 { "hash_elements", KSTAT_DATA_UINT64 }, 670 { "hash_elements_max", KSTAT_DATA_UINT64 }, 671 { "hash_collisions", KSTAT_DATA_UINT64 }, 672 { "hash_chains", KSTAT_DATA_UINT64 }, 673 { "hash_chain_max", KSTAT_DATA_UINT64 }, 674 { "p", KSTAT_DATA_UINT64 }, 675 { "c", KSTAT_DATA_UINT64 }, 676 { "c_min", KSTAT_DATA_UINT64 }, 677 { "c_max", KSTAT_DATA_UINT64 }, 678 { "size", KSTAT_DATA_UINT64 }, 679 { "compressed_size", KSTAT_DATA_UINT64 }, 680 { "uncompressed_size", KSTAT_DATA_UINT64 }, 681 { "overhead_size", KSTAT_DATA_UINT64 }, 682 { "hdr_size", KSTAT_DATA_UINT64 }, 683 { "data_size", KSTAT_DATA_UINT64 }, 684 { "metadata_size", KSTAT_DATA_UINT64 }, 685 { "other_size", KSTAT_DATA_UINT64 }, 686 { "anon_size", KSTAT_DATA_UINT64 }, 687 { "anon_evictable_data", KSTAT_DATA_UINT64 }, 688 { "anon_evictable_metadata", KSTAT_DATA_UINT64 }, 689 { "mru_size", KSTAT_DATA_UINT64 }, 690 { "mru_evictable_data", KSTAT_DATA_UINT64 }, 691 { "mru_evictable_metadata", KSTAT_DATA_UINT64 }, 692 { "mru_ghost_size", KSTAT_DATA_UINT64 }, 693 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 }, 694 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 695 { "mfu_size", KSTAT_DATA_UINT64 }, 696 { "mfu_evictable_data", KSTAT_DATA_UINT64 }, 697 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 }, 698 { "mfu_ghost_size", KSTAT_DATA_UINT64 }, 699 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 }, 700 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 701 { "l2_hits", KSTAT_DATA_UINT64 }, 702 { "l2_misses", KSTAT_DATA_UINT64 }, 703 { "l2_feeds", KSTAT_DATA_UINT64 }, 704 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 705 { "l2_read_bytes", KSTAT_DATA_UINT64 }, 706 { "l2_write_bytes", KSTAT_DATA_UINT64 }, 707 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 708 { "l2_writes_done", KSTAT_DATA_UINT64 }, 709 { "l2_writes_error", KSTAT_DATA_UINT64 }, 710 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 }, 711 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 712 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 713 { "l2_evict_l1cached", KSTAT_DATA_UINT64 }, 714 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 715 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 716 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 717 { "l2_io_error", KSTAT_DATA_UINT64 }, 718 { "l2_size", KSTAT_DATA_UINT64 }, 719 { "l2_asize", KSTAT_DATA_UINT64 }, 720 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 721 { "memory_throttle_count", KSTAT_DATA_UINT64 }, 722 { "arc_meta_used", KSTAT_DATA_UINT64 }, 723 { "arc_meta_limit", KSTAT_DATA_UINT64 }, 724 { "arc_meta_max", KSTAT_DATA_UINT64 }, 725 { "arc_meta_min", KSTAT_DATA_UINT64 }, 726 { "sync_wait_for_async", KSTAT_DATA_UINT64 }, 727 { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, 728 }; 729 730 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 731 732 #define ARCSTAT_INCR(stat, val) \ 733 atomic_add_64(&arc_stats.stat.value.ui64, (val)) 734 735 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 736 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 737 738 #define ARCSTAT_MAX(stat, val) { \ 739 uint64_t m; \ 740 while ((val) > (m = arc_stats.stat.value.ui64) && \ 741 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 742 continue; \ 743 } 744 745 #define ARCSTAT_MAXSTAT(stat) \ 746 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 747 748 /* 749 * We define a macro to allow ARC hits/misses to be easily broken down by 750 * two separate conditions, giving a total of four different subtypes for 751 * each of hits and misses (so eight statistics total). 752 */ 753 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 754 if (cond1) { \ 755 if (cond2) { \ 756 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 757 } else { \ 758 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 759 } \ 760 } else { \ 761 if (cond2) { \ 762 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 763 } else { \ 764 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 765 } \ 766 } 767 768 kstat_t *arc_ksp; 769 static arc_state_t *arc_anon; 770 static arc_state_t *arc_mru; 771 static arc_state_t *arc_mru_ghost; 772 static arc_state_t *arc_mfu; 773 static arc_state_t *arc_mfu_ghost; 774 static arc_state_t *arc_l2c_only; 775 776 /* 777 * There are several ARC variables that are critical to export as kstats -- 778 * but we don't want to have to grovel around in the kstat whenever we wish to 779 * manipulate them. For these variables, we therefore define them to be in 780 * terms of the statistic variable. This assures that we are not introducing 781 * the possibility of inconsistency by having shadow copies of the variables, 782 * while still allowing the code to be readable. 783 */ 784 #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 785 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 786 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 787 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 788 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 789 #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ 790 #define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */ 791 #define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */ 792 #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ 793 794 /* compressed size of entire arc */ 795 #define arc_compressed_size ARCSTAT(arcstat_compressed_size) 796 /* uncompressed size of entire arc */ 797 #define arc_uncompressed_size ARCSTAT(arcstat_uncompressed_size) 798 /* number of bytes in the arc from arc_buf_t's */ 799 #define arc_overhead_size ARCSTAT(arcstat_overhead_size) 800 801 static int arc_no_grow; /* Don't try to grow cache size */ 802 static uint64_t arc_tempreserve; 803 static uint64_t arc_loaned_bytes; 804 805 typedef struct arc_callback arc_callback_t; 806 807 struct arc_callback { 808 void *acb_private; 809 arc_done_func_t *acb_done; 810 arc_buf_t *acb_buf; 811 boolean_t acb_compressed; 812 zio_t *acb_zio_dummy; 813 arc_callback_t *acb_next; 814 }; 815 816 typedef struct arc_write_callback arc_write_callback_t; 817 818 struct arc_write_callback { 819 void *awcb_private; 820 arc_done_func_t *awcb_ready; 821 arc_done_func_t *awcb_children_ready; 822 arc_done_func_t *awcb_physdone; 823 arc_done_func_t *awcb_done; 824 arc_buf_t *awcb_buf; 825 }; 826 827 /* 828 * ARC buffers are separated into multiple structs as a memory saving measure: 829 * - Common fields struct, always defined, and embedded within it: 830 * - L2-only fields, always allocated but undefined when not in L2ARC 831 * - L1-only fields, only allocated when in L1ARC 832 * 833 * Buffer in L1 Buffer only in L2 834 * +------------------------+ +------------------------+ 835 * | arc_buf_hdr_t | | arc_buf_hdr_t | 836 * | | | | 837 * | | | | 838 * | | | | 839 * +------------------------+ +------------------------+ 840 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t | 841 * | (undefined if L1-only) | | | 842 * +------------------------+ +------------------------+ 843 * | l1arc_buf_hdr_t | 844 * | | 845 * | | 846 * | | 847 * | | 848 * +------------------------+ 849 * 850 * Because it's possible for the L2ARC to become extremely large, we can wind 851 * up eating a lot of memory in L2ARC buffer headers, so the size of a header 852 * is minimized by only allocating the fields necessary for an L1-cached buffer 853 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and 854 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple 855 * words in pointers. arc_hdr_realloc() is used to switch a header between 856 * these two allocation states. 857 */ 858 typedef struct l1arc_buf_hdr { 859 kmutex_t b_freeze_lock; 860 zio_cksum_t *b_freeze_cksum; 861 #ifdef ZFS_DEBUG 862 /* 863 * Used for debugging with kmem_flags - by allocating and freeing 864 * b_thawed when the buffer is thawed, we get a record of the stack 865 * trace that thawed it. 866 */ 867 void *b_thawed; 868 #endif 869 870 arc_buf_t *b_buf; 871 uint32_t b_bufcnt; 872 /* for waiting on writes to complete */ 873 kcondvar_t b_cv; 874 uint8_t b_byteswap; 875 876 /* protected by arc state mutex */ 877 arc_state_t *b_state; 878 multilist_node_t b_arc_node; 879 880 /* updated atomically */ 881 clock_t b_arc_access; 882 883 /* self protecting */ 884 refcount_t b_refcnt; 885 886 arc_callback_t *b_acb; 887 abd_t *b_pabd; 888 } l1arc_buf_hdr_t; 889 890 typedef struct l2arc_dev l2arc_dev_t; 891 892 typedef struct l2arc_buf_hdr { 893 /* protected by arc_buf_hdr mutex */ 894 l2arc_dev_t *b_dev; /* L2ARC device */ 895 uint64_t b_daddr; /* disk address, offset byte */ 896 897 list_node_t b_l2node; 898 } l2arc_buf_hdr_t; 899 900 struct arc_buf_hdr { 901 /* protected by hash lock */ 902 dva_t b_dva; 903 uint64_t b_birth; 904 905 arc_buf_contents_t b_type; 906 arc_buf_hdr_t *b_hash_next; 907 arc_flags_t b_flags; 908 909 /* 910 * This field stores the size of the data buffer after 911 * compression, and is set in the arc's zio completion handlers. 912 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes). 913 * 914 * While the block pointers can store up to 32MB in their psize 915 * field, we can only store up to 32MB minus 512B. This is due 916 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e. 917 * a field of zeros represents 512B in the bp). We can't use a 918 * bias of 1 since we need to reserve a psize of zero, here, to 919 * represent holes and embedded blocks. 920 * 921 * This isn't a problem in practice, since the maximum size of a 922 * buffer is limited to 16MB, so we never need to store 32MB in 923 * this field. Even in the upstream illumos code base, the 924 * maximum size of a buffer is limited to 16MB. 925 */ 926 uint16_t b_psize; 927 928 /* 929 * This field stores the size of the data buffer before 930 * compression, and cannot change once set. It is in units 931 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes) 932 */ 933 uint16_t b_lsize; /* immutable */ 934 uint64_t b_spa; /* immutable */ 935 936 /* L2ARC fields. Undefined when not in L2ARC. */ 937 l2arc_buf_hdr_t b_l2hdr; 938 /* L1ARC fields. Undefined when in l2arc_only state */ 939 l1arc_buf_hdr_t b_l1hdr; 940 }; 941 942 #define GHOST_STATE(state) \ 943 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 944 (state) == arc_l2c_only) 945 946 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) 947 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) 948 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) 949 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) 950 #define HDR_COMPRESSION_ENABLED(hdr) \ 951 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) 952 953 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) 954 #define HDR_L2_READING(hdr) \ 955 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \ 956 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)) 957 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) 958 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) 959 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) 960 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) 961 962 #define HDR_ISTYPE_METADATA(hdr) \ 963 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA) 964 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) 965 966 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) 967 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) 968 969 /* For storing compression mode in b_flags */ 970 #define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1) 971 972 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ 973 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS)) 974 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ 975 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp)); 976 977 #define ARC_BUF_LAST(buf) ((buf)->b_next == NULL) 978 #define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED) 979 #define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED) 980 981 /* 982 * Other sizes 983 */ 984 985 #define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 986 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr)) 987 988 /* 989 * Hash table routines 990 */ 991 992 #define HT_LOCK_PAD 64 993 994 struct ht_lock { 995 kmutex_t ht_lock; 996 #ifdef _KERNEL 997 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 998 #endif 999 }; 1000 1001 #define BUF_LOCKS 256 1002 typedef struct buf_hash_table { 1003 uint64_t ht_mask; 1004 arc_buf_hdr_t **ht_table; 1005 struct ht_lock ht_locks[BUF_LOCKS]; 1006 } buf_hash_table_t; 1007 1008 static buf_hash_table_t buf_hash_table; 1009 1010 #define BUF_HASH_INDEX(spa, dva, birth) \ 1011 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 1012 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 1013 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 1014 #define HDR_LOCK(hdr) \ 1015 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth))) 1016 1017 uint64_t zfs_crc64_table[256]; 1018 1019 /* 1020 * Level 2 ARC 1021 */ 1022 1023 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 1024 #define L2ARC_HEADROOM 2 /* num of writes */ 1025 /* 1026 * If we discover during ARC scan any buffers to be compressed, we boost 1027 * our headroom for the next scanning cycle by this percentage multiple. 1028 */ 1029 #define L2ARC_HEADROOM_BOOST 200 1030 #define L2ARC_FEED_SECS 1 /* caching interval secs */ 1031 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 1032 1033 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 1034 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 1035 1036 /* L2ARC Performance Tunables */ 1037 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 1038 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 1039 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 1040 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST; 1041 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 1042 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 1043 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 1044 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 1045 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 1046 1047 /* 1048 * L2ARC Internals 1049 */ 1050 struct l2arc_dev { 1051 vdev_t *l2ad_vdev; /* vdev */ 1052 spa_t *l2ad_spa; /* spa */ 1053 uint64_t l2ad_hand; /* next write location */ 1054 uint64_t l2ad_start; /* first addr on device */ 1055 uint64_t l2ad_end; /* last addr on device */ 1056 boolean_t l2ad_first; /* first sweep through */ 1057 boolean_t l2ad_writing; /* currently writing */ 1058 kmutex_t l2ad_mtx; /* lock for buffer list */ 1059 list_t l2ad_buflist; /* buffer list */ 1060 list_node_t l2ad_node; /* device list node */ 1061 refcount_t l2ad_alloc; /* allocated bytes */ 1062 }; 1063 1064 static list_t L2ARC_dev_list; /* device list */ 1065 static list_t *l2arc_dev_list; /* device list pointer */ 1066 static kmutex_t l2arc_dev_mtx; /* device list mutex */ 1067 static l2arc_dev_t *l2arc_dev_last; /* last device used */ 1068 static list_t L2ARC_free_on_write; /* free after write buf list */ 1069 static list_t *l2arc_free_on_write; /* free after write list ptr */ 1070 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 1071 static uint64_t l2arc_ndev; /* number of devices */ 1072 1073 typedef struct l2arc_read_callback { 1074 arc_buf_hdr_t *l2rcb_hdr; /* read header */ 1075 blkptr_t l2rcb_bp; /* original blkptr */ 1076 zbookmark_phys_t l2rcb_zb; /* original bookmark */ 1077 int l2rcb_flags; /* original flags */ 1078 abd_t *l2rcb_abd; /* temporary buffer */ 1079 } l2arc_read_callback_t; 1080 1081 typedef struct l2arc_write_callback { 1082 l2arc_dev_t *l2wcb_dev; /* device info */ 1083 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 1084 } l2arc_write_callback_t; 1085 1086 typedef struct l2arc_data_free { 1087 /* protected by l2arc_free_on_write_mtx */ 1088 abd_t *l2df_abd; 1089 size_t l2df_size; 1090 arc_buf_contents_t l2df_type; 1091 list_node_t l2df_list_node; 1092 } l2arc_data_free_t; 1093 1094 static kmutex_t l2arc_feed_thr_lock; 1095 static kcondvar_t l2arc_feed_thr_cv; 1096 static uint8_t l2arc_thread_exit; 1097 1098 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *); 1099 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *); 1100 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *); 1101 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *); 1102 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *); 1103 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag); 1104 static void arc_hdr_free_pabd(arc_buf_hdr_t *); 1105 static void arc_hdr_alloc_pabd(arc_buf_hdr_t *); 1106 static void arc_access(arc_buf_hdr_t *, kmutex_t *); 1107 static boolean_t arc_is_overflowing(); 1108 static void arc_buf_watch(arc_buf_t *); 1109 1110 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *); 1111 static uint32_t arc_bufc_to_flags(arc_buf_contents_t); 1112 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1113 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1114 1115 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *); 1116 static void l2arc_read_done(zio_t *); 1117 1118 static uint64_t 1119 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 1120 { 1121 uint8_t *vdva = (uint8_t *)dva; 1122 uint64_t crc = -1ULL; 1123 int i; 1124 1125 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 1126 1127 for (i = 0; i < sizeof (dva_t); i++) 1128 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 1129 1130 crc ^= (spa>>8) ^ birth; 1131 1132 return (crc); 1133 } 1134 1135 #define HDR_EMPTY(hdr) \ 1136 ((hdr)->b_dva.dva_word[0] == 0 && \ 1137 (hdr)->b_dva.dva_word[1] == 0) 1138 1139 #define HDR_EQUAL(spa, dva, birth, hdr) \ 1140 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 1141 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 1142 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa) 1143 1144 static void 1145 buf_discard_identity(arc_buf_hdr_t *hdr) 1146 { 1147 hdr->b_dva.dva_word[0] = 0; 1148 hdr->b_dva.dva_word[1] = 0; 1149 hdr->b_birth = 0; 1150 } 1151 1152 static arc_buf_hdr_t * 1153 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp) 1154 { 1155 const dva_t *dva = BP_IDENTITY(bp); 1156 uint64_t birth = BP_PHYSICAL_BIRTH(bp); 1157 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 1158 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1159 arc_buf_hdr_t *hdr; 1160 1161 mutex_enter(hash_lock); 1162 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; 1163 hdr = hdr->b_hash_next) { 1164 if (HDR_EQUAL(spa, dva, birth, hdr)) { 1165 *lockp = hash_lock; 1166 return (hdr); 1167 } 1168 } 1169 mutex_exit(hash_lock); 1170 *lockp = NULL; 1171 return (NULL); 1172 } 1173 1174 /* 1175 * Insert an entry into the hash table. If there is already an element 1176 * equal to elem in the hash table, then the already existing element 1177 * will be returned and the new element will not be inserted. 1178 * Otherwise returns NULL. 1179 * If lockp == NULL, the caller is assumed to already hold the hash lock. 1180 */ 1181 static arc_buf_hdr_t * 1182 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) 1183 { 1184 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1185 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1186 arc_buf_hdr_t *fhdr; 1187 uint32_t i; 1188 1189 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); 1190 ASSERT(hdr->b_birth != 0); 1191 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1192 1193 if (lockp != NULL) { 1194 *lockp = hash_lock; 1195 mutex_enter(hash_lock); 1196 } else { 1197 ASSERT(MUTEX_HELD(hash_lock)); 1198 } 1199 1200 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL; 1201 fhdr = fhdr->b_hash_next, i++) { 1202 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) 1203 return (fhdr); 1204 } 1205 1206 hdr->b_hash_next = buf_hash_table.ht_table[idx]; 1207 buf_hash_table.ht_table[idx] = hdr; 1208 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1209 1210 /* collect some hash table performance data */ 1211 if (i > 0) { 1212 ARCSTAT_BUMP(arcstat_hash_collisions); 1213 if (i == 1) 1214 ARCSTAT_BUMP(arcstat_hash_chains); 1215 1216 ARCSTAT_MAX(arcstat_hash_chain_max, i); 1217 } 1218 1219 ARCSTAT_BUMP(arcstat_hash_elements); 1220 ARCSTAT_MAXSTAT(arcstat_hash_elements); 1221 1222 return (NULL); 1223 } 1224 1225 static void 1226 buf_hash_remove(arc_buf_hdr_t *hdr) 1227 { 1228 arc_buf_hdr_t *fhdr, **hdrp; 1229 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1230 1231 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 1232 ASSERT(HDR_IN_HASH_TABLE(hdr)); 1233 1234 hdrp = &buf_hash_table.ht_table[idx]; 1235 while ((fhdr = *hdrp) != hdr) { 1236 ASSERT3P(fhdr, !=, NULL); 1237 hdrp = &fhdr->b_hash_next; 1238 } 1239 *hdrp = hdr->b_hash_next; 1240 hdr->b_hash_next = NULL; 1241 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1242 1243 /* collect some hash table performance data */ 1244 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 1245 1246 if (buf_hash_table.ht_table[idx] && 1247 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 1248 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 1249 } 1250 1251 /* 1252 * Global data structures and functions for the buf kmem cache. 1253 */ 1254 static kmem_cache_t *hdr_full_cache; 1255 static kmem_cache_t *hdr_l2only_cache; 1256 static kmem_cache_t *buf_cache; 1257 1258 static void 1259 buf_fini(void) 1260 { 1261 int i; 1262 1263 kmem_free(buf_hash_table.ht_table, 1264 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 1265 for (i = 0; i < BUF_LOCKS; i++) 1266 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 1267 kmem_cache_destroy(hdr_full_cache); 1268 kmem_cache_destroy(hdr_l2only_cache); 1269 kmem_cache_destroy(buf_cache); 1270 } 1271 1272 /* 1273 * Constructor callback - called when the cache is empty 1274 * and a new buf is requested. 1275 */ 1276 /* ARGSUSED */ 1277 static int 1278 hdr_full_cons(void *vbuf, void *unused, int kmflag) 1279 { 1280 arc_buf_hdr_t *hdr = vbuf; 1281 1282 bzero(hdr, HDR_FULL_SIZE); 1283 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); 1284 refcount_create(&hdr->b_l1hdr.b_refcnt); 1285 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 1286 multilist_link_init(&hdr->b_l1hdr.b_arc_node); 1287 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1288 1289 return (0); 1290 } 1291 1292 /* ARGSUSED */ 1293 static int 1294 hdr_l2only_cons(void *vbuf, void *unused, int kmflag) 1295 { 1296 arc_buf_hdr_t *hdr = vbuf; 1297 1298 bzero(hdr, HDR_L2ONLY_SIZE); 1299 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1300 1301 return (0); 1302 } 1303 1304 /* ARGSUSED */ 1305 static int 1306 buf_cons(void *vbuf, void *unused, int kmflag) 1307 { 1308 arc_buf_t *buf = vbuf; 1309 1310 bzero(buf, sizeof (arc_buf_t)); 1311 mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL); 1312 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1313 1314 return (0); 1315 } 1316 1317 /* 1318 * Destructor callback - called when a cached buf is 1319 * no longer required. 1320 */ 1321 /* ARGSUSED */ 1322 static void 1323 hdr_full_dest(void *vbuf, void *unused) 1324 { 1325 arc_buf_hdr_t *hdr = vbuf; 1326 1327 ASSERT(HDR_EMPTY(hdr)); 1328 cv_destroy(&hdr->b_l1hdr.b_cv); 1329 refcount_destroy(&hdr->b_l1hdr.b_refcnt); 1330 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); 1331 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 1332 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1333 } 1334 1335 /* ARGSUSED */ 1336 static void 1337 hdr_l2only_dest(void *vbuf, void *unused) 1338 { 1339 arc_buf_hdr_t *hdr = vbuf; 1340 1341 ASSERT(HDR_EMPTY(hdr)); 1342 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1343 } 1344 1345 /* ARGSUSED */ 1346 static void 1347 buf_dest(void *vbuf, void *unused) 1348 { 1349 arc_buf_t *buf = vbuf; 1350 1351 mutex_destroy(&buf->b_evict_lock); 1352 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1353 } 1354 1355 /* 1356 * Reclaim callback -- invoked when memory is low. 1357 */ 1358 /* ARGSUSED */ 1359 static void 1360 hdr_recl(void *unused) 1361 { 1362 dprintf("hdr_recl called\n"); 1363 /* 1364 * umem calls the reclaim func when we destroy the buf cache, 1365 * which is after we do arc_fini(). 1366 */ 1367 if (!arc_dead) 1368 cv_signal(&arc_reclaim_thread_cv); 1369 } 1370 1371 static void 1372 buf_init(void) 1373 { 1374 uint64_t *ct; 1375 uint64_t hsize = 1ULL << 12; 1376 int i, j; 1377 1378 /* 1379 * The hash table is big enough to fill all of physical memory 1380 * with an average block size of zfs_arc_average_blocksize (default 8K). 1381 * By default, the table will take up 1382 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers). 1383 */ 1384 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE) 1385 hsize <<= 1; 1386 retry: 1387 buf_hash_table.ht_mask = hsize - 1; 1388 buf_hash_table.ht_table = 1389 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 1390 if (buf_hash_table.ht_table == NULL) { 1391 ASSERT(hsize > (1ULL << 8)); 1392 hsize >>= 1; 1393 goto retry; 1394 } 1395 1396 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE, 1397 0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0); 1398 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only", 1399 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl, 1400 NULL, NULL, 0); 1401 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 1402 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 1403 1404 for (i = 0; i < 256; i++) 1405 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 1406 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 1407 1408 for (i = 0; i < BUF_LOCKS; i++) { 1409 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 1410 NULL, MUTEX_DEFAULT, NULL); 1411 } 1412 } 1413 1414 /* 1415 * This is the size that the buf occupies in memory. If the buf is compressed, 1416 * it will correspond to the compressed size. You should use this method of 1417 * getting the buf size unless you explicitly need the logical size. 1418 */ 1419 int32_t 1420 arc_buf_size(arc_buf_t *buf) 1421 { 1422 return (ARC_BUF_COMPRESSED(buf) ? 1423 HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr)); 1424 } 1425 1426 int32_t 1427 arc_buf_lsize(arc_buf_t *buf) 1428 { 1429 return (HDR_GET_LSIZE(buf->b_hdr)); 1430 } 1431 1432 enum zio_compress 1433 arc_get_compression(arc_buf_t *buf) 1434 { 1435 return (ARC_BUF_COMPRESSED(buf) ? 1436 HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF); 1437 } 1438 1439 #define ARC_MINTIME (hz>>4) /* 62 ms */ 1440 1441 static inline boolean_t 1442 arc_buf_is_shared(arc_buf_t *buf) 1443 { 1444 boolean_t shared = (buf->b_data != NULL && 1445 buf->b_hdr->b_l1hdr.b_pabd != NULL && 1446 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) && 1447 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd)); 1448 IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr)); 1449 IMPLY(shared, ARC_BUF_SHARED(buf)); 1450 IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf)); 1451 1452 /* 1453 * It would be nice to assert arc_can_share() too, but the "hdr isn't 1454 * already being shared" requirement prevents us from doing that. 1455 */ 1456 1457 return (shared); 1458 } 1459 1460 /* 1461 * Free the checksum associated with this header. If there is no checksum, this 1462 * is a no-op. 1463 */ 1464 static inline void 1465 arc_cksum_free(arc_buf_hdr_t *hdr) 1466 { 1467 ASSERT(HDR_HAS_L1HDR(hdr)); 1468 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1469 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { 1470 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); 1471 hdr->b_l1hdr.b_freeze_cksum = NULL; 1472 } 1473 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1474 } 1475 1476 /* 1477 * Return true iff at least one of the bufs on hdr is not compressed. 1478 */ 1479 static boolean_t 1480 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) 1481 { 1482 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { 1483 if (!ARC_BUF_COMPRESSED(b)) { 1484 return (B_TRUE); 1485 } 1486 } 1487 return (B_FALSE); 1488 } 1489 1490 /* 1491 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data 1492 * matches the checksum that is stored in the hdr. If there is no checksum, 1493 * or if the buf is compressed, this is a no-op. 1494 */ 1495 static void 1496 arc_cksum_verify(arc_buf_t *buf) 1497 { 1498 arc_buf_hdr_t *hdr = buf->b_hdr; 1499 zio_cksum_t zc; 1500 1501 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1502 return; 1503 1504 if (ARC_BUF_COMPRESSED(buf)) { 1505 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1506 arc_hdr_has_uncompressed_buf(hdr)); 1507 return; 1508 } 1509 1510 ASSERT(HDR_HAS_L1HDR(hdr)); 1511 1512 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1513 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { 1514 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1515 return; 1516 } 1517 1518 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc); 1519 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) 1520 panic("buffer modified while frozen!"); 1521 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1522 } 1523 1524 static boolean_t 1525 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) 1526 { 1527 enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp); 1528 boolean_t valid_cksum; 1529 1530 ASSERT(!BP_IS_EMBEDDED(zio->io_bp)); 1531 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); 1532 1533 /* 1534 * We rely on the blkptr's checksum to determine if the block 1535 * is valid or not. When compressed arc is enabled, the l2arc 1536 * writes the block to the l2arc just as it appears in the pool. 1537 * This allows us to use the blkptr's checksum to validate the 1538 * data that we just read off of the l2arc without having to store 1539 * a separate checksum in the arc_buf_hdr_t. However, if compressed 1540 * arc is disabled, then the data written to the l2arc is always 1541 * uncompressed and won't match the block as it exists in the main 1542 * pool. When this is the case, we must first compress it if it is 1543 * compressed on the main pool before we can validate the checksum. 1544 */ 1545 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) { 1546 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 1547 uint64_t lsize = HDR_GET_LSIZE(hdr); 1548 uint64_t csize; 1549 1550 void *cbuf = zio_buf_alloc(HDR_GET_PSIZE(hdr)); 1551 csize = zio_compress_data(compress, zio->io_abd, cbuf, lsize); 1552 1553 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr)); 1554 if (csize < HDR_GET_PSIZE(hdr)) { 1555 /* 1556 * Compressed blocks are always a multiple of the 1557 * smallest ashift in the pool. Ideally, we would 1558 * like to round up the csize to the next 1559 * spa_min_ashift but that value may have changed 1560 * since the block was last written. Instead, 1561 * we rely on the fact that the hdr's psize 1562 * was set to the psize of the block when it was 1563 * last written. We set the csize to that value 1564 * and zero out any part that should not contain 1565 * data. 1566 */ 1567 bzero((char *)cbuf + csize, HDR_GET_PSIZE(hdr) - csize); 1568 csize = HDR_GET_PSIZE(hdr); 1569 } 1570 zio_push_transform(zio, cbuf, csize, HDR_GET_PSIZE(hdr), NULL); 1571 } 1572 1573 /* 1574 * Block pointers always store the checksum for the logical data. 1575 * If the block pointer has the gang bit set, then the checksum 1576 * it represents is for the reconstituted data and not for an 1577 * individual gang member. The zio pipeline, however, must be able to 1578 * determine the checksum of each of the gang constituents so it 1579 * treats the checksum comparison differently than what we need 1580 * for l2arc blocks. This prevents us from using the 1581 * zio_checksum_error() interface directly. Instead we must call the 1582 * zio_checksum_error_impl() so that we can ensure the checksum is 1583 * generated using the correct checksum algorithm and accounts for the 1584 * logical I/O size and not just a gang fragment. 1585 */ 1586 valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp, 1587 BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size, 1588 zio->io_offset, NULL) == 0); 1589 zio_pop_transforms(zio); 1590 return (valid_cksum); 1591 } 1592 1593 /* 1594 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a 1595 * checksum and attaches it to the buf's hdr so that we can ensure that the buf 1596 * isn't modified later on. If buf is compressed or there is already a checksum 1597 * on the hdr, this is a no-op (we only checksum uncompressed bufs). 1598 */ 1599 static void 1600 arc_cksum_compute(arc_buf_t *buf) 1601 { 1602 arc_buf_hdr_t *hdr = buf->b_hdr; 1603 1604 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1605 return; 1606 1607 ASSERT(HDR_HAS_L1HDR(hdr)); 1608 1609 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock); 1610 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { 1611 ASSERT(arc_hdr_has_uncompressed_buf(hdr)); 1612 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1613 return; 1614 } else if (ARC_BUF_COMPRESSED(buf)) { 1615 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1616 return; 1617 } 1618 1619 ASSERT(!ARC_BUF_COMPRESSED(buf)); 1620 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), 1621 KM_SLEEP); 1622 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, 1623 hdr->b_l1hdr.b_freeze_cksum); 1624 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1625 arc_buf_watch(buf); 1626 } 1627 1628 #ifndef _KERNEL 1629 typedef struct procctl { 1630 long cmd; 1631 prwatch_t prwatch; 1632 } procctl_t; 1633 #endif 1634 1635 /* ARGSUSED */ 1636 static void 1637 arc_buf_unwatch(arc_buf_t *buf) 1638 { 1639 #ifndef _KERNEL 1640 if (arc_watch) { 1641 int result; 1642 procctl_t ctl; 1643 ctl.cmd = PCWATCH; 1644 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1645 ctl.prwatch.pr_size = 0; 1646 ctl.prwatch.pr_wflags = 0; 1647 result = write(arc_procfd, &ctl, sizeof (ctl)); 1648 ASSERT3U(result, ==, sizeof (ctl)); 1649 } 1650 #endif 1651 } 1652 1653 /* ARGSUSED */ 1654 static void 1655 arc_buf_watch(arc_buf_t *buf) 1656 { 1657 #ifndef _KERNEL 1658 if (arc_watch) { 1659 int result; 1660 procctl_t ctl; 1661 ctl.cmd = PCWATCH; 1662 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1663 ctl.prwatch.pr_size = arc_buf_size(buf); 1664 ctl.prwatch.pr_wflags = WA_WRITE; 1665 result = write(arc_procfd, &ctl, sizeof (ctl)); 1666 ASSERT3U(result, ==, sizeof (ctl)); 1667 } 1668 #endif 1669 } 1670 1671 static arc_buf_contents_t 1672 arc_buf_type(arc_buf_hdr_t *hdr) 1673 { 1674 arc_buf_contents_t type; 1675 if (HDR_ISTYPE_METADATA(hdr)) { 1676 type = ARC_BUFC_METADATA; 1677 } else { 1678 type = ARC_BUFC_DATA; 1679 } 1680 VERIFY3U(hdr->b_type, ==, type); 1681 return (type); 1682 } 1683 1684 boolean_t 1685 arc_is_metadata(arc_buf_t *buf) 1686 { 1687 return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0); 1688 } 1689 1690 static uint32_t 1691 arc_bufc_to_flags(arc_buf_contents_t type) 1692 { 1693 switch (type) { 1694 case ARC_BUFC_DATA: 1695 /* metadata field is 0 if buffer contains normal data */ 1696 return (0); 1697 case ARC_BUFC_METADATA: 1698 return (ARC_FLAG_BUFC_METADATA); 1699 default: 1700 break; 1701 } 1702 panic("undefined ARC buffer type!"); 1703 return ((uint32_t)-1); 1704 } 1705 1706 void 1707 arc_buf_thaw(arc_buf_t *buf) 1708 { 1709 arc_buf_hdr_t *hdr = buf->b_hdr; 1710 1711 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 1712 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1713 1714 arc_cksum_verify(buf); 1715 1716 /* 1717 * Compressed buffers do not manipulate the b_freeze_cksum or 1718 * allocate b_thawed. 1719 */ 1720 if (ARC_BUF_COMPRESSED(buf)) { 1721 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1722 arc_hdr_has_uncompressed_buf(hdr)); 1723 return; 1724 } 1725 1726 ASSERT(HDR_HAS_L1HDR(hdr)); 1727 arc_cksum_free(hdr); 1728 1729 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1730 #ifdef ZFS_DEBUG 1731 if (zfs_flags & ZFS_DEBUG_MODIFY) { 1732 if (hdr->b_l1hdr.b_thawed != NULL) 1733 kmem_free(hdr->b_l1hdr.b_thawed, 1); 1734 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP); 1735 } 1736 #endif 1737 1738 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1739 1740 arc_buf_unwatch(buf); 1741 } 1742 1743 void 1744 arc_buf_freeze(arc_buf_t *buf) 1745 { 1746 arc_buf_hdr_t *hdr = buf->b_hdr; 1747 kmutex_t *hash_lock; 1748 1749 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1750 return; 1751 1752 if (ARC_BUF_COMPRESSED(buf)) { 1753 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1754 arc_hdr_has_uncompressed_buf(hdr)); 1755 return; 1756 } 1757 1758 hash_lock = HDR_LOCK(hdr); 1759 mutex_enter(hash_lock); 1760 1761 ASSERT(HDR_HAS_L1HDR(hdr)); 1762 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL || 1763 hdr->b_l1hdr.b_state == arc_anon); 1764 arc_cksum_compute(buf); 1765 mutex_exit(hash_lock); 1766 } 1767 1768 /* 1769 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead, 1770 * the following functions should be used to ensure that the flags are 1771 * updated in a thread-safe way. When manipulating the flags either 1772 * the hash_lock must be held or the hdr must be undiscoverable. This 1773 * ensures that we're not racing with any other threads when updating 1774 * the flags. 1775 */ 1776 static inline void 1777 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 1778 { 1779 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1780 hdr->b_flags |= flags; 1781 } 1782 1783 static inline void 1784 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 1785 { 1786 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1787 hdr->b_flags &= ~flags; 1788 } 1789 1790 /* 1791 * Setting the compression bits in the arc_buf_hdr_t's b_flags is 1792 * done in a special way since we have to clear and set bits 1793 * at the same time. Consumers that wish to set the compression bits 1794 * must use this function to ensure that the flags are updated in 1795 * thread-safe manner. 1796 */ 1797 static void 1798 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) 1799 { 1800 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1801 1802 /* 1803 * Holes and embedded blocks will always have a psize = 0 so 1804 * we ignore the compression of the blkptr and set the 1805 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF. 1806 * Holes and embedded blocks remain anonymous so we don't 1807 * want to uncompress them. Mark them as uncompressed. 1808 */ 1809 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { 1810 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 1811 HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF); 1812 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); 1813 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 1814 } else { 1815 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 1816 HDR_SET_COMPRESS(hdr, cmp); 1817 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); 1818 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); 1819 } 1820 } 1821 1822 /* 1823 * Looks for another buf on the same hdr which has the data decompressed, copies 1824 * from it, and returns true. If no such buf exists, returns false. 1825 */ 1826 static boolean_t 1827 arc_buf_try_copy_decompressed_data(arc_buf_t *buf) 1828 { 1829 arc_buf_hdr_t *hdr = buf->b_hdr; 1830 boolean_t copied = B_FALSE; 1831 1832 ASSERT(HDR_HAS_L1HDR(hdr)); 1833 ASSERT3P(buf->b_data, !=, NULL); 1834 ASSERT(!ARC_BUF_COMPRESSED(buf)); 1835 1836 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; 1837 from = from->b_next) { 1838 /* can't use our own data buffer */ 1839 if (from == buf) { 1840 continue; 1841 } 1842 1843 if (!ARC_BUF_COMPRESSED(from)) { 1844 bcopy(from->b_data, buf->b_data, arc_buf_size(buf)); 1845 copied = B_TRUE; 1846 break; 1847 } 1848 } 1849 1850 /* 1851 * There were no decompressed bufs, so there should not be a 1852 * checksum on the hdr either. 1853 */ 1854 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL); 1855 1856 return (copied); 1857 } 1858 1859 /* 1860 * Given a buf that has a data buffer attached to it, this function will 1861 * efficiently fill the buf with data of the specified compression setting from 1862 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr 1863 * are already sharing a data buf, no copy is performed. 1864 * 1865 * If the buf is marked as compressed but uncompressed data was requested, this 1866 * will allocate a new data buffer for the buf, remove that flag, and fill the 1867 * buf with uncompressed data. You can't request a compressed buf on a hdr with 1868 * uncompressed data, and (since we haven't added support for it yet) if you 1869 * want compressed data your buf must already be marked as compressed and have 1870 * the correct-sized data buffer. 1871 */ 1872 static int 1873 arc_buf_fill(arc_buf_t *buf, boolean_t compressed) 1874 { 1875 arc_buf_hdr_t *hdr = buf->b_hdr; 1876 boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 1877 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; 1878 1879 ASSERT3P(buf->b_data, !=, NULL); 1880 IMPLY(compressed, hdr_compressed); 1881 IMPLY(compressed, ARC_BUF_COMPRESSED(buf)); 1882 1883 if (hdr_compressed == compressed) { 1884 if (!arc_buf_is_shared(buf)) { 1885 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd, 1886 arc_buf_size(buf)); 1887 } 1888 } else { 1889 ASSERT(hdr_compressed); 1890 ASSERT(!compressed); 1891 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); 1892 1893 /* 1894 * If the buf is sharing its data with the hdr, unlink it and 1895 * allocate a new data buffer for the buf. 1896 */ 1897 if (arc_buf_is_shared(buf)) { 1898 ASSERT(ARC_BUF_COMPRESSED(buf)); 1899 1900 /* We need to give the buf it's own b_data */ 1901 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 1902 buf->b_data = 1903 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 1904 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 1905 1906 /* Previously overhead was 0; just add new overhead */ 1907 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); 1908 } else if (ARC_BUF_COMPRESSED(buf)) { 1909 /* We need to reallocate the buf's b_data */ 1910 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), 1911 buf); 1912 buf->b_data = 1913 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 1914 1915 /* We increased the size of b_data; update overhead */ 1916 ARCSTAT_INCR(arcstat_overhead_size, 1917 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); 1918 } 1919 1920 /* 1921 * Regardless of the buf's previous compression settings, it 1922 * should not be compressed at the end of this function. 1923 */ 1924 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; 1925 1926 /* 1927 * Try copying the data from another buf which already has a 1928 * decompressed version. If that's not possible, it's time to 1929 * bite the bullet and decompress the data from the hdr. 1930 */ 1931 if (arc_buf_try_copy_decompressed_data(buf)) { 1932 /* Skip byteswapping and checksumming (already done) */ 1933 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL); 1934 return (0); 1935 } else { 1936 int error = zio_decompress_data(HDR_GET_COMPRESS(hdr), 1937 hdr->b_l1hdr.b_pabd, buf->b_data, 1938 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 1939 1940 /* 1941 * Absent hardware errors or software bugs, this should 1942 * be impossible, but log it anyway so we can debug it. 1943 */ 1944 if (error != 0) { 1945 zfs_dbgmsg( 1946 "hdr %p, compress %d, psize %d, lsize %d", 1947 hdr, HDR_GET_COMPRESS(hdr), 1948 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 1949 return (SET_ERROR(EIO)); 1950 } 1951 } 1952 } 1953 1954 /* Byteswap the buf's data if necessary */ 1955 if (bswap != DMU_BSWAP_NUMFUNCS) { 1956 ASSERT(!HDR_SHARED_DATA(hdr)); 1957 ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS); 1958 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); 1959 } 1960 1961 /* Compute the hdr's checksum if necessary */ 1962 arc_cksum_compute(buf); 1963 1964 return (0); 1965 } 1966 1967 int 1968 arc_decompress(arc_buf_t *buf) 1969 { 1970 return (arc_buf_fill(buf, B_FALSE)); 1971 } 1972 1973 /* 1974 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t. 1975 */ 1976 static uint64_t 1977 arc_hdr_size(arc_buf_hdr_t *hdr) 1978 { 1979 uint64_t size; 1980 1981 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && 1982 HDR_GET_PSIZE(hdr) > 0) { 1983 size = HDR_GET_PSIZE(hdr); 1984 } else { 1985 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); 1986 size = HDR_GET_LSIZE(hdr); 1987 } 1988 return (size); 1989 } 1990 1991 /* 1992 * Increment the amount of evictable space in the arc_state_t's refcount. 1993 * We account for the space used by the hdr and the arc buf individually 1994 * so that we can add and remove them from the refcount individually. 1995 */ 1996 static void 1997 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) 1998 { 1999 arc_buf_contents_t type = arc_buf_type(hdr); 2000 2001 ASSERT(HDR_HAS_L1HDR(hdr)); 2002 2003 if (GHOST_STATE(state)) { 2004 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2005 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2006 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2007 (void) refcount_add_many(&state->arcs_esize[type], 2008 HDR_GET_LSIZE(hdr), hdr); 2009 return; 2010 } 2011 2012 ASSERT(!GHOST_STATE(state)); 2013 if (hdr->b_l1hdr.b_pabd != NULL) { 2014 (void) refcount_add_many(&state->arcs_esize[type], 2015 arc_hdr_size(hdr), hdr); 2016 } 2017 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2018 buf = buf->b_next) { 2019 if (arc_buf_is_shared(buf)) 2020 continue; 2021 (void) refcount_add_many(&state->arcs_esize[type], 2022 arc_buf_size(buf), buf); 2023 } 2024 } 2025 2026 /* 2027 * Decrement the amount of evictable space in the arc_state_t's refcount. 2028 * We account for the space used by the hdr and the arc buf individually 2029 * so that we can add and remove them from the refcount individually. 2030 */ 2031 static void 2032 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) 2033 { 2034 arc_buf_contents_t type = arc_buf_type(hdr); 2035 2036 ASSERT(HDR_HAS_L1HDR(hdr)); 2037 2038 if (GHOST_STATE(state)) { 2039 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2040 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2041 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2042 (void) refcount_remove_many(&state->arcs_esize[type], 2043 HDR_GET_LSIZE(hdr), hdr); 2044 return; 2045 } 2046 2047 ASSERT(!GHOST_STATE(state)); 2048 if (hdr->b_l1hdr.b_pabd != NULL) { 2049 (void) refcount_remove_many(&state->arcs_esize[type], 2050 arc_hdr_size(hdr), hdr); 2051 } 2052 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2053 buf = buf->b_next) { 2054 if (arc_buf_is_shared(buf)) 2055 continue; 2056 (void) refcount_remove_many(&state->arcs_esize[type], 2057 arc_buf_size(buf), buf); 2058 } 2059 } 2060 2061 /* 2062 * Add a reference to this hdr indicating that someone is actively 2063 * referencing that memory. When the refcount transitions from 0 to 1, 2064 * we remove it from the respective arc_state_t list to indicate that 2065 * it is not evictable. 2066 */ 2067 static void 2068 add_reference(arc_buf_hdr_t *hdr, void *tag) 2069 { 2070 ASSERT(HDR_HAS_L1HDR(hdr)); 2071 if (!MUTEX_HELD(HDR_LOCK(hdr))) { 2072 ASSERT(hdr->b_l1hdr.b_state == arc_anon); 2073 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2074 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2075 } 2076 2077 arc_state_t *state = hdr->b_l1hdr.b_state; 2078 2079 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && 2080 (state != arc_anon)) { 2081 /* We don't use the L2-only state list. */ 2082 if (state != arc_l2c_only) { 2083 multilist_remove(state->arcs_list[arc_buf_type(hdr)], 2084 hdr); 2085 arc_evictable_space_decrement(hdr, state); 2086 } 2087 /* remove the prefetch flag if we get a reference */ 2088 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 2089 } 2090 } 2091 2092 /* 2093 * Remove a reference from this hdr. When the reference transitions from 2094 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's 2095 * list making it eligible for eviction. 2096 */ 2097 static int 2098 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) 2099 { 2100 int cnt; 2101 arc_state_t *state = hdr->b_l1hdr.b_state; 2102 2103 ASSERT(HDR_HAS_L1HDR(hdr)); 2104 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 2105 ASSERT(!GHOST_STATE(state)); 2106 2107 /* 2108 * arc_l2c_only counts as a ghost state so we don't need to explicitly 2109 * check to prevent usage of the arc_l2c_only list. 2110 */ 2111 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && 2112 (state != arc_anon)) { 2113 multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr); 2114 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); 2115 arc_evictable_space_increment(hdr, state); 2116 } 2117 return (cnt); 2118 } 2119 2120 /* 2121 * Move the supplied buffer to the indicated state. The hash lock 2122 * for the buffer must be held by the caller. 2123 */ 2124 static void 2125 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, 2126 kmutex_t *hash_lock) 2127 { 2128 arc_state_t *old_state; 2129 int64_t refcnt; 2130 uint32_t bufcnt; 2131 boolean_t update_old, update_new; 2132 arc_buf_contents_t buftype = arc_buf_type(hdr); 2133 2134 /* 2135 * We almost always have an L1 hdr here, since we call arc_hdr_realloc() 2136 * in arc_read() when bringing a buffer out of the L2ARC. However, the 2137 * L1 hdr doesn't always exist when we change state to arc_anon before 2138 * destroying a header, in which case reallocating to add the L1 hdr is 2139 * pointless. 2140 */ 2141 if (HDR_HAS_L1HDR(hdr)) { 2142 old_state = hdr->b_l1hdr.b_state; 2143 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt); 2144 bufcnt = hdr->b_l1hdr.b_bufcnt; 2145 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL); 2146 } else { 2147 old_state = arc_l2c_only; 2148 refcnt = 0; 2149 bufcnt = 0; 2150 update_old = B_FALSE; 2151 } 2152 update_new = update_old; 2153 2154 ASSERT(MUTEX_HELD(hash_lock)); 2155 ASSERT3P(new_state, !=, old_state); 2156 ASSERT(!GHOST_STATE(new_state) || bufcnt == 0); 2157 ASSERT(old_state != arc_anon || bufcnt <= 1); 2158 2159 /* 2160 * If this buffer is evictable, transfer it from the 2161 * old state list to the new state list. 2162 */ 2163 if (refcnt == 0) { 2164 if (old_state != arc_anon && old_state != arc_l2c_only) { 2165 ASSERT(HDR_HAS_L1HDR(hdr)); 2166 multilist_remove(old_state->arcs_list[buftype], hdr); 2167 2168 if (GHOST_STATE(old_state)) { 2169 ASSERT0(bufcnt); 2170 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2171 update_old = B_TRUE; 2172 } 2173 arc_evictable_space_decrement(hdr, old_state); 2174 } 2175 if (new_state != arc_anon && new_state != arc_l2c_only) { 2176 2177 /* 2178 * An L1 header always exists here, since if we're 2179 * moving to some L1-cached state (i.e. not l2c_only or 2180 * anonymous), we realloc the header to add an L1hdr 2181 * beforehand. 2182 */ 2183 ASSERT(HDR_HAS_L1HDR(hdr)); 2184 multilist_insert(new_state->arcs_list[buftype], hdr); 2185 2186 if (GHOST_STATE(new_state)) { 2187 ASSERT0(bufcnt); 2188 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2189 update_new = B_TRUE; 2190 } 2191 arc_evictable_space_increment(hdr, new_state); 2192 } 2193 } 2194 2195 ASSERT(!HDR_EMPTY(hdr)); 2196 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) 2197 buf_hash_remove(hdr); 2198 2199 /* adjust state sizes (ignore arc_l2c_only) */ 2200 2201 if (update_new && new_state != arc_l2c_only) { 2202 ASSERT(HDR_HAS_L1HDR(hdr)); 2203 if (GHOST_STATE(new_state)) { 2204 ASSERT0(bufcnt); 2205 2206 /* 2207 * When moving a header to a ghost state, we first 2208 * remove all arc buffers. Thus, we'll have a 2209 * bufcnt of zero, and no arc buffer to use for 2210 * the reference. As a result, we use the arc 2211 * header pointer for the reference. 2212 */ 2213 (void) refcount_add_many(&new_state->arcs_size, 2214 HDR_GET_LSIZE(hdr), hdr); 2215 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2216 } else { 2217 uint32_t buffers = 0; 2218 2219 /* 2220 * Each individual buffer holds a unique reference, 2221 * thus we must remove each of these references one 2222 * at a time. 2223 */ 2224 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2225 buf = buf->b_next) { 2226 ASSERT3U(bufcnt, !=, 0); 2227 buffers++; 2228 2229 /* 2230 * When the arc_buf_t is sharing the data 2231 * block with the hdr, the owner of the 2232 * reference belongs to the hdr. Only 2233 * add to the refcount if the arc_buf_t is 2234 * not shared. 2235 */ 2236 if (arc_buf_is_shared(buf)) 2237 continue; 2238 2239 (void) refcount_add_many(&new_state->arcs_size, 2240 arc_buf_size(buf), buf); 2241 } 2242 ASSERT3U(bufcnt, ==, buffers); 2243 2244 if (hdr->b_l1hdr.b_pabd != NULL) { 2245 (void) refcount_add_many(&new_state->arcs_size, 2246 arc_hdr_size(hdr), hdr); 2247 } else { 2248 ASSERT(GHOST_STATE(old_state)); 2249 } 2250 } 2251 } 2252 2253 if (update_old && old_state != arc_l2c_only) { 2254 ASSERT(HDR_HAS_L1HDR(hdr)); 2255 if (GHOST_STATE(old_state)) { 2256 ASSERT0(bufcnt); 2257 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2258 2259 /* 2260 * When moving a header off of a ghost state, 2261 * the header will not contain any arc buffers. 2262 * We use the arc header pointer for the reference 2263 * which is exactly what we did when we put the 2264 * header on the ghost state. 2265 */ 2266 2267 (void) refcount_remove_many(&old_state->arcs_size, 2268 HDR_GET_LSIZE(hdr), hdr); 2269 } else { 2270 uint32_t buffers = 0; 2271 2272 /* 2273 * Each individual buffer holds a unique reference, 2274 * thus we must remove each of these references one 2275 * at a time. 2276 */ 2277 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2278 buf = buf->b_next) { 2279 ASSERT3U(bufcnt, !=, 0); 2280 buffers++; 2281 2282 /* 2283 * When the arc_buf_t is sharing the data 2284 * block with the hdr, the owner of the 2285 * reference belongs to the hdr. Only 2286 * add to the refcount if the arc_buf_t is 2287 * not shared. 2288 */ 2289 if (arc_buf_is_shared(buf)) 2290 continue; 2291 2292 (void) refcount_remove_many( 2293 &old_state->arcs_size, arc_buf_size(buf), 2294 buf); 2295 } 2296 ASSERT3U(bufcnt, ==, buffers); 2297 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2298 (void) refcount_remove_many( 2299 &old_state->arcs_size, arc_hdr_size(hdr), hdr); 2300 } 2301 } 2302 2303 if (HDR_HAS_L1HDR(hdr)) 2304 hdr->b_l1hdr.b_state = new_state; 2305 2306 /* 2307 * L2 headers should never be on the L2 state list since they don't 2308 * have L1 headers allocated. 2309 */ 2310 ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) && 2311 multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA])); 2312 } 2313 2314 void 2315 arc_space_consume(uint64_t space, arc_space_type_t type) 2316 { 2317 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2318 2319 switch (type) { 2320 case ARC_SPACE_DATA: 2321 ARCSTAT_INCR(arcstat_data_size, space); 2322 break; 2323 case ARC_SPACE_META: 2324 ARCSTAT_INCR(arcstat_metadata_size, space); 2325 break; 2326 case ARC_SPACE_OTHER: 2327 ARCSTAT_INCR(arcstat_other_size, space); 2328 break; 2329 case ARC_SPACE_HDRS: 2330 ARCSTAT_INCR(arcstat_hdr_size, space); 2331 break; 2332 case ARC_SPACE_L2HDRS: 2333 ARCSTAT_INCR(arcstat_l2_hdr_size, space); 2334 break; 2335 } 2336 2337 if (type != ARC_SPACE_DATA) 2338 ARCSTAT_INCR(arcstat_meta_used, space); 2339 2340 atomic_add_64(&arc_size, space); 2341 } 2342 2343 void 2344 arc_space_return(uint64_t space, arc_space_type_t type) 2345 { 2346 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2347 2348 switch (type) { 2349 case ARC_SPACE_DATA: 2350 ARCSTAT_INCR(arcstat_data_size, -space); 2351 break; 2352 case ARC_SPACE_META: 2353 ARCSTAT_INCR(arcstat_metadata_size, -space); 2354 break; 2355 case ARC_SPACE_OTHER: 2356 ARCSTAT_INCR(arcstat_other_size, -space); 2357 break; 2358 case ARC_SPACE_HDRS: 2359 ARCSTAT_INCR(arcstat_hdr_size, -space); 2360 break; 2361 case ARC_SPACE_L2HDRS: 2362 ARCSTAT_INCR(arcstat_l2_hdr_size, -space); 2363 break; 2364 } 2365 2366 if (type != ARC_SPACE_DATA) { 2367 ASSERT(arc_meta_used >= space); 2368 if (arc_meta_max < arc_meta_used) 2369 arc_meta_max = arc_meta_used; 2370 ARCSTAT_INCR(arcstat_meta_used, -space); 2371 } 2372 2373 ASSERT(arc_size >= space); 2374 atomic_add_64(&arc_size, -space); 2375 } 2376 2377 /* 2378 * Given a hdr and a buf, returns whether that buf can share its b_data buffer 2379 * with the hdr's b_pabd. 2380 */ 2381 static boolean_t 2382 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2383 { 2384 /* 2385 * The criteria for sharing a hdr's data are: 2386 * 1. the hdr's compression matches the buf's compression 2387 * 2. the hdr doesn't need to be byteswapped 2388 * 3. the hdr isn't already being shared 2389 * 4. the buf is either compressed or it is the last buf in the hdr list 2390 * 2391 * Criterion #4 maintains the invariant that shared uncompressed 2392 * bufs must be the final buf in the hdr's b_buf list. Reading this, you 2393 * might ask, "if a compressed buf is allocated first, won't that be the 2394 * last thing in the list?", but in that case it's impossible to create 2395 * a shared uncompressed buf anyway (because the hdr must be compressed 2396 * to have the compressed buf). You might also think that #3 is 2397 * sufficient to make this guarantee, however it's possible 2398 * (specifically in the rare L2ARC write race mentioned in 2399 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that 2400 * is sharable, but wasn't at the time of its allocation. Rather than 2401 * allow a new shared uncompressed buf to be created and then shuffle 2402 * the list around to make it the last element, this simply disallows 2403 * sharing if the new buf isn't the first to be added. 2404 */ 2405 ASSERT3P(buf->b_hdr, ==, hdr); 2406 boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF; 2407 boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0; 2408 return (buf_compressed == hdr_compressed && 2409 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && 2410 !HDR_SHARED_DATA(hdr) && 2411 (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf))); 2412 } 2413 2414 /* 2415 * Allocate a buf for this hdr. If you care about the data that's in the hdr, 2416 * or if you want a compressed buffer, pass those flags in. Returns 0 if the 2417 * copy was made successfully, or an error code otherwise. 2418 */ 2419 static int 2420 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed, 2421 boolean_t fill, arc_buf_t **ret) 2422 { 2423 arc_buf_t *buf; 2424 2425 ASSERT(HDR_HAS_L1HDR(hdr)); 2426 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 2427 VERIFY(hdr->b_type == ARC_BUFC_DATA || 2428 hdr->b_type == ARC_BUFC_METADATA); 2429 ASSERT3P(ret, !=, NULL); 2430 ASSERT3P(*ret, ==, NULL); 2431 2432 buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2433 buf->b_hdr = hdr; 2434 buf->b_data = NULL; 2435 buf->b_next = hdr->b_l1hdr.b_buf; 2436 buf->b_flags = 0; 2437 2438 add_reference(hdr, tag); 2439 2440 /* 2441 * We're about to change the hdr's b_flags. We must either 2442 * hold the hash_lock or be undiscoverable. 2443 */ 2444 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2445 2446 /* 2447 * Only honor requests for compressed bufs if the hdr is actually 2448 * compressed. 2449 */ 2450 if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) 2451 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED; 2452 2453 /* 2454 * If the hdr's data can be shared then we share the data buffer and 2455 * set the appropriate bit in the hdr's b_flags to indicate the hdr is 2456 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new 2457 * buffer to store the buf's data. 2458 * 2459 * There are two additional restrictions here because we're sharing 2460 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be 2461 * actively involved in an L2ARC write, because if this buf is used by 2462 * an arc_write() then the hdr's data buffer will be released when the 2463 * write completes, even though the L2ARC write might still be using it. 2464 * Second, the hdr's ABD must be linear so that the buf's user doesn't 2465 * need to be ABD-aware. 2466 */ 2467 boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) && 2468 abd_is_linear(hdr->b_l1hdr.b_pabd); 2469 2470 /* Set up b_data and sharing */ 2471 if (can_share) { 2472 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd); 2473 buf->b_flags |= ARC_BUF_FLAG_SHARED; 2474 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 2475 } else { 2476 buf->b_data = 2477 arc_get_data_buf(hdr, arc_buf_size(buf), buf); 2478 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 2479 } 2480 VERIFY3P(buf->b_data, !=, NULL); 2481 2482 hdr->b_l1hdr.b_buf = buf; 2483 hdr->b_l1hdr.b_bufcnt += 1; 2484 2485 /* 2486 * If the user wants the data from the hdr, we need to either copy or 2487 * decompress the data. 2488 */ 2489 if (fill) { 2490 return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0)); 2491 } 2492 2493 return (0); 2494 } 2495 2496 static char *arc_onloan_tag = "onloan"; 2497 2498 static inline void 2499 arc_loaned_bytes_update(int64_t delta) 2500 { 2501 atomic_add_64(&arc_loaned_bytes, delta); 2502 2503 /* assert that it did not wrap around */ 2504 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); 2505 } 2506 2507 /* 2508 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 2509 * flight data by arc_tempreserve_space() until they are "returned". Loaned 2510 * buffers must be returned to the arc before they can be used by the DMU or 2511 * freed. 2512 */ 2513 arc_buf_t * 2514 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size) 2515 { 2516 arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag, 2517 is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size); 2518 2519 arc_loaned_bytes_update(size); 2520 2521 return (buf); 2522 } 2523 2524 arc_buf_t * 2525 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize, 2526 enum zio_compress compression_type) 2527 { 2528 arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag, 2529 psize, lsize, compression_type); 2530 2531 arc_loaned_bytes_update(psize); 2532 2533 return (buf); 2534 } 2535 2536 2537 /* 2538 * Return a loaned arc buffer to the arc. 2539 */ 2540 void 2541 arc_return_buf(arc_buf_t *buf, void *tag) 2542 { 2543 arc_buf_hdr_t *hdr = buf->b_hdr; 2544 2545 ASSERT3P(buf->b_data, !=, NULL); 2546 ASSERT(HDR_HAS_L1HDR(hdr)); 2547 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag); 2548 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 2549 2550 arc_loaned_bytes_update(-arc_buf_size(buf)); 2551 } 2552 2553 /* Detach an arc_buf from a dbuf (tag) */ 2554 void 2555 arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 2556 { 2557 arc_buf_hdr_t *hdr = buf->b_hdr; 2558 2559 ASSERT3P(buf->b_data, !=, NULL); 2560 ASSERT(HDR_HAS_L1HDR(hdr)); 2561 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 2562 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); 2563 2564 arc_loaned_bytes_update(arc_buf_size(buf)); 2565 } 2566 2567 static void 2568 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type) 2569 { 2570 l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP); 2571 2572 df->l2df_abd = abd; 2573 df->l2df_size = size; 2574 df->l2df_type = type; 2575 mutex_enter(&l2arc_free_on_write_mtx); 2576 list_insert_head(l2arc_free_on_write, df); 2577 mutex_exit(&l2arc_free_on_write_mtx); 2578 } 2579 2580 static void 2581 arc_hdr_free_on_write(arc_buf_hdr_t *hdr) 2582 { 2583 arc_state_t *state = hdr->b_l1hdr.b_state; 2584 arc_buf_contents_t type = arc_buf_type(hdr); 2585 uint64_t size = arc_hdr_size(hdr); 2586 2587 /* protected by hash lock, if in the hash table */ 2588 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 2589 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2590 ASSERT(state != arc_anon && state != arc_l2c_only); 2591 2592 (void) refcount_remove_many(&state->arcs_esize[type], 2593 size, hdr); 2594 } 2595 (void) refcount_remove_many(&state->arcs_size, size, hdr); 2596 if (type == ARC_BUFC_METADATA) { 2597 arc_space_return(size, ARC_SPACE_META); 2598 } else { 2599 ASSERT(type == ARC_BUFC_DATA); 2600 arc_space_return(size, ARC_SPACE_DATA); 2601 } 2602 2603 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type); 2604 } 2605 2606 /* 2607 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the 2608 * data buffer, we transfer the refcount ownership to the hdr and update 2609 * the appropriate kstats. 2610 */ 2611 static void 2612 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2613 { 2614 arc_state_t *state = hdr->b_l1hdr.b_state; 2615 2616 ASSERT(arc_can_share(hdr, buf)); 2617 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2618 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2619 2620 /* 2621 * Start sharing the data buffer. We transfer the 2622 * refcount ownership to the hdr since it always owns 2623 * the refcount whenever an arc_buf_t is shared. 2624 */ 2625 refcount_transfer_ownership(&state->arcs_size, buf, hdr); 2626 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf)); 2627 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd, 2628 HDR_ISTYPE_METADATA(hdr)); 2629 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 2630 buf->b_flags |= ARC_BUF_FLAG_SHARED; 2631 2632 /* 2633 * Since we've transferred ownership to the hdr we need 2634 * to increment its compressed and uncompressed kstats and 2635 * decrement the overhead size. 2636 */ 2637 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); 2638 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 2639 ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf)); 2640 } 2641 2642 static void 2643 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2644 { 2645 arc_state_t *state = hdr->b_l1hdr.b_state; 2646 2647 ASSERT(arc_buf_is_shared(buf)); 2648 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2649 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2650 2651 /* 2652 * We are no longer sharing this buffer so we need 2653 * to transfer its ownership to the rightful owner. 2654 */ 2655 refcount_transfer_ownership(&state->arcs_size, hdr, buf); 2656 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 2657 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); 2658 abd_put(hdr->b_l1hdr.b_pabd); 2659 hdr->b_l1hdr.b_pabd = NULL; 2660 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 2661 2662 /* 2663 * Since the buffer is no longer shared between 2664 * the arc buf and the hdr, count it as overhead. 2665 */ 2666 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); 2667 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 2668 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 2669 } 2670 2671 /* 2672 * Remove an arc_buf_t from the hdr's buf list and return the last 2673 * arc_buf_t on the list. If no buffers remain on the list then return 2674 * NULL. 2675 */ 2676 static arc_buf_t * 2677 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2678 { 2679 ASSERT(HDR_HAS_L1HDR(hdr)); 2680 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2681 2682 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf; 2683 arc_buf_t *lastbuf = NULL; 2684 2685 /* 2686 * Remove the buf from the hdr list and locate the last 2687 * remaining buffer on the list. 2688 */ 2689 while (*bufp != NULL) { 2690 if (*bufp == buf) 2691 *bufp = buf->b_next; 2692 2693 /* 2694 * If we've removed a buffer in the middle of 2695 * the list then update the lastbuf and update 2696 * bufp. 2697 */ 2698 if (*bufp != NULL) { 2699 lastbuf = *bufp; 2700 bufp = &(*bufp)->b_next; 2701 } 2702 } 2703 buf->b_next = NULL; 2704 ASSERT3P(lastbuf, !=, buf); 2705 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL); 2706 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL); 2707 IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf)); 2708 2709 return (lastbuf); 2710 } 2711 2712 /* 2713 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's 2714 * list and free it. 2715 */ 2716 static void 2717 arc_buf_destroy_impl(arc_buf_t *buf) 2718 { 2719 arc_buf_hdr_t *hdr = buf->b_hdr; 2720 2721 /* 2722 * Free up the data associated with the buf but only if we're not 2723 * sharing this with the hdr. If we are sharing it with the hdr, the 2724 * hdr is responsible for doing the free. 2725 */ 2726 if (buf->b_data != NULL) { 2727 /* 2728 * We're about to change the hdr's b_flags. We must either 2729 * hold the hash_lock or be undiscoverable. 2730 */ 2731 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2732 2733 arc_cksum_verify(buf); 2734 arc_buf_unwatch(buf); 2735 2736 if (arc_buf_is_shared(buf)) { 2737 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 2738 } else { 2739 uint64_t size = arc_buf_size(buf); 2740 arc_free_data_buf(hdr, buf->b_data, size, buf); 2741 ARCSTAT_INCR(arcstat_overhead_size, -size); 2742 } 2743 buf->b_data = NULL; 2744 2745 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 2746 hdr->b_l1hdr.b_bufcnt -= 1; 2747 } 2748 2749 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); 2750 2751 if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) { 2752 /* 2753 * If the current arc_buf_t is sharing its data buffer with the 2754 * hdr, then reassign the hdr's b_pabd to share it with the new 2755 * buffer at the end of the list. The shared buffer is always 2756 * the last one on the hdr's buffer list. 2757 * 2758 * There is an equivalent case for compressed bufs, but since 2759 * they aren't guaranteed to be the last buf in the list and 2760 * that is an exceedingly rare case, we just allow that space be 2761 * wasted temporarily. 2762 */ 2763 if (lastbuf != NULL) { 2764 /* Only one buf can be shared at once */ 2765 VERIFY(!arc_buf_is_shared(lastbuf)); 2766 /* hdr is uncompressed so can't have compressed buf */ 2767 VERIFY(!ARC_BUF_COMPRESSED(lastbuf)); 2768 2769 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2770 arc_hdr_free_pabd(hdr); 2771 2772 /* 2773 * We must setup a new shared block between the 2774 * last buffer and the hdr. The data would have 2775 * been allocated by the arc buf so we need to transfer 2776 * ownership to the hdr since it's now being shared. 2777 */ 2778 arc_share_buf(hdr, lastbuf); 2779 } 2780 } else if (HDR_SHARED_DATA(hdr)) { 2781 /* 2782 * Uncompressed shared buffers are always at the end 2783 * of the list. Compressed buffers don't have the 2784 * same requirements. This makes it hard to 2785 * simply assert that the lastbuf is shared so 2786 * we rely on the hdr's compression flags to determine 2787 * if we have a compressed, shared buffer. 2788 */ 2789 ASSERT3P(lastbuf, !=, NULL); 2790 ASSERT(arc_buf_is_shared(lastbuf) || 2791 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 2792 } 2793 2794 /* 2795 * Free the checksum if we're removing the last uncompressed buf from 2796 * this hdr. 2797 */ 2798 if (!arc_hdr_has_uncompressed_buf(hdr)) { 2799 arc_cksum_free(hdr); 2800 } 2801 2802 /* clean up the buf */ 2803 buf->b_hdr = NULL; 2804 kmem_cache_free(buf_cache, buf); 2805 } 2806 2807 static void 2808 arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr) 2809 { 2810 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 2811 ASSERT(HDR_HAS_L1HDR(hdr)); 2812 ASSERT(!HDR_SHARED_DATA(hdr)); 2813 2814 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2815 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr); 2816 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 2817 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2818 2819 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); 2820 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 2821 } 2822 2823 static void 2824 arc_hdr_free_pabd(arc_buf_hdr_t *hdr) 2825 { 2826 ASSERT(HDR_HAS_L1HDR(hdr)); 2827 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2828 2829 /* 2830 * If the hdr is currently being written to the l2arc then 2831 * we defer freeing the data by adding it to the l2arc_free_on_write 2832 * list. The l2arc will free the data once it's finished 2833 * writing it to the l2arc device. 2834 */ 2835 if (HDR_L2_WRITING(hdr)) { 2836 arc_hdr_free_on_write(hdr); 2837 ARCSTAT_BUMP(arcstat_l2_free_on_write); 2838 } else { 2839 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, 2840 arc_hdr_size(hdr), hdr); 2841 } 2842 hdr->b_l1hdr.b_pabd = NULL; 2843 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 2844 2845 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); 2846 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 2847 } 2848 2849 static arc_buf_hdr_t * 2850 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize, 2851 enum zio_compress compression_type, arc_buf_contents_t type) 2852 { 2853 arc_buf_hdr_t *hdr; 2854 2855 VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA); 2856 2857 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); 2858 ASSERT(HDR_EMPTY(hdr)); 2859 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 2860 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL); 2861 HDR_SET_PSIZE(hdr, psize); 2862 HDR_SET_LSIZE(hdr, lsize); 2863 hdr->b_spa = spa; 2864 hdr->b_type = type; 2865 hdr->b_flags = 0; 2866 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); 2867 arc_hdr_set_compress(hdr, compression_type); 2868 2869 hdr->b_l1hdr.b_state = arc_anon; 2870 hdr->b_l1hdr.b_arc_access = 0; 2871 hdr->b_l1hdr.b_bufcnt = 0; 2872 hdr->b_l1hdr.b_buf = NULL; 2873 2874 /* 2875 * Allocate the hdr's buffer. This will contain either 2876 * the compressed or uncompressed data depending on the block 2877 * it references and compressed arc enablement. 2878 */ 2879 arc_hdr_alloc_pabd(hdr); 2880 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2881 2882 return (hdr); 2883 } 2884 2885 /* 2886 * Transition between the two allocation states for the arc_buf_hdr struct. 2887 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without 2888 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller 2889 * version is used when a cache buffer is only in the L2ARC in order to reduce 2890 * memory usage. 2891 */ 2892 static arc_buf_hdr_t * 2893 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) 2894 { 2895 ASSERT(HDR_HAS_L2HDR(hdr)); 2896 2897 arc_buf_hdr_t *nhdr; 2898 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 2899 2900 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) || 2901 (old == hdr_l2only_cache && new == hdr_full_cache)); 2902 2903 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE); 2904 2905 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); 2906 buf_hash_remove(hdr); 2907 2908 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); 2909 2910 if (new == hdr_full_cache) { 2911 arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR); 2912 /* 2913 * arc_access and arc_change_state need to be aware that a 2914 * header has just come out of L2ARC, so we set its state to 2915 * l2c_only even though it's about to change. 2916 */ 2917 nhdr->b_l1hdr.b_state = arc_l2c_only; 2918 2919 /* Verify previous threads set to NULL before freeing */ 2920 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL); 2921 } else { 2922 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2923 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2924 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 2925 2926 /* 2927 * If we've reached here, We must have been called from 2928 * arc_evict_hdr(), as such we should have already been 2929 * removed from any ghost list we were previously on 2930 * (which protects us from racing with arc_evict_state), 2931 * thus no locking is needed during this check. 2932 */ 2933 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 2934 2935 /* 2936 * A buffer must not be moved into the arc_l2c_only 2937 * state if it's not finished being written out to the 2938 * l2arc device. Otherwise, the b_l1hdr.b_pabd field 2939 * might try to be accessed, even though it was removed. 2940 */ 2941 VERIFY(!HDR_L2_WRITING(hdr)); 2942 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2943 2944 #ifdef ZFS_DEBUG 2945 if (hdr->b_l1hdr.b_thawed != NULL) { 2946 kmem_free(hdr->b_l1hdr.b_thawed, 1); 2947 hdr->b_l1hdr.b_thawed = NULL; 2948 } 2949 #endif 2950 2951 arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR); 2952 } 2953 /* 2954 * The header has been reallocated so we need to re-insert it into any 2955 * lists it was on. 2956 */ 2957 (void) buf_hash_insert(nhdr, NULL); 2958 2959 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); 2960 2961 mutex_enter(&dev->l2ad_mtx); 2962 2963 /* 2964 * We must place the realloc'ed header back into the list at 2965 * the same spot. Otherwise, if it's placed earlier in the list, 2966 * l2arc_write_buffers() could find it during the function's 2967 * write phase, and try to write it out to the l2arc. 2968 */ 2969 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); 2970 list_remove(&dev->l2ad_buflist, hdr); 2971 2972 mutex_exit(&dev->l2ad_mtx); 2973 2974 /* 2975 * Since we're using the pointer address as the tag when 2976 * incrementing and decrementing the l2ad_alloc refcount, we 2977 * must remove the old pointer (that we're about to destroy) and 2978 * add the new pointer to the refcount. Otherwise we'd remove 2979 * the wrong pointer address when calling arc_hdr_destroy() later. 2980 */ 2981 2982 (void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr); 2983 (void) refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), nhdr); 2984 2985 buf_discard_identity(hdr); 2986 kmem_cache_free(old, hdr); 2987 2988 return (nhdr); 2989 } 2990 2991 /* 2992 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller. 2993 * The buf is returned thawed since we expect the consumer to modify it. 2994 */ 2995 arc_buf_t * 2996 arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size) 2997 { 2998 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, 2999 ZIO_COMPRESS_OFF, type); 3000 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3001 3002 arc_buf_t *buf = NULL; 3003 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf)); 3004 arc_buf_thaw(buf); 3005 3006 return (buf); 3007 } 3008 3009 /* 3010 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this 3011 * for bufs containing metadata. 3012 */ 3013 arc_buf_t * 3014 arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize, 3015 enum zio_compress compression_type) 3016 { 3017 ASSERT3U(lsize, >, 0); 3018 ASSERT3U(lsize, >=, psize); 3019 ASSERT(compression_type > ZIO_COMPRESS_OFF); 3020 ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS); 3021 3022 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 3023 compression_type, ARC_BUFC_DATA); 3024 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3025 3026 arc_buf_t *buf = NULL; 3027 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf)); 3028 arc_buf_thaw(buf); 3029 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3030 3031 if (!arc_buf_is_shared(buf)) { 3032 /* 3033 * To ensure that the hdr has the correct data in it if we call 3034 * arc_decompress() on this buf before it's been written to 3035 * disk, it's easiest if we just set up sharing between the 3036 * buf and the hdr. 3037 */ 3038 ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd)); 3039 arc_hdr_free_pabd(hdr); 3040 arc_share_buf(hdr, buf); 3041 } 3042 3043 return (buf); 3044 } 3045 3046 static void 3047 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) 3048 { 3049 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; 3050 l2arc_dev_t *dev = l2hdr->b_dev; 3051 uint64_t psize = arc_hdr_size(hdr); 3052 3053 ASSERT(MUTEX_HELD(&dev->l2ad_mtx)); 3054 ASSERT(HDR_HAS_L2HDR(hdr)); 3055 3056 list_remove(&dev->l2ad_buflist, hdr); 3057 3058 ARCSTAT_INCR(arcstat_l2_psize, -psize); 3059 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr)); 3060 3061 vdev_space_update(dev->l2ad_vdev, -psize, 0, 0); 3062 3063 (void) refcount_remove_many(&dev->l2ad_alloc, psize, hdr); 3064 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); 3065 } 3066 3067 static void 3068 arc_hdr_destroy(arc_buf_hdr_t *hdr) 3069 { 3070 if (HDR_HAS_L1HDR(hdr)) { 3071 ASSERT(hdr->b_l1hdr.b_buf == NULL || 3072 hdr->b_l1hdr.b_bufcnt > 0); 3073 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 3074 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 3075 } 3076 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3077 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 3078 3079 if (!HDR_EMPTY(hdr)) 3080 buf_discard_identity(hdr); 3081 3082 if (HDR_HAS_L2HDR(hdr)) { 3083 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 3084 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx); 3085 3086 if (!buflist_held) 3087 mutex_enter(&dev->l2ad_mtx); 3088 3089 /* 3090 * Even though we checked this conditional above, we 3091 * need to check this again now that we have the 3092 * l2ad_mtx. This is because we could be racing with 3093 * another thread calling l2arc_evict() which might have 3094 * destroyed this header's L2 portion as we were waiting 3095 * to acquire the l2ad_mtx. If that happens, we don't 3096 * want to re-destroy the header's L2 portion. 3097 */ 3098 if (HDR_HAS_L2HDR(hdr)) 3099 arc_hdr_l2hdr_destroy(hdr); 3100 3101 if (!buflist_held) 3102 mutex_exit(&dev->l2ad_mtx); 3103 } 3104 3105 if (HDR_HAS_L1HDR(hdr)) { 3106 arc_cksum_free(hdr); 3107 3108 while (hdr->b_l1hdr.b_buf != NULL) 3109 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf); 3110 3111 #ifdef ZFS_DEBUG 3112 if (hdr->b_l1hdr.b_thawed != NULL) { 3113 kmem_free(hdr->b_l1hdr.b_thawed, 1); 3114 hdr->b_l1hdr.b_thawed = NULL; 3115 } 3116 #endif 3117 3118 if (hdr->b_l1hdr.b_pabd != NULL) { 3119 arc_hdr_free_pabd(hdr); 3120 } 3121 } 3122 3123 ASSERT3P(hdr->b_hash_next, ==, NULL); 3124 if (HDR_HAS_L1HDR(hdr)) { 3125 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 3126 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 3127 kmem_cache_free(hdr_full_cache, hdr); 3128 } else { 3129 kmem_cache_free(hdr_l2only_cache, hdr); 3130 } 3131 } 3132 3133 void 3134 arc_buf_destroy(arc_buf_t *buf, void* tag) 3135 { 3136 arc_buf_hdr_t *hdr = buf->b_hdr; 3137 kmutex_t *hash_lock = HDR_LOCK(hdr); 3138 3139 if (hdr->b_l1hdr.b_state == arc_anon) { 3140 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 3141 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3142 VERIFY0(remove_reference(hdr, NULL, tag)); 3143 arc_hdr_destroy(hdr); 3144 return; 3145 } 3146 3147 mutex_enter(hash_lock); 3148 ASSERT3P(hdr, ==, buf->b_hdr); 3149 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 3150 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 3151 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); 3152 ASSERT3P(buf->b_data, !=, NULL); 3153 3154 (void) remove_reference(hdr, hash_lock, tag); 3155 arc_buf_destroy_impl(buf); 3156 mutex_exit(hash_lock); 3157 } 3158 3159 /* 3160 * Evict the arc_buf_hdr that is provided as a parameter. The resultant 3161 * state of the header is dependent on it's state prior to entering this 3162 * function. The following transitions are possible: 3163 * 3164 * - arc_mru -> arc_mru_ghost 3165 * - arc_mfu -> arc_mfu_ghost 3166 * - arc_mru_ghost -> arc_l2c_only 3167 * - arc_mru_ghost -> deleted 3168 * - arc_mfu_ghost -> arc_l2c_only 3169 * - arc_mfu_ghost -> deleted 3170 */ 3171 static int64_t 3172 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 3173 { 3174 arc_state_t *evicted_state, *state; 3175 int64_t bytes_evicted = 0; 3176 3177 ASSERT(MUTEX_HELD(hash_lock)); 3178 ASSERT(HDR_HAS_L1HDR(hdr)); 3179 3180 state = hdr->b_l1hdr.b_state; 3181 if (GHOST_STATE(state)) { 3182 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3183 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 3184 3185 /* 3186 * l2arc_write_buffers() relies on a header's L1 portion 3187 * (i.e. its b_pabd field) during it's write phase. 3188 * Thus, we cannot push a header onto the arc_l2c_only 3189 * state (removing it's L1 piece) until the header is 3190 * done being written to the l2arc. 3191 */ 3192 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { 3193 ARCSTAT_BUMP(arcstat_evict_l2_skip); 3194 return (bytes_evicted); 3195 } 3196 3197 ARCSTAT_BUMP(arcstat_deleted); 3198 bytes_evicted += HDR_GET_LSIZE(hdr); 3199 3200 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); 3201 3202 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3203 if (HDR_HAS_L2HDR(hdr)) { 3204 /* 3205 * This buffer is cached on the 2nd Level ARC; 3206 * don't destroy the header. 3207 */ 3208 arc_change_state(arc_l2c_only, hdr, hash_lock); 3209 /* 3210 * dropping from L1+L2 cached to L2-only, 3211 * realloc to remove the L1 header. 3212 */ 3213 hdr = arc_hdr_realloc(hdr, hdr_full_cache, 3214 hdr_l2only_cache); 3215 } else { 3216 arc_change_state(arc_anon, hdr, hash_lock); 3217 arc_hdr_destroy(hdr); 3218 } 3219 return (bytes_evicted); 3220 } 3221 3222 ASSERT(state == arc_mru || state == arc_mfu); 3223 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 3224 3225 /* prefetch buffers have a minimum lifespan */ 3226 if (HDR_IO_IN_PROGRESS(hdr) || 3227 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && 3228 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < 3229 arc_min_prefetch_lifespan)) { 3230 ARCSTAT_BUMP(arcstat_evict_skip); 3231 return (bytes_evicted); 3232 } 3233 3234 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); 3235 while (hdr->b_l1hdr.b_buf) { 3236 arc_buf_t *buf = hdr->b_l1hdr.b_buf; 3237 if (!mutex_tryenter(&buf->b_evict_lock)) { 3238 ARCSTAT_BUMP(arcstat_mutex_miss); 3239 break; 3240 } 3241 if (buf->b_data != NULL) 3242 bytes_evicted += HDR_GET_LSIZE(hdr); 3243 mutex_exit(&buf->b_evict_lock); 3244 arc_buf_destroy_impl(buf); 3245 } 3246 3247 if (HDR_HAS_L2HDR(hdr)) { 3248 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); 3249 } else { 3250 if (l2arc_write_eligible(hdr->b_spa, hdr)) { 3251 ARCSTAT_INCR(arcstat_evict_l2_eligible, 3252 HDR_GET_LSIZE(hdr)); 3253 } else { 3254 ARCSTAT_INCR(arcstat_evict_l2_ineligible, 3255 HDR_GET_LSIZE(hdr)); 3256 } 3257 } 3258 3259 if (hdr->b_l1hdr.b_bufcnt == 0) { 3260 arc_cksum_free(hdr); 3261 3262 bytes_evicted += arc_hdr_size(hdr); 3263 3264 /* 3265 * If this hdr is being evicted and has a compressed 3266 * buffer then we discard it here before we change states. 3267 * This ensures that the accounting is updated correctly 3268 * in arc_free_data_impl(). 3269 */ 3270 arc_hdr_free_pabd(hdr); 3271 3272 arc_change_state(evicted_state, hdr, hash_lock); 3273 ASSERT(HDR_IN_HASH_TABLE(hdr)); 3274 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 3275 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); 3276 } 3277 3278 return (bytes_evicted); 3279 } 3280 3281 static uint64_t 3282 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker, 3283 uint64_t spa, int64_t bytes) 3284 { 3285 multilist_sublist_t *mls; 3286 uint64_t bytes_evicted = 0; 3287 arc_buf_hdr_t *hdr; 3288 kmutex_t *hash_lock; 3289 int evict_count = 0; 3290 3291 ASSERT3P(marker, !=, NULL); 3292 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 3293 3294 mls = multilist_sublist_lock(ml, idx); 3295 3296 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; 3297 hdr = multilist_sublist_prev(mls, marker)) { 3298 if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) || 3299 (evict_count >= zfs_arc_evict_batch_limit)) 3300 break; 3301 3302 /* 3303 * To keep our iteration location, move the marker 3304 * forward. Since we're not holding hdr's hash lock, we 3305 * must be very careful and not remove 'hdr' from the 3306 * sublist. Otherwise, other consumers might mistake the 3307 * 'hdr' as not being on a sublist when they call the 3308 * multilist_link_active() function (they all rely on 3309 * the hash lock protecting concurrent insertions and 3310 * removals). multilist_sublist_move_forward() was 3311 * specifically implemented to ensure this is the case 3312 * (only 'marker' will be removed and re-inserted). 3313 */ 3314 multilist_sublist_move_forward(mls, marker); 3315 3316 /* 3317 * The only case where the b_spa field should ever be 3318 * zero, is the marker headers inserted by 3319 * arc_evict_state(). It's possible for multiple threads 3320 * to be calling arc_evict_state() concurrently (e.g. 3321 * dsl_pool_close() and zio_inject_fault()), so we must 3322 * skip any markers we see from these other threads. 3323 */ 3324 if (hdr->b_spa == 0) 3325 continue; 3326 3327 /* we're only interested in evicting buffers of a certain spa */ 3328 if (spa != 0 && hdr->b_spa != spa) { 3329 ARCSTAT_BUMP(arcstat_evict_skip); 3330 continue; 3331 } 3332 3333 hash_lock = HDR_LOCK(hdr); 3334 3335 /* 3336 * We aren't calling this function from any code path 3337 * that would already be holding a hash lock, so we're 3338 * asserting on this assumption to be defensive in case 3339 * this ever changes. Without this check, it would be 3340 * possible to incorrectly increment arcstat_mutex_miss 3341 * below (e.g. if the code changed such that we called 3342 * this function with a hash lock held). 3343 */ 3344 ASSERT(!MUTEX_HELD(hash_lock)); 3345 3346 if (mutex_tryenter(hash_lock)) { 3347 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); 3348 mutex_exit(hash_lock); 3349 3350 bytes_evicted += evicted; 3351 3352 /* 3353 * If evicted is zero, arc_evict_hdr() must have 3354 * decided to skip this header, don't increment 3355 * evict_count in this case. 3356 */ 3357 if (evicted != 0) 3358 evict_count++; 3359 3360 /* 3361 * If arc_size isn't overflowing, signal any 3362 * threads that might happen to be waiting. 3363 * 3364 * For each header evicted, we wake up a single 3365 * thread. If we used cv_broadcast, we could 3366 * wake up "too many" threads causing arc_size 3367 * to significantly overflow arc_c; since 3368 * arc_get_data_impl() doesn't check for overflow 3369 * when it's woken up (it doesn't because it's 3370 * possible for the ARC to be overflowing while 3371 * full of un-evictable buffers, and the 3372 * function should proceed in this case). 3373 * 3374 * If threads are left sleeping, due to not 3375 * using cv_broadcast, they will be woken up 3376 * just before arc_reclaim_thread() sleeps. 3377 */ 3378 mutex_enter(&arc_reclaim_lock); 3379 if (!arc_is_overflowing()) 3380 cv_signal(&arc_reclaim_waiters_cv); 3381 mutex_exit(&arc_reclaim_lock); 3382 } else { 3383 ARCSTAT_BUMP(arcstat_mutex_miss); 3384 } 3385 } 3386 3387 multilist_sublist_unlock(mls); 3388 3389 return (bytes_evicted); 3390 } 3391 3392 /* 3393 * Evict buffers from the given arc state, until we've removed the 3394 * specified number of bytes. Move the removed buffers to the 3395 * appropriate evict state. 3396 * 3397 * This function makes a "best effort". It skips over any buffers 3398 * it can't get a hash_lock on, and so, may not catch all candidates. 3399 * It may also return without evicting as much space as requested. 3400 * 3401 * If bytes is specified using the special value ARC_EVICT_ALL, this 3402 * will evict all available (i.e. unlocked and evictable) buffers from 3403 * the given arc state; which is used by arc_flush(). 3404 */ 3405 static uint64_t 3406 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes, 3407 arc_buf_contents_t type) 3408 { 3409 uint64_t total_evicted = 0; 3410 multilist_t *ml = state->arcs_list[type]; 3411 int num_sublists; 3412 arc_buf_hdr_t **markers; 3413 3414 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 3415 3416 num_sublists = multilist_get_num_sublists(ml); 3417 3418 /* 3419 * If we've tried to evict from each sublist, made some 3420 * progress, but still have not hit the target number of bytes 3421 * to evict, we want to keep trying. The markers allow us to 3422 * pick up where we left off for each individual sublist, rather 3423 * than starting from the tail each time. 3424 */ 3425 markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP); 3426 for (int i = 0; i < num_sublists; i++) { 3427 markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP); 3428 3429 /* 3430 * A b_spa of 0 is used to indicate that this header is 3431 * a marker. This fact is used in arc_adjust_type() and 3432 * arc_evict_state_impl(). 3433 */ 3434 markers[i]->b_spa = 0; 3435 3436 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 3437 multilist_sublist_insert_tail(mls, markers[i]); 3438 multilist_sublist_unlock(mls); 3439 } 3440 3441 /* 3442 * While we haven't hit our target number of bytes to evict, or 3443 * we're evicting all available buffers. 3444 */ 3445 while (total_evicted < bytes || bytes == ARC_EVICT_ALL) { 3446 /* 3447 * Start eviction using a randomly selected sublist, 3448 * this is to try and evenly balance eviction across all 3449 * sublists. Always starting at the same sublist 3450 * (e.g. index 0) would cause evictions to favor certain 3451 * sublists over others. 3452 */ 3453 int sublist_idx = multilist_get_random_index(ml); 3454 uint64_t scan_evicted = 0; 3455 3456 for (int i = 0; i < num_sublists; i++) { 3457 uint64_t bytes_remaining; 3458 uint64_t bytes_evicted; 3459 3460 if (bytes == ARC_EVICT_ALL) 3461 bytes_remaining = ARC_EVICT_ALL; 3462 else if (total_evicted < bytes) 3463 bytes_remaining = bytes - total_evicted; 3464 else 3465 break; 3466 3467 bytes_evicted = arc_evict_state_impl(ml, sublist_idx, 3468 markers[sublist_idx], spa, bytes_remaining); 3469 3470 scan_evicted += bytes_evicted; 3471 total_evicted += bytes_evicted; 3472 3473 /* we've reached the end, wrap to the beginning */ 3474 if (++sublist_idx >= num_sublists) 3475 sublist_idx = 0; 3476 } 3477 3478 /* 3479 * If we didn't evict anything during this scan, we have 3480 * no reason to believe we'll evict more during another 3481 * scan, so break the loop. 3482 */ 3483 if (scan_evicted == 0) { 3484 /* This isn't possible, let's make that obvious */ 3485 ASSERT3S(bytes, !=, 0); 3486 3487 /* 3488 * When bytes is ARC_EVICT_ALL, the only way to 3489 * break the loop is when scan_evicted is zero. 3490 * In that case, we actually have evicted enough, 3491 * so we don't want to increment the kstat. 3492 */ 3493 if (bytes != ARC_EVICT_ALL) { 3494 ASSERT3S(total_evicted, <, bytes); 3495 ARCSTAT_BUMP(arcstat_evict_not_enough); 3496 } 3497 3498 break; 3499 } 3500 } 3501 3502 for (int i = 0; i < num_sublists; i++) { 3503 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 3504 multilist_sublist_remove(mls, markers[i]); 3505 multilist_sublist_unlock(mls); 3506 3507 kmem_cache_free(hdr_full_cache, markers[i]); 3508 } 3509 kmem_free(markers, sizeof (*markers) * num_sublists); 3510 3511 return (total_evicted); 3512 } 3513 3514 /* 3515 * Flush all "evictable" data of the given type from the arc state 3516 * specified. This will not evict any "active" buffers (i.e. referenced). 3517 * 3518 * When 'retry' is set to B_FALSE, the function will make a single pass 3519 * over the state and evict any buffers that it can. Since it doesn't 3520 * continually retry the eviction, it might end up leaving some buffers 3521 * in the ARC due to lock misses. 3522 * 3523 * When 'retry' is set to B_TRUE, the function will continually retry the 3524 * eviction until *all* evictable buffers have been removed from the 3525 * state. As a result, if concurrent insertions into the state are 3526 * allowed (e.g. if the ARC isn't shutting down), this function might 3527 * wind up in an infinite loop, continually trying to evict buffers. 3528 */ 3529 static uint64_t 3530 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type, 3531 boolean_t retry) 3532 { 3533 uint64_t evicted = 0; 3534 3535 while (refcount_count(&state->arcs_esize[type]) != 0) { 3536 evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type); 3537 3538 if (!retry) 3539 break; 3540 } 3541 3542 return (evicted); 3543 } 3544 3545 /* 3546 * Evict the specified number of bytes from the state specified, 3547 * restricting eviction to the spa and type given. This function 3548 * prevents us from trying to evict more from a state's list than 3549 * is "evictable", and to skip evicting altogether when passed a 3550 * negative value for "bytes". In contrast, arc_evict_state() will 3551 * evict everything it can, when passed a negative value for "bytes". 3552 */ 3553 static uint64_t 3554 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes, 3555 arc_buf_contents_t type) 3556 { 3557 int64_t delta; 3558 3559 if (bytes > 0 && refcount_count(&state->arcs_esize[type]) > 0) { 3560 delta = MIN(refcount_count(&state->arcs_esize[type]), bytes); 3561 return (arc_evict_state(state, spa, delta, type)); 3562 } 3563 3564 return (0); 3565 } 3566 3567 /* 3568 * Evict metadata buffers from the cache, such that arc_meta_used is 3569 * capped by the arc_meta_limit tunable. 3570 */ 3571 static uint64_t 3572 arc_adjust_meta(void) 3573 { 3574 uint64_t total_evicted = 0; 3575 int64_t target; 3576 3577 /* 3578 * If we're over the meta limit, we want to evict enough 3579 * metadata to get back under the meta limit. We don't want to 3580 * evict so much that we drop the MRU below arc_p, though. If 3581 * we're over the meta limit more than we're over arc_p, we 3582 * evict some from the MRU here, and some from the MFU below. 3583 */ 3584 target = MIN((int64_t)(arc_meta_used - arc_meta_limit), 3585 (int64_t)(refcount_count(&arc_anon->arcs_size) + 3586 refcount_count(&arc_mru->arcs_size) - arc_p)); 3587 3588 total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3589 3590 /* 3591 * Similar to the above, we want to evict enough bytes to get us 3592 * below the meta limit, but not so much as to drop us below the 3593 * space allotted to the MFU (which is defined as arc_c - arc_p). 3594 */ 3595 target = MIN((int64_t)(arc_meta_used - arc_meta_limit), 3596 (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p))); 3597 3598 total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3599 3600 return (total_evicted); 3601 } 3602 3603 /* 3604 * Return the type of the oldest buffer in the given arc state 3605 * 3606 * This function will select a random sublist of type ARC_BUFC_DATA and 3607 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist 3608 * is compared, and the type which contains the "older" buffer will be 3609 * returned. 3610 */ 3611 static arc_buf_contents_t 3612 arc_adjust_type(arc_state_t *state) 3613 { 3614 multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA]; 3615 multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA]; 3616 int data_idx = multilist_get_random_index(data_ml); 3617 int meta_idx = multilist_get_random_index(meta_ml); 3618 multilist_sublist_t *data_mls; 3619 multilist_sublist_t *meta_mls; 3620 arc_buf_contents_t type; 3621 arc_buf_hdr_t *data_hdr; 3622 arc_buf_hdr_t *meta_hdr; 3623 3624 /* 3625 * We keep the sublist lock until we're finished, to prevent 3626 * the headers from being destroyed via arc_evict_state(). 3627 */ 3628 data_mls = multilist_sublist_lock(data_ml, data_idx); 3629 meta_mls = multilist_sublist_lock(meta_ml, meta_idx); 3630 3631 /* 3632 * These two loops are to ensure we skip any markers that 3633 * might be at the tail of the lists due to arc_evict_state(). 3634 */ 3635 3636 for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL; 3637 data_hdr = multilist_sublist_prev(data_mls, data_hdr)) { 3638 if (data_hdr->b_spa != 0) 3639 break; 3640 } 3641 3642 for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL; 3643 meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) { 3644 if (meta_hdr->b_spa != 0) 3645 break; 3646 } 3647 3648 if (data_hdr == NULL && meta_hdr == NULL) { 3649 type = ARC_BUFC_DATA; 3650 } else if (data_hdr == NULL) { 3651 ASSERT3P(meta_hdr, !=, NULL); 3652 type = ARC_BUFC_METADATA; 3653 } else if (meta_hdr == NULL) { 3654 ASSERT3P(data_hdr, !=, NULL); 3655 type = ARC_BUFC_DATA; 3656 } else { 3657 ASSERT3P(data_hdr, !=, NULL); 3658 ASSERT3P(meta_hdr, !=, NULL); 3659 3660 /* The headers can't be on the sublist without an L1 header */ 3661 ASSERT(HDR_HAS_L1HDR(data_hdr)); 3662 ASSERT(HDR_HAS_L1HDR(meta_hdr)); 3663 3664 if (data_hdr->b_l1hdr.b_arc_access < 3665 meta_hdr->b_l1hdr.b_arc_access) { 3666 type = ARC_BUFC_DATA; 3667 } else { 3668 type = ARC_BUFC_METADATA; 3669 } 3670 } 3671 3672 multilist_sublist_unlock(meta_mls); 3673 multilist_sublist_unlock(data_mls); 3674 3675 return (type); 3676 } 3677 3678 /* 3679 * Evict buffers from the cache, such that arc_size is capped by arc_c. 3680 */ 3681 static uint64_t 3682 arc_adjust(void) 3683 { 3684 uint64_t total_evicted = 0; 3685 uint64_t bytes; 3686 int64_t target; 3687 3688 /* 3689 * If we're over arc_meta_limit, we want to correct that before 3690 * potentially evicting data buffers below. 3691 */ 3692 total_evicted += arc_adjust_meta(); 3693 3694 /* 3695 * Adjust MRU size 3696 * 3697 * If we're over the target cache size, we want to evict enough 3698 * from the list to get back to our target size. We don't want 3699 * to evict too much from the MRU, such that it drops below 3700 * arc_p. So, if we're over our target cache size more than 3701 * the MRU is over arc_p, we'll evict enough to get back to 3702 * arc_p here, and then evict more from the MFU below. 3703 */ 3704 target = MIN((int64_t)(arc_size - arc_c), 3705 (int64_t)(refcount_count(&arc_anon->arcs_size) + 3706 refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p)); 3707 3708 /* 3709 * If we're below arc_meta_min, always prefer to evict data. 3710 * Otherwise, try to satisfy the requested number of bytes to 3711 * evict from the type which contains older buffers; in an 3712 * effort to keep newer buffers in the cache regardless of their 3713 * type. If we cannot satisfy the number of bytes from this 3714 * type, spill over into the next type. 3715 */ 3716 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA && 3717 arc_meta_used > arc_meta_min) { 3718 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3719 total_evicted += bytes; 3720 3721 /* 3722 * If we couldn't evict our target number of bytes from 3723 * metadata, we try to get the rest from data. 3724 */ 3725 target -= bytes; 3726 3727 total_evicted += 3728 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 3729 } else { 3730 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 3731 total_evicted += bytes; 3732 3733 /* 3734 * If we couldn't evict our target number of bytes from 3735 * data, we try to get the rest from metadata. 3736 */ 3737 target -= bytes; 3738 3739 total_evicted += 3740 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3741 } 3742 3743 /* 3744 * Adjust MFU size 3745 * 3746 * Now that we've tried to evict enough from the MRU to get its 3747 * size back to arc_p, if we're still above the target cache 3748 * size, we evict the rest from the MFU. 3749 */ 3750 target = arc_size - arc_c; 3751 3752 if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA && 3753 arc_meta_used > arc_meta_min) { 3754 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3755 total_evicted += bytes; 3756 3757 /* 3758 * If we couldn't evict our target number of bytes from 3759 * metadata, we try to get the rest from data. 3760 */ 3761 target -= bytes; 3762 3763 total_evicted += 3764 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 3765 } else { 3766 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 3767 total_evicted += bytes; 3768 3769 /* 3770 * If we couldn't evict our target number of bytes from 3771 * data, we try to get the rest from data. 3772 */ 3773 target -= bytes; 3774 3775 total_evicted += 3776 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3777 } 3778 3779 /* 3780 * Adjust ghost lists 3781 * 3782 * In addition to the above, the ARC also defines target values 3783 * for the ghost lists. The sum of the mru list and mru ghost 3784 * list should never exceed the target size of the cache, and 3785 * the sum of the mru list, mfu list, mru ghost list, and mfu 3786 * ghost list should never exceed twice the target size of the 3787 * cache. The following logic enforces these limits on the ghost 3788 * caches, and evicts from them as needed. 3789 */ 3790 target = refcount_count(&arc_mru->arcs_size) + 3791 refcount_count(&arc_mru_ghost->arcs_size) - arc_c; 3792 3793 bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA); 3794 total_evicted += bytes; 3795 3796 target -= bytes; 3797 3798 total_evicted += 3799 arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA); 3800 3801 /* 3802 * We assume the sum of the mru list and mfu list is less than 3803 * or equal to arc_c (we enforced this above), which means we 3804 * can use the simpler of the two equations below: 3805 * 3806 * mru + mfu + mru ghost + mfu ghost <= 2 * arc_c 3807 * mru ghost + mfu ghost <= arc_c 3808 */ 3809 target = refcount_count(&arc_mru_ghost->arcs_size) + 3810 refcount_count(&arc_mfu_ghost->arcs_size) - arc_c; 3811 3812 bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA); 3813 total_evicted += bytes; 3814 3815 target -= bytes; 3816 3817 total_evicted += 3818 arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA); 3819 3820 return (total_evicted); 3821 } 3822 3823 void 3824 arc_flush(spa_t *spa, boolean_t retry) 3825 { 3826 uint64_t guid = 0; 3827 3828 /* 3829 * If retry is B_TRUE, a spa must not be specified since we have 3830 * no good way to determine if all of a spa's buffers have been 3831 * evicted from an arc state. 3832 */ 3833 ASSERT(!retry || spa == 0); 3834 3835 if (spa != NULL) 3836 guid = spa_load_guid(spa); 3837 3838 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry); 3839 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry); 3840 3841 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry); 3842 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry); 3843 3844 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry); 3845 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry); 3846 3847 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry); 3848 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry); 3849 } 3850 3851 void 3852 arc_shrink(int64_t to_free) 3853 { 3854 if (arc_c > arc_c_min) { 3855 3856 if (arc_c > arc_c_min + to_free) 3857 atomic_add_64(&arc_c, -to_free); 3858 else 3859 arc_c = arc_c_min; 3860 3861 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 3862 if (arc_c > arc_size) 3863 arc_c = MAX(arc_size, arc_c_min); 3864 if (arc_p > arc_c) 3865 arc_p = (arc_c >> 1); 3866 ASSERT(arc_c >= arc_c_min); 3867 ASSERT((int64_t)arc_p >= 0); 3868 } 3869 3870 if (arc_size > arc_c) 3871 (void) arc_adjust(); 3872 } 3873 3874 typedef enum free_memory_reason_t { 3875 FMR_UNKNOWN, 3876 FMR_NEEDFREE, 3877 FMR_LOTSFREE, 3878 FMR_SWAPFS_MINFREE, 3879 FMR_PAGES_PP_MAXIMUM, 3880 FMR_HEAP_ARENA, 3881 FMR_ZIO_ARENA, 3882 } free_memory_reason_t; 3883 3884 int64_t last_free_memory; 3885 free_memory_reason_t last_free_reason; 3886 3887 /* 3888 * Additional reserve of pages for pp_reserve. 3889 */ 3890 int64_t arc_pages_pp_reserve = 64; 3891 3892 /* 3893 * Additional reserve of pages for swapfs. 3894 */ 3895 int64_t arc_swapfs_reserve = 64; 3896 3897 /* 3898 * Return the amount of memory that can be consumed before reclaim will be 3899 * needed. Positive if there is sufficient free memory, negative indicates 3900 * the amount of memory that needs to be freed up. 3901 */ 3902 static int64_t 3903 arc_available_memory(void) 3904 { 3905 int64_t lowest = INT64_MAX; 3906 int64_t n; 3907 free_memory_reason_t r = FMR_UNKNOWN; 3908 3909 #ifdef _KERNEL 3910 if (needfree > 0) { 3911 n = PAGESIZE * (-needfree); 3912 if (n < lowest) { 3913 lowest = n; 3914 r = FMR_NEEDFREE; 3915 } 3916 } 3917 3918 /* 3919 * check that we're out of range of the pageout scanner. It starts to 3920 * schedule paging if freemem is less than lotsfree and needfree. 3921 * lotsfree is the high-water mark for pageout, and needfree is the 3922 * number of needed free pages. We add extra pages here to make sure 3923 * the scanner doesn't start up while we're freeing memory. 3924 */ 3925 n = PAGESIZE * (freemem - lotsfree - needfree - desfree); 3926 if (n < lowest) { 3927 lowest = n; 3928 r = FMR_LOTSFREE; 3929 } 3930 3931 /* 3932 * check to make sure that swapfs has enough space so that anon 3933 * reservations can still succeed. anon_resvmem() checks that the 3934 * availrmem is greater than swapfs_minfree, and the number of reserved 3935 * swap pages. We also add a bit of extra here just to prevent 3936 * circumstances from getting really dire. 3937 */ 3938 n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve - 3939 desfree - arc_swapfs_reserve); 3940 if (n < lowest) { 3941 lowest = n; 3942 r = FMR_SWAPFS_MINFREE; 3943 } 3944 3945 3946 /* 3947 * Check that we have enough availrmem that memory locking (e.g., via 3948 * mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum 3949 * stores the number of pages that cannot be locked; when availrmem 3950 * drops below pages_pp_maximum, page locking mechanisms such as 3951 * page_pp_lock() will fail.) 3952 */ 3953 n = PAGESIZE * (availrmem - pages_pp_maximum - 3954 arc_pages_pp_reserve); 3955 if (n < lowest) { 3956 lowest = n; 3957 r = FMR_PAGES_PP_MAXIMUM; 3958 } 3959 3960 #if defined(__i386) 3961 /* 3962 * If we're on an i386 platform, it's possible that we'll exhaust the 3963 * kernel heap space before we ever run out of available physical 3964 * memory. Most checks of the size of the heap_area compare against 3965 * tune.t_minarmem, which is the minimum available real memory that we 3966 * can have in the system. However, this is generally fixed at 25 pages 3967 * which is so low that it's useless. In this comparison, we seek to 3968 * calculate the total heap-size, and reclaim if more than 3/4ths of the 3969 * heap is allocated. (Or, in the calculation, if less than 1/4th is 3970 * free) 3971 */ 3972 n = (int64_t)vmem_size(heap_arena, VMEM_FREE) - 3973 (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2); 3974 if (n < lowest) { 3975 lowest = n; 3976 r = FMR_HEAP_ARENA; 3977 } 3978 #endif 3979 3980 /* 3981 * If zio data pages are being allocated out of a separate heap segment, 3982 * then enforce that the size of available vmem for this arena remains 3983 * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free. 3984 * 3985 * Note that reducing the arc_zio_arena_free_shift keeps more virtual 3986 * memory (in the zio_arena) free, which can avoid memory 3987 * fragmentation issues. 3988 */ 3989 if (zio_arena != NULL) { 3990 n = (int64_t)vmem_size(zio_arena, VMEM_FREE) - 3991 (vmem_size(zio_arena, VMEM_ALLOC) >> 3992 arc_zio_arena_free_shift); 3993 if (n < lowest) { 3994 lowest = n; 3995 r = FMR_ZIO_ARENA; 3996 } 3997 } 3998 #else 3999 /* Every 100 calls, free a small amount */ 4000 if (spa_get_random(100) == 0) 4001 lowest = -1024; 4002 #endif 4003 4004 last_free_memory = lowest; 4005 last_free_reason = r; 4006 4007 return (lowest); 4008 } 4009 4010 4011 /* 4012 * Determine if the system is under memory pressure and is asking 4013 * to reclaim memory. A return value of B_TRUE indicates that the system 4014 * is under memory pressure and that the arc should adjust accordingly. 4015 */ 4016 static boolean_t 4017 arc_reclaim_needed(void) 4018 { 4019 return (arc_available_memory() < 0); 4020 } 4021 4022 static void 4023 arc_kmem_reap_now(void) 4024 { 4025 size_t i; 4026 kmem_cache_t *prev_cache = NULL; 4027 kmem_cache_t *prev_data_cache = NULL; 4028 extern kmem_cache_t *zio_buf_cache[]; 4029 extern kmem_cache_t *zio_data_buf_cache[]; 4030 extern kmem_cache_t *range_seg_cache; 4031 extern kmem_cache_t *abd_chunk_cache; 4032 4033 #ifdef _KERNEL 4034 if (arc_meta_used >= arc_meta_limit) { 4035 /* 4036 * We are exceeding our meta-data cache limit. 4037 * Purge some DNLC entries to release holds on meta-data. 4038 */ 4039 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 4040 } 4041 #if defined(__i386) 4042 /* 4043 * Reclaim unused memory from all kmem caches. 4044 */ 4045 kmem_reap(); 4046 #endif 4047 #endif 4048 4049 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 4050 if (zio_buf_cache[i] != prev_cache) { 4051 prev_cache = zio_buf_cache[i]; 4052 kmem_cache_reap_now(zio_buf_cache[i]); 4053 } 4054 if (zio_data_buf_cache[i] != prev_data_cache) { 4055 prev_data_cache = zio_data_buf_cache[i]; 4056 kmem_cache_reap_now(zio_data_buf_cache[i]); 4057 } 4058 } 4059 kmem_cache_reap_now(abd_chunk_cache); 4060 kmem_cache_reap_now(buf_cache); 4061 kmem_cache_reap_now(hdr_full_cache); 4062 kmem_cache_reap_now(hdr_l2only_cache); 4063 kmem_cache_reap_now(range_seg_cache); 4064 4065 if (zio_arena != NULL) { 4066 /* 4067 * Ask the vmem arena to reclaim unused memory from its 4068 * quantum caches. 4069 */ 4070 vmem_qcache_reap(zio_arena); 4071 } 4072 } 4073 4074 /* 4075 * Threads can block in arc_get_data_impl() waiting for this thread to evict 4076 * enough data and signal them to proceed. When this happens, the threads in 4077 * arc_get_data_impl() are sleeping while holding the hash lock for their 4078 * particular arc header. Thus, we must be careful to never sleep on a 4079 * hash lock in this thread. This is to prevent the following deadlock: 4080 * 4081 * - Thread A sleeps on CV in arc_get_data_impl() holding hash lock "L", 4082 * waiting for the reclaim thread to signal it. 4083 * 4084 * - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter, 4085 * fails, and goes to sleep forever. 4086 * 4087 * This possible deadlock is avoided by always acquiring a hash lock 4088 * using mutex_tryenter() from arc_reclaim_thread(). 4089 */ 4090 /* ARGSUSED */ 4091 static void 4092 arc_reclaim_thread(void *unused) 4093 { 4094 hrtime_t growtime = 0; 4095 callb_cpr_t cpr; 4096 4097 CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG); 4098 4099 mutex_enter(&arc_reclaim_lock); 4100 while (!arc_reclaim_thread_exit) { 4101 uint64_t evicted = 0; 4102 4103 /* 4104 * This is necessary in order for the mdb ::arc dcmd to 4105 * show up to date information. Since the ::arc command 4106 * does not call the kstat's update function, without 4107 * this call, the command may show stale stats for the 4108 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even 4109 * with this change, the data might be up to 1 second 4110 * out of date; but that should suffice. The arc_state_t 4111 * structures can be queried directly if more accurate 4112 * information is needed. 4113 */ 4114 if (arc_ksp != NULL) 4115 arc_ksp->ks_update(arc_ksp, KSTAT_READ); 4116 4117 mutex_exit(&arc_reclaim_lock); 4118 4119 /* 4120 * We call arc_adjust() before (possibly) calling 4121 * arc_kmem_reap_now(), so that we can wake up 4122 * arc_get_data_impl() sooner. 4123 */ 4124 evicted = arc_adjust(); 4125 4126 int64_t free_memory = arc_available_memory(); 4127 if (free_memory < 0) { 4128 4129 arc_no_grow = B_TRUE; 4130 arc_warm = B_TRUE; 4131 4132 /* 4133 * Wait at least zfs_grow_retry (default 60) seconds 4134 * before considering growing. 4135 */ 4136 growtime = gethrtime() + SEC2NSEC(arc_grow_retry); 4137 4138 arc_kmem_reap_now(); 4139 4140 /* 4141 * If we are still low on memory, shrink the ARC 4142 * so that we have arc_shrink_min free space. 4143 */ 4144 free_memory = arc_available_memory(); 4145 4146 int64_t to_free = 4147 (arc_c >> arc_shrink_shift) - free_memory; 4148 if (to_free > 0) { 4149 #ifdef _KERNEL 4150 to_free = MAX(to_free, ptob(needfree)); 4151 #endif 4152 arc_shrink(to_free); 4153 } 4154 } else if (free_memory < arc_c >> arc_no_grow_shift) { 4155 arc_no_grow = B_TRUE; 4156 } else if (gethrtime() >= growtime) { 4157 arc_no_grow = B_FALSE; 4158 } 4159 4160 mutex_enter(&arc_reclaim_lock); 4161 4162 /* 4163 * If evicted is zero, we couldn't evict anything via 4164 * arc_adjust(). This could be due to hash lock 4165 * collisions, but more likely due to the majority of 4166 * arc buffers being unevictable. Therefore, even if 4167 * arc_size is above arc_c, another pass is unlikely to 4168 * be helpful and could potentially cause us to enter an 4169 * infinite loop. 4170 */ 4171 if (arc_size <= arc_c || evicted == 0) { 4172 /* 4173 * We're either no longer overflowing, or we 4174 * can't evict anything more, so we should wake 4175 * up any threads before we go to sleep. 4176 */ 4177 cv_broadcast(&arc_reclaim_waiters_cv); 4178 4179 /* 4180 * Block until signaled, or after one second (we 4181 * might need to perform arc_kmem_reap_now() 4182 * even if we aren't being signalled) 4183 */ 4184 CALLB_CPR_SAFE_BEGIN(&cpr); 4185 (void) cv_timedwait_hires(&arc_reclaim_thread_cv, 4186 &arc_reclaim_lock, SEC2NSEC(1), MSEC2NSEC(1), 0); 4187 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock); 4188 } 4189 } 4190 4191 arc_reclaim_thread_exit = B_FALSE; 4192 cv_broadcast(&arc_reclaim_thread_cv); 4193 CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_lock */ 4194 thread_exit(); 4195 } 4196 4197 /* 4198 * Adapt arc info given the number of bytes we are trying to add and 4199 * the state that we are comming from. This function is only called 4200 * when we are adding new content to the cache. 4201 */ 4202 static void 4203 arc_adapt(int bytes, arc_state_t *state) 4204 { 4205 int mult; 4206 uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 4207 int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size); 4208 int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size); 4209 4210 if (state == arc_l2c_only) 4211 return; 4212 4213 ASSERT(bytes > 0); 4214 /* 4215 * Adapt the target size of the MRU list: 4216 * - if we just hit in the MRU ghost list, then increase 4217 * the target size of the MRU list. 4218 * - if we just hit in the MFU ghost list, then increase 4219 * the target size of the MFU list by decreasing the 4220 * target size of the MRU list. 4221 */ 4222 if (state == arc_mru_ghost) { 4223 mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size); 4224 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */ 4225 4226 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 4227 } else if (state == arc_mfu_ghost) { 4228 uint64_t delta; 4229 4230 mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size); 4231 mult = MIN(mult, 10); 4232 4233 delta = MIN(bytes * mult, arc_p); 4234 arc_p = MAX(arc_p_min, arc_p - delta); 4235 } 4236 ASSERT((int64_t)arc_p >= 0); 4237 4238 if (arc_reclaim_needed()) { 4239 cv_signal(&arc_reclaim_thread_cv); 4240 return; 4241 } 4242 4243 if (arc_no_grow) 4244 return; 4245 4246 if (arc_c >= arc_c_max) 4247 return; 4248 4249 /* 4250 * If we're within (2 * maxblocksize) bytes of the target 4251 * cache size, increment the target cache size 4252 */ 4253 if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 4254 atomic_add_64(&arc_c, (int64_t)bytes); 4255 if (arc_c > arc_c_max) 4256 arc_c = arc_c_max; 4257 else if (state == arc_anon) 4258 atomic_add_64(&arc_p, (int64_t)bytes); 4259 if (arc_p > arc_c) 4260 arc_p = arc_c; 4261 } 4262 ASSERT((int64_t)arc_p >= 0); 4263 } 4264 4265 /* 4266 * Check if arc_size has grown past our upper threshold, determined by 4267 * zfs_arc_overflow_shift. 4268 */ 4269 static boolean_t 4270 arc_is_overflowing(void) 4271 { 4272 /* Always allow at least one block of overflow */ 4273 uint64_t overflow = MAX(SPA_MAXBLOCKSIZE, 4274 arc_c >> zfs_arc_overflow_shift); 4275 4276 return (arc_size >= arc_c + overflow); 4277 } 4278 4279 static abd_t * 4280 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4281 { 4282 arc_buf_contents_t type = arc_buf_type(hdr); 4283 4284 arc_get_data_impl(hdr, size, tag); 4285 if (type == ARC_BUFC_METADATA) { 4286 return (abd_alloc(size, B_TRUE)); 4287 } else { 4288 ASSERT(type == ARC_BUFC_DATA); 4289 return (abd_alloc(size, B_FALSE)); 4290 } 4291 } 4292 4293 static void * 4294 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4295 { 4296 arc_buf_contents_t type = arc_buf_type(hdr); 4297 4298 arc_get_data_impl(hdr, size, tag); 4299 if (type == ARC_BUFC_METADATA) { 4300 return (zio_buf_alloc(size)); 4301 } else { 4302 ASSERT(type == ARC_BUFC_DATA); 4303 return (zio_data_buf_alloc(size)); 4304 } 4305 } 4306 4307 /* 4308 * Allocate a block and return it to the caller. If we are hitting the 4309 * hard limit for the cache size, we must sleep, waiting for the eviction 4310 * thread to catch up. If we're past the target size but below the hard 4311 * limit, we'll only signal the reclaim thread and continue on. 4312 */ 4313 static void 4314 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4315 { 4316 arc_state_t *state = hdr->b_l1hdr.b_state; 4317 arc_buf_contents_t type = arc_buf_type(hdr); 4318 4319 arc_adapt(size, state); 4320 4321 /* 4322 * If arc_size is currently overflowing, and has grown past our 4323 * upper limit, we must be adding data faster than the evict 4324 * thread can evict. Thus, to ensure we don't compound the 4325 * problem by adding more data and forcing arc_size to grow even 4326 * further past it's target size, we halt and wait for the 4327 * eviction thread to catch up. 4328 * 4329 * It's also possible that the reclaim thread is unable to evict 4330 * enough buffers to get arc_size below the overflow limit (e.g. 4331 * due to buffers being un-evictable, or hash lock collisions). 4332 * In this case, we want to proceed regardless if we're 4333 * overflowing; thus we don't use a while loop here. 4334 */ 4335 if (arc_is_overflowing()) { 4336 mutex_enter(&arc_reclaim_lock); 4337 4338 /* 4339 * Now that we've acquired the lock, we may no longer be 4340 * over the overflow limit, lets check. 4341 * 4342 * We're ignoring the case of spurious wake ups. If that 4343 * were to happen, it'd let this thread consume an ARC 4344 * buffer before it should have (i.e. before we're under 4345 * the overflow limit and were signalled by the reclaim 4346 * thread). As long as that is a rare occurrence, it 4347 * shouldn't cause any harm. 4348 */ 4349 if (arc_is_overflowing()) { 4350 cv_signal(&arc_reclaim_thread_cv); 4351 cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock); 4352 } 4353 4354 mutex_exit(&arc_reclaim_lock); 4355 } 4356 4357 VERIFY3U(hdr->b_type, ==, type); 4358 if (type == ARC_BUFC_METADATA) { 4359 arc_space_consume(size, ARC_SPACE_META); 4360 } else { 4361 arc_space_consume(size, ARC_SPACE_DATA); 4362 } 4363 4364 /* 4365 * Update the state size. Note that ghost states have a 4366 * "ghost size" and so don't need to be updated. 4367 */ 4368 if (!GHOST_STATE(state)) { 4369 4370 (void) refcount_add_many(&state->arcs_size, size, tag); 4371 4372 /* 4373 * If this is reached via arc_read, the link is 4374 * protected by the hash lock. If reached via 4375 * arc_buf_alloc, the header should not be accessed by 4376 * any other thread. And, if reached via arc_read_done, 4377 * the hash lock will protect it if it's found in the 4378 * hash table; otherwise no other thread should be 4379 * trying to [add|remove]_reference it. 4380 */ 4381 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 4382 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4383 (void) refcount_add_many(&state->arcs_esize[type], 4384 size, tag); 4385 } 4386 4387 /* 4388 * If we are growing the cache, and we are adding anonymous 4389 * data, and we have outgrown arc_p, update arc_p 4390 */ 4391 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon && 4392 (refcount_count(&arc_anon->arcs_size) + 4393 refcount_count(&arc_mru->arcs_size) > arc_p)) 4394 arc_p = MIN(arc_c, arc_p + size); 4395 } 4396 } 4397 4398 static void 4399 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag) 4400 { 4401 arc_free_data_impl(hdr, size, tag); 4402 abd_free(abd); 4403 } 4404 4405 static void 4406 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag) 4407 { 4408 arc_buf_contents_t type = arc_buf_type(hdr); 4409 4410 arc_free_data_impl(hdr, size, tag); 4411 if (type == ARC_BUFC_METADATA) { 4412 zio_buf_free(buf, size); 4413 } else { 4414 ASSERT(type == ARC_BUFC_DATA); 4415 zio_data_buf_free(buf, size); 4416 } 4417 } 4418 4419 /* 4420 * Free the arc data buffer. 4421 */ 4422 static void 4423 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4424 { 4425 arc_state_t *state = hdr->b_l1hdr.b_state; 4426 arc_buf_contents_t type = arc_buf_type(hdr); 4427 4428 /* protected by hash lock, if in the hash table */ 4429 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 4430 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4431 ASSERT(state != arc_anon && state != arc_l2c_only); 4432 4433 (void) refcount_remove_many(&state->arcs_esize[type], 4434 size, tag); 4435 } 4436 (void) refcount_remove_many(&state->arcs_size, size, tag); 4437 4438 VERIFY3U(hdr->b_type, ==, type); 4439 if (type == ARC_BUFC_METADATA) { 4440 arc_space_return(size, ARC_SPACE_META); 4441 } else { 4442 ASSERT(type == ARC_BUFC_DATA); 4443 arc_space_return(size, ARC_SPACE_DATA); 4444 } 4445 } 4446 4447 /* 4448 * This routine is called whenever a buffer is accessed. 4449 * NOTE: the hash lock is dropped in this function. 4450 */ 4451 static void 4452 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 4453 { 4454 clock_t now; 4455 4456 ASSERT(MUTEX_HELD(hash_lock)); 4457 ASSERT(HDR_HAS_L1HDR(hdr)); 4458 4459 if (hdr->b_l1hdr.b_state == arc_anon) { 4460 /* 4461 * This buffer is not in the cache, and does not 4462 * appear in our "ghost" list. Add the new buffer 4463 * to the MRU state. 4464 */ 4465 4466 ASSERT0(hdr->b_l1hdr.b_arc_access); 4467 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4468 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 4469 arc_change_state(arc_mru, hdr, hash_lock); 4470 4471 } else if (hdr->b_l1hdr.b_state == arc_mru) { 4472 now = ddi_get_lbolt(); 4473 4474 /* 4475 * If this buffer is here because of a prefetch, then either: 4476 * - clear the flag if this is a "referencing" read 4477 * (any subsequent access will bump this into the MFU state). 4478 * or 4479 * - move the buffer to the head of the list if this is 4480 * another prefetch (to make it less likely to be evicted). 4481 */ 4482 if (HDR_PREFETCH(hdr)) { 4483 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 4484 /* link protected by hash lock */ 4485 ASSERT(multilist_link_active( 4486 &hdr->b_l1hdr.b_arc_node)); 4487 } else { 4488 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 4489 ARCSTAT_BUMP(arcstat_mru_hits); 4490 } 4491 hdr->b_l1hdr.b_arc_access = now; 4492 return; 4493 } 4494 4495 /* 4496 * This buffer has been "accessed" only once so far, 4497 * but it is still in the cache. Move it to the MFU 4498 * state. 4499 */ 4500 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { 4501 /* 4502 * More than 125ms have passed since we 4503 * instantiated this buffer. Move it to the 4504 * most frequently used state. 4505 */ 4506 hdr->b_l1hdr.b_arc_access = now; 4507 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4508 arc_change_state(arc_mfu, hdr, hash_lock); 4509 } 4510 ARCSTAT_BUMP(arcstat_mru_hits); 4511 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { 4512 arc_state_t *new_state; 4513 /* 4514 * This buffer has been "accessed" recently, but 4515 * was evicted from the cache. Move it to the 4516 * MFU state. 4517 */ 4518 4519 if (HDR_PREFETCH(hdr)) { 4520 new_state = arc_mru; 4521 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) 4522 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 4523 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 4524 } else { 4525 new_state = arc_mfu; 4526 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4527 } 4528 4529 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4530 arc_change_state(new_state, hdr, hash_lock); 4531 4532 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 4533 } else if (hdr->b_l1hdr.b_state == arc_mfu) { 4534 /* 4535 * This buffer has been accessed more than once and is 4536 * still in the cache. Keep it in the MFU state. 4537 * 4538 * NOTE: an add_reference() that occurred when we did 4539 * the arc_read() will have kicked this off the list. 4540 * If it was a prefetch, we will explicitly move it to 4541 * the head of the list now. 4542 */ 4543 if ((HDR_PREFETCH(hdr)) != 0) { 4544 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4545 /* link protected by hash_lock */ 4546 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 4547 } 4548 ARCSTAT_BUMP(arcstat_mfu_hits); 4549 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4550 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { 4551 arc_state_t *new_state = arc_mfu; 4552 /* 4553 * This buffer has been accessed more than once but has 4554 * been evicted from the cache. Move it back to the 4555 * MFU state. 4556 */ 4557 4558 if (HDR_PREFETCH(hdr)) { 4559 /* 4560 * This is a prefetch access... 4561 * move this block back to the MRU state. 4562 */ 4563 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); 4564 new_state = arc_mru; 4565 } 4566 4567 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4568 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4569 arc_change_state(new_state, hdr, hash_lock); 4570 4571 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 4572 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { 4573 /* 4574 * This buffer is on the 2nd Level ARC. 4575 */ 4576 4577 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4578 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4579 arc_change_state(arc_mfu, hdr, hash_lock); 4580 } else { 4581 ASSERT(!"invalid arc state"); 4582 } 4583 } 4584 4585 /* a generic arc_done_func_t which you can use */ 4586 /* ARGSUSED */ 4587 void 4588 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 4589 { 4590 if (zio == NULL || zio->io_error == 0) 4591 bcopy(buf->b_data, arg, arc_buf_size(buf)); 4592 arc_buf_destroy(buf, arg); 4593 } 4594 4595 /* a generic arc_done_func_t */ 4596 void 4597 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 4598 { 4599 arc_buf_t **bufp = arg; 4600 if (zio && zio->io_error) { 4601 arc_buf_destroy(buf, arg); 4602 *bufp = NULL; 4603 } else { 4604 *bufp = buf; 4605 ASSERT(buf->b_data); 4606 } 4607 } 4608 4609 static void 4610 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp) 4611 { 4612 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) { 4613 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); 4614 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 4615 } else { 4616 if (HDR_COMPRESSION_ENABLED(hdr)) { 4617 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, 4618 BP_GET_COMPRESS(bp)); 4619 } 4620 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); 4621 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); 4622 } 4623 } 4624 4625 static void 4626 arc_read_done(zio_t *zio) 4627 { 4628 arc_buf_hdr_t *hdr = zio->io_private; 4629 kmutex_t *hash_lock = NULL; 4630 arc_callback_t *callback_list; 4631 arc_callback_t *acb; 4632 boolean_t freeable = B_FALSE; 4633 boolean_t no_zio_error = (zio->io_error == 0); 4634 4635 /* 4636 * The hdr was inserted into hash-table and removed from lists 4637 * prior to starting I/O. We should find this header, since 4638 * it's in the hash table, and it should be legit since it's 4639 * not possible to evict it during the I/O. The only possible 4640 * reason for it not to be found is if we were freed during the 4641 * read. 4642 */ 4643 if (HDR_IN_HASH_TABLE(hdr)) { 4644 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); 4645 ASSERT3U(hdr->b_dva.dva_word[0], ==, 4646 BP_IDENTITY(zio->io_bp)->dva_word[0]); 4647 ASSERT3U(hdr->b_dva.dva_word[1], ==, 4648 BP_IDENTITY(zio->io_bp)->dva_word[1]); 4649 4650 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, 4651 &hash_lock); 4652 4653 ASSERT((found == hdr && 4654 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 4655 (found == hdr && HDR_L2_READING(hdr))); 4656 ASSERT3P(hash_lock, !=, NULL); 4657 } 4658 4659 if (no_zio_error) { 4660 /* byteswap if necessary */ 4661 if (BP_SHOULD_BYTESWAP(zio->io_bp)) { 4662 if (BP_GET_LEVEL(zio->io_bp) > 0) { 4663 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; 4664 } else { 4665 hdr->b_l1hdr.b_byteswap = 4666 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); 4667 } 4668 } else { 4669 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 4670 } 4671 } 4672 4673 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); 4674 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) 4675 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); 4676 4677 callback_list = hdr->b_l1hdr.b_acb; 4678 ASSERT3P(callback_list, !=, NULL); 4679 4680 if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) { 4681 /* 4682 * Only call arc_access on anonymous buffers. This is because 4683 * if we've issued an I/O for an evicted buffer, we've already 4684 * called arc_access (to prevent any simultaneous readers from 4685 * getting confused). 4686 */ 4687 arc_access(hdr, hash_lock); 4688 } 4689 4690 /* 4691 * If a read request has a callback (i.e. acb_done is not NULL), then we 4692 * make a buf containing the data according to the parameters which were 4693 * passed in. The implementation of arc_buf_alloc_impl() ensures that we 4694 * aren't needlessly decompressing the data multiple times. 4695 */ 4696 int callback_cnt = 0; 4697 for (acb = callback_list; acb != NULL; acb = acb->acb_next) { 4698 if (!acb->acb_done) 4699 continue; 4700 4701 /* This is a demand read since prefetches don't use callbacks */ 4702 callback_cnt++; 4703 4704 int error = arc_buf_alloc_impl(hdr, acb->acb_private, 4705 acb->acb_compressed, no_zio_error, &acb->acb_buf); 4706 if (no_zio_error) { 4707 zio->io_error = error; 4708 } 4709 } 4710 hdr->b_l1hdr.b_acb = NULL; 4711 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 4712 if (callback_cnt == 0) { 4713 ASSERT(HDR_PREFETCH(hdr)); 4714 ASSERT0(hdr->b_l1hdr.b_bufcnt); 4715 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 4716 } 4717 4718 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || 4719 callback_list != NULL); 4720 4721 if (no_zio_error) { 4722 arc_hdr_verify(hdr, zio->io_bp); 4723 } else { 4724 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); 4725 if (hdr->b_l1hdr.b_state != arc_anon) 4726 arc_change_state(arc_anon, hdr, hash_lock); 4727 if (HDR_IN_HASH_TABLE(hdr)) 4728 buf_hash_remove(hdr); 4729 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 4730 } 4731 4732 /* 4733 * Broadcast before we drop the hash_lock to avoid the possibility 4734 * that the hdr (and hence the cv) might be freed before we get to 4735 * the cv_broadcast(). 4736 */ 4737 cv_broadcast(&hdr->b_l1hdr.b_cv); 4738 4739 if (hash_lock != NULL) { 4740 mutex_exit(hash_lock); 4741 } else { 4742 /* 4743 * This block was freed while we waited for the read to 4744 * complete. It has been removed from the hash table and 4745 * moved to the anonymous state (so that it won't show up 4746 * in the cache). 4747 */ 4748 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 4749 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 4750 } 4751 4752 /* execute each callback and free its structure */ 4753 while ((acb = callback_list) != NULL) { 4754 if (acb->acb_done) 4755 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 4756 4757 if (acb->acb_zio_dummy != NULL) { 4758 acb->acb_zio_dummy->io_error = zio->io_error; 4759 zio_nowait(acb->acb_zio_dummy); 4760 } 4761 4762 callback_list = acb->acb_next; 4763 kmem_free(acb, sizeof (arc_callback_t)); 4764 } 4765 4766 if (freeable) 4767 arc_hdr_destroy(hdr); 4768 } 4769 4770 /* 4771 * "Read" the block at the specified DVA (in bp) via the 4772 * cache. If the block is found in the cache, invoke the provided 4773 * callback immediately and return. Note that the `zio' parameter 4774 * in the callback will be NULL in this case, since no IO was 4775 * required. If the block is not in the cache pass the read request 4776 * on to the spa with a substitute callback function, so that the 4777 * requested block will be added to the cache. 4778 * 4779 * If a read request arrives for a block that has a read in-progress, 4780 * either wait for the in-progress read to complete (and return the 4781 * results); or, if this is a read with a "done" func, add a record 4782 * to the read to invoke the "done" func when the read completes, 4783 * and return; or just return. 4784 * 4785 * arc_read_done() will invoke all the requested "done" functions 4786 * for readers of this block. 4787 */ 4788 int 4789 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done, 4790 void *private, zio_priority_t priority, int zio_flags, 4791 arc_flags_t *arc_flags, const zbookmark_phys_t *zb) 4792 { 4793 arc_buf_hdr_t *hdr = NULL; 4794 kmutex_t *hash_lock = NULL; 4795 zio_t *rzio; 4796 uint64_t guid = spa_load_guid(spa); 4797 boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0; 4798 4799 ASSERT(!BP_IS_EMBEDDED(bp) || 4800 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); 4801 4802 top: 4803 if (!BP_IS_EMBEDDED(bp)) { 4804 /* 4805 * Embedded BP's have no DVA and require no I/O to "read". 4806 * Create an anonymous arc buf to back it. 4807 */ 4808 hdr = buf_hash_find(guid, bp, &hash_lock); 4809 } 4810 4811 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) { 4812 arc_buf_t *buf = NULL; 4813 *arc_flags |= ARC_FLAG_CACHED; 4814 4815 if (HDR_IO_IN_PROGRESS(hdr)) { 4816 4817 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && 4818 priority == ZIO_PRIORITY_SYNC_READ) { 4819 /* 4820 * This sync read must wait for an 4821 * in-progress async read (e.g. a predictive 4822 * prefetch). Async reads are queued 4823 * separately at the vdev_queue layer, so 4824 * this is a form of priority inversion. 4825 * Ideally, we would "inherit" the demand 4826 * i/o's priority by moving the i/o from 4827 * the async queue to the synchronous queue, 4828 * but there is currently no mechanism to do 4829 * so. Track this so that we can evaluate 4830 * the magnitude of this potential performance 4831 * problem. 4832 * 4833 * Note that if the prefetch i/o is already 4834 * active (has been issued to the device), 4835 * the prefetch improved performance, because 4836 * we issued it sooner than we would have 4837 * without the prefetch. 4838 */ 4839 DTRACE_PROBE1(arc__sync__wait__for__async, 4840 arc_buf_hdr_t *, hdr); 4841 ARCSTAT_BUMP(arcstat_sync_wait_for_async); 4842 } 4843 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 4844 arc_hdr_clear_flags(hdr, 4845 ARC_FLAG_PREDICTIVE_PREFETCH); 4846 } 4847 4848 if (*arc_flags & ARC_FLAG_WAIT) { 4849 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); 4850 mutex_exit(hash_lock); 4851 goto top; 4852 } 4853 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 4854 4855 if (done) { 4856 arc_callback_t *acb = NULL; 4857 4858 acb = kmem_zalloc(sizeof (arc_callback_t), 4859 KM_SLEEP); 4860 acb->acb_done = done; 4861 acb->acb_private = private; 4862 acb->acb_compressed = compressed_read; 4863 if (pio != NULL) 4864 acb->acb_zio_dummy = zio_null(pio, 4865 spa, NULL, NULL, NULL, zio_flags); 4866 4867 ASSERT3P(acb->acb_done, !=, NULL); 4868 acb->acb_next = hdr->b_l1hdr.b_acb; 4869 hdr->b_l1hdr.b_acb = acb; 4870 mutex_exit(hash_lock); 4871 return (0); 4872 } 4873 mutex_exit(hash_lock); 4874 return (0); 4875 } 4876 4877 ASSERT(hdr->b_l1hdr.b_state == arc_mru || 4878 hdr->b_l1hdr.b_state == arc_mfu); 4879 4880 if (done) { 4881 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 4882 /* 4883 * This is a demand read which does not have to 4884 * wait for i/o because we did a predictive 4885 * prefetch i/o for it, which has completed. 4886 */ 4887 DTRACE_PROBE1( 4888 arc__demand__hit__predictive__prefetch, 4889 arc_buf_hdr_t *, hdr); 4890 ARCSTAT_BUMP( 4891 arcstat_demand_hit_predictive_prefetch); 4892 arc_hdr_clear_flags(hdr, 4893 ARC_FLAG_PREDICTIVE_PREFETCH); 4894 } 4895 ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); 4896 4897 /* Get a buf with the desired data in it. */ 4898 VERIFY0(arc_buf_alloc_impl(hdr, private, 4899 compressed_read, B_TRUE, &buf)); 4900 } else if (*arc_flags & ARC_FLAG_PREFETCH && 4901 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 4902 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 4903 } 4904 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 4905 arc_access(hdr, hash_lock); 4906 if (*arc_flags & ARC_FLAG_L2CACHE) 4907 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 4908 mutex_exit(hash_lock); 4909 ARCSTAT_BUMP(arcstat_hits); 4910 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 4911 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 4912 data, metadata, hits); 4913 4914 if (done) 4915 done(NULL, buf, private); 4916 } else { 4917 uint64_t lsize = BP_GET_LSIZE(bp); 4918 uint64_t psize = BP_GET_PSIZE(bp); 4919 arc_callback_t *acb; 4920 vdev_t *vd = NULL; 4921 uint64_t addr = 0; 4922 boolean_t devw = B_FALSE; 4923 uint64_t size; 4924 4925 if (hdr == NULL) { 4926 /* this block is not in the cache */ 4927 arc_buf_hdr_t *exists = NULL; 4928 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 4929 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 4930 BP_GET_COMPRESS(bp), type); 4931 4932 if (!BP_IS_EMBEDDED(bp)) { 4933 hdr->b_dva = *BP_IDENTITY(bp); 4934 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 4935 exists = buf_hash_insert(hdr, &hash_lock); 4936 } 4937 if (exists != NULL) { 4938 /* somebody beat us to the hash insert */ 4939 mutex_exit(hash_lock); 4940 buf_discard_identity(hdr); 4941 arc_hdr_destroy(hdr); 4942 goto top; /* restart the IO request */ 4943 } 4944 } else { 4945 /* 4946 * This block is in the ghost cache. If it was L2-only 4947 * (and thus didn't have an L1 hdr), we realloc the 4948 * header to add an L1 hdr. 4949 */ 4950 if (!HDR_HAS_L1HDR(hdr)) { 4951 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, 4952 hdr_full_cache); 4953 } 4954 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 4955 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state)); 4956 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 4957 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4958 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 4959 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 4960 4961 /* 4962 * This is a delicate dance that we play here. 4963 * This hdr is in the ghost list so we access it 4964 * to move it out of the ghost list before we 4965 * initiate the read. If it's a prefetch then 4966 * it won't have a callback so we'll remove the 4967 * reference that arc_buf_alloc_impl() created. We 4968 * do this after we've called arc_access() to 4969 * avoid hitting an assert in remove_reference(). 4970 */ 4971 arc_access(hdr, hash_lock); 4972 arc_hdr_alloc_pabd(hdr); 4973 } 4974 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 4975 size = arc_hdr_size(hdr); 4976 4977 /* 4978 * If compression is enabled on the hdr, then will do 4979 * RAW I/O and will store the compressed data in the hdr's 4980 * data block. Otherwise, the hdr's data block will contain 4981 * the uncompressed data. 4982 */ 4983 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { 4984 zio_flags |= ZIO_FLAG_RAW; 4985 } 4986 4987 if (*arc_flags & ARC_FLAG_PREFETCH) 4988 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 4989 if (*arc_flags & ARC_FLAG_L2CACHE) 4990 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 4991 if (BP_GET_LEVEL(bp) > 0) 4992 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); 4993 if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH) 4994 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); 4995 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); 4996 4997 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 4998 acb->acb_done = done; 4999 acb->acb_private = private; 5000 acb->acb_compressed = compressed_read; 5001 5002 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5003 hdr->b_l1hdr.b_acb = acb; 5004 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5005 5006 if (HDR_HAS_L2HDR(hdr) && 5007 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { 5008 devw = hdr->b_l2hdr.b_dev->l2ad_writing; 5009 addr = hdr->b_l2hdr.b_daddr; 5010 /* 5011 * Lock out device removal. 5012 */ 5013 if (vdev_is_dead(vd) || 5014 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 5015 vd = NULL; 5016 } 5017 5018 if (priority == ZIO_PRIORITY_ASYNC_READ) 5019 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 5020 else 5021 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 5022 5023 if (hash_lock != NULL) 5024 mutex_exit(hash_lock); 5025 5026 /* 5027 * At this point, we have a level 1 cache miss. Try again in 5028 * L2ARC if possible. 5029 */ 5030 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); 5031 5032 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 5033 uint64_t, lsize, zbookmark_phys_t *, zb); 5034 ARCSTAT_BUMP(arcstat_misses); 5035 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 5036 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 5037 data, metadata, misses); 5038 5039 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 5040 /* 5041 * Read from the L2ARC if the following are true: 5042 * 1. The L2ARC vdev was previously cached. 5043 * 2. This buffer still has L2ARC metadata. 5044 * 3. This buffer isn't currently writing to the L2ARC. 5045 * 4. The L2ARC entry wasn't evicted, which may 5046 * also have invalidated the vdev. 5047 * 5. This isn't prefetch and l2arc_noprefetch is set. 5048 */ 5049 if (HDR_HAS_L2HDR(hdr) && 5050 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 5051 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 5052 l2arc_read_callback_t *cb; 5053 abd_t *abd; 5054 uint64_t asize; 5055 5056 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 5057 ARCSTAT_BUMP(arcstat_l2_hits); 5058 5059 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 5060 KM_SLEEP); 5061 cb->l2rcb_hdr = hdr; 5062 cb->l2rcb_bp = *bp; 5063 cb->l2rcb_zb = *zb; 5064 cb->l2rcb_flags = zio_flags; 5065 5066 asize = vdev_psize_to_asize(vd, size); 5067 if (asize != size) { 5068 abd = abd_alloc_for_io(asize, 5069 HDR_ISTYPE_METADATA(hdr)); 5070 cb->l2rcb_abd = abd; 5071 } else { 5072 abd = hdr->b_l1hdr.b_pabd; 5073 } 5074 5075 ASSERT(addr >= VDEV_LABEL_START_SIZE && 5076 addr + asize <= vd->vdev_psize - 5077 VDEV_LABEL_END_SIZE); 5078 5079 /* 5080 * l2arc read. The SCL_L2ARC lock will be 5081 * released by l2arc_read_done(). 5082 * Issue a null zio if the underlying buffer 5083 * was squashed to zero size by compression. 5084 */ 5085 ASSERT3U(HDR_GET_COMPRESS(hdr), !=, 5086 ZIO_COMPRESS_EMPTY); 5087 rzio = zio_read_phys(pio, vd, addr, 5088 asize, abd, 5089 ZIO_CHECKSUM_OFF, 5090 l2arc_read_done, cb, priority, 5091 zio_flags | ZIO_FLAG_DONT_CACHE | 5092 ZIO_FLAG_CANFAIL | 5093 ZIO_FLAG_DONT_PROPAGATE | 5094 ZIO_FLAG_DONT_RETRY, B_FALSE); 5095 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 5096 zio_t *, rzio); 5097 ARCSTAT_INCR(arcstat_l2_read_bytes, size); 5098 5099 if (*arc_flags & ARC_FLAG_NOWAIT) { 5100 zio_nowait(rzio); 5101 return (0); 5102 } 5103 5104 ASSERT(*arc_flags & ARC_FLAG_WAIT); 5105 if (zio_wait(rzio) == 0) 5106 return (0); 5107 5108 /* l2arc read error; goto zio_read() */ 5109 } else { 5110 DTRACE_PROBE1(l2arc__miss, 5111 arc_buf_hdr_t *, hdr); 5112 ARCSTAT_BUMP(arcstat_l2_misses); 5113 if (HDR_L2_WRITING(hdr)) 5114 ARCSTAT_BUMP(arcstat_l2_rw_clash); 5115 spa_config_exit(spa, SCL_L2ARC, vd); 5116 } 5117 } else { 5118 if (vd != NULL) 5119 spa_config_exit(spa, SCL_L2ARC, vd); 5120 if (l2arc_ndev != 0) { 5121 DTRACE_PROBE1(l2arc__miss, 5122 arc_buf_hdr_t *, hdr); 5123 ARCSTAT_BUMP(arcstat_l2_misses); 5124 } 5125 } 5126 5127 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size, 5128 arc_read_done, hdr, priority, zio_flags, zb); 5129 5130 if (*arc_flags & ARC_FLAG_WAIT) 5131 return (zio_wait(rzio)); 5132 5133 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 5134 zio_nowait(rzio); 5135 } 5136 return (0); 5137 } 5138 5139 /* 5140 * Notify the arc that a block was freed, and thus will never be used again. 5141 */ 5142 void 5143 arc_freed(spa_t *spa, const blkptr_t *bp) 5144 { 5145 arc_buf_hdr_t *hdr; 5146 kmutex_t *hash_lock; 5147 uint64_t guid = spa_load_guid(spa); 5148 5149 ASSERT(!BP_IS_EMBEDDED(bp)); 5150 5151 hdr = buf_hash_find(guid, bp, &hash_lock); 5152 if (hdr == NULL) 5153 return; 5154 5155 /* 5156 * We might be trying to free a block that is still doing I/O 5157 * (i.e. prefetch) or has a reference (i.e. a dedup-ed, 5158 * dmu_sync-ed block). If this block is being prefetched, then it 5159 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr 5160 * until the I/O completes. A block may also have a reference if it is 5161 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would 5162 * have written the new block to its final resting place on disk but 5163 * without the dedup flag set. This would have left the hdr in the MRU 5164 * state and discoverable. When the txg finally syncs it detects that 5165 * the block was overridden in open context and issues an override I/O. 5166 * Since this is a dedup block, the override I/O will determine if the 5167 * block is already in the DDT. If so, then it will replace the io_bp 5168 * with the bp from the DDT and allow the I/O to finish. When the I/O 5169 * reaches the done callback, dbuf_write_override_done, it will 5170 * check to see if the io_bp and io_bp_override are identical. 5171 * If they are not, then it indicates that the bp was replaced with 5172 * the bp in the DDT and the override bp is freed. This allows 5173 * us to arrive here with a reference on a block that is being 5174 * freed. So if we have an I/O in progress, or a reference to 5175 * this hdr, then we don't destroy the hdr. 5176 */ 5177 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) && 5178 refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) { 5179 arc_change_state(arc_anon, hdr, hash_lock); 5180 arc_hdr_destroy(hdr); 5181 mutex_exit(hash_lock); 5182 } else { 5183 mutex_exit(hash_lock); 5184 } 5185 5186 } 5187 5188 /* 5189 * Release this buffer from the cache, making it an anonymous buffer. This 5190 * must be done after a read and prior to modifying the buffer contents. 5191 * If the buffer has more than one reference, we must make 5192 * a new hdr for the buffer. 5193 */ 5194 void 5195 arc_release(arc_buf_t *buf, void *tag) 5196 { 5197 arc_buf_hdr_t *hdr = buf->b_hdr; 5198 5199 /* 5200 * It would be nice to assert that if it's DMU metadata (level > 5201 * 0 || it's the dnode file), then it must be syncing context. 5202 * But we don't know that information at this level. 5203 */ 5204 5205 mutex_enter(&buf->b_evict_lock); 5206 5207 ASSERT(HDR_HAS_L1HDR(hdr)); 5208 5209 /* 5210 * We don't grab the hash lock prior to this check, because if 5211 * the buffer's header is in the arc_anon state, it won't be 5212 * linked into the hash table. 5213 */ 5214 if (hdr->b_l1hdr.b_state == arc_anon) { 5215 mutex_exit(&buf->b_evict_lock); 5216 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5217 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 5218 ASSERT(!HDR_HAS_L2HDR(hdr)); 5219 ASSERT(HDR_EMPTY(hdr)); 5220 5221 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 5222 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); 5223 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); 5224 5225 hdr->b_l1hdr.b_arc_access = 0; 5226 5227 /* 5228 * If the buf is being overridden then it may already 5229 * have a hdr that is not empty. 5230 */ 5231 buf_discard_identity(hdr); 5232 arc_buf_thaw(buf); 5233 5234 return; 5235 } 5236 5237 kmutex_t *hash_lock = HDR_LOCK(hdr); 5238 mutex_enter(hash_lock); 5239 5240 /* 5241 * This assignment is only valid as long as the hash_lock is 5242 * held, we must be careful not to reference state or the 5243 * b_state field after dropping the lock. 5244 */ 5245 arc_state_t *state = hdr->b_l1hdr.b_state; 5246 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 5247 ASSERT3P(state, !=, arc_anon); 5248 5249 /* this buffer is not on any list */ 5250 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0); 5251 5252 if (HDR_HAS_L2HDR(hdr)) { 5253 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); 5254 5255 /* 5256 * We have to recheck this conditional again now that 5257 * we're holding the l2ad_mtx to prevent a race with 5258 * another thread which might be concurrently calling 5259 * l2arc_evict(). In that case, l2arc_evict() might have 5260 * destroyed the header's L2 portion as we were waiting 5261 * to acquire the l2ad_mtx. 5262 */ 5263 if (HDR_HAS_L2HDR(hdr)) 5264 arc_hdr_l2hdr_destroy(hdr); 5265 5266 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); 5267 } 5268 5269 /* 5270 * Do we have more than one buf? 5271 */ 5272 if (hdr->b_l1hdr.b_bufcnt > 1) { 5273 arc_buf_hdr_t *nhdr; 5274 uint64_t spa = hdr->b_spa; 5275 uint64_t psize = HDR_GET_PSIZE(hdr); 5276 uint64_t lsize = HDR_GET_LSIZE(hdr); 5277 enum zio_compress compress = HDR_GET_COMPRESS(hdr); 5278 arc_buf_contents_t type = arc_buf_type(hdr); 5279 VERIFY3U(hdr->b_type, ==, type); 5280 5281 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); 5282 (void) remove_reference(hdr, hash_lock, tag); 5283 5284 if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) { 5285 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); 5286 ASSERT(ARC_BUF_LAST(buf)); 5287 } 5288 5289 /* 5290 * Pull the data off of this hdr and attach it to 5291 * a new anonymous hdr. Also find the last buffer 5292 * in the hdr's buffer list. 5293 */ 5294 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); 5295 ASSERT3P(lastbuf, !=, NULL); 5296 5297 /* 5298 * If the current arc_buf_t and the hdr are sharing their data 5299 * buffer, then we must stop sharing that block. 5300 */ 5301 if (arc_buf_is_shared(buf)) { 5302 VERIFY(!arc_buf_is_shared(lastbuf)); 5303 5304 /* 5305 * First, sever the block sharing relationship between 5306 * buf and the arc_buf_hdr_t. 5307 */ 5308 arc_unshare_buf(hdr, buf); 5309 5310 /* 5311 * Now we need to recreate the hdr's b_pabd. Since we 5312 * have lastbuf handy, we try to share with it, but if 5313 * we can't then we allocate a new b_pabd and copy the 5314 * data from buf into it. 5315 */ 5316 if (arc_can_share(hdr, lastbuf)) { 5317 arc_share_buf(hdr, lastbuf); 5318 } else { 5319 arc_hdr_alloc_pabd(hdr); 5320 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, 5321 buf->b_data, psize); 5322 } 5323 VERIFY3P(lastbuf->b_data, !=, NULL); 5324 } else if (HDR_SHARED_DATA(hdr)) { 5325 /* 5326 * Uncompressed shared buffers are always at the end 5327 * of the list. Compressed buffers don't have the 5328 * same requirements. This makes it hard to 5329 * simply assert that the lastbuf is shared so 5330 * we rely on the hdr's compression flags to determine 5331 * if we have a compressed, shared buffer. 5332 */ 5333 ASSERT(arc_buf_is_shared(lastbuf) || 5334 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 5335 ASSERT(!ARC_BUF_SHARED(buf)); 5336 } 5337 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 5338 ASSERT3P(state, !=, arc_l2c_only); 5339 5340 (void) refcount_remove_many(&state->arcs_size, 5341 arc_buf_size(buf), buf); 5342 5343 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { 5344 ASSERT3P(state, !=, arc_l2c_only); 5345 (void) refcount_remove_many(&state->arcs_esize[type], 5346 arc_buf_size(buf), buf); 5347 } 5348 5349 hdr->b_l1hdr.b_bufcnt -= 1; 5350 arc_cksum_verify(buf); 5351 arc_buf_unwatch(buf); 5352 5353 mutex_exit(hash_lock); 5354 5355 /* 5356 * Allocate a new hdr. The new hdr will contain a b_pabd 5357 * buffer which will be freed in arc_write(). 5358 */ 5359 nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type); 5360 ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL); 5361 ASSERT0(nhdr->b_l1hdr.b_bufcnt); 5362 ASSERT0(refcount_count(&nhdr->b_l1hdr.b_refcnt)); 5363 VERIFY3U(nhdr->b_type, ==, type); 5364 ASSERT(!HDR_SHARED_DATA(nhdr)); 5365 5366 nhdr->b_l1hdr.b_buf = buf; 5367 nhdr->b_l1hdr.b_bufcnt = 1; 5368 (void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag); 5369 buf->b_hdr = nhdr; 5370 5371 mutex_exit(&buf->b_evict_lock); 5372 (void) refcount_add_many(&arc_anon->arcs_size, 5373 arc_buf_size(buf), buf); 5374 } else { 5375 mutex_exit(&buf->b_evict_lock); 5376 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); 5377 /* protected by hash lock, or hdr is on arc_anon */ 5378 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 5379 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5380 arc_change_state(arc_anon, hdr, hash_lock); 5381 hdr->b_l1hdr.b_arc_access = 0; 5382 mutex_exit(hash_lock); 5383 5384 buf_discard_identity(hdr); 5385 arc_buf_thaw(buf); 5386 } 5387 } 5388 5389 int 5390 arc_released(arc_buf_t *buf) 5391 { 5392 int released; 5393 5394 mutex_enter(&buf->b_evict_lock); 5395 released = (buf->b_data != NULL && 5396 buf->b_hdr->b_l1hdr.b_state == arc_anon); 5397 mutex_exit(&buf->b_evict_lock); 5398 return (released); 5399 } 5400 5401 #ifdef ZFS_DEBUG 5402 int 5403 arc_referenced(arc_buf_t *buf) 5404 { 5405 int referenced; 5406 5407 mutex_enter(&buf->b_evict_lock); 5408 referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt)); 5409 mutex_exit(&buf->b_evict_lock); 5410 return (referenced); 5411 } 5412 #endif 5413 5414 static void 5415 arc_write_ready(zio_t *zio) 5416 { 5417 arc_write_callback_t *callback = zio->io_private; 5418 arc_buf_t *buf = callback->awcb_buf; 5419 arc_buf_hdr_t *hdr = buf->b_hdr; 5420 uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp); 5421 5422 ASSERT(HDR_HAS_L1HDR(hdr)); 5423 ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt)); 5424 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 5425 5426 /* 5427 * If we're reexecuting this zio because the pool suspended, then 5428 * cleanup any state that was previously set the first time the 5429 * callback was invoked. 5430 */ 5431 if (zio->io_flags & ZIO_FLAG_REEXECUTED) { 5432 arc_cksum_free(hdr); 5433 arc_buf_unwatch(buf); 5434 if (hdr->b_l1hdr.b_pabd != NULL) { 5435 if (arc_buf_is_shared(buf)) { 5436 arc_unshare_buf(hdr, buf); 5437 } else { 5438 arc_hdr_free_pabd(hdr); 5439 } 5440 } 5441 } 5442 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 5443 ASSERT(!HDR_SHARED_DATA(hdr)); 5444 ASSERT(!arc_buf_is_shared(buf)); 5445 5446 callback->awcb_ready(zio, buf, callback->awcb_private); 5447 5448 if (HDR_IO_IN_PROGRESS(hdr)) 5449 ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED); 5450 5451 arc_cksum_compute(buf); 5452 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5453 5454 enum zio_compress compress; 5455 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) { 5456 compress = ZIO_COMPRESS_OFF; 5457 } else { 5458 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp)); 5459 compress = BP_GET_COMPRESS(zio->io_bp); 5460 } 5461 HDR_SET_PSIZE(hdr, psize); 5462 arc_hdr_set_compress(hdr, compress); 5463 5464 5465 /* 5466 * Fill the hdr with data. If the hdr is compressed, the data we want 5467 * is available from the zio, otherwise we can take it from the buf. 5468 * 5469 * We might be able to share the buf's data with the hdr here. However, 5470 * doing so would cause the ARC to be full of linear ABDs if we write a 5471 * lot of shareable data. As a compromise, we check whether scattered 5472 * ABDs are allowed, and assume that if they are then the user wants 5473 * the ARC to be primarily filled with them regardless of the data being 5474 * written. Therefore, if they're allowed then we allocate one and copy 5475 * the data into it; otherwise, we share the data directly if we can. 5476 */ 5477 if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) { 5478 arc_hdr_alloc_pabd(hdr); 5479 5480 /* 5481 * Ideally, we would always copy the io_abd into b_pabd, but the 5482 * user may have disabled compressed ARC, thus we must check the 5483 * hdr's compression setting rather than the io_bp's. 5484 */ 5485 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { 5486 ASSERT3U(BP_GET_COMPRESS(zio->io_bp), !=, 5487 ZIO_COMPRESS_OFF); 5488 ASSERT3U(psize, >, 0); 5489 5490 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize); 5491 } else { 5492 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr)); 5493 5494 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data, 5495 arc_buf_size(buf)); 5496 } 5497 } else { 5498 ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd)); 5499 ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf)); 5500 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 5501 5502 arc_share_buf(hdr, buf); 5503 } 5504 5505 arc_hdr_verify(hdr, zio->io_bp); 5506 } 5507 5508 static void 5509 arc_write_children_ready(zio_t *zio) 5510 { 5511 arc_write_callback_t *callback = zio->io_private; 5512 arc_buf_t *buf = callback->awcb_buf; 5513 5514 callback->awcb_children_ready(zio, buf, callback->awcb_private); 5515 } 5516 5517 /* 5518 * The SPA calls this callback for each physical write that happens on behalf 5519 * of a logical write. See the comment in dbuf_write_physdone() for details. 5520 */ 5521 static void 5522 arc_write_physdone(zio_t *zio) 5523 { 5524 arc_write_callback_t *cb = zio->io_private; 5525 if (cb->awcb_physdone != NULL) 5526 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private); 5527 } 5528 5529 static void 5530 arc_write_done(zio_t *zio) 5531 { 5532 arc_write_callback_t *callback = zio->io_private; 5533 arc_buf_t *buf = callback->awcb_buf; 5534 arc_buf_hdr_t *hdr = buf->b_hdr; 5535 5536 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5537 5538 if (zio->io_error == 0) { 5539 arc_hdr_verify(hdr, zio->io_bp); 5540 5541 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) { 5542 buf_discard_identity(hdr); 5543 } else { 5544 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 5545 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 5546 } 5547 } else { 5548 ASSERT(HDR_EMPTY(hdr)); 5549 } 5550 5551 /* 5552 * If the block to be written was all-zero or compressed enough to be 5553 * embedded in the BP, no write was performed so there will be no 5554 * dva/birth/checksum. The buffer must therefore remain anonymous 5555 * (and uncached). 5556 */ 5557 if (!HDR_EMPTY(hdr)) { 5558 arc_buf_hdr_t *exists; 5559 kmutex_t *hash_lock; 5560 5561 ASSERT3U(zio->io_error, ==, 0); 5562 5563 arc_cksum_verify(buf); 5564 5565 exists = buf_hash_insert(hdr, &hash_lock); 5566 if (exists != NULL) { 5567 /* 5568 * This can only happen if we overwrite for 5569 * sync-to-convergence, because we remove 5570 * buffers from the hash table when we arc_free(). 5571 */ 5572 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 5573 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 5574 panic("bad overwrite, hdr=%p exists=%p", 5575 (void *)hdr, (void *)exists); 5576 ASSERT(refcount_is_zero( 5577 &exists->b_l1hdr.b_refcnt)); 5578 arc_change_state(arc_anon, exists, hash_lock); 5579 mutex_exit(hash_lock); 5580 arc_hdr_destroy(exists); 5581 exists = buf_hash_insert(hdr, &hash_lock); 5582 ASSERT3P(exists, ==, NULL); 5583 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) { 5584 /* nopwrite */ 5585 ASSERT(zio->io_prop.zp_nopwrite); 5586 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 5587 panic("bad nopwrite, hdr=%p exists=%p", 5588 (void *)hdr, (void *)exists); 5589 } else { 5590 /* Dedup */ 5591 ASSERT(hdr->b_l1hdr.b_bufcnt == 1); 5592 ASSERT(hdr->b_l1hdr.b_state == arc_anon); 5593 ASSERT(BP_GET_DEDUP(zio->io_bp)); 5594 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 5595 } 5596 } 5597 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5598 /* if it's not anon, we are doing a scrub */ 5599 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) 5600 arc_access(hdr, hash_lock); 5601 mutex_exit(hash_lock); 5602 } else { 5603 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5604 } 5605 5606 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 5607 callback->awcb_done(zio, buf, callback->awcb_private); 5608 5609 abd_put(zio->io_abd); 5610 kmem_free(callback, sizeof (arc_write_callback_t)); 5611 } 5612 5613 zio_t * 5614 arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 5615 boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready, 5616 arc_done_func_t *children_ready, arc_done_func_t *physdone, 5617 arc_done_func_t *done, void *private, zio_priority_t priority, 5618 int zio_flags, const zbookmark_phys_t *zb) 5619 { 5620 arc_buf_hdr_t *hdr = buf->b_hdr; 5621 arc_write_callback_t *callback; 5622 zio_t *zio; 5623 zio_prop_t localprop = *zp; 5624 5625 ASSERT3P(ready, !=, NULL); 5626 ASSERT3P(done, !=, NULL); 5627 ASSERT(!HDR_IO_ERROR(hdr)); 5628 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5629 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5630 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); 5631 if (l2arc) 5632 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 5633 if (ARC_BUF_COMPRESSED(buf)) { 5634 /* 5635 * We're writing a pre-compressed buffer. Make the 5636 * compression algorithm requested by the zio_prop_t match 5637 * the pre-compressed buffer's compression algorithm. 5638 */ 5639 localprop.zp_compress = HDR_GET_COMPRESS(hdr); 5640 5641 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf)); 5642 zio_flags |= ZIO_FLAG_RAW; 5643 } 5644 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 5645 callback->awcb_ready = ready; 5646 callback->awcb_children_ready = children_ready; 5647 callback->awcb_physdone = physdone; 5648 callback->awcb_done = done; 5649 callback->awcb_private = private; 5650 callback->awcb_buf = buf; 5651 5652 /* 5653 * The hdr's b_pabd is now stale, free it now. A new data block 5654 * will be allocated when the zio pipeline calls arc_write_ready(). 5655 */ 5656 if (hdr->b_l1hdr.b_pabd != NULL) { 5657 /* 5658 * If the buf is currently sharing the data block with 5659 * the hdr then we need to break that relationship here. 5660 * The hdr will remain with a NULL data pointer and the 5661 * buf will take sole ownership of the block. 5662 */ 5663 if (arc_buf_is_shared(buf)) { 5664 arc_unshare_buf(hdr, buf); 5665 } else { 5666 arc_hdr_free_pabd(hdr); 5667 } 5668 VERIFY3P(buf->b_data, !=, NULL); 5669 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF); 5670 } 5671 ASSERT(!arc_buf_is_shared(buf)); 5672 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 5673 5674 zio = zio_write(pio, spa, txg, bp, 5675 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)), 5676 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready, 5677 (children_ready != NULL) ? arc_write_children_ready : NULL, 5678 arc_write_physdone, arc_write_done, callback, 5679 priority, zio_flags, zb); 5680 5681 return (zio); 5682 } 5683 5684 static int 5685 arc_memory_throttle(uint64_t reserve, uint64_t txg) 5686 { 5687 #ifdef _KERNEL 5688 uint64_t available_memory = ptob(freemem); 5689 static uint64_t page_load = 0; 5690 static uint64_t last_txg = 0; 5691 5692 #if defined(__i386) 5693 available_memory = 5694 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 5695 #endif 5696 5697 if (freemem > physmem * arc_lotsfree_percent / 100) 5698 return (0); 5699 5700 if (txg > last_txg) { 5701 last_txg = txg; 5702 page_load = 0; 5703 } 5704 /* 5705 * If we are in pageout, we know that memory is already tight, 5706 * the arc is already going to be evicting, so we just want to 5707 * continue to let page writes occur as quickly as possible. 5708 */ 5709 if (curproc == proc_pageout) { 5710 if (page_load > MAX(ptob(minfree), available_memory) / 4) 5711 return (SET_ERROR(ERESTART)); 5712 /* Note: reserve is inflated, so we deflate */ 5713 page_load += reserve / 8; 5714 return (0); 5715 } else if (page_load > 0 && arc_reclaim_needed()) { 5716 /* memory is low, delay before restarting */ 5717 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 5718 return (SET_ERROR(EAGAIN)); 5719 } 5720 page_load = 0; 5721 #endif 5722 return (0); 5723 } 5724 5725 void 5726 arc_tempreserve_clear(uint64_t reserve) 5727 { 5728 atomic_add_64(&arc_tempreserve, -reserve); 5729 ASSERT((int64_t)arc_tempreserve >= 0); 5730 } 5731 5732 int 5733 arc_tempreserve_space(uint64_t reserve, uint64_t txg) 5734 { 5735 int error; 5736 uint64_t anon_size; 5737 5738 if (reserve > arc_c/4 && !arc_no_grow) 5739 arc_c = MIN(arc_c_max, reserve * 4); 5740 if (reserve > arc_c) 5741 return (SET_ERROR(ENOMEM)); 5742 5743 /* 5744 * Don't count loaned bufs as in flight dirty data to prevent long 5745 * network delays from blocking transactions that are ready to be 5746 * assigned to a txg. 5747 */ 5748 5749 /* assert that it has not wrapped around */ 5750 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); 5751 5752 anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) - 5753 arc_loaned_bytes), 0); 5754 5755 /* 5756 * Writes will, almost always, require additional memory allocations 5757 * in order to compress/encrypt/etc the data. We therefore need to 5758 * make sure that there is sufficient available memory for this. 5759 */ 5760 error = arc_memory_throttle(reserve, txg); 5761 if (error != 0) 5762 return (error); 5763 5764 /* 5765 * Throttle writes when the amount of dirty data in the cache 5766 * gets too large. We try to keep the cache less than half full 5767 * of dirty blocks so that our sync times don't grow too large. 5768 * Note: if two requests come in concurrently, we might let them 5769 * both succeed, when one of them should fail. Not a huge deal. 5770 */ 5771 5772 if (reserve + arc_tempreserve + anon_size > arc_c / 2 && 5773 anon_size > arc_c / 4) { 5774 uint64_t meta_esize = 5775 refcount_count(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); 5776 uint64_t data_esize = 5777 refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 5778 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 5779 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 5780 arc_tempreserve >> 10, meta_esize >> 10, 5781 data_esize >> 10, reserve >> 10, arc_c >> 10); 5782 return (SET_ERROR(ERESTART)); 5783 } 5784 atomic_add_64(&arc_tempreserve, reserve); 5785 return (0); 5786 } 5787 5788 static void 5789 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size, 5790 kstat_named_t *evict_data, kstat_named_t *evict_metadata) 5791 { 5792 size->value.ui64 = refcount_count(&state->arcs_size); 5793 evict_data->value.ui64 = 5794 refcount_count(&state->arcs_esize[ARC_BUFC_DATA]); 5795 evict_metadata->value.ui64 = 5796 refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]); 5797 } 5798 5799 static int 5800 arc_kstat_update(kstat_t *ksp, int rw) 5801 { 5802 arc_stats_t *as = ksp->ks_data; 5803 5804 if (rw == KSTAT_WRITE) { 5805 return (EACCES); 5806 } else { 5807 arc_kstat_update_state(arc_anon, 5808 &as->arcstat_anon_size, 5809 &as->arcstat_anon_evictable_data, 5810 &as->arcstat_anon_evictable_metadata); 5811 arc_kstat_update_state(arc_mru, 5812 &as->arcstat_mru_size, 5813 &as->arcstat_mru_evictable_data, 5814 &as->arcstat_mru_evictable_metadata); 5815 arc_kstat_update_state(arc_mru_ghost, 5816 &as->arcstat_mru_ghost_size, 5817 &as->arcstat_mru_ghost_evictable_data, 5818 &as->arcstat_mru_ghost_evictable_metadata); 5819 arc_kstat_update_state(arc_mfu, 5820 &as->arcstat_mfu_size, 5821 &as->arcstat_mfu_evictable_data, 5822 &as->arcstat_mfu_evictable_metadata); 5823 arc_kstat_update_state(arc_mfu_ghost, 5824 &as->arcstat_mfu_ghost_size, 5825 &as->arcstat_mfu_ghost_evictable_data, 5826 &as->arcstat_mfu_ghost_evictable_metadata); 5827 } 5828 5829 return (0); 5830 } 5831 5832 /* 5833 * This function *must* return indices evenly distributed between all 5834 * sublists of the multilist. This is needed due to how the ARC eviction 5835 * code is laid out; arc_evict_state() assumes ARC buffers are evenly 5836 * distributed between all sublists and uses this assumption when 5837 * deciding which sublist to evict from and how much to evict from it. 5838 */ 5839 unsigned int 5840 arc_state_multilist_index_func(multilist_t *ml, void *obj) 5841 { 5842 arc_buf_hdr_t *hdr = obj; 5843 5844 /* 5845 * We rely on b_dva to generate evenly distributed index 5846 * numbers using buf_hash below. So, as an added precaution, 5847 * let's make sure we never add empty buffers to the arc lists. 5848 */ 5849 ASSERT(!HDR_EMPTY(hdr)); 5850 5851 /* 5852 * The assumption here, is the hash value for a given 5853 * arc_buf_hdr_t will remain constant throughout it's lifetime 5854 * (i.e. it's b_spa, b_dva, and b_birth fields don't change). 5855 * Thus, we don't need to store the header's sublist index 5856 * on insertion, as this index can be recalculated on removal. 5857 * 5858 * Also, the low order bits of the hash value are thought to be 5859 * distributed evenly. Otherwise, in the case that the multilist 5860 * has a power of two number of sublists, each sublists' usage 5861 * would not be evenly distributed. 5862 */ 5863 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % 5864 multilist_get_num_sublists(ml)); 5865 } 5866 5867 static void 5868 arc_state_init(void) 5869 { 5870 arc_anon = &ARC_anon; 5871 arc_mru = &ARC_mru; 5872 arc_mru_ghost = &ARC_mru_ghost; 5873 arc_mfu = &ARC_mfu; 5874 arc_mfu_ghost = &ARC_mfu_ghost; 5875 arc_l2c_only = &ARC_l2c_only; 5876 5877 arc_mru->arcs_list[ARC_BUFC_METADATA] = 5878 multilist_create(sizeof (arc_buf_hdr_t), 5879 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5880 arc_state_multilist_index_func); 5881 arc_mru->arcs_list[ARC_BUFC_DATA] = 5882 multilist_create(sizeof (arc_buf_hdr_t), 5883 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5884 arc_state_multilist_index_func); 5885 arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] = 5886 multilist_create(sizeof (arc_buf_hdr_t), 5887 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5888 arc_state_multilist_index_func); 5889 arc_mru_ghost->arcs_list[ARC_BUFC_DATA] = 5890 multilist_create(sizeof (arc_buf_hdr_t), 5891 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5892 arc_state_multilist_index_func); 5893 arc_mfu->arcs_list[ARC_BUFC_METADATA] = 5894 multilist_create(sizeof (arc_buf_hdr_t), 5895 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5896 arc_state_multilist_index_func); 5897 arc_mfu->arcs_list[ARC_BUFC_DATA] = 5898 multilist_create(sizeof (arc_buf_hdr_t), 5899 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5900 arc_state_multilist_index_func); 5901 arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] = 5902 multilist_create(sizeof (arc_buf_hdr_t), 5903 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5904 arc_state_multilist_index_func); 5905 arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] = 5906 multilist_create(sizeof (arc_buf_hdr_t), 5907 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5908 arc_state_multilist_index_func); 5909 arc_l2c_only->arcs_list[ARC_BUFC_METADATA] = 5910 multilist_create(sizeof (arc_buf_hdr_t), 5911 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5912 arc_state_multilist_index_func); 5913 arc_l2c_only->arcs_list[ARC_BUFC_DATA] = 5914 multilist_create(sizeof (arc_buf_hdr_t), 5915 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 5916 arc_state_multilist_index_func); 5917 5918 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); 5919 refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 5920 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]); 5921 refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]); 5922 refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]); 5923 refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]); 5924 refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 5925 refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]); 5926 refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]); 5927 refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]); 5928 refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]); 5929 refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]); 5930 5931 refcount_create(&arc_anon->arcs_size); 5932 refcount_create(&arc_mru->arcs_size); 5933 refcount_create(&arc_mru_ghost->arcs_size); 5934 refcount_create(&arc_mfu->arcs_size); 5935 refcount_create(&arc_mfu_ghost->arcs_size); 5936 refcount_create(&arc_l2c_only->arcs_size); 5937 } 5938 5939 static void 5940 arc_state_fini(void) 5941 { 5942 refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); 5943 refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 5944 refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]); 5945 refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]); 5946 refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]); 5947 refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]); 5948 refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 5949 refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]); 5950 refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]); 5951 refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]); 5952 refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]); 5953 refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]); 5954 5955 refcount_destroy(&arc_anon->arcs_size); 5956 refcount_destroy(&arc_mru->arcs_size); 5957 refcount_destroy(&arc_mru_ghost->arcs_size); 5958 refcount_destroy(&arc_mfu->arcs_size); 5959 refcount_destroy(&arc_mfu_ghost->arcs_size); 5960 refcount_destroy(&arc_l2c_only->arcs_size); 5961 5962 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]); 5963 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 5964 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]); 5965 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 5966 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]); 5967 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 5968 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]); 5969 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 5970 } 5971 5972 uint64_t 5973 arc_max_bytes(void) 5974 { 5975 return (arc_c_max); 5976 } 5977 5978 void 5979 arc_init(void) 5980 { 5981 /* 5982 * allmem is "all memory that we could possibly use". 5983 */ 5984 #ifdef _KERNEL 5985 uint64_t allmem = ptob(physmem - swapfs_minfree); 5986 #else 5987 uint64_t allmem = (physmem * PAGESIZE) / 2; 5988 #endif 5989 5990 mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL); 5991 cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL); 5992 cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL); 5993 5994 /* Convert seconds to clock ticks */ 5995 arc_min_prefetch_lifespan = 1 * hz; 5996 5997 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 5998 arc_c_min = MAX(allmem / 32, 64 << 20); 5999 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 6000 if (allmem >= 1 << 30) 6001 arc_c_max = allmem - (1 << 30); 6002 else 6003 arc_c_max = arc_c_min; 6004 arc_c_max = MAX(allmem * 3 / 4, arc_c_max); 6005 6006 /* 6007 * In userland, there's only the memory pressure that we artificially 6008 * create (see arc_available_memory()). Don't let arc_c get too 6009 * small, because it can cause transactions to be larger than 6010 * arc_c, causing arc_tempreserve_space() to fail. 6011 */ 6012 #ifndef _KERNEL 6013 arc_c_min = arc_c_max / 2; 6014 #endif 6015 6016 /* 6017 * Allow the tunables to override our calculations if they are 6018 * reasonable (ie. over 64MB) 6019 */ 6020 if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem) { 6021 arc_c_max = zfs_arc_max; 6022 arc_c_min = MIN(arc_c_min, arc_c_max); 6023 } 6024 if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max) 6025 arc_c_min = zfs_arc_min; 6026 6027 arc_c = arc_c_max; 6028 arc_p = (arc_c >> 1); 6029 arc_size = 0; 6030 6031 /* limit meta-data to 1/4 of the arc capacity */ 6032 arc_meta_limit = arc_c_max / 4; 6033 6034 #ifdef _KERNEL 6035 /* 6036 * Metadata is stored in the kernel's heap. Don't let us 6037 * use more than half the heap for the ARC. 6038 */ 6039 arc_meta_limit = MIN(arc_meta_limit, 6040 vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 2); 6041 #endif 6042 6043 /* Allow the tunable to override if it is reasonable */ 6044 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 6045 arc_meta_limit = zfs_arc_meta_limit; 6046 6047 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 6048 arc_c_min = arc_meta_limit / 2; 6049 6050 if (zfs_arc_meta_min > 0) { 6051 arc_meta_min = zfs_arc_meta_min; 6052 } else { 6053 arc_meta_min = arc_c_min / 2; 6054 } 6055 6056 if (zfs_arc_grow_retry > 0) 6057 arc_grow_retry = zfs_arc_grow_retry; 6058 6059 if (zfs_arc_shrink_shift > 0) 6060 arc_shrink_shift = zfs_arc_shrink_shift; 6061 6062 /* 6063 * Ensure that arc_no_grow_shift is less than arc_shrink_shift. 6064 */ 6065 if (arc_no_grow_shift >= arc_shrink_shift) 6066 arc_no_grow_shift = arc_shrink_shift - 1; 6067 6068 if (zfs_arc_p_min_shift > 0) 6069 arc_p_min_shift = zfs_arc_p_min_shift; 6070 6071 /* if kmem_flags are set, lets try to use less memory */ 6072 if (kmem_debugging()) 6073 arc_c = arc_c / 2; 6074 if (arc_c < arc_c_min) 6075 arc_c = arc_c_min; 6076 6077 arc_state_init(); 6078 buf_init(); 6079 6080 arc_reclaim_thread_exit = B_FALSE; 6081 6082 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 6083 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 6084 6085 if (arc_ksp != NULL) { 6086 arc_ksp->ks_data = &arc_stats; 6087 arc_ksp->ks_update = arc_kstat_update; 6088 kstat_install(arc_ksp); 6089 } 6090 6091 (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 6092 TS_RUN, minclsyspri); 6093 6094 arc_dead = B_FALSE; 6095 arc_warm = B_FALSE; 6096 6097 /* 6098 * Calculate maximum amount of dirty data per pool. 6099 * 6100 * If it has been set by /etc/system, take that. 6101 * Otherwise, use a percentage of physical memory defined by 6102 * zfs_dirty_data_max_percent (default 10%) with a cap at 6103 * zfs_dirty_data_max_max (default 4GB). 6104 */ 6105 if (zfs_dirty_data_max == 0) { 6106 zfs_dirty_data_max = physmem * PAGESIZE * 6107 zfs_dirty_data_max_percent / 100; 6108 zfs_dirty_data_max = MIN(zfs_dirty_data_max, 6109 zfs_dirty_data_max_max); 6110 } 6111 } 6112 6113 void 6114 arc_fini(void) 6115 { 6116 mutex_enter(&arc_reclaim_lock); 6117 arc_reclaim_thread_exit = B_TRUE; 6118 /* 6119 * The reclaim thread will set arc_reclaim_thread_exit back to 6120 * B_FALSE when it is finished exiting; we're waiting for that. 6121 */ 6122 while (arc_reclaim_thread_exit) { 6123 cv_signal(&arc_reclaim_thread_cv); 6124 cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock); 6125 } 6126 mutex_exit(&arc_reclaim_lock); 6127 6128 /* Use B_TRUE to ensure *all* buffers are evicted */ 6129 arc_flush(NULL, B_TRUE); 6130 6131 arc_dead = B_TRUE; 6132 6133 if (arc_ksp != NULL) { 6134 kstat_delete(arc_ksp); 6135 arc_ksp = NULL; 6136 } 6137 6138 mutex_destroy(&arc_reclaim_lock); 6139 cv_destroy(&arc_reclaim_thread_cv); 6140 cv_destroy(&arc_reclaim_waiters_cv); 6141 6142 arc_state_fini(); 6143 buf_fini(); 6144 6145 ASSERT0(arc_loaned_bytes); 6146 } 6147 6148 /* 6149 * Level 2 ARC 6150 * 6151 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 6152 * It uses dedicated storage devices to hold cached data, which are populated 6153 * using large infrequent writes. The main role of this cache is to boost 6154 * the performance of random read workloads. The intended L2ARC devices 6155 * include short-stroked disks, solid state disks, and other media with 6156 * substantially faster read latency than disk. 6157 * 6158 * +-----------------------+ 6159 * | ARC | 6160 * +-----------------------+ 6161 * | ^ ^ 6162 * | | | 6163 * l2arc_feed_thread() arc_read() 6164 * | | | 6165 * | l2arc read | 6166 * V | | 6167 * +---------------+ | 6168 * | L2ARC | | 6169 * +---------------+ | 6170 * | ^ | 6171 * l2arc_write() | | 6172 * | | | 6173 * V | | 6174 * +-------+ +-------+ 6175 * | vdev | | vdev | 6176 * | cache | | cache | 6177 * +-------+ +-------+ 6178 * +=========+ .-----. 6179 * : L2ARC : |-_____-| 6180 * : devices : | Disks | 6181 * +=========+ `-_____-' 6182 * 6183 * Read requests are satisfied from the following sources, in order: 6184 * 6185 * 1) ARC 6186 * 2) vdev cache of L2ARC devices 6187 * 3) L2ARC devices 6188 * 4) vdev cache of disks 6189 * 5) disks 6190 * 6191 * Some L2ARC device types exhibit extremely slow write performance. 6192 * To accommodate for this there are some significant differences between 6193 * the L2ARC and traditional cache design: 6194 * 6195 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 6196 * the ARC behave as usual, freeing buffers and placing headers on ghost 6197 * lists. The ARC does not send buffers to the L2ARC during eviction as 6198 * this would add inflated write latencies for all ARC memory pressure. 6199 * 6200 * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 6201 * It does this by periodically scanning buffers from the eviction-end of 6202 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 6203 * not already there. It scans until a headroom of buffers is satisfied, 6204 * which itself is a buffer for ARC eviction. If a compressible buffer is 6205 * found during scanning and selected for writing to an L2ARC device, we 6206 * temporarily boost scanning headroom during the next scan cycle to make 6207 * sure we adapt to compression effects (which might significantly reduce 6208 * the data volume we write to L2ARC). The thread that does this is 6209 * l2arc_feed_thread(), illustrated below; example sizes are included to 6210 * provide a better sense of ratio than this diagram: 6211 * 6212 * head --> tail 6213 * +---------------------+----------+ 6214 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 6215 * +---------------------+----------+ | o L2ARC eligible 6216 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 6217 * +---------------------+----------+ | 6218 * 15.9 Gbytes ^ 32 Mbytes | 6219 * headroom | 6220 * l2arc_feed_thread() 6221 * | 6222 * l2arc write hand <--[oooo]--' 6223 * | 8 Mbyte 6224 * | write max 6225 * V 6226 * +==============================+ 6227 * L2ARC dev |####|#|###|###| |####| ... | 6228 * +==============================+ 6229 * 32 Gbytes 6230 * 6231 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 6232 * evicted, then the L2ARC has cached a buffer much sooner than it probably 6233 * needed to, potentially wasting L2ARC device bandwidth and storage. It is 6234 * safe to say that this is an uncommon case, since buffers at the end of 6235 * the ARC lists have moved there due to inactivity. 6236 * 6237 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 6238 * then the L2ARC simply misses copying some buffers. This serves as a 6239 * pressure valve to prevent heavy read workloads from both stalling the ARC 6240 * with waits and clogging the L2ARC with writes. This also helps prevent 6241 * the potential for the L2ARC to churn if it attempts to cache content too 6242 * quickly, such as during backups of the entire pool. 6243 * 6244 * 5. After system boot and before the ARC has filled main memory, there are 6245 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 6246 * lists can remain mostly static. Instead of searching from tail of these 6247 * lists as pictured, the l2arc_feed_thread() will search from the list heads 6248 * for eligible buffers, greatly increasing its chance of finding them. 6249 * 6250 * The L2ARC device write speed is also boosted during this time so that 6251 * the L2ARC warms up faster. Since there have been no ARC evictions yet, 6252 * there are no L2ARC reads, and no fear of degrading read performance 6253 * through increased writes. 6254 * 6255 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 6256 * the vdev queue can aggregate them into larger and fewer writes. Each 6257 * device is written to in a rotor fashion, sweeping writes through 6258 * available space then repeating. 6259 * 6260 * 7. The L2ARC does not store dirty content. It never needs to flush 6261 * write buffers back to disk based storage. 6262 * 6263 * 8. If an ARC buffer is written (and dirtied) which also exists in the 6264 * L2ARC, the now stale L2ARC buffer is immediately dropped. 6265 * 6266 * The performance of the L2ARC can be tweaked by a number of tunables, which 6267 * may be necessary for different workloads: 6268 * 6269 * l2arc_write_max max write bytes per interval 6270 * l2arc_write_boost extra write bytes during device warmup 6271 * l2arc_noprefetch skip caching prefetched buffers 6272 * l2arc_headroom number of max device writes to precache 6273 * l2arc_headroom_boost when we find compressed buffers during ARC 6274 * scanning, we multiply headroom by this 6275 * percentage factor for the next scan cycle, 6276 * since more compressed buffers are likely to 6277 * be present 6278 * l2arc_feed_secs seconds between L2ARC writing 6279 * 6280 * Tunables may be removed or added as future performance improvements are 6281 * integrated, and also may become zpool properties. 6282 * 6283 * There are three key functions that control how the L2ARC warms up: 6284 * 6285 * l2arc_write_eligible() check if a buffer is eligible to cache 6286 * l2arc_write_size() calculate how much to write 6287 * l2arc_write_interval() calculate sleep delay between writes 6288 * 6289 * These three functions determine what to write, how much, and how quickly 6290 * to send writes. 6291 */ 6292 6293 static boolean_t 6294 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr) 6295 { 6296 /* 6297 * A buffer is *not* eligible for the L2ARC if it: 6298 * 1. belongs to a different spa. 6299 * 2. is already cached on the L2ARC. 6300 * 3. has an I/O in progress (it may be an incomplete read). 6301 * 4. is flagged not eligible (zfs property). 6302 */ 6303 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) || 6304 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr)) 6305 return (B_FALSE); 6306 6307 return (B_TRUE); 6308 } 6309 6310 static uint64_t 6311 l2arc_write_size(void) 6312 { 6313 uint64_t size; 6314 6315 /* 6316 * Make sure our globals have meaningful values in case the user 6317 * altered them. 6318 */ 6319 size = l2arc_write_max; 6320 if (size == 0) { 6321 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must " 6322 "be greater than zero, resetting it to the default (%d)", 6323 L2ARC_WRITE_SIZE); 6324 size = l2arc_write_max = L2ARC_WRITE_SIZE; 6325 } 6326 6327 if (arc_warm == B_FALSE) 6328 size += l2arc_write_boost; 6329 6330 return (size); 6331 6332 } 6333 6334 static clock_t 6335 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 6336 { 6337 clock_t interval, next, now; 6338 6339 /* 6340 * If the ARC lists are busy, increase our write rate; if the 6341 * lists are stale, idle back. This is achieved by checking 6342 * how much we previously wrote - if it was more than half of 6343 * what we wanted, schedule the next write much sooner. 6344 */ 6345 if (l2arc_feed_again && wrote > (wanted / 2)) 6346 interval = (hz * l2arc_feed_min_ms) / 1000; 6347 else 6348 interval = hz * l2arc_feed_secs; 6349 6350 now = ddi_get_lbolt(); 6351 next = MAX(now, MIN(now + interval, began + interval)); 6352 6353 return (next); 6354 } 6355 6356 /* 6357 * Cycle through L2ARC devices. This is how L2ARC load balances. 6358 * If a device is returned, this also returns holding the spa config lock. 6359 */ 6360 static l2arc_dev_t * 6361 l2arc_dev_get_next(void) 6362 { 6363 l2arc_dev_t *first, *next = NULL; 6364 6365 /* 6366 * Lock out the removal of spas (spa_namespace_lock), then removal 6367 * of cache devices (l2arc_dev_mtx). Once a device has been selected, 6368 * both locks will be dropped and a spa config lock held instead. 6369 */ 6370 mutex_enter(&spa_namespace_lock); 6371 mutex_enter(&l2arc_dev_mtx); 6372 6373 /* if there are no vdevs, there is nothing to do */ 6374 if (l2arc_ndev == 0) 6375 goto out; 6376 6377 first = NULL; 6378 next = l2arc_dev_last; 6379 do { 6380 /* loop around the list looking for a non-faulted vdev */ 6381 if (next == NULL) { 6382 next = list_head(l2arc_dev_list); 6383 } else { 6384 next = list_next(l2arc_dev_list, next); 6385 if (next == NULL) 6386 next = list_head(l2arc_dev_list); 6387 } 6388 6389 /* if we have come back to the start, bail out */ 6390 if (first == NULL) 6391 first = next; 6392 else if (next == first) 6393 break; 6394 6395 } while (vdev_is_dead(next->l2ad_vdev)); 6396 6397 /* if we were unable to find any usable vdevs, return NULL */ 6398 if (vdev_is_dead(next->l2ad_vdev)) 6399 next = NULL; 6400 6401 l2arc_dev_last = next; 6402 6403 out: 6404 mutex_exit(&l2arc_dev_mtx); 6405 6406 /* 6407 * Grab the config lock to prevent the 'next' device from being 6408 * removed while we are writing to it. 6409 */ 6410 if (next != NULL) 6411 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 6412 mutex_exit(&spa_namespace_lock); 6413 6414 return (next); 6415 } 6416 6417 /* 6418 * Free buffers that were tagged for destruction. 6419 */ 6420 static void 6421 l2arc_do_free_on_write() 6422 { 6423 list_t *buflist; 6424 l2arc_data_free_t *df, *df_prev; 6425 6426 mutex_enter(&l2arc_free_on_write_mtx); 6427 buflist = l2arc_free_on_write; 6428 6429 for (df = list_tail(buflist); df; df = df_prev) { 6430 df_prev = list_prev(buflist, df); 6431 ASSERT3P(df->l2df_abd, !=, NULL); 6432 abd_free(df->l2df_abd); 6433 list_remove(buflist, df); 6434 kmem_free(df, sizeof (l2arc_data_free_t)); 6435 } 6436 6437 mutex_exit(&l2arc_free_on_write_mtx); 6438 } 6439 6440 /* 6441 * A write to a cache device has completed. Update all headers to allow 6442 * reads from these buffers to begin. 6443 */ 6444 static void 6445 l2arc_write_done(zio_t *zio) 6446 { 6447 l2arc_write_callback_t *cb; 6448 l2arc_dev_t *dev; 6449 list_t *buflist; 6450 arc_buf_hdr_t *head, *hdr, *hdr_prev; 6451 kmutex_t *hash_lock; 6452 int64_t bytes_dropped = 0; 6453 6454 cb = zio->io_private; 6455 ASSERT3P(cb, !=, NULL); 6456 dev = cb->l2wcb_dev; 6457 ASSERT3P(dev, !=, NULL); 6458 head = cb->l2wcb_head; 6459 ASSERT3P(head, !=, NULL); 6460 buflist = &dev->l2ad_buflist; 6461 ASSERT3P(buflist, !=, NULL); 6462 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 6463 l2arc_write_callback_t *, cb); 6464 6465 if (zio->io_error != 0) 6466 ARCSTAT_BUMP(arcstat_l2_writes_error); 6467 6468 /* 6469 * All writes completed, or an error was hit. 6470 */ 6471 top: 6472 mutex_enter(&dev->l2ad_mtx); 6473 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { 6474 hdr_prev = list_prev(buflist, hdr); 6475 6476 hash_lock = HDR_LOCK(hdr); 6477 6478 /* 6479 * We cannot use mutex_enter or else we can deadlock 6480 * with l2arc_write_buffers (due to swapping the order 6481 * the hash lock and l2ad_mtx are taken). 6482 */ 6483 if (!mutex_tryenter(hash_lock)) { 6484 /* 6485 * Missed the hash lock. We must retry so we 6486 * don't leave the ARC_FLAG_L2_WRITING bit set. 6487 */ 6488 ARCSTAT_BUMP(arcstat_l2_writes_lock_retry); 6489 6490 /* 6491 * We don't want to rescan the headers we've 6492 * already marked as having been written out, so 6493 * we reinsert the head node so we can pick up 6494 * where we left off. 6495 */ 6496 list_remove(buflist, head); 6497 list_insert_after(buflist, hdr, head); 6498 6499 mutex_exit(&dev->l2ad_mtx); 6500 6501 /* 6502 * We wait for the hash lock to become available 6503 * to try and prevent busy waiting, and increase 6504 * the chance we'll be able to acquire the lock 6505 * the next time around. 6506 */ 6507 mutex_enter(hash_lock); 6508 mutex_exit(hash_lock); 6509 goto top; 6510 } 6511 6512 /* 6513 * We could not have been moved into the arc_l2c_only 6514 * state while in-flight due to our ARC_FLAG_L2_WRITING 6515 * bit being set. Let's just ensure that's being enforced. 6516 */ 6517 ASSERT(HDR_HAS_L1HDR(hdr)); 6518 6519 if (zio->io_error != 0) { 6520 /* 6521 * Error - drop L2ARC entry. 6522 */ 6523 list_remove(buflist, hdr); 6524 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); 6525 6526 ARCSTAT_INCR(arcstat_l2_psize, -arc_hdr_size(hdr)); 6527 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr)); 6528 6529 bytes_dropped += arc_hdr_size(hdr); 6530 (void) refcount_remove_many(&dev->l2ad_alloc, 6531 arc_hdr_size(hdr), hdr); 6532 } 6533 6534 /* 6535 * Allow ARC to begin reads and ghost list evictions to 6536 * this L2ARC entry. 6537 */ 6538 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING); 6539 6540 mutex_exit(hash_lock); 6541 } 6542 6543 atomic_inc_64(&l2arc_writes_done); 6544 list_remove(buflist, head); 6545 ASSERT(!HDR_HAS_L1HDR(head)); 6546 kmem_cache_free(hdr_l2only_cache, head); 6547 mutex_exit(&dev->l2ad_mtx); 6548 6549 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0); 6550 6551 l2arc_do_free_on_write(); 6552 6553 kmem_free(cb, sizeof (l2arc_write_callback_t)); 6554 } 6555 6556 /* 6557 * A read to a cache device completed. Validate buffer contents before 6558 * handing over to the regular ARC routines. 6559 */ 6560 static void 6561 l2arc_read_done(zio_t *zio) 6562 { 6563 l2arc_read_callback_t *cb; 6564 arc_buf_hdr_t *hdr; 6565 kmutex_t *hash_lock; 6566 boolean_t valid_cksum; 6567 6568 ASSERT3P(zio->io_vd, !=, NULL); 6569 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 6570 6571 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 6572 6573 cb = zio->io_private; 6574 ASSERT3P(cb, !=, NULL); 6575 hdr = cb->l2rcb_hdr; 6576 ASSERT3P(hdr, !=, NULL); 6577 6578 hash_lock = HDR_LOCK(hdr); 6579 mutex_enter(hash_lock); 6580 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 6581 6582 /* 6583 * If the data was read into a temporary buffer, 6584 * move it and free the buffer. 6585 */ 6586 if (cb->l2rcb_abd != NULL) { 6587 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size); 6588 if (zio->io_error == 0) { 6589 abd_copy(hdr->b_l1hdr.b_pabd, cb->l2rcb_abd, 6590 arc_hdr_size(hdr)); 6591 } 6592 6593 /* 6594 * The following must be done regardless of whether 6595 * there was an error: 6596 * - free the temporary buffer 6597 * - point zio to the real ARC buffer 6598 * - set zio size accordingly 6599 * These are required because zio is either re-used for 6600 * an I/O of the block in the case of the error 6601 * or the zio is passed to arc_read_done() and it 6602 * needs real data. 6603 */ 6604 abd_free(cb->l2rcb_abd); 6605 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr); 6606 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd; 6607 } 6608 6609 ASSERT3P(zio->io_abd, !=, NULL); 6610 6611 /* 6612 * Check this survived the L2ARC journey. 6613 */ 6614 ASSERT3P(zio->io_abd, ==, hdr->b_l1hdr.b_pabd); 6615 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 6616 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 6617 6618 valid_cksum = arc_cksum_is_equal(hdr, zio); 6619 if (valid_cksum && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 6620 mutex_exit(hash_lock); 6621 zio->io_private = hdr; 6622 arc_read_done(zio); 6623 } else { 6624 mutex_exit(hash_lock); 6625 /* 6626 * Buffer didn't survive caching. Increment stats and 6627 * reissue to the original storage device. 6628 */ 6629 if (zio->io_error != 0) { 6630 ARCSTAT_BUMP(arcstat_l2_io_error); 6631 } else { 6632 zio->io_error = SET_ERROR(EIO); 6633 } 6634 if (!valid_cksum) 6635 ARCSTAT_BUMP(arcstat_l2_cksum_bad); 6636 6637 /* 6638 * If there's no waiter, issue an async i/o to the primary 6639 * storage now. If there *is* a waiter, the caller must 6640 * issue the i/o in a context where it's OK to block. 6641 */ 6642 if (zio->io_waiter == NULL) { 6643 zio_t *pio = zio_unique_parent(zio); 6644 6645 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 6646 6647 zio_nowait(zio_read(pio, zio->io_spa, zio->io_bp, 6648 hdr->b_l1hdr.b_pabd, zio->io_size, arc_read_done, 6649 hdr, zio->io_priority, cb->l2rcb_flags, 6650 &cb->l2rcb_zb)); 6651 } 6652 } 6653 6654 kmem_free(cb, sizeof (l2arc_read_callback_t)); 6655 } 6656 6657 /* 6658 * This is the list priority from which the L2ARC will search for pages to 6659 * cache. This is used within loops (0..3) to cycle through lists in the 6660 * desired order. This order can have a significant effect on cache 6661 * performance. 6662 * 6663 * Currently the metadata lists are hit first, MFU then MRU, followed by 6664 * the data lists. This function returns a locked list, and also returns 6665 * the lock pointer. 6666 */ 6667 static multilist_sublist_t * 6668 l2arc_sublist_lock(int list_num) 6669 { 6670 multilist_t *ml = NULL; 6671 unsigned int idx; 6672 6673 ASSERT(list_num >= 0 && list_num <= 3); 6674 6675 switch (list_num) { 6676 case 0: 6677 ml = arc_mfu->arcs_list[ARC_BUFC_METADATA]; 6678 break; 6679 case 1: 6680 ml = arc_mru->arcs_list[ARC_BUFC_METADATA]; 6681 break; 6682 case 2: 6683 ml = arc_mfu->arcs_list[ARC_BUFC_DATA]; 6684 break; 6685 case 3: 6686 ml = arc_mru->arcs_list[ARC_BUFC_DATA]; 6687 break; 6688 } 6689 6690 /* 6691 * Return a randomly-selected sublist. This is acceptable 6692 * because the caller feeds only a little bit of data for each 6693 * call (8MB). Subsequent calls will result in different 6694 * sublists being selected. 6695 */ 6696 idx = multilist_get_random_index(ml); 6697 return (multilist_sublist_lock(ml, idx)); 6698 } 6699 6700 /* 6701 * Evict buffers from the device write hand to the distance specified in 6702 * bytes. This distance may span populated buffers, it may span nothing. 6703 * This is clearing a region on the L2ARC device ready for writing. 6704 * If the 'all' boolean is set, every buffer is evicted. 6705 */ 6706 static void 6707 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 6708 { 6709 list_t *buflist; 6710 arc_buf_hdr_t *hdr, *hdr_prev; 6711 kmutex_t *hash_lock; 6712 uint64_t taddr; 6713 6714 buflist = &dev->l2ad_buflist; 6715 6716 if (!all && dev->l2ad_first) { 6717 /* 6718 * This is the first sweep through the device. There is 6719 * nothing to evict. 6720 */ 6721 return; 6722 } 6723 6724 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 6725 /* 6726 * When nearing the end of the device, evict to the end 6727 * before the device write hand jumps to the start. 6728 */ 6729 taddr = dev->l2ad_end; 6730 } else { 6731 taddr = dev->l2ad_hand + distance; 6732 } 6733 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 6734 uint64_t, taddr, boolean_t, all); 6735 6736 top: 6737 mutex_enter(&dev->l2ad_mtx); 6738 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { 6739 hdr_prev = list_prev(buflist, hdr); 6740 6741 hash_lock = HDR_LOCK(hdr); 6742 6743 /* 6744 * We cannot use mutex_enter or else we can deadlock 6745 * with l2arc_write_buffers (due to swapping the order 6746 * the hash lock and l2ad_mtx are taken). 6747 */ 6748 if (!mutex_tryenter(hash_lock)) { 6749 /* 6750 * Missed the hash lock. Retry. 6751 */ 6752 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 6753 mutex_exit(&dev->l2ad_mtx); 6754 mutex_enter(hash_lock); 6755 mutex_exit(hash_lock); 6756 goto top; 6757 } 6758 6759 /* 6760 * A header can't be on this list if it doesn't have L2 header. 6761 */ 6762 ASSERT(HDR_HAS_L2HDR(hdr)); 6763 6764 /* Ensure this header has finished being written. */ 6765 ASSERT(!HDR_L2_WRITING(hdr)); 6766 ASSERT(!HDR_L2_WRITE_HEAD(hdr)); 6767 6768 if (!all && (hdr->b_l2hdr.b_daddr >= taddr || 6769 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { 6770 /* 6771 * We've evicted to the target address, 6772 * or the end of the device. 6773 */ 6774 mutex_exit(hash_lock); 6775 break; 6776 } 6777 6778 if (!HDR_HAS_L1HDR(hdr)) { 6779 ASSERT(!HDR_L2_READING(hdr)); 6780 /* 6781 * This doesn't exist in the ARC. Destroy. 6782 * arc_hdr_destroy() will call list_remove() 6783 * and decrement arcstat_l2_lsize. 6784 */ 6785 arc_change_state(arc_anon, hdr, hash_lock); 6786 arc_hdr_destroy(hdr); 6787 } else { 6788 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); 6789 ARCSTAT_BUMP(arcstat_l2_evict_l1cached); 6790 /* 6791 * Invalidate issued or about to be issued 6792 * reads, since we may be about to write 6793 * over this location. 6794 */ 6795 if (HDR_L2_READING(hdr)) { 6796 ARCSTAT_BUMP(arcstat_l2_evict_reading); 6797 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED); 6798 } 6799 6800 arc_hdr_l2hdr_destroy(hdr); 6801 } 6802 mutex_exit(hash_lock); 6803 } 6804 mutex_exit(&dev->l2ad_mtx); 6805 } 6806 6807 /* 6808 * Find and write ARC buffers to the L2ARC device. 6809 * 6810 * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid 6811 * for reading until they have completed writing. 6812 * The headroom_boost is an in-out parameter used to maintain headroom boost 6813 * state between calls to this function. 6814 * 6815 * Returns the number of bytes actually written (which may be smaller than 6816 * the delta by which the device hand has changed due to alignment). 6817 */ 6818 static uint64_t 6819 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 6820 { 6821 arc_buf_hdr_t *hdr, *hdr_prev, *head; 6822 uint64_t write_asize, write_psize, write_lsize, headroom; 6823 boolean_t full; 6824 l2arc_write_callback_t *cb; 6825 zio_t *pio, *wzio; 6826 uint64_t guid = spa_load_guid(spa); 6827 6828 ASSERT3P(dev->l2ad_vdev, !=, NULL); 6829 6830 pio = NULL; 6831 write_lsize = write_asize = write_psize = 0; 6832 full = B_FALSE; 6833 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE); 6834 arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR); 6835 6836 /* 6837 * Copy buffers for L2ARC writing. 6838 */ 6839 for (int try = 0; try <= 3; try++) { 6840 multilist_sublist_t *mls = l2arc_sublist_lock(try); 6841 uint64_t passed_sz = 0; 6842 6843 /* 6844 * L2ARC fast warmup. 6845 * 6846 * Until the ARC is warm and starts to evict, read from the 6847 * head of the ARC lists rather than the tail. 6848 */ 6849 if (arc_warm == B_FALSE) 6850 hdr = multilist_sublist_head(mls); 6851 else 6852 hdr = multilist_sublist_tail(mls); 6853 6854 headroom = target_sz * l2arc_headroom; 6855 if (zfs_compressed_arc_enabled) 6856 headroom = (headroom * l2arc_headroom_boost) / 100; 6857 6858 for (; hdr; hdr = hdr_prev) { 6859 kmutex_t *hash_lock; 6860 6861 if (arc_warm == B_FALSE) 6862 hdr_prev = multilist_sublist_next(mls, hdr); 6863 else 6864 hdr_prev = multilist_sublist_prev(mls, hdr); 6865 6866 hash_lock = HDR_LOCK(hdr); 6867 if (!mutex_tryenter(hash_lock)) { 6868 /* 6869 * Skip this buffer rather than waiting. 6870 */ 6871 continue; 6872 } 6873 6874 passed_sz += HDR_GET_LSIZE(hdr); 6875 if (passed_sz > headroom) { 6876 /* 6877 * Searched too far. 6878 */ 6879 mutex_exit(hash_lock); 6880 break; 6881 } 6882 6883 if (!l2arc_write_eligible(guid, hdr)) { 6884 mutex_exit(hash_lock); 6885 continue; 6886 } 6887 6888 /* 6889 * We rely on the L1 portion of the header below, so 6890 * it's invalid for this header to have been evicted out 6891 * of the ghost cache, prior to being written out. The 6892 * ARC_FLAG_L2_WRITING bit ensures this won't happen. 6893 */ 6894 ASSERT(HDR_HAS_L1HDR(hdr)); 6895 6896 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0); 6897 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 6898 ASSERT3U(arc_hdr_size(hdr), >, 0); 6899 uint64_t psize = arc_hdr_size(hdr); 6900 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, 6901 psize); 6902 6903 if ((write_asize + asize) > target_sz) { 6904 full = B_TRUE; 6905 mutex_exit(hash_lock); 6906 break; 6907 } 6908 6909 if (pio == NULL) { 6910 /* 6911 * Insert a dummy header on the buflist so 6912 * l2arc_write_done() can find where the 6913 * write buffers begin without searching. 6914 */ 6915 mutex_enter(&dev->l2ad_mtx); 6916 list_insert_head(&dev->l2ad_buflist, head); 6917 mutex_exit(&dev->l2ad_mtx); 6918 6919 cb = kmem_alloc( 6920 sizeof (l2arc_write_callback_t), KM_SLEEP); 6921 cb->l2wcb_dev = dev; 6922 cb->l2wcb_head = head; 6923 pio = zio_root(spa, l2arc_write_done, cb, 6924 ZIO_FLAG_CANFAIL); 6925 } 6926 6927 hdr->b_l2hdr.b_dev = dev; 6928 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; 6929 arc_hdr_set_flags(hdr, 6930 ARC_FLAG_L2_WRITING | ARC_FLAG_HAS_L2HDR); 6931 6932 mutex_enter(&dev->l2ad_mtx); 6933 list_insert_head(&dev->l2ad_buflist, hdr); 6934 mutex_exit(&dev->l2ad_mtx); 6935 6936 (void) refcount_add_many(&dev->l2ad_alloc, psize, hdr); 6937 6938 /* 6939 * Normally the L2ARC can use the hdr's data, but if 6940 * we're sharing data between the hdr and one of its 6941 * bufs, L2ARC needs its own copy of the data so that 6942 * the ZIO below can't race with the buf consumer. 6943 * Another case where we need to create a copy of the 6944 * data is when the buffer size is not device-aligned 6945 * and we need to pad the block to make it such. 6946 * That also keeps the clock hand suitably aligned. 6947 * 6948 * To ensure that the copy will be available for the 6949 * lifetime of the ZIO and be cleaned up afterwards, we 6950 * add it to the l2arc_free_on_write queue. 6951 */ 6952 abd_t *to_write; 6953 if (!HDR_SHARED_DATA(hdr) && psize == asize) { 6954 to_write = hdr->b_l1hdr.b_pabd; 6955 } else { 6956 to_write = abd_alloc_for_io(asize, 6957 HDR_ISTYPE_METADATA(hdr)); 6958 abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize); 6959 if (asize != psize) { 6960 abd_zero_off(to_write, psize, 6961 asize - psize); 6962 } 6963 l2arc_free_abd_on_write(to_write, asize, 6964 arc_buf_type(hdr)); 6965 } 6966 wzio = zio_write_phys(pio, dev->l2ad_vdev, 6967 hdr->b_l2hdr.b_daddr, asize, to_write, 6968 ZIO_CHECKSUM_OFF, NULL, hdr, 6969 ZIO_PRIORITY_ASYNC_WRITE, 6970 ZIO_FLAG_CANFAIL, B_FALSE); 6971 6972 write_lsize += HDR_GET_LSIZE(hdr); 6973 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 6974 zio_t *, wzio); 6975 6976 write_psize += psize; 6977 write_asize += asize; 6978 dev->l2ad_hand += asize; 6979 6980 mutex_exit(hash_lock); 6981 6982 (void) zio_nowait(wzio); 6983 } 6984 6985 multilist_sublist_unlock(mls); 6986 6987 if (full == B_TRUE) 6988 break; 6989 } 6990 6991 /* No buffers selected for writing? */ 6992 if (pio == NULL) { 6993 ASSERT0(write_lsize); 6994 ASSERT(!HDR_HAS_L1HDR(head)); 6995 kmem_cache_free(hdr_l2only_cache, head); 6996 return (0); 6997 } 6998 6999 ASSERT3U(write_asize, <=, target_sz); 7000 ARCSTAT_BUMP(arcstat_l2_writes_sent); 7001 ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize); 7002 ARCSTAT_INCR(arcstat_l2_lsize, write_lsize); 7003 ARCSTAT_INCR(arcstat_l2_psize, write_psize); 7004 vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0); 7005 7006 /* 7007 * Bump device hand to the device start if it is approaching the end. 7008 * l2arc_evict() will already have evicted ahead for this case. 7009 */ 7010 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 7011 dev->l2ad_hand = dev->l2ad_start; 7012 dev->l2ad_first = B_FALSE; 7013 } 7014 7015 dev->l2ad_writing = B_TRUE; 7016 (void) zio_wait(pio); 7017 dev->l2ad_writing = B_FALSE; 7018 7019 return (write_asize); 7020 } 7021 7022 /* 7023 * This thread feeds the L2ARC at regular intervals. This is the beating 7024 * heart of the L2ARC. 7025 */ 7026 /* ARGSUSED */ 7027 static void 7028 l2arc_feed_thread(void *unused) 7029 { 7030 callb_cpr_t cpr; 7031 l2arc_dev_t *dev; 7032 spa_t *spa; 7033 uint64_t size, wrote; 7034 clock_t begin, next = ddi_get_lbolt(); 7035 7036 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 7037 7038 mutex_enter(&l2arc_feed_thr_lock); 7039 7040 while (l2arc_thread_exit == 0) { 7041 CALLB_CPR_SAFE_BEGIN(&cpr); 7042 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 7043 next); 7044 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 7045 next = ddi_get_lbolt() + hz; 7046 7047 /* 7048 * Quick check for L2ARC devices. 7049 */ 7050 mutex_enter(&l2arc_dev_mtx); 7051 if (l2arc_ndev == 0) { 7052 mutex_exit(&l2arc_dev_mtx); 7053 continue; 7054 } 7055 mutex_exit(&l2arc_dev_mtx); 7056 begin = ddi_get_lbolt(); 7057 7058 /* 7059 * This selects the next l2arc device to write to, and in 7060 * doing so the next spa to feed from: dev->l2ad_spa. This 7061 * will return NULL if there are now no l2arc devices or if 7062 * they are all faulted. 7063 * 7064 * If a device is returned, its spa's config lock is also 7065 * held to prevent device removal. l2arc_dev_get_next() 7066 * will grab and release l2arc_dev_mtx. 7067 */ 7068 if ((dev = l2arc_dev_get_next()) == NULL) 7069 continue; 7070 7071 spa = dev->l2ad_spa; 7072 ASSERT3P(spa, !=, NULL); 7073 7074 /* 7075 * If the pool is read-only then force the feed thread to 7076 * sleep a little longer. 7077 */ 7078 if (!spa_writeable(spa)) { 7079 next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz; 7080 spa_config_exit(spa, SCL_L2ARC, dev); 7081 continue; 7082 } 7083 7084 /* 7085 * Avoid contributing to memory pressure. 7086 */ 7087 if (arc_reclaim_needed()) { 7088 ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 7089 spa_config_exit(spa, SCL_L2ARC, dev); 7090 continue; 7091 } 7092 7093 ARCSTAT_BUMP(arcstat_l2_feeds); 7094 7095 size = l2arc_write_size(); 7096 7097 /* 7098 * Evict L2ARC buffers that will be overwritten. 7099 */ 7100 l2arc_evict(dev, size, B_FALSE); 7101 7102 /* 7103 * Write ARC buffers. 7104 */ 7105 wrote = l2arc_write_buffers(spa, dev, size); 7106 7107 /* 7108 * Calculate interval between writes. 7109 */ 7110 next = l2arc_write_interval(begin, size, wrote); 7111 spa_config_exit(spa, SCL_L2ARC, dev); 7112 } 7113 7114 l2arc_thread_exit = 0; 7115 cv_broadcast(&l2arc_feed_thr_cv); 7116 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 7117 thread_exit(); 7118 } 7119 7120 boolean_t 7121 l2arc_vdev_present(vdev_t *vd) 7122 { 7123 l2arc_dev_t *dev; 7124 7125 mutex_enter(&l2arc_dev_mtx); 7126 for (dev = list_head(l2arc_dev_list); dev != NULL; 7127 dev = list_next(l2arc_dev_list, dev)) { 7128 if (dev->l2ad_vdev == vd) 7129 break; 7130 } 7131 mutex_exit(&l2arc_dev_mtx); 7132 7133 return (dev != NULL); 7134 } 7135 7136 /* 7137 * Add a vdev for use by the L2ARC. By this point the spa has already 7138 * validated the vdev and opened it. 7139 */ 7140 void 7141 l2arc_add_vdev(spa_t *spa, vdev_t *vd) 7142 { 7143 l2arc_dev_t *adddev; 7144 7145 ASSERT(!l2arc_vdev_present(vd)); 7146 7147 /* 7148 * Create a new l2arc device entry. 7149 */ 7150 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 7151 adddev->l2ad_spa = spa; 7152 adddev->l2ad_vdev = vd; 7153 adddev->l2ad_start = VDEV_LABEL_START_SIZE; 7154 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 7155 adddev->l2ad_hand = adddev->l2ad_start; 7156 adddev->l2ad_first = B_TRUE; 7157 adddev->l2ad_writing = B_FALSE; 7158 7159 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL); 7160 /* 7161 * This is a list of all ARC buffers that are still valid on the 7162 * device. 7163 */ 7164 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 7165 offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node)); 7166 7167 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 7168 refcount_create(&adddev->l2ad_alloc); 7169 7170 /* 7171 * Add device to global list 7172 */ 7173 mutex_enter(&l2arc_dev_mtx); 7174 list_insert_head(l2arc_dev_list, adddev); 7175 atomic_inc_64(&l2arc_ndev); 7176 mutex_exit(&l2arc_dev_mtx); 7177 } 7178 7179 /* 7180 * Remove a vdev from the L2ARC. 7181 */ 7182 void 7183 l2arc_remove_vdev(vdev_t *vd) 7184 { 7185 l2arc_dev_t *dev, *nextdev, *remdev = NULL; 7186 7187 /* 7188 * Find the device by vdev 7189 */ 7190 mutex_enter(&l2arc_dev_mtx); 7191 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 7192 nextdev = list_next(l2arc_dev_list, dev); 7193 if (vd == dev->l2ad_vdev) { 7194 remdev = dev; 7195 break; 7196 } 7197 } 7198 ASSERT3P(remdev, !=, NULL); 7199 7200 /* 7201 * Remove device from global list 7202 */ 7203 list_remove(l2arc_dev_list, remdev); 7204 l2arc_dev_last = NULL; /* may have been invalidated */ 7205 atomic_dec_64(&l2arc_ndev); 7206 mutex_exit(&l2arc_dev_mtx); 7207 7208 /* 7209 * Clear all buflists and ARC references. L2ARC device flush. 7210 */ 7211 l2arc_evict(remdev, 0, B_TRUE); 7212 list_destroy(&remdev->l2ad_buflist); 7213 mutex_destroy(&remdev->l2ad_mtx); 7214 refcount_destroy(&remdev->l2ad_alloc); 7215 kmem_free(remdev, sizeof (l2arc_dev_t)); 7216 } 7217 7218 void 7219 l2arc_init(void) 7220 { 7221 l2arc_thread_exit = 0; 7222 l2arc_ndev = 0; 7223 l2arc_writes_sent = 0; 7224 l2arc_writes_done = 0; 7225 7226 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 7227 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 7228 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 7229 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 7230 7231 l2arc_dev_list = &L2ARC_dev_list; 7232 l2arc_free_on_write = &L2ARC_free_on_write; 7233 list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 7234 offsetof(l2arc_dev_t, l2ad_node)); 7235 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 7236 offsetof(l2arc_data_free_t, l2df_list_node)); 7237 } 7238 7239 void 7240 l2arc_fini(void) 7241 { 7242 /* 7243 * This is called from dmu_fini(), which is called from spa_fini(); 7244 * Because of this, we can assume that all l2arc devices have 7245 * already been removed when the pools themselves were removed. 7246 */ 7247 7248 l2arc_do_free_on_write(); 7249 7250 mutex_destroy(&l2arc_feed_thr_lock); 7251 cv_destroy(&l2arc_feed_thr_cv); 7252 mutex_destroy(&l2arc_dev_mtx); 7253 mutex_destroy(&l2arc_free_on_write_mtx); 7254 7255 list_destroy(l2arc_dev_list); 7256 list_destroy(l2arc_free_on_write); 7257 } 7258 7259 void 7260 l2arc_start(void) 7261 { 7262 if (!(spa_mode_global & FWRITE)) 7263 return; 7264 7265 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 7266 TS_RUN, minclsyspri); 7267 } 7268 7269 void 7270 l2arc_stop(void) 7271 { 7272 if (!(spa_mode_global & FWRITE)) 7273 return; 7274 7275 mutex_enter(&l2arc_feed_thr_lock); 7276 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 7277 l2arc_thread_exit = 1; 7278 while (l2arc_thread_exit != 0) 7279 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 7280 mutex_exit(&l2arc_feed_thr_lock); 7281 } 7282