1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 14 * Copyright (c) 2013, Delphix. All rights reserved. 15 * Copyright (c) 2013, Saso Kiselkov. All rights reserved. 16 * Copyright (c) 2013, Nexenta Systems, Inc. All rights reserved. 17 * Copyright (c) 2020, George Amanakis. All rights reserved. 18 */ 19 20 #ifndef _SYS_ARC_IMPL_H 21 #define _SYS_ARC_IMPL_H 22 23 #include <sys/arc.h> 24 #include <sys/multilist.h> 25 #include <sys/zio_crypt.h> 26 #include <sys/zthr.h> 27 #include <sys/aggsum.h> 28 #include <sys/wmsum.h> 29 30 #ifdef __cplusplus 31 extern "C" { 32 #endif 33 34 /* 35 * We can feed L2ARC from two states of ARC buffers, mru and mfu, 36 * and each of the states has two types: data and metadata. 37 */ 38 #define L2ARC_FEED_TYPES 4 39 #define L2ARC_MFU_META 0 40 #define L2ARC_MRU_META 1 41 #define L2ARC_MFU_DATA 2 42 #define L2ARC_MRU_DATA 3 43 44 /* 45 * L2ARC state and statistics for persistent marker management. 46 */ 47 typedef struct l2arc_info { 48 arc_buf_hdr_t **l2arc_markers[L2ARC_FEED_TYPES]; 49 uint64_t l2arc_total_writes; /* total writes for reset */ 50 uint64_t l2arc_total_capacity; /* total L2ARC capacity */ 51 uint64_t l2arc_smallest_capacity; /* smallest device capacity */ 52 /* 53 * Per-device thread coordination for sublist processing. 54 * reset: flags sublist marker for lazy reset to tail. 55 */ 56 boolean_t *l2arc_sublist_busy[L2ARC_FEED_TYPES]; 57 boolean_t *l2arc_sublist_reset[L2ARC_FEED_TYPES]; 58 kmutex_t l2arc_sublist_lock; /* protects busy/reset flags */ 59 /* 60 * Cumulative bytes scanned per pass since marker reset. 61 * Limits how far persistent markers advance from tail 62 * before resetting, based on % of state size. 63 */ 64 uint64_t l2arc_ext_scanned[L2ARC_FEED_TYPES]; 65 int l2arc_next_sublist[L2ARC_FEED_TYPES]; /* round-robin */ 66 } l2arc_info_t; 67 68 /* 69 * Note that buffers can be in one of 6 states: 70 * ARC_anon - anonymous (discussed below) 71 * ARC_mru - recently used, currently cached 72 * ARC_mru_ghost - recently used, no longer in cache 73 * ARC_mfu - frequently used, currently cached 74 * ARC_mfu_ghost - frequently used, no longer in cache 75 * ARC_uncached - uncacheable prefetch, to be evicted 76 * ARC_l2c_only - exists in L2ARC but not other states 77 * When there are no active references to the buffer, they are 78 * are linked onto a list in one of these arc states. These are 79 * the only buffers that can be evicted or deleted. Within each 80 * state there are multiple lists, one for meta-data and one for 81 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 82 * etc.) is tracked separately so that it can be managed more 83 * explicitly: favored over data, limited explicitly. 84 * 85 * Anonymous buffers are buffers that are not associated with 86 * a DVA. These are buffers that hold dirty block copies 87 * before they are written to stable storage. By definition, 88 * they are "ref'd" and are considered part of arc_mru 89 * that cannot be freed. Generally, they will acquire a DVA 90 * as they are written and migrate onto the arc_mru list. 91 * 92 * The ARC_l2c_only state is for buffers that are in the second 93 * level ARC but no longer in any of the ARC_m* lists. The second 94 * level ARC itself may also contain buffers that are in any of 95 * the ARC_m* states - meaning that a buffer can exist in two 96 * places. The reason for the ARC_l2c_only state is to keep the 97 * buffer header in the hash table, so that reads that hit the 98 * second level ARC benefit from these fast lookups. 99 */ 100 101 typedef struct arc_state { 102 /* 103 * list of evictable buffers 104 */ 105 multilist_t arcs_list[ARC_BUFC_NUMTYPES]; 106 /* 107 * supports the "dbufs" kstat 108 */ 109 arc_state_type_t arcs_state; 110 /* 111 * total amount of data in this state. 112 */ 113 zfs_refcount_t arcs_size[ARC_BUFC_NUMTYPES] ____cacheline_aligned; 114 /* 115 * total amount of evictable data in this state 116 */ 117 zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES]; 118 /* 119 * amount of hit bytes for this state (counted only for ghost states) 120 */ 121 wmsum_t arcs_hits[ARC_BUFC_NUMTYPES]; 122 } arc_state_t; 123 124 typedef struct arc_callback arc_callback_t; 125 126 struct arc_callback { 127 void *acb_private; 128 arc_read_done_func_t *acb_done; 129 arc_buf_t *acb_buf; 130 boolean_t acb_encrypted; 131 boolean_t acb_compressed; 132 boolean_t acb_noauth; 133 boolean_t acb_nobuf; 134 boolean_t acb_wait; 135 int acb_wait_error; 136 kmutex_t acb_wait_lock; 137 kcondvar_t acb_wait_cv; 138 zbookmark_phys_t acb_zb; 139 zio_t *acb_zio_dummy; 140 zio_t *acb_zio_head; 141 arc_callback_t *acb_prev; 142 arc_callback_t *acb_next; 143 }; 144 145 typedef struct arc_write_callback arc_write_callback_t; 146 147 struct arc_write_callback { 148 void *awcb_private; 149 arc_write_done_func_t *awcb_ready; 150 arc_write_done_func_t *awcb_children_ready; 151 arc_write_done_func_t *awcb_done; 152 arc_buf_t *awcb_buf; 153 }; 154 155 /* 156 * ARC buffers are separated into multiple structs as a memory saving measure: 157 * - Common fields struct, always defined, and embedded within it: 158 * - L2-only fields, always allocated but undefined when not in L2ARC 159 * - L1-only fields, only allocated when in L1ARC 160 * 161 * Buffer in L1 Buffer only in L2 162 * +------------------------+ +------------------------+ 163 * | arc_buf_hdr_t | | arc_buf_hdr_t | 164 * | | | | 165 * | | | | 166 * | | | | 167 * +------------------------+ +------------------------+ 168 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t | 169 * | (undefined if L1-only) | | | 170 * +------------------------+ +------------------------+ 171 * | l1arc_buf_hdr_t | 172 * | | 173 * | | 174 * | | 175 * | | 176 * +------------------------+ 177 * 178 * Because it's possible for the L2ARC to become extremely large, we can wind 179 * up eating a lot of memory in L2ARC buffer headers, so the size of a header 180 * is minimized by only allocating the fields necessary for an L1-cached buffer 181 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and 182 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple 183 * words in pointers. arc_hdr_realloc() is used to switch a header between 184 * these two allocation states. 185 */ 186 typedef struct l1arc_buf_hdr { 187 /* protected by arc state mutex */ 188 arc_state_t *b_state; 189 multilist_node_t b_arc_node; 190 191 /* protected by hash lock */ 192 clock_t b_arc_access; 193 uint32_t b_mru_hits; 194 uint32_t b_mru_ghost_hits; 195 uint32_t b_mfu_hits; 196 uint32_t b_mfu_ghost_hits; 197 uint8_t b_byteswap; 198 arc_buf_t *b_buf; 199 200 /* self protecting */ 201 zfs_refcount_t b_refcnt; 202 203 arc_callback_t *b_acb; 204 abd_t *b_pabd; 205 206 #ifdef ZFS_DEBUG 207 zio_cksum_t *b_freeze_cksum; 208 kmutex_t b_freeze_lock; 209 #endif 210 } l1arc_buf_hdr_t; 211 212 typedef enum l2arc_dev_hdr_flags_t { 213 L2ARC_DEV_HDR_EVICT_FIRST = (1 << 0) /* mirror of l2ad_first */ 214 } l2arc_dev_hdr_flags_t; 215 216 /* 217 * Pointer used in persistent L2ARC (for pointing to log blocks). 218 */ 219 typedef struct l2arc_log_blkptr { 220 /* 221 * Offset of log block within the device, in bytes 222 */ 223 uint64_t lbp_daddr; 224 /* 225 * Aligned payload size (in bytes) of the log block 226 */ 227 uint64_t lbp_payload_asize; 228 /* 229 * Offset in bytes of the first buffer in the payload 230 */ 231 uint64_t lbp_payload_start; 232 /* 233 * lbp_prop has the following format: 234 * * logical size (in bytes) 235 * * aligned (after compression) size (in bytes) 236 * * compression algorithm (we always LZ4-compress l2arc logs) 237 * * checksum algorithm (used for lbp_cksum) 238 */ 239 uint64_t lbp_prop; 240 zio_cksum_t lbp_cksum; /* checksum of log */ 241 } l2arc_log_blkptr_t; 242 243 /* 244 * The persistent L2ARC device header. 245 * Byte order of magic determines whether 64-bit bswap of fields is necessary. 246 */ 247 typedef struct l2arc_dev_hdr_phys { 248 uint64_t dh_magic; /* L2ARC_DEV_HDR_MAGIC */ 249 uint64_t dh_version; /* Persistent L2ARC version */ 250 251 /* 252 * Global L2ARC device state and metadata. 253 */ 254 uint64_t dh_spa_guid; 255 uint64_t dh_vdev_guid; 256 uint64_t dh_log_entries; /* mirror of l2ad_log_entries */ 257 uint64_t dh_evict; /* evicted offset in bytes */ 258 uint64_t dh_flags; /* l2arc_dev_hdr_flags_t */ 259 /* 260 * Used in zdb.c for determining if a log block is valid, in the same 261 * way that l2arc_rebuild() does. 262 */ 263 uint64_t dh_start; /* mirror of l2ad_start */ 264 uint64_t dh_end; /* mirror of l2ad_end */ 265 /* 266 * Start of log block chain. [0] -> newest log, [1] -> one older (used 267 * for initiating prefetch). 268 */ 269 l2arc_log_blkptr_t dh_start_lbps[2]; 270 /* 271 * Aligned size of all log blocks as accounted by vdev_space_update(). 272 */ 273 uint64_t dh_lb_asize; /* mirror of l2ad_lb_asize */ 274 uint64_t dh_lb_count; /* mirror of l2ad_lb_count */ 275 /* 276 * Mirrors of vdev_trim_action_time and vdev_trim_state, used to 277 * display when the cache device was fully trimmed for the last 278 * time. 279 */ 280 uint64_t dh_trim_action_time; 281 uint64_t dh_trim_state; 282 const uint64_t dh_pad[30]; /* pad to 512 bytes */ 283 zio_eck_t dh_tail; 284 } l2arc_dev_hdr_phys_t; 285 _Static_assert(sizeof (l2arc_dev_hdr_phys_t) == SPA_MINBLOCKSIZE, 286 "l2arc_dev_hdr_phys_t wrong size"); 287 288 /* 289 * A single ARC buffer header entry in a l2arc_log_blk_phys_t. 290 */ 291 typedef struct l2arc_log_ent_phys { 292 dva_t le_dva; /* dva of buffer */ 293 uint64_t le_birth; /* birth txg of buffer */ 294 /* 295 * le_prop has the following format: 296 * * logical size (in bytes) 297 * * physical (compressed) size (in bytes) 298 * * compression algorithm 299 * * object type (used to restore arc_buf_contents_t) 300 * * protected status (used for encryption) 301 * * prefetch status (used in l2arc_read_done()) 302 */ 303 uint64_t le_prop; 304 uint64_t le_daddr; /* buf location on l2dev */ 305 uint64_t le_complevel; 306 /* 307 * We pad the size of each entry to a power of 2 so that the size of 308 * l2arc_log_blk_phys_t is power-of-2 aligned with SPA_MINBLOCKSHIFT, 309 * because of the L2ARC_SET_*SIZE macros. 310 */ 311 const uint64_t le_pad[2]; /* pad to 64 bytes */ 312 } l2arc_log_ent_phys_t; 313 314 #define L2ARC_LOG_BLK_MAX_ENTRIES (1022) 315 316 /* 317 * A log block of up to 1022 ARC buffer log entries, chained into the 318 * persistent L2ARC metadata linked list. Byte order of magic determines 319 * whether 64-bit bswap of fields is necessary. 320 */ 321 typedef struct l2arc_log_blk_phys { 322 uint64_t lb_magic; /* L2ARC_LOG_BLK_MAGIC */ 323 /* 324 * There are 2 chains (headed by dh_start_lbps[2]), and this field 325 * points back to the previous block in this chain. We alternate 326 * which chain we append to, so they are time-wise and offset-wise 327 * interleaved, but that is an optimization rather than for 328 * correctness. 329 */ 330 l2arc_log_blkptr_t lb_prev_lbp; /* pointer to prev log block */ 331 /* 332 * Pad header section to 128 bytes 333 */ 334 uint64_t lb_pad[7]; 335 /* Payload */ 336 l2arc_log_ent_phys_t lb_entries[L2ARC_LOG_BLK_MAX_ENTRIES]; 337 } l2arc_log_blk_phys_t; /* 64K total */ 338 339 /* 340 * The size of l2arc_log_blk_phys_t has to be power-of-2 aligned with 341 * SPA_MINBLOCKSHIFT because of L2BLK_SET_*SIZE macros. 342 */ 343 _Static_assert(IS_P2ALIGNED(sizeof (l2arc_log_blk_phys_t), 344 1ULL << SPA_MINBLOCKSHIFT), "l2arc_log_blk_phys_t misaligned"); 345 _Static_assert(sizeof (l2arc_log_blk_phys_t) >= SPA_MINBLOCKSIZE, 346 "l2arc_log_blk_phys_t too small"); 347 _Static_assert(sizeof (l2arc_log_blk_phys_t) <= SPA_MAXBLOCKSIZE, 348 "l2arc_log_blk_phys_t too big"); 349 350 /* 351 * These structures hold in-flight abd buffers for log blocks as they're being 352 * written to the L2ARC device. 353 */ 354 typedef struct l2arc_lb_abd_buf { 355 abd_t *abd; 356 list_node_t node; 357 } l2arc_lb_abd_buf_t; 358 359 /* 360 * These structures hold pointers to log blocks present on the L2ARC device. 361 */ 362 typedef struct l2arc_lb_ptr_buf { 363 l2arc_log_blkptr_t *lb_ptr; 364 list_node_t node; 365 } l2arc_lb_ptr_buf_t; 366 367 /* Macros for setting fields in le_prop and lbp_prop */ 368 #define L2BLK_GET_LSIZE(field) \ 369 BF64_GET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1) 370 #define L2BLK_SET_LSIZE(field, x) \ 371 BF64_SET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1, x) 372 #define L2BLK_GET_PSIZE(field) \ 373 BF64_GET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1) 374 #define L2BLK_SET_PSIZE(field, x) \ 375 BF64_SET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1, x) 376 #define L2BLK_GET_COMPRESS(field) \ 377 BF64_GET((field), 32, SPA_COMPRESSBITS) 378 #define L2BLK_SET_COMPRESS(field, x) \ 379 BF64_SET((field), 32, SPA_COMPRESSBITS, x) 380 #define L2BLK_GET_PREFETCH(field) BF64_GET((field), 39, 1) 381 #define L2BLK_SET_PREFETCH(field, x) BF64_SET((field), 39, 1, x) 382 #define L2BLK_GET_CHECKSUM(field) BF64_GET((field), 40, 8) 383 #define L2BLK_SET_CHECKSUM(field, x) BF64_SET((field), 40, 8, x) 384 /* +/- 1 here are to keep compatibility after ARC_BUFC_INVALID removal. */ 385 #define L2BLK_GET_TYPE(field) (BF64_GET((field), 48, 8) - 1) 386 #define L2BLK_SET_TYPE(field, x) BF64_SET((field), 48, 8, (x) + 1) 387 #define L2BLK_GET_PROTECTED(field) BF64_GET((field), 56, 1) 388 #define L2BLK_SET_PROTECTED(field, x) BF64_SET((field), 56, 1, x) 389 #define L2BLK_GET_STATE(field) BF64_GET((field), 57, 4) 390 #define L2BLK_SET_STATE(field, x) BF64_SET((field), 57, 4, x) 391 392 #define PTR_SWAP(x, y) \ 393 do { \ 394 void *tmp = (x);\ 395 x = y; \ 396 y = tmp; \ 397 } while (0) 398 399 #define L2ARC_DEV_HDR_MAGIC 0x5a46534341434845LLU /* ASCII: "ZFSCACHE" */ 400 #define L2ARC_LOG_BLK_MAGIC 0x4c4f47424c4b4844LLU /* ASCII: "LOGBLKHD" */ 401 402 /* 403 * L2ARC Internals 404 */ 405 typedef struct l2arc_dev { 406 vdev_t *l2ad_vdev; /* can be NULL during remove */ 407 spa_t *l2ad_spa; /* can be NULL during remove */ 408 uint64_t l2ad_hand; /* next write location */ 409 uint64_t l2ad_start; /* first addr on device */ 410 uint64_t l2ad_end; /* last addr on device */ 411 boolean_t l2ad_first; /* first sweep through */ 412 boolean_t l2ad_writing; /* currently writing */ 413 kmutex_t l2ad_mtx; /* lock for buffer list */ 414 list_t l2ad_buflist; /* buffer list */ 415 list_node_t l2ad_node; /* device list node */ 416 zfs_refcount_t l2ad_alloc; /* allocated bytes */ 417 /* 418 * Persistence-related stuff 419 */ 420 l2arc_dev_hdr_phys_t *l2ad_dev_hdr; /* persistent device header */ 421 uint64_t l2ad_dev_hdr_asize; /* aligned hdr size */ 422 l2arc_log_blk_phys_t l2ad_log_blk; /* currently open log block */ 423 int l2ad_log_ent_idx; /* index into cur log blk */ 424 /* Number of bytes in current log block's payload */ 425 uint64_t l2ad_log_blk_payload_asize; 426 /* 427 * Offset (in bytes) of the first buffer in current log block's 428 * payload. 429 */ 430 uint64_t l2ad_log_blk_payload_start; 431 /* Flag indicating whether a rebuild is scheduled or is going on */ 432 boolean_t l2ad_rebuild; 433 boolean_t l2ad_rebuild_cancel; 434 boolean_t l2ad_rebuild_began; 435 uint64_t l2ad_log_entries; /* entries per log blk */ 436 uint64_t l2ad_evict; /* evicted offset in bytes */ 437 /* List of pointers to log blocks present in the L2ARC device */ 438 list_t l2ad_lbptr_list; 439 /* 440 * Aligned size of all log blocks as accounted by vdev_space_update(). 441 */ 442 zfs_refcount_t l2ad_lb_asize; 443 /* 444 * Number of log blocks present on the device. 445 */ 446 zfs_refcount_t l2ad_lb_count; 447 boolean_t l2ad_trim_all; /* TRIM whole device */ 448 /* 449 * DWPD tracking with daily reset 450 */ 451 uint64_t l2ad_dwpd_writes; /* 24h bytes written */ 452 uint64_t l2ad_dwpd_start; /* 24h period start */ 453 uint64_t l2ad_dwpd_accumulated; /* Accumulated */ 454 uint64_t l2ad_dwpd_bump; /* Reset trigger */ 455 /* 456 * Per-device feed thread for parallel L2ARC writes 457 */ 458 kthread_t *l2ad_feed_thread; /* feed thread handle */ 459 boolean_t l2ad_thread_exit; /* signal thread exit */ 460 kmutex_t l2ad_feed_thr_lock; /* thread sleep/wake */ 461 kcondvar_t l2ad_feed_cv; /* thread wakeup cv */ 462 /* 463 * Consecutive cycles where metadata filled write budget 464 * while data passes got nothing written. Used to detect 465 * monopolization and skip metadata to give data a chance. 466 */ 467 uint64_t l2ad_meta_cycles; 468 } l2arc_dev_t; 469 470 /* 471 * Encrypted blocks will need to be stored encrypted on the L2ARC 472 * disk as they appear in the main pool. In order for this to work we 473 * need to pass around the encryption parameters so they can be used 474 * to write data to the L2ARC. This struct is only defined in the 475 * arc_buf_hdr_t if the L1 header is defined and has the ARC_FLAG_ENCRYPTED 476 * flag set. 477 */ 478 typedef struct arc_buf_hdr_crypt { 479 abd_t *b_rabd; /* raw encrypted data */ 480 481 /* dsobj for looking up encryption key for l2arc encryption */ 482 uint64_t b_dsobj; 483 484 dmu_object_type_t b_ot; /* object type */ 485 486 /* encryption parameters */ 487 uint8_t b_salt[ZIO_DATA_SALT_LEN]; 488 uint8_t b_iv[ZIO_DATA_IV_LEN]; 489 490 /* 491 * Technically this could be removed since we will always be able to 492 * get the mac from the bp when we need it. However, it is inconvenient 493 * for callers of arc code to have to pass a bp in all the time. This 494 * also allows us to assert that L2ARC data is properly encrypted to 495 * match the data in the main storage pool. 496 */ 497 uint8_t b_mac[ZIO_DATA_MAC_LEN]; 498 } arc_buf_hdr_crypt_t; 499 500 typedef struct l2arc_buf_hdr { 501 /* protected by arc_buf_hdr mutex */ 502 l2arc_dev_t *b_dev; /* L2ARC device */ 503 uint64_t b_daddr; /* disk address, offset byte */ 504 uint32_t b_hits; 505 arc_state_type_t b_arcs_state; 506 list_node_t b_l2node; 507 } l2arc_buf_hdr_t; 508 509 typedef struct l2arc_write_callback { 510 l2arc_dev_t *l2wcb_dev; /* device info */ 511 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 512 /* in-flight list of log blocks */ 513 list_t l2wcb_abd_list; 514 } l2arc_write_callback_t; 515 516 struct arc_buf_hdr { 517 /* protected by hash lock */ 518 dva_t b_dva; 519 uint64_t b_birth; 520 521 arc_buf_contents_t b_type; 522 uint8_t b_complevel; 523 uint8_t b_reserved1; /* used for 4 byte alignment */ 524 uint16_t b_l2size; /* alignment or L2-only size */ 525 arc_buf_hdr_t *b_hash_next; 526 arc_flags_t b_flags; 527 528 /* 529 * This field stores the size of the data buffer after 530 * compression, and is set in the arc's zio completion handlers. 531 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes). 532 * 533 * While the block pointers can store up to 32MB in their psize 534 * field, we can only store up to 32MB minus 512B. This is due 535 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e. 536 * a field of zeros represents 512B in the bp). We can't use a 537 * bias of 1 since we need to reserve a psize of zero, here, to 538 * represent holes and embedded blocks. 539 * 540 * This isn't a problem in practice, since the maximum size of a 541 * buffer is limited to 16MB, so we never need to store 32MB in 542 * this field. Even in the upstream illumos code base, the 543 * maximum size of a buffer is limited to 16MB. 544 */ 545 uint16_t b_psize; 546 547 /* 548 * This field stores the size of the data buffer before 549 * compression, and cannot change once set. It is in units 550 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes) 551 */ 552 uint16_t b_lsize; /* immutable */ 553 uint64_t b_spa; /* immutable */ 554 555 /* L2ARC fields. Undefined when not in L2ARC. */ 556 l2arc_buf_hdr_t b_l2hdr; 557 /* L1ARC fields. Undefined when in l2arc_only state */ 558 l1arc_buf_hdr_t b_l1hdr; 559 /* 560 * Encryption parameters. Defined only when ARC_FLAG_ENCRYPTED 561 * is set and the L1 header exists. 562 */ 563 arc_buf_hdr_crypt_t b_crypt_hdr; 564 }; 565 566 typedef struct arc_stats { 567 /* Number of requests that were satisfied without I/O. */ 568 kstat_named_t arcstat_hits; 569 /* Number of requests for which I/O was already running. */ 570 kstat_named_t arcstat_iohits; 571 /* Number of requests for which I/O has to be issued. */ 572 kstat_named_t arcstat_misses; 573 /* Same three, but specifically for demand data. */ 574 kstat_named_t arcstat_demand_data_hits; 575 kstat_named_t arcstat_demand_data_iohits; 576 kstat_named_t arcstat_demand_data_misses; 577 /* Same three, but specifically for demand metadata. */ 578 kstat_named_t arcstat_demand_metadata_hits; 579 kstat_named_t arcstat_demand_metadata_iohits; 580 kstat_named_t arcstat_demand_metadata_misses; 581 /* Same three, but specifically for prefetch data. */ 582 kstat_named_t arcstat_prefetch_data_hits; 583 kstat_named_t arcstat_prefetch_data_iohits; 584 kstat_named_t arcstat_prefetch_data_misses; 585 /* Same three, but specifically for prefetch metadata. */ 586 kstat_named_t arcstat_prefetch_metadata_hits; 587 kstat_named_t arcstat_prefetch_metadata_iohits; 588 kstat_named_t arcstat_prefetch_metadata_misses; 589 kstat_named_t arcstat_mru_hits; 590 kstat_named_t arcstat_mru_ghost_hits; 591 kstat_named_t arcstat_mfu_hits; 592 kstat_named_t arcstat_mfu_ghost_hits; 593 kstat_named_t arcstat_uncached_hits; 594 kstat_named_t arcstat_deleted; 595 /* 596 * Number of buffers that could not be evicted because the hash lock 597 * was held by another thread. The lock may not necessarily be held 598 * by something using the same buffer, since hash locks are shared 599 * by multiple buffers. 600 */ 601 kstat_named_t arcstat_mutex_miss; 602 /* 603 * Number of buffers skipped when updating the access state due to the 604 * header having already been released after acquiring the hash lock. 605 */ 606 kstat_named_t arcstat_access_skip; 607 /* 608 * Number of buffers skipped because they have I/O in progress, are 609 * indirect prefetch buffers that have not lived long enough, or are 610 * not from the spa we're trying to evict from. 611 */ 612 kstat_named_t arcstat_evict_skip; 613 /* 614 * Number of times arc_evict_state() was unable to evict enough 615 * buffers to reach its target amount. 616 */ 617 kstat_named_t arcstat_evict_not_enough; 618 kstat_named_t arcstat_evict_l2_cached; 619 kstat_named_t arcstat_evict_l2_eligible; 620 kstat_named_t arcstat_evict_l2_eligible_mfu; 621 kstat_named_t arcstat_evict_l2_eligible_mru; 622 kstat_named_t arcstat_evict_l2_ineligible; 623 kstat_named_t arcstat_evict_l2_skip; 624 kstat_named_t arcstat_hash_elements; 625 kstat_named_t arcstat_hash_elements_max; 626 kstat_named_t arcstat_hash_collisions; 627 kstat_named_t arcstat_hash_chains; 628 kstat_named_t arcstat_hash_chain_max; 629 kstat_named_t arcstat_meta; 630 kstat_named_t arcstat_pd; 631 kstat_named_t arcstat_pm; 632 kstat_named_t arcstat_c; 633 kstat_named_t arcstat_c_min; 634 kstat_named_t arcstat_c_max; 635 kstat_named_t arcstat_size; 636 /* 637 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd. 638 * Note that the compressed bytes may match the uncompressed bytes 639 * if the block is either not compressed or compressed arc is disabled. 640 */ 641 kstat_named_t arcstat_compressed_size; 642 /* 643 * Uncompressed size of the data stored in b_pabd. If compressed 644 * arc is disabled then this value will be identical to the stat 645 * above. 646 */ 647 kstat_named_t arcstat_uncompressed_size; 648 /* 649 * Number of bytes stored in all the arc_buf_t's. This is classified 650 * as "overhead" since this data is typically short-lived and will 651 * be evicted from the arc when it becomes unreferenced unless the 652 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level 653 * values have been set (see comment in dbuf.c for more information). 654 */ 655 kstat_named_t arcstat_overhead_size; 656 /* 657 * Number of bytes consumed by internal ARC structures necessary 658 * for tracking purposes; these structures are not actually 659 * backed by ARC buffers. This includes arc_buf_hdr_t structures 660 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only 661 * caches), and arc_buf_t structures (allocated via arc_buf_t 662 * cache). 663 */ 664 kstat_named_t arcstat_hdr_size; 665 /* 666 * Number of bytes consumed by ARC buffers of type equal to 667 * ARC_BUFC_DATA. This is generally consumed by buffers backing 668 * on disk user data (e.g. plain file contents). 669 */ 670 kstat_named_t arcstat_data_size; 671 /* 672 * Number of bytes consumed by ARC buffers of type equal to 673 * ARC_BUFC_METADATA. This is generally consumed by buffers 674 * backing on disk data that is used for internal ZFS 675 * structures (e.g. ZAP, dnode, indirect blocks, etc). 676 */ 677 kstat_named_t arcstat_metadata_size; 678 /* 679 * Number of bytes consumed by dmu_buf_impl_t objects. 680 */ 681 kstat_named_t arcstat_dbuf_size; 682 /* 683 * Number of bytes consumed by dnode_t objects. 684 */ 685 kstat_named_t arcstat_dnode_size; 686 /* 687 * Number of bytes consumed by bonus buffers. 688 */ 689 kstat_named_t arcstat_bonus_size; 690 #if defined(COMPAT_FREEBSD11) 691 /* 692 * Sum of the previous three counters, provided for compatibility. 693 */ 694 kstat_named_t arcstat_other_size; 695 #endif 696 697 /* 698 * Total number of bytes consumed by ARC buffers residing in the 699 * arc_anon state. This includes *all* buffers in the arc_anon 700 * state; e.g. data, metadata, evictable, and unevictable buffers 701 * are all included in this value. 702 */ 703 kstat_named_t arcstat_anon_size; 704 kstat_named_t arcstat_anon_data; 705 kstat_named_t arcstat_anon_metadata; 706 /* 707 * Number of bytes consumed by ARC buffers that meet the 708 * following criteria: backing buffers of type ARC_BUFC_DATA, 709 * residing in the arc_anon state, and are eligible for eviction 710 * (e.g. have no outstanding holds on the buffer). 711 */ 712 kstat_named_t arcstat_anon_evictable_data; 713 /* 714 * Number of bytes consumed by ARC buffers that meet the 715 * following criteria: backing buffers of type ARC_BUFC_METADATA, 716 * residing in the arc_anon state, and are eligible for eviction 717 * (e.g. have no outstanding holds on the buffer). 718 */ 719 kstat_named_t arcstat_anon_evictable_metadata; 720 /* 721 * Total number of bytes consumed by ARC buffers residing in the 722 * arc_mru state. This includes *all* buffers in the arc_mru 723 * state; e.g. data, metadata, evictable, and unevictable buffers 724 * are all included in this value. 725 */ 726 kstat_named_t arcstat_mru_size; 727 kstat_named_t arcstat_mru_data; 728 kstat_named_t arcstat_mru_metadata; 729 /* 730 * Number of bytes consumed by ARC buffers that meet the 731 * following criteria: backing buffers of type ARC_BUFC_DATA, 732 * residing in the arc_mru state, and are eligible for eviction 733 * (e.g. have no outstanding holds on the buffer). 734 */ 735 kstat_named_t arcstat_mru_evictable_data; 736 /* 737 * Number of bytes consumed by ARC buffers that meet the 738 * following criteria: backing buffers of type ARC_BUFC_METADATA, 739 * residing in the arc_mru state, and are eligible for eviction 740 * (e.g. have no outstanding holds on the buffer). 741 */ 742 kstat_named_t arcstat_mru_evictable_metadata; 743 /* 744 * Total number of bytes that *would have been* consumed by ARC 745 * buffers in the arc_mru_ghost state. The key thing to note 746 * here, is the fact that this size doesn't actually indicate 747 * RAM consumption. The ghost lists only consist of headers and 748 * don't actually have ARC buffers linked off of these headers. 749 * Thus, *if* the headers had associated ARC buffers, these 750 * buffers *would have* consumed this number of bytes. 751 */ 752 kstat_named_t arcstat_mru_ghost_size; 753 kstat_named_t arcstat_mru_ghost_data; 754 kstat_named_t arcstat_mru_ghost_metadata; 755 /* 756 * Number of bytes that *would have been* consumed by ARC 757 * buffers that are eligible for eviction, of type 758 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state. 759 */ 760 kstat_named_t arcstat_mru_ghost_evictable_data; 761 /* 762 * Number of bytes that *would have been* consumed by ARC 763 * buffers that are eligible for eviction, of type 764 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 765 */ 766 kstat_named_t arcstat_mru_ghost_evictable_metadata; 767 /* 768 * Total number of bytes consumed by ARC buffers residing in the 769 * arc_mfu state. This includes *all* buffers in the arc_mfu 770 * state; e.g. data, metadata, evictable, and unevictable buffers 771 * are all included in this value. 772 */ 773 kstat_named_t arcstat_mfu_size; 774 kstat_named_t arcstat_mfu_data; 775 kstat_named_t arcstat_mfu_metadata; 776 /* 777 * Number of bytes consumed by ARC buffers that are eligible for 778 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu 779 * state. 780 */ 781 kstat_named_t arcstat_mfu_evictable_data; 782 /* 783 * Number of bytes consumed by ARC buffers that are eligible for 784 * eviction, of type ARC_BUFC_METADATA, and reside in the 785 * arc_mfu state. 786 */ 787 kstat_named_t arcstat_mfu_evictable_metadata; 788 /* 789 * Total number of bytes that *would have been* consumed by ARC 790 * buffers in the arc_mfu_ghost state. See the comment above 791 * arcstat_mru_ghost_size for more details. 792 */ 793 kstat_named_t arcstat_mfu_ghost_size; 794 kstat_named_t arcstat_mfu_ghost_data; 795 kstat_named_t arcstat_mfu_ghost_metadata; 796 /* 797 * Number of bytes that *would have been* consumed by ARC 798 * buffers that are eligible for eviction, of type 799 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state. 800 */ 801 kstat_named_t arcstat_mfu_ghost_evictable_data; 802 /* 803 * Number of bytes that *would have been* consumed by ARC 804 * buffers that are eligible for eviction, of type 805 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 806 */ 807 kstat_named_t arcstat_mfu_ghost_evictable_metadata; 808 /* 809 * Total number of bytes that are going to be evicted from ARC due to 810 * ARC_FLAG_UNCACHED being set. 811 */ 812 kstat_named_t arcstat_uncached_size; 813 kstat_named_t arcstat_uncached_data; 814 kstat_named_t arcstat_uncached_metadata; 815 /* 816 * Number of data bytes that are going to be evicted from ARC due to 817 * ARC_FLAG_UNCACHED being set. 818 */ 819 kstat_named_t arcstat_uncached_evictable_data; 820 /* 821 * Number of metadata bytes that that are going to be evicted from ARC 822 * due to ARC_FLAG_UNCACHED being set. 823 */ 824 kstat_named_t arcstat_uncached_evictable_metadata; 825 /* Number of L2ARC devices currently attached across all pools. */ 826 kstat_named_t arcstat_l2_ndev; 827 kstat_named_t arcstat_l2_hits; 828 kstat_named_t arcstat_l2_misses; 829 /* 830 * Allocated size (in bytes) of L2ARC cached buffers by ARC state. 831 */ 832 kstat_named_t arcstat_l2_prefetch_asize; 833 kstat_named_t arcstat_l2_mru_asize; 834 kstat_named_t arcstat_l2_mfu_asize; 835 /* 836 * Allocated size (in bytes) of L2ARC cached buffers by buffer content 837 * type. 838 */ 839 kstat_named_t arcstat_l2_bufc_data_asize; 840 kstat_named_t arcstat_l2_bufc_metadata_asize; 841 kstat_named_t arcstat_l2_feeds; 842 kstat_named_t arcstat_l2_rw_clash; 843 kstat_named_t arcstat_l2_read_bytes; 844 kstat_named_t arcstat_l2_write_bytes; 845 kstat_named_t arcstat_l2_writes_sent; 846 kstat_named_t arcstat_l2_writes_done; 847 kstat_named_t arcstat_l2_writes_error; 848 kstat_named_t arcstat_l2_writes_lock_retry; 849 kstat_named_t arcstat_l2_evict_lock_retry; 850 kstat_named_t arcstat_l2_evict_reading; 851 kstat_named_t arcstat_l2_evict_l1cached; 852 kstat_named_t arcstat_l2_free_on_write; 853 kstat_named_t arcstat_l2_abort_lowmem; 854 kstat_named_t arcstat_l2_cksum_bad; 855 kstat_named_t arcstat_l2_io_error; 856 kstat_named_t arcstat_l2_lsize; 857 kstat_named_t arcstat_l2_psize; 858 kstat_named_t arcstat_l2_hdr_size; 859 /* 860 * Number of L2ARC log blocks written. These are used for restoring the 861 * L2ARC. Updated during writing of L2ARC log blocks. 862 */ 863 kstat_named_t arcstat_l2_log_blk_writes; 864 /* 865 * Moving average of the aligned size of the L2ARC log blocks, in 866 * bytes. Updated during L2ARC rebuild and during writing of L2ARC 867 * log blocks. 868 */ 869 kstat_named_t arcstat_l2_log_blk_avg_asize; 870 /* Aligned size of L2ARC log blocks on L2ARC devices. */ 871 kstat_named_t arcstat_l2_log_blk_asize; 872 /* Number of L2ARC log blocks present on L2ARC devices. */ 873 kstat_named_t arcstat_l2_log_blk_count; 874 /* 875 * Moving average of the aligned size of L2ARC restored data, in bytes, 876 * to the aligned size of their metadata in L2ARC, in bytes. 877 * Updated during L2ARC rebuild and during writing of L2ARC log blocks. 878 */ 879 kstat_named_t arcstat_l2_data_to_meta_ratio; 880 /* 881 * Number of times the L2ARC rebuild was successful for an L2ARC device. 882 */ 883 kstat_named_t arcstat_l2_rebuild_success; 884 /* 885 * Number of times the L2ARC rebuild failed because the device header 886 * was in an unsupported format or corrupted. 887 */ 888 kstat_named_t arcstat_l2_rebuild_abort_unsupported; 889 /* 890 * Number of times the L2ARC rebuild failed because of IO errors 891 * while reading a log block. 892 */ 893 kstat_named_t arcstat_l2_rebuild_abort_io_errors; 894 /* 895 * Number of times the L2ARC rebuild failed because of IO errors when 896 * reading the device header. 897 */ 898 kstat_named_t arcstat_l2_rebuild_abort_dh_errors; 899 /* 900 * Number of L2ARC log blocks which failed to be restored due to 901 * checksum errors. 902 */ 903 kstat_named_t arcstat_l2_rebuild_abort_cksum_lb_errors; 904 /* 905 * Number of times the L2ARC rebuild was aborted due to low system 906 * memory. 907 */ 908 kstat_named_t arcstat_l2_rebuild_abort_lowmem; 909 /* Logical size of L2ARC restored data, in bytes. */ 910 kstat_named_t arcstat_l2_rebuild_size; 911 /* Aligned size of L2ARC restored data, in bytes. */ 912 kstat_named_t arcstat_l2_rebuild_asize; 913 /* 914 * Number of L2ARC log entries (buffers) that were successfully 915 * restored in ARC. 916 */ 917 kstat_named_t arcstat_l2_rebuild_bufs; 918 /* 919 * Number of L2ARC log entries (buffers) already cached in ARC. These 920 * were not restored again. 921 */ 922 kstat_named_t arcstat_l2_rebuild_bufs_precached; 923 /* 924 * Number of L2ARC log blocks that were restored successfully. Each 925 * log block may hold up to L2ARC_LOG_BLK_MAX_ENTRIES buffers. 926 */ 927 kstat_named_t arcstat_l2_rebuild_log_blks; 928 kstat_named_t arcstat_memory_throttle_count; 929 kstat_named_t arcstat_memory_direct_count; 930 kstat_named_t arcstat_memory_indirect_count; 931 kstat_named_t arcstat_memory_all_bytes; 932 kstat_named_t arcstat_memory_free_bytes; 933 kstat_named_t arcstat_memory_available_bytes; 934 kstat_named_t arcstat_no_grow; 935 kstat_named_t arcstat_tempreserve; 936 kstat_named_t arcstat_loaned_bytes; 937 kstat_named_t arcstat_prune; 938 kstat_named_t arcstat_meta_used; 939 kstat_named_t arcstat_dnode_limit; 940 kstat_named_t arcstat_async_upgrade_sync; 941 /* Number of predictive prefetch requests. */ 942 kstat_named_t arcstat_predictive_prefetch; 943 /* Number of requests for which predictive prefetch has completed. */ 944 kstat_named_t arcstat_demand_hit_predictive_prefetch; 945 /* Number of requests for which predictive prefetch was running. */ 946 kstat_named_t arcstat_demand_iohit_predictive_prefetch; 947 /* Number of prescient prefetch requests. */ 948 kstat_named_t arcstat_prescient_prefetch; 949 /* Number of requests for which prescient prefetch has completed. */ 950 kstat_named_t arcstat_demand_hit_prescient_prefetch; 951 /* Number of requests for which prescient prefetch was running. */ 952 kstat_named_t arcstat_demand_iohit_prescient_prefetch; 953 kstat_named_t arcstat_need_free; 954 kstat_named_t arcstat_sys_free; 955 kstat_named_t arcstat_raw_size; 956 kstat_named_t arcstat_cached_only_in_progress; 957 kstat_named_t arcstat_abd_chunk_waste_size; 958 } arc_stats_t; 959 960 typedef struct arc_sums { 961 wmsum_t arcstat_hits; 962 wmsum_t arcstat_iohits; 963 wmsum_t arcstat_misses; 964 wmsum_t arcstat_demand_data_hits; 965 wmsum_t arcstat_demand_data_iohits; 966 wmsum_t arcstat_demand_data_misses; 967 wmsum_t arcstat_demand_metadata_hits; 968 wmsum_t arcstat_demand_metadata_iohits; 969 wmsum_t arcstat_demand_metadata_misses; 970 wmsum_t arcstat_prefetch_data_hits; 971 wmsum_t arcstat_prefetch_data_iohits; 972 wmsum_t arcstat_prefetch_data_misses; 973 wmsum_t arcstat_prefetch_metadata_hits; 974 wmsum_t arcstat_prefetch_metadata_iohits; 975 wmsum_t arcstat_prefetch_metadata_misses; 976 wmsum_t arcstat_mru_hits; 977 wmsum_t arcstat_mru_ghost_hits; 978 wmsum_t arcstat_mfu_hits; 979 wmsum_t arcstat_mfu_ghost_hits; 980 wmsum_t arcstat_uncached_hits; 981 wmsum_t arcstat_deleted; 982 wmsum_t arcstat_mutex_miss; 983 wmsum_t arcstat_access_skip; 984 wmsum_t arcstat_evict_skip; 985 wmsum_t arcstat_evict_not_enough; 986 wmsum_t arcstat_evict_l2_cached; 987 wmsum_t arcstat_evict_l2_eligible; 988 wmsum_t arcstat_evict_l2_eligible_mfu; 989 wmsum_t arcstat_evict_l2_eligible_mru; 990 wmsum_t arcstat_evict_l2_ineligible; 991 wmsum_t arcstat_evict_l2_skip; 992 wmsum_t arcstat_hash_elements; 993 wmsum_t arcstat_hash_collisions; 994 wmsum_t arcstat_hash_chains; 995 aggsum_t arcstat_size; 996 wmsum_t arcstat_compressed_size; 997 wmsum_t arcstat_uncompressed_size; 998 wmsum_t arcstat_overhead_size; 999 wmsum_t arcstat_hdr_size; 1000 wmsum_t arcstat_data_size; 1001 wmsum_t arcstat_metadata_size; 1002 wmsum_t arcstat_dbuf_size; 1003 aggsum_t arcstat_dnode_size; 1004 wmsum_t arcstat_bonus_size; 1005 wmsum_t arcstat_l2_hits; 1006 wmsum_t arcstat_l2_misses; 1007 wmsum_t arcstat_l2_prefetch_asize; 1008 wmsum_t arcstat_l2_mru_asize; 1009 wmsum_t arcstat_l2_mfu_asize; 1010 wmsum_t arcstat_l2_bufc_data_asize; 1011 wmsum_t arcstat_l2_bufc_metadata_asize; 1012 wmsum_t arcstat_l2_feeds; 1013 wmsum_t arcstat_l2_rw_clash; 1014 wmsum_t arcstat_l2_read_bytes; 1015 wmsum_t arcstat_l2_write_bytes; 1016 wmsum_t arcstat_l2_writes_sent; 1017 wmsum_t arcstat_l2_writes_done; 1018 wmsum_t arcstat_l2_writes_error; 1019 wmsum_t arcstat_l2_writes_lock_retry; 1020 wmsum_t arcstat_l2_evict_lock_retry; 1021 wmsum_t arcstat_l2_evict_reading; 1022 wmsum_t arcstat_l2_evict_l1cached; 1023 wmsum_t arcstat_l2_free_on_write; 1024 wmsum_t arcstat_l2_abort_lowmem; 1025 wmsum_t arcstat_l2_cksum_bad; 1026 wmsum_t arcstat_l2_io_error; 1027 wmsum_t arcstat_l2_lsize; 1028 wmsum_t arcstat_l2_psize; 1029 aggsum_t arcstat_l2_hdr_size; 1030 wmsum_t arcstat_l2_log_blk_writes; 1031 wmsum_t arcstat_l2_log_blk_asize; 1032 wmsum_t arcstat_l2_log_blk_count; 1033 wmsum_t arcstat_l2_rebuild_success; 1034 wmsum_t arcstat_l2_rebuild_abort_unsupported; 1035 wmsum_t arcstat_l2_rebuild_abort_io_errors; 1036 wmsum_t arcstat_l2_rebuild_abort_dh_errors; 1037 wmsum_t arcstat_l2_rebuild_abort_cksum_lb_errors; 1038 wmsum_t arcstat_l2_rebuild_abort_lowmem; 1039 wmsum_t arcstat_l2_rebuild_size; 1040 wmsum_t arcstat_l2_rebuild_asize; 1041 wmsum_t arcstat_l2_rebuild_bufs; 1042 wmsum_t arcstat_l2_rebuild_bufs_precached; 1043 wmsum_t arcstat_l2_rebuild_log_blks; 1044 wmsum_t arcstat_memory_throttle_count; 1045 wmsum_t arcstat_memory_direct_count; 1046 wmsum_t arcstat_memory_indirect_count; 1047 wmsum_t arcstat_prune; 1048 wmsum_t arcstat_meta_used; 1049 wmsum_t arcstat_async_upgrade_sync; 1050 wmsum_t arcstat_predictive_prefetch; 1051 wmsum_t arcstat_demand_hit_predictive_prefetch; 1052 wmsum_t arcstat_demand_iohit_predictive_prefetch; 1053 wmsum_t arcstat_prescient_prefetch; 1054 wmsum_t arcstat_demand_hit_prescient_prefetch; 1055 wmsum_t arcstat_demand_iohit_prescient_prefetch; 1056 wmsum_t arcstat_raw_size; 1057 wmsum_t arcstat_cached_only_in_progress; 1058 wmsum_t arcstat_abd_chunk_waste_size; 1059 } arc_sums_t; 1060 1061 typedef struct arc_evict_waiter { 1062 list_node_t aew_node; 1063 kcondvar_t aew_cv; 1064 uint64_t aew_count; 1065 } arc_evict_waiter_t; 1066 1067 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 1068 1069 #define ARCSTAT_INCR(stat, val) \ 1070 wmsum_add(&arc_sums.stat, (val)) 1071 1072 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 1073 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 1074 1075 #define arc_no_grow ARCSTAT(arcstat_no_grow) /* do not grow cache size */ 1076 #define arc_meta ARCSTAT(arcstat_meta) /* target frac of metadata */ 1077 #define arc_pd ARCSTAT(arcstat_pd) /* target frac of data MRU */ 1078 #define arc_pm ARCSTAT(arcstat_pm) /* target frac of meta MRU */ 1079 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 1080 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 1081 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 1082 #define arc_sys_free ARCSTAT(arcstat_sys_free) /* target system free bytes */ 1083 1084 #define arc_anon (&ARC_anon) 1085 #define arc_mru (&ARC_mru) 1086 #define arc_mru_ghost (&ARC_mru_ghost) 1087 #define arc_mfu (&ARC_mfu) 1088 #define arc_mfu_ghost (&ARC_mfu_ghost) 1089 #define arc_l2c_only (&ARC_l2c_only) 1090 #define arc_uncached (&ARC_uncached) 1091 1092 extern taskq_t *arc_prune_taskq; 1093 extern arc_stats_t arc_stats; 1094 extern arc_sums_t arc_sums; 1095 extern hrtime_t arc_growtime; 1096 extern boolean_t arc_warm; 1097 extern uint_t arc_grow_retry; 1098 extern uint_t zfs_arc_no_grow_shift; 1099 extern uint_t arc_shrink_shift; 1100 extern kmutex_t arc_prune_mtx; 1101 extern list_t arc_prune_list; 1102 extern arc_state_t ARC_mfu; 1103 extern arc_state_t ARC_mru; 1104 extern uint_t zfs_arc_pc_percent; 1105 extern uint_t arc_lotsfree_percent; 1106 extern uint64_t zfs_arc_min; 1107 extern uint64_t zfs_arc_max; 1108 extern uint64_t l2arc_dwpd_limit; 1109 1110 extern uint64_t arc_reduce_target_size(uint64_t to_free); 1111 extern boolean_t arc_reclaim_needed(void); 1112 extern void arc_kmem_reap_soon(void); 1113 extern void arc_wait_for_eviction(uint64_t, boolean_t, boolean_t); 1114 1115 extern void arc_lowmem_init(void); 1116 extern void arc_lowmem_fini(void); 1117 extern int arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg); 1118 extern uint64_t arc_free_memory(void); 1119 extern int64_t arc_available_memory(void); 1120 extern void arc_tuning_update(boolean_t); 1121 extern void arc_register_hotplug(void); 1122 extern void arc_unregister_hotplug(void); 1123 1124 extern int param_set_arc_u64(ZFS_MODULE_PARAM_ARGS); 1125 extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS); 1126 extern int param_set_arc_min(ZFS_MODULE_PARAM_ARGS); 1127 extern int param_set_arc_max(ZFS_MODULE_PARAM_ARGS); 1128 extern int param_set_l2arc_dwpd_limit(ZFS_MODULE_PARAM_ARGS); 1129 extern int param_set_arc_no_grow_shift(ZFS_MODULE_PARAM_ARGS); 1130 extern void l2arc_dwpd_bump_reset(void); 1131 1132 /* used in zdb.c */ 1133 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev, 1134 const l2arc_log_blkptr_t *lbp); 1135 1136 /* used in vdev_trim.c */ 1137 void l2arc_dev_hdr_update(l2arc_dev_t *dev); 1138 l2arc_dev_t *l2arc_vdev_get(vdev_t *vd); 1139 1140 #ifdef __cplusplus 1141 } 1142 #endif 1143 1144 #endif /* _SYS_ARC_IMPL_H */ 1145