xref: /freebsd/sys/contrib/openzfs/include/sys/arc_impl.h (revision 22649d4dba730d46244fd2dff4fd174903c8379f)
1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3  * This file and its contents are supplied under the terms of the
4  * Common Development and Distribution License ("CDDL"), version 1.0.
5  * You may only use this file in accordance with the terms of version
6  * 1.0 of the CDDL.
7  *
8  * A full copy of the text of the CDDL should have accompanied this
9  * source.  A copy of the CDDL is also available via the Internet at
10  * https://opensource.org/license/CDDL-1.0.
11  */
12 /*
13  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
14  * Copyright (c) 2013, Delphix. All rights reserved.
15  * Copyright (c) 2013, Saso Kiselkov. All rights reserved.
16  * Copyright (c) 2013, Nexenta Systems, Inc.  All rights reserved.
17  * Copyright (c) 2020, George Amanakis. All rights reserved.
18  */
19 
20 #ifndef _SYS_ARC_IMPL_H
21 #define	_SYS_ARC_IMPL_H
22 
23 #include <sys/arc.h>
24 #include <sys/multilist.h>
25 #include <sys/zio_crypt.h>
26 #include <sys/zthr.h>
27 #include <sys/aggsum.h>
28 #include <sys/wmsum.h>
29 
30 #ifdef __cplusplus
31 extern "C" {
32 #endif
33 
34 /*
35  * We can feed L2ARC from two states of ARC buffers, mru and mfu,
36  * and each of the states has two types: data and metadata.
37  */
38 #define	L2ARC_FEED_TYPES	4
39 #define	L2ARC_MFU_META		0
40 #define	L2ARC_MRU_META		1
41 #define	L2ARC_MFU_DATA		2
42 #define	L2ARC_MRU_DATA		3
43 
44 /*
45  * L2ARC state and statistics for persistent marker management.
46  */
47 typedef struct l2arc_info {
48 	arc_buf_hdr_t	**l2arc_markers[L2ARC_FEED_TYPES];
49 	uint64_t	l2arc_total_writes;	/* total writes for reset */
50 	uint64_t	l2arc_total_capacity;	/* total L2ARC capacity */
51 	uint64_t	l2arc_smallest_capacity; /* smallest device capacity */
52 	/*
53 	 * Per-device thread coordination for sublist processing.
54 	 * reset: flags sublist marker for lazy reset to tail.
55 	 */
56 	boolean_t	*l2arc_sublist_busy[L2ARC_FEED_TYPES];
57 	boolean_t	*l2arc_sublist_reset[L2ARC_FEED_TYPES];
58 	kmutex_t	l2arc_sublist_lock;	/* protects busy/reset flags */
59 	/*
60 	 * Cumulative bytes scanned per pass since marker reset.
61 	 * Limits how far persistent markers advance from tail
62 	 * before resetting, based on % of state size.
63 	 */
64 	uint64_t	l2arc_ext_scanned[L2ARC_FEED_TYPES];
65 	int		l2arc_next_sublist[L2ARC_FEED_TYPES]; /* round-robin */
66 } l2arc_info_t;
67 
68 /*
69  * Note that buffers can be in one of 6 states:
70  *	ARC_anon	- anonymous (discussed below)
71  *	ARC_mru		- recently used, currently cached
72  *	ARC_mru_ghost	- recently used, no longer in cache
73  *	ARC_mfu		- frequently used, currently cached
74  *	ARC_mfu_ghost	- frequently used, no longer in cache
75  *	ARC_uncached	- uncacheable prefetch, to be evicted
76  *	ARC_l2c_only	- exists in L2ARC but not other states
77  * When there are no active references to the buffer, they are
78  * are linked onto a list in one of these arc states.  These are
79  * the only buffers that can be evicted or deleted.  Within each
80  * state there are multiple lists, one for meta-data and one for
81  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
82  * etc.) is tracked separately so that it can be managed more
83  * explicitly: favored over data, limited explicitly.
84  *
85  * Anonymous buffers are buffers that are not associated with
86  * a DVA.  These are buffers that hold dirty block copies
87  * before they are written to stable storage.  By definition,
88  * they are "ref'd" and are considered part of arc_mru
89  * that cannot be freed.  Generally, they will acquire a DVA
90  * as they are written and migrate onto the arc_mru list.
91  *
92  * The ARC_l2c_only state is for buffers that are in the second
93  * level ARC but no longer in any of the ARC_m* lists.  The second
94  * level ARC itself may also contain buffers that are in any of
95  * the ARC_m* states - meaning that a buffer can exist in two
96  * places.  The reason for the ARC_l2c_only state is to keep the
97  * buffer header in the hash table, so that reads that hit the
98  * second level ARC benefit from these fast lookups.
99  */
100 
101 typedef struct arc_state {
102 	/*
103 	 * list of evictable buffers
104 	 */
105 	multilist_t arcs_list[ARC_BUFC_NUMTYPES];
106 	/*
107 	 * supports the "dbufs" kstat
108 	 */
109 	arc_state_type_t arcs_state;
110 	/*
111 	 * total amount of data in this state.
112 	 */
113 	zfs_refcount_t arcs_size[ARC_BUFC_NUMTYPES] ____cacheline_aligned;
114 	/*
115 	 * total amount of evictable data in this state
116 	 */
117 	zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES];
118 	/*
119 	 * amount of hit bytes for this state (counted only for ghost states)
120 	 */
121 	wmsum_t arcs_hits[ARC_BUFC_NUMTYPES];
122 } arc_state_t;
123 
124 typedef struct arc_callback arc_callback_t;
125 
126 struct arc_callback {
127 	void			*acb_private;
128 	arc_read_done_func_t	*acb_done;
129 	arc_buf_t		*acb_buf;
130 	boolean_t		acb_encrypted;
131 	boolean_t		acb_compressed;
132 	boolean_t		acb_noauth;
133 	boolean_t		acb_nobuf;
134 	boolean_t		acb_wait;
135 	int			acb_wait_error;
136 	kmutex_t		acb_wait_lock;
137 	kcondvar_t		acb_wait_cv;
138 	zbookmark_phys_t	acb_zb;
139 	zio_t			*acb_zio_dummy;
140 	zio_t			*acb_zio_head;
141 	arc_callback_t		*acb_prev;
142 	arc_callback_t		*acb_next;
143 };
144 
145 typedef struct arc_write_callback arc_write_callback_t;
146 
147 struct arc_write_callback {
148 	void			*awcb_private;
149 	arc_write_done_func_t	*awcb_ready;
150 	arc_write_done_func_t	*awcb_children_ready;
151 	arc_write_done_func_t	*awcb_done;
152 	arc_buf_t		*awcb_buf;
153 };
154 
155 /*
156  * ARC buffers are separated into multiple structs as a memory saving measure:
157  *   - Common fields struct, always defined, and embedded within it:
158  *       - L2-only fields, always allocated but undefined when not in L2ARC
159  *       - L1-only fields, only allocated when in L1ARC
160  *
161  *           Buffer in L1                     Buffer only in L2
162  *    +------------------------+          +------------------------+
163  *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
164  *    |                        |          |                        |
165  *    |                        |          |                        |
166  *    |                        |          |                        |
167  *    +------------------------+          +------------------------+
168  *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
169  *    | (undefined if L1-only) |          |                        |
170  *    +------------------------+          +------------------------+
171  *    | l1arc_buf_hdr_t        |
172  *    |                        |
173  *    |                        |
174  *    |                        |
175  *    |                        |
176  *    +------------------------+
177  *
178  * Because it's possible for the L2ARC to become extremely large, we can wind
179  * up eating a lot of memory in L2ARC buffer headers, so the size of a header
180  * is minimized by only allocating the fields necessary for an L1-cached buffer
181  * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
182  * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
183  * words in pointers. arc_hdr_realloc() is used to switch a header between
184  * these two allocation states.
185  */
186 typedef struct l1arc_buf_hdr {
187 	/* protected by arc state mutex */
188 	arc_state_t		*b_state;
189 	multilist_node_t	b_arc_node;
190 
191 	/* protected by hash lock */
192 	clock_t			b_arc_access;
193 	uint32_t		b_mru_hits;
194 	uint32_t		b_mru_ghost_hits;
195 	uint32_t		b_mfu_hits;
196 	uint32_t		b_mfu_ghost_hits;
197 	uint8_t			b_byteswap;
198 	arc_buf_t		*b_buf;
199 
200 	/* self protecting */
201 	zfs_refcount_t		b_refcnt;
202 
203 	arc_callback_t		*b_acb;
204 	abd_t			*b_pabd;
205 
206 #ifdef ZFS_DEBUG
207 	zio_cksum_t		*b_freeze_cksum;
208 	kmutex_t		b_freeze_lock;
209 #endif
210 } l1arc_buf_hdr_t;
211 
212 typedef enum l2arc_dev_hdr_flags_t {
213 	L2ARC_DEV_HDR_EVICT_FIRST = (1 << 0)	/* mirror of l2ad_first */
214 } l2arc_dev_hdr_flags_t;
215 
216 /*
217  * Pointer used in persistent L2ARC (for pointing to log blocks).
218  */
219 typedef struct l2arc_log_blkptr {
220 	/*
221 	 * Offset of log block within the device, in bytes
222 	 */
223 	uint64_t	lbp_daddr;
224 	/*
225 	 * Aligned payload size (in bytes) of the log block
226 	 */
227 	uint64_t	lbp_payload_asize;
228 	/*
229 	 * Offset in bytes of the first buffer in the payload
230 	 */
231 	uint64_t	lbp_payload_start;
232 	/*
233 	 * lbp_prop has the following format:
234 	 *	* logical size (in bytes)
235 	 *	* aligned (after compression) size (in bytes)
236 	 *	* compression algorithm (we always LZ4-compress l2arc logs)
237 	 *	* checksum algorithm (used for lbp_cksum)
238 	 */
239 	uint64_t	lbp_prop;
240 	zio_cksum_t	lbp_cksum;	/* checksum of log */
241 } l2arc_log_blkptr_t;
242 
243 /*
244  * The persistent L2ARC device header.
245  * Byte order of magic determines whether 64-bit bswap of fields is necessary.
246  */
247 typedef struct l2arc_dev_hdr_phys {
248 	uint64_t	dh_magic;	/* L2ARC_DEV_HDR_MAGIC */
249 	uint64_t	dh_version;	/* Persistent L2ARC version */
250 
251 	/*
252 	 * Global L2ARC device state and metadata.
253 	 */
254 	uint64_t	dh_spa_guid;
255 	uint64_t	dh_vdev_guid;
256 	uint64_t	dh_log_entries;		/* mirror of l2ad_log_entries */
257 	uint64_t	dh_evict;		/* evicted offset in bytes */
258 	uint64_t	dh_flags;		/* l2arc_dev_hdr_flags_t */
259 	/*
260 	 * Used in zdb.c for determining if a log block is valid, in the same
261 	 * way that l2arc_rebuild() does.
262 	 */
263 	uint64_t	dh_start;		/* mirror of l2ad_start */
264 	uint64_t	dh_end;			/* mirror of l2ad_end */
265 	/*
266 	 * Start of log block chain. [0] -> newest log, [1] -> one older (used
267 	 * for initiating prefetch).
268 	 */
269 	l2arc_log_blkptr_t	dh_start_lbps[2];
270 	/*
271 	 * Aligned size of all log blocks as accounted by vdev_space_update().
272 	 */
273 	uint64_t	dh_lb_asize;		/* mirror of l2ad_lb_asize */
274 	uint64_t	dh_lb_count;		/* mirror of l2ad_lb_count */
275 	/*
276 	 * Mirrors of vdev_trim_action_time and vdev_trim_state, used to
277 	 * display when the cache device was fully trimmed for the last
278 	 * time.
279 	 */
280 	uint64_t		dh_trim_action_time;
281 	uint64_t		dh_trim_state;
282 	const uint64_t		dh_pad[30];	/* pad to 512 bytes */
283 	zio_eck_t		dh_tail;
284 } l2arc_dev_hdr_phys_t;
285 _Static_assert(sizeof (l2arc_dev_hdr_phys_t) == SPA_MINBLOCKSIZE,
286 	"l2arc_dev_hdr_phys_t wrong size");
287 
288 /*
289  * A single ARC buffer header entry in a l2arc_log_blk_phys_t.
290  */
291 typedef struct l2arc_log_ent_phys {
292 	dva_t			le_dva;		/* dva of buffer */
293 	uint64_t		le_birth;	/* birth txg of buffer */
294 	/*
295 	 * le_prop has the following format:
296 	 *	* logical size (in bytes)
297 	 *	* physical (compressed) size (in bytes)
298 	 *	* compression algorithm
299 	 *	* object type (used to restore arc_buf_contents_t)
300 	 *	* protected status (used for encryption)
301 	 *	* prefetch status (used in l2arc_read_done())
302 	 */
303 	uint64_t		le_prop;
304 	uint64_t		le_daddr;	/* buf location on l2dev */
305 	uint64_t		le_complevel;
306 	/*
307 	 * We pad the size of each entry to a power of 2 so that the size of
308 	 * l2arc_log_blk_phys_t is power-of-2 aligned with SPA_MINBLOCKSHIFT,
309 	 * because of the L2ARC_SET_*SIZE macros.
310 	 */
311 	const uint64_t		le_pad[2];	/* pad to 64 bytes	 */
312 } l2arc_log_ent_phys_t;
313 
314 #define	L2ARC_LOG_BLK_MAX_ENTRIES	(1022)
315 
316 /*
317  * A log block of up to 1022 ARC buffer log entries, chained into the
318  * persistent L2ARC metadata linked list. Byte order of magic determines
319  * whether 64-bit bswap of fields is necessary.
320  */
321 typedef struct l2arc_log_blk_phys {
322 	uint64_t		lb_magic;	/* L2ARC_LOG_BLK_MAGIC */
323 	/*
324 	 * There are 2 chains (headed by dh_start_lbps[2]), and this field
325 	 * points back to the previous block in this chain. We alternate
326 	 * which chain we append to, so they are time-wise and offset-wise
327 	 * interleaved, but that is an optimization rather than for
328 	 * correctness.
329 	 */
330 	l2arc_log_blkptr_t	lb_prev_lbp;	/* pointer to prev log block */
331 	/*
332 	 * Pad header section to 128 bytes
333 	 */
334 	uint64_t		lb_pad[7];
335 	/* Payload */
336 	l2arc_log_ent_phys_t	lb_entries[L2ARC_LOG_BLK_MAX_ENTRIES];
337 } l2arc_log_blk_phys_t;				/* 64K total */
338 
339 /*
340  * The size of l2arc_log_blk_phys_t has to be power-of-2 aligned with
341  * SPA_MINBLOCKSHIFT because of L2BLK_SET_*SIZE macros.
342  */
343 _Static_assert(IS_P2ALIGNED(sizeof (l2arc_log_blk_phys_t),
344     1ULL << SPA_MINBLOCKSHIFT), "l2arc_log_blk_phys_t misaligned");
345 _Static_assert(sizeof (l2arc_log_blk_phys_t) >= SPA_MINBLOCKSIZE,
346 	"l2arc_log_blk_phys_t too small");
347 _Static_assert(sizeof (l2arc_log_blk_phys_t) <= SPA_MAXBLOCKSIZE,
348 	"l2arc_log_blk_phys_t too big");
349 
350 /*
351  * These structures hold in-flight abd buffers for log blocks as they're being
352  * written to the L2ARC device.
353  */
354 typedef struct l2arc_lb_abd_buf {
355 	abd_t		*abd;
356 	list_node_t	node;
357 } l2arc_lb_abd_buf_t;
358 
359 /*
360  * These structures hold pointers to log blocks present on the L2ARC device.
361  */
362 typedef struct l2arc_lb_ptr_buf {
363 	l2arc_log_blkptr_t	*lb_ptr;
364 	list_node_t		node;
365 } l2arc_lb_ptr_buf_t;
366 
367 /* Macros for setting fields in le_prop and lbp_prop */
368 #define	L2BLK_GET_LSIZE(field)	\
369 	BF64_GET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1)
370 #define	L2BLK_SET_LSIZE(field, x)	\
371 	BF64_SET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1, x)
372 #define	L2BLK_GET_PSIZE(field)	\
373 	BF64_GET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1)
374 #define	L2BLK_SET_PSIZE(field, x)	\
375 	BF64_SET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1, x)
376 #define	L2BLK_GET_COMPRESS(field)	\
377 	BF64_GET((field), 32, SPA_COMPRESSBITS)
378 #define	L2BLK_SET_COMPRESS(field, x)	\
379 	BF64_SET((field), 32, SPA_COMPRESSBITS, x)
380 #define	L2BLK_GET_PREFETCH(field)	BF64_GET((field), 39, 1)
381 #define	L2BLK_SET_PREFETCH(field, x)	BF64_SET((field), 39, 1, x)
382 #define	L2BLK_GET_CHECKSUM(field)	BF64_GET((field), 40, 8)
383 #define	L2BLK_SET_CHECKSUM(field, x)	BF64_SET((field), 40, 8, x)
384 /* +/- 1 here are to keep compatibility after ARC_BUFC_INVALID removal. */
385 #define	L2BLK_GET_TYPE(field)		(BF64_GET((field), 48, 8) - 1)
386 #define	L2BLK_SET_TYPE(field, x)	BF64_SET((field), 48, 8, (x) + 1)
387 #define	L2BLK_GET_PROTECTED(field)	BF64_GET((field), 56, 1)
388 #define	L2BLK_SET_PROTECTED(field, x)	BF64_SET((field), 56, 1, x)
389 #define	L2BLK_GET_STATE(field)		BF64_GET((field), 57, 4)
390 #define	L2BLK_SET_STATE(field, x)	BF64_SET((field), 57, 4, x)
391 
392 #define	PTR_SWAP(x, y)		\
393 	do {			\
394 		void *tmp = (x);\
395 		x = y;		\
396 		y = tmp;	\
397 	} while (0)
398 
399 #define	L2ARC_DEV_HDR_MAGIC	0x5a46534341434845LLU	/* ASCII: "ZFSCACHE" */
400 #define	L2ARC_LOG_BLK_MAGIC	0x4c4f47424c4b4844LLU	/* ASCII: "LOGBLKHD" */
401 
402 /*
403  * L2ARC Internals
404  */
405 typedef struct l2arc_dev {
406 	vdev_t			*l2ad_vdev;	/* can be NULL during remove */
407 	spa_t			*l2ad_spa;	/* can be NULL during remove */
408 	uint64_t		l2ad_hand;	/* next write location */
409 	uint64_t		l2ad_start;	/* first addr on device */
410 	uint64_t		l2ad_end;	/* last addr on device */
411 	boolean_t		l2ad_first;	/* first sweep through */
412 	boolean_t		l2ad_writing;	/* currently writing */
413 	kmutex_t		l2ad_mtx;	/* lock for buffer list */
414 	list_t			l2ad_buflist;	/* buffer list */
415 	list_node_t		l2ad_node;	/* device list node */
416 	zfs_refcount_t		l2ad_alloc;	/* allocated bytes */
417 	/*
418 	 * Persistence-related stuff
419 	 */
420 	l2arc_dev_hdr_phys_t	*l2ad_dev_hdr;	/* persistent device header */
421 	uint64_t		l2ad_dev_hdr_asize; /* aligned hdr size */
422 	l2arc_log_blk_phys_t	l2ad_log_blk;	/* currently open log block */
423 	int			l2ad_log_ent_idx; /* index into cur log blk */
424 	/* Number of bytes in current log block's payload */
425 	uint64_t		l2ad_log_blk_payload_asize;
426 	/*
427 	 * Offset (in bytes) of the first buffer in current log block's
428 	 * payload.
429 	 */
430 	uint64_t		l2ad_log_blk_payload_start;
431 	/* Flag indicating whether a rebuild is scheduled or is going on */
432 	boolean_t		l2ad_rebuild;
433 	boolean_t		l2ad_rebuild_cancel;
434 	boolean_t		l2ad_rebuild_began;
435 	uint64_t		l2ad_log_entries;   /* entries per log blk  */
436 	uint64_t		l2ad_evict;	 /* evicted offset in bytes */
437 	/* List of pointers to log blocks present in the L2ARC device */
438 	list_t			l2ad_lbptr_list;
439 	/*
440 	 * Aligned size of all log blocks as accounted by vdev_space_update().
441 	 */
442 	zfs_refcount_t		l2ad_lb_asize;
443 	/*
444 	 * Number of log blocks present on the device.
445 	 */
446 	zfs_refcount_t		l2ad_lb_count;
447 	boolean_t		l2ad_trim_all; /* TRIM whole device */
448 	/*
449 	 * DWPD tracking with daily reset
450 	 */
451 	uint64_t		l2ad_dwpd_writes;	/* 24h bytes written */
452 	uint64_t		l2ad_dwpd_start;	/* 24h period start */
453 	uint64_t		l2ad_dwpd_accumulated;	/* Accumulated */
454 	uint64_t		l2ad_dwpd_bump;		/* Reset trigger */
455 	/*
456 	 * Per-device feed thread for parallel L2ARC writes
457 	 */
458 	kthread_t		*l2ad_feed_thread;	/* feed thread handle */
459 	boolean_t		l2ad_thread_exit;	/* signal thread exit */
460 	kmutex_t		l2ad_feed_thr_lock;	/* thread sleep/wake */
461 	kcondvar_t		l2ad_feed_cv;		/* thread wakeup cv */
462 	/*
463 	 * Consecutive cycles where metadata filled write budget
464 	 * while data passes got nothing written. Used to detect
465 	 * monopolization and skip metadata to give data a chance.
466 	 */
467 	uint64_t		l2ad_meta_cycles;
468 } l2arc_dev_t;
469 
470 /*
471  * Encrypted blocks will need to be stored encrypted on the L2ARC
472  * disk as they appear in the main pool. In order for this to work we
473  * need to pass around the encryption parameters so they can be used
474  * to write data to the L2ARC. This struct is only defined in the
475  * arc_buf_hdr_t if the L1 header is defined and has the ARC_FLAG_ENCRYPTED
476  * flag set.
477  */
478 typedef struct arc_buf_hdr_crypt {
479 	abd_t			*b_rabd;	/* raw encrypted data */
480 
481 	/* dsobj for looking up encryption key for l2arc encryption */
482 	uint64_t		b_dsobj;
483 
484 	dmu_object_type_t	b_ot;		/* object type */
485 
486 	/* encryption parameters */
487 	uint8_t			b_salt[ZIO_DATA_SALT_LEN];
488 	uint8_t			b_iv[ZIO_DATA_IV_LEN];
489 
490 	/*
491 	 * Technically this could be removed since we will always be able to
492 	 * get the mac from the bp when we need it. However, it is inconvenient
493 	 * for callers of arc code to have to pass a bp in all the time. This
494 	 * also allows us to assert that L2ARC data is properly encrypted to
495 	 * match the data in the main storage pool.
496 	 */
497 	uint8_t			b_mac[ZIO_DATA_MAC_LEN];
498 } arc_buf_hdr_crypt_t;
499 
500 typedef struct l2arc_buf_hdr {
501 	/* protected by arc_buf_hdr mutex */
502 	l2arc_dev_t		*b_dev;		/* L2ARC device */
503 	uint64_t		b_daddr;	/* disk address, offset byte */
504 	uint32_t		b_hits;
505 	arc_state_type_t	b_arcs_state;
506 	list_node_t		b_l2node;
507 } l2arc_buf_hdr_t;
508 
509 typedef struct l2arc_write_callback {
510 	l2arc_dev_t	*l2wcb_dev;		/* device info */
511 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
512 	/* in-flight list of log blocks */
513 	list_t		l2wcb_abd_list;
514 } l2arc_write_callback_t;
515 
516 struct arc_buf_hdr {
517 	/* protected by hash lock */
518 	dva_t			b_dva;
519 	uint64_t		b_birth;
520 
521 	arc_buf_contents_t	b_type;
522 	uint8_t			b_complevel;
523 	uint8_t			b_reserved1;	/* used for 4 byte alignment */
524 	uint16_t		b_l2size;	/* alignment or L2-only size */
525 	arc_buf_hdr_t		*b_hash_next;
526 	arc_flags_t		b_flags;
527 
528 	/*
529 	 * This field stores the size of the data buffer after
530 	 * compression, and is set in the arc's zio completion handlers.
531 	 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes).
532 	 *
533 	 * While the block pointers can store up to 32MB in their psize
534 	 * field, we can only store up to 32MB minus 512B. This is due
535 	 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e.
536 	 * a field of zeros represents 512B in the bp). We can't use a
537 	 * bias of 1 since we need to reserve a psize of zero, here, to
538 	 * represent holes and embedded blocks.
539 	 *
540 	 * This isn't a problem in practice, since the maximum size of a
541 	 * buffer is limited to 16MB, so we never need to store 32MB in
542 	 * this field. Even in the upstream illumos code base, the
543 	 * maximum size of a buffer is limited to 16MB.
544 	 */
545 	uint16_t		b_psize;
546 
547 	/*
548 	 * This field stores the size of the data buffer before
549 	 * compression, and cannot change once set. It is in units
550 	 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes)
551 	 */
552 	uint16_t		b_lsize;	/* immutable */
553 	uint64_t		b_spa;		/* immutable */
554 
555 	/* L2ARC fields. Undefined when not in L2ARC. */
556 	l2arc_buf_hdr_t		b_l2hdr;
557 	/* L1ARC fields. Undefined when in l2arc_only state */
558 	l1arc_buf_hdr_t		b_l1hdr;
559 	/*
560 	 * Encryption parameters. Defined only when ARC_FLAG_ENCRYPTED
561 	 * is set and the L1 header exists.
562 	 */
563 	arc_buf_hdr_crypt_t b_crypt_hdr;
564 };
565 
566 typedef struct arc_stats {
567 	/* Number of requests that were satisfied without I/O. */
568 	kstat_named_t arcstat_hits;
569 	/* Number of requests for which I/O was already running. */
570 	kstat_named_t arcstat_iohits;
571 	/* Number of requests for which I/O has to be issued. */
572 	kstat_named_t arcstat_misses;
573 	/* Same three, but specifically for demand data. */
574 	kstat_named_t arcstat_demand_data_hits;
575 	kstat_named_t arcstat_demand_data_iohits;
576 	kstat_named_t arcstat_demand_data_misses;
577 	/* Same three, but specifically for demand metadata. */
578 	kstat_named_t arcstat_demand_metadata_hits;
579 	kstat_named_t arcstat_demand_metadata_iohits;
580 	kstat_named_t arcstat_demand_metadata_misses;
581 	/* Same three, but specifically for prefetch data. */
582 	kstat_named_t arcstat_prefetch_data_hits;
583 	kstat_named_t arcstat_prefetch_data_iohits;
584 	kstat_named_t arcstat_prefetch_data_misses;
585 	/* Same three, but specifically for prefetch metadata. */
586 	kstat_named_t arcstat_prefetch_metadata_hits;
587 	kstat_named_t arcstat_prefetch_metadata_iohits;
588 	kstat_named_t arcstat_prefetch_metadata_misses;
589 	kstat_named_t arcstat_mru_hits;
590 	kstat_named_t arcstat_mru_ghost_hits;
591 	kstat_named_t arcstat_mfu_hits;
592 	kstat_named_t arcstat_mfu_ghost_hits;
593 	kstat_named_t arcstat_uncached_hits;
594 	kstat_named_t arcstat_deleted;
595 	/*
596 	 * Number of buffers that could not be evicted because the hash lock
597 	 * was held by another thread.  The lock may not necessarily be held
598 	 * by something using the same buffer, since hash locks are shared
599 	 * by multiple buffers.
600 	 */
601 	kstat_named_t arcstat_mutex_miss;
602 	/*
603 	 * Number of buffers skipped when updating the access state due to the
604 	 * header having already been released after acquiring the hash lock.
605 	 */
606 	kstat_named_t arcstat_access_skip;
607 	/*
608 	 * Number of buffers skipped because they have I/O in progress, are
609 	 * indirect prefetch buffers that have not lived long enough, or are
610 	 * not from the spa we're trying to evict from.
611 	 */
612 	kstat_named_t arcstat_evict_skip;
613 	/*
614 	 * Number of times arc_evict_state() was unable to evict enough
615 	 * buffers to reach its target amount.
616 	 */
617 	kstat_named_t arcstat_evict_not_enough;
618 	kstat_named_t arcstat_evict_l2_cached;
619 	kstat_named_t arcstat_evict_l2_eligible;
620 	kstat_named_t arcstat_evict_l2_eligible_mfu;
621 	kstat_named_t arcstat_evict_l2_eligible_mru;
622 	kstat_named_t arcstat_evict_l2_ineligible;
623 	kstat_named_t arcstat_evict_l2_skip;
624 	kstat_named_t arcstat_hash_elements;
625 	kstat_named_t arcstat_hash_elements_max;
626 	kstat_named_t arcstat_hash_collisions;
627 	kstat_named_t arcstat_hash_chains;
628 	kstat_named_t arcstat_hash_chain_max;
629 	kstat_named_t arcstat_meta;
630 	kstat_named_t arcstat_pd;
631 	kstat_named_t arcstat_pm;
632 	kstat_named_t arcstat_c;
633 	kstat_named_t arcstat_c_min;
634 	kstat_named_t arcstat_c_max;
635 	kstat_named_t arcstat_size;
636 	/*
637 	 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd.
638 	 * Note that the compressed bytes may match the uncompressed bytes
639 	 * if the block is either not compressed or compressed arc is disabled.
640 	 */
641 	kstat_named_t arcstat_compressed_size;
642 	/*
643 	 * Uncompressed size of the data stored in b_pabd. If compressed
644 	 * arc is disabled then this value will be identical to the stat
645 	 * above.
646 	 */
647 	kstat_named_t arcstat_uncompressed_size;
648 	/*
649 	 * Number of bytes stored in all the arc_buf_t's. This is classified
650 	 * as "overhead" since this data is typically short-lived and will
651 	 * be evicted from the arc when it becomes unreferenced unless the
652 	 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level
653 	 * values have been set (see comment in dbuf.c for more information).
654 	 */
655 	kstat_named_t arcstat_overhead_size;
656 	/*
657 	 * Number of bytes consumed by internal ARC structures necessary
658 	 * for tracking purposes; these structures are not actually
659 	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
660 	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
661 	 * caches), and arc_buf_t structures (allocated via arc_buf_t
662 	 * cache).
663 	 */
664 	kstat_named_t arcstat_hdr_size;
665 	/*
666 	 * Number of bytes consumed by ARC buffers of type equal to
667 	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
668 	 * on disk user data (e.g. plain file contents).
669 	 */
670 	kstat_named_t arcstat_data_size;
671 	/*
672 	 * Number of bytes consumed by ARC buffers of type equal to
673 	 * ARC_BUFC_METADATA. This is generally consumed by buffers
674 	 * backing on disk data that is used for internal ZFS
675 	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
676 	 */
677 	kstat_named_t arcstat_metadata_size;
678 	/*
679 	 * Number of bytes consumed by dmu_buf_impl_t objects.
680 	 */
681 	kstat_named_t arcstat_dbuf_size;
682 	/*
683 	 * Number of bytes consumed by dnode_t objects.
684 	 */
685 	kstat_named_t arcstat_dnode_size;
686 	/*
687 	 * Number of bytes consumed by bonus buffers.
688 	 */
689 	kstat_named_t arcstat_bonus_size;
690 #if defined(COMPAT_FREEBSD11)
691 	/*
692 	 * Sum of the previous three counters, provided for compatibility.
693 	 */
694 	kstat_named_t arcstat_other_size;
695 #endif
696 
697 	/*
698 	 * Total number of bytes consumed by ARC buffers residing in the
699 	 * arc_anon state. This includes *all* buffers in the arc_anon
700 	 * state; e.g. data, metadata, evictable, and unevictable buffers
701 	 * are all included in this value.
702 	 */
703 	kstat_named_t arcstat_anon_size;
704 	kstat_named_t arcstat_anon_data;
705 	kstat_named_t arcstat_anon_metadata;
706 	/*
707 	 * Number of bytes consumed by ARC buffers that meet the
708 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
709 	 * residing in the arc_anon state, and are eligible for eviction
710 	 * (e.g. have no outstanding holds on the buffer).
711 	 */
712 	kstat_named_t arcstat_anon_evictable_data;
713 	/*
714 	 * Number of bytes consumed by ARC buffers that meet the
715 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
716 	 * residing in the arc_anon state, and are eligible for eviction
717 	 * (e.g. have no outstanding holds on the buffer).
718 	 */
719 	kstat_named_t arcstat_anon_evictable_metadata;
720 	/*
721 	 * Total number of bytes consumed by ARC buffers residing in the
722 	 * arc_mru state. This includes *all* buffers in the arc_mru
723 	 * state; e.g. data, metadata, evictable, and unevictable buffers
724 	 * are all included in this value.
725 	 */
726 	kstat_named_t arcstat_mru_size;
727 	kstat_named_t arcstat_mru_data;
728 	kstat_named_t arcstat_mru_metadata;
729 	/*
730 	 * Number of bytes consumed by ARC buffers that meet the
731 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
732 	 * residing in the arc_mru state, and are eligible for eviction
733 	 * (e.g. have no outstanding holds on the buffer).
734 	 */
735 	kstat_named_t arcstat_mru_evictable_data;
736 	/*
737 	 * Number of bytes consumed by ARC buffers that meet the
738 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
739 	 * residing in the arc_mru state, and are eligible for eviction
740 	 * (e.g. have no outstanding holds on the buffer).
741 	 */
742 	kstat_named_t arcstat_mru_evictable_metadata;
743 	/*
744 	 * Total number of bytes that *would have been* consumed by ARC
745 	 * buffers in the arc_mru_ghost state. The key thing to note
746 	 * here, is the fact that this size doesn't actually indicate
747 	 * RAM consumption. The ghost lists only consist of headers and
748 	 * don't actually have ARC buffers linked off of these headers.
749 	 * Thus, *if* the headers had associated ARC buffers, these
750 	 * buffers *would have* consumed this number of bytes.
751 	 */
752 	kstat_named_t arcstat_mru_ghost_size;
753 	kstat_named_t arcstat_mru_ghost_data;
754 	kstat_named_t arcstat_mru_ghost_metadata;
755 	/*
756 	 * Number of bytes that *would have been* consumed by ARC
757 	 * buffers that are eligible for eviction, of type
758 	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
759 	 */
760 	kstat_named_t arcstat_mru_ghost_evictable_data;
761 	/*
762 	 * Number of bytes that *would have been* consumed by ARC
763 	 * buffers that are eligible for eviction, of type
764 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
765 	 */
766 	kstat_named_t arcstat_mru_ghost_evictable_metadata;
767 	/*
768 	 * Total number of bytes consumed by ARC buffers residing in the
769 	 * arc_mfu state. This includes *all* buffers in the arc_mfu
770 	 * state; e.g. data, metadata, evictable, and unevictable buffers
771 	 * are all included in this value.
772 	 */
773 	kstat_named_t arcstat_mfu_size;
774 	kstat_named_t arcstat_mfu_data;
775 	kstat_named_t arcstat_mfu_metadata;
776 	/*
777 	 * Number of bytes consumed by ARC buffers that are eligible for
778 	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
779 	 * state.
780 	 */
781 	kstat_named_t arcstat_mfu_evictable_data;
782 	/*
783 	 * Number of bytes consumed by ARC buffers that are eligible for
784 	 * eviction, of type ARC_BUFC_METADATA, and reside in the
785 	 * arc_mfu state.
786 	 */
787 	kstat_named_t arcstat_mfu_evictable_metadata;
788 	/*
789 	 * Total number of bytes that *would have been* consumed by ARC
790 	 * buffers in the arc_mfu_ghost state. See the comment above
791 	 * arcstat_mru_ghost_size for more details.
792 	 */
793 	kstat_named_t arcstat_mfu_ghost_size;
794 	kstat_named_t arcstat_mfu_ghost_data;
795 	kstat_named_t arcstat_mfu_ghost_metadata;
796 	/*
797 	 * Number of bytes that *would have been* consumed by ARC
798 	 * buffers that are eligible for eviction, of type
799 	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
800 	 */
801 	kstat_named_t arcstat_mfu_ghost_evictable_data;
802 	/*
803 	 * Number of bytes that *would have been* consumed by ARC
804 	 * buffers that are eligible for eviction, of type
805 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
806 	 */
807 	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
808 	/*
809 	 * Total number of bytes that are going to be evicted from ARC due to
810 	 * ARC_FLAG_UNCACHED being set.
811 	 */
812 	kstat_named_t arcstat_uncached_size;
813 	kstat_named_t arcstat_uncached_data;
814 	kstat_named_t arcstat_uncached_metadata;
815 	/*
816 	 * Number of data bytes that are going to be evicted from ARC due to
817 	 * ARC_FLAG_UNCACHED being set.
818 	 */
819 	kstat_named_t arcstat_uncached_evictable_data;
820 	/*
821 	 * Number of metadata bytes that that are going to be evicted from ARC
822 	 * due to ARC_FLAG_UNCACHED being set.
823 	 */
824 	kstat_named_t arcstat_uncached_evictable_metadata;
825 	/* Number of L2ARC devices currently attached across all pools. */
826 	kstat_named_t arcstat_l2_ndev;
827 	kstat_named_t arcstat_l2_hits;
828 	kstat_named_t arcstat_l2_misses;
829 	/*
830 	 * Allocated size (in bytes) of L2ARC cached buffers by ARC state.
831 	 */
832 	kstat_named_t arcstat_l2_prefetch_asize;
833 	kstat_named_t arcstat_l2_mru_asize;
834 	kstat_named_t arcstat_l2_mfu_asize;
835 	/*
836 	 * Allocated size (in bytes) of L2ARC cached buffers by buffer content
837 	 * type.
838 	 */
839 	kstat_named_t arcstat_l2_bufc_data_asize;
840 	kstat_named_t arcstat_l2_bufc_metadata_asize;
841 	kstat_named_t arcstat_l2_feeds;
842 	kstat_named_t arcstat_l2_rw_clash;
843 	kstat_named_t arcstat_l2_read_bytes;
844 	kstat_named_t arcstat_l2_write_bytes;
845 	kstat_named_t arcstat_l2_writes_sent;
846 	kstat_named_t arcstat_l2_writes_done;
847 	kstat_named_t arcstat_l2_writes_error;
848 	kstat_named_t arcstat_l2_writes_lock_retry;
849 	kstat_named_t arcstat_l2_evict_lock_retry;
850 	kstat_named_t arcstat_l2_evict_reading;
851 	kstat_named_t arcstat_l2_evict_l1cached;
852 	kstat_named_t arcstat_l2_free_on_write;
853 	kstat_named_t arcstat_l2_abort_lowmem;
854 	kstat_named_t arcstat_l2_cksum_bad;
855 	kstat_named_t arcstat_l2_io_error;
856 	kstat_named_t arcstat_l2_lsize;
857 	kstat_named_t arcstat_l2_psize;
858 	kstat_named_t arcstat_l2_hdr_size;
859 	/*
860 	 * Number of L2ARC log blocks written. These are used for restoring the
861 	 * L2ARC. Updated during writing of L2ARC log blocks.
862 	 */
863 	kstat_named_t arcstat_l2_log_blk_writes;
864 	/*
865 	 * Moving average of the aligned size of the L2ARC log blocks, in
866 	 * bytes. Updated during L2ARC rebuild and during writing of L2ARC
867 	 * log blocks.
868 	 */
869 	kstat_named_t arcstat_l2_log_blk_avg_asize;
870 	/* Aligned size of L2ARC log blocks on L2ARC devices. */
871 	kstat_named_t arcstat_l2_log_blk_asize;
872 	/* Number of L2ARC log blocks present on L2ARC devices. */
873 	kstat_named_t arcstat_l2_log_blk_count;
874 	/*
875 	 * Moving average of the aligned size of L2ARC restored data, in bytes,
876 	 * to the aligned size of their metadata in L2ARC, in bytes.
877 	 * Updated during L2ARC rebuild and during writing of L2ARC log blocks.
878 	 */
879 	kstat_named_t arcstat_l2_data_to_meta_ratio;
880 	/*
881 	 * Number of times the L2ARC rebuild was successful for an L2ARC device.
882 	 */
883 	kstat_named_t arcstat_l2_rebuild_success;
884 	/*
885 	 * Number of times the L2ARC rebuild failed because the device header
886 	 * was in an unsupported format or corrupted.
887 	 */
888 	kstat_named_t arcstat_l2_rebuild_abort_unsupported;
889 	/*
890 	 * Number of times the L2ARC rebuild failed because of IO errors
891 	 * while reading a log block.
892 	 */
893 	kstat_named_t arcstat_l2_rebuild_abort_io_errors;
894 	/*
895 	 * Number of times the L2ARC rebuild failed because of IO errors when
896 	 * reading the device header.
897 	 */
898 	kstat_named_t arcstat_l2_rebuild_abort_dh_errors;
899 	/*
900 	 * Number of L2ARC log blocks which failed to be restored due to
901 	 * checksum errors.
902 	 */
903 	kstat_named_t arcstat_l2_rebuild_abort_cksum_lb_errors;
904 	/*
905 	 * Number of times the L2ARC rebuild was aborted due to low system
906 	 * memory.
907 	 */
908 	kstat_named_t arcstat_l2_rebuild_abort_lowmem;
909 	/* Logical size of L2ARC restored data, in bytes. */
910 	kstat_named_t arcstat_l2_rebuild_size;
911 	/* Aligned size of L2ARC restored data, in bytes. */
912 	kstat_named_t arcstat_l2_rebuild_asize;
913 	/*
914 	 * Number of L2ARC log entries (buffers) that were successfully
915 	 * restored in ARC.
916 	 */
917 	kstat_named_t arcstat_l2_rebuild_bufs;
918 	/*
919 	 * Number of L2ARC log entries (buffers) already cached in ARC. These
920 	 * were not restored again.
921 	 */
922 	kstat_named_t arcstat_l2_rebuild_bufs_precached;
923 	/*
924 	 * Number of L2ARC log blocks that were restored successfully. Each
925 	 * log block may hold up to L2ARC_LOG_BLK_MAX_ENTRIES buffers.
926 	 */
927 	kstat_named_t arcstat_l2_rebuild_log_blks;
928 	kstat_named_t arcstat_memory_throttle_count;
929 	kstat_named_t arcstat_memory_direct_count;
930 	kstat_named_t arcstat_memory_indirect_count;
931 	kstat_named_t arcstat_memory_all_bytes;
932 	kstat_named_t arcstat_memory_free_bytes;
933 	kstat_named_t arcstat_memory_available_bytes;
934 	kstat_named_t arcstat_no_grow;
935 	kstat_named_t arcstat_tempreserve;
936 	kstat_named_t arcstat_loaned_bytes;
937 	kstat_named_t arcstat_prune;
938 	kstat_named_t arcstat_meta_used;
939 	kstat_named_t arcstat_dnode_limit;
940 	kstat_named_t arcstat_async_upgrade_sync;
941 	/* Number of predictive prefetch requests. */
942 	kstat_named_t arcstat_predictive_prefetch;
943 	/* Number of requests for which predictive prefetch has completed. */
944 	kstat_named_t arcstat_demand_hit_predictive_prefetch;
945 	/* Number of requests for which predictive prefetch was running. */
946 	kstat_named_t arcstat_demand_iohit_predictive_prefetch;
947 	/* Number of prescient prefetch requests. */
948 	kstat_named_t arcstat_prescient_prefetch;
949 	/* Number of requests for which prescient prefetch has completed. */
950 	kstat_named_t arcstat_demand_hit_prescient_prefetch;
951 	/* Number of requests for which prescient prefetch was running. */
952 	kstat_named_t arcstat_demand_iohit_prescient_prefetch;
953 	kstat_named_t arcstat_need_free;
954 	kstat_named_t arcstat_sys_free;
955 	kstat_named_t arcstat_raw_size;
956 	kstat_named_t arcstat_cached_only_in_progress;
957 	kstat_named_t arcstat_abd_chunk_waste_size;
958 } arc_stats_t;
959 
960 typedef struct arc_sums {
961 	wmsum_t arcstat_hits;
962 	wmsum_t arcstat_iohits;
963 	wmsum_t arcstat_misses;
964 	wmsum_t arcstat_demand_data_hits;
965 	wmsum_t arcstat_demand_data_iohits;
966 	wmsum_t arcstat_demand_data_misses;
967 	wmsum_t arcstat_demand_metadata_hits;
968 	wmsum_t arcstat_demand_metadata_iohits;
969 	wmsum_t arcstat_demand_metadata_misses;
970 	wmsum_t arcstat_prefetch_data_hits;
971 	wmsum_t arcstat_prefetch_data_iohits;
972 	wmsum_t arcstat_prefetch_data_misses;
973 	wmsum_t arcstat_prefetch_metadata_hits;
974 	wmsum_t arcstat_prefetch_metadata_iohits;
975 	wmsum_t arcstat_prefetch_metadata_misses;
976 	wmsum_t arcstat_mru_hits;
977 	wmsum_t arcstat_mru_ghost_hits;
978 	wmsum_t arcstat_mfu_hits;
979 	wmsum_t arcstat_mfu_ghost_hits;
980 	wmsum_t arcstat_uncached_hits;
981 	wmsum_t arcstat_deleted;
982 	wmsum_t arcstat_mutex_miss;
983 	wmsum_t arcstat_access_skip;
984 	wmsum_t arcstat_evict_skip;
985 	wmsum_t arcstat_evict_not_enough;
986 	wmsum_t arcstat_evict_l2_cached;
987 	wmsum_t arcstat_evict_l2_eligible;
988 	wmsum_t arcstat_evict_l2_eligible_mfu;
989 	wmsum_t arcstat_evict_l2_eligible_mru;
990 	wmsum_t arcstat_evict_l2_ineligible;
991 	wmsum_t arcstat_evict_l2_skip;
992 	wmsum_t arcstat_hash_elements;
993 	wmsum_t arcstat_hash_collisions;
994 	wmsum_t arcstat_hash_chains;
995 	aggsum_t arcstat_size;
996 	wmsum_t arcstat_compressed_size;
997 	wmsum_t arcstat_uncompressed_size;
998 	wmsum_t arcstat_overhead_size;
999 	wmsum_t arcstat_hdr_size;
1000 	wmsum_t arcstat_data_size;
1001 	wmsum_t arcstat_metadata_size;
1002 	wmsum_t arcstat_dbuf_size;
1003 	aggsum_t arcstat_dnode_size;
1004 	wmsum_t arcstat_bonus_size;
1005 	wmsum_t arcstat_l2_hits;
1006 	wmsum_t arcstat_l2_misses;
1007 	wmsum_t arcstat_l2_prefetch_asize;
1008 	wmsum_t arcstat_l2_mru_asize;
1009 	wmsum_t arcstat_l2_mfu_asize;
1010 	wmsum_t arcstat_l2_bufc_data_asize;
1011 	wmsum_t arcstat_l2_bufc_metadata_asize;
1012 	wmsum_t arcstat_l2_feeds;
1013 	wmsum_t arcstat_l2_rw_clash;
1014 	wmsum_t arcstat_l2_read_bytes;
1015 	wmsum_t arcstat_l2_write_bytes;
1016 	wmsum_t arcstat_l2_writes_sent;
1017 	wmsum_t arcstat_l2_writes_done;
1018 	wmsum_t arcstat_l2_writes_error;
1019 	wmsum_t arcstat_l2_writes_lock_retry;
1020 	wmsum_t arcstat_l2_evict_lock_retry;
1021 	wmsum_t arcstat_l2_evict_reading;
1022 	wmsum_t arcstat_l2_evict_l1cached;
1023 	wmsum_t arcstat_l2_free_on_write;
1024 	wmsum_t arcstat_l2_abort_lowmem;
1025 	wmsum_t arcstat_l2_cksum_bad;
1026 	wmsum_t arcstat_l2_io_error;
1027 	wmsum_t arcstat_l2_lsize;
1028 	wmsum_t arcstat_l2_psize;
1029 	aggsum_t arcstat_l2_hdr_size;
1030 	wmsum_t arcstat_l2_log_blk_writes;
1031 	wmsum_t arcstat_l2_log_blk_asize;
1032 	wmsum_t arcstat_l2_log_blk_count;
1033 	wmsum_t arcstat_l2_rebuild_success;
1034 	wmsum_t arcstat_l2_rebuild_abort_unsupported;
1035 	wmsum_t arcstat_l2_rebuild_abort_io_errors;
1036 	wmsum_t arcstat_l2_rebuild_abort_dh_errors;
1037 	wmsum_t arcstat_l2_rebuild_abort_cksum_lb_errors;
1038 	wmsum_t arcstat_l2_rebuild_abort_lowmem;
1039 	wmsum_t arcstat_l2_rebuild_size;
1040 	wmsum_t arcstat_l2_rebuild_asize;
1041 	wmsum_t arcstat_l2_rebuild_bufs;
1042 	wmsum_t arcstat_l2_rebuild_bufs_precached;
1043 	wmsum_t arcstat_l2_rebuild_log_blks;
1044 	wmsum_t arcstat_memory_throttle_count;
1045 	wmsum_t arcstat_memory_direct_count;
1046 	wmsum_t arcstat_memory_indirect_count;
1047 	wmsum_t arcstat_prune;
1048 	wmsum_t arcstat_meta_used;
1049 	wmsum_t arcstat_async_upgrade_sync;
1050 	wmsum_t arcstat_predictive_prefetch;
1051 	wmsum_t arcstat_demand_hit_predictive_prefetch;
1052 	wmsum_t arcstat_demand_iohit_predictive_prefetch;
1053 	wmsum_t arcstat_prescient_prefetch;
1054 	wmsum_t arcstat_demand_hit_prescient_prefetch;
1055 	wmsum_t arcstat_demand_iohit_prescient_prefetch;
1056 	wmsum_t arcstat_raw_size;
1057 	wmsum_t arcstat_cached_only_in_progress;
1058 	wmsum_t arcstat_abd_chunk_waste_size;
1059 } arc_sums_t;
1060 
1061 typedef struct arc_evict_waiter {
1062 	list_node_t aew_node;
1063 	kcondvar_t aew_cv;
1064 	uint64_t aew_count;
1065 } arc_evict_waiter_t;
1066 
1067 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
1068 
1069 #define	ARCSTAT_INCR(stat, val) \
1070 	wmsum_add(&arc_sums.stat, (val))
1071 
1072 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
1073 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
1074 
1075 #define	arc_no_grow	ARCSTAT(arcstat_no_grow) /* do not grow cache size */
1076 #define	arc_meta	ARCSTAT(arcstat_meta)	/* target frac of metadata */
1077 #define	arc_pd		ARCSTAT(arcstat_pd)	/* target frac of data MRU */
1078 #define	arc_pm		ARCSTAT(arcstat_pm)	/* target frac of meta MRU */
1079 #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
1080 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
1081 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
1082 #define	arc_sys_free	ARCSTAT(arcstat_sys_free) /* target system free bytes */
1083 
1084 #define	arc_anon	(&ARC_anon)
1085 #define	arc_mru		(&ARC_mru)
1086 #define	arc_mru_ghost	(&ARC_mru_ghost)
1087 #define	arc_mfu		(&ARC_mfu)
1088 #define	arc_mfu_ghost	(&ARC_mfu_ghost)
1089 #define	arc_l2c_only	(&ARC_l2c_only)
1090 #define	arc_uncached	(&ARC_uncached)
1091 
1092 extern taskq_t *arc_prune_taskq;
1093 extern arc_stats_t arc_stats;
1094 extern arc_sums_t arc_sums;
1095 extern hrtime_t arc_growtime;
1096 extern boolean_t arc_warm;
1097 extern uint_t arc_grow_retry;
1098 extern uint_t zfs_arc_no_grow_shift;
1099 extern uint_t arc_shrink_shift;
1100 extern kmutex_t arc_prune_mtx;
1101 extern list_t arc_prune_list;
1102 extern arc_state_t	ARC_mfu;
1103 extern arc_state_t	ARC_mru;
1104 extern uint_t zfs_arc_pc_percent;
1105 extern uint_t arc_lotsfree_percent;
1106 extern uint64_t zfs_arc_min;
1107 extern uint64_t zfs_arc_max;
1108 extern uint64_t l2arc_dwpd_limit;
1109 
1110 extern uint64_t arc_reduce_target_size(uint64_t to_free);
1111 extern boolean_t arc_reclaim_needed(void);
1112 extern void arc_kmem_reap_soon(void);
1113 extern void arc_wait_for_eviction(uint64_t, boolean_t, boolean_t);
1114 
1115 extern void arc_lowmem_init(void);
1116 extern void arc_lowmem_fini(void);
1117 extern int arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg);
1118 extern uint64_t arc_free_memory(void);
1119 extern int64_t arc_available_memory(void);
1120 extern void arc_tuning_update(boolean_t);
1121 extern void arc_register_hotplug(void);
1122 extern void arc_unregister_hotplug(void);
1123 
1124 extern int param_set_arc_u64(ZFS_MODULE_PARAM_ARGS);
1125 extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS);
1126 extern int param_set_arc_min(ZFS_MODULE_PARAM_ARGS);
1127 extern int param_set_arc_max(ZFS_MODULE_PARAM_ARGS);
1128 extern int param_set_l2arc_dwpd_limit(ZFS_MODULE_PARAM_ARGS);
1129 extern int param_set_arc_no_grow_shift(ZFS_MODULE_PARAM_ARGS);
1130 extern void l2arc_dwpd_bump_reset(void);
1131 
1132 /* used in zdb.c */
1133 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
1134     const l2arc_log_blkptr_t *lbp);
1135 
1136 /* used in vdev_trim.c */
1137 void l2arc_dev_hdr_update(l2arc_dev_t *dev);
1138 l2arc_dev_t *l2arc_vdev_get(vdev_t *vd);
1139 
1140 #ifdef __cplusplus
1141 }
1142 #endif
1143 
1144 #endif /* _SYS_ARC_IMPL_H */
1145