xref: /freebsd/sys/contrib/openzfs/module/zfs/arc.c (revision edf8578117e8844e02c0121147f45e4609b30680)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2018, Joyent, Inc.
24  * Copyright (c) 2011, 2020, Delphix. All rights reserved.
25  * Copyright (c) 2014, Saso Kiselkov. All rights reserved.
26  * Copyright (c) 2017, Nexenta Systems, Inc.  All rights reserved.
27  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
28  * Copyright (c) 2020, George Amanakis. All rights reserved.
29  * Copyright (c) 2019, Klara Inc.
30  * Copyright (c) 2019, Allan Jude
31  * Copyright (c) 2020, The FreeBSD Foundation [1]
32  *
33  * [1] Portions of this software were developed by Allan Jude
34  *     under sponsorship from the FreeBSD Foundation.
35  */
36 
37 /*
38  * DVA-based Adjustable Replacement Cache
39  *
40  * While much of the theory of operation used here is
41  * based on the self-tuning, low overhead replacement cache
42  * presented by Megiddo and Modha at FAST 2003, there are some
43  * significant differences:
44  *
45  * 1. The Megiddo and Modha model assumes any page is evictable.
46  * Pages in its cache cannot be "locked" into memory.  This makes
47  * the eviction algorithm simple: evict the last page in the list.
48  * This also make the performance characteristics easy to reason
49  * about.  Our cache is not so simple.  At any given moment, some
50  * subset of the blocks in the cache are un-evictable because we
51  * have handed out a reference to them.  Blocks are only evictable
52  * when there are no external references active.  This makes
53  * eviction far more problematic:  we choose to evict the evictable
54  * blocks that are the "lowest" in the list.
55  *
56  * There are times when it is not possible to evict the requested
57  * space.  In these circumstances we are unable to adjust the cache
58  * size.  To prevent the cache growing unbounded at these times we
59  * implement a "cache throttle" that slows the flow of new data
60  * into the cache until we can make space available.
61  *
62  * 2. The Megiddo and Modha model assumes a fixed cache size.
63  * Pages are evicted when the cache is full and there is a cache
64  * miss.  Our model has a variable sized cache.  It grows with
65  * high use, but also tries to react to memory pressure from the
66  * operating system: decreasing its size when system memory is
67  * tight.
68  *
69  * 3. The Megiddo and Modha model assumes a fixed page size. All
70  * elements of the cache are therefore exactly the same size.  So
71  * when adjusting the cache size following a cache miss, its simply
72  * a matter of choosing a single page to evict.  In our model, we
73  * have variable sized cache blocks (ranging from 512 bytes to
74  * 128K bytes).  We therefore choose a set of blocks to evict to make
75  * space for a cache miss that approximates as closely as possible
76  * the space used by the new block.
77  *
78  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
79  * by N. Megiddo & D. Modha, FAST 2003
80  */
81 
82 /*
83  * The locking model:
84  *
85  * A new reference to a cache buffer can be obtained in two
86  * ways: 1) via a hash table lookup using the DVA as a key,
87  * or 2) via one of the ARC lists.  The arc_read() interface
88  * uses method 1, while the internal ARC algorithms for
89  * adjusting the cache use method 2.  We therefore provide two
90  * types of locks: 1) the hash table lock array, and 2) the
91  * ARC list locks.
92  *
93  * Buffers do not have their own mutexes, rather they rely on the
94  * hash table mutexes for the bulk of their protection (i.e. most
95  * fields in the arc_buf_hdr_t are protected by these mutexes).
96  *
97  * buf_hash_find() returns the appropriate mutex (held) when it
98  * locates the requested buffer in the hash table.  It returns
99  * NULL for the mutex if the buffer was not in the table.
100  *
101  * buf_hash_remove() expects the appropriate hash mutex to be
102  * already held before it is invoked.
103  *
104  * Each ARC state also has a mutex which is used to protect the
105  * buffer list associated with the state.  When attempting to
106  * obtain a hash table lock while holding an ARC list lock you
107  * must use: mutex_tryenter() to avoid deadlock.  Also note that
108  * the active state mutex must be held before the ghost state mutex.
109  *
110  * It as also possible to register a callback which is run when the
111  * metadata limit is reached and no buffers can be safely evicted.  In
112  * this case the arc user should drop a reference on some arc buffers so
113  * they can be reclaimed.  For example, when using the ZPL each dentry
114  * holds a references on a znode.  These dentries must be pruned before
115  * the arc buffer holding the znode can be safely evicted.
116  *
117  * Note that the majority of the performance stats are manipulated
118  * with atomic operations.
119  *
120  * The L2ARC uses the l2ad_mtx on each vdev for the following:
121  *
122  *	- L2ARC buflist creation
123  *	- L2ARC buflist eviction
124  *	- L2ARC write completion, which walks L2ARC buflists
125  *	- ARC header destruction, as it removes from L2ARC buflists
126  *	- ARC header release, as it removes from L2ARC buflists
127  */
128 
129 /*
130  * ARC operation:
131  *
132  * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
133  * This structure can point either to a block that is still in the cache or to
134  * one that is only accessible in an L2 ARC device, or it can provide
135  * information about a block that was recently evicted. If a block is
136  * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
137  * information to retrieve it from the L2ARC device. This information is
138  * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
139  * that is in this state cannot access the data directly.
140  *
141  * Blocks that are actively being referenced or have not been evicted
142  * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
143  * the arc_buf_hdr_t that will point to the data block in memory. A block can
144  * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
145  * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
146  * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
147  *
148  * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
149  * ability to store the physical data (b_pabd) associated with the DVA of the
150  * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
151  * it will match its on-disk compression characteristics. This behavior can be
152  * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
153  * compressed ARC functionality is disabled, the b_pabd will point to an
154  * uncompressed version of the on-disk data.
155  *
156  * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
157  * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
158  * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
159  * consumer. The ARC will provide references to this data and will keep it
160  * cached until it is no longer in use. The ARC caches only the L1ARC's physical
161  * data block and will evict any arc_buf_t that is no longer referenced. The
162  * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
163  * "overhead_size" kstat.
164  *
165  * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
166  * compressed form. The typical case is that consumers will want uncompressed
167  * data, and when that happens a new data buffer is allocated where the data is
168  * decompressed for them to use. Currently the only consumer who wants
169  * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
170  * exists on disk. When this happens, the arc_buf_t's data buffer is shared
171  * with the arc_buf_hdr_t.
172  *
173  * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
174  * first one is owned by a compressed send consumer (and therefore references
175  * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
176  * used by any other consumer (and has its own uncompressed copy of the data
177  * buffer).
178  *
179  *   arc_buf_hdr_t
180  *   +-----------+
181  *   | fields    |
182  *   | common to |
183  *   | L1- and   |
184  *   | L2ARC     |
185  *   +-----------+
186  *   | l2arc_buf_hdr_t
187  *   |           |
188  *   +-----------+
189  *   | l1arc_buf_hdr_t
190  *   |           |              arc_buf_t
191  *   | b_buf     +------------>+-----------+      arc_buf_t
192  *   | b_pabd    +-+           |b_next     +---->+-----------+
193  *   +-----------+ |           |-----------|     |b_next     +-->NULL
194  *                 |           |b_comp = T |     +-----------+
195  *                 |           |b_data     +-+   |b_comp = F |
196  *                 |           +-----------+ |   |b_data     +-+
197  *                 +->+------+               |   +-----------+ |
198  *        compressed  |      |               |                 |
199  *           data     |      |<--------------+                 | uncompressed
200  *                    +------+          compressed,            |     data
201  *                                        shared               +-->+------+
202  *                                         data                    |      |
203  *                                                                 |      |
204  *                                                                 +------+
205  *
206  * When a consumer reads a block, the ARC must first look to see if the
207  * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
208  * arc_buf_t and either copies uncompressed data into a new data buffer from an
209  * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
210  * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
211  * hdr is compressed and the desired compression characteristics of the
212  * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
213  * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
214  * the last buffer in the hdr's b_buf list, however a shared compressed buf can
215  * be anywhere in the hdr's list.
216  *
217  * The diagram below shows an example of an uncompressed ARC hdr that is
218  * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
219  * the last element in the buf list):
220  *
221  *                arc_buf_hdr_t
222  *                +-----------+
223  *                |           |
224  *                |           |
225  *                |           |
226  *                +-----------+
227  * l2arc_buf_hdr_t|           |
228  *                |           |
229  *                +-----------+
230  * l1arc_buf_hdr_t|           |
231  *                |           |                 arc_buf_t    (shared)
232  *                |    b_buf  +------------>+---------+      arc_buf_t
233  *                |           |             |b_next   +---->+---------+
234  *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
235  *                +-----------+ |           |         |     +---------+
236  *                              |           |b_data   +-+   |         |
237  *                              |           +---------+ |   |b_data   +-+
238  *                              +->+------+             |   +---------+ |
239  *                                 |      |             |               |
240  *                   uncompressed  |      |             |               |
241  *                        data     +------+             |               |
242  *                                    ^                 +->+------+     |
243  *                                    |       uncompressed |      |     |
244  *                                    |           data     |      |     |
245  *                                    |                    +------+     |
246  *                                    +---------------------------------+
247  *
248  * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
249  * since the physical block is about to be rewritten. The new data contents
250  * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
251  * it may compress the data before writing it to disk. The ARC will be called
252  * with the transformed data and will memcpy the transformed on-disk block into
253  * a newly allocated b_pabd. Writes are always done into buffers which have
254  * either been loaned (and hence are new and don't have other readers) or
255  * buffers which have been released (and hence have their own hdr, if there
256  * were originally other readers of the buf's original hdr). This ensures that
257  * the ARC only needs to update a single buf and its hdr after a write occurs.
258  *
259  * When the L2ARC is in use, it will also take advantage of the b_pabd. The
260  * L2ARC will always write the contents of b_pabd to the L2ARC. This means
261  * that when compressed ARC is enabled that the L2ARC blocks are identical
262  * to the on-disk block in the main data pool. This provides a significant
263  * advantage since the ARC can leverage the bp's checksum when reading from the
264  * L2ARC to determine if the contents are valid. However, if the compressed
265  * ARC is disabled, then the L2ARC's block must be transformed to look
266  * like the physical block in the main data pool before comparing the
267  * checksum and determining its validity.
268  *
269  * The L1ARC has a slightly different system for storing encrypted data.
270  * Raw (encrypted + possibly compressed) data has a few subtle differences from
271  * data that is just compressed. The biggest difference is that it is not
272  * possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
273  * The other difference is that encryption cannot be treated as a suggestion.
274  * If a caller would prefer compressed data, but they actually wind up with
275  * uncompressed data the worst thing that could happen is there might be a
276  * performance hit. If the caller requests encrypted data, however, we must be
277  * sure they actually get it or else secret information could be leaked. Raw
278  * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
279  * may have both an encrypted version and a decrypted version of its data at
280  * once. When a caller needs a raw arc_buf_t, it is allocated and the data is
281  * copied out of this header. To avoid complications with b_pabd, raw buffers
282  * cannot be shared.
283  */
284 
285 #include <sys/spa.h>
286 #include <sys/zio.h>
287 #include <sys/spa_impl.h>
288 #include <sys/zio_compress.h>
289 #include <sys/zio_checksum.h>
290 #include <sys/zfs_context.h>
291 #include <sys/arc.h>
292 #include <sys/zfs_refcount.h>
293 #include <sys/vdev.h>
294 #include <sys/vdev_impl.h>
295 #include <sys/dsl_pool.h>
296 #include <sys/multilist.h>
297 #include <sys/abd.h>
298 #include <sys/zil.h>
299 #include <sys/fm/fs/zfs.h>
300 #include <sys/callb.h>
301 #include <sys/kstat.h>
302 #include <sys/zthr.h>
303 #include <zfs_fletcher.h>
304 #include <sys/arc_impl.h>
305 #include <sys/trace_zfs.h>
306 #include <sys/aggsum.h>
307 #include <sys/wmsum.h>
308 #include <cityhash.h>
309 #include <sys/vdev_trim.h>
310 #include <sys/zfs_racct.h>
311 #include <sys/zstd/zstd.h>
312 
313 #ifndef _KERNEL
314 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
315 boolean_t arc_watch = B_FALSE;
316 #endif
317 
318 /*
319  * This thread's job is to keep enough free memory in the system, by
320  * calling arc_kmem_reap_soon() plus arc_reduce_target_size(), which improves
321  * arc_available_memory().
322  */
323 static zthr_t *arc_reap_zthr;
324 
325 /*
326  * This thread's job is to keep arc_size under arc_c, by calling
327  * arc_evict(), which improves arc_is_overflowing().
328  */
329 static zthr_t *arc_evict_zthr;
330 static arc_buf_hdr_t **arc_state_evict_markers;
331 static int arc_state_evict_marker_count;
332 
333 static kmutex_t arc_evict_lock;
334 static boolean_t arc_evict_needed = B_FALSE;
335 static clock_t arc_last_uncached_flush;
336 
337 /*
338  * Count of bytes evicted since boot.
339  */
340 static uint64_t arc_evict_count;
341 
342 /*
343  * List of arc_evict_waiter_t's, representing threads waiting for the
344  * arc_evict_count to reach specific values.
345  */
346 static list_t arc_evict_waiters;
347 
348 /*
349  * When arc_is_overflowing(), arc_get_data_impl() waits for this percent of
350  * the requested amount of data to be evicted.  For example, by default for
351  * every 2KB that's evicted, 1KB of it may be "reused" by a new allocation.
352  * Since this is above 100%, it ensures that progress is made towards getting
353  * arc_size under arc_c.  Since this is finite, it ensures that allocations
354  * can still happen, even during the potentially long time that arc_size is
355  * more than arc_c.
356  */
357 static uint_t zfs_arc_eviction_pct = 200;
358 
359 /*
360  * The number of headers to evict in arc_evict_state_impl() before
361  * dropping the sublist lock and evicting from another sublist. A lower
362  * value means we're more likely to evict the "correct" header (i.e. the
363  * oldest header in the arc state), but comes with higher overhead
364  * (i.e. more invocations of arc_evict_state_impl()).
365  */
366 static uint_t zfs_arc_evict_batch_limit = 10;
367 
368 /* number of seconds before growing cache again */
369 uint_t arc_grow_retry = 5;
370 
371 /*
372  * Minimum time between calls to arc_kmem_reap_soon().
373  */
374 static const int arc_kmem_cache_reap_retry_ms = 1000;
375 
376 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */
377 static int zfs_arc_overflow_shift = 8;
378 
379 /* log2(fraction of arc to reclaim) */
380 uint_t arc_shrink_shift = 7;
381 
382 /* percent of pagecache to reclaim arc to */
383 #ifdef _KERNEL
384 uint_t zfs_arc_pc_percent = 0;
385 #endif
386 
387 /*
388  * log2(fraction of ARC which must be free to allow growing).
389  * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
390  * when reading a new block into the ARC, we will evict an equal-sized block
391  * from the ARC.
392  *
393  * This must be less than arc_shrink_shift, so that when we shrink the ARC,
394  * we will still not allow it to grow.
395  */
396 uint_t		arc_no_grow_shift = 5;
397 
398 
399 /*
400  * minimum lifespan of a prefetch block in clock ticks
401  * (initialized in arc_init())
402  */
403 static uint_t		arc_min_prefetch_ms;
404 static uint_t		arc_min_prescient_prefetch_ms;
405 
406 /*
407  * If this percent of memory is free, don't throttle.
408  */
409 uint_t arc_lotsfree_percent = 10;
410 
411 /*
412  * The arc has filled available memory and has now warmed up.
413  */
414 boolean_t arc_warm;
415 
416 /*
417  * These tunables are for performance analysis.
418  */
419 uint64_t zfs_arc_max = 0;
420 uint64_t zfs_arc_min = 0;
421 static uint64_t zfs_arc_dnode_limit = 0;
422 static uint_t zfs_arc_dnode_reduce_percent = 10;
423 static uint_t zfs_arc_grow_retry = 0;
424 static uint_t zfs_arc_shrink_shift = 0;
425 uint_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
426 
427 /*
428  * ARC dirty data constraints for arc_tempreserve_space() throttle:
429  * * total dirty data limit
430  * * anon block dirty limit
431  * * each pool's anon allowance
432  */
433 static const unsigned long zfs_arc_dirty_limit_percent = 50;
434 static const unsigned long zfs_arc_anon_limit_percent = 25;
435 static const unsigned long zfs_arc_pool_dirty_percent = 20;
436 
437 /*
438  * Enable or disable compressed arc buffers.
439  */
440 int zfs_compressed_arc_enabled = B_TRUE;
441 
442 /*
443  * Balance between metadata and data on ghost hits.  Values above 100
444  * increase metadata caching by proportionally reducing effect of ghost
445  * data hits on target data/metadata rate.
446  */
447 static uint_t zfs_arc_meta_balance = 500;
448 
449 /*
450  * Percentage that can be consumed by dnodes of ARC meta buffers.
451  */
452 static uint_t zfs_arc_dnode_limit_percent = 10;
453 
454 /*
455  * These tunables are Linux-specific
456  */
457 static uint64_t zfs_arc_sys_free = 0;
458 static uint_t zfs_arc_min_prefetch_ms = 0;
459 static uint_t zfs_arc_min_prescient_prefetch_ms = 0;
460 static uint_t zfs_arc_lotsfree_percent = 10;
461 
462 /*
463  * Number of arc_prune threads
464  */
465 static int zfs_arc_prune_task_threads = 1;
466 
467 /* The 7 states: */
468 arc_state_t ARC_anon;
469 arc_state_t ARC_mru;
470 arc_state_t ARC_mru_ghost;
471 arc_state_t ARC_mfu;
472 arc_state_t ARC_mfu_ghost;
473 arc_state_t ARC_l2c_only;
474 arc_state_t ARC_uncached;
475 
476 arc_stats_t arc_stats = {
477 	{ "hits",			KSTAT_DATA_UINT64 },
478 	{ "iohits",			KSTAT_DATA_UINT64 },
479 	{ "misses",			KSTAT_DATA_UINT64 },
480 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
481 	{ "demand_data_iohits",		KSTAT_DATA_UINT64 },
482 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
483 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
484 	{ "demand_metadata_iohits",	KSTAT_DATA_UINT64 },
485 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
486 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
487 	{ "prefetch_data_iohits",	KSTAT_DATA_UINT64 },
488 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
489 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
490 	{ "prefetch_metadata_iohits",	KSTAT_DATA_UINT64 },
491 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
492 	{ "mru_hits",			KSTAT_DATA_UINT64 },
493 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
494 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
495 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
496 	{ "uncached_hits",		KSTAT_DATA_UINT64 },
497 	{ "deleted",			KSTAT_DATA_UINT64 },
498 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
499 	{ "access_skip",		KSTAT_DATA_UINT64 },
500 	{ "evict_skip",			KSTAT_DATA_UINT64 },
501 	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
502 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
503 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
504 	{ "evict_l2_eligible_mfu",	KSTAT_DATA_UINT64 },
505 	{ "evict_l2_eligible_mru",	KSTAT_DATA_UINT64 },
506 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
507 	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
508 	{ "hash_elements",		KSTAT_DATA_UINT64 },
509 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
510 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
511 	{ "hash_chains",		KSTAT_DATA_UINT64 },
512 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
513 	{ "meta",			KSTAT_DATA_UINT64 },
514 	{ "pd",				KSTAT_DATA_UINT64 },
515 	{ "pm",				KSTAT_DATA_UINT64 },
516 	{ "c",				KSTAT_DATA_UINT64 },
517 	{ "c_min",			KSTAT_DATA_UINT64 },
518 	{ "c_max",			KSTAT_DATA_UINT64 },
519 	{ "size",			KSTAT_DATA_UINT64 },
520 	{ "compressed_size",		KSTAT_DATA_UINT64 },
521 	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
522 	{ "overhead_size",		KSTAT_DATA_UINT64 },
523 	{ "hdr_size",			KSTAT_DATA_UINT64 },
524 	{ "data_size",			KSTAT_DATA_UINT64 },
525 	{ "metadata_size",		KSTAT_DATA_UINT64 },
526 	{ "dbuf_size",			KSTAT_DATA_UINT64 },
527 	{ "dnode_size",			KSTAT_DATA_UINT64 },
528 	{ "bonus_size",			KSTAT_DATA_UINT64 },
529 #if defined(COMPAT_FREEBSD11)
530 	{ "other_size",			KSTAT_DATA_UINT64 },
531 #endif
532 	{ "anon_size",			KSTAT_DATA_UINT64 },
533 	{ "anon_data",			KSTAT_DATA_UINT64 },
534 	{ "anon_metadata",		KSTAT_DATA_UINT64 },
535 	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
536 	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
537 	{ "mru_size",			KSTAT_DATA_UINT64 },
538 	{ "mru_data",			KSTAT_DATA_UINT64 },
539 	{ "mru_metadata",		KSTAT_DATA_UINT64 },
540 	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
541 	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
542 	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
543 	{ "mru_ghost_data",		KSTAT_DATA_UINT64 },
544 	{ "mru_ghost_metadata",		KSTAT_DATA_UINT64 },
545 	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
546 	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
547 	{ "mfu_size",			KSTAT_DATA_UINT64 },
548 	{ "mfu_data",			KSTAT_DATA_UINT64 },
549 	{ "mfu_metadata",		KSTAT_DATA_UINT64 },
550 	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
551 	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
552 	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
553 	{ "mfu_ghost_data",		KSTAT_DATA_UINT64 },
554 	{ "mfu_ghost_metadata",		KSTAT_DATA_UINT64 },
555 	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
556 	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
557 	{ "uncached_size",		KSTAT_DATA_UINT64 },
558 	{ "uncached_data",		KSTAT_DATA_UINT64 },
559 	{ "uncached_metadata",		KSTAT_DATA_UINT64 },
560 	{ "uncached_evictable_data",	KSTAT_DATA_UINT64 },
561 	{ "uncached_evictable_metadata", KSTAT_DATA_UINT64 },
562 	{ "l2_hits",			KSTAT_DATA_UINT64 },
563 	{ "l2_misses",			KSTAT_DATA_UINT64 },
564 	{ "l2_prefetch_asize",		KSTAT_DATA_UINT64 },
565 	{ "l2_mru_asize",		KSTAT_DATA_UINT64 },
566 	{ "l2_mfu_asize",		KSTAT_DATA_UINT64 },
567 	{ "l2_bufc_data_asize",		KSTAT_DATA_UINT64 },
568 	{ "l2_bufc_metadata_asize",	KSTAT_DATA_UINT64 },
569 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
570 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
571 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
572 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
573 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
574 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
575 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
576 	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
577 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
578 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
579 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
580 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
581 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
582 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
583 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
584 	{ "l2_size",			KSTAT_DATA_UINT64 },
585 	{ "l2_asize",			KSTAT_DATA_UINT64 },
586 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
587 	{ "l2_log_blk_writes",		KSTAT_DATA_UINT64 },
588 	{ "l2_log_blk_avg_asize",	KSTAT_DATA_UINT64 },
589 	{ "l2_log_blk_asize",		KSTAT_DATA_UINT64 },
590 	{ "l2_log_blk_count",		KSTAT_DATA_UINT64 },
591 	{ "l2_data_to_meta_ratio",	KSTAT_DATA_UINT64 },
592 	{ "l2_rebuild_success",		KSTAT_DATA_UINT64 },
593 	{ "l2_rebuild_unsupported",	KSTAT_DATA_UINT64 },
594 	{ "l2_rebuild_io_errors",	KSTAT_DATA_UINT64 },
595 	{ "l2_rebuild_dh_errors",	KSTAT_DATA_UINT64 },
596 	{ "l2_rebuild_cksum_lb_errors",	KSTAT_DATA_UINT64 },
597 	{ "l2_rebuild_lowmem",		KSTAT_DATA_UINT64 },
598 	{ "l2_rebuild_size",		KSTAT_DATA_UINT64 },
599 	{ "l2_rebuild_asize",		KSTAT_DATA_UINT64 },
600 	{ "l2_rebuild_bufs",		KSTAT_DATA_UINT64 },
601 	{ "l2_rebuild_bufs_precached",	KSTAT_DATA_UINT64 },
602 	{ "l2_rebuild_log_blks",	KSTAT_DATA_UINT64 },
603 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
604 	{ "memory_direct_count",	KSTAT_DATA_UINT64 },
605 	{ "memory_indirect_count",	KSTAT_DATA_UINT64 },
606 	{ "memory_all_bytes",		KSTAT_DATA_UINT64 },
607 	{ "memory_free_bytes",		KSTAT_DATA_UINT64 },
608 	{ "memory_available_bytes",	KSTAT_DATA_INT64 },
609 	{ "arc_no_grow",		KSTAT_DATA_UINT64 },
610 	{ "arc_tempreserve",		KSTAT_DATA_UINT64 },
611 	{ "arc_loaned_bytes",		KSTAT_DATA_UINT64 },
612 	{ "arc_prune",			KSTAT_DATA_UINT64 },
613 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
614 	{ "arc_dnode_limit",		KSTAT_DATA_UINT64 },
615 	{ "async_upgrade_sync",		KSTAT_DATA_UINT64 },
616 	{ "predictive_prefetch", KSTAT_DATA_UINT64 },
617 	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
618 	{ "demand_iohit_predictive_prefetch", KSTAT_DATA_UINT64 },
619 	{ "prescient_prefetch", KSTAT_DATA_UINT64 },
620 	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
621 	{ "demand_iohit_prescient_prefetch", KSTAT_DATA_UINT64 },
622 	{ "arc_need_free",		KSTAT_DATA_UINT64 },
623 	{ "arc_sys_free",		KSTAT_DATA_UINT64 },
624 	{ "arc_raw_size",		KSTAT_DATA_UINT64 },
625 	{ "cached_only_in_progress",	KSTAT_DATA_UINT64 },
626 	{ "abd_chunk_waste_size",	KSTAT_DATA_UINT64 },
627 };
628 
629 arc_sums_t arc_sums;
630 
631 #define	ARCSTAT_MAX(stat, val) {					\
632 	uint64_t m;							\
633 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
634 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
635 		continue;						\
636 }
637 
638 /*
639  * We define a macro to allow ARC hits/misses to be easily broken down by
640  * two separate conditions, giving a total of four different subtypes for
641  * each of hits and misses (so eight statistics total).
642  */
643 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
644 	if (cond1) {							\
645 		if (cond2) {						\
646 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
647 		} else {						\
648 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
649 		}							\
650 	} else {							\
651 		if (cond2) {						\
652 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
653 		} else {						\
654 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
655 		}							\
656 	}
657 
658 /*
659  * This macro allows us to use kstats as floating averages. Each time we
660  * update this kstat, we first factor it and the update value by
661  * ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall
662  * average. This macro assumes that integer loads and stores are atomic, but
663  * is not safe for multiple writers updating the kstat in parallel (only the
664  * last writer's update will remain).
665  */
666 #define	ARCSTAT_F_AVG_FACTOR	3
667 #define	ARCSTAT_F_AVG(stat, value) \
668 	do { \
669 		uint64_t x = ARCSTAT(stat); \
670 		x = x - x / ARCSTAT_F_AVG_FACTOR + \
671 		    (value) / ARCSTAT_F_AVG_FACTOR; \
672 		ARCSTAT(stat) = x; \
673 	} while (0)
674 
675 static kstat_t			*arc_ksp;
676 
677 /*
678  * There are several ARC variables that are critical to export as kstats --
679  * but we don't want to have to grovel around in the kstat whenever we wish to
680  * manipulate them.  For these variables, we therefore define them to be in
681  * terms of the statistic variable.  This assures that we are not introducing
682  * the possibility of inconsistency by having shadow copies of the variables,
683  * while still allowing the code to be readable.
684  */
685 #define	arc_tempreserve	ARCSTAT(arcstat_tempreserve)
686 #define	arc_loaned_bytes	ARCSTAT(arcstat_loaned_bytes)
687 #define	arc_dnode_limit	ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
688 #define	arc_need_free	ARCSTAT(arcstat_need_free) /* waiting to be evicted */
689 
690 hrtime_t arc_growtime;
691 list_t arc_prune_list;
692 kmutex_t arc_prune_mtx;
693 taskq_t *arc_prune_taskq;
694 
695 #define	GHOST_STATE(state)	\
696 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
697 	(state) == arc_l2c_only)
698 
699 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
700 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
701 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
702 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
703 #define	HDR_PRESCIENT_PREFETCH(hdr)	\
704 	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
705 #define	HDR_COMPRESSION_ENABLED(hdr)	\
706 	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
707 
708 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
709 #define	HDR_UNCACHED(hdr)	((hdr)->b_flags & ARC_FLAG_UNCACHED)
710 #define	HDR_L2_READING(hdr)	\
711 	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
712 	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
713 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
714 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
715 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
716 #define	HDR_PROTECTED(hdr)	((hdr)->b_flags & ARC_FLAG_PROTECTED)
717 #define	HDR_NOAUTH(hdr)		((hdr)->b_flags & ARC_FLAG_NOAUTH)
718 #define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
719 
720 #define	HDR_ISTYPE_METADATA(hdr)	\
721 	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
722 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
723 
724 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
725 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
726 #define	HDR_HAS_RABD(hdr)	\
727 	(HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) &&	\
728 	(hdr)->b_crypt_hdr.b_rabd != NULL)
729 #define	HDR_ENCRYPTED(hdr)	\
730 	(HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
731 #define	HDR_AUTHENTICATED(hdr)	\
732 	(HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
733 
734 /* For storing compression mode in b_flags */
735 #define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
736 
737 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
738 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
739 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
740 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
741 
742 #define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
743 #define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
744 #define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
745 #define	ARC_BUF_ENCRYPTED(buf)	((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)
746 
747 /*
748  * Other sizes
749  */
750 
751 #define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
752 #define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
753 
754 /*
755  * Hash table routines
756  */
757 
758 #define	BUF_LOCKS 2048
759 typedef struct buf_hash_table {
760 	uint64_t ht_mask;
761 	arc_buf_hdr_t **ht_table;
762 	kmutex_t ht_locks[BUF_LOCKS] ____cacheline_aligned;
763 } buf_hash_table_t;
764 
765 static buf_hash_table_t buf_hash_table;
766 
767 #define	BUF_HASH_INDEX(spa, dva, birth) \
768 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
769 #define	BUF_HASH_LOCK(idx)	(&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
770 #define	HDR_LOCK(hdr) \
771 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
772 
773 uint64_t zfs_crc64_table[256];
774 
775 /*
776  * Level 2 ARC
777  */
778 
779 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
780 #define	L2ARC_HEADROOM		2			/* num of writes */
781 
782 /*
783  * If we discover during ARC scan any buffers to be compressed, we boost
784  * our headroom for the next scanning cycle by this percentage multiple.
785  */
786 #define	L2ARC_HEADROOM_BOOST	200
787 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
788 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
789 
790 /*
791  * We can feed L2ARC from two states of ARC buffers, mru and mfu,
792  * and each of the state has two types: data and metadata.
793  */
794 #define	L2ARC_FEED_TYPES	4
795 
796 /* L2ARC Performance Tunables */
797 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* def max write size */
798 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra warmup write */
799 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* # of dev writes */
800 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
801 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
802 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval msecs */
803 int l2arc_noprefetch = B_TRUE;			/* don't cache prefetch bufs */
804 int l2arc_feed_again = B_TRUE;			/* turbo warmup */
805 int l2arc_norw = B_FALSE;			/* no reads during writes */
806 static uint_t l2arc_meta_percent = 33;	/* limit on headers size */
807 
808 /*
809  * L2ARC Internals
810  */
811 static list_t L2ARC_dev_list;			/* device list */
812 static list_t *l2arc_dev_list;			/* device list pointer */
813 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
814 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
815 static list_t L2ARC_free_on_write;		/* free after write buf list */
816 static list_t *l2arc_free_on_write;		/* free after write list ptr */
817 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
818 static uint64_t l2arc_ndev;			/* number of devices */
819 
820 typedef struct l2arc_read_callback {
821 	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
822 	blkptr_t		l2rcb_bp;		/* original blkptr */
823 	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
824 	int			l2rcb_flags;		/* original flags */
825 	abd_t			*l2rcb_abd;		/* temporary buffer */
826 } l2arc_read_callback_t;
827 
828 typedef struct l2arc_data_free {
829 	/* protected by l2arc_free_on_write_mtx */
830 	abd_t		*l2df_abd;
831 	size_t		l2df_size;
832 	arc_buf_contents_t l2df_type;
833 	list_node_t	l2df_list_node;
834 } l2arc_data_free_t;
835 
836 typedef enum arc_fill_flags {
837 	ARC_FILL_LOCKED		= 1 << 0, /* hdr lock is held */
838 	ARC_FILL_COMPRESSED	= 1 << 1, /* fill with compressed data */
839 	ARC_FILL_ENCRYPTED	= 1 << 2, /* fill with encrypted data */
840 	ARC_FILL_NOAUTH		= 1 << 3, /* don't attempt to authenticate */
841 	ARC_FILL_IN_PLACE	= 1 << 4  /* fill in place (special case) */
842 } arc_fill_flags_t;
843 
844 typedef enum arc_ovf_level {
845 	ARC_OVF_NONE,			/* ARC within target size. */
846 	ARC_OVF_SOME,			/* ARC is slightly overflowed. */
847 	ARC_OVF_SEVERE			/* ARC is severely overflowed. */
848 } arc_ovf_level_t;
849 
850 static kmutex_t l2arc_feed_thr_lock;
851 static kcondvar_t l2arc_feed_thr_cv;
852 static uint8_t l2arc_thread_exit;
853 
854 static kmutex_t l2arc_rebuild_thr_lock;
855 static kcondvar_t l2arc_rebuild_thr_cv;
856 
857 enum arc_hdr_alloc_flags {
858 	ARC_HDR_ALLOC_RDATA = 0x1,
859 	ARC_HDR_USE_RESERVE = 0x4,
860 	ARC_HDR_ALLOC_LINEAR = 0x8,
861 };
862 
863 
864 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, const void *, int);
865 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, const void *);
866 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, const void *, int);
867 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, const void *);
868 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, const void *);
869 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
870     const void *tag);
871 static void arc_hdr_free_abd(arc_buf_hdr_t *, boolean_t);
872 static void arc_hdr_alloc_abd(arc_buf_hdr_t *, int);
873 static void arc_hdr_destroy(arc_buf_hdr_t *);
874 static void arc_access(arc_buf_hdr_t *, arc_flags_t, boolean_t);
875 static void arc_buf_watch(arc_buf_t *);
876 static void arc_change_state(arc_state_t *, arc_buf_hdr_t *);
877 
878 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
879 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
880 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
881 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
882 
883 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
884 static void l2arc_read_done(zio_t *);
885 static void l2arc_do_free_on_write(void);
886 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
887     boolean_t state_only);
888 
889 #define	l2arc_hdr_arcstats_increment(hdr) \
890 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
891 #define	l2arc_hdr_arcstats_decrement(hdr) \
892 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
893 #define	l2arc_hdr_arcstats_increment_state(hdr) \
894 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
895 #define	l2arc_hdr_arcstats_decrement_state(hdr) \
896 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
897 
898 /*
899  * l2arc_exclude_special : A zfs module parameter that controls whether buffers
900  * 		present on special vdevs are eligibile for caching in L2ARC. If
901  * 		set to 1, exclude dbufs on special vdevs from being cached to
902  * 		L2ARC.
903  */
904 int l2arc_exclude_special = 0;
905 
906 /*
907  * l2arc_mfuonly : A ZFS module parameter that controls whether only MFU
908  * 		metadata and data are cached from ARC into L2ARC.
909  */
910 static int l2arc_mfuonly = 0;
911 
912 /*
913  * L2ARC TRIM
914  * l2arc_trim_ahead : A ZFS module parameter that controls how much ahead of
915  * 		the current write size (l2arc_write_max) we should TRIM if we
916  * 		have filled the device. It is defined as a percentage of the
917  * 		write size. If set to 100 we trim twice the space required to
918  * 		accommodate upcoming writes. A minimum of 64MB will be trimmed.
919  * 		It also enables TRIM of the whole L2ARC device upon creation or
920  * 		addition to an existing pool or if the header of the device is
921  * 		invalid upon importing a pool or onlining a cache device. The
922  * 		default is 0, which disables TRIM on L2ARC altogether as it can
923  * 		put significant stress on the underlying storage devices. This
924  * 		will vary depending of how well the specific device handles
925  * 		these commands.
926  */
927 static uint64_t l2arc_trim_ahead = 0;
928 
929 /*
930  * Performance tuning of L2ARC persistence:
931  *
932  * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
933  * 		an L2ARC device (either at pool import or later) will attempt
934  * 		to rebuild L2ARC buffer contents.
935  * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
936  * 		whether log blocks are written to the L2ARC device. If the L2ARC
937  * 		device is less than 1GB, the amount of data l2arc_evict()
938  * 		evicts is significant compared to the amount of restored L2ARC
939  * 		data. In this case do not write log blocks in L2ARC in order
940  * 		not to waste space.
941  */
942 static int l2arc_rebuild_enabled = B_TRUE;
943 static uint64_t l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
944 
945 /* L2ARC persistence rebuild control routines. */
946 void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
947 static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
948 static int l2arc_rebuild(l2arc_dev_t *dev);
949 
950 /* L2ARC persistence read I/O routines. */
951 static int l2arc_dev_hdr_read(l2arc_dev_t *dev);
952 static int l2arc_log_blk_read(l2arc_dev_t *dev,
953     const l2arc_log_blkptr_t *this_lp, const l2arc_log_blkptr_t *next_lp,
954     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
955     zio_t *this_io, zio_t **next_io);
956 static zio_t *l2arc_log_blk_fetch(vdev_t *vd,
957     const l2arc_log_blkptr_t *lp, l2arc_log_blk_phys_t *lb);
958 static void l2arc_log_blk_fetch_abort(zio_t *zio);
959 
960 /* L2ARC persistence block restoration routines. */
961 static void l2arc_log_blk_restore(l2arc_dev_t *dev,
962     const l2arc_log_blk_phys_t *lb, uint64_t lb_asize);
963 static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le,
964     l2arc_dev_t *dev);
965 
966 /* L2ARC persistence write I/O routines. */
967 static uint64_t l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio,
968     l2arc_write_callback_t *cb);
969 
970 /* L2ARC persistence auxiliary routines. */
971 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
972     const l2arc_log_blkptr_t *lbp);
973 static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev,
974     const arc_buf_hdr_t *ab);
975 boolean_t l2arc_range_check_overlap(uint64_t bottom,
976     uint64_t top, uint64_t check);
977 static void l2arc_blk_fetch_done(zio_t *zio);
978 static inline uint64_t
979     l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev);
980 
981 /*
982  * We use Cityhash for this. It's fast, and has good hash properties without
983  * requiring any large static buffers.
984  */
985 static uint64_t
986 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
987 {
988 	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
989 }
990 
991 #define	HDR_EMPTY(hdr)						\
992 	((hdr)->b_dva.dva_word[0] == 0 &&			\
993 	(hdr)->b_dva.dva_word[1] == 0)
994 
995 #define	HDR_EMPTY_OR_LOCKED(hdr)				\
996 	(HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
997 
998 #define	HDR_EQUAL(spa, dva, birth, hdr)				\
999 	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1000 	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1001 	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1002 
1003 static void
1004 buf_discard_identity(arc_buf_hdr_t *hdr)
1005 {
1006 	hdr->b_dva.dva_word[0] = 0;
1007 	hdr->b_dva.dva_word[1] = 0;
1008 	hdr->b_birth = 0;
1009 }
1010 
1011 static arc_buf_hdr_t *
1012 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1013 {
1014 	const dva_t *dva = BP_IDENTITY(bp);
1015 	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
1016 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1017 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1018 	arc_buf_hdr_t *hdr;
1019 
1020 	mutex_enter(hash_lock);
1021 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1022 	    hdr = hdr->b_hash_next) {
1023 		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1024 			*lockp = hash_lock;
1025 			return (hdr);
1026 		}
1027 	}
1028 	mutex_exit(hash_lock);
1029 	*lockp = NULL;
1030 	return (NULL);
1031 }
1032 
1033 /*
1034  * Insert an entry into the hash table.  If there is already an element
1035  * equal to elem in the hash table, then the already existing element
1036  * will be returned and the new element will not be inserted.
1037  * Otherwise returns NULL.
1038  * If lockp == NULL, the caller is assumed to already hold the hash lock.
1039  */
1040 static arc_buf_hdr_t *
1041 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1042 {
1043 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1044 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1045 	arc_buf_hdr_t *fhdr;
1046 	uint32_t i;
1047 
1048 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1049 	ASSERT(hdr->b_birth != 0);
1050 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1051 
1052 	if (lockp != NULL) {
1053 		*lockp = hash_lock;
1054 		mutex_enter(hash_lock);
1055 	} else {
1056 		ASSERT(MUTEX_HELD(hash_lock));
1057 	}
1058 
1059 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1060 	    fhdr = fhdr->b_hash_next, i++) {
1061 		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1062 			return (fhdr);
1063 	}
1064 
1065 	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1066 	buf_hash_table.ht_table[idx] = hdr;
1067 	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1068 
1069 	/* collect some hash table performance data */
1070 	if (i > 0) {
1071 		ARCSTAT_BUMP(arcstat_hash_collisions);
1072 		if (i == 1)
1073 			ARCSTAT_BUMP(arcstat_hash_chains);
1074 
1075 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1076 	}
1077 	uint64_t he = atomic_inc_64_nv(
1078 	    &arc_stats.arcstat_hash_elements.value.ui64);
1079 	ARCSTAT_MAX(arcstat_hash_elements_max, he);
1080 
1081 	return (NULL);
1082 }
1083 
1084 static void
1085 buf_hash_remove(arc_buf_hdr_t *hdr)
1086 {
1087 	arc_buf_hdr_t *fhdr, **hdrp;
1088 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1089 
1090 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1091 	ASSERT(HDR_IN_HASH_TABLE(hdr));
1092 
1093 	hdrp = &buf_hash_table.ht_table[idx];
1094 	while ((fhdr = *hdrp) != hdr) {
1095 		ASSERT3P(fhdr, !=, NULL);
1096 		hdrp = &fhdr->b_hash_next;
1097 	}
1098 	*hdrp = hdr->b_hash_next;
1099 	hdr->b_hash_next = NULL;
1100 	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1101 
1102 	/* collect some hash table performance data */
1103 	atomic_dec_64(&arc_stats.arcstat_hash_elements.value.ui64);
1104 
1105 	if (buf_hash_table.ht_table[idx] &&
1106 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1107 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1108 }
1109 
1110 /*
1111  * Global data structures and functions for the buf kmem cache.
1112  */
1113 
1114 static kmem_cache_t *hdr_full_cache;
1115 static kmem_cache_t *hdr_l2only_cache;
1116 static kmem_cache_t *buf_cache;
1117 
1118 static void
1119 buf_fini(void)
1120 {
1121 #if defined(_KERNEL)
1122 	/*
1123 	 * Large allocations which do not require contiguous pages
1124 	 * should be using vmem_free() in the linux kernel\
1125 	 */
1126 	vmem_free(buf_hash_table.ht_table,
1127 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1128 #else
1129 	kmem_free(buf_hash_table.ht_table,
1130 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1131 #endif
1132 	for (int i = 0; i < BUF_LOCKS; i++)
1133 		mutex_destroy(BUF_HASH_LOCK(i));
1134 	kmem_cache_destroy(hdr_full_cache);
1135 	kmem_cache_destroy(hdr_l2only_cache);
1136 	kmem_cache_destroy(buf_cache);
1137 }
1138 
1139 /*
1140  * Constructor callback - called when the cache is empty
1141  * and a new buf is requested.
1142  */
1143 static int
1144 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1145 {
1146 	(void) unused, (void) kmflag;
1147 	arc_buf_hdr_t *hdr = vbuf;
1148 
1149 	memset(hdr, 0, HDR_FULL_SIZE);
1150 	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
1151 	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1152 #ifdef ZFS_DEBUG
1153 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1154 #endif
1155 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1156 	list_link_init(&hdr->b_l2hdr.b_l2node);
1157 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1158 
1159 	return (0);
1160 }
1161 
1162 static int
1163 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1164 {
1165 	(void) unused, (void) kmflag;
1166 	arc_buf_hdr_t *hdr = vbuf;
1167 
1168 	memset(hdr, 0, HDR_L2ONLY_SIZE);
1169 	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1170 
1171 	return (0);
1172 }
1173 
1174 static int
1175 buf_cons(void *vbuf, void *unused, int kmflag)
1176 {
1177 	(void) unused, (void) kmflag;
1178 	arc_buf_t *buf = vbuf;
1179 
1180 	memset(buf, 0, sizeof (arc_buf_t));
1181 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1182 
1183 	return (0);
1184 }
1185 
1186 /*
1187  * Destructor callback - called when a cached buf is
1188  * no longer required.
1189  */
1190 static void
1191 hdr_full_dest(void *vbuf, void *unused)
1192 {
1193 	(void) unused;
1194 	arc_buf_hdr_t *hdr = vbuf;
1195 
1196 	ASSERT(HDR_EMPTY(hdr));
1197 	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1198 #ifdef ZFS_DEBUG
1199 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1200 #endif
1201 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1202 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1203 }
1204 
1205 static void
1206 hdr_l2only_dest(void *vbuf, void *unused)
1207 {
1208 	(void) unused;
1209 	arc_buf_hdr_t *hdr = vbuf;
1210 
1211 	ASSERT(HDR_EMPTY(hdr));
1212 	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1213 }
1214 
1215 static void
1216 buf_dest(void *vbuf, void *unused)
1217 {
1218 	(void) unused;
1219 	(void) vbuf;
1220 
1221 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1222 }
1223 
1224 static void
1225 buf_init(void)
1226 {
1227 	uint64_t *ct = NULL;
1228 	uint64_t hsize = 1ULL << 12;
1229 	int i, j;
1230 
1231 	/*
1232 	 * The hash table is big enough to fill all of physical memory
1233 	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1234 	 * By default, the table will take up
1235 	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1236 	 */
1237 	while (hsize * zfs_arc_average_blocksize < arc_all_memory())
1238 		hsize <<= 1;
1239 retry:
1240 	buf_hash_table.ht_mask = hsize - 1;
1241 #if defined(_KERNEL)
1242 	/*
1243 	 * Large allocations which do not require contiguous pages
1244 	 * should be using vmem_alloc() in the linux kernel
1245 	 */
1246 	buf_hash_table.ht_table =
1247 	    vmem_zalloc(hsize * sizeof (void*), KM_SLEEP);
1248 #else
1249 	buf_hash_table.ht_table =
1250 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1251 #endif
1252 	if (buf_hash_table.ht_table == NULL) {
1253 		ASSERT(hsize > (1ULL << 8));
1254 		hsize >>= 1;
1255 		goto retry;
1256 	}
1257 
1258 	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1259 	    0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, 0);
1260 	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1261 	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL,
1262 	    NULL, NULL, 0);
1263 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1264 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1265 
1266 	for (i = 0; i < 256; i++)
1267 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1268 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1269 
1270 	for (i = 0; i < BUF_LOCKS; i++)
1271 		mutex_init(BUF_HASH_LOCK(i), NULL, MUTEX_DEFAULT, NULL);
1272 }
1273 
1274 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
1275 
1276 /*
1277  * This is the size that the buf occupies in memory. If the buf is compressed,
1278  * it will correspond to the compressed size. You should use this method of
1279  * getting the buf size unless you explicitly need the logical size.
1280  */
1281 uint64_t
1282 arc_buf_size(arc_buf_t *buf)
1283 {
1284 	return (ARC_BUF_COMPRESSED(buf) ?
1285 	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1286 }
1287 
1288 uint64_t
1289 arc_buf_lsize(arc_buf_t *buf)
1290 {
1291 	return (HDR_GET_LSIZE(buf->b_hdr));
1292 }
1293 
1294 /*
1295  * This function will return B_TRUE if the buffer is encrypted in memory.
1296  * This buffer can be decrypted by calling arc_untransform().
1297  */
1298 boolean_t
1299 arc_is_encrypted(arc_buf_t *buf)
1300 {
1301 	return (ARC_BUF_ENCRYPTED(buf) != 0);
1302 }
1303 
1304 /*
1305  * Returns B_TRUE if the buffer represents data that has not had its MAC
1306  * verified yet.
1307  */
1308 boolean_t
1309 arc_is_unauthenticated(arc_buf_t *buf)
1310 {
1311 	return (HDR_NOAUTH(buf->b_hdr) != 0);
1312 }
1313 
1314 void
1315 arc_get_raw_params(arc_buf_t *buf, boolean_t *byteorder, uint8_t *salt,
1316     uint8_t *iv, uint8_t *mac)
1317 {
1318 	arc_buf_hdr_t *hdr = buf->b_hdr;
1319 
1320 	ASSERT(HDR_PROTECTED(hdr));
1321 
1322 	memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
1323 	memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
1324 	memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
1325 	*byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
1326 	    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
1327 }
1328 
1329 /*
1330  * Indicates how this buffer is compressed in memory. If it is not compressed
1331  * the value will be ZIO_COMPRESS_OFF. It can be made normally readable with
1332  * arc_untransform() as long as it is also unencrypted.
1333  */
1334 enum zio_compress
1335 arc_get_compression(arc_buf_t *buf)
1336 {
1337 	return (ARC_BUF_COMPRESSED(buf) ?
1338 	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1339 }
1340 
1341 /*
1342  * Return the compression algorithm used to store this data in the ARC. If ARC
1343  * compression is enabled or this is an encrypted block, this will be the same
1344  * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
1345  */
1346 static inline enum zio_compress
1347 arc_hdr_get_compress(arc_buf_hdr_t *hdr)
1348 {
1349 	return (HDR_COMPRESSION_ENABLED(hdr) ?
1350 	    HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
1351 }
1352 
1353 uint8_t
1354 arc_get_complevel(arc_buf_t *buf)
1355 {
1356 	return (buf->b_hdr->b_complevel);
1357 }
1358 
1359 static inline boolean_t
1360 arc_buf_is_shared(arc_buf_t *buf)
1361 {
1362 	boolean_t shared = (buf->b_data != NULL &&
1363 	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1364 	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1365 	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1366 	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1367 	IMPLY(shared, ARC_BUF_SHARED(buf));
1368 	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1369 
1370 	/*
1371 	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1372 	 * already being shared" requirement prevents us from doing that.
1373 	 */
1374 
1375 	return (shared);
1376 }
1377 
1378 /*
1379  * Free the checksum associated with this header. If there is no checksum, this
1380  * is a no-op.
1381  */
1382 static inline void
1383 arc_cksum_free(arc_buf_hdr_t *hdr)
1384 {
1385 #ifdef ZFS_DEBUG
1386 	ASSERT(HDR_HAS_L1HDR(hdr));
1387 
1388 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1389 	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1390 		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1391 		hdr->b_l1hdr.b_freeze_cksum = NULL;
1392 	}
1393 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1394 #endif
1395 }
1396 
1397 /*
1398  * Return true iff at least one of the bufs on hdr is not compressed.
1399  * Encrypted buffers count as compressed.
1400  */
1401 static boolean_t
1402 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1403 {
1404 	ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
1405 
1406 	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1407 		if (!ARC_BUF_COMPRESSED(b)) {
1408 			return (B_TRUE);
1409 		}
1410 	}
1411 	return (B_FALSE);
1412 }
1413 
1414 
1415 /*
1416  * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1417  * matches the checksum that is stored in the hdr. If there is no checksum,
1418  * or if the buf is compressed, this is a no-op.
1419  */
1420 static void
1421 arc_cksum_verify(arc_buf_t *buf)
1422 {
1423 #ifdef ZFS_DEBUG
1424 	arc_buf_hdr_t *hdr = buf->b_hdr;
1425 	zio_cksum_t zc;
1426 
1427 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1428 		return;
1429 
1430 	if (ARC_BUF_COMPRESSED(buf))
1431 		return;
1432 
1433 	ASSERT(HDR_HAS_L1HDR(hdr));
1434 
1435 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1436 
1437 	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1438 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1439 		return;
1440 	}
1441 
1442 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1443 	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1444 		panic("buffer modified while frozen!");
1445 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1446 #endif
1447 }
1448 
1449 /*
1450  * This function makes the assumption that data stored in the L2ARC
1451  * will be transformed exactly as it is in the main pool. Because of
1452  * this we can verify the checksum against the reading process's bp.
1453  */
1454 static boolean_t
1455 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1456 {
1457 	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1458 	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1459 
1460 	/*
1461 	 * Block pointers always store the checksum for the logical data.
1462 	 * If the block pointer has the gang bit set, then the checksum
1463 	 * it represents is for the reconstituted data and not for an
1464 	 * individual gang member. The zio pipeline, however, must be able to
1465 	 * determine the checksum of each of the gang constituents so it
1466 	 * treats the checksum comparison differently than what we need
1467 	 * for l2arc blocks. This prevents us from using the
1468 	 * zio_checksum_error() interface directly. Instead we must call the
1469 	 * zio_checksum_error_impl() so that we can ensure the checksum is
1470 	 * generated using the correct checksum algorithm and accounts for the
1471 	 * logical I/O size and not just a gang fragment.
1472 	 */
1473 	return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1474 	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1475 	    zio->io_offset, NULL) == 0);
1476 }
1477 
1478 /*
1479  * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
1480  * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1481  * isn't modified later on. If buf is compressed or there is already a checksum
1482  * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1483  */
1484 static void
1485 arc_cksum_compute(arc_buf_t *buf)
1486 {
1487 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1488 		return;
1489 
1490 #ifdef ZFS_DEBUG
1491 	arc_buf_hdr_t *hdr = buf->b_hdr;
1492 	ASSERT(HDR_HAS_L1HDR(hdr));
1493 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1494 	if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
1495 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1496 		return;
1497 	}
1498 
1499 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
1500 	ASSERT(!ARC_BUF_COMPRESSED(buf));
1501 	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1502 	    KM_SLEEP);
1503 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1504 	    hdr->b_l1hdr.b_freeze_cksum);
1505 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1506 #endif
1507 	arc_buf_watch(buf);
1508 }
1509 
1510 #ifndef _KERNEL
1511 void
1512 arc_buf_sigsegv(int sig, siginfo_t *si, void *unused)
1513 {
1514 	(void) sig, (void) unused;
1515 	panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
1516 }
1517 #endif
1518 
1519 static void
1520 arc_buf_unwatch(arc_buf_t *buf)
1521 {
1522 #ifndef _KERNEL
1523 	if (arc_watch) {
1524 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1525 		    PROT_READ | PROT_WRITE));
1526 	}
1527 #else
1528 	(void) buf;
1529 #endif
1530 }
1531 
1532 static void
1533 arc_buf_watch(arc_buf_t *buf)
1534 {
1535 #ifndef _KERNEL
1536 	if (arc_watch)
1537 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1538 		    PROT_READ));
1539 #else
1540 	(void) buf;
1541 #endif
1542 }
1543 
1544 static arc_buf_contents_t
1545 arc_buf_type(arc_buf_hdr_t *hdr)
1546 {
1547 	arc_buf_contents_t type;
1548 	if (HDR_ISTYPE_METADATA(hdr)) {
1549 		type = ARC_BUFC_METADATA;
1550 	} else {
1551 		type = ARC_BUFC_DATA;
1552 	}
1553 	VERIFY3U(hdr->b_type, ==, type);
1554 	return (type);
1555 }
1556 
1557 boolean_t
1558 arc_is_metadata(arc_buf_t *buf)
1559 {
1560 	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
1561 }
1562 
1563 static uint32_t
1564 arc_bufc_to_flags(arc_buf_contents_t type)
1565 {
1566 	switch (type) {
1567 	case ARC_BUFC_DATA:
1568 		/* metadata field is 0 if buffer contains normal data */
1569 		return (0);
1570 	case ARC_BUFC_METADATA:
1571 		return (ARC_FLAG_BUFC_METADATA);
1572 	default:
1573 		break;
1574 	}
1575 	panic("undefined ARC buffer type!");
1576 	return ((uint32_t)-1);
1577 }
1578 
1579 void
1580 arc_buf_thaw(arc_buf_t *buf)
1581 {
1582 	arc_buf_hdr_t *hdr = buf->b_hdr;
1583 
1584 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1585 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1586 
1587 	arc_cksum_verify(buf);
1588 
1589 	/*
1590 	 * Compressed buffers do not manipulate the b_freeze_cksum.
1591 	 */
1592 	if (ARC_BUF_COMPRESSED(buf))
1593 		return;
1594 
1595 	ASSERT(HDR_HAS_L1HDR(hdr));
1596 	arc_cksum_free(hdr);
1597 	arc_buf_unwatch(buf);
1598 }
1599 
1600 void
1601 arc_buf_freeze(arc_buf_t *buf)
1602 {
1603 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1604 		return;
1605 
1606 	if (ARC_BUF_COMPRESSED(buf))
1607 		return;
1608 
1609 	ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
1610 	arc_cksum_compute(buf);
1611 }
1612 
1613 /*
1614  * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
1615  * the following functions should be used to ensure that the flags are
1616  * updated in a thread-safe way. When manipulating the flags either
1617  * the hash_lock must be held or the hdr must be undiscoverable. This
1618  * ensures that we're not racing with any other threads when updating
1619  * the flags.
1620  */
1621 static inline void
1622 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1623 {
1624 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1625 	hdr->b_flags |= flags;
1626 }
1627 
1628 static inline void
1629 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1630 {
1631 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1632 	hdr->b_flags &= ~flags;
1633 }
1634 
1635 /*
1636  * Setting the compression bits in the arc_buf_hdr_t's b_flags is
1637  * done in a special way since we have to clear and set bits
1638  * at the same time. Consumers that wish to set the compression bits
1639  * must use this function to ensure that the flags are updated in
1640  * thread-safe manner.
1641  */
1642 static void
1643 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
1644 {
1645 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1646 
1647 	/*
1648 	 * Holes and embedded blocks will always have a psize = 0 so
1649 	 * we ignore the compression of the blkptr and set the
1650 	 * want to uncompress them. Mark them as uncompressed.
1651 	 */
1652 	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
1653 		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1654 		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
1655 	} else {
1656 		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1657 		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
1658 	}
1659 
1660 	HDR_SET_COMPRESS(hdr, cmp);
1661 	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
1662 }
1663 
1664 /*
1665  * Looks for another buf on the same hdr which has the data decompressed, copies
1666  * from it, and returns true. If no such buf exists, returns false.
1667  */
1668 static boolean_t
1669 arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
1670 {
1671 	arc_buf_hdr_t *hdr = buf->b_hdr;
1672 	boolean_t copied = B_FALSE;
1673 
1674 	ASSERT(HDR_HAS_L1HDR(hdr));
1675 	ASSERT3P(buf->b_data, !=, NULL);
1676 	ASSERT(!ARC_BUF_COMPRESSED(buf));
1677 
1678 	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1679 	    from = from->b_next) {
1680 		/* can't use our own data buffer */
1681 		if (from == buf) {
1682 			continue;
1683 		}
1684 
1685 		if (!ARC_BUF_COMPRESSED(from)) {
1686 			memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
1687 			copied = B_TRUE;
1688 			break;
1689 		}
1690 	}
1691 
1692 #ifdef ZFS_DEBUG
1693 	/*
1694 	 * There were no decompressed bufs, so there should not be a
1695 	 * checksum on the hdr either.
1696 	 */
1697 	if (zfs_flags & ZFS_DEBUG_MODIFY)
1698 		EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1699 #endif
1700 
1701 	return (copied);
1702 }
1703 
1704 /*
1705  * Allocates an ARC buf header that's in an evicted & L2-cached state.
1706  * This is used during l2arc reconstruction to make empty ARC buffers
1707  * which circumvent the regular disk->arc->l2arc path and instead come
1708  * into being in the reverse order, i.e. l2arc->arc.
1709  */
1710 static arc_buf_hdr_t *
1711 arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
1712     dva_t dva, uint64_t daddr, int32_t psize, uint64_t birth,
1713     enum zio_compress compress, uint8_t complevel, boolean_t protected,
1714     boolean_t prefetch, arc_state_type_t arcs_state)
1715 {
1716 	arc_buf_hdr_t	*hdr;
1717 
1718 	ASSERT(size != 0);
1719 	hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
1720 	hdr->b_birth = birth;
1721 	hdr->b_type = type;
1722 	hdr->b_flags = 0;
1723 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);
1724 	HDR_SET_LSIZE(hdr, size);
1725 	HDR_SET_PSIZE(hdr, psize);
1726 	arc_hdr_set_compress(hdr, compress);
1727 	hdr->b_complevel = complevel;
1728 	if (protected)
1729 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
1730 	if (prefetch)
1731 		arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
1732 	hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
1733 
1734 	hdr->b_dva = dva;
1735 
1736 	hdr->b_l2hdr.b_dev = dev;
1737 	hdr->b_l2hdr.b_daddr = daddr;
1738 	hdr->b_l2hdr.b_arcs_state = arcs_state;
1739 
1740 	return (hdr);
1741 }
1742 
1743 /*
1744  * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
1745  */
1746 static uint64_t
1747 arc_hdr_size(arc_buf_hdr_t *hdr)
1748 {
1749 	uint64_t size;
1750 
1751 	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
1752 	    HDR_GET_PSIZE(hdr) > 0) {
1753 		size = HDR_GET_PSIZE(hdr);
1754 	} else {
1755 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
1756 		size = HDR_GET_LSIZE(hdr);
1757 	}
1758 	return (size);
1759 }
1760 
1761 static int
1762 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
1763 {
1764 	int ret;
1765 	uint64_t csize;
1766 	uint64_t lsize = HDR_GET_LSIZE(hdr);
1767 	uint64_t psize = HDR_GET_PSIZE(hdr);
1768 	void *tmpbuf = NULL;
1769 	abd_t *abd = hdr->b_l1hdr.b_pabd;
1770 
1771 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1772 	ASSERT(HDR_AUTHENTICATED(hdr));
1773 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1774 
1775 	/*
1776 	 * The MAC is calculated on the compressed data that is stored on disk.
1777 	 * However, if compressed arc is disabled we will only have the
1778 	 * decompressed data available to us now. Compress it into a temporary
1779 	 * abd so we can verify the MAC. The performance overhead of this will
1780 	 * be relatively low, since most objects in an encrypted objset will
1781 	 * be encrypted (instead of authenticated) anyway.
1782 	 */
1783 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1784 	    !HDR_COMPRESSION_ENABLED(hdr)) {
1785 
1786 		csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
1787 		    hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel);
1788 		ASSERT3P(tmpbuf, !=, NULL);
1789 		ASSERT3U(csize, <=, psize);
1790 		abd = abd_get_from_buf(tmpbuf, lsize);
1791 		abd_take_ownership_of_buf(abd, B_TRUE);
1792 		abd_zero_off(abd, csize, psize - csize);
1793 	}
1794 
1795 	/*
1796 	 * Authentication is best effort. We authenticate whenever the key is
1797 	 * available. If we succeed we clear ARC_FLAG_NOAUTH.
1798 	 */
1799 	if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
1800 		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1801 		ASSERT3U(lsize, ==, psize);
1802 		ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd,
1803 		    psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1804 	} else {
1805 		ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize,
1806 		    hdr->b_crypt_hdr.b_mac);
1807 	}
1808 
1809 	if (ret == 0)
1810 		arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
1811 	else if (ret != ENOENT)
1812 		goto error;
1813 
1814 	if (tmpbuf != NULL)
1815 		abd_free(abd);
1816 
1817 	return (0);
1818 
1819 error:
1820 	if (tmpbuf != NULL)
1821 		abd_free(abd);
1822 
1823 	return (ret);
1824 }
1825 
1826 /*
1827  * This function will take a header that only has raw encrypted data in
1828  * b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in
1829  * b_l1hdr.b_pabd. If designated in the header flags, this function will
1830  * also decompress the data.
1831  */
1832 static int
1833 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
1834 {
1835 	int ret;
1836 	abd_t *cabd = NULL;
1837 	void *tmp = NULL;
1838 	boolean_t no_crypt = B_FALSE;
1839 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1840 
1841 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1842 	ASSERT(HDR_ENCRYPTED(hdr));
1843 
1844 	arc_hdr_alloc_abd(hdr, 0);
1845 
1846 	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
1847 	    B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
1848 	    hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
1849 	    hdr->b_crypt_hdr.b_rabd, &no_crypt);
1850 	if (ret != 0)
1851 		goto error;
1852 
1853 	if (no_crypt) {
1854 		abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
1855 		    HDR_GET_PSIZE(hdr));
1856 	}
1857 
1858 	/*
1859 	 * If this header has disabled arc compression but the b_pabd is
1860 	 * compressed after decrypting it, we need to decompress the newly
1861 	 * decrypted data.
1862 	 */
1863 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1864 	    !HDR_COMPRESSION_ENABLED(hdr)) {
1865 		/*
1866 		 * We want to make sure that we are correctly honoring the
1867 		 * zfs_abd_scatter_enabled setting, so we allocate an abd here
1868 		 * and then loan a buffer from it, rather than allocating a
1869 		 * linear buffer and wrapping it in an abd later.
1870 		 */
1871 		cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
1872 		tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
1873 
1874 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
1875 		    hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
1876 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
1877 		if (ret != 0) {
1878 			abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
1879 			goto error;
1880 		}
1881 
1882 		abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
1883 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
1884 		    arc_hdr_size(hdr), hdr);
1885 		hdr->b_l1hdr.b_pabd = cabd;
1886 	}
1887 
1888 	return (0);
1889 
1890 error:
1891 	arc_hdr_free_abd(hdr, B_FALSE);
1892 	if (cabd != NULL)
1893 		arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);
1894 
1895 	return (ret);
1896 }
1897 
1898 /*
1899  * This function is called during arc_buf_fill() to prepare the header's
1900  * abd plaintext pointer for use. This involves authenticated protected
1901  * data and decrypting encrypted data into the plaintext abd.
1902  */
1903 static int
1904 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
1905     const zbookmark_phys_t *zb, boolean_t noauth)
1906 {
1907 	int ret;
1908 
1909 	ASSERT(HDR_PROTECTED(hdr));
1910 
1911 	if (hash_lock != NULL)
1912 		mutex_enter(hash_lock);
1913 
1914 	if (HDR_NOAUTH(hdr) && !noauth) {
1915 		/*
1916 		 * The caller requested authenticated data but our data has
1917 		 * not been authenticated yet. Verify the MAC now if we can.
1918 		 */
1919 		ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
1920 		if (ret != 0)
1921 			goto error;
1922 	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
1923 		/*
1924 		 * If we only have the encrypted version of the data, but the
1925 		 * unencrypted version was requested we take this opportunity
1926 		 * to store the decrypted version in the header for future use.
1927 		 */
1928 		ret = arc_hdr_decrypt(hdr, spa, zb);
1929 		if (ret != 0)
1930 			goto error;
1931 	}
1932 
1933 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1934 
1935 	if (hash_lock != NULL)
1936 		mutex_exit(hash_lock);
1937 
1938 	return (0);
1939 
1940 error:
1941 	if (hash_lock != NULL)
1942 		mutex_exit(hash_lock);
1943 
1944 	return (ret);
1945 }
1946 
1947 /*
1948  * This function is used by the dbuf code to decrypt bonus buffers in place.
1949  * The dbuf code itself doesn't have any locking for decrypting a shared dnode
1950  * block, so we use the hash lock here to protect against concurrent calls to
1951  * arc_buf_fill().
1952  */
1953 static void
1954 arc_buf_untransform_in_place(arc_buf_t *buf)
1955 {
1956 	arc_buf_hdr_t *hdr = buf->b_hdr;
1957 
1958 	ASSERT(HDR_ENCRYPTED(hdr));
1959 	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
1960 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1961 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1962 
1963 	zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
1964 	    arc_buf_size(buf));
1965 	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
1966 	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
1967 }
1968 
1969 /*
1970  * Given a buf that has a data buffer attached to it, this function will
1971  * efficiently fill the buf with data of the specified compression setting from
1972  * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
1973  * are already sharing a data buf, no copy is performed.
1974  *
1975  * If the buf is marked as compressed but uncompressed data was requested, this
1976  * will allocate a new data buffer for the buf, remove that flag, and fill the
1977  * buf with uncompressed data. You can't request a compressed buf on a hdr with
1978  * uncompressed data, and (since we haven't added support for it yet) if you
1979  * want compressed data your buf must already be marked as compressed and have
1980  * the correct-sized data buffer.
1981  */
1982 static int
1983 arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
1984     arc_fill_flags_t flags)
1985 {
1986 	int error = 0;
1987 	arc_buf_hdr_t *hdr = buf->b_hdr;
1988 	boolean_t hdr_compressed =
1989 	    (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
1990 	boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0;
1991 	boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0;
1992 	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
1993 	kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);
1994 
1995 	ASSERT3P(buf->b_data, !=, NULL);
1996 	IMPLY(compressed, hdr_compressed || ARC_BUF_ENCRYPTED(buf));
1997 	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
1998 	IMPLY(encrypted, HDR_ENCRYPTED(hdr));
1999 	IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf));
2000 	IMPLY(encrypted, ARC_BUF_COMPRESSED(buf));
2001 	IMPLY(encrypted, !ARC_BUF_SHARED(buf));
2002 
2003 	/*
2004 	 * If the caller wanted encrypted data we just need to copy it from
2005 	 * b_rabd and potentially byteswap it. We won't be able to do any
2006 	 * further transforms on it.
2007 	 */
2008 	if (encrypted) {
2009 		ASSERT(HDR_HAS_RABD(hdr));
2010 		abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
2011 		    HDR_GET_PSIZE(hdr));
2012 		goto byteswap;
2013 	}
2014 
2015 	/*
2016 	 * Adjust encrypted and authenticated headers to accommodate
2017 	 * the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are
2018 	 * allowed to fail decryption due to keys not being loaded
2019 	 * without being marked as an IO error.
2020 	 */
2021 	if (HDR_PROTECTED(hdr)) {
2022 		error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
2023 		    zb, !!(flags & ARC_FILL_NOAUTH));
2024 		if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) {
2025 			return (error);
2026 		} else if (error != 0) {
2027 			if (hash_lock != NULL)
2028 				mutex_enter(hash_lock);
2029 			arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2030 			if (hash_lock != NULL)
2031 				mutex_exit(hash_lock);
2032 			return (error);
2033 		}
2034 	}
2035 
2036 	/*
2037 	 * There is a special case here for dnode blocks which are
2038 	 * decrypting their bonus buffers. These blocks may request to
2039 	 * be decrypted in-place. This is necessary because there may
2040 	 * be many dnodes pointing into this buffer and there is
2041 	 * currently no method to synchronize replacing the backing
2042 	 * b_data buffer and updating all of the pointers. Here we use
2043 	 * the hash lock to ensure there are no races. If the need
2044 	 * arises for other types to be decrypted in-place, they must
2045 	 * add handling here as well.
2046 	 */
2047 	if ((flags & ARC_FILL_IN_PLACE) != 0) {
2048 		ASSERT(!hdr_compressed);
2049 		ASSERT(!compressed);
2050 		ASSERT(!encrypted);
2051 
2052 		if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
2053 			ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
2054 
2055 			if (hash_lock != NULL)
2056 				mutex_enter(hash_lock);
2057 			arc_buf_untransform_in_place(buf);
2058 			if (hash_lock != NULL)
2059 				mutex_exit(hash_lock);
2060 
2061 			/* Compute the hdr's checksum if necessary */
2062 			arc_cksum_compute(buf);
2063 		}
2064 
2065 		return (0);
2066 	}
2067 
2068 	if (hdr_compressed == compressed) {
2069 		if (!arc_buf_is_shared(buf)) {
2070 			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2071 			    arc_buf_size(buf));
2072 		}
2073 	} else {
2074 		ASSERT(hdr_compressed);
2075 		ASSERT(!compressed);
2076 
2077 		/*
2078 		 * If the buf is sharing its data with the hdr, unlink it and
2079 		 * allocate a new data buffer for the buf.
2080 		 */
2081 		if (arc_buf_is_shared(buf)) {
2082 			ASSERT(ARC_BUF_COMPRESSED(buf));
2083 
2084 			/* We need to give the buf its own b_data */
2085 			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2086 			buf->b_data =
2087 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2088 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2089 
2090 			/* Previously overhead was 0; just add new overhead */
2091 			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2092 		} else if (ARC_BUF_COMPRESSED(buf)) {
2093 			/* We need to reallocate the buf's b_data */
2094 			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2095 			    buf);
2096 			buf->b_data =
2097 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2098 
2099 			/* We increased the size of b_data; update overhead */
2100 			ARCSTAT_INCR(arcstat_overhead_size,
2101 			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2102 		}
2103 
2104 		/*
2105 		 * Regardless of the buf's previous compression settings, it
2106 		 * should not be compressed at the end of this function.
2107 		 */
2108 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2109 
2110 		/*
2111 		 * Try copying the data from another buf which already has a
2112 		 * decompressed version. If that's not possible, it's time to
2113 		 * bite the bullet and decompress the data from the hdr.
2114 		 */
2115 		if (arc_buf_try_copy_decompressed_data(buf)) {
2116 			/* Skip byteswapping and checksumming (already done) */
2117 			return (0);
2118 		} else {
2119 			error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2120 			    hdr->b_l1hdr.b_pabd, buf->b_data,
2121 			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
2122 			    &hdr->b_complevel);
2123 
2124 			/*
2125 			 * Absent hardware errors or software bugs, this should
2126 			 * be impossible, but log it anyway so we can debug it.
2127 			 */
2128 			if (error != 0) {
2129 				zfs_dbgmsg(
2130 				    "hdr %px, compress %d, psize %d, lsize %d",
2131 				    hdr, arc_hdr_get_compress(hdr),
2132 				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2133 				if (hash_lock != NULL)
2134 					mutex_enter(hash_lock);
2135 				arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2136 				if (hash_lock != NULL)
2137 					mutex_exit(hash_lock);
2138 				return (SET_ERROR(EIO));
2139 			}
2140 		}
2141 	}
2142 
2143 byteswap:
2144 	/* Byteswap the buf's data if necessary */
2145 	if (bswap != DMU_BSWAP_NUMFUNCS) {
2146 		ASSERT(!HDR_SHARED_DATA(hdr));
2147 		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2148 		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2149 	}
2150 
2151 	/* Compute the hdr's checksum if necessary */
2152 	arc_cksum_compute(buf);
2153 
2154 	return (0);
2155 }
2156 
2157 /*
2158  * If this function is being called to decrypt an encrypted buffer or verify an
2159  * authenticated one, the key must be loaded and a mapping must be made
2160  * available in the keystore via spa_keystore_create_mapping() or one of its
2161  * callers.
2162  */
2163 int
2164 arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
2165     boolean_t in_place)
2166 {
2167 	int ret;
2168 	arc_fill_flags_t flags = 0;
2169 
2170 	if (in_place)
2171 		flags |= ARC_FILL_IN_PLACE;
2172 
2173 	ret = arc_buf_fill(buf, spa, zb, flags);
2174 	if (ret == ECKSUM) {
2175 		/*
2176 		 * Convert authentication and decryption errors to EIO
2177 		 * (and generate an ereport) before leaving the ARC.
2178 		 */
2179 		ret = SET_ERROR(EIO);
2180 		spa_log_error(spa, zb, &buf->b_hdr->b_birth);
2181 		(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
2182 		    spa, NULL, zb, NULL, 0);
2183 	}
2184 
2185 	return (ret);
2186 }
2187 
2188 /*
2189  * Increment the amount of evictable space in the arc_state_t's refcount.
2190  * We account for the space used by the hdr and the arc buf individually
2191  * so that we can add and remove them from the refcount individually.
2192  */
2193 static void
2194 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2195 {
2196 	arc_buf_contents_t type = arc_buf_type(hdr);
2197 
2198 	ASSERT(HDR_HAS_L1HDR(hdr));
2199 
2200 	if (GHOST_STATE(state)) {
2201 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2202 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2203 		ASSERT(!HDR_HAS_RABD(hdr));
2204 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2205 		    HDR_GET_LSIZE(hdr), hdr);
2206 		return;
2207 	}
2208 
2209 	if (hdr->b_l1hdr.b_pabd != NULL) {
2210 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2211 		    arc_hdr_size(hdr), hdr);
2212 	}
2213 	if (HDR_HAS_RABD(hdr)) {
2214 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2215 		    HDR_GET_PSIZE(hdr), hdr);
2216 	}
2217 
2218 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2219 	    buf = buf->b_next) {
2220 		if (arc_buf_is_shared(buf))
2221 			continue;
2222 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2223 		    arc_buf_size(buf), buf);
2224 	}
2225 }
2226 
2227 /*
2228  * Decrement the amount of evictable space in the arc_state_t's refcount.
2229  * We account for the space used by the hdr and the arc buf individually
2230  * so that we can add and remove them from the refcount individually.
2231  */
2232 static void
2233 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2234 {
2235 	arc_buf_contents_t type = arc_buf_type(hdr);
2236 
2237 	ASSERT(HDR_HAS_L1HDR(hdr));
2238 
2239 	if (GHOST_STATE(state)) {
2240 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2241 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2242 		ASSERT(!HDR_HAS_RABD(hdr));
2243 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2244 		    HDR_GET_LSIZE(hdr), hdr);
2245 		return;
2246 	}
2247 
2248 	if (hdr->b_l1hdr.b_pabd != NULL) {
2249 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2250 		    arc_hdr_size(hdr), hdr);
2251 	}
2252 	if (HDR_HAS_RABD(hdr)) {
2253 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2254 		    HDR_GET_PSIZE(hdr), hdr);
2255 	}
2256 
2257 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2258 	    buf = buf->b_next) {
2259 		if (arc_buf_is_shared(buf))
2260 			continue;
2261 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2262 		    arc_buf_size(buf), buf);
2263 	}
2264 }
2265 
2266 /*
2267  * Add a reference to this hdr indicating that someone is actively
2268  * referencing that memory. When the refcount transitions from 0 to 1,
2269  * we remove it from the respective arc_state_t list to indicate that
2270  * it is not evictable.
2271  */
2272 static void
2273 add_reference(arc_buf_hdr_t *hdr, const void *tag)
2274 {
2275 	arc_state_t *state = hdr->b_l1hdr.b_state;
2276 
2277 	ASSERT(HDR_HAS_L1HDR(hdr));
2278 	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
2279 		ASSERT(state == arc_anon);
2280 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2281 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2282 	}
2283 
2284 	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2285 	    state != arc_anon && state != arc_l2c_only) {
2286 		/* We don't use the L2-only state list. */
2287 		multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
2288 		arc_evictable_space_decrement(hdr, state);
2289 	}
2290 }
2291 
2292 /*
2293  * Remove a reference from this hdr. When the reference transitions from
2294  * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2295  * list making it eligible for eviction.
2296  */
2297 static int
2298 remove_reference(arc_buf_hdr_t *hdr, const void *tag)
2299 {
2300 	int cnt;
2301 	arc_state_t *state = hdr->b_l1hdr.b_state;
2302 
2303 	ASSERT(HDR_HAS_L1HDR(hdr));
2304 	ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));
2305 	ASSERT(!GHOST_STATE(state));	/* arc_l2c_only counts as a ghost. */
2306 
2307 	if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
2308 		return (cnt);
2309 
2310 	if (state == arc_anon) {
2311 		arc_hdr_destroy(hdr);
2312 		return (0);
2313 	}
2314 	if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
2315 		arc_change_state(arc_anon, hdr);
2316 		arc_hdr_destroy(hdr);
2317 		return (0);
2318 	}
2319 	multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2320 	arc_evictable_space_increment(hdr, state);
2321 	return (0);
2322 }
2323 
2324 /*
2325  * Returns detailed information about a specific arc buffer.  When the
2326  * state_index argument is set the function will calculate the arc header
2327  * list position for its arc state.  Since this requires a linear traversal
2328  * callers are strongly encourage not to do this.  However, it can be helpful
2329  * for targeted analysis so the functionality is provided.
2330  */
2331 void
2332 arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
2333 {
2334 	(void) state_index;
2335 	arc_buf_hdr_t *hdr = ab->b_hdr;
2336 	l1arc_buf_hdr_t *l1hdr = NULL;
2337 	l2arc_buf_hdr_t *l2hdr = NULL;
2338 	arc_state_t *state = NULL;
2339 
2340 	memset(abi, 0, sizeof (arc_buf_info_t));
2341 
2342 	if (hdr == NULL)
2343 		return;
2344 
2345 	abi->abi_flags = hdr->b_flags;
2346 
2347 	if (HDR_HAS_L1HDR(hdr)) {
2348 		l1hdr = &hdr->b_l1hdr;
2349 		state = l1hdr->b_state;
2350 	}
2351 	if (HDR_HAS_L2HDR(hdr))
2352 		l2hdr = &hdr->b_l2hdr;
2353 
2354 	if (l1hdr) {
2355 		abi->abi_bufcnt = 0;
2356 		for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
2357 			abi->abi_bufcnt++;
2358 		abi->abi_access = l1hdr->b_arc_access;
2359 		abi->abi_mru_hits = l1hdr->b_mru_hits;
2360 		abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2361 		abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2362 		abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2363 		abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
2364 	}
2365 
2366 	if (l2hdr) {
2367 		abi->abi_l2arc_dattr = l2hdr->b_daddr;
2368 		abi->abi_l2arc_hits = l2hdr->b_hits;
2369 	}
2370 
2371 	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2372 	abi->abi_state_contents = arc_buf_type(hdr);
2373 	abi->abi_size = arc_hdr_size(hdr);
2374 }
2375 
2376 /*
2377  * Move the supplied buffer to the indicated state. The hash lock
2378  * for the buffer must be held by the caller.
2379  */
2380 static void
2381 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
2382 {
2383 	arc_state_t *old_state;
2384 	int64_t refcnt;
2385 	boolean_t update_old, update_new;
2386 	arc_buf_contents_t type = arc_buf_type(hdr);
2387 
2388 	/*
2389 	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2390 	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2391 	 * L1 hdr doesn't always exist when we change state to arc_anon before
2392 	 * destroying a header, in which case reallocating to add the L1 hdr is
2393 	 * pointless.
2394 	 */
2395 	if (HDR_HAS_L1HDR(hdr)) {
2396 		old_state = hdr->b_l1hdr.b_state;
2397 		refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2398 		update_old = (hdr->b_l1hdr.b_buf != NULL ||
2399 		    hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
2400 
2401 		IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
2402 		IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
2403 		IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
2404 		    ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
2405 	} else {
2406 		old_state = arc_l2c_only;
2407 		refcnt = 0;
2408 		update_old = B_FALSE;
2409 	}
2410 	update_new = update_old;
2411 	if (GHOST_STATE(old_state))
2412 		update_old = B_TRUE;
2413 	if (GHOST_STATE(new_state))
2414 		update_new = B_TRUE;
2415 
2416 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2417 	ASSERT3P(new_state, !=, old_state);
2418 
2419 	/*
2420 	 * If this buffer is evictable, transfer it from the
2421 	 * old state list to the new state list.
2422 	 */
2423 	if (refcnt == 0) {
2424 		if (old_state != arc_anon && old_state != arc_l2c_only) {
2425 			ASSERT(HDR_HAS_L1HDR(hdr));
2426 			/* remove_reference() saves on insert. */
2427 			if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2428 				multilist_remove(&old_state->arcs_list[type],
2429 				    hdr);
2430 				arc_evictable_space_decrement(hdr, old_state);
2431 			}
2432 		}
2433 		if (new_state != arc_anon && new_state != arc_l2c_only) {
2434 			/*
2435 			 * An L1 header always exists here, since if we're
2436 			 * moving to some L1-cached state (i.e. not l2c_only or
2437 			 * anonymous), we realloc the header to add an L1hdr
2438 			 * beforehand.
2439 			 */
2440 			ASSERT(HDR_HAS_L1HDR(hdr));
2441 			multilist_insert(&new_state->arcs_list[type], hdr);
2442 			arc_evictable_space_increment(hdr, new_state);
2443 		}
2444 	}
2445 
2446 	ASSERT(!HDR_EMPTY(hdr));
2447 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2448 		buf_hash_remove(hdr);
2449 
2450 	/* adjust state sizes (ignore arc_l2c_only) */
2451 
2452 	if (update_new && new_state != arc_l2c_only) {
2453 		ASSERT(HDR_HAS_L1HDR(hdr));
2454 		if (GHOST_STATE(new_state)) {
2455 
2456 			/*
2457 			 * When moving a header to a ghost state, we first
2458 			 * remove all arc buffers. Thus, we'll have no arc
2459 			 * buffer to use for the reference. As a result, we
2460 			 * use the arc header pointer for the reference.
2461 			 */
2462 			(void) zfs_refcount_add_many(
2463 			    &new_state->arcs_size[type],
2464 			    HDR_GET_LSIZE(hdr), hdr);
2465 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2466 			ASSERT(!HDR_HAS_RABD(hdr));
2467 		} else {
2468 
2469 			/*
2470 			 * Each individual buffer holds a unique reference,
2471 			 * thus we must remove each of these references one
2472 			 * at a time.
2473 			 */
2474 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2475 			    buf = buf->b_next) {
2476 
2477 				/*
2478 				 * When the arc_buf_t is sharing the data
2479 				 * block with the hdr, the owner of the
2480 				 * reference belongs to the hdr. Only
2481 				 * add to the refcount if the arc_buf_t is
2482 				 * not shared.
2483 				 */
2484 				if (arc_buf_is_shared(buf))
2485 					continue;
2486 
2487 				(void) zfs_refcount_add_many(
2488 				    &new_state->arcs_size[type],
2489 				    arc_buf_size(buf), buf);
2490 			}
2491 
2492 			if (hdr->b_l1hdr.b_pabd != NULL) {
2493 				(void) zfs_refcount_add_many(
2494 				    &new_state->arcs_size[type],
2495 				    arc_hdr_size(hdr), hdr);
2496 			}
2497 
2498 			if (HDR_HAS_RABD(hdr)) {
2499 				(void) zfs_refcount_add_many(
2500 				    &new_state->arcs_size[type],
2501 				    HDR_GET_PSIZE(hdr), hdr);
2502 			}
2503 		}
2504 	}
2505 
2506 	if (update_old && old_state != arc_l2c_only) {
2507 		ASSERT(HDR_HAS_L1HDR(hdr));
2508 		if (GHOST_STATE(old_state)) {
2509 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2510 			ASSERT(!HDR_HAS_RABD(hdr));
2511 
2512 			/*
2513 			 * When moving a header off of a ghost state,
2514 			 * the header will not contain any arc buffers.
2515 			 * We use the arc header pointer for the reference
2516 			 * which is exactly what we did when we put the
2517 			 * header on the ghost state.
2518 			 */
2519 
2520 			(void) zfs_refcount_remove_many(
2521 			    &old_state->arcs_size[type],
2522 			    HDR_GET_LSIZE(hdr), hdr);
2523 		} else {
2524 
2525 			/*
2526 			 * Each individual buffer holds a unique reference,
2527 			 * thus we must remove each of these references one
2528 			 * at a time.
2529 			 */
2530 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2531 			    buf = buf->b_next) {
2532 
2533 				/*
2534 				 * When the arc_buf_t is sharing the data
2535 				 * block with the hdr, the owner of the
2536 				 * reference belongs to the hdr. Only
2537 				 * add to the refcount if the arc_buf_t is
2538 				 * not shared.
2539 				 */
2540 				if (arc_buf_is_shared(buf))
2541 					continue;
2542 
2543 				(void) zfs_refcount_remove_many(
2544 				    &old_state->arcs_size[type],
2545 				    arc_buf_size(buf), buf);
2546 			}
2547 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
2548 			    HDR_HAS_RABD(hdr));
2549 
2550 			if (hdr->b_l1hdr.b_pabd != NULL) {
2551 				(void) zfs_refcount_remove_many(
2552 				    &old_state->arcs_size[type],
2553 				    arc_hdr_size(hdr), hdr);
2554 			}
2555 
2556 			if (HDR_HAS_RABD(hdr)) {
2557 				(void) zfs_refcount_remove_many(
2558 				    &old_state->arcs_size[type],
2559 				    HDR_GET_PSIZE(hdr), hdr);
2560 			}
2561 		}
2562 	}
2563 
2564 	if (HDR_HAS_L1HDR(hdr)) {
2565 		hdr->b_l1hdr.b_state = new_state;
2566 
2567 		if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
2568 			l2arc_hdr_arcstats_decrement_state(hdr);
2569 			hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
2570 			l2arc_hdr_arcstats_increment_state(hdr);
2571 		}
2572 	}
2573 }
2574 
2575 void
2576 arc_space_consume(uint64_t space, arc_space_type_t type)
2577 {
2578 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2579 
2580 	switch (type) {
2581 	default:
2582 		break;
2583 	case ARC_SPACE_DATA:
2584 		ARCSTAT_INCR(arcstat_data_size, space);
2585 		break;
2586 	case ARC_SPACE_META:
2587 		ARCSTAT_INCR(arcstat_metadata_size, space);
2588 		break;
2589 	case ARC_SPACE_BONUS:
2590 		ARCSTAT_INCR(arcstat_bonus_size, space);
2591 		break;
2592 	case ARC_SPACE_DNODE:
2593 		ARCSTAT_INCR(arcstat_dnode_size, space);
2594 		break;
2595 	case ARC_SPACE_DBUF:
2596 		ARCSTAT_INCR(arcstat_dbuf_size, space);
2597 		break;
2598 	case ARC_SPACE_HDRS:
2599 		ARCSTAT_INCR(arcstat_hdr_size, space);
2600 		break;
2601 	case ARC_SPACE_L2HDRS:
2602 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
2603 		break;
2604 	case ARC_SPACE_ABD_CHUNK_WASTE:
2605 		/*
2606 		 * Note: this includes space wasted by all scatter ABD's, not
2607 		 * just those allocated by the ARC.  But the vast majority of
2608 		 * scatter ABD's come from the ARC, because other users are
2609 		 * very short-lived.
2610 		 */
2611 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
2612 		break;
2613 	}
2614 
2615 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2616 		ARCSTAT_INCR(arcstat_meta_used, space);
2617 
2618 	aggsum_add(&arc_sums.arcstat_size, space);
2619 }
2620 
2621 void
2622 arc_space_return(uint64_t space, arc_space_type_t type)
2623 {
2624 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2625 
2626 	switch (type) {
2627 	default:
2628 		break;
2629 	case ARC_SPACE_DATA:
2630 		ARCSTAT_INCR(arcstat_data_size, -space);
2631 		break;
2632 	case ARC_SPACE_META:
2633 		ARCSTAT_INCR(arcstat_metadata_size, -space);
2634 		break;
2635 	case ARC_SPACE_BONUS:
2636 		ARCSTAT_INCR(arcstat_bonus_size, -space);
2637 		break;
2638 	case ARC_SPACE_DNODE:
2639 		ARCSTAT_INCR(arcstat_dnode_size, -space);
2640 		break;
2641 	case ARC_SPACE_DBUF:
2642 		ARCSTAT_INCR(arcstat_dbuf_size, -space);
2643 		break;
2644 	case ARC_SPACE_HDRS:
2645 		ARCSTAT_INCR(arcstat_hdr_size, -space);
2646 		break;
2647 	case ARC_SPACE_L2HDRS:
2648 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
2649 		break;
2650 	case ARC_SPACE_ABD_CHUNK_WASTE:
2651 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
2652 		break;
2653 	}
2654 
2655 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2656 		ARCSTAT_INCR(arcstat_meta_used, -space);
2657 
2658 	ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
2659 	aggsum_add(&arc_sums.arcstat_size, -space);
2660 }
2661 
2662 /*
2663  * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2664  * with the hdr's b_pabd.
2665  */
2666 static boolean_t
2667 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2668 {
2669 	/*
2670 	 * The criteria for sharing a hdr's data are:
2671 	 * 1. the buffer is not encrypted
2672 	 * 2. the hdr's compression matches the buf's compression
2673 	 * 3. the hdr doesn't need to be byteswapped
2674 	 * 4. the hdr isn't already being shared
2675 	 * 5. the buf is either compressed or it is the last buf in the hdr list
2676 	 *
2677 	 * Criterion #5 maintains the invariant that shared uncompressed
2678 	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2679 	 * might ask, "if a compressed buf is allocated first, won't that be the
2680 	 * last thing in the list?", but in that case it's impossible to create
2681 	 * a shared uncompressed buf anyway (because the hdr must be compressed
2682 	 * to have the compressed buf). You might also think that #3 is
2683 	 * sufficient to make this guarantee, however it's possible
2684 	 * (specifically in the rare L2ARC write race mentioned in
2685 	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2686 	 * is shareable, but wasn't at the time of its allocation. Rather than
2687 	 * allow a new shared uncompressed buf to be created and then shuffle
2688 	 * the list around to make it the last element, this simply disallows
2689 	 * sharing if the new buf isn't the first to be added.
2690 	 */
2691 	ASSERT3P(buf->b_hdr, ==, hdr);
2692 	boolean_t hdr_compressed =
2693 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
2694 	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2695 	return (!ARC_BUF_ENCRYPTED(buf) &&
2696 	    buf_compressed == hdr_compressed &&
2697 	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2698 	    !HDR_SHARED_DATA(hdr) &&
2699 	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2700 }
2701 
2702 /*
2703  * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2704  * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2705  * copy was made successfully, or an error code otherwise.
2706  */
2707 static int
2708 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
2709     const void *tag, boolean_t encrypted, boolean_t compressed,
2710     boolean_t noauth, boolean_t fill, arc_buf_t **ret)
2711 {
2712 	arc_buf_t *buf;
2713 	arc_fill_flags_t flags = ARC_FILL_LOCKED;
2714 
2715 	ASSERT(HDR_HAS_L1HDR(hdr));
2716 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2717 	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2718 	    hdr->b_type == ARC_BUFC_METADATA);
2719 	ASSERT3P(ret, !=, NULL);
2720 	ASSERT3P(*ret, ==, NULL);
2721 	IMPLY(encrypted, compressed);
2722 
2723 	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2724 	buf->b_hdr = hdr;
2725 	buf->b_data = NULL;
2726 	buf->b_next = hdr->b_l1hdr.b_buf;
2727 	buf->b_flags = 0;
2728 
2729 	add_reference(hdr, tag);
2730 
2731 	/*
2732 	 * We're about to change the hdr's b_flags. We must either
2733 	 * hold the hash_lock or be undiscoverable.
2734 	 */
2735 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2736 
2737 	/*
2738 	 * Only honor requests for compressed bufs if the hdr is actually
2739 	 * compressed. This must be overridden if the buffer is encrypted since
2740 	 * encrypted buffers cannot be decompressed.
2741 	 */
2742 	if (encrypted) {
2743 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2744 		buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED;
2745 		flags |= ARC_FILL_COMPRESSED | ARC_FILL_ENCRYPTED;
2746 	} else if (compressed &&
2747 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
2748 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2749 		flags |= ARC_FILL_COMPRESSED;
2750 	}
2751 
2752 	if (noauth) {
2753 		ASSERT0(encrypted);
2754 		flags |= ARC_FILL_NOAUTH;
2755 	}
2756 
2757 	/*
2758 	 * If the hdr's data can be shared then we share the data buffer and
2759 	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2760 	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2761 	 * buffer to store the buf's data.
2762 	 *
2763 	 * There are two additional restrictions here because we're sharing
2764 	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2765 	 * actively involved in an L2ARC write, because if this buf is used by
2766 	 * an arc_write() then the hdr's data buffer will be released when the
2767 	 * write completes, even though the L2ARC write might still be using it.
2768 	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2769 	 * need to be ABD-aware.  It must be allocated via
2770 	 * zio_[data_]buf_alloc(), not as a page, because we need to be able
2771 	 * to abd_release_ownership_of_buf(), which isn't allowed on "linear
2772 	 * page" buffers because the ABD code needs to handle freeing them
2773 	 * specially.
2774 	 */
2775 	boolean_t can_share = arc_can_share(hdr, buf) &&
2776 	    !HDR_L2_WRITING(hdr) &&
2777 	    hdr->b_l1hdr.b_pabd != NULL &&
2778 	    abd_is_linear(hdr->b_l1hdr.b_pabd) &&
2779 	    !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
2780 
2781 	/* Set up b_data and sharing */
2782 	if (can_share) {
2783 		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2784 		buf->b_flags |= ARC_BUF_FLAG_SHARED;
2785 		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2786 	} else {
2787 		buf->b_data =
2788 		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2789 		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2790 	}
2791 	VERIFY3P(buf->b_data, !=, NULL);
2792 
2793 	hdr->b_l1hdr.b_buf = buf;
2794 
2795 	/*
2796 	 * If the user wants the data from the hdr, we need to either copy or
2797 	 * decompress the data.
2798 	 */
2799 	if (fill) {
2800 		ASSERT3P(zb, !=, NULL);
2801 		return (arc_buf_fill(buf, spa, zb, flags));
2802 	}
2803 
2804 	return (0);
2805 }
2806 
2807 static const char *arc_onloan_tag = "onloan";
2808 
2809 static inline void
2810 arc_loaned_bytes_update(int64_t delta)
2811 {
2812 	atomic_add_64(&arc_loaned_bytes, delta);
2813 
2814 	/* assert that it did not wrap around */
2815 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
2816 }
2817 
2818 /*
2819  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
2820  * flight data by arc_tempreserve_space() until they are "returned". Loaned
2821  * buffers must be returned to the arc before they can be used by the DMU or
2822  * freed.
2823  */
2824 arc_buf_t *
2825 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
2826 {
2827 	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
2828 	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
2829 
2830 	arc_loaned_bytes_update(arc_buf_size(buf));
2831 
2832 	return (buf);
2833 }
2834 
2835 arc_buf_t *
2836 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
2837     enum zio_compress compression_type, uint8_t complevel)
2838 {
2839 	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
2840 	    psize, lsize, compression_type, complevel);
2841 
2842 	arc_loaned_bytes_update(arc_buf_size(buf));
2843 
2844 	return (buf);
2845 }
2846 
2847 arc_buf_t *
2848 arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
2849     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
2850     dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
2851     enum zio_compress compression_type, uint8_t complevel)
2852 {
2853 	arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
2854 	    byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
2855 	    complevel);
2856 
2857 	atomic_add_64(&arc_loaned_bytes, psize);
2858 	return (buf);
2859 }
2860 
2861 
2862 /*
2863  * Return a loaned arc buffer to the arc.
2864  */
2865 void
2866 arc_return_buf(arc_buf_t *buf, const void *tag)
2867 {
2868 	arc_buf_hdr_t *hdr = buf->b_hdr;
2869 
2870 	ASSERT3P(buf->b_data, !=, NULL);
2871 	ASSERT(HDR_HAS_L1HDR(hdr));
2872 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2873 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2874 
2875 	arc_loaned_bytes_update(-arc_buf_size(buf));
2876 }
2877 
2878 /* Detach an arc_buf from a dbuf (tag) */
2879 void
2880 arc_loan_inuse_buf(arc_buf_t *buf, const void *tag)
2881 {
2882 	arc_buf_hdr_t *hdr = buf->b_hdr;
2883 
2884 	ASSERT3P(buf->b_data, !=, NULL);
2885 	ASSERT(HDR_HAS_L1HDR(hdr));
2886 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2887 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2888 
2889 	arc_loaned_bytes_update(arc_buf_size(buf));
2890 }
2891 
2892 static void
2893 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
2894 {
2895 	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
2896 
2897 	df->l2df_abd = abd;
2898 	df->l2df_size = size;
2899 	df->l2df_type = type;
2900 	mutex_enter(&l2arc_free_on_write_mtx);
2901 	list_insert_head(l2arc_free_on_write, df);
2902 	mutex_exit(&l2arc_free_on_write_mtx);
2903 }
2904 
2905 static void
2906 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
2907 {
2908 	arc_state_t *state = hdr->b_l1hdr.b_state;
2909 	arc_buf_contents_t type = arc_buf_type(hdr);
2910 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
2911 
2912 	/* protected by hash lock, if in the hash table */
2913 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2914 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2915 		ASSERT(state != arc_anon && state != arc_l2c_only);
2916 
2917 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2918 		    size, hdr);
2919 	}
2920 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
2921 	if (type == ARC_BUFC_METADATA) {
2922 		arc_space_return(size, ARC_SPACE_META);
2923 	} else {
2924 		ASSERT(type == ARC_BUFC_DATA);
2925 		arc_space_return(size, ARC_SPACE_DATA);
2926 	}
2927 
2928 	if (free_rdata) {
2929 		l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
2930 	} else {
2931 		l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2932 	}
2933 }
2934 
2935 /*
2936  * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2937  * data buffer, we transfer the refcount ownership to the hdr and update
2938  * the appropriate kstats.
2939  */
2940 static void
2941 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2942 {
2943 	ASSERT(arc_can_share(hdr, buf));
2944 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2945 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
2946 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2947 
2948 	/*
2949 	 * Start sharing the data buffer. We transfer the
2950 	 * refcount ownership to the hdr since it always owns
2951 	 * the refcount whenever an arc_buf_t is shared.
2952 	 */
2953 	zfs_refcount_transfer_ownership_many(
2954 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2955 	    arc_hdr_size(hdr), buf, hdr);
2956 	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2957 	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2958 	    HDR_ISTYPE_METADATA(hdr));
2959 	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2960 	buf->b_flags |= ARC_BUF_FLAG_SHARED;
2961 
2962 	/*
2963 	 * Since we've transferred ownership to the hdr we need
2964 	 * to increment its compressed and uncompressed kstats and
2965 	 * decrement the overhead size.
2966 	 */
2967 	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2968 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2969 	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
2970 }
2971 
2972 static void
2973 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2974 {
2975 	ASSERT(arc_buf_is_shared(buf));
2976 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
2977 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2978 
2979 	/*
2980 	 * We are no longer sharing this buffer so we need
2981 	 * to transfer its ownership to the rightful owner.
2982 	 */
2983 	zfs_refcount_transfer_ownership_many(
2984 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2985 	    arc_hdr_size(hdr), hdr, buf);
2986 	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2987 	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
2988 	abd_free(hdr->b_l1hdr.b_pabd);
2989 	hdr->b_l1hdr.b_pabd = NULL;
2990 	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2991 
2992 	/*
2993 	 * Since the buffer is no longer shared between
2994 	 * the arc buf and the hdr, count it as overhead.
2995 	 */
2996 	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2997 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2998 	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2999 }
3000 
3001 /*
3002  * Remove an arc_buf_t from the hdr's buf list and return the last
3003  * arc_buf_t on the list. If no buffers remain on the list then return
3004  * NULL.
3005  */
3006 static arc_buf_t *
3007 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3008 {
3009 	ASSERT(HDR_HAS_L1HDR(hdr));
3010 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3011 
3012 	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3013 	arc_buf_t *lastbuf = NULL;
3014 
3015 	/*
3016 	 * Remove the buf from the hdr list and locate the last
3017 	 * remaining buffer on the list.
3018 	 */
3019 	while (*bufp != NULL) {
3020 		if (*bufp == buf)
3021 			*bufp = buf->b_next;
3022 
3023 		/*
3024 		 * If we've removed a buffer in the middle of
3025 		 * the list then update the lastbuf and update
3026 		 * bufp.
3027 		 */
3028 		if (*bufp != NULL) {
3029 			lastbuf = *bufp;
3030 			bufp = &(*bufp)->b_next;
3031 		}
3032 	}
3033 	buf->b_next = NULL;
3034 	ASSERT3P(lastbuf, !=, buf);
3035 	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3036 
3037 	return (lastbuf);
3038 }
3039 
3040 /*
3041  * Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
3042  * list and free it.
3043  */
3044 static void
3045 arc_buf_destroy_impl(arc_buf_t *buf)
3046 {
3047 	arc_buf_hdr_t *hdr = buf->b_hdr;
3048 
3049 	/*
3050 	 * Free up the data associated with the buf but only if we're not
3051 	 * sharing this with the hdr. If we are sharing it with the hdr, the
3052 	 * hdr is responsible for doing the free.
3053 	 */
3054 	if (buf->b_data != NULL) {
3055 		/*
3056 		 * We're about to change the hdr's b_flags. We must either
3057 		 * hold the hash_lock or be undiscoverable.
3058 		 */
3059 		ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3060 
3061 		arc_cksum_verify(buf);
3062 		arc_buf_unwatch(buf);
3063 
3064 		if (arc_buf_is_shared(buf)) {
3065 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3066 		} else {
3067 			uint64_t size = arc_buf_size(buf);
3068 			arc_free_data_buf(hdr, buf->b_data, size, buf);
3069 			ARCSTAT_INCR(arcstat_overhead_size, -size);
3070 		}
3071 		buf->b_data = NULL;
3072 
3073 		/*
3074 		 * If we have no more encrypted buffers and we've already
3075 		 * gotten a copy of the decrypted data we can free b_rabd
3076 		 * to save some space.
3077 		 */
3078 		if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
3079 		    hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
3080 			arc_buf_t *b;
3081 			for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
3082 				if (b != buf && ARC_BUF_ENCRYPTED(b))
3083 					break;
3084 			}
3085 			if (b == NULL)
3086 				arc_hdr_free_abd(hdr, B_TRUE);
3087 		}
3088 	}
3089 
3090 	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3091 
3092 	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3093 		/*
3094 		 * If the current arc_buf_t is sharing its data buffer with the
3095 		 * hdr, then reassign the hdr's b_pabd to share it with the new
3096 		 * buffer at the end of the list. The shared buffer is always
3097 		 * the last one on the hdr's buffer list.
3098 		 *
3099 		 * There is an equivalent case for compressed bufs, but since
3100 		 * they aren't guaranteed to be the last buf in the list and
3101 		 * that is an exceedingly rare case, we just allow that space be
3102 		 * wasted temporarily. We must also be careful not to share
3103 		 * encrypted buffers, since they cannot be shared.
3104 		 */
3105 		if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) {
3106 			/* Only one buf can be shared at once */
3107 			VERIFY(!arc_buf_is_shared(lastbuf));
3108 			/* hdr is uncompressed so can't have compressed buf */
3109 			VERIFY(!ARC_BUF_COMPRESSED(lastbuf));
3110 
3111 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3112 			arc_hdr_free_abd(hdr, B_FALSE);
3113 
3114 			/*
3115 			 * We must setup a new shared block between the
3116 			 * last buffer and the hdr. The data would have
3117 			 * been allocated by the arc buf so we need to transfer
3118 			 * ownership to the hdr since it's now being shared.
3119 			 */
3120 			arc_share_buf(hdr, lastbuf);
3121 		}
3122 	} else if (HDR_SHARED_DATA(hdr)) {
3123 		/*
3124 		 * Uncompressed shared buffers are always at the end
3125 		 * of the list. Compressed buffers don't have the
3126 		 * same requirements. This makes it hard to
3127 		 * simply assert that the lastbuf is shared so
3128 		 * we rely on the hdr's compression flags to determine
3129 		 * if we have a compressed, shared buffer.
3130 		 */
3131 		ASSERT3P(lastbuf, !=, NULL);
3132 		ASSERT(arc_buf_is_shared(lastbuf) ||
3133 		    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
3134 	}
3135 
3136 	/*
3137 	 * Free the checksum if we're removing the last uncompressed buf from
3138 	 * this hdr.
3139 	 */
3140 	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3141 		arc_cksum_free(hdr);
3142 	}
3143 
3144 	/* clean up the buf */
3145 	buf->b_hdr = NULL;
3146 	kmem_cache_free(buf_cache, buf);
3147 }
3148 
3149 static void
3150 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
3151 {
3152 	uint64_t size;
3153 	boolean_t alloc_rdata = ((alloc_flags & ARC_HDR_ALLOC_RDATA) != 0);
3154 
3155 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3156 	ASSERT(HDR_HAS_L1HDR(hdr));
3157 	ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);
3158 	IMPLY(alloc_rdata, HDR_PROTECTED(hdr));
3159 
3160 	if (alloc_rdata) {
3161 		size = HDR_GET_PSIZE(hdr);
3162 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
3163 		hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
3164 		    alloc_flags);
3165 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
3166 		ARCSTAT_INCR(arcstat_raw_size, size);
3167 	} else {
3168 		size = arc_hdr_size(hdr);
3169 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3170 		hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
3171 		    alloc_flags);
3172 		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3173 	}
3174 
3175 	ARCSTAT_INCR(arcstat_compressed_size, size);
3176 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3177 }
3178 
3179 static void
3180 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
3181 {
3182 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
3183 
3184 	ASSERT(HDR_HAS_L1HDR(hdr));
3185 	ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
3186 	IMPLY(free_rdata, HDR_HAS_RABD(hdr));
3187 
3188 	/*
3189 	 * If the hdr is currently being written to the l2arc then
3190 	 * we defer freeing the data by adding it to the l2arc_free_on_write
3191 	 * list. The l2arc will free the data once it's finished
3192 	 * writing it to the l2arc device.
3193 	 */
3194 	if (HDR_L2_WRITING(hdr)) {
3195 		arc_hdr_free_on_write(hdr, free_rdata);
3196 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3197 	} else if (free_rdata) {
3198 		arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
3199 	} else {
3200 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
3201 	}
3202 
3203 	if (free_rdata) {
3204 		hdr->b_crypt_hdr.b_rabd = NULL;
3205 		ARCSTAT_INCR(arcstat_raw_size, -size);
3206 	} else {
3207 		hdr->b_l1hdr.b_pabd = NULL;
3208 	}
3209 
3210 	if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
3211 		hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3212 
3213 	ARCSTAT_INCR(arcstat_compressed_size, -size);
3214 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3215 }
3216 
3217 /*
3218  * Allocate empty anonymous ARC header.  The header will get its identity
3219  * assigned and buffers attached later as part of read or write operations.
3220  *
3221  * In case of read arc_read() assigns header its identify (b_dva + b_birth),
3222  * inserts it into ARC hash to become globally visible and allocates physical
3223  * (b_pabd) or raw (b_rabd) ABD buffer to read into from disk.  On disk read
3224  * completion arc_read_done() allocates ARC buffer(s) as needed, potentially
3225  * sharing one of them with the physical ABD buffer.
3226  *
3227  * In case of write arc_alloc_buf() allocates ARC buffer to be filled with
3228  * data.  Then after compression and/or encryption arc_write_ready() allocates
3229  * and fills (or potentially shares) physical (b_pabd) or raw (b_rabd) ABD
3230  * buffer.  On disk write completion arc_write_done() assigns the header its
3231  * new identity (b_dva + b_birth) and inserts into ARC hash.
3232  *
3233  * In case of partial overwrite the old data is read first as described. Then
3234  * arc_release() either allocates new anonymous ARC header and moves the ARC
3235  * buffer to it, or reuses the old ARC header by discarding its identity and
3236  * removing it from ARC hash.  After buffer modification normal write process
3237  * follows as described.
3238  */
3239 static arc_buf_hdr_t *
3240 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3241     boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
3242     arc_buf_contents_t type)
3243 {
3244 	arc_buf_hdr_t *hdr;
3245 
3246 	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3247 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3248 
3249 	ASSERT(HDR_EMPTY(hdr));
3250 #ifdef ZFS_DEBUG
3251 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3252 #endif
3253 	HDR_SET_PSIZE(hdr, psize);
3254 	HDR_SET_LSIZE(hdr, lsize);
3255 	hdr->b_spa = spa;
3256 	hdr->b_type = type;
3257 	hdr->b_flags = 0;
3258 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3259 	arc_hdr_set_compress(hdr, compression_type);
3260 	hdr->b_complevel = complevel;
3261 	if (protected)
3262 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3263 
3264 	hdr->b_l1hdr.b_state = arc_anon;
3265 	hdr->b_l1hdr.b_arc_access = 0;
3266 	hdr->b_l1hdr.b_mru_hits = 0;
3267 	hdr->b_l1hdr.b_mru_ghost_hits = 0;
3268 	hdr->b_l1hdr.b_mfu_hits = 0;
3269 	hdr->b_l1hdr.b_mfu_ghost_hits = 0;
3270 	hdr->b_l1hdr.b_buf = NULL;
3271 
3272 	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3273 
3274 	return (hdr);
3275 }
3276 
3277 /*
3278  * Transition between the two allocation states for the arc_buf_hdr struct.
3279  * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3280  * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3281  * version is used when a cache buffer is only in the L2ARC in order to reduce
3282  * memory usage.
3283  */
3284 static arc_buf_hdr_t *
3285 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3286 {
3287 	ASSERT(HDR_HAS_L2HDR(hdr));
3288 
3289 	arc_buf_hdr_t *nhdr;
3290 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3291 
3292 	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3293 	    (old == hdr_l2only_cache && new == hdr_full_cache));
3294 
3295 	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3296 
3297 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3298 	buf_hash_remove(hdr);
3299 
3300 	memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);
3301 
3302 	if (new == hdr_full_cache) {
3303 		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3304 		/*
3305 		 * arc_access and arc_change_state need to be aware that a
3306 		 * header has just come out of L2ARC, so we set its state to
3307 		 * l2c_only even though it's about to change.
3308 		 */
3309 		nhdr->b_l1hdr.b_state = arc_l2c_only;
3310 
3311 		/* Verify previous threads set to NULL before freeing */
3312 		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3313 		ASSERT(!HDR_HAS_RABD(hdr));
3314 	} else {
3315 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3316 #ifdef ZFS_DEBUG
3317 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3318 #endif
3319 
3320 		/*
3321 		 * If we've reached here, We must have been called from
3322 		 * arc_evict_hdr(), as such we should have already been
3323 		 * removed from any ghost list we were previously on
3324 		 * (which protects us from racing with arc_evict_state),
3325 		 * thus no locking is needed during this check.
3326 		 */
3327 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3328 
3329 		/*
3330 		 * A buffer must not be moved into the arc_l2c_only
3331 		 * state if it's not finished being written out to the
3332 		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3333 		 * might try to be accessed, even though it was removed.
3334 		 */
3335 		VERIFY(!HDR_L2_WRITING(hdr));
3336 		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3337 		ASSERT(!HDR_HAS_RABD(hdr));
3338 
3339 		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3340 	}
3341 	/*
3342 	 * The header has been reallocated so we need to re-insert it into any
3343 	 * lists it was on.
3344 	 */
3345 	(void) buf_hash_insert(nhdr, NULL);
3346 
3347 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3348 
3349 	mutex_enter(&dev->l2ad_mtx);
3350 
3351 	/*
3352 	 * We must place the realloc'ed header back into the list at
3353 	 * the same spot. Otherwise, if it's placed earlier in the list,
3354 	 * l2arc_write_buffers() could find it during the function's
3355 	 * write phase, and try to write it out to the l2arc.
3356 	 */
3357 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3358 	list_remove(&dev->l2ad_buflist, hdr);
3359 
3360 	mutex_exit(&dev->l2ad_mtx);
3361 
3362 	/*
3363 	 * Since we're using the pointer address as the tag when
3364 	 * incrementing and decrementing the l2ad_alloc refcount, we
3365 	 * must remove the old pointer (that we're about to destroy) and
3366 	 * add the new pointer to the refcount. Otherwise we'd remove
3367 	 * the wrong pointer address when calling arc_hdr_destroy() later.
3368 	 */
3369 
3370 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
3371 	    arc_hdr_size(hdr), hdr);
3372 	(void) zfs_refcount_add_many(&dev->l2ad_alloc,
3373 	    arc_hdr_size(nhdr), nhdr);
3374 
3375 	buf_discard_identity(hdr);
3376 	kmem_cache_free(old, hdr);
3377 
3378 	return (nhdr);
3379 }
3380 
3381 /*
3382  * This function is used by the send / receive code to convert a newly
3383  * allocated arc_buf_t to one that is suitable for a raw encrypted write. It
3384  * is also used to allow the root objset block to be updated without altering
3385  * its embedded MACs. Both block types will always be uncompressed so we do not
3386  * have to worry about compression type or psize.
3387  */
3388 void
3389 arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder,
3390     dmu_object_type_t ot, const uint8_t *salt, const uint8_t *iv,
3391     const uint8_t *mac)
3392 {
3393 	arc_buf_hdr_t *hdr = buf->b_hdr;
3394 
3395 	ASSERT(ot == DMU_OT_DNODE || ot == DMU_OT_OBJSET);
3396 	ASSERT(HDR_HAS_L1HDR(hdr));
3397 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3398 
3399 	buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED);
3400 	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3401 	hdr->b_crypt_hdr.b_dsobj = dsobj;
3402 	hdr->b_crypt_hdr.b_ot = ot;
3403 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3404 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3405 	if (!arc_hdr_has_uncompressed_buf(hdr))
3406 		arc_cksum_free(hdr);
3407 
3408 	if (salt != NULL)
3409 		memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3410 	if (iv != NULL)
3411 		memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3412 	if (mac != NULL)
3413 		memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3414 }
3415 
3416 /*
3417  * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3418  * The buf is returned thawed since we expect the consumer to modify it.
3419  */
3420 arc_buf_t *
3421 arc_alloc_buf(spa_t *spa, const void *tag, arc_buf_contents_t type,
3422     int32_t size)
3423 {
3424 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3425 	    B_FALSE, ZIO_COMPRESS_OFF, 0, type);
3426 
3427 	arc_buf_t *buf = NULL;
3428 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
3429 	    B_FALSE, B_FALSE, &buf));
3430 	arc_buf_thaw(buf);
3431 
3432 	return (buf);
3433 }
3434 
3435 /*
3436  * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3437  * for bufs containing metadata.
3438  */
3439 arc_buf_t *
3440 arc_alloc_compressed_buf(spa_t *spa, const void *tag, uint64_t psize,
3441     uint64_t lsize, enum zio_compress compression_type, uint8_t complevel)
3442 {
3443 	ASSERT3U(lsize, >, 0);
3444 	ASSERT3U(lsize, >=, psize);
3445 	ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF);
3446 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3447 
3448 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3449 	    B_FALSE, compression_type, complevel, ARC_BUFC_DATA);
3450 
3451 	arc_buf_t *buf = NULL;
3452 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
3453 	    B_TRUE, B_FALSE, B_FALSE, &buf));
3454 	arc_buf_thaw(buf);
3455 
3456 	/*
3457 	 * To ensure that the hdr has the correct data in it if we call
3458 	 * arc_untransform() on this buf before it's been written to disk,
3459 	 * it's easiest if we just set up sharing between the buf and the hdr.
3460 	 */
3461 	arc_share_buf(hdr, buf);
3462 
3463 	return (buf);
3464 }
3465 
3466 arc_buf_t *
3467 arc_alloc_raw_buf(spa_t *spa, const void *tag, uint64_t dsobj,
3468     boolean_t byteorder, const uint8_t *salt, const uint8_t *iv,
3469     const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
3470     enum zio_compress compression_type, uint8_t complevel)
3471 {
3472 	arc_buf_hdr_t *hdr;
3473 	arc_buf_t *buf;
3474 	arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ?
3475 	    ARC_BUFC_METADATA : ARC_BUFC_DATA;
3476 
3477 	ASSERT3U(lsize, >, 0);
3478 	ASSERT3U(lsize, >=, psize);
3479 	ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF);
3480 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3481 
3482 	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
3483 	    compression_type, complevel, type);
3484 
3485 	hdr->b_crypt_hdr.b_dsobj = dsobj;
3486 	hdr->b_crypt_hdr.b_ot = ot;
3487 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3488 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3489 	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3490 	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3491 	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3492 
3493 	/*
3494 	 * This buffer will be considered encrypted even if the ot is not an
3495 	 * encrypted type. It will become authenticated instead in
3496 	 * arc_write_ready().
3497 	 */
3498 	buf = NULL;
3499 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
3500 	    B_FALSE, B_FALSE, &buf));
3501 	arc_buf_thaw(buf);
3502 
3503 	return (buf);
3504 }
3505 
3506 static void
3507 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
3508     boolean_t state_only)
3509 {
3510 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3511 	l2arc_dev_t *dev = l2hdr->b_dev;
3512 	uint64_t lsize = HDR_GET_LSIZE(hdr);
3513 	uint64_t psize = HDR_GET_PSIZE(hdr);
3514 	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
3515 	arc_buf_contents_t type = hdr->b_type;
3516 	int64_t lsize_s;
3517 	int64_t psize_s;
3518 	int64_t asize_s;
3519 
3520 	if (incr) {
3521 		lsize_s = lsize;
3522 		psize_s = psize;
3523 		asize_s = asize;
3524 	} else {
3525 		lsize_s = -lsize;
3526 		psize_s = -psize;
3527 		asize_s = -asize;
3528 	}
3529 
3530 	/* If the buffer is a prefetch, count it as such. */
3531 	if (HDR_PREFETCH(hdr)) {
3532 		ARCSTAT_INCR(arcstat_l2_prefetch_asize, asize_s);
3533 	} else {
3534 		/*
3535 		 * We use the value stored in the L2 header upon initial
3536 		 * caching in L2ARC. This value will be updated in case
3537 		 * an MRU/MRU_ghost buffer transitions to MFU but the L2ARC
3538 		 * metadata (log entry) cannot currently be updated. Having
3539 		 * the ARC state in the L2 header solves the problem of a
3540 		 * possibly absent L1 header (apparent in buffers restored
3541 		 * from persistent L2ARC).
3542 		 */
3543 		switch (hdr->b_l2hdr.b_arcs_state) {
3544 			case ARC_STATE_MRU_GHOST:
3545 			case ARC_STATE_MRU:
3546 				ARCSTAT_INCR(arcstat_l2_mru_asize, asize_s);
3547 				break;
3548 			case ARC_STATE_MFU_GHOST:
3549 			case ARC_STATE_MFU:
3550 				ARCSTAT_INCR(arcstat_l2_mfu_asize, asize_s);
3551 				break;
3552 			default:
3553 				break;
3554 		}
3555 	}
3556 
3557 	if (state_only)
3558 		return;
3559 
3560 	ARCSTAT_INCR(arcstat_l2_psize, psize_s);
3561 	ARCSTAT_INCR(arcstat_l2_lsize, lsize_s);
3562 
3563 	switch (type) {
3564 		case ARC_BUFC_DATA:
3565 			ARCSTAT_INCR(arcstat_l2_bufc_data_asize, asize_s);
3566 			break;
3567 		case ARC_BUFC_METADATA:
3568 			ARCSTAT_INCR(arcstat_l2_bufc_metadata_asize, asize_s);
3569 			break;
3570 		default:
3571 			break;
3572 	}
3573 }
3574 
3575 
3576 static void
3577 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3578 {
3579 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3580 	l2arc_dev_t *dev = l2hdr->b_dev;
3581 	uint64_t psize = HDR_GET_PSIZE(hdr);
3582 	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
3583 
3584 	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3585 	ASSERT(HDR_HAS_L2HDR(hdr));
3586 
3587 	list_remove(&dev->l2ad_buflist, hdr);
3588 
3589 	l2arc_hdr_arcstats_decrement(hdr);
3590 	vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3591 
3592 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3593 	    hdr);
3594 	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3595 }
3596 
3597 static void
3598 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3599 {
3600 	if (HDR_HAS_L1HDR(hdr)) {
3601 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3602 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3603 	}
3604 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3605 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3606 
3607 	if (HDR_HAS_L2HDR(hdr)) {
3608 		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3609 		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3610 
3611 		if (!buflist_held)
3612 			mutex_enter(&dev->l2ad_mtx);
3613 
3614 		/*
3615 		 * Even though we checked this conditional above, we
3616 		 * need to check this again now that we have the
3617 		 * l2ad_mtx. This is because we could be racing with
3618 		 * another thread calling l2arc_evict() which might have
3619 		 * destroyed this header's L2 portion as we were waiting
3620 		 * to acquire the l2ad_mtx. If that happens, we don't
3621 		 * want to re-destroy the header's L2 portion.
3622 		 */
3623 		if (HDR_HAS_L2HDR(hdr)) {
3624 
3625 			if (!HDR_EMPTY(hdr))
3626 				buf_discard_identity(hdr);
3627 
3628 			arc_hdr_l2hdr_destroy(hdr);
3629 		}
3630 
3631 		if (!buflist_held)
3632 			mutex_exit(&dev->l2ad_mtx);
3633 	}
3634 
3635 	/*
3636 	 * The header's identify can only be safely discarded once it is no
3637 	 * longer discoverable.  This requires removing it from the hash table
3638 	 * and the l2arc header list.  After this point the hash lock can not
3639 	 * be used to protect the header.
3640 	 */
3641 	if (!HDR_EMPTY(hdr))
3642 		buf_discard_identity(hdr);
3643 
3644 	if (HDR_HAS_L1HDR(hdr)) {
3645 		arc_cksum_free(hdr);
3646 
3647 		while (hdr->b_l1hdr.b_buf != NULL)
3648 			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3649 
3650 		if (hdr->b_l1hdr.b_pabd != NULL)
3651 			arc_hdr_free_abd(hdr, B_FALSE);
3652 
3653 		if (HDR_HAS_RABD(hdr))
3654 			arc_hdr_free_abd(hdr, B_TRUE);
3655 	}
3656 
3657 	ASSERT3P(hdr->b_hash_next, ==, NULL);
3658 	if (HDR_HAS_L1HDR(hdr)) {
3659 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3660 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3661 #ifdef ZFS_DEBUG
3662 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3663 #endif
3664 		kmem_cache_free(hdr_full_cache, hdr);
3665 	} else {
3666 		kmem_cache_free(hdr_l2only_cache, hdr);
3667 	}
3668 }
3669 
3670 void
3671 arc_buf_destroy(arc_buf_t *buf, const void *tag)
3672 {
3673 	arc_buf_hdr_t *hdr = buf->b_hdr;
3674 
3675 	if (hdr->b_l1hdr.b_state == arc_anon) {
3676 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
3677 		ASSERT(ARC_BUF_LAST(buf));
3678 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3679 		VERIFY0(remove_reference(hdr, tag));
3680 		return;
3681 	}
3682 
3683 	kmutex_t *hash_lock = HDR_LOCK(hdr);
3684 	mutex_enter(hash_lock);
3685 
3686 	ASSERT3P(hdr, ==, buf->b_hdr);
3687 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
3688 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3689 	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3690 	ASSERT3P(buf->b_data, !=, NULL);
3691 
3692 	arc_buf_destroy_impl(buf);
3693 	(void) remove_reference(hdr, tag);
3694 	mutex_exit(hash_lock);
3695 }
3696 
3697 /*
3698  * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3699  * state of the header is dependent on its state prior to entering this
3700  * function. The following transitions are possible:
3701  *
3702  *    - arc_mru -> arc_mru_ghost
3703  *    - arc_mfu -> arc_mfu_ghost
3704  *    - arc_mru_ghost -> arc_l2c_only
3705  *    - arc_mru_ghost -> deleted
3706  *    - arc_mfu_ghost -> arc_l2c_only
3707  *    - arc_mfu_ghost -> deleted
3708  *    - arc_uncached -> deleted
3709  *
3710  * Return total size of evicted data buffers for eviction progress tracking.
3711  * When evicting from ghost states return logical buffer size to make eviction
3712  * progress at the same (or at least comparable) rate as from non-ghost states.
3713  *
3714  * Return *real_evicted for actual ARC size reduction to wake up threads
3715  * waiting for it.  For non-ghost states it includes size of evicted data
3716  * buffers (the headers are not freed there).  For ghost states it includes
3717  * only the evicted headers size.
3718  */
3719 static int64_t
3720 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
3721 {
3722 	arc_state_t *evicted_state, *state;
3723 	int64_t bytes_evicted = 0;
3724 	uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3725 	    arc_min_prescient_prefetch_ms : arc_min_prefetch_ms;
3726 
3727 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3728 	ASSERT(HDR_HAS_L1HDR(hdr));
3729 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3730 	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3731 	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3732 
3733 	*real_evicted = 0;
3734 	state = hdr->b_l1hdr.b_state;
3735 	if (GHOST_STATE(state)) {
3736 
3737 		/*
3738 		 * l2arc_write_buffers() relies on a header's L1 portion
3739 		 * (i.e. its b_pabd field) during it's write phase.
3740 		 * Thus, we cannot push a header onto the arc_l2c_only
3741 		 * state (removing its L1 piece) until the header is
3742 		 * done being written to the l2arc.
3743 		 */
3744 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3745 			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3746 			return (bytes_evicted);
3747 		}
3748 
3749 		ARCSTAT_BUMP(arcstat_deleted);
3750 		bytes_evicted += HDR_GET_LSIZE(hdr);
3751 
3752 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3753 
3754 		if (HDR_HAS_L2HDR(hdr)) {
3755 			ASSERT(hdr->b_l1hdr.b_pabd == NULL);
3756 			ASSERT(!HDR_HAS_RABD(hdr));
3757 			/*
3758 			 * This buffer is cached on the 2nd Level ARC;
3759 			 * don't destroy the header.
3760 			 */
3761 			arc_change_state(arc_l2c_only, hdr);
3762 			/*
3763 			 * dropping from L1+L2 cached to L2-only,
3764 			 * realloc to remove the L1 header.
3765 			 */
3766 			(void) arc_hdr_realloc(hdr, hdr_full_cache,
3767 			    hdr_l2only_cache);
3768 			*real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
3769 		} else {
3770 			arc_change_state(arc_anon, hdr);
3771 			arc_hdr_destroy(hdr);
3772 			*real_evicted += HDR_FULL_SIZE;
3773 		}
3774 		return (bytes_evicted);
3775 	}
3776 
3777 	ASSERT(state == arc_mru || state == arc_mfu || state == arc_uncached);
3778 	evicted_state = (state == arc_uncached) ? arc_anon :
3779 	    ((state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost);
3780 
3781 	/* prefetch buffers have a minimum lifespan */
3782 	if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3783 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3784 	    MSEC_TO_TICK(min_lifetime)) {
3785 		ARCSTAT_BUMP(arcstat_evict_skip);
3786 		return (bytes_evicted);
3787 	}
3788 
3789 	if (HDR_HAS_L2HDR(hdr)) {
3790 		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3791 	} else {
3792 		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3793 			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3794 			    HDR_GET_LSIZE(hdr));
3795 
3796 			switch (state->arcs_state) {
3797 				case ARC_STATE_MRU:
3798 					ARCSTAT_INCR(
3799 					    arcstat_evict_l2_eligible_mru,
3800 					    HDR_GET_LSIZE(hdr));
3801 					break;
3802 				case ARC_STATE_MFU:
3803 					ARCSTAT_INCR(
3804 					    arcstat_evict_l2_eligible_mfu,
3805 					    HDR_GET_LSIZE(hdr));
3806 					break;
3807 				default:
3808 					break;
3809 			}
3810 		} else {
3811 			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3812 			    HDR_GET_LSIZE(hdr));
3813 		}
3814 	}
3815 
3816 	bytes_evicted += arc_hdr_size(hdr);
3817 	*real_evicted += arc_hdr_size(hdr);
3818 
3819 	/*
3820 	 * If this hdr is being evicted and has a compressed buffer then we
3821 	 * discard it here before we change states.  This ensures that the
3822 	 * accounting is updated correctly in arc_free_data_impl().
3823 	 */
3824 	if (hdr->b_l1hdr.b_pabd != NULL)
3825 		arc_hdr_free_abd(hdr, B_FALSE);
3826 
3827 	if (HDR_HAS_RABD(hdr))
3828 		arc_hdr_free_abd(hdr, B_TRUE);
3829 
3830 	arc_change_state(evicted_state, hdr);
3831 	DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3832 	if (evicted_state == arc_anon) {
3833 		arc_hdr_destroy(hdr);
3834 		*real_evicted += HDR_FULL_SIZE;
3835 	} else {
3836 		ASSERT(HDR_IN_HASH_TABLE(hdr));
3837 	}
3838 
3839 	return (bytes_evicted);
3840 }
3841 
3842 static void
3843 arc_set_need_free(void)
3844 {
3845 	ASSERT(MUTEX_HELD(&arc_evict_lock));
3846 	int64_t remaining = arc_free_memory() - arc_sys_free / 2;
3847 	arc_evict_waiter_t *aw = list_tail(&arc_evict_waiters);
3848 	if (aw == NULL) {
3849 		arc_need_free = MAX(-remaining, 0);
3850 	} else {
3851 		arc_need_free =
3852 		    MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
3853 	}
3854 }
3855 
3856 static uint64_t
3857 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3858     uint64_t spa, uint64_t bytes)
3859 {
3860 	multilist_sublist_t *mls;
3861 	uint64_t bytes_evicted = 0, real_evicted = 0;
3862 	arc_buf_hdr_t *hdr;
3863 	kmutex_t *hash_lock;
3864 	uint_t evict_count = zfs_arc_evict_batch_limit;
3865 
3866 	ASSERT3P(marker, !=, NULL);
3867 
3868 	mls = multilist_sublist_lock(ml, idx);
3869 
3870 	for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
3871 	    hdr = multilist_sublist_prev(mls, marker)) {
3872 		if ((evict_count == 0) || (bytes_evicted >= bytes))
3873 			break;
3874 
3875 		/*
3876 		 * To keep our iteration location, move the marker
3877 		 * forward. Since we're not holding hdr's hash lock, we
3878 		 * must be very careful and not remove 'hdr' from the
3879 		 * sublist. Otherwise, other consumers might mistake the
3880 		 * 'hdr' as not being on a sublist when they call the
3881 		 * multilist_link_active() function (they all rely on
3882 		 * the hash lock protecting concurrent insertions and
3883 		 * removals). multilist_sublist_move_forward() was
3884 		 * specifically implemented to ensure this is the case
3885 		 * (only 'marker' will be removed and re-inserted).
3886 		 */
3887 		multilist_sublist_move_forward(mls, marker);
3888 
3889 		/*
3890 		 * The only case where the b_spa field should ever be
3891 		 * zero, is the marker headers inserted by
3892 		 * arc_evict_state(). It's possible for multiple threads
3893 		 * to be calling arc_evict_state() concurrently (e.g.
3894 		 * dsl_pool_close() and zio_inject_fault()), so we must
3895 		 * skip any markers we see from these other threads.
3896 		 */
3897 		if (hdr->b_spa == 0)
3898 			continue;
3899 
3900 		/* we're only interested in evicting buffers of a certain spa */
3901 		if (spa != 0 && hdr->b_spa != spa) {
3902 			ARCSTAT_BUMP(arcstat_evict_skip);
3903 			continue;
3904 		}
3905 
3906 		hash_lock = HDR_LOCK(hdr);
3907 
3908 		/*
3909 		 * We aren't calling this function from any code path
3910 		 * that would already be holding a hash lock, so we're
3911 		 * asserting on this assumption to be defensive in case
3912 		 * this ever changes. Without this check, it would be
3913 		 * possible to incorrectly increment arcstat_mutex_miss
3914 		 * below (e.g. if the code changed such that we called
3915 		 * this function with a hash lock held).
3916 		 */
3917 		ASSERT(!MUTEX_HELD(hash_lock));
3918 
3919 		if (mutex_tryenter(hash_lock)) {
3920 			uint64_t revicted;
3921 			uint64_t evicted = arc_evict_hdr(hdr, &revicted);
3922 			mutex_exit(hash_lock);
3923 
3924 			bytes_evicted += evicted;
3925 			real_evicted += revicted;
3926 
3927 			/*
3928 			 * If evicted is zero, arc_evict_hdr() must have
3929 			 * decided to skip this header, don't increment
3930 			 * evict_count in this case.
3931 			 */
3932 			if (evicted != 0)
3933 				evict_count--;
3934 
3935 		} else {
3936 			ARCSTAT_BUMP(arcstat_mutex_miss);
3937 		}
3938 	}
3939 
3940 	multilist_sublist_unlock(mls);
3941 
3942 	/*
3943 	 * Increment the count of evicted bytes, and wake up any threads that
3944 	 * are waiting for the count to reach this value.  Since the list is
3945 	 * ordered by ascending aew_count, we pop off the beginning of the
3946 	 * list until we reach the end, or a waiter that's past the current
3947 	 * "count".  Doing this outside the loop reduces the number of times
3948 	 * we need to acquire the global arc_evict_lock.
3949 	 *
3950 	 * Only wake when there's sufficient free memory in the system
3951 	 * (specifically, arc_sys_free/2, which by default is a bit more than
3952 	 * 1/64th of RAM).  See the comments in arc_wait_for_eviction().
3953 	 */
3954 	mutex_enter(&arc_evict_lock);
3955 	arc_evict_count += real_evicted;
3956 
3957 	if (arc_free_memory() > arc_sys_free / 2) {
3958 		arc_evict_waiter_t *aw;
3959 		while ((aw = list_head(&arc_evict_waiters)) != NULL &&
3960 		    aw->aew_count <= arc_evict_count) {
3961 			list_remove(&arc_evict_waiters, aw);
3962 			cv_broadcast(&aw->aew_cv);
3963 		}
3964 	}
3965 	arc_set_need_free();
3966 	mutex_exit(&arc_evict_lock);
3967 
3968 	/*
3969 	 * If the ARC size is reduced from arc_c_max to arc_c_min (especially
3970 	 * if the average cached block is small), eviction can be on-CPU for
3971 	 * many seconds.  To ensure that other threads that may be bound to
3972 	 * this CPU are able to make progress, make a voluntary preemption
3973 	 * call here.
3974 	 */
3975 	kpreempt(KPREEMPT_SYNC);
3976 
3977 	return (bytes_evicted);
3978 }
3979 
3980 /*
3981  * Allocate an array of buffer headers used as placeholders during arc state
3982  * eviction.
3983  */
3984 static arc_buf_hdr_t **
3985 arc_state_alloc_markers(int count)
3986 {
3987 	arc_buf_hdr_t **markers;
3988 
3989 	markers = kmem_zalloc(sizeof (*markers) * count, KM_SLEEP);
3990 	for (int i = 0; i < count; i++) {
3991 		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
3992 
3993 		/*
3994 		 * A b_spa of 0 is used to indicate that this header is
3995 		 * a marker. This fact is used in arc_evict_state_impl().
3996 		 */
3997 		markers[i]->b_spa = 0;
3998 
3999 	}
4000 	return (markers);
4001 }
4002 
4003 static void
4004 arc_state_free_markers(arc_buf_hdr_t **markers, int count)
4005 {
4006 	for (int i = 0; i < count; i++)
4007 		kmem_cache_free(hdr_full_cache, markers[i]);
4008 	kmem_free(markers, sizeof (*markers) * count);
4009 }
4010 
4011 /*
4012  * Evict buffers from the given arc state, until we've removed the
4013  * specified number of bytes. Move the removed buffers to the
4014  * appropriate evict state.
4015  *
4016  * This function makes a "best effort". It skips over any buffers
4017  * it can't get a hash_lock on, and so, may not catch all candidates.
4018  * It may also return without evicting as much space as requested.
4019  *
4020  * If bytes is specified using the special value ARC_EVICT_ALL, this
4021  * will evict all available (i.e. unlocked and evictable) buffers from
4022  * the given arc state; which is used by arc_flush().
4023  */
4024 static uint64_t
4025 arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
4026     uint64_t bytes)
4027 {
4028 	uint64_t total_evicted = 0;
4029 	multilist_t *ml = &state->arcs_list[type];
4030 	int num_sublists;
4031 	arc_buf_hdr_t **markers;
4032 
4033 	num_sublists = multilist_get_num_sublists(ml);
4034 
4035 	/*
4036 	 * If we've tried to evict from each sublist, made some
4037 	 * progress, but still have not hit the target number of bytes
4038 	 * to evict, we want to keep trying. The markers allow us to
4039 	 * pick up where we left off for each individual sublist, rather
4040 	 * than starting from the tail each time.
4041 	 */
4042 	if (zthr_iscurthread(arc_evict_zthr)) {
4043 		markers = arc_state_evict_markers;
4044 		ASSERT3S(num_sublists, <=, arc_state_evict_marker_count);
4045 	} else {
4046 		markers = arc_state_alloc_markers(num_sublists);
4047 	}
4048 	for (int i = 0; i < num_sublists; i++) {
4049 		multilist_sublist_t *mls;
4050 
4051 		mls = multilist_sublist_lock(ml, i);
4052 		multilist_sublist_insert_tail(mls, markers[i]);
4053 		multilist_sublist_unlock(mls);
4054 	}
4055 
4056 	/*
4057 	 * While we haven't hit our target number of bytes to evict, or
4058 	 * we're evicting all available buffers.
4059 	 */
4060 	while (total_evicted < bytes) {
4061 		int sublist_idx = multilist_get_random_index(ml);
4062 		uint64_t scan_evicted = 0;
4063 
4064 		/*
4065 		 * Start eviction using a randomly selected sublist,
4066 		 * this is to try and evenly balance eviction across all
4067 		 * sublists. Always starting at the same sublist
4068 		 * (e.g. index 0) would cause evictions to favor certain
4069 		 * sublists over others.
4070 		 */
4071 		for (int i = 0; i < num_sublists; i++) {
4072 			uint64_t bytes_remaining;
4073 			uint64_t bytes_evicted;
4074 
4075 			if (total_evicted < bytes)
4076 				bytes_remaining = bytes - total_evicted;
4077 			else
4078 				break;
4079 
4080 			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
4081 			    markers[sublist_idx], spa, bytes_remaining);
4082 
4083 			scan_evicted += bytes_evicted;
4084 			total_evicted += bytes_evicted;
4085 
4086 			/* we've reached the end, wrap to the beginning */
4087 			if (++sublist_idx >= num_sublists)
4088 				sublist_idx = 0;
4089 		}
4090 
4091 		/*
4092 		 * If we didn't evict anything during this scan, we have
4093 		 * no reason to believe we'll evict more during another
4094 		 * scan, so break the loop.
4095 		 */
4096 		if (scan_evicted == 0) {
4097 			/* This isn't possible, let's make that obvious */
4098 			ASSERT3S(bytes, !=, 0);
4099 
4100 			/*
4101 			 * When bytes is ARC_EVICT_ALL, the only way to
4102 			 * break the loop is when scan_evicted is zero.
4103 			 * In that case, we actually have evicted enough,
4104 			 * so we don't want to increment the kstat.
4105 			 */
4106 			if (bytes != ARC_EVICT_ALL) {
4107 				ASSERT3S(total_evicted, <, bytes);
4108 				ARCSTAT_BUMP(arcstat_evict_not_enough);
4109 			}
4110 
4111 			break;
4112 		}
4113 	}
4114 
4115 	for (int i = 0; i < num_sublists; i++) {
4116 		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
4117 		multilist_sublist_remove(mls, markers[i]);
4118 		multilist_sublist_unlock(mls);
4119 	}
4120 	if (markers != arc_state_evict_markers)
4121 		arc_state_free_markers(markers, num_sublists);
4122 
4123 	return (total_evicted);
4124 }
4125 
4126 /*
4127  * Flush all "evictable" data of the given type from the arc state
4128  * specified. This will not evict any "active" buffers (i.e. referenced).
4129  *
4130  * When 'retry' is set to B_FALSE, the function will make a single pass
4131  * over the state and evict any buffers that it can. Since it doesn't
4132  * continually retry the eviction, it might end up leaving some buffers
4133  * in the ARC due to lock misses.
4134  *
4135  * When 'retry' is set to B_TRUE, the function will continually retry the
4136  * eviction until *all* evictable buffers have been removed from the
4137  * state. As a result, if concurrent insertions into the state are
4138  * allowed (e.g. if the ARC isn't shutting down), this function might
4139  * wind up in an infinite loop, continually trying to evict buffers.
4140  */
4141 static uint64_t
4142 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
4143     boolean_t retry)
4144 {
4145 	uint64_t evicted = 0;
4146 
4147 	while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
4148 		evicted += arc_evict_state(state, type, spa, ARC_EVICT_ALL);
4149 
4150 		if (!retry)
4151 			break;
4152 	}
4153 
4154 	return (evicted);
4155 }
4156 
4157 /*
4158  * Evict the specified number of bytes from the state specified. This
4159  * function prevents us from trying to evict more from a state's list
4160  * than is "evictable", and to skip evicting altogether when passed a
4161  * negative value for "bytes". In contrast, arc_evict_state() will
4162  * evict everything it can, when passed a negative value for "bytes".
4163  */
4164 static uint64_t
4165 arc_evict_impl(arc_state_t *state, arc_buf_contents_t type, int64_t bytes)
4166 {
4167 	uint64_t delta;
4168 
4169 	if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
4170 		delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
4171 		    bytes);
4172 		return (arc_evict_state(state, type, 0, delta));
4173 	}
4174 
4175 	return (0);
4176 }
4177 
4178 /*
4179  * Adjust specified fraction, taking into account initial ghost state(s) size,
4180  * ghost hit bytes towards increasing the fraction, ghost hit bytes towards
4181  * decreasing it, plus a balance factor, controlling the decrease rate, used
4182  * to balance metadata vs data.
4183  */
4184 static uint64_t
4185 arc_evict_adj(uint64_t frac, uint64_t total, uint64_t up, uint64_t down,
4186     uint_t balance)
4187 {
4188 	if (total < 8 || up + down == 0)
4189 		return (frac);
4190 
4191 	/*
4192 	 * We should not have more ghost hits than ghost size, but they
4193 	 * may get close.  Restrict maximum adjustment in that case.
4194 	 */
4195 	if (up + down >= total / 4) {
4196 		uint64_t scale = (up + down) / (total / 8);
4197 		up /= scale;
4198 		down /= scale;
4199 	}
4200 
4201 	/* Get maximal dynamic range by choosing optimal shifts. */
4202 	int s = highbit64(total);
4203 	s = MIN(64 - s, 32);
4204 
4205 	uint64_t ofrac = (1ULL << 32) - frac;
4206 
4207 	if (frac >= 4 * ofrac)
4208 		up /= frac / (2 * ofrac + 1);
4209 	up = (up << s) / (total >> (32 - s));
4210 	if (ofrac >= 4 * frac)
4211 		down /= ofrac / (2 * frac + 1);
4212 	down = (down << s) / (total >> (32 - s));
4213 	down = down * 100 / balance;
4214 
4215 	return (frac + up - down);
4216 }
4217 
4218 /*
4219  * Evict buffers from the cache, such that arcstat_size is capped by arc_c.
4220  */
4221 static uint64_t
4222 arc_evict(void)
4223 {
4224 	uint64_t asize, bytes, total_evicted = 0;
4225 	int64_t e, mrud, mrum, mfud, mfum, w;
4226 	static uint64_t ogrd, ogrm, ogfd, ogfm;
4227 	static uint64_t gsrd, gsrm, gsfd, gsfm;
4228 	uint64_t ngrd, ngrm, ngfd, ngfm;
4229 
4230 	/* Get current size of ARC states we can evict from. */
4231 	mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
4232 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
4233 	mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
4234 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
4235 	mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
4236 	mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
4237 	uint64_t d = mrud + mfud;
4238 	uint64_t m = mrum + mfum;
4239 	uint64_t t = d + m;
4240 
4241 	/* Get ARC ghost hits since last eviction. */
4242 	ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
4243 	uint64_t grd = ngrd - ogrd;
4244 	ogrd = ngrd;
4245 	ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
4246 	uint64_t grm = ngrm - ogrm;
4247 	ogrm = ngrm;
4248 	ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
4249 	uint64_t gfd = ngfd - ogfd;
4250 	ogfd = ngfd;
4251 	ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
4252 	uint64_t gfm = ngfm - ogfm;
4253 	ogfm = ngfm;
4254 
4255 	/* Adjust ARC states balance based on ghost hits. */
4256 	arc_meta = arc_evict_adj(arc_meta, gsrd + gsrm + gsfd + gsfm,
4257 	    grm + gfm, grd + gfd, zfs_arc_meta_balance);
4258 	arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
4259 	arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);
4260 
4261 	asize = aggsum_value(&arc_sums.arcstat_size);
4262 	int64_t wt = t - (asize - arc_c);
4263 
4264 	/*
4265 	 * Try to reduce pinned dnodes if more than 3/4 of wanted metadata
4266 	 * target is not evictable or if they go over arc_dnode_limit.
4267 	 */
4268 	int64_t prune = 0;
4269 	int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
4270 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
4271 	if (zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
4272 	    zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]) -
4273 	    zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) -
4274 	    zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]) >
4275 	    w * 3 / 4) {
4276 		prune = dn / sizeof (dnode_t) *
4277 		    zfs_arc_dnode_reduce_percent / 100;
4278 	} else if (dn > arc_dnode_limit) {
4279 		prune = (dn - arc_dnode_limit) / sizeof (dnode_t) *
4280 		    zfs_arc_dnode_reduce_percent / 100;
4281 	}
4282 	if (prune > 0)
4283 		arc_prune_async(prune);
4284 
4285 	/* Evict MRU metadata. */
4286 	w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
4287 	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrum - w));
4288 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
4289 	total_evicted += bytes;
4290 	mrum -= bytes;
4291 	asize -= bytes;
4292 
4293 	/* Evict MFU metadata. */
4294 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
4295 	e = MIN((int64_t)(asize - arc_c), (int64_t)(m - w));
4296 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
4297 	total_evicted += bytes;
4298 	mfum -= bytes;
4299 	asize -= bytes;
4300 
4301 	/* Evict MRU data. */
4302 	wt -= m - total_evicted;
4303 	w = wt * (int64_t)(arc_pd >> 16) >> 16;
4304 	e = MIN((int64_t)(asize - arc_c), (int64_t)(mrud - w));
4305 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
4306 	total_evicted += bytes;
4307 	mrud -= bytes;
4308 	asize -= bytes;
4309 
4310 	/* Evict MFU data. */
4311 	e = asize - arc_c;
4312 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
4313 	mfud -= bytes;
4314 	total_evicted += bytes;
4315 
4316 	/*
4317 	 * Evict ghost lists
4318 	 *
4319 	 * Size of each state's ghost list represents how much that state
4320 	 * may grow by shrinking the other states.  Would it need to shrink
4321 	 * other states to zero (that is unlikely), its ghost size would be
4322 	 * equal to sum of other three state sizes.  But excessive ghost
4323 	 * size may result in false ghost hits (too far back), that may
4324 	 * never result in real cache hits if several states are competing.
4325 	 * So choose some arbitraty point of 1/2 of other state sizes.
4326 	 */
4327 	gsrd = (mrum + mfud + mfum) / 2;
4328 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
4329 	    gsrd;
4330 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_DATA, e);
4331 
4332 	gsrm = (mrud + mfud + mfum) / 2;
4333 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
4334 	    gsrm;
4335 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_METADATA, e);
4336 
4337 	gsfd = (mrud + mrum + mfum) / 2;
4338 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
4339 	    gsfd;
4340 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_DATA, e);
4341 
4342 	gsfm = (mrud + mrum + mfud) / 2;
4343 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
4344 	    gsfm;
4345 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_METADATA, e);
4346 
4347 	return (total_evicted);
4348 }
4349 
4350 void
4351 arc_flush(spa_t *spa, boolean_t retry)
4352 {
4353 	uint64_t guid = 0;
4354 
4355 	/*
4356 	 * If retry is B_TRUE, a spa must not be specified since we have
4357 	 * no good way to determine if all of a spa's buffers have been
4358 	 * evicted from an arc state.
4359 	 */
4360 	ASSERT(!retry || spa == NULL);
4361 
4362 	if (spa != NULL)
4363 		guid = spa_load_guid(spa);
4364 
4365 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4366 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4367 
4368 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4369 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4370 
4371 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4372 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4373 
4374 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4375 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4376 
4377 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_DATA, retry);
4378 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_METADATA, retry);
4379 }
4380 
4381 void
4382 arc_reduce_target_size(int64_t to_free)
4383 {
4384 	uint64_t c = arc_c;
4385 
4386 	if (c <= arc_c_min)
4387 		return;
4388 
4389 	/*
4390 	 * All callers want the ARC to actually evict (at least) this much
4391 	 * memory.  Therefore we reduce from the lower of the current size and
4392 	 * the target size.  This way, even if arc_c is much higher than
4393 	 * arc_size (as can be the case after many calls to arc_freed(), we will
4394 	 * immediately have arc_c < arc_size and therefore the arc_evict_zthr
4395 	 * will evict.
4396 	 */
4397 	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
4398 	if (asize < c)
4399 		to_free += c - asize;
4400 	arc_c = MAX((int64_t)c - to_free, (int64_t)arc_c_min);
4401 
4402 	/* See comment in arc_evict_cb_check() on why lock+flag */
4403 	mutex_enter(&arc_evict_lock);
4404 	arc_evict_needed = B_TRUE;
4405 	mutex_exit(&arc_evict_lock);
4406 	zthr_wakeup(arc_evict_zthr);
4407 }
4408 
4409 /*
4410  * Determine if the system is under memory pressure and is asking
4411  * to reclaim memory. A return value of B_TRUE indicates that the system
4412  * is under memory pressure and that the arc should adjust accordingly.
4413  */
4414 boolean_t
4415 arc_reclaim_needed(void)
4416 {
4417 	return (arc_available_memory() < 0);
4418 }
4419 
4420 void
4421 arc_kmem_reap_soon(void)
4422 {
4423 	size_t			i;
4424 	kmem_cache_t		*prev_cache = NULL;
4425 	kmem_cache_t		*prev_data_cache = NULL;
4426 
4427 #ifdef _KERNEL
4428 #if defined(_ILP32)
4429 	/*
4430 	 * Reclaim unused memory from all kmem caches.
4431 	 */
4432 	kmem_reap();
4433 #endif
4434 #endif
4435 
4436 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4437 #if defined(_ILP32)
4438 		/* reach upper limit of cache size on 32-bit */
4439 		if (zio_buf_cache[i] == NULL)
4440 			break;
4441 #endif
4442 		if (zio_buf_cache[i] != prev_cache) {
4443 			prev_cache = zio_buf_cache[i];
4444 			kmem_cache_reap_now(zio_buf_cache[i]);
4445 		}
4446 		if (zio_data_buf_cache[i] != prev_data_cache) {
4447 			prev_data_cache = zio_data_buf_cache[i];
4448 			kmem_cache_reap_now(zio_data_buf_cache[i]);
4449 		}
4450 	}
4451 	kmem_cache_reap_now(buf_cache);
4452 	kmem_cache_reap_now(hdr_full_cache);
4453 	kmem_cache_reap_now(hdr_l2only_cache);
4454 	kmem_cache_reap_now(zfs_btree_leaf_cache);
4455 	abd_cache_reap_now();
4456 }
4457 
4458 static boolean_t
4459 arc_evict_cb_check(void *arg, zthr_t *zthr)
4460 {
4461 	(void) arg, (void) zthr;
4462 
4463 #ifdef ZFS_DEBUG
4464 	/*
4465 	 * This is necessary in order to keep the kstat information
4466 	 * up to date for tools that display kstat data such as the
4467 	 * mdb ::arc dcmd and the Linux crash utility.  These tools
4468 	 * typically do not call kstat's update function, but simply
4469 	 * dump out stats from the most recent update.  Without
4470 	 * this call, these commands may show stale stats for the
4471 	 * anon, mru, mru_ghost, mfu, and mfu_ghost lists.  Even
4472 	 * with this call, the data might be out of date if the
4473 	 * evict thread hasn't been woken recently; but that should
4474 	 * suffice.  The arc_state_t structures can be queried
4475 	 * directly if more accurate information is needed.
4476 	 */
4477 	if (arc_ksp != NULL)
4478 		arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4479 #endif
4480 
4481 	/*
4482 	 * We have to rely on arc_wait_for_eviction() to tell us when to
4483 	 * evict, rather than checking if we are overflowing here, so that we
4484 	 * are sure to not leave arc_wait_for_eviction() waiting on aew_cv.
4485 	 * If we have become "not overflowing" since arc_wait_for_eviction()
4486 	 * checked, we need to wake it up.  We could broadcast the CV here,
4487 	 * but arc_wait_for_eviction() may have not yet gone to sleep.  We
4488 	 * would need to use a mutex to ensure that this function doesn't
4489 	 * broadcast until arc_wait_for_eviction() has gone to sleep (e.g.
4490 	 * the arc_evict_lock).  However, the lock ordering of such a lock
4491 	 * would necessarily be incorrect with respect to the zthr_lock,
4492 	 * which is held before this function is called, and is held by
4493 	 * arc_wait_for_eviction() when it calls zthr_wakeup().
4494 	 */
4495 	if (arc_evict_needed)
4496 		return (B_TRUE);
4497 
4498 	/*
4499 	 * If we have buffers in uncached state, evict them periodically.
4500 	 */
4501 	return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
4502 	    zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
4503 	    ddi_get_lbolt() - arc_last_uncached_flush >
4504 	    MSEC_TO_TICK(arc_min_prefetch_ms / 2)));
4505 }
4506 
4507 /*
4508  * Keep arc_size under arc_c by running arc_evict which evicts data
4509  * from the ARC.
4510  */
4511 static void
4512 arc_evict_cb(void *arg, zthr_t *zthr)
4513 {
4514 	(void) arg, (void) zthr;
4515 
4516 	uint64_t evicted = 0;
4517 	fstrans_cookie_t cookie = spl_fstrans_mark();
4518 
4519 	/* Always try to evict from uncached state. */
4520 	arc_last_uncached_flush = ddi_get_lbolt();
4521 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_DATA, B_FALSE);
4522 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_METADATA, B_FALSE);
4523 
4524 	/* Evict from other states only if told to. */
4525 	if (arc_evict_needed)
4526 		evicted += arc_evict();
4527 
4528 	/*
4529 	 * If evicted is zero, we couldn't evict anything
4530 	 * via arc_evict(). This could be due to hash lock
4531 	 * collisions, but more likely due to the majority of
4532 	 * arc buffers being unevictable. Therefore, even if
4533 	 * arc_size is above arc_c, another pass is unlikely to
4534 	 * be helpful and could potentially cause us to enter an
4535 	 * infinite loop.  Additionally, zthr_iscancelled() is
4536 	 * checked here so that if the arc is shutting down, the
4537 	 * broadcast will wake any remaining arc evict waiters.
4538 	 */
4539 	mutex_enter(&arc_evict_lock);
4540 	arc_evict_needed = !zthr_iscancelled(arc_evict_zthr) &&
4541 	    evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
4542 	if (!arc_evict_needed) {
4543 		/*
4544 		 * We're either no longer overflowing, or we
4545 		 * can't evict anything more, so we should wake
4546 		 * arc_get_data_impl() sooner.
4547 		 */
4548 		arc_evict_waiter_t *aw;
4549 		while ((aw = list_remove_head(&arc_evict_waiters)) != NULL) {
4550 			cv_broadcast(&aw->aew_cv);
4551 		}
4552 		arc_set_need_free();
4553 	}
4554 	mutex_exit(&arc_evict_lock);
4555 	spl_fstrans_unmark(cookie);
4556 }
4557 
4558 static boolean_t
4559 arc_reap_cb_check(void *arg, zthr_t *zthr)
4560 {
4561 	(void) arg, (void) zthr;
4562 
4563 	int64_t free_memory = arc_available_memory();
4564 	static int reap_cb_check_counter = 0;
4565 
4566 	/*
4567 	 * If a kmem reap is already active, don't schedule more.  We must
4568 	 * check for this because kmem_cache_reap_soon() won't actually
4569 	 * block on the cache being reaped (this is to prevent callers from
4570 	 * becoming implicitly blocked by a system-wide kmem reap -- which,
4571 	 * on a system with many, many full magazines, can take minutes).
4572 	 */
4573 	if (!kmem_cache_reap_active() && free_memory < 0) {
4574 
4575 		arc_no_grow = B_TRUE;
4576 		arc_warm = B_TRUE;
4577 		/*
4578 		 * Wait at least zfs_grow_retry (default 5) seconds
4579 		 * before considering growing.
4580 		 */
4581 		arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4582 		return (B_TRUE);
4583 	} else if (free_memory < arc_c >> arc_no_grow_shift) {
4584 		arc_no_grow = B_TRUE;
4585 	} else if (gethrtime() >= arc_growtime) {
4586 		arc_no_grow = B_FALSE;
4587 	}
4588 
4589 	/*
4590 	 * Called unconditionally every 60 seconds to reclaim unused
4591 	 * zstd compression and decompression context. This is done
4592 	 * here to avoid the need for an independent thread.
4593 	 */
4594 	if (!((reap_cb_check_counter++) % 60))
4595 		zfs_zstd_cache_reap_now();
4596 
4597 	return (B_FALSE);
4598 }
4599 
4600 /*
4601  * Keep enough free memory in the system by reaping the ARC's kmem
4602  * caches.  To cause more slabs to be reapable, we may reduce the
4603  * target size of the cache (arc_c), causing the arc_evict_cb()
4604  * to free more buffers.
4605  */
4606 static void
4607 arc_reap_cb(void *arg, zthr_t *zthr)
4608 {
4609 	(void) arg, (void) zthr;
4610 
4611 	int64_t free_memory;
4612 	fstrans_cookie_t cookie = spl_fstrans_mark();
4613 
4614 	/*
4615 	 * Kick off asynchronous kmem_reap()'s of all our caches.
4616 	 */
4617 	arc_kmem_reap_soon();
4618 
4619 	/*
4620 	 * Wait at least arc_kmem_cache_reap_retry_ms between
4621 	 * arc_kmem_reap_soon() calls. Without this check it is possible to
4622 	 * end up in a situation where we spend lots of time reaping
4623 	 * caches, while we're near arc_c_min.  Waiting here also gives the
4624 	 * subsequent free memory check a chance of finding that the
4625 	 * asynchronous reap has already freed enough memory, and we don't
4626 	 * need to call arc_reduce_target_size().
4627 	 */
4628 	delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);
4629 
4630 	/*
4631 	 * Reduce the target size as needed to maintain the amount of free
4632 	 * memory in the system at a fraction of the arc_size (1/128th by
4633 	 * default).  If oversubscribed (free_memory < 0) then reduce the
4634 	 * target arc_size by the deficit amount plus the fractional
4635 	 * amount.  If free memory is positive but less than the fractional
4636 	 * amount, reduce by what is needed to hit the fractional amount.
4637 	 */
4638 	free_memory = arc_available_memory();
4639 
4640 	int64_t can_free = arc_c - arc_c_min;
4641 	if (can_free > 0) {
4642 		int64_t to_free = (can_free >> arc_shrink_shift) - free_memory;
4643 		if (to_free > 0)
4644 			arc_reduce_target_size(to_free);
4645 	}
4646 	spl_fstrans_unmark(cookie);
4647 }
4648 
4649 #ifdef _KERNEL
4650 /*
4651  * Determine the amount of memory eligible for eviction contained in the
4652  * ARC. All clean data reported by the ghost lists can always be safely
4653  * evicted. Due to arc_c_min, the same does not hold for all clean data
4654  * contained by the regular mru and mfu lists.
4655  *
4656  * In the case of the regular mru and mfu lists, we need to report as
4657  * much clean data as possible, such that evicting that same reported
4658  * data will not bring arc_size below arc_c_min. Thus, in certain
4659  * circumstances, the total amount of clean data in the mru and mfu
4660  * lists might not actually be evictable.
4661  *
4662  * The following two distinct cases are accounted for:
4663  *
4664  * 1. The sum of the amount of dirty data contained by both the mru and
4665  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
4666  *    is greater than or equal to arc_c_min.
4667  *    (i.e. amount of dirty data >= arc_c_min)
4668  *
4669  *    This is the easy case; all clean data contained by the mru and mfu
4670  *    lists is evictable. Evicting all clean data can only drop arc_size
4671  *    to the amount of dirty data, which is greater than arc_c_min.
4672  *
4673  * 2. The sum of the amount of dirty data contained by both the mru and
4674  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
4675  *    is less than arc_c_min.
4676  *    (i.e. arc_c_min > amount of dirty data)
4677  *
4678  *    2.1. arc_size is greater than or equal arc_c_min.
4679  *         (i.e. arc_size >= arc_c_min > amount of dirty data)
4680  *
4681  *         In this case, not all clean data from the regular mru and mfu
4682  *         lists is actually evictable; we must leave enough clean data
4683  *         to keep arc_size above arc_c_min. Thus, the maximum amount of
4684  *         evictable data from the two lists combined, is exactly the
4685  *         difference between arc_size and arc_c_min.
4686  *
4687  *    2.2. arc_size is less than arc_c_min
4688  *         (i.e. arc_c_min > arc_size > amount of dirty data)
4689  *
4690  *         In this case, none of the data contained in the mru and mfu
4691  *         lists is evictable, even if it's clean. Since arc_size is
4692  *         already below arc_c_min, evicting any more would only
4693  *         increase this negative difference.
4694  */
4695 
4696 #endif /* _KERNEL */
4697 
4698 /*
4699  * Adapt arc info given the number of bytes we are trying to add and
4700  * the state that we are coming from.  This function is only called
4701  * when we are adding new content to the cache.
4702  */
4703 static void
4704 arc_adapt(uint64_t bytes)
4705 {
4706 	/*
4707 	 * Wake reap thread if we do not have any available memory
4708 	 */
4709 	if (arc_reclaim_needed()) {
4710 		zthr_wakeup(arc_reap_zthr);
4711 		return;
4712 	}
4713 
4714 	if (arc_no_grow)
4715 		return;
4716 
4717 	if (arc_c >= arc_c_max)
4718 		return;
4719 
4720 	/*
4721 	 * If we're within (2 * maxblocksize) bytes of the target
4722 	 * cache size, increment the target cache size
4723 	 */
4724 	if (aggsum_upper_bound(&arc_sums.arcstat_size) +
4725 	    2 * SPA_MAXBLOCKSIZE >= arc_c) {
4726 		uint64_t dc = MAX(bytes, SPA_OLD_MAXBLOCKSIZE);
4727 		if (atomic_add_64_nv(&arc_c, dc) > arc_c_max)
4728 			arc_c = arc_c_max;
4729 	}
4730 }
4731 
4732 /*
4733  * Check if arc_size has grown past our upper threshold, determined by
4734  * zfs_arc_overflow_shift.
4735  */
4736 static arc_ovf_level_t
4737 arc_is_overflowing(boolean_t use_reserve)
4738 {
4739 	/* Always allow at least one block of overflow */
4740 	int64_t overflow = MAX(SPA_MAXBLOCKSIZE,
4741 	    arc_c >> zfs_arc_overflow_shift);
4742 
4743 	/*
4744 	 * We just compare the lower bound here for performance reasons. Our
4745 	 * primary goals are to make sure that the arc never grows without
4746 	 * bound, and that it can reach its maximum size. This check
4747 	 * accomplishes both goals. The maximum amount we could run over by is
4748 	 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
4749 	 * in the ARC. In practice, that's in the tens of MB, which is low
4750 	 * enough to be safe.
4751 	 */
4752 	int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) -
4753 	    arc_c - overflow / 2;
4754 	if (!use_reserve)
4755 		overflow /= 2;
4756 	return (over < 0 ? ARC_OVF_NONE :
4757 	    over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
4758 }
4759 
4760 static abd_t *
4761 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
4762     int alloc_flags)
4763 {
4764 	arc_buf_contents_t type = arc_buf_type(hdr);
4765 
4766 	arc_get_data_impl(hdr, size, tag, alloc_flags);
4767 	if (alloc_flags & ARC_HDR_ALLOC_LINEAR)
4768 		return (abd_alloc_linear(size, type == ARC_BUFC_METADATA));
4769 	else
4770 		return (abd_alloc(size, type == ARC_BUFC_METADATA));
4771 }
4772 
4773 static void *
4774 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
4775 {
4776 	arc_buf_contents_t type = arc_buf_type(hdr);
4777 
4778 	arc_get_data_impl(hdr, size, tag, 0);
4779 	if (type == ARC_BUFC_METADATA) {
4780 		return (zio_buf_alloc(size));
4781 	} else {
4782 		ASSERT(type == ARC_BUFC_DATA);
4783 		return (zio_data_buf_alloc(size));
4784 	}
4785 }
4786 
4787 /*
4788  * Wait for the specified amount of data (in bytes) to be evicted from the
4789  * ARC, and for there to be sufficient free memory in the system.  Waiting for
4790  * eviction ensures that the memory used by the ARC decreases.  Waiting for
4791  * free memory ensures that the system won't run out of free pages, regardless
4792  * of ARC behavior and settings.  See arc_lowmem_init().
4793  */
4794 void
4795 arc_wait_for_eviction(uint64_t amount, boolean_t use_reserve)
4796 {
4797 	switch (arc_is_overflowing(use_reserve)) {
4798 	case ARC_OVF_NONE:
4799 		return;
4800 	case ARC_OVF_SOME:
4801 		/*
4802 		 * This is a bit racy without taking arc_evict_lock, but the
4803 		 * worst that can happen is we either call zthr_wakeup() extra
4804 		 * time due to race with other thread here, or the set flag
4805 		 * get cleared by arc_evict_cb(), which is unlikely due to
4806 		 * big hysteresis, but also not important since at this level
4807 		 * of overflow the eviction is purely advisory.  Same time
4808 		 * taking the global lock here every time without waiting for
4809 		 * the actual eviction creates a significant lock contention.
4810 		 */
4811 		if (!arc_evict_needed) {
4812 			arc_evict_needed = B_TRUE;
4813 			zthr_wakeup(arc_evict_zthr);
4814 		}
4815 		return;
4816 	case ARC_OVF_SEVERE:
4817 	default:
4818 	{
4819 		arc_evict_waiter_t aw;
4820 		list_link_init(&aw.aew_node);
4821 		cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL);
4822 
4823 		uint64_t last_count = 0;
4824 		mutex_enter(&arc_evict_lock);
4825 		if (!list_is_empty(&arc_evict_waiters)) {
4826 			arc_evict_waiter_t *last =
4827 			    list_tail(&arc_evict_waiters);
4828 			last_count = last->aew_count;
4829 		} else if (!arc_evict_needed) {
4830 			arc_evict_needed = B_TRUE;
4831 			zthr_wakeup(arc_evict_zthr);
4832 		}
4833 		/*
4834 		 * Note, the last waiter's count may be less than
4835 		 * arc_evict_count if we are low on memory in which
4836 		 * case arc_evict_state_impl() may have deferred
4837 		 * wakeups (but still incremented arc_evict_count).
4838 		 */
4839 		aw.aew_count = MAX(last_count, arc_evict_count) + amount;
4840 
4841 		list_insert_tail(&arc_evict_waiters, &aw);
4842 
4843 		arc_set_need_free();
4844 
4845 		DTRACE_PROBE3(arc__wait__for__eviction,
4846 		    uint64_t, amount,
4847 		    uint64_t, arc_evict_count,
4848 		    uint64_t, aw.aew_count);
4849 
4850 		/*
4851 		 * We will be woken up either when arc_evict_count reaches
4852 		 * aew_count, or when the ARC is no longer overflowing and
4853 		 * eviction completes.
4854 		 * In case of "false" wakeup, we will still be on the list.
4855 		 */
4856 		do {
4857 			cv_wait(&aw.aew_cv, &arc_evict_lock);
4858 		} while (list_link_active(&aw.aew_node));
4859 		mutex_exit(&arc_evict_lock);
4860 
4861 		cv_destroy(&aw.aew_cv);
4862 	}
4863 	}
4864 }
4865 
4866 /*
4867  * Allocate a block and return it to the caller. If we are hitting the
4868  * hard limit for the cache size, we must sleep, waiting for the eviction
4869  * thread to catch up. If we're past the target size but below the hard
4870  * limit, we'll only signal the reclaim thread and continue on.
4871  */
4872 static void
4873 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
4874     int alloc_flags)
4875 {
4876 	arc_adapt(size);
4877 
4878 	/*
4879 	 * If arc_size is currently overflowing, we must be adding data
4880 	 * faster than we are evicting.  To ensure we don't compound the
4881 	 * problem by adding more data and forcing arc_size to grow even
4882 	 * further past it's target size, we wait for the eviction thread to
4883 	 * make some progress.  We also wait for there to be sufficient free
4884 	 * memory in the system, as measured by arc_free_memory().
4885 	 *
4886 	 * Specifically, we wait for zfs_arc_eviction_pct percent of the
4887 	 * requested size to be evicted.  This should be more than 100%, to
4888 	 * ensure that that progress is also made towards getting arc_size
4889 	 * under arc_c.  See the comment above zfs_arc_eviction_pct.
4890 	 */
4891 	arc_wait_for_eviction(size * zfs_arc_eviction_pct / 100,
4892 	    alloc_flags & ARC_HDR_USE_RESERVE);
4893 
4894 	arc_buf_contents_t type = arc_buf_type(hdr);
4895 	if (type == ARC_BUFC_METADATA) {
4896 		arc_space_consume(size, ARC_SPACE_META);
4897 	} else {
4898 		arc_space_consume(size, ARC_SPACE_DATA);
4899 	}
4900 
4901 	/*
4902 	 * Update the state size.  Note that ghost states have a
4903 	 * "ghost size" and so don't need to be updated.
4904 	 */
4905 	arc_state_t *state = hdr->b_l1hdr.b_state;
4906 	if (!GHOST_STATE(state)) {
4907 
4908 		(void) zfs_refcount_add_many(&state->arcs_size[type], size,
4909 		    tag);
4910 
4911 		/*
4912 		 * If this is reached via arc_read, the link is
4913 		 * protected by the hash lock. If reached via
4914 		 * arc_buf_alloc, the header should not be accessed by
4915 		 * any other thread. And, if reached via arc_read_done,
4916 		 * the hash lock will protect it if it's found in the
4917 		 * hash table; otherwise no other thread should be
4918 		 * trying to [add|remove]_reference it.
4919 		 */
4920 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4921 			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4922 			(void) zfs_refcount_add_many(&state->arcs_esize[type],
4923 			    size, tag);
4924 		}
4925 	}
4926 }
4927 
4928 static void
4929 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,
4930     const void *tag)
4931 {
4932 	arc_free_data_impl(hdr, size, tag);
4933 	abd_free(abd);
4934 }
4935 
4936 static void
4937 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)
4938 {
4939 	arc_buf_contents_t type = arc_buf_type(hdr);
4940 
4941 	arc_free_data_impl(hdr, size, tag);
4942 	if (type == ARC_BUFC_METADATA) {
4943 		zio_buf_free(buf, size);
4944 	} else {
4945 		ASSERT(type == ARC_BUFC_DATA);
4946 		zio_data_buf_free(buf, size);
4947 	}
4948 }
4949 
4950 /*
4951  * Free the arc data buffer.
4952  */
4953 static void
4954 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
4955 {
4956 	arc_state_t *state = hdr->b_l1hdr.b_state;
4957 	arc_buf_contents_t type = arc_buf_type(hdr);
4958 
4959 	/* protected by hash lock, if in the hash table */
4960 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4961 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4962 		ASSERT(state != arc_anon && state != arc_l2c_only);
4963 
4964 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
4965 		    size, tag);
4966 	}
4967 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);
4968 
4969 	VERIFY3U(hdr->b_type, ==, type);
4970 	if (type == ARC_BUFC_METADATA) {
4971 		arc_space_return(size, ARC_SPACE_META);
4972 	} else {
4973 		ASSERT(type == ARC_BUFC_DATA);
4974 		arc_space_return(size, ARC_SPACE_DATA);
4975 	}
4976 }
4977 
4978 /*
4979  * This routine is called whenever a buffer is accessed.
4980  */
4981 static void
4982 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
4983 {
4984 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
4985 	ASSERT(HDR_HAS_L1HDR(hdr));
4986 
4987 	/*
4988 	 * Update buffer prefetch status.
4989 	 */
4990 	boolean_t was_prefetch = HDR_PREFETCH(hdr);
4991 	boolean_t now_prefetch = arc_flags & ARC_FLAG_PREFETCH;
4992 	if (was_prefetch != now_prefetch) {
4993 		if (was_prefetch) {
4994 			ARCSTAT_CONDSTAT(hit, demand_hit, demand_iohit,
4995 			    HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
4996 			    prefetch);
4997 		}
4998 		if (HDR_HAS_L2HDR(hdr))
4999 			l2arc_hdr_arcstats_decrement_state(hdr);
5000 		if (was_prefetch) {
5001 			arc_hdr_clear_flags(hdr,
5002 			    ARC_FLAG_PREFETCH | ARC_FLAG_PRESCIENT_PREFETCH);
5003 		} else {
5004 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5005 		}
5006 		if (HDR_HAS_L2HDR(hdr))
5007 			l2arc_hdr_arcstats_increment_state(hdr);
5008 	}
5009 	if (now_prefetch) {
5010 		if (arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
5011 			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5012 			ARCSTAT_BUMP(arcstat_prescient_prefetch);
5013 		} else {
5014 			ARCSTAT_BUMP(arcstat_predictive_prefetch);
5015 		}
5016 	}
5017 	if (arc_flags & ARC_FLAG_L2CACHE)
5018 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5019 
5020 	clock_t now = ddi_get_lbolt();
5021 	if (hdr->b_l1hdr.b_state == arc_anon) {
5022 		arc_state_t	*new_state;
5023 		/*
5024 		 * This buffer is not in the cache, and does not appear in
5025 		 * our "ghost" lists.  Add it to the MRU or uncached state.
5026 		 */
5027 		ASSERT0(hdr->b_l1hdr.b_arc_access);
5028 		hdr->b_l1hdr.b_arc_access = now;
5029 		if (HDR_UNCACHED(hdr)) {
5030 			new_state = arc_uncached;
5031 			DTRACE_PROBE1(new_state__uncached, arc_buf_hdr_t *,
5032 			    hdr);
5033 		} else {
5034 			new_state = arc_mru;
5035 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5036 		}
5037 		arc_change_state(new_state, hdr);
5038 	} else if (hdr->b_l1hdr.b_state == arc_mru) {
5039 		/*
5040 		 * This buffer has been accessed once recently and either
5041 		 * its read is still in progress or it is in the cache.
5042 		 */
5043 		if (HDR_IO_IN_PROGRESS(hdr)) {
5044 			hdr->b_l1hdr.b_arc_access = now;
5045 			return;
5046 		}
5047 		hdr->b_l1hdr.b_mru_hits++;
5048 		ARCSTAT_BUMP(arcstat_mru_hits);
5049 
5050 		/*
5051 		 * If the previous access was a prefetch, then it already
5052 		 * handled possible promotion, so nothing more to do for now.
5053 		 */
5054 		if (was_prefetch) {
5055 			hdr->b_l1hdr.b_arc_access = now;
5056 			return;
5057 		}
5058 
5059 		/*
5060 		 * If more than ARC_MINTIME have passed from the previous
5061 		 * hit, promote the buffer to the MFU state.
5062 		 */
5063 		if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
5064 		    ARC_MINTIME)) {
5065 			hdr->b_l1hdr.b_arc_access = now;
5066 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5067 			arc_change_state(arc_mfu, hdr);
5068 		}
5069 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5070 		arc_state_t	*new_state;
5071 		/*
5072 		 * This buffer has been accessed once recently, but was
5073 		 * evicted from the cache.  Would we have bigger MRU, it
5074 		 * would be an MRU hit, so handle it the same way, except
5075 		 * we don't need to check the previous access time.
5076 		 */
5077 		hdr->b_l1hdr.b_mru_ghost_hits++;
5078 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
5079 		hdr->b_l1hdr.b_arc_access = now;
5080 		wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
5081 		    arc_hdr_size(hdr));
5082 		if (was_prefetch) {
5083 			new_state = arc_mru;
5084 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5085 		} else {
5086 			new_state = arc_mfu;
5087 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5088 		}
5089 		arc_change_state(new_state, hdr);
5090 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
5091 		/*
5092 		 * This buffer has been accessed more than once and either
5093 		 * still in the cache or being restored from one of ghosts.
5094 		 */
5095 		if (!HDR_IO_IN_PROGRESS(hdr)) {
5096 			hdr->b_l1hdr.b_mfu_hits++;
5097 			ARCSTAT_BUMP(arcstat_mfu_hits);
5098 		}
5099 		hdr->b_l1hdr.b_arc_access = now;
5100 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5101 		/*
5102 		 * This buffer has been accessed more than once recently, but
5103 		 * has been evicted from the cache.  Would we have bigger MFU
5104 		 * it would stay in cache, so move it back to MFU state.
5105 		 */
5106 		hdr->b_l1hdr.b_mfu_ghost_hits++;
5107 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
5108 		hdr->b_l1hdr.b_arc_access = now;
5109 		wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
5110 		    arc_hdr_size(hdr));
5111 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5112 		arc_change_state(arc_mfu, hdr);
5113 	} else if (hdr->b_l1hdr.b_state == arc_uncached) {
5114 		/*
5115 		 * This buffer is uncacheable, but we got a hit.  Probably
5116 		 * a demand read after prefetch.  Nothing more to do here.
5117 		 */
5118 		if (!HDR_IO_IN_PROGRESS(hdr))
5119 			ARCSTAT_BUMP(arcstat_uncached_hits);
5120 		hdr->b_l1hdr.b_arc_access = now;
5121 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5122 		/*
5123 		 * This buffer is on the 2nd Level ARC and was not accessed
5124 		 * for a long time, so treat it as new and put into MRU.
5125 		 */
5126 		hdr->b_l1hdr.b_arc_access = now;
5127 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5128 		arc_change_state(arc_mru, hdr);
5129 	} else {
5130 		cmn_err(CE_PANIC, "invalid arc state 0x%p",
5131 		    hdr->b_l1hdr.b_state);
5132 	}
5133 }
5134 
5135 /*
5136  * This routine is called by dbuf_hold() to update the arc_access() state
5137  * which otherwise would be skipped for entries in the dbuf cache.
5138  */
5139 void
5140 arc_buf_access(arc_buf_t *buf)
5141 {
5142 	arc_buf_hdr_t *hdr = buf->b_hdr;
5143 
5144 	/*
5145 	 * Avoid taking the hash_lock when possible as an optimization.
5146 	 * The header must be checked again under the hash_lock in order
5147 	 * to handle the case where it is concurrently being released.
5148 	 */
5149 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
5150 		return;
5151 
5152 	kmutex_t *hash_lock = HDR_LOCK(hdr);
5153 	mutex_enter(hash_lock);
5154 
5155 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5156 		mutex_exit(hash_lock);
5157 		ARCSTAT_BUMP(arcstat_access_skip);
5158 		return;
5159 	}
5160 
5161 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5162 	    hdr->b_l1hdr.b_state == arc_mfu ||
5163 	    hdr->b_l1hdr.b_state == arc_uncached);
5164 
5165 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5166 	arc_access(hdr, 0, B_TRUE);
5167 	mutex_exit(hash_lock);
5168 
5169 	ARCSTAT_BUMP(arcstat_hits);
5170 	ARCSTAT_CONDSTAT(B_TRUE /* demand */, demand, prefetch,
5171 	    !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5172 }
5173 
5174 /* a generic arc_read_done_func_t which you can use */
5175 void
5176 arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5177     arc_buf_t *buf, void *arg)
5178 {
5179 	(void) zio, (void) zb, (void) bp;
5180 
5181 	if (buf == NULL)
5182 		return;
5183 
5184 	memcpy(arg, buf->b_data, arc_buf_size(buf));
5185 	arc_buf_destroy(buf, arg);
5186 }
5187 
5188 /* a generic arc_read_done_func_t */
5189 void
5190 arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5191     arc_buf_t *buf, void *arg)
5192 {
5193 	(void) zb, (void) bp;
5194 	arc_buf_t **bufp = arg;
5195 
5196 	if (buf == NULL) {
5197 		ASSERT(zio == NULL || zio->io_error != 0);
5198 		*bufp = NULL;
5199 	} else {
5200 		ASSERT(zio == NULL || zio->io_error == 0);
5201 		*bufp = buf;
5202 		ASSERT(buf->b_data != NULL);
5203 	}
5204 }
5205 
5206 static void
5207 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5208 {
5209 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5210 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5211 		ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
5212 	} else {
5213 		if (HDR_COMPRESSION_ENABLED(hdr)) {
5214 			ASSERT3U(arc_hdr_get_compress(hdr), ==,
5215 			    BP_GET_COMPRESS(bp));
5216 		}
5217 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5218 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5219 		ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
5220 	}
5221 }
5222 
5223 static void
5224 arc_read_done(zio_t *zio)
5225 {
5226 	blkptr_t 	*bp = zio->io_bp;
5227 	arc_buf_hdr_t	*hdr = zio->io_private;
5228 	kmutex_t	*hash_lock = NULL;
5229 	arc_callback_t	*callback_list;
5230 	arc_callback_t	*acb;
5231 
5232 	/*
5233 	 * The hdr was inserted into hash-table and removed from lists
5234 	 * prior to starting I/O.  We should find this header, since
5235 	 * it's in the hash table, and it should be legit since it's
5236 	 * not possible to evict it during the I/O.  The only possible
5237 	 * reason for it not to be found is if we were freed during the
5238 	 * read.
5239 	 */
5240 	if (HDR_IN_HASH_TABLE(hdr)) {
5241 		arc_buf_hdr_t *found;
5242 
5243 		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
5244 		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5245 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5246 		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5247 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5248 
5249 		found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
5250 
5251 		ASSERT((found == hdr &&
5252 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5253 		    (found == hdr && HDR_L2_READING(hdr)));
5254 		ASSERT3P(hash_lock, !=, NULL);
5255 	}
5256 
5257 	if (BP_IS_PROTECTED(bp)) {
5258 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
5259 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
5260 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
5261 		    hdr->b_crypt_hdr.b_iv);
5262 
5263 		if (zio->io_error == 0) {
5264 			if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) {
5265 				void *tmpbuf;
5266 
5267 				tmpbuf = abd_borrow_buf_copy(zio->io_abd,
5268 				    sizeof (zil_chain_t));
5269 				zio_crypt_decode_mac_zil(tmpbuf,
5270 				    hdr->b_crypt_hdr.b_mac);
5271 				abd_return_buf(zio->io_abd, tmpbuf,
5272 				    sizeof (zil_chain_t));
5273 			} else {
5274 				zio_crypt_decode_mac_bp(bp,
5275 				    hdr->b_crypt_hdr.b_mac);
5276 			}
5277 		}
5278 	}
5279 
5280 	if (zio->io_error == 0) {
5281 		/* byteswap if necessary */
5282 		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5283 			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5284 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5285 			} else {
5286 				hdr->b_l1hdr.b_byteswap =
5287 				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5288 			}
5289 		} else {
5290 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5291 		}
5292 		if (!HDR_L2_READING(hdr)) {
5293 			hdr->b_complevel = zio->io_prop.zp_complevel;
5294 		}
5295 	}
5296 
5297 	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5298 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5299 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5300 
5301 	callback_list = hdr->b_l1hdr.b_acb;
5302 	ASSERT3P(callback_list, !=, NULL);
5303 	hdr->b_l1hdr.b_acb = NULL;
5304 
5305 	/*
5306 	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5307 	 * make a buf containing the data according to the parameters which were
5308 	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5309 	 * aren't needlessly decompressing the data multiple times.
5310 	 */
5311 	int callback_cnt = 0;
5312 	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5313 
5314 		/* We need the last one to call below in original order. */
5315 		callback_list = acb;
5316 
5317 		if (!acb->acb_done || acb->acb_nobuf)
5318 			continue;
5319 
5320 		callback_cnt++;
5321 
5322 		if (zio->io_error != 0)
5323 			continue;
5324 
5325 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,
5326 		    &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
5327 		    acb->acb_compressed, acb->acb_noauth, B_TRUE,
5328 		    &acb->acb_buf);
5329 
5330 		/*
5331 		 * Assert non-speculative zios didn't fail because an
5332 		 * encryption key wasn't loaded
5333 		 */
5334 		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
5335 		    error != EACCES);
5336 
5337 		/*
5338 		 * If we failed to decrypt, report an error now (as the zio
5339 		 * layer would have done if it had done the transforms).
5340 		 */
5341 		if (error == ECKSUM) {
5342 			ASSERT(BP_IS_PROTECTED(bp));
5343 			error = SET_ERROR(EIO);
5344 			if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5345 				spa_log_error(zio->io_spa, &acb->acb_zb,
5346 				    &zio->io_bp->blk_birth);
5347 				(void) zfs_ereport_post(
5348 				    FM_EREPORT_ZFS_AUTHENTICATION,
5349 				    zio->io_spa, NULL, &acb->acb_zb, zio, 0);
5350 			}
5351 		}
5352 
5353 		if (error != 0) {
5354 			/*
5355 			 * Decompression or decryption failed.  Set
5356 			 * io_error so that when we call acb_done
5357 			 * (below), we will indicate that the read
5358 			 * failed. Note that in the unusual case
5359 			 * where one callback is compressed and another
5360 			 * uncompressed, we will mark all of them
5361 			 * as failed, even though the uncompressed
5362 			 * one can't actually fail.  In this case,
5363 			 * the hdr will not be anonymous, because
5364 			 * if there are multiple callbacks, it's
5365 			 * because multiple threads found the same
5366 			 * arc buf in the hash table.
5367 			 */
5368 			zio->io_error = error;
5369 		}
5370 	}
5371 
5372 	/*
5373 	 * If there are multiple callbacks, we must have the hash lock,
5374 	 * because the only way for multiple threads to find this hdr is
5375 	 * in the hash table.  This ensures that if there are multiple
5376 	 * callbacks, the hdr is not anonymous.  If it were anonymous,
5377 	 * we couldn't use arc_buf_destroy() in the error case below.
5378 	 */
5379 	ASSERT(callback_cnt < 2 || hash_lock != NULL);
5380 
5381 	if (zio->io_error == 0) {
5382 		arc_hdr_verify(hdr, zio->io_bp);
5383 	} else {
5384 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5385 		if (hdr->b_l1hdr.b_state != arc_anon)
5386 			arc_change_state(arc_anon, hdr);
5387 		if (HDR_IN_HASH_TABLE(hdr))
5388 			buf_hash_remove(hdr);
5389 	}
5390 
5391 	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5392 	(void) remove_reference(hdr, hdr);
5393 
5394 	if (hash_lock != NULL)
5395 		mutex_exit(hash_lock);
5396 
5397 	/* execute each callback and free its structure */
5398 	while ((acb = callback_list) != NULL) {
5399 		if (acb->acb_done != NULL) {
5400 			if (zio->io_error != 0 && acb->acb_buf != NULL) {
5401 				/*
5402 				 * If arc_buf_alloc_impl() fails during
5403 				 * decompression, the buf will still be
5404 				 * allocated, and needs to be freed here.
5405 				 */
5406 				arc_buf_destroy(acb->acb_buf,
5407 				    acb->acb_private);
5408 				acb->acb_buf = NULL;
5409 			}
5410 			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5411 			    acb->acb_buf, acb->acb_private);
5412 		}
5413 
5414 		if (acb->acb_zio_dummy != NULL) {
5415 			acb->acb_zio_dummy->io_error = zio->io_error;
5416 			zio_nowait(acb->acb_zio_dummy);
5417 		}
5418 
5419 		callback_list = acb->acb_prev;
5420 		if (acb->acb_wait) {
5421 			mutex_enter(&acb->acb_wait_lock);
5422 			acb->acb_wait_error = zio->io_error;
5423 			acb->acb_wait = B_FALSE;
5424 			cv_signal(&acb->acb_wait_cv);
5425 			mutex_exit(&acb->acb_wait_lock);
5426 			/* acb will be freed by the waiting thread. */
5427 		} else {
5428 			kmem_free(acb, sizeof (arc_callback_t));
5429 		}
5430 	}
5431 }
5432 
5433 /*
5434  * "Read" the block at the specified DVA (in bp) via the
5435  * cache.  If the block is found in the cache, invoke the provided
5436  * callback immediately and return.  Note that the `zio' parameter
5437  * in the callback will be NULL in this case, since no IO was
5438  * required.  If the block is not in the cache pass the read request
5439  * on to the spa with a substitute callback function, so that the
5440  * requested block will be added to the cache.
5441  *
5442  * If a read request arrives for a block that has a read in-progress,
5443  * either wait for the in-progress read to complete (and return the
5444  * results); or, if this is a read with a "done" func, add a record
5445  * to the read to invoke the "done" func when the read completes,
5446  * and return; or just return.
5447  *
5448  * arc_read_done() will invoke all the requested "done" functions
5449  * for readers of this block.
5450  */
5451 int
5452 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
5453     arc_read_done_func_t *done, void *private, zio_priority_t priority,
5454     int zio_flags, arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5455 {
5456 	arc_buf_hdr_t *hdr = NULL;
5457 	kmutex_t *hash_lock = NULL;
5458 	zio_t *rzio;
5459 	uint64_t guid = spa_load_guid(spa);
5460 	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0;
5461 	boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) &&
5462 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5463 	boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) &&
5464 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5465 	boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
5466 	boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
5467 	arc_buf_t *buf = NULL;
5468 	int rc = 0;
5469 
5470 	ASSERT(!embedded_bp ||
5471 	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5472 	ASSERT(!BP_IS_HOLE(bp));
5473 	ASSERT(!BP_IS_REDACTED(bp));
5474 
5475 	/*
5476 	 * Normally SPL_FSTRANS will already be set since kernel threads which
5477 	 * expect to call the DMU interfaces will set it when created.  System
5478 	 * calls are similarly handled by setting/cleaning the bit in the
5479 	 * registered callback (module/os/.../zfs/zpl_*).
5480 	 *
5481 	 * External consumers such as Lustre which call the exported DMU
5482 	 * interfaces may not have set SPL_FSTRANS.  To avoid a deadlock
5483 	 * on the hash_lock always set and clear the bit.
5484 	 */
5485 	fstrans_cookie_t cookie = spl_fstrans_mark();
5486 top:
5487 	/*
5488 	 * Verify the block pointer contents are reasonable.  This should
5489 	 * always be the case since the blkptr is protected by a checksum.
5490 	 * However, if there is damage it's desirable to detect this early
5491 	 * and treat it as a checksum error.  This allows an alternate blkptr
5492 	 * to be tried when one is available (e.g. ditto blocks).
5493 	 */
5494 	if (!zfs_blkptr_verify(spa, bp, (zio_flags & ZIO_FLAG_CONFIG_WRITER) ?
5495 	    BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
5496 		rc = SET_ERROR(ECKSUM);
5497 		goto done;
5498 	}
5499 
5500 	if (!embedded_bp) {
5501 		/*
5502 		 * Embedded BP's have no DVA and require no I/O to "read".
5503 		 * Create an anonymous arc buf to back it.
5504 		 */
5505 		hdr = buf_hash_find(guid, bp, &hash_lock);
5506 	}
5507 
5508 	/*
5509 	 * Determine if we have an L1 cache hit or a cache miss. For simplicity
5510 	 * we maintain encrypted data separately from compressed / uncompressed
5511 	 * data. If the user is requesting raw encrypted data and we don't have
5512 	 * that in the header we will read from disk to guarantee that we can
5513 	 * get it even if the encryption keys aren't loaded.
5514 	 */
5515 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
5516 	    (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
5517 		boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);
5518 
5519 		if (HDR_IO_IN_PROGRESS(hdr)) {
5520 			if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
5521 				mutex_exit(hash_lock);
5522 				ARCSTAT_BUMP(arcstat_cached_only_in_progress);
5523 				rc = SET_ERROR(ENOENT);
5524 				goto done;
5525 			}
5526 
5527 			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5528 			ASSERT3P(head_zio, !=, NULL);
5529 			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5530 			    priority == ZIO_PRIORITY_SYNC_READ) {
5531 				/*
5532 				 * This is a sync read that needs to wait for
5533 				 * an in-flight async read. Request that the
5534 				 * zio have its priority upgraded.
5535 				 */
5536 				zio_change_priority(head_zio, priority);
5537 				DTRACE_PROBE1(arc__async__upgrade__sync,
5538 				    arc_buf_hdr_t *, hdr);
5539 				ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5540 			}
5541 
5542 			DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
5543 			arc_access(hdr, *arc_flags, B_FALSE);
5544 
5545 			/*
5546 			 * If there are multiple threads reading the same block
5547 			 * and that block is not yet in the ARC, then only one
5548 			 * thread will do the physical I/O and all other
5549 			 * threads will wait until that I/O completes.
5550 			 * Synchronous reads use the acb_wait_cv whereas nowait
5551 			 * reads register a callback. Both are signalled/called
5552 			 * in arc_read_done.
5553 			 *
5554 			 * Errors of the physical I/O may need to be propagated.
5555 			 * Synchronous read errors are returned here from
5556 			 * arc_read_done via acb_wait_error.  Nowait reads
5557 			 * attach the acb_zio_dummy zio to pio and
5558 			 * arc_read_done propagates the physical I/O's io_error
5559 			 * to acb_zio_dummy, and thereby to pio.
5560 			 */
5561 			arc_callback_t *acb = NULL;
5562 			if (done || pio || *arc_flags & ARC_FLAG_WAIT) {
5563 				acb = kmem_zalloc(sizeof (arc_callback_t),
5564 				    KM_SLEEP);
5565 				acb->acb_done = done;
5566 				acb->acb_private = private;
5567 				acb->acb_compressed = compressed_read;
5568 				acb->acb_encrypted = encrypted_read;
5569 				acb->acb_noauth = noauth_read;
5570 				acb->acb_nobuf = no_buf;
5571 				if (*arc_flags & ARC_FLAG_WAIT) {
5572 					acb->acb_wait = B_TRUE;
5573 					mutex_init(&acb->acb_wait_lock, NULL,
5574 					    MUTEX_DEFAULT, NULL);
5575 					cv_init(&acb->acb_wait_cv, NULL,
5576 					    CV_DEFAULT, NULL);
5577 				}
5578 				acb->acb_zb = *zb;
5579 				if (pio != NULL) {
5580 					acb->acb_zio_dummy = zio_null(pio,
5581 					    spa, NULL, NULL, NULL, zio_flags);
5582 				}
5583 				acb->acb_zio_head = head_zio;
5584 				acb->acb_next = hdr->b_l1hdr.b_acb;
5585 				hdr->b_l1hdr.b_acb->acb_prev = acb;
5586 				hdr->b_l1hdr.b_acb = acb;
5587 			}
5588 			mutex_exit(hash_lock);
5589 
5590 			ARCSTAT_BUMP(arcstat_iohits);
5591 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5592 			    demand, prefetch, is_data, data, metadata, iohits);
5593 
5594 			if (*arc_flags & ARC_FLAG_WAIT) {
5595 				mutex_enter(&acb->acb_wait_lock);
5596 				while (acb->acb_wait) {
5597 					cv_wait(&acb->acb_wait_cv,
5598 					    &acb->acb_wait_lock);
5599 				}
5600 				rc = acb->acb_wait_error;
5601 				mutex_exit(&acb->acb_wait_lock);
5602 				mutex_destroy(&acb->acb_wait_lock);
5603 				cv_destroy(&acb->acb_wait_cv);
5604 				kmem_free(acb, sizeof (arc_callback_t));
5605 			}
5606 			goto out;
5607 		}
5608 
5609 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5610 		    hdr->b_l1hdr.b_state == arc_mfu ||
5611 		    hdr->b_l1hdr.b_state == arc_uncached);
5612 
5613 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5614 		arc_access(hdr, *arc_flags, B_TRUE);
5615 
5616 		if (done && !no_buf) {
5617 			ASSERT(!embedded_bp || !BP_IS_HOLE(bp));
5618 
5619 			/* Get a buf with the desired data in it. */
5620 			rc = arc_buf_alloc_impl(hdr, spa, zb, private,
5621 			    encrypted_read, compressed_read, noauth_read,
5622 			    B_TRUE, &buf);
5623 			if (rc == ECKSUM) {
5624 				/*
5625 				 * Convert authentication and decryption errors
5626 				 * to EIO (and generate an ereport if needed)
5627 				 * before leaving the ARC.
5628 				 */
5629 				rc = SET_ERROR(EIO);
5630 				if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5631 					spa_log_error(spa, zb, &hdr->b_birth);
5632 					(void) zfs_ereport_post(
5633 					    FM_EREPORT_ZFS_AUTHENTICATION,
5634 					    spa, NULL, zb, NULL, 0);
5635 				}
5636 			}
5637 			if (rc != 0) {
5638 				arc_buf_destroy_impl(buf);
5639 				buf = NULL;
5640 				(void) remove_reference(hdr, private);
5641 			}
5642 
5643 			/* assert any errors weren't due to unloaded keys */
5644 			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
5645 			    rc != EACCES);
5646 		}
5647 		mutex_exit(hash_lock);
5648 		ARCSTAT_BUMP(arcstat_hits);
5649 		ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5650 		    demand, prefetch, is_data, data, metadata, hits);
5651 		*arc_flags |= ARC_FLAG_CACHED;
5652 		goto done;
5653 	} else {
5654 		uint64_t lsize = BP_GET_LSIZE(bp);
5655 		uint64_t psize = BP_GET_PSIZE(bp);
5656 		arc_callback_t *acb;
5657 		vdev_t *vd = NULL;
5658 		uint64_t addr = 0;
5659 		boolean_t devw = B_FALSE;
5660 		uint64_t size;
5661 		abd_t *hdr_abd;
5662 		int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;
5663 		arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5664 
5665 		if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
5666 			if (hash_lock != NULL)
5667 				mutex_exit(hash_lock);
5668 			rc = SET_ERROR(ENOENT);
5669 			goto done;
5670 		}
5671 
5672 		if (hdr == NULL) {
5673 			/*
5674 			 * This block is not in the cache or it has
5675 			 * embedded data.
5676 			 */
5677 			arc_buf_hdr_t *exists = NULL;
5678 			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
5679 			    BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type);
5680 
5681 			if (!embedded_bp) {
5682 				hdr->b_dva = *BP_IDENTITY(bp);
5683 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
5684 				exists = buf_hash_insert(hdr, &hash_lock);
5685 			}
5686 			if (exists != NULL) {
5687 				/* somebody beat us to the hash insert */
5688 				mutex_exit(hash_lock);
5689 				buf_discard_identity(hdr);
5690 				arc_hdr_destroy(hdr);
5691 				goto top; /* restart the IO request */
5692 			}
5693 		} else {
5694 			/*
5695 			 * This block is in the ghost cache or encrypted data
5696 			 * was requested and we didn't have it. If it was
5697 			 * L2-only (and thus didn't have an L1 hdr),
5698 			 * we realloc the header to add an L1 hdr.
5699 			 */
5700 			if (!HDR_HAS_L1HDR(hdr)) {
5701 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5702 				    hdr_full_cache);
5703 			}
5704 
5705 			if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
5706 				ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5707 				ASSERT(!HDR_HAS_RABD(hdr));
5708 				ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5709 				ASSERT0(zfs_refcount_count(
5710 				    &hdr->b_l1hdr.b_refcnt));
5711 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5712 #ifdef ZFS_DEBUG
5713 				ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5714 #endif
5715 			} else if (HDR_IO_IN_PROGRESS(hdr)) {
5716 				/*
5717 				 * If this header already had an IO in progress
5718 				 * and we are performing another IO to fetch
5719 				 * encrypted data we must wait until the first
5720 				 * IO completes so as not to confuse
5721 				 * arc_read_done(). This should be very rare
5722 				 * and so the performance impact shouldn't
5723 				 * matter.
5724 				 */
5725 				arc_callback_t *acb = kmem_zalloc(
5726 				    sizeof (arc_callback_t), KM_SLEEP);
5727 				acb->acb_wait = B_TRUE;
5728 				mutex_init(&acb->acb_wait_lock, NULL,
5729 				    MUTEX_DEFAULT, NULL);
5730 				cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
5731 				    NULL);
5732 				acb->acb_zio_head =
5733 				    hdr->b_l1hdr.b_acb->acb_zio_head;
5734 				acb->acb_next = hdr->b_l1hdr.b_acb;
5735 				hdr->b_l1hdr.b_acb->acb_prev = acb;
5736 				hdr->b_l1hdr.b_acb = acb;
5737 				mutex_exit(hash_lock);
5738 				mutex_enter(&acb->acb_wait_lock);
5739 				while (acb->acb_wait) {
5740 					cv_wait(&acb->acb_wait_cv,
5741 					    &acb->acb_wait_lock);
5742 				}
5743 				mutex_exit(&acb->acb_wait_lock);
5744 				mutex_destroy(&acb->acb_wait_lock);
5745 				cv_destroy(&acb->acb_wait_cv);
5746 				kmem_free(acb, sizeof (arc_callback_t));
5747 				goto top;
5748 			}
5749 		}
5750 		if (*arc_flags & ARC_FLAG_UNCACHED) {
5751 			arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
5752 			if (!encrypted_read)
5753 				alloc_flags |= ARC_HDR_ALLOC_LINEAR;
5754 		}
5755 
5756 		/*
5757 		 * Take additional reference for IO_IN_PROGRESS.  It stops
5758 		 * arc_access() from putting this header without any buffers
5759 		 * and so other references but obviously nonevictable onto
5760 		 * the evictable list of MRU or MFU state.
5761 		 */
5762 		add_reference(hdr, hdr);
5763 		if (!embedded_bp)
5764 			arc_access(hdr, *arc_flags, B_FALSE);
5765 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5766 		arc_hdr_alloc_abd(hdr, alloc_flags);
5767 		if (encrypted_read) {
5768 			ASSERT(HDR_HAS_RABD(hdr));
5769 			size = HDR_GET_PSIZE(hdr);
5770 			hdr_abd = hdr->b_crypt_hdr.b_rabd;
5771 			zio_flags |= ZIO_FLAG_RAW;
5772 		} else {
5773 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
5774 			size = arc_hdr_size(hdr);
5775 			hdr_abd = hdr->b_l1hdr.b_pabd;
5776 
5777 			if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
5778 				zio_flags |= ZIO_FLAG_RAW_COMPRESS;
5779 			}
5780 
5781 			/*
5782 			 * For authenticated bp's, we do not ask the ZIO layer
5783 			 * to authenticate them since this will cause the entire
5784 			 * IO to fail if the key isn't loaded. Instead, we
5785 			 * defer authentication until arc_buf_fill(), which will
5786 			 * verify the data when the key is available.
5787 			 */
5788 			if (BP_IS_AUTHENTICATED(bp))
5789 				zio_flags |= ZIO_FLAG_RAW_ENCRYPT;
5790 		}
5791 
5792 		if (BP_IS_AUTHENTICATED(bp))
5793 			arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
5794 		if (BP_GET_LEVEL(bp) > 0)
5795 			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
5796 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
5797 
5798 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
5799 		acb->acb_done = done;
5800 		acb->acb_private = private;
5801 		acb->acb_compressed = compressed_read;
5802 		acb->acb_encrypted = encrypted_read;
5803 		acb->acb_noauth = noauth_read;
5804 		acb->acb_zb = *zb;
5805 
5806 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5807 		hdr->b_l1hdr.b_acb = acb;
5808 
5809 		if (HDR_HAS_L2HDR(hdr) &&
5810 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
5811 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
5812 			addr = hdr->b_l2hdr.b_daddr;
5813 			/*
5814 			 * Lock out L2ARC device removal.
5815 			 */
5816 			if (vdev_is_dead(vd) ||
5817 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
5818 				vd = NULL;
5819 		}
5820 
5821 		/*
5822 		 * We count both async reads and scrub IOs as asynchronous so
5823 		 * that both can be upgraded in the event of a cache hit while
5824 		 * the read IO is still in-flight.
5825 		 */
5826 		if (priority == ZIO_PRIORITY_ASYNC_READ ||
5827 		    priority == ZIO_PRIORITY_SCRUB)
5828 			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5829 		else
5830 			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
5831 
5832 		/*
5833 		 * At this point, we have a level 1 cache miss or a blkptr
5834 		 * with embedded data.  Try again in L2ARC if possible.
5835 		 */
5836 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
5837 
5838 		/*
5839 		 * Skip ARC stat bump for block pointers with embedded
5840 		 * data. The data are read from the blkptr itself via
5841 		 * decode_embedded_bp_compressed().
5842 		 */
5843 		if (!embedded_bp) {
5844 			DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
5845 			    blkptr_t *, bp, uint64_t, lsize,
5846 			    zbookmark_phys_t *, zb);
5847 			ARCSTAT_BUMP(arcstat_misses);
5848 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5849 			    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
5850 			    metadata, misses);
5851 			zfs_racct_read(size, 1);
5852 		}
5853 
5854 		/* Check if the spa even has l2 configured */
5855 		const boolean_t spa_has_l2 = l2arc_ndev != 0 &&
5856 		    spa->spa_l2cache.sav_count > 0;
5857 
5858 		if (vd != NULL && spa_has_l2 && !(l2arc_norw && devw)) {
5859 			/*
5860 			 * Read from the L2ARC if the following are true:
5861 			 * 1. The L2ARC vdev was previously cached.
5862 			 * 2. This buffer still has L2ARC metadata.
5863 			 * 3. This buffer isn't currently writing to the L2ARC.
5864 			 * 4. The L2ARC entry wasn't evicted, which may
5865 			 *    also have invalidated the vdev.
5866 			 * 5. This isn't prefetch or l2arc_noprefetch is 0.
5867 			 */
5868 			if (HDR_HAS_L2HDR(hdr) &&
5869 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
5870 			    !(l2arc_noprefetch &&
5871 			    (*arc_flags & ARC_FLAG_PREFETCH))) {
5872 				l2arc_read_callback_t *cb;
5873 				abd_t *abd;
5874 				uint64_t asize;
5875 
5876 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
5877 				ARCSTAT_BUMP(arcstat_l2_hits);
5878 				hdr->b_l2hdr.b_hits++;
5879 
5880 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
5881 				    KM_SLEEP);
5882 				cb->l2rcb_hdr = hdr;
5883 				cb->l2rcb_bp = *bp;
5884 				cb->l2rcb_zb = *zb;
5885 				cb->l2rcb_flags = zio_flags;
5886 
5887 				/*
5888 				 * When Compressed ARC is disabled, but the
5889 				 * L2ARC block is compressed, arc_hdr_size()
5890 				 * will have returned LSIZE rather than PSIZE.
5891 				 */
5892 				if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
5893 				    !HDR_COMPRESSION_ENABLED(hdr) &&
5894 				    HDR_GET_PSIZE(hdr) != 0) {
5895 					size = HDR_GET_PSIZE(hdr);
5896 				}
5897 
5898 				asize = vdev_psize_to_asize(vd, size);
5899 				if (asize != size) {
5900 					abd = abd_alloc_for_io(asize,
5901 					    HDR_ISTYPE_METADATA(hdr));
5902 					cb->l2rcb_abd = abd;
5903 				} else {
5904 					abd = hdr_abd;
5905 				}
5906 
5907 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
5908 				    addr + asize <= vd->vdev_psize -
5909 				    VDEV_LABEL_END_SIZE);
5910 
5911 				/*
5912 				 * l2arc read.  The SCL_L2ARC lock will be
5913 				 * released by l2arc_read_done().
5914 				 * Issue a null zio if the underlying buffer
5915 				 * was squashed to zero size by compression.
5916 				 */
5917 				ASSERT3U(arc_hdr_get_compress(hdr), !=,
5918 				    ZIO_COMPRESS_EMPTY);
5919 				rzio = zio_read_phys(pio, vd, addr,
5920 				    asize, abd,
5921 				    ZIO_CHECKSUM_OFF,
5922 				    l2arc_read_done, cb, priority,
5923 				    zio_flags | ZIO_FLAG_CANFAIL |
5924 				    ZIO_FLAG_DONT_PROPAGATE |
5925 				    ZIO_FLAG_DONT_RETRY, B_FALSE);
5926 				acb->acb_zio_head = rzio;
5927 
5928 				if (hash_lock != NULL)
5929 					mutex_exit(hash_lock);
5930 
5931 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
5932 				    zio_t *, rzio);
5933 				ARCSTAT_INCR(arcstat_l2_read_bytes,
5934 				    HDR_GET_PSIZE(hdr));
5935 
5936 				if (*arc_flags & ARC_FLAG_NOWAIT) {
5937 					zio_nowait(rzio);
5938 					goto out;
5939 				}
5940 
5941 				ASSERT(*arc_flags & ARC_FLAG_WAIT);
5942 				if (zio_wait(rzio) == 0)
5943 					goto out;
5944 
5945 				/* l2arc read error; goto zio_read() */
5946 				if (hash_lock != NULL)
5947 					mutex_enter(hash_lock);
5948 			} else {
5949 				DTRACE_PROBE1(l2arc__miss,
5950 				    arc_buf_hdr_t *, hdr);
5951 				ARCSTAT_BUMP(arcstat_l2_misses);
5952 				if (HDR_L2_WRITING(hdr))
5953 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
5954 				spa_config_exit(spa, SCL_L2ARC, vd);
5955 			}
5956 		} else {
5957 			if (vd != NULL)
5958 				spa_config_exit(spa, SCL_L2ARC, vd);
5959 
5960 			/*
5961 			 * Only a spa with l2 should contribute to l2
5962 			 * miss stats.  (Including the case of having a
5963 			 * faulted cache device - that's also a miss.)
5964 			 */
5965 			if (spa_has_l2) {
5966 				/*
5967 				 * Skip ARC stat bump for block pointers with
5968 				 * embedded data. The data are read from the
5969 				 * blkptr itself via
5970 				 * decode_embedded_bp_compressed().
5971 				 */
5972 				if (!embedded_bp) {
5973 					DTRACE_PROBE1(l2arc__miss,
5974 					    arc_buf_hdr_t *, hdr);
5975 					ARCSTAT_BUMP(arcstat_l2_misses);
5976 				}
5977 			}
5978 		}
5979 
5980 		rzio = zio_read(pio, spa, bp, hdr_abd, size,
5981 		    arc_read_done, hdr, priority, zio_flags, zb);
5982 		acb->acb_zio_head = rzio;
5983 
5984 		if (hash_lock != NULL)
5985 			mutex_exit(hash_lock);
5986 
5987 		if (*arc_flags & ARC_FLAG_WAIT) {
5988 			rc = zio_wait(rzio);
5989 			goto out;
5990 		}
5991 
5992 		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
5993 		zio_nowait(rzio);
5994 	}
5995 
5996 out:
5997 	/* embedded bps don't actually go to disk */
5998 	if (!embedded_bp)
5999 		spa_read_history_add(spa, zb, *arc_flags);
6000 	spl_fstrans_unmark(cookie);
6001 	return (rc);
6002 
6003 done:
6004 	if (done)
6005 		done(NULL, zb, bp, buf, private);
6006 	if (pio && rc != 0) {
6007 		zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
6008 		zio->io_error = rc;
6009 		zio_nowait(zio);
6010 	}
6011 	goto out;
6012 }
6013 
6014 arc_prune_t *
6015 arc_add_prune_callback(arc_prune_func_t *func, void *private)
6016 {
6017 	arc_prune_t *p;
6018 
6019 	p = kmem_alloc(sizeof (*p), KM_SLEEP);
6020 	p->p_pfunc = func;
6021 	p->p_private = private;
6022 	list_link_init(&p->p_node);
6023 	zfs_refcount_create(&p->p_refcnt);
6024 
6025 	mutex_enter(&arc_prune_mtx);
6026 	zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
6027 	list_insert_head(&arc_prune_list, p);
6028 	mutex_exit(&arc_prune_mtx);
6029 
6030 	return (p);
6031 }
6032 
6033 void
6034 arc_remove_prune_callback(arc_prune_t *p)
6035 {
6036 	boolean_t wait = B_FALSE;
6037 	mutex_enter(&arc_prune_mtx);
6038 	list_remove(&arc_prune_list, p);
6039 	if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6040 		wait = B_TRUE;
6041 	mutex_exit(&arc_prune_mtx);
6042 
6043 	/* wait for arc_prune_task to finish */
6044 	if (wait)
6045 		taskq_wait_outstanding(arc_prune_taskq, 0);
6046 	ASSERT0(zfs_refcount_count(&p->p_refcnt));
6047 	zfs_refcount_destroy(&p->p_refcnt);
6048 	kmem_free(p, sizeof (*p));
6049 }
6050 
6051 /*
6052  * Notify the arc that a block was freed, and thus will never be used again.
6053  */
6054 void
6055 arc_freed(spa_t *spa, const blkptr_t *bp)
6056 {
6057 	arc_buf_hdr_t *hdr;
6058 	kmutex_t *hash_lock;
6059 	uint64_t guid = spa_load_guid(spa);
6060 
6061 	ASSERT(!BP_IS_EMBEDDED(bp));
6062 
6063 	hdr = buf_hash_find(guid, bp, &hash_lock);
6064 	if (hdr == NULL)
6065 		return;
6066 
6067 	/*
6068 	 * We might be trying to free a block that is still doing I/O
6069 	 * (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
6070 	 * dmu_sync-ed block). A block may also have a reference if it is
6071 	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6072 	 * have written the new block to its final resting place on disk but
6073 	 * without the dedup flag set. This would have left the hdr in the MRU
6074 	 * state and discoverable. When the txg finally syncs it detects that
6075 	 * the block was overridden in open context and issues an override I/O.
6076 	 * Since this is a dedup block, the override I/O will determine if the
6077 	 * block is already in the DDT. If so, then it will replace the io_bp
6078 	 * with the bp from the DDT and allow the I/O to finish. When the I/O
6079 	 * reaches the done callback, dbuf_write_override_done, it will
6080 	 * check to see if the io_bp and io_bp_override are identical.
6081 	 * If they are not, then it indicates that the bp was replaced with
6082 	 * the bp in the DDT and the override bp is freed. This allows
6083 	 * us to arrive here with a reference on a block that is being
6084 	 * freed. So if we have an I/O in progress, or a reference to
6085 	 * this hdr, then we don't destroy the hdr.
6086 	 */
6087 	if (!HDR_HAS_L1HDR(hdr) ||
6088 	    zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6089 		arc_change_state(arc_anon, hdr);
6090 		arc_hdr_destroy(hdr);
6091 		mutex_exit(hash_lock);
6092 	} else {
6093 		mutex_exit(hash_lock);
6094 	}
6095 
6096 }
6097 
6098 /*
6099  * Release this buffer from the cache, making it an anonymous buffer.  This
6100  * must be done after a read and prior to modifying the buffer contents.
6101  * If the buffer has more than one reference, we must make
6102  * a new hdr for the buffer.
6103  */
6104 void
6105 arc_release(arc_buf_t *buf, const void *tag)
6106 {
6107 	arc_buf_hdr_t *hdr = buf->b_hdr;
6108 
6109 	/*
6110 	 * It would be nice to assert that if its DMU metadata (level >
6111 	 * 0 || it's the dnode file), then it must be syncing context.
6112 	 * But we don't know that information at this level.
6113 	 */
6114 
6115 	ASSERT(HDR_HAS_L1HDR(hdr));
6116 
6117 	/*
6118 	 * We don't grab the hash lock prior to this check, because if
6119 	 * the buffer's header is in the arc_anon state, it won't be
6120 	 * linked into the hash table.
6121 	 */
6122 	if (hdr->b_l1hdr.b_state == arc_anon) {
6123 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6124 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
6125 		ASSERT(!HDR_HAS_L2HDR(hdr));
6126 
6127 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6128 		ASSERT(ARC_BUF_LAST(buf));
6129 		ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6130 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6131 
6132 		hdr->b_l1hdr.b_arc_access = 0;
6133 
6134 		/*
6135 		 * If the buf is being overridden then it may already
6136 		 * have a hdr that is not empty.
6137 		 */
6138 		buf_discard_identity(hdr);
6139 		arc_buf_thaw(buf);
6140 
6141 		return;
6142 	}
6143 
6144 	kmutex_t *hash_lock = HDR_LOCK(hdr);
6145 	mutex_enter(hash_lock);
6146 
6147 	/*
6148 	 * This assignment is only valid as long as the hash_lock is
6149 	 * held, we must be careful not to reference state or the
6150 	 * b_state field after dropping the lock.
6151 	 */
6152 	arc_state_t *state = hdr->b_l1hdr.b_state;
6153 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6154 	ASSERT3P(state, !=, arc_anon);
6155 
6156 	/* this buffer is not on any list */
6157 	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6158 
6159 	if (HDR_HAS_L2HDR(hdr)) {
6160 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6161 
6162 		/*
6163 		 * We have to recheck this conditional again now that
6164 		 * we're holding the l2ad_mtx to prevent a race with
6165 		 * another thread which might be concurrently calling
6166 		 * l2arc_evict(). In that case, l2arc_evict() might have
6167 		 * destroyed the header's L2 portion as we were waiting
6168 		 * to acquire the l2ad_mtx.
6169 		 */
6170 		if (HDR_HAS_L2HDR(hdr))
6171 			arc_hdr_l2hdr_destroy(hdr);
6172 
6173 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6174 	}
6175 
6176 	/*
6177 	 * Do we have more than one buf?
6178 	 */
6179 	if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
6180 		arc_buf_hdr_t *nhdr;
6181 		uint64_t spa = hdr->b_spa;
6182 		uint64_t psize = HDR_GET_PSIZE(hdr);
6183 		uint64_t lsize = HDR_GET_LSIZE(hdr);
6184 		boolean_t protected = HDR_PROTECTED(hdr);
6185 		enum zio_compress compress = arc_hdr_get_compress(hdr);
6186 		arc_buf_contents_t type = arc_buf_type(hdr);
6187 		VERIFY3U(hdr->b_type, ==, type);
6188 
6189 		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6190 		VERIFY3S(remove_reference(hdr, tag), >, 0);
6191 
6192 		if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) {
6193 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6194 			ASSERT(ARC_BUF_LAST(buf));
6195 		}
6196 
6197 		/*
6198 		 * Pull the data off of this hdr and attach it to
6199 		 * a new anonymous hdr. Also find the last buffer
6200 		 * in the hdr's buffer list.
6201 		 */
6202 		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6203 		ASSERT3P(lastbuf, !=, NULL);
6204 
6205 		/*
6206 		 * If the current arc_buf_t and the hdr are sharing their data
6207 		 * buffer, then we must stop sharing that block.
6208 		 */
6209 		if (arc_buf_is_shared(buf)) {
6210 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6211 			VERIFY(!arc_buf_is_shared(lastbuf));
6212 
6213 			/*
6214 			 * First, sever the block sharing relationship between
6215 			 * buf and the arc_buf_hdr_t.
6216 			 */
6217 			arc_unshare_buf(hdr, buf);
6218 
6219 			/*
6220 			 * Now we need to recreate the hdr's b_pabd. Since we
6221 			 * have lastbuf handy, we try to share with it, but if
6222 			 * we can't then we allocate a new b_pabd and copy the
6223 			 * data from buf into it.
6224 			 */
6225 			if (arc_can_share(hdr, lastbuf)) {
6226 				arc_share_buf(hdr, lastbuf);
6227 			} else {
6228 				arc_hdr_alloc_abd(hdr, 0);
6229 				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6230 				    buf->b_data, psize);
6231 			}
6232 			VERIFY3P(lastbuf->b_data, !=, NULL);
6233 		} else if (HDR_SHARED_DATA(hdr)) {
6234 			/*
6235 			 * Uncompressed shared buffers are always at the end
6236 			 * of the list. Compressed buffers don't have the
6237 			 * same requirements. This makes it hard to
6238 			 * simply assert that the lastbuf is shared so
6239 			 * we rely on the hdr's compression flags to determine
6240 			 * if we have a compressed, shared buffer.
6241 			 */
6242 			ASSERT(arc_buf_is_shared(lastbuf) ||
6243 			    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
6244 			ASSERT(!ARC_BUF_SHARED(buf));
6245 		}
6246 
6247 		ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
6248 		ASSERT3P(state, !=, arc_l2c_only);
6249 
6250 		(void) zfs_refcount_remove_many(&state->arcs_size[type],
6251 		    arc_buf_size(buf), buf);
6252 
6253 		if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6254 			ASSERT3P(state, !=, arc_l2c_only);
6255 			(void) zfs_refcount_remove_many(
6256 			    &state->arcs_esize[type],
6257 			    arc_buf_size(buf), buf);
6258 		}
6259 
6260 		arc_cksum_verify(buf);
6261 		arc_buf_unwatch(buf);
6262 
6263 		/* if this is the last uncompressed buf free the checksum */
6264 		if (!arc_hdr_has_uncompressed_buf(hdr))
6265 			arc_cksum_free(hdr);
6266 
6267 		mutex_exit(hash_lock);
6268 
6269 		nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
6270 		    compress, hdr->b_complevel, type);
6271 		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6272 		ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
6273 		VERIFY3U(nhdr->b_type, ==, type);
6274 		ASSERT(!HDR_SHARED_DATA(nhdr));
6275 
6276 		nhdr->b_l1hdr.b_buf = buf;
6277 		(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6278 		buf->b_hdr = nhdr;
6279 
6280 		(void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
6281 		    arc_buf_size(buf), buf);
6282 	} else {
6283 		ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6284 		/* protected by hash lock, or hdr is on arc_anon */
6285 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6286 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6287 		hdr->b_l1hdr.b_mru_hits = 0;
6288 		hdr->b_l1hdr.b_mru_ghost_hits = 0;
6289 		hdr->b_l1hdr.b_mfu_hits = 0;
6290 		hdr->b_l1hdr.b_mfu_ghost_hits = 0;
6291 		arc_change_state(arc_anon, hdr);
6292 		hdr->b_l1hdr.b_arc_access = 0;
6293 
6294 		mutex_exit(hash_lock);
6295 		buf_discard_identity(hdr);
6296 		arc_buf_thaw(buf);
6297 	}
6298 }
6299 
6300 int
6301 arc_released(arc_buf_t *buf)
6302 {
6303 	return (buf->b_data != NULL &&
6304 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
6305 }
6306 
6307 #ifdef ZFS_DEBUG
6308 int
6309 arc_referenced(arc_buf_t *buf)
6310 {
6311 	return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6312 }
6313 #endif
6314 
6315 static void
6316 arc_write_ready(zio_t *zio)
6317 {
6318 	arc_write_callback_t *callback = zio->io_private;
6319 	arc_buf_t *buf = callback->awcb_buf;
6320 	arc_buf_hdr_t *hdr = buf->b_hdr;
6321 	blkptr_t *bp = zio->io_bp;
6322 	uint64_t psize = BP_IS_HOLE(bp) ? 0 : BP_GET_PSIZE(bp);
6323 	fstrans_cookie_t cookie = spl_fstrans_mark();
6324 
6325 	ASSERT(HDR_HAS_L1HDR(hdr));
6326 	ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6327 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6328 
6329 	/*
6330 	 * If we're reexecuting this zio because the pool suspended, then
6331 	 * cleanup any state that was previously set the first time the
6332 	 * callback was invoked.
6333 	 */
6334 	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6335 		arc_cksum_free(hdr);
6336 		arc_buf_unwatch(buf);
6337 		if (hdr->b_l1hdr.b_pabd != NULL) {
6338 			if (arc_buf_is_shared(buf)) {
6339 				arc_unshare_buf(hdr, buf);
6340 			} else {
6341 				arc_hdr_free_abd(hdr, B_FALSE);
6342 			}
6343 		}
6344 
6345 		if (HDR_HAS_RABD(hdr))
6346 			arc_hdr_free_abd(hdr, B_TRUE);
6347 	}
6348 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6349 	ASSERT(!HDR_HAS_RABD(hdr));
6350 	ASSERT(!HDR_SHARED_DATA(hdr));
6351 	ASSERT(!arc_buf_is_shared(buf));
6352 
6353 	callback->awcb_ready(zio, buf, callback->awcb_private);
6354 
6355 	if (HDR_IO_IN_PROGRESS(hdr)) {
6356 		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6357 	} else {
6358 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6359 		add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
6360 	}
6361 
6362 	if (BP_IS_PROTECTED(bp)) {
6363 		/* ZIL blocks are written through zio_rewrite */
6364 		ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
6365 
6366 		if (BP_SHOULD_BYTESWAP(bp)) {
6367 			if (BP_GET_LEVEL(bp) > 0) {
6368 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
6369 			} else {
6370 				hdr->b_l1hdr.b_byteswap =
6371 				    DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
6372 			}
6373 		} else {
6374 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
6375 		}
6376 
6377 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
6378 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
6379 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
6380 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
6381 		    hdr->b_crypt_hdr.b_iv);
6382 		zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
6383 	} else {
6384 		arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
6385 	}
6386 
6387 	/*
6388 	 * If this block was written for raw encryption but the zio layer
6389 	 * ended up only authenticating it, adjust the buffer flags now.
6390 	 */
6391 	if (BP_IS_AUTHENTICATED(bp) && ARC_BUF_ENCRYPTED(buf)) {
6392 		arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6393 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6394 		if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF)
6395 			buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6396 	} else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) {
6397 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6398 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6399 	}
6400 
6401 	/* this must be done after the buffer flags are adjusted */
6402 	arc_cksum_compute(buf);
6403 
6404 	enum zio_compress compress;
6405 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
6406 		compress = ZIO_COMPRESS_OFF;
6407 	} else {
6408 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
6409 		compress = BP_GET_COMPRESS(bp);
6410 	}
6411 	HDR_SET_PSIZE(hdr, psize);
6412 	arc_hdr_set_compress(hdr, compress);
6413 	hdr->b_complevel = zio->io_prop.zp_complevel;
6414 
6415 	if (zio->io_error != 0 || psize == 0)
6416 		goto out;
6417 
6418 	/*
6419 	 * Fill the hdr with data. If the buffer is encrypted we have no choice
6420 	 * but to copy the data into b_radb. If the hdr is compressed, the data
6421 	 * we want is available from the zio, otherwise we can take it from
6422 	 * the buf.
6423 	 *
6424 	 * We might be able to share the buf's data with the hdr here. However,
6425 	 * doing so would cause the ARC to be full of linear ABDs if we write a
6426 	 * lot of shareable data. As a compromise, we check whether scattered
6427 	 * ABDs are allowed, and assume that if they are then the user wants
6428 	 * the ARC to be primarily filled with them regardless of the data being
6429 	 * written. Therefore, if they're allowed then we allocate one and copy
6430 	 * the data into it; otherwise, we share the data directly if we can.
6431 	 */
6432 	if (ARC_BUF_ENCRYPTED(buf)) {
6433 		ASSERT3U(psize, >, 0);
6434 		ASSERT(ARC_BUF_COMPRESSED(buf));
6435 		arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6436 		    ARC_HDR_USE_RESERVE);
6437 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6438 	} else if (!(HDR_UNCACHED(hdr) ||
6439 	    abd_size_alloc_linear(arc_buf_size(buf))) ||
6440 	    !arc_can_share(hdr, buf)) {
6441 		/*
6442 		 * Ideally, we would always copy the io_abd into b_pabd, but the
6443 		 * user may have disabled compressed ARC, thus we must check the
6444 		 * hdr's compression setting rather than the io_bp's.
6445 		 */
6446 		if (BP_IS_ENCRYPTED(bp)) {
6447 			ASSERT3U(psize, >, 0);
6448 			arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6449 			    ARC_HDR_USE_RESERVE);
6450 			abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6451 		} else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
6452 		    !ARC_BUF_COMPRESSED(buf)) {
6453 			ASSERT3U(psize, >, 0);
6454 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6455 			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6456 		} else {
6457 			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6458 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6459 			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6460 			    arc_buf_size(buf));
6461 		}
6462 	} else {
6463 		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
6464 		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
6465 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6466 		ASSERT(ARC_BUF_LAST(buf));
6467 
6468 		arc_share_buf(hdr, buf);
6469 	}
6470 
6471 out:
6472 	arc_hdr_verify(hdr, bp);
6473 	spl_fstrans_unmark(cookie);
6474 }
6475 
6476 static void
6477 arc_write_children_ready(zio_t *zio)
6478 {
6479 	arc_write_callback_t *callback = zio->io_private;
6480 	arc_buf_t *buf = callback->awcb_buf;
6481 
6482 	callback->awcb_children_ready(zio, buf, callback->awcb_private);
6483 }
6484 
6485 static void
6486 arc_write_done(zio_t *zio)
6487 {
6488 	arc_write_callback_t *callback = zio->io_private;
6489 	arc_buf_t *buf = callback->awcb_buf;
6490 	arc_buf_hdr_t *hdr = buf->b_hdr;
6491 
6492 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6493 
6494 	if (zio->io_error == 0) {
6495 		arc_hdr_verify(hdr, zio->io_bp);
6496 
6497 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6498 			buf_discard_identity(hdr);
6499 		} else {
6500 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
6501 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
6502 		}
6503 	} else {
6504 		ASSERT(HDR_EMPTY(hdr));
6505 	}
6506 
6507 	/*
6508 	 * If the block to be written was all-zero or compressed enough to be
6509 	 * embedded in the BP, no write was performed so there will be no
6510 	 * dva/birth/checksum.  The buffer must therefore remain anonymous
6511 	 * (and uncached).
6512 	 */
6513 	if (!HDR_EMPTY(hdr)) {
6514 		arc_buf_hdr_t *exists;
6515 		kmutex_t *hash_lock;
6516 
6517 		ASSERT3U(zio->io_error, ==, 0);
6518 
6519 		arc_cksum_verify(buf);
6520 
6521 		exists = buf_hash_insert(hdr, &hash_lock);
6522 		if (exists != NULL) {
6523 			/*
6524 			 * This can only happen if we overwrite for
6525 			 * sync-to-convergence, because we remove
6526 			 * buffers from the hash table when we arc_free().
6527 			 */
6528 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
6529 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6530 					panic("bad overwrite, hdr=%p exists=%p",
6531 					    (void *)hdr, (void *)exists);
6532 				ASSERT(zfs_refcount_is_zero(
6533 				    &exists->b_l1hdr.b_refcnt));
6534 				arc_change_state(arc_anon, exists);
6535 				arc_hdr_destroy(exists);
6536 				mutex_exit(hash_lock);
6537 				exists = buf_hash_insert(hdr, &hash_lock);
6538 				ASSERT3P(exists, ==, NULL);
6539 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
6540 				/* nopwrite */
6541 				ASSERT(zio->io_prop.zp_nopwrite);
6542 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6543 					panic("bad nopwrite, hdr=%p exists=%p",
6544 					    (void *)hdr, (void *)exists);
6545 			} else {
6546 				/* Dedup */
6547 				ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6548 				ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
6549 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6550 				ASSERT(BP_GET_DEDUP(zio->io_bp));
6551 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
6552 			}
6553 		}
6554 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6555 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
6556 		/* if it's not anon, we are doing a scrub */
6557 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6558 			arc_access(hdr, 0, B_FALSE);
6559 		mutex_exit(hash_lock);
6560 	} else {
6561 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6562 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
6563 	}
6564 
6565 	callback->awcb_done(zio, buf, callback->awcb_private);
6566 
6567 	abd_free(zio->io_abd);
6568 	kmem_free(callback, sizeof (arc_write_callback_t));
6569 }
6570 
6571 zio_t *
6572 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
6573     blkptr_t *bp, arc_buf_t *buf, boolean_t uncached, boolean_t l2arc,
6574     const zio_prop_t *zp, arc_write_done_func_t *ready,
6575     arc_write_done_func_t *children_ready, arc_write_done_func_t *done,
6576     void *private, zio_priority_t priority, int zio_flags,
6577     const zbookmark_phys_t *zb)
6578 {
6579 	arc_buf_hdr_t *hdr = buf->b_hdr;
6580 	arc_write_callback_t *callback;
6581 	zio_t *zio;
6582 	zio_prop_t localprop = *zp;
6583 
6584 	ASSERT3P(ready, !=, NULL);
6585 	ASSERT3P(done, !=, NULL);
6586 	ASSERT(!HDR_IO_ERROR(hdr));
6587 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6588 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6589 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6590 	if (uncached)
6591 		arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
6592 	else if (l2arc)
6593 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
6594 
6595 	if (ARC_BUF_ENCRYPTED(buf)) {
6596 		ASSERT(ARC_BUF_COMPRESSED(buf));
6597 		localprop.zp_encrypt = B_TRUE;
6598 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6599 		localprop.zp_complevel = hdr->b_complevel;
6600 		localprop.zp_byteorder =
6601 		    (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
6602 		    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
6603 		memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
6604 		    ZIO_DATA_SALT_LEN);
6605 		memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
6606 		    ZIO_DATA_IV_LEN);
6607 		memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
6608 		    ZIO_DATA_MAC_LEN);
6609 		if (DMU_OT_IS_ENCRYPTED(localprop.zp_type)) {
6610 			localprop.zp_nopwrite = B_FALSE;
6611 			localprop.zp_copies =
6612 			    MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
6613 		}
6614 		zio_flags |= ZIO_FLAG_RAW;
6615 	} else if (ARC_BUF_COMPRESSED(buf)) {
6616 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
6617 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6618 		localprop.zp_complevel = hdr->b_complevel;
6619 		zio_flags |= ZIO_FLAG_RAW_COMPRESS;
6620 	}
6621 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
6622 	callback->awcb_ready = ready;
6623 	callback->awcb_children_ready = children_ready;
6624 	callback->awcb_done = done;
6625 	callback->awcb_private = private;
6626 	callback->awcb_buf = buf;
6627 
6628 	/*
6629 	 * The hdr's b_pabd is now stale, free it now. A new data block
6630 	 * will be allocated when the zio pipeline calls arc_write_ready().
6631 	 */
6632 	if (hdr->b_l1hdr.b_pabd != NULL) {
6633 		/*
6634 		 * If the buf is currently sharing the data block with
6635 		 * the hdr then we need to break that relationship here.
6636 		 * The hdr will remain with a NULL data pointer and the
6637 		 * buf will take sole ownership of the block.
6638 		 */
6639 		if (arc_buf_is_shared(buf)) {
6640 			arc_unshare_buf(hdr, buf);
6641 		} else {
6642 			arc_hdr_free_abd(hdr, B_FALSE);
6643 		}
6644 		VERIFY3P(buf->b_data, !=, NULL);
6645 	}
6646 
6647 	if (HDR_HAS_RABD(hdr))
6648 		arc_hdr_free_abd(hdr, B_TRUE);
6649 
6650 	if (!(zio_flags & ZIO_FLAG_RAW))
6651 		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
6652 
6653 	ASSERT(!arc_buf_is_shared(buf));
6654 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6655 
6656 	zio = zio_write(pio, spa, txg, bp,
6657 	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6658 	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
6659 	    (children_ready != NULL) ? arc_write_children_ready : NULL,
6660 	    arc_write_done, callback, priority, zio_flags, zb);
6661 
6662 	return (zio);
6663 }
6664 
6665 void
6666 arc_tempreserve_clear(uint64_t reserve)
6667 {
6668 	atomic_add_64(&arc_tempreserve, -reserve);
6669 	ASSERT((int64_t)arc_tempreserve >= 0);
6670 }
6671 
6672 int
6673 arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg)
6674 {
6675 	int error;
6676 	uint64_t anon_size;
6677 
6678 	if (!arc_no_grow &&
6679 	    reserve > arc_c/4 &&
6680 	    reserve * 4 > (2ULL << SPA_MAXBLOCKSHIFT))
6681 		arc_c = MIN(arc_c_max, reserve * 4);
6682 
6683 	/*
6684 	 * Throttle when the calculated memory footprint for the TXG
6685 	 * exceeds the target ARC size.
6686 	 */
6687 	if (reserve > arc_c) {
6688 		DMU_TX_STAT_BUMP(dmu_tx_memory_reserve);
6689 		return (SET_ERROR(ERESTART));
6690 	}
6691 
6692 	/*
6693 	 * Don't count loaned bufs as in flight dirty data to prevent long
6694 	 * network delays from blocking transactions that are ready to be
6695 	 * assigned to a txg.
6696 	 */
6697 
6698 	/* assert that it has not wrapped around */
6699 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
6700 
6701 	anon_size = MAX((int64_t)
6702 	    (zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
6703 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
6704 	    arc_loaned_bytes), 0);
6705 
6706 	/*
6707 	 * Writes will, almost always, require additional memory allocations
6708 	 * in order to compress/encrypt/etc the data.  We therefore need to
6709 	 * make sure that there is sufficient available memory for this.
6710 	 */
6711 	error = arc_memory_throttle(spa, reserve, txg);
6712 	if (error != 0)
6713 		return (error);
6714 
6715 	/*
6716 	 * Throttle writes when the amount of dirty data in the cache
6717 	 * gets too large.  We try to keep the cache less than half full
6718 	 * of dirty blocks so that our sync times don't grow too large.
6719 	 *
6720 	 * In the case of one pool being built on another pool, we want
6721 	 * to make sure we don't end up throttling the lower (backing)
6722 	 * pool when the upper pool is the majority contributor to dirty
6723 	 * data. To insure we make forward progress during throttling, we
6724 	 * also check the current pool's net dirty data and only throttle
6725 	 * if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty
6726 	 * data in the cache.
6727 	 *
6728 	 * Note: if two requests come in concurrently, we might let them
6729 	 * both succeed, when one of them should fail.  Not a huge deal.
6730 	 */
6731 	uint64_t total_dirty = reserve + arc_tempreserve + anon_size;
6732 	uint64_t spa_dirty_anon = spa_dirty_data(spa);
6733 	uint64_t rarc_c = arc_warm ? arc_c : arc_c_max;
6734 	if (total_dirty > rarc_c * zfs_arc_dirty_limit_percent / 100 &&
6735 	    anon_size > rarc_c * zfs_arc_anon_limit_percent / 100 &&
6736 	    spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) {
6737 #ifdef ZFS_DEBUG
6738 		uint64_t meta_esize = zfs_refcount_count(
6739 		    &arc_anon->arcs_esize[ARC_BUFC_METADATA]);
6740 		uint64_t data_esize =
6741 		    zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
6742 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
6743 		    "anon_data=%lluK tempreserve=%lluK rarc_c=%lluK\n",
6744 		    (u_longlong_t)arc_tempreserve >> 10,
6745 		    (u_longlong_t)meta_esize >> 10,
6746 		    (u_longlong_t)data_esize >> 10,
6747 		    (u_longlong_t)reserve >> 10,
6748 		    (u_longlong_t)rarc_c >> 10);
6749 #endif
6750 		DMU_TX_STAT_BUMP(dmu_tx_dirty_throttle);
6751 		return (SET_ERROR(ERESTART));
6752 	}
6753 	atomic_add_64(&arc_tempreserve, reserve);
6754 	return (0);
6755 }
6756 
6757 static void
6758 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
6759     kstat_named_t *data, kstat_named_t *metadata,
6760     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
6761 {
6762 	data->value.ui64 =
6763 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
6764 	metadata->value.ui64 =
6765 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
6766 	size->value.ui64 = data->value.ui64 + metadata->value.ui64;
6767 	evict_data->value.ui64 =
6768 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
6769 	evict_metadata->value.ui64 =
6770 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
6771 }
6772 
6773 static int
6774 arc_kstat_update(kstat_t *ksp, int rw)
6775 {
6776 	arc_stats_t *as = ksp->ks_data;
6777 
6778 	if (rw == KSTAT_WRITE)
6779 		return (SET_ERROR(EACCES));
6780 
6781 	as->arcstat_hits.value.ui64 =
6782 	    wmsum_value(&arc_sums.arcstat_hits);
6783 	as->arcstat_iohits.value.ui64 =
6784 	    wmsum_value(&arc_sums.arcstat_iohits);
6785 	as->arcstat_misses.value.ui64 =
6786 	    wmsum_value(&arc_sums.arcstat_misses);
6787 	as->arcstat_demand_data_hits.value.ui64 =
6788 	    wmsum_value(&arc_sums.arcstat_demand_data_hits);
6789 	as->arcstat_demand_data_iohits.value.ui64 =
6790 	    wmsum_value(&arc_sums.arcstat_demand_data_iohits);
6791 	as->arcstat_demand_data_misses.value.ui64 =
6792 	    wmsum_value(&arc_sums.arcstat_demand_data_misses);
6793 	as->arcstat_demand_metadata_hits.value.ui64 =
6794 	    wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
6795 	as->arcstat_demand_metadata_iohits.value.ui64 =
6796 	    wmsum_value(&arc_sums.arcstat_demand_metadata_iohits);
6797 	as->arcstat_demand_metadata_misses.value.ui64 =
6798 	    wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
6799 	as->arcstat_prefetch_data_hits.value.ui64 =
6800 	    wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
6801 	as->arcstat_prefetch_data_iohits.value.ui64 =
6802 	    wmsum_value(&arc_sums.arcstat_prefetch_data_iohits);
6803 	as->arcstat_prefetch_data_misses.value.ui64 =
6804 	    wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
6805 	as->arcstat_prefetch_metadata_hits.value.ui64 =
6806 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
6807 	as->arcstat_prefetch_metadata_iohits.value.ui64 =
6808 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_iohits);
6809 	as->arcstat_prefetch_metadata_misses.value.ui64 =
6810 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
6811 	as->arcstat_mru_hits.value.ui64 =
6812 	    wmsum_value(&arc_sums.arcstat_mru_hits);
6813 	as->arcstat_mru_ghost_hits.value.ui64 =
6814 	    wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
6815 	as->arcstat_mfu_hits.value.ui64 =
6816 	    wmsum_value(&arc_sums.arcstat_mfu_hits);
6817 	as->arcstat_mfu_ghost_hits.value.ui64 =
6818 	    wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
6819 	as->arcstat_uncached_hits.value.ui64 =
6820 	    wmsum_value(&arc_sums.arcstat_uncached_hits);
6821 	as->arcstat_deleted.value.ui64 =
6822 	    wmsum_value(&arc_sums.arcstat_deleted);
6823 	as->arcstat_mutex_miss.value.ui64 =
6824 	    wmsum_value(&arc_sums.arcstat_mutex_miss);
6825 	as->arcstat_access_skip.value.ui64 =
6826 	    wmsum_value(&arc_sums.arcstat_access_skip);
6827 	as->arcstat_evict_skip.value.ui64 =
6828 	    wmsum_value(&arc_sums.arcstat_evict_skip);
6829 	as->arcstat_evict_not_enough.value.ui64 =
6830 	    wmsum_value(&arc_sums.arcstat_evict_not_enough);
6831 	as->arcstat_evict_l2_cached.value.ui64 =
6832 	    wmsum_value(&arc_sums.arcstat_evict_l2_cached);
6833 	as->arcstat_evict_l2_eligible.value.ui64 =
6834 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
6835 	as->arcstat_evict_l2_eligible_mfu.value.ui64 =
6836 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
6837 	as->arcstat_evict_l2_eligible_mru.value.ui64 =
6838 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
6839 	as->arcstat_evict_l2_ineligible.value.ui64 =
6840 	    wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
6841 	as->arcstat_evict_l2_skip.value.ui64 =
6842 	    wmsum_value(&arc_sums.arcstat_evict_l2_skip);
6843 	as->arcstat_hash_collisions.value.ui64 =
6844 	    wmsum_value(&arc_sums.arcstat_hash_collisions);
6845 	as->arcstat_hash_chains.value.ui64 =
6846 	    wmsum_value(&arc_sums.arcstat_hash_chains);
6847 	as->arcstat_size.value.ui64 =
6848 	    aggsum_value(&arc_sums.arcstat_size);
6849 	as->arcstat_compressed_size.value.ui64 =
6850 	    wmsum_value(&arc_sums.arcstat_compressed_size);
6851 	as->arcstat_uncompressed_size.value.ui64 =
6852 	    wmsum_value(&arc_sums.arcstat_uncompressed_size);
6853 	as->arcstat_overhead_size.value.ui64 =
6854 	    wmsum_value(&arc_sums.arcstat_overhead_size);
6855 	as->arcstat_hdr_size.value.ui64 =
6856 	    wmsum_value(&arc_sums.arcstat_hdr_size);
6857 	as->arcstat_data_size.value.ui64 =
6858 	    wmsum_value(&arc_sums.arcstat_data_size);
6859 	as->arcstat_metadata_size.value.ui64 =
6860 	    wmsum_value(&arc_sums.arcstat_metadata_size);
6861 	as->arcstat_dbuf_size.value.ui64 =
6862 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
6863 #if defined(COMPAT_FREEBSD11)
6864 	as->arcstat_other_size.value.ui64 =
6865 	    wmsum_value(&arc_sums.arcstat_bonus_size) +
6866 	    wmsum_value(&arc_sums.arcstat_dnode_size) +
6867 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
6868 #endif
6869 
6870 	arc_kstat_update_state(arc_anon,
6871 	    &as->arcstat_anon_size,
6872 	    &as->arcstat_anon_data,
6873 	    &as->arcstat_anon_metadata,
6874 	    &as->arcstat_anon_evictable_data,
6875 	    &as->arcstat_anon_evictable_metadata);
6876 	arc_kstat_update_state(arc_mru,
6877 	    &as->arcstat_mru_size,
6878 	    &as->arcstat_mru_data,
6879 	    &as->arcstat_mru_metadata,
6880 	    &as->arcstat_mru_evictable_data,
6881 	    &as->arcstat_mru_evictable_metadata);
6882 	arc_kstat_update_state(arc_mru_ghost,
6883 	    &as->arcstat_mru_ghost_size,
6884 	    &as->arcstat_mru_ghost_data,
6885 	    &as->arcstat_mru_ghost_metadata,
6886 	    &as->arcstat_mru_ghost_evictable_data,
6887 	    &as->arcstat_mru_ghost_evictable_metadata);
6888 	arc_kstat_update_state(arc_mfu,
6889 	    &as->arcstat_mfu_size,
6890 	    &as->arcstat_mfu_data,
6891 	    &as->arcstat_mfu_metadata,
6892 	    &as->arcstat_mfu_evictable_data,
6893 	    &as->arcstat_mfu_evictable_metadata);
6894 	arc_kstat_update_state(arc_mfu_ghost,
6895 	    &as->arcstat_mfu_ghost_size,
6896 	    &as->arcstat_mfu_ghost_data,
6897 	    &as->arcstat_mfu_ghost_metadata,
6898 	    &as->arcstat_mfu_ghost_evictable_data,
6899 	    &as->arcstat_mfu_ghost_evictable_metadata);
6900 	arc_kstat_update_state(arc_uncached,
6901 	    &as->arcstat_uncached_size,
6902 	    &as->arcstat_uncached_data,
6903 	    &as->arcstat_uncached_metadata,
6904 	    &as->arcstat_uncached_evictable_data,
6905 	    &as->arcstat_uncached_evictable_metadata);
6906 
6907 	as->arcstat_dnode_size.value.ui64 =
6908 	    wmsum_value(&arc_sums.arcstat_dnode_size);
6909 	as->arcstat_bonus_size.value.ui64 =
6910 	    wmsum_value(&arc_sums.arcstat_bonus_size);
6911 	as->arcstat_l2_hits.value.ui64 =
6912 	    wmsum_value(&arc_sums.arcstat_l2_hits);
6913 	as->arcstat_l2_misses.value.ui64 =
6914 	    wmsum_value(&arc_sums.arcstat_l2_misses);
6915 	as->arcstat_l2_prefetch_asize.value.ui64 =
6916 	    wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
6917 	as->arcstat_l2_mru_asize.value.ui64 =
6918 	    wmsum_value(&arc_sums.arcstat_l2_mru_asize);
6919 	as->arcstat_l2_mfu_asize.value.ui64 =
6920 	    wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
6921 	as->arcstat_l2_bufc_data_asize.value.ui64 =
6922 	    wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
6923 	as->arcstat_l2_bufc_metadata_asize.value.ui64 =
6924 	    wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
6925 	as->arcstat_l2_feeds.value.ui64 =
6926 	    wmsum_value(&arc_sums.arcstat_l2_feeds);
6927 	as->arcstat_l2_rw_clash.value.ui64 =
6928 	    wmsum_value(&arc_sums.arcstat_l2_rw_clash);
6929 	as->arcstat_l2_read_bytes.value.ui64 =
6930 	    wmsum_value(&arc_sums.arcstat_l2_read_bytes);
6931 	as->arcstat_l2_write_bytes.value.ui64 =
6932 	    wmsum_value(&arc_sums.arcstat_l2_write_bytes);
6933 	as->arcstat_l2_writes_sent.value.ui64 =
6934 	    wmsum_value(&arc_sums.arcstat_l2_writes_sent);
6935 	as->arcstat_l2_writes_done.value.ui64 =
6936 	    wmsum_value(&arc_sums.arcstat_l2_writes_done);
6937 	as->arcstat_l2_writes_error.value.ui64 =
6938 	    wmsum_value(&arc_sums.arcstat_l2_writes_error);
6939 	as->arcstat_l2_writes_lock_retry.value.ui64 =
6940 	    wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
6941 	as->arcstat_l2_evict_lock_retry.value.ui64 =
6942 	    wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
6943 	as->arcstat_l2_evict_reading.value.ui64 =
6944 	    wmsum_value(&arc_sums.arcstat_l2_evict_reading);
6945 	as->arcstat_l2_evict_l1cached.value.ui64 =
6946 	    wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
6947 	as->arcstat_l2_free_on_write.value.ui64 =
6948 	    wmsum_value(&arc_sums.arcstat_l2_free_on_write);
6949 	as->arcstat_l2_abort_lowmem.value.ui64 =
6950 	    wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
6951 	as->arcstat_l2_cksum_bad.value.ui64 =
6952 	    wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
6953 	as->arcstat_l2_io_error.value.ui64 =
6954 	    wmsum_value(&arc_sums.arcstat_l2_io_error);
6955 	as->arcstat_l2_lsize.value.ui64 =
6956 	    wmsum_value(&arc_sums.arcstat_l2_lsize);
6957 	as->arcstat_l2_psize.value.ui64 =
6958 	    wmsum_value(&arc_sums.arcstat_l2_psize);
6959 	as->arcstat_l2_hdr_size.value.ui64 =
6960 	    aggsum_value(&arc_sums.arcstat_l2_hdr_size);
6961 	as->arcstat_l2_log_blk_writes.value.ui64 =
6962 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
6963 	as->arcstat_l2_log_blk_asize.value.ui64 =
6964 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
6965 	as->arcstat_l2_log_blk_count.value.ui64 =
6966 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
6967 	as->arcstat_l2_rebuild_success.value.ui64 =
6968 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
6969 	as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
6970 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
6971 	as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
6972 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
6973 	as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
6974 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
6975 	as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
6976 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
6977 	as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
6978 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
6979 	as->arcstat_l2_rebuild_size.value.ui64 =
6980 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
6981 	as->arcstat_l2_rebuild_asize.value.ui64 =
6982 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
6983 	as->arcstat_l2_rebuild_bufs.value.ui64 =
6984 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
6985 	as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
6986 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
6987 	as->arcstat_l2_rebuild_log_blks.value.ui64 =
6988 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
6989 	as->arcstat_memory_throttle_count.value.ui64 =
6990 	    wmsum_value(&arc_sums.arcstat_memory_throttle_count);
6991 	as->arcstat_memory_direct_count.value.ui64 =
6992 	    wmsum_value(&arc_sums.arcstat_memory_direct_count);
6993 	as->arcstat_memory_indirect_count.value.ui64 =
6994 	    wmsum_value(&arc_sums.arcstat_memory_indirect_count);
6995 
6996 	as->arcstat_memory_all_bytes.value.ui64 =
6997 	    arc_all_memory();
6998 	as->arcstat_memory_free_bytes.value.ui64 =
6999 	    arc_free_memory();
7000 	as->arcstat_memory_available_bytes.value.i64 =
7001 	    arc_available_memory();
7002 
7003 	as->arcstat_prune.value.ui64 =
7004 	    wmsum_value(&arc_sums.arcstat_prune);
7005 	as->arcstat_meta_used.value.ui64 =
7006 	    wmsum_value(&arc_sums.arcstat_meta_used);
7007 	as->arcstat_async_upgrade_sync.value.ui64 =
7008 	    wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
7009 	as->arcstat_predictive_prefetch.value.ui64 =
7010 	    wmsum_value(&arc_sums.arcstat_predictive_prefetch);
7011 	as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
7012 	    wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7013 	as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
7014 	    wmsum_value(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7015 	as->arcstat_prescient_prefetch.value.ui64 =
7016 	    wmsum_value(&arc_sums.arcstat_prescient_prefetch);
7017 	as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
7018 	    wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7019 	as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
7020 	    wmsum_value(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7021 	as->arcstat_raw_size.value.ui64 =
7022 	    wmsum_value(&arc_sums.arcstat_raw_size);
7023 	as->arcstat_cached_only_in_progress.value.ui64 =
7024 	    wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
7025 	as->arcstat_abd_chunk_waste_size.value.ui64 =
7026 	    wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);
7027 
7028 	return (0);
7029 }
7030 
7031 /*
7032  * This function *must* return indices evenly distributed between all
7033  * sublists of the multilist. This is needed due to how the ARC eviction
7034  * code is laid out; arc_evict_state() assumes ARC buffers are evenly
7035  * distributed between all sublists and uses this assumption when
7036  * deciding which sublist to evict from and how much to evict from it.
7037  */
7038 static unsigned int
7039 arc_state_multilist_index_func(multilist_t *ml, void *obj)
7040 {
7041 	arc_buf_hdr_t *hdr = obj;
7042 
7043 	/*
7044 	 * We rely on b_dva to generate evenly distributed index
7045 	 * numbers using buf_hash below. So, as an added precaution,
7046 	 * let's make sure we never add empty buffers to the arc lists.
7047 	 */
7048 	ASSERT(!HDR_EMPTY(hdr));
7049 
7050 	/*
7051 	 * The assumption here, is the hash value for a given
7052 	 * arc_buf_hdr_t will remain constant throughout its lifetime
7053 	 * (i.e. its b_spa, b_dva, and b_birth fields don't change).
7054 	 * Thus, we don't need to store the header's sublist index
7055 	 * on insertion, as this index can be recalculated on removal.
7056 	 *
7057 	 * Also, the low order bits of the hash value are thought to be
7058 	 * distributed evenly. Otherwise, in the case that the multilist
7059 	 * has a power of two number of sublists, each sublists' usage
7060 	 * would not be evenly distributed. In this context full 64bit
7061 	 * division would be a waste of time, so limit it to 32 bits.
7062 	 */
7063 	return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
7064 	    multilist_get_num_sublists(ml));
7065 }
7066 
7067 static unsigned int
7068 arc_state_l2c_multilist_index_func(multilist_t *ml, void *obj)
7069 {
7070 	panic("Header %p insert into arc_l2c_only %p", obj, ml);
7071 }
7072 
7073 #define	WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do {	\
7074 	if ((do_warn) && (tuning) && ((tuning) != (value))) {	\
7075 		cmn_err(CE_WARN,				\
7076 		    "ignoring tunable %s (using %llu instead)",	\
7077 		    (#tuning), (u_longlong_t)(value));	\
7078 	}							\
7079 } while (0)
7080 
7081 /*
7082  * Called during module initialization and periodically thereafter to
7083  * apply reasonable changes to the exposed performance tunings.  Can also be
7084  * called explicitly by param_set_arc_*() functions when ARC tunables are
7085  * updated manually.  Non-zero zfs_* values which differ from the currently set
7086  * values will be applied.
7087  */
7088 void
7089 arc_tuning_update(boolean_t verbose)
7090 {
7091 	uint64_t allmem = arc_all_memory();
7092 
7093 	/* Valid range: 32M - <arc_c_max> */
7094 	if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) &&
7095 	    (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) &&
7096 	    (zfs_arc_min <= arc_c_max)) {
7097 		arc_c_min = zfs_arc_min;
7098 		arc_c = MAX(arc_c, arc_c_min);
7099 	}
7100 	WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose);
7101 
7102 	/* Valid range: 64M - <all physical memory> */
7103 	if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
7104 	    (zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
7105 	    (zfs_arc_max > arc_c_min)) {
7106 		arc_c_max = zfs_arc_max;
7107 		arc_c = MIN(arc_c, arc_c_max);
7108 		if (arc_dnode_limit > arc_c_max)
7109 			arc_dnode_limit = arc_c_max;
7110 	}
7111 	WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose);
7112 
7113 	/* Valid range: 0 - <all physical memory> */
7114 	arc_dnode_limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit :
7115 	    MIN(zfs_arc_dnode_limit_percent, 100) * arc_c_max / 100;
7116 	WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_limit, verbose);
7117 
7118 	/* Valid range: 1 - N */
7119 	if (zfs_arc_grow_retry)
7120 		arc_grow_retry = zfs_arc_grow_retry;
7121 
7122 	/* Valid range: 1 - N */
7123 	if (zfs_arc_shrink_shift) {
7124 		arc_shrink_shift = zfs_arc_shrink_shift;
7125 		arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
7126 	}
7127 
7128 	/* Valid range: 1 - N ms */
7129 	if (zfs_arc_min_prefetch_ms)
7130 		arc_min_prefetch_ms = zfs_arc_min_prefetch_ms;
7131 
7132 	/* Valid range: 1 - N ms */
7133 	if (zfs_arc_min_prescient_prefetch_ms) {
7134 		arc_min_prescient_prefetch_ms =
7135 		    zfs_arc_min_prescient_prefetch_ms;
7136 	}
7137 
7138 	/* Valid range: 0 - 100 */
7139 	if (zfs_arc_lotsfree_percent <= 100)
7140 		arc_lotsfree_percent = zfs_arc_lotsfree_percent;
7141 	WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent,
7142 	    verbose);
7143 
7144 	/* Valid range: 0 - <all physical memory> */
7145 	if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free))
7146 		arc_sys_free = MIN(zfs_arc_sys_free, allmem);
7147 	WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose);
7148 }
7149 
7150 static void
7151 arc_state_multilist_init(multilist_t *ml,
7152     multilist_sublist_index_func_t *index_func, int *maxcountp)
7153 {
7154 	multilist_create(ml, sizeof (arc_buf_hdr_t),
7155 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func);
7156 	*maxcountp = MAX(*maxcountp, multilist_get_num_sublists(ml));
7157 }
7158 
7159 static void
7160 arc_state_init(void)
7161 {
7162 	int num_sublists = 0;
7163 
7164 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
7165 	    arc_state_multilist_index_func, &num_sublists);
7166 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
7167 	    arc_state_multilist_index_func, &num_sublists);
7168 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
7169 	    arc_state_multilist_index_func, &num_sublists);
7170 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
7171 	    arc_state_multilist_index_func, &num_sublists);
7172 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
7173 	    arc_state_multilist_index_func, &num_sublists);
7174 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
7175 	    arc_state_multilist_index_func, &num_sublists);
7176 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
7177 	    arc_state_multilist_index_func, &num_sublists);
7178 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
7179 	    arc_state_multilist_index_func, &num_sublists);
7180 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
7181 	    arc_state_multilist_index_func, &num_sublists);
7182 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
7183 	    arc_state_multilist_index_func, &num_sublists);
7184 
7185 	/*
7186 	 * L2 headers should never be on the L2 state list since they don't
7187 	 * have L1 headers allocated.  Special index function asserts that.
7188 	 */
7189 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
7190 	    arc_state_l2c_multilist_index_func, &num_sublists);
7191 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
7192 	    arc_state_l2c_multilist_index_func, &num_sublists);
7193 
7194 	/*
7195 	 * Keep track of the number of markers needed to reclaim buffers from
7196 	 * any ARC state.  The markers will be pre-allocated so as to minimize
7197 	 * the number of memory allocations performed by the eviction thread.
7198 	 */
7199 	arc_state_evict_marker_count = num_sublists;
7200 
7201 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7202 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7203 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7204 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7205 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7206 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7207 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7208 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7209 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7210 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7211 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7212 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7213 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7214 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7215 
7216 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7217 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7218 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7219 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7220 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7221 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7222 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7223 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7224 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7225 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7226 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7227 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7228 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7229 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7230 
7231 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7232 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7233 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7234 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7235 
7236 	wmsum_init(&arc_sums.arcstat_hits, 0);
7237 	wmsum_init(&arc_sums.arcstat_iohits, 0);
7238 	wmsum_init(&arc_sums.arcstat_misses, 0);
7239 	wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
7240 	wmsum_init(&arc_sums.arcstat_demand_data_iohits, 0);
7241 	wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
7242 	wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
7243 	wmsum_init(&arc_sums.arcstat_demand_metadata_iohits, 0);
7244 	wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
7245 	wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
7246 	wmsum_init(&arc_sums.arcstat_prefetch_data_iohits, 0);
7247 	wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
7248 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
7249 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_iohits, 0);
7250 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
7251 	wmsum_init(&arc_sums.arcstat_mru_hits, 0);
7252 	wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
7253 	wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
7254 	wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
7255 	wmsum_init(&arc_sums.arcstat_uncached_hits, 0);
7256 	wmsum_init(&arc_sums.arcstat_deleted, 0);
7257 	wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
7258 	wmsum_init(&arc_sums.arcstat_access_skip, 0);
7259 	wmsum_init(&arc_sums.arcstat_evict_skip, 0);
7260 	wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
7261 	wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
7262 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
7263 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
7264 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
7265 	wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
7266 	wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
7267 	wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
7268 	wmsum_init(&arc_sums.arcstat_hash_chains, 0);
7269 	aggsum_init(&arc_sums.arcstat_size, 0);
7270 	wmsum_init(&arc_sums.arcstat_compressed_size, 0);
7271 	wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
7272 	wmsum_init(&arc_sums.arcstat_overhead_size, 0);
7273 	wmsum_init(&arc_sums.arcstat_hdr_size, 0);
7274 	wmsum_init(&arc_sums.arcstat_data_size, 0);
7275 	wmsum_init(&arc_sums.arcstat_metadata_size, 0);
7276 	wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
7277 	wmsum_init(&arc_sums.arcstat_dnode_size, 0);
7278 	wmsum_init(&arc_sums.arcstat_bonus_size, 0);
7279 	wmsum_init(&arc_sums.arcstat_l2_hits, 0);
7280 	wmsum_init(&arc_sums.arcstat_l2_misses, 0);
7281 	wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
7282 	wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
7283 	wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
7284 	wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
7285 	wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
7286 	wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
7287 	wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
7288 	wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
7289 	wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
7290 	wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
7291 	wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
7292 	wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
7293 	wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
7294 	wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
7295 	wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
7296 	wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
7297 	wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
7298 	wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
7299 	wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
7300 	wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
7301 	wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
7302 	wmsum_init(&arc_sums.arcstat_l2_psize, 0);
7303 	aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
7304 	wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
7305 	wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
7306 	wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
7307 	wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
7308 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
7309 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
7310 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
7311 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
7312 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
7313 	wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
7314 	wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
7315 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
7316 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
7317 	wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
7318 	wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
7319 	wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
7320 	wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
7321 	wmsum_init(&arc_sums.arcstat_prune, 0);
7322 	wmsum_init(&arc_sums.arcstat_meta_used, 0);
7323 	wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
7324 	wmsum_init(&arc_sums.arcstat_predictive_prefetch, 0);
7325 	wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
7326 	wmsum_init(&arc_sums.arcstat_demand_iohit_predictive_prefetch, 0);
7327 	wmsum_init(&arc_sums.arcstat_prescient_prefetch, 0);
7328 	wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
7329 	wmsum_init(&arc_sums.arcstat_demand_iohit_prescient_prefetch, 0);
7330 	wmsum_init(&arc_sums.arcstat_raw_size, 0);
7331 	wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
7332 	wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);
7333 
7334 	arc_anon->arcs_state = ARC_STATE_ANON;
7335 	arc_mru->arcs_state = ARC_STATE_MRU;
7336 	arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
7337 	arc_mfu->arcs_state = ARC_STATE_MFU;
7338 	arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
7339 	arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
7340 	arc_uncached->arcs_state = ARC_STATE_UNCACHED;
7341 }
7342 
7343 static void
7344 arc_state_fini(void)
7345 {
7346 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7347 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7348 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7349 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7350 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7351 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7352 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7353 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7354 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7355 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7356 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7357 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7358 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7359 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7360 
7361 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7362 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7363 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7364 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7365 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7366 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7367 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7368 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7369 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7370 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7371 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7372 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7373 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7374 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7375 
7376 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
7377 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7378 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7379 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7380 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
7381 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7382 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
7383 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7384 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7385 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
7386 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
7387 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);
7388 
7389 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
7390 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
7391 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
7392 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
7393 
7394 	wmsum_fini(&arc_sums.arcstat_hits);
7395 	wmsum_fini(&arc_sums.arcstat_iohits);
7396 	wmsum_fini(&arc_sums.arcstat_misses);
7397 	wmsum_fini(&arc_sums.arcstat_demand_data_hits);
7398 	wmsum_fini(&arc_sums.arcstat_demand_data_iohits);
7399 	wmsum_fini(&arc_sums.arcstat_demand_data_misses);
7400 	wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
7401 	wmsum_fini(&arc_sums.arcstat_demand_metadata_iohits);
7402 	wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
7403 	wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
7404 	wmsum_fini(&arc_sums.arcstat_prefetch_data_iohits);
7405 	wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
7406 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
7407 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_iohits);
7408 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
7409 	wmsum_fini(&arc_sums.arcstat_mru_hits);
7410 	wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
7411 	wmsum_fini(&arc_sums.arcstat_mfu_hits);
7412 	wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
7413 	wmsum_fini(&arc_sums.arcstat_uncached_hits);
7414 	wmsum_fini(&arc_sums.arcstat_deleted);
7415 	wmsum_fini(&arc_sums.arcstat_mutex_miss);
7416 	wmsum_fini(&arc_sums.arcstat_access_skip);
7417 	wmsum_fini(&arc_sums.arcstat_evict_skip);
7418 	wmsum_fini(&arc_sums.arcstat_evict_not_enough);
7419 	wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
7420 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
7421 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
7422 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
7423 	wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
7424 	wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
7425 	wmsum_fini(&arc_sums.arcstat_hash_collisions);
7426 	wmsum_fini(&arc_sums.arcstat_hash_chains);
7427 	aggsum_fini(&arc_sums.arcstat_size);
7428 	wmsum_fini(&arc_sums.arcstat_compressed_size);
7429 	wmsum_fini(&arc_sums.arcstat_uncompressed_size);
7430 	wmsum_fini(&arc_sums.arcstat_overhead_size);
7431 	wmsum_fini(&arc_sums.arcstat_hdr_size);
7432 	wmsum_fini(&arc_sums.arcstat_data_size);
7433 	wmsum_fini(&arc_sums.arcstat_metadata_size);
7434 	wmsum_fini(&arc_sums.arcstat_dbuf_size);
7435 	wmsum_fini(&arc_sums.arcstat_dnode_size);
7436 	wmsum_fini(&arc_sums.arcstat_bonus_size);
7437 	wmsum_fini(&arc_sums.arcstat_l2_hits);
7438 	wmsum_fini(&arc_sums.arcstat_l2_misses);
7439 	wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
7440 	wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
7441 	wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
7442 	wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
7443 	wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
7444 	wmsum_fini(&arc_sums.arcstat_l2_feeds);
7445 	wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
7446 	wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
7447 	wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
7448 	wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
7449 	wmsum_fini(&arc_sums.arcstat_l2_writes_done);
7450 	wmsum_fini(&arc_sums.arcstat_l2_writes_error);
7451 	wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
7452 	wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
7453 	wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
7454 	wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
7455 	wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
7456 	wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
7457 	wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
7458 	wmsum_fini(&arc_sums.arcstat_l2_io_error);
7459 	wmsum_fini(&arc_sums.arcstat_l2_lsize);
7460 	wmsum_fini(&arc_sums.arcstat_l2_psize);
7461 	aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
7462 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
7463 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
7464 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
7465 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
7466 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
7467 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
7468 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
7469 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
7470 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
7471 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
7472 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
7473 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
7474 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
7475 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
7476 	wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
7477 	wmsum_fini(&arc_sums.arcstat_memory_direct_count);
7478 	wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
7479 	wmsum_fini(&arc_sums.arcstat_prune);
7480 	wmsum_fini(&arc_sums.arcstat_meta_used);
7481 	wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
7482 	wmsum_fini(&arc_sums.arcstat_predictive_prefetch);
7483 	wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7484 	wmsum_fini(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7485 	wmsum_fini(&arc_sums.arcstat_prescient_prefetch);
7486 	wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7487 	wmsum_fini(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7488 	wmsum_fini(&arc_sums.arcstat_raw_size);
7489 	wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
7490 	wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
7491 }
7492 
7493 uint64_t
7494 arc_target_bytes(void)
7495 {
7496 	return (arc_c);
7497 }
7498 
7499 void
7500 arc_set_limits(uint64_t allmem)
7501 {
7502 	/* Set min cache to 1/32 of all memory, or 32MB, whichever is more. */
7503 	arc_c_min = MAX(allmem / 32, 2ULL << SPA_MAXBLOCKSHIFT);
7504 
7505 	/* How to set default max varies by platform. */
7506 	arc_c_max = arc_default_max(arc_c_min, allmem);
7507 }
7508 void
7509 arc_init(void)
7510 {
7511 	uint64_t percent, allmem = arc_all_memory();
7512 	mutex_init(&arc_evict_lock, NULL, MUTEX_DEFAULT, NULL);
7513 	list_create(&arc_evict_waiters, sizeof (arc_evict_waiter_t),
7514 	    offsetof(arc_evict_waiter_t, aew_node));
7515 
7516 	arc_min_prefetch_ms = 1000;
7517 	arc_min_prescient_prefetch_ms = 6000;
7518 
7519 #if defined(_KERNEL)
7520 	arc_lowmem_init();
7521 #endif
7522 
7523 	arc_set_limits(allmem);
7524 
7525 #ifdef _KERNEL
7526 	/*
7527 	 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
7528 	 * environment before the module was loaded, don't block setting the
7529 	 * maximum because it is less than arc_c_min, instead, reset arc_c_min
7530 	 * to a lower value.
7531 	 * zfs_arc_min will be handled by arc_tuning_update().
7532 	 */
7533 	if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
7534 	    zfs_arc_max < allmem) {
7535 		arc_c_max = zfs_arc_max;
7536 		if (arc_c_min >= arc_c_max) {
7537 			arc_c_min = MAX(zfs_arc_max / 2,
7538 			    2ULL << SPA_MAXBLOCKSHIFT);
7539 		}
7540 	}
7541 #else
7542 	/*
7543 	 * In userland, there's only the memory pressure that we artificially
7544 	 * create (see arc_available_memory()).  Don't let arc_c get too
7545 	 * small, because it can cause transactions to be larger than
7546 	 * arc_c, causing arc_tempreserve_space() to fail.
7547 	 */
7548 	arc_c_min = MAX(arc_c_max / 2, 2ULL << SPA_MAXBLOCKSHIFT);
7549 #endif
7550 
7551 	arc_c = arc_c_min;
7552 	/*
7553 	 * 32-bit fixed point fractions of metadata from total ARC size,
7554 	 * MRU data from all data and MRU metadata from all metadata.
7555 	 */
7556 	arc_meta = (1ULL << 32) / 4;	/* Metadata is 25% of arc_c. */
7557 	arc_pd = (1ULL << 32) / 2;	/* Data MRU is 50% of data. */
7558 	arc_pm = (1ULL << 32) / 2;	/* Metadata MRU is 50% of metadata. */
7559 
7560 	percent = MIN(zfs_arc_dnode_limit_percent, 100);
7561 	arc_dnode_limit = arc_c_max * percent / 100;
7562 
7563 	/* Apply user specified tunings */
7564 	arc_tuning_update(B_TRUE);
7565 
7566 	/* if kmem_flags are set, lets try to use less memory */
7567 	if (kmem_debugging())
7568 		arc_c = arc_c / 2;
7569 	if (arc_c < arc_c_min)
7570 		arc_c = arc_c_min;
7571 
7572 	arc_register_hotplug();
7573 
7574 	arc_state_init();
7575 
7576 	buf_init();
7577 
7578 	list_create(&arc_prune_list, sizeof (arc_prune_t),
7579 	    offsetof(arc_prune_t, p_node));
7580 	mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL);
7581 
7582 	arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
7583 	    defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
7584 
7585 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
7586 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
7587 
7588 	if (arc_ksp != NULL) {
7589 		arc_ksp->ks_data = &arc_stats;
7590 		arc_ksp->ks_update = arc_kstat_update;
7591 		kstat_install(arc_ksp);
7592 	}
7593 
7594 	arc_state_evict_markers =
7595 	    arc_state_alloc_markers(arc_state_evict_marker_count);
7596 	arc_evict_zthr = zthr_create_timer("arc_evict",
7597 	    arc_evict_cb_check, arc_evict_cb, NULL, SEC2NSEC(1), defclsyspri);
7598 	arc_reap_zthr = zthr_create_timer("arc_reap",
7599 	    arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);
7600 
7601 	arc_warm = B_FALSE;
7602 
7603 	/*
7604 	 * Calculate maximum amount of dirty data per pool.
7605 	 *
7606 	 * If it has been set by a module parameter, take that.
7607 	 * Otherwise, use a percentage of physical memory defined by
7608 	 * zfs_dirty_data_max_percent (default 10%) with a cap at
7609 	 * zfs_dirty_data_max_max (default 4G or 25% of physical memory).
7610 	 */
7611 #ifdef __LP64__
7612 	if (zfs_dirty_data_max_max == 0)
7613 		zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
7614 		    allmem * zfs_dirty_data_max_max_percent / 100);
7615 #else
7616 	if (zfs_dirty_data_max_max == 0)
7617 		zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
7618 		    allmem * zfs_dirty_data_max_max_percent / 100);
7619 #endif
7620 
7621 	if (zfs_dirty_data_max == 0) {
7622 		zfs_dirty_data_max = allmem *
7623 		    zfs_dirty_data_max_percent / 100;
7624 		zfs_dirty_data_max = MIN(zfs_dirty_data_max,
7625 		    zfs_dirty_data_max_max);
7626 	}
7627 
7628 	if (zfs_wrlog_data_max == 0) {
7629 
7630 		/*
7631 		 * dp_wrlog_total is reduced for each txg at the end of
7632 		 * spa_sync(). However, dp_dirty_total is reduced every time
7633 		 * a block is written out. Thus under normal operation,
7634 		 * dp_wrlog_total could grow 2 times as big as
7635 		 * zfs_dirty_data_max.
7636 		 */
7637 		zfs_wrlog_data_max = zfs_dirty_data_max * 2;
7638 	}
7639 }
7640 
7641 void
7642 arc_fini(void)
7643 {
7644 	arc_prune_t *p;
7645 
7646 #ifdef _KERNEL
7647 	arc_lowmem_fini();
7648 #endif /* _KERNEL */
7649 
7650 	/* Use B_TRUE to ensure *all* buffers are evicted */
7651 	arc_flush(NULL, B_TRUE);
7652 
7653 	if (arc_ksp != NULL) {
7654 		kstat_delete(arc_ksp);
7655 		arc_ksp = NULL;
7656 	}
7657 
7658 	taskq_wait(arc_prune_taskq);
7659 	taskq_destroy(arc_prune_taskq);
7660 
7661 	mutex_enter(&arc_prune_mtx);
7662 	while ((p = list_remove_head(&arc_prune_list)) != NULL) {
7663 		zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
7664 		zfs_refcount_destroy(&p->p_refcnt);
7665 		kmem_free(p, sizeof (*p));
7666 	}
7667 	mutex_exit(&arc_prune_mtx);
7668 
7669 	list_destroy(&arc_prune_list);
7670 	mutex_destroy(&arc_prune_mtx);
7671 
7672 	(void) zthr_cancel(arc_evict_zthr);
7673 	(void) zthr_cancel(arc_reap_zthr);
7674 	arc_state_free_markers(arc_state_evict_markers,
7675 	    arc_state_evict_marker_count);
7676 
7677 	mutex_destroy(&arc_evict_lock);
7678 	list_destroy(&arc_evict_waiters);
7679 
7680 	/*
7681 	 * Free any buffers that were tagged for destruction.  This needs
7682 	 * to occur before arc_state_fini() runs and destroys the aggsum
7683 	 * values which are updated when freeing scatter ABDs.
7684 	 */
7685 	l2arc_do_free_on_write();
7686 
7687 	/*
7688 	 * buf_fini() must proceed arc_state_fini() because buf_fin() may
7689 	 * trigger the release of kmem magazines, which can callback to
7690 	 * arc_space_return() which accesses aggsums freed in act_state_fini().
7691 	 */
7692 	buf_fini();
7693 	arc_state_fini();
7694 
7695 	arc_unregister_hotplug();
7696 
7697 	/*
7698 	 * We destroy the zthrs after all the ARC state has been
7699 	 * torn down to avoid the case of them receiving any
7700 	 * wakeup() signals after they are destroyed.
7701 	 */
7702 	zthr_destroy(arc_evict_zthr);
7703 	zthr_destroy(arc_reap_zthr);
7704 
7705 	ASSERT0(arc_loaned_bytes);
7706 }
7707 
7708 /*
7709  * Level 2 ARC
7710  *
7711  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
7712  * It uses dedicated storage devices to hold cached data, which are populated
7713  * using large infrequent writes.  The main role of this cache is to boost
7714  * the performance of random read workloads.  The intended L2ARC devices
7715  * include short-stroked disks, solid state disks, and other media with
7716  * substantially faster read latency than disk.
7717  *
7718  *                 +-----------------------+
7719  *                 |         ARC           |
7720  *                 +-----------------------+
7721  *                    |         ^     ^
7722  *                    |         |     |
7723  *      l2arc_feed_thread()    arc_read()
7724  *                    |         |     |
7725  *                    |  l2arc read   |
7726  *                    V         |     |
7727  *               +---------------+    |
7728  *               |     L2ARC     |    |
7729  *               +---------------+    |
7730  *                   |    ^           |
7731  *          l2arc_write() |           |
7732  *                   |    |           |
7733  *                   V    |           |
7734  *                 +-------+      +-------+
7735  *                 | vdev  |      | vdev  |
7736  *                 | cache |      | cache |
7737  *                 +-------+      +-------+
7738  *                 +=========+     .-----.
7739  *                 :  L2ARC  :    |-_____-|
7740  *                 : devices :    | Disks |
7741  *                 +=========+    `-_____-'
7742  *
7743  * Read requests are satisfied from the following sources, in order:
7744  *
7745  *	1) ARC
7746  *	2) vdev cache of L2ARC devices
7747  *	3) L2ARC devices
7748  *	4) vdev cache of disks
7749  *	5) disks
7750  *
7751  * Some L2ARC device types exhibit extremely slow write performance.
7752  * To accommodate for this there are some significant differences between
7753  * the L2ARC and traditional cache design:
7754  *
7755  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
7756  * the ARC behave as usual, freeing buffers and placing headers on ghost
7757  * lists.  The ARC does not send buffers to the L2ARC during eviction as
7758  * this would add inflated write latencies for all ARC memory pressure.
7759  *
7760  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
7761  * It does this by periodically scanning buffers from the eviction-end of
7762  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
7763  * not already there. It scans until a headroom of buffers is satisfied,
7764  * which itself is a buffer for ARC eviction. If a compressible buffer is
7765  * found during scanning and selected for writing to an L2ARC device, we
7766  * temporarily boost scanning headroom during the next scan cycle to make
7767  * sure we adapt to compression effects (which might significantly reduce
7768  * the data volume we write to L2ARC). The thread that does this is
7769  * l2arc_feed_thread(), illustrated below; example sizes are included to
7770  * provide a better sense of ratio than this diagram:
7771  *
7772  *	       head -->                        tail
7773  *	        +---------------------+----------+
7774  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
7775  *	        +---------------------+----------+   |   o L2ARC eligible
7776  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
7777  *	        +---------------------+----------+   |
7778  *	             15.9 Gbytes      ^ 32 Mbytes    |
7779  *	                           headroom          |
7780  *	                                      l2arc_feed_thread()
7781  *	                                             |
7782  *	                 l2arc write hand <--[oooo]--'
7783  *	                         |           8 Mbyte
7784  *	                         |          write max
7785  *	                         V
7786  *		  +==============================+
7787  *	L2ARC dev |####|#|###|###|    |####| ... |
7788  *	          +==============================+
7789  *	                     32 Gbytes
7790  *
7791  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
7792  * evicted, then the L2ARC has cached a buffer much sooner than it probably
7793  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
7794  * safe to say that this is an uncommon case, since buffers at the end of
7795  * the ARC lists have moved there due to inactivity.
7796  *
7797  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
7798  * then the L2ARC simply misses copying some buffers.  This serves as a
7799  * pressure valve to prevent heavy read workloads from both stalling the ARC
7800  * with waits and clogging the L2ARC with writes.  This also helps prevent
7801  * the potential for the L2ARC to churn if it attempts to cache content too
7802  * quickly, such as during backups of the entire pool.
7803  *
7804  * 5. After system boot and before the ARC has filled main memory, there are
7805  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
7806  * lists can remain mostly static.  Instead of searching from tail of these
7807  * lists as pictured, the l2arc_feed_thread() will search from the list heads
7808  * for eligible buffers, greatly increasing its chance of finding them.
7809  *
7810  * The L2ARC device write speed is also boosted during this time so that
7811  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
7812  * there are no L2ARC reads, and no fear of degrading read performance
7813  * through increased writes.
7814  *
7815  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
7816  * the vdev queue can aggregate them into larger and fewer writes.  Each
7817  * device is written to in a rotor fashion, sweeping writes through
7818  * available space then repeating.
7819  *
7820  * 7. The L2ARC does not store dirty content.  It never needs to flush
7821  * write buffers back to disk based storage.
7822  *
7823  * 8. If an ARC buffer is written (and dirtied) which also exists in the
7824  * L2ARC, the now stale L2ARC buffer is immediately dropped.
7825  *
7826  * The performance of the L2ARC can be tweaked by a number of tunables, which
7827  * may be necessary for different workloads:
7828  *
7829  *	l2arc_write_max		max write bytes per interval
7830  *	l2arc_write_boost	extra write bytes during device warmup
7831  *	l2arc_noprefetch	skip caching prefetched buffers
7832  *	l2arc_headroom		number of max device writes to precache
7833  *	l2arc_headroom_boost	when we find compressed buffers during ARC
7834  *				scanning, we multiply headroom by this
7835  *				percentage factor for the next scan cycle,
7836  *				since more compressed buffers are likely to
7837  *				be present
7838  *	l2arc_feed_secs		seconds between L2ARC writing
7839  *
7840  * Tunables may be removed or added as future performance improvements are
7841  * integrated, and also may become zpool properties.
7842  *
7843  * There are three key functions that control how the L2ARC warms up:
7844  *
7845  *	l2arc_write_eligible()	check if a buffer is eligible to cache
7846  *	l2arc_write_size()	calculate how much to write
7847  *	l2arc_write_interval()	calculate sleep delay between writes
7848  *
7849  * These three functions determine what to write, how much, and how quickly
7850  * to send writes.
7851  *
7852  * L2ARC persistence:
7853  *
7854  * When writing buffers to L2ARC, we periodically add some metadata to
7855  * make sure we can pick them up after reboot, thus dramatically reducing
7856  * the impact that any downtime has on the performance of storage systems
7857  * with large caches.
7858  *
7859  * The implementation works fairly simply by integrating the following two
7860  * modifications:
7861  *
7862  * *) When writing to the L2ARC, we occasionally write a "l2arc log block",
7863  *    which is an additional piece of metadata which describes what's been
7864  *    written. This allows us to rebuild the arc_buf_hdr_t structures of the
7865  *    main ARC buffers. There are 2 linked-lists of log blocks headed by
7866  *    dh_start_lbps[2]. We alternate which chain we append to, so they are
7867  *    time-wise and offset-wise interleaved, but that is an optimization rather
7868  *    than for correctness. The log block also includes a pointer to the
7869  *    previous block in its chain.
7870  *
7871  * *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device
7872  *    for our header bookkeeping purposes. This contains a device header,
7873  *    which contains our top-level reference structures. We update it each
7874  *    time we write a new log block, so that we're able to locate it in the
7875  *    L2ARC device. If this write results in an inconsistent device header
7876  *    (e.g. due to power failure), we detect this by verifying the header's
7877  *    checksum and simply fail to reconstruct the L2ARC after reboot.
7878  *
7879  * Implementation diagram:
7880  *
7881  * +=== L2ARC device (not to scale) ======================================+
7882  * |       ___two newest log block pointers__.__________                  |
7883  * |      /                                   \dh_start_lbps[1]           |
7884  * |	 /				       \         \dh_start_lbps[0]|
7885  * |.___/__.                                    V         V               |
7886  * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---|
7887  * ||   hdr|      ^         /^       /^        /         /                |
7888  * |+------+  ...--\-------/  \-----/--\------/         /                 |
7889  * |                \--------------/    \--------------/                  |
7890  * +======================================================================+
7891  *
7892  * As can be seen on the diagram, rather than using a simple linked list,
7893  * we use a pair of linked lists with alternating elements. This is a
7894  * performance enhancement due to the fact that we only find out the
7895  * address of the next log block access once the current block has been
7896  * completely read in. Obviously, this hurts performance, because we'd be
7897  * keeping the device's I/O queue at only a 1 operation deep, thus
7898  * incurring a large amount of I/O round-trip latency. Having two lists
7899  * allows us to fetch two log blocks ahead of where we are currently
7900  * rebuilding L2ARC buffers.
7901  *
7902  * On-device data structures:
7903  *
7904  * L2ARC device header:	l2arc_dev_hdr_phys_t
7905  * L2ARC log block:	l2arc_log_blk_phys_t
7906  *
7907  * L2ARC reconstruction:
7908  *
7909  * When writing data, we simply write in the standard rotary fashion,
7910  * evicting buffers as we go and simply writing new data over them (writing
7911  * a new log block every now and then). This obviously means that once we
7912  * loop around the end of the device, we will start cutting into an already
7913  * committed log block (and its referenced data buffers), like so:
7914  *
7915  *    current write head__       __old tail
7916  *                        \     /
7917  *                        V    V
7918  * <--|bufs |lb |bufs |lb |    |bufs |lb |bufs |lb |-->
7919  *                         ^    ^^^^^^^^^___________________________________
7920  *                         |                                                \
7921  *                   <<nextwrite>> may overwrite this blk and/or its bufs --'
7922  *
7923  * When importing the pool, we detect this situation and use it to stop
7924  * our scanning process (see l2arc_rebuild).
7925  *
7926  * There is one significant caveat to consider when rebuilding ARC contents
7927  * from an L2ARC device: what about invalidated buffers? Given the above
7928  * construction, we cannot update blocks which we've already written to amend
7929  * them to remove buffers which were invalidated. Thus, during reconstruction,
7930  * we might be populating the cache with buffers for data that's not on the
7931  * main pool anymore, or may have been overwritten!
7932  *
7933  * As it turns out, this isn't a problem. Every arc_read request includes
7934  * both the DVA and, crucially, the birth TXG of the BP the caller is
7935  * looking for. So even if the cache were populated by completely rotten
7936  * blocks for data that had been long deleted and/or overwritten, we'll
7937  * never actually return bad data from the cache, since the DVA with the
7938  * birth TXG uniquely identify a block in space and time - once created,
7939  * a block is immutable on disk. The worst thing we have done is wasted
7940  * some time and memory at l2arc rebuild to reconstruct outdated ARC
7941  * entries that will get dropped from the l2arc as it is being updated
7942  * with new blocks.
7943  *
7944  * L2ARC buffers that have been evicted by l2arc_evict() ahead of the write
7945  * hand are not restored. This is done by saving the offset (in bytes)
7946  * l2arc_evict() has evicted to in the L2ARC device header and taking it
7947  * into account when restoring buffers.
7948  */
7949 
7950 static boolean_t
7951 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
7952 {
7953 	/*
7954 	 * A buffer is *not* eligible for the L2ARC if it:
7955 	 * 1. belongs to a different spa.
7956 	 * 2. is already cached on the L2ARC.
7957 	 * 3. has an I/O in progress (it may be an incomplete read).
7958 	 * 4. is flagged not eligible (zfs property).
7959 	 */
7960 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
7961 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
7962 		return (B_FALSE);
7963 
7964 	return (B_TRUE);
7965 }
7966 
7967 static uint64_t
7968 l2arc_write_size(l2arc_dev_t *dev)
7969 {
7970 	uint64_t size;
7971 
7972 	/*
7973 	 * Make sure our globals have meaningful values in case the user
7974 	 * altered them.
7975 	 */
7976 	size = l2arc_write_max;
7977 	if (size == 0) {
7978 		cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
7979 		    "be greater than zero, resetting it to the default (%d)",
7980 		    L2ARC_WRITE_SIZE);
7981 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
7982 	}
7983 
7984 	if (arc_warm == B_FALSE)
7985 		size += l2arc_write_boost;
7986 
7987 	/* We need to add in the worst case scenario of log block overhead. */
7988 	size += l2arc_log_blk_overhead(size, dev);
7989 	if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
7990 		/*
7991 		 * Trim ahead of the write size 64MB or (l2arc_trim_ahead/100)
7992 		 * times the writesize, whichever is greater.
7993 		 */
7994 		size += MAX(64 * 1024 * 1024,
7995 		    (size * l2arc_trim_ahead) / 100);
7996 	}
7997 
7998 	/*
7999 	 * Make sure the write size does not exceed the size of the cache
8000 	 * device. This is important in l2arc_evict(), otherwise infinite
8001 	 * iteration can occur.
8002 	 */
8003 	if (size > dev->l2ad_end - dev->l2ad_start) {
8004 		cmn_err(CE_NOTE, "l2arc_write_max or l2arc_write_boost "
8005 		    "plus the overhead of log blocks (persistent L2ARC, "
8006 		    "%llu bytes) exceeds the size of the cache device "
8007 		    "(guid %llu), resetting them to the default (%d)",
8008 		    (u_longlong_t)l2arc_log_blk_overhead(size, dev),
8009 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid, L2ARC_WRITE_SIZE);
8010 
8011 		size = l2arc_write_max = l2arc_write_boost = L2ARC_WRITE_SIZE;
8012 
8013 		if (l2arc_trim_ahead > 1) {
8014 			cmn_err(CE_NOTE, "l2arc_trim_ahead set to 1");
8015 			l2arc_trim_ahead = 1;
8016 		}
8017 
8018 		if (arc_warm == B_FALSE)
8019 			size += l2arc_write_boost;
8020 
8021 		size += l2arc_log_blk_overhead(size, dev);
8022 		if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
8023 			size += MAX(64 * 1024 * 1024,
8024 			    (size * l2arc_trim_ahead) / 100);
8025 		}
8026 	}
8027 
8028 	return (size);
8029 
8030 }
8031 
8032 static clock_t
8033 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
8034 {
8035 	clock_t interval, next, now;
8036 
8037 	/*
8038 	 * If the ARC lists are busy, increase our write rate; if the
8039 	 * lists are stale, idle back.  This is achieved by checking
8040 	 * how much we previously wrote - if it was more than half of
8041 	 * what we wanted, schedule the next write much sooner.
8042 	 */
8043 	if (l2arc_feed_again && wrote > (wanted / 2))
8044 		interval = (hz * l2arc_feed_min_ms) / 1000;
8045 	else
8046 		interval = hz * l2arc_feed_secs;
8047 
8048 	now = ddi_get_lbolt();
8049 	next = MAX(now, MIN(now + interval, began + interval));
8050 
8051 	return (next);
8052 }
8053 
8054 /*
8055  * Cycle through L2ARC devices.  This is how L2ARC load balances.
8056  * If a device is returned, this also returns holding the spa config lock.
8057  */
8058 static l2arc_dev_t *
8059 l2arc_dev_get_next(void)
8060 {
8061 	l2arc_dev_t *first, *next = NULL;
8062 
8063 	/*
8064 	 * Lock out the removal of spas (spa_namespace_lock), then removal
8065 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
8066 	 * both locks will be dropped and a spa config lock held instead.
8067 	 */
8068 	mutex_enter(&spa_namespace_lock);
8069 	mutex_enter(&l2arc_dev_mtx);
8070 
8071 	/* if there are no vdevs, there is nothing to do */
8072 	if (l2arc_ndev == 0)
8073 		goto out;
8074 
8075 	first = NULL;
8076 	next = l2arc_dev_last;
8077 	do {
8078 		/* loop around the list looking for a non-faulted vdev */
8079 		if (next == NULL) {
8080 			next = list_head(l2arc_dev_list);
8081 		} else {
8082 			next = list_next(l2arc_dev_list, next);
8083 			if (next == NULL)
8084 				next = list_head(l2arc_dev_list);
8085 		}
8086 
8087 		/* if we have come back to the start, bail out */
8088 		if (first == NULL)
8089 			first = next;
8090 		else if (next == first)
8091 			break;
8092 
8093 		ASSERT3P(next, !=, NULL);
8094 	} while (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild ||
8095 	    next->l2ad_trim_all);
8096 
8097 	/* if we were unable to find any usable vdevs, return NULL */
8098 	if (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild ||
8099 	    next->l2ad_trim_all)
8100 		next = NULL;
8101 
8102 	l2arc_dev_last = next;
8103 
8104 out:
8105 	mutex_exit(&l2arc_dev_mtx);
8106 
8107 	/*
8108 	 * Grab the config lock to prevent the 'next' device from being
8109 	 * removed while we are writing to it.
8110 	 */
8111 	if (next != NULL)
8112 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
8113 	mutex_exit(&spa_namespace_lock);
8114 
8115 	return (next);
8116 }
8117 
8118 /*
8119  * Free buffers that were tagged for destruction.
8120  */
8121 static void
8122 l2arc_do_free_on_write(void)
8123 {
8124 	l2arc_data_free_t *df;
8125 
8126 	mutex_enter(&l2arc_free_on_write_mtx);
8127 	while ((df = list_remove_head(l2arc_free_on_write)) != NULL) {
8128 		ASSERT3P(df->l2df_abd, !=, NULL);
8129 		abd_free(df->l2df_abd);
8130 		kmem_free(df, sizeof (l2arc_data_free_t));
8131 	}
8132 	mutex_exit(&l2arc_free_on_write_mtx);
8133 }
8134 
8135 /*
8136  * A write to a cache device has completed.  Update all headers to allow
8137  * reads from these buffers to begin.
8138  */
8139 static void
8140 l2arc_write_done(zio_t *zio)
8141 {
8142 	l2arc_write_callback_t	*cb;
8143 	l2arc_lb_abd_buf_t	*abd_buf;
8144 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
8145 	l2arc_dev_t		*dev;
8146 	l2arc_dev_hdr_phys_t	*l2dhdr;
8147 	list_t			*buflist;
8148 	arc_buf_hdr_t		*head, *hdr, *hdr_prev;
8149 	kmutex_t		*hash_lock;
8150 	int64_t			bytes_dropped = 0;
8151 
8152 	cb = zio->io_private;
8153 	ASSERT3P(cb, !=, NULL);
8154 	dev = cb->l2wcb_dev;
8155 	l2dhdr = dev->l2ad_dev_hdr;
8156 	ASSERT3P(dev, !=, NULL);
8157 	head = cb->l2wcb_head;
8158 	ASSERT3P(head, !=, NULL);
8159 	buflist = &dev->l2ad_buflist;
8160 	ASSERT3P(buflist, !=, NULL);
8161 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
8162 	    l2arc_write_callback_t *, cb);
8163 
8164 	/*
8165 	 * All writes completed, or an error was hit.
8166 	 */
8167 top:
8168 	mutex_enter(&dev->l2ad_mtx);
8169 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
8170 		hdr_prev = list_prev(buflist, hdr);
8171 
8172 		hash_lock = HDR_LOCK(hdr);
8173 
8174 		/*
8175 		 * We cannot use mutex_enter or else we can deadlock
8176 		 * with l2arc_write_buffers (due to swapping the order
8177 		 * the hash lock and l2ad_mtx are taken).
8178 		 */
8179 		if (!mutex_tryenter(hash_lock)) {
8180 			/*
8181 			 * Missed the hash lock. We must retry so we
8182 			 * don't leave the ARC_FLAG_L2_WRITING bit set.
8183 			 */
8184 			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
8185 
8186 			/*
8187 			 * We don't want to rescan the headers we've
8188 			 * already marked as having been written out, so
8189 			 * we reinsert the head node so we can pick up
8190 			 * where we left off.
8191 			 */
8192 			list_remove(buflist, head);
8193 			list_insert_after(buflist, hdr, head);
8194 
8195 			mutex_exit(&dev->l2ad_mtx);
8196 
8197 			/*
8198 			 * We wait for the hash lock to become available
8199 			 * to try and prevent busy waiting, and increase
8200 			 * the chance we'll be able to acquire the lock
8201 			 * the next time around.
8202 			 */
8203 			mutex_enter(hash_lock);
8204 			mutex_exit(hash_lock);
8205 			goto top;
8206 		}
8207 
8208 		/*
8209 		 * We could not have been moved into the arc_l2c_only
8210 		 * state while in-flight due to our ARC_FLAG_L2_WRITING
8211 		 * bit being set. Let's just ensure that's being enforced.
8212 		 */
8213 		ASSERT(HDR_HAS_L1HDR(hdr));
8214 
8215 		/*
8216 		 * Skipped - drop L2ARC entry and mark the header as no
8217 		 * longer L2 eligibile.
8218 		 */
8219 		if (zio->io_error != 0) {
8220 			/*
8221 			 * Error - drop L2ARC entry.
8222 			 */
8223 			list_remove(buflist, hdr);
8224 			arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
8225 
8226 			uint64_t psize = HDR_GET_PSIZE(hdr);
8227 			l2arc_hdr_arcstats_decrement(hdr);
8228 
8229 			bytes_dropped +=
8230 			    vdev_psize_to_asize(dev->l2ad_vdev, psize);
8231 			(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
8232 			    arc_hdr_size(hdr), hdr);
8233 		}
8234 
8235 		/*
8236 		 * Allow ARC to begin reads and ghost list evictions to
8237 		 * this L2ARC entry.
8238 		 */
8239 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
8240 
8241 		mutex_exit(hash_lock);
8242 	}
8243 
8244 	/*
8245 	 * Free the allocated abd buffers for writing the log blocks.
8246 	 * If the zio failed reclaim the allocated space and remove the
8247 	 * pointers to these log blocks from the log block pointer list
8248 	 * of the L2ARC device.
8249 	 */
8250 	while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
8251 		abd_free(abd_buf->abd);
8252 		zio_buf_free(abd_buf, sizeof (*abd_buf));
8253 		if (zio->io_error != 0) {
8254 			lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
8255 			/*
8256 			 * L2BLK_GET_PSIZE returns aligned size for log
8257 			 * blocks.
8258 			 */
8259 			uint64_t asize =
8260 			    L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
8261 			bytes_dropped += asize;
8262 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
8263 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
8264 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
8265 			    lb_ptr_buf);
8266 			zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
8267 			kmem_free(lb_ptr_buf->lb_ptr,
8268 			    sizeof (l2arc_log_blkptr_t));
8269 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
8270 		}
8271 	}
8272 	list_destroy(&cb->l2wcb_abd_list);
8273 
8274 	if (zio->io_error != 0) {
8275 		ARCSTAT_BUMP(arcstat_l2_writes_error);
8276 
8277 		/*
8278 		 * Restore the lbps array in the header to its previous state.
8279 		 * If the list of log block pointers is empty, zero out the
8280 		 * log block pointers in the device header.
8281 		 */
8282 		lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
8283 		for (int i = 0; i < 2; i++) {
8284 			if (lb_ptr_buf == NULL) {
8285 				/*
8286 				 * If the list is empty zero out the device
8287 				 * header. Otherwise zero out the second log
8288 				 * block pointer in the header.
8289 				 */
8290 				if (i == 0) {
8291 					memset(l2dhdr, 0,
8292 					    dev->l2ad_dev_hdr_asize);
8293 				} else {
8294 					memset(&l2dhdr->dh_start_lbps[i], 0,
8295 					    sizeof (l2arc_log_blkptr_t));
8296 				}
8297 				break;
8298 			}
8299 			memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
8300 			    sizeof (l2arc_log_blkptr_t));
8301 			lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
8302 			    lb_ptr_buf);
8303 		}
8304 	}
8305 
8306 	ARCSTAT_BUMP(arcstat_l2_writes_done);
8307 	list_remove(buflist, head);
8308 	ASSERT(!HDR_HAS_L1HDR(head));
8309 	kmem_cache_free(hdr_l2only_cache, head);
8310 	mutex_exit(&dev->l2ad_mtx);
8311 
8312 	ASSERT(dev->l2ad_vdev != NULL);
8313 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
8314 
8315 	l2arc_do_free_on_write();
8316 
8317 	kmem_free(cb, sizeof (l2arc_write_callback_t));
8318 }
8319 
8320 static int
8321 l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
8322 {
8323 	int ret;
8324 	spa_t *spa = zio->io_spa;
8325 	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
8326 	blkptr_t *bp = zio->io_bp;
8327 	uint8_t salt[ZIO_DATA_SALT_LEN];
8328 	uint8_t iv[ZIO_DATA_IV_LEN];
8329 	uint8_t mac[ZIO_DATA_MAC_LEN];
8330 	boolean_t no_crypt = B_FALSE;
8331 
8332 	/*
8333 	 * ZIL data is never be written to the L2ARC, so we don't need
8334 	 * special handling for its unique MAC storage.
8335 	 */
8336 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
8337 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
8338 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8339 
8340 	/*
8341 	 * If the data was encrypted, decrypt it now. Note that
8342 	 * we must check the bp here and not the hdr, since the
8343 	 * hdr does not have its encryption parameters updated
8344 	 * until arc_read_done().
8345 	 */
8346 	if (BP_IS_ENCRYPTED(bp)) {
8347 		abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8348 		    ARC_HDR_USE_RESERVE);
8349 
8350 		zio_crypt_decode_params_bp(bp, salt, iv);
8351 		zio_crypt_decode_mac_bp(bp, mac);
8352 
8353 		ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
8354 		    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
8355 		    salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
8356 		    hdr->b_l1hdr.b_pabd, &no_crypt);
8357 		if (ret != 0) {
8358 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8359 			goto error;
8360 		}
8361 
8362 		/*
8363 		 * If we actually performed decryption, replace b_pabd
8364 		 * with the decrypted data. Otherwise we can just throw
8365 		 * our decryption buffer away.
8366 		 */
8367 		if (!no_crypt) {
8368 			arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8369 			    arc_hdr_size(hdr), hdr);
8370 			hdr->b_l1hdr.b_pabd = eabd;
8371 			zio->io_abd = eabd;
8372 		} else {
8373 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8374 		}
8375 	}
8376 
8377 	/*
8378 	 * If the L2ARC block was compressed, but ARC compression
8379 	 * is disabled we decompress the data into a new buffer and
8380 	 * replace the existing data.
8381 	 */
8382 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
8383 	    !HDR_COMPRESSION_ENABLED(hdr)) {
8384 		abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8385 		    ARC_HDR_USE_RESERVE);
8386 		void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
8387 
8388 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
8389 		    hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
8390 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
8391 		if (ret != 0) {
8392 			abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
8393 			arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
8394 			goto error;
8395 		}
8396 
8397 		abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
8398 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8399 		    arc_hdr_size(hdr), hdr);
8400 		hdr->b_l1hdr.b_pabd = cabd;
8401 		zio->io_abd = cabd;
8402 		zio->io_size = HDR_GET_LSIZE(hdr);
8403 	}
8404 
8405 	return (0);
8406 
8407 error:
8408 	return (ret);
8409 }
8410 
8411 
8412 /*
8413  * A read to a cache device completed.  Validate buffer contents before
8414  * handing over to the regular ARC routines.
8415  */
8416 static void
8417 l2arc_read_done(zio_t *zio)
8418 {
8419 	int tfm_error = 0;
8420 	l2arc_read_callback_t *cb = zio->io_private;
8421 	arc_buf_hdr_t *hdr;
8422 	kmutex_t *hash_lock;
8423 	boolean_t valid_cksum;
8424 	boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
8425 	    (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));
8426 
8427 	ASSERT3P(zio->io_vd, !=, NULL);
8428 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
8429 
8430 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
8431 
8432 	ASSERT3P(cb, !=, NULL);
8433 	hdr = cb->l2rcb_hdr;
8434 	ASSERT3P(hdr, !=, NULL);
8435 
8436 	hash_lock = HDR_LOCK(hdr);
8437 	mutex_enter(hash_lock);
8438 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
8439 
8440 	/*
8441 	 * If the data was read into a temporary buffer,
8442 	 * move it and free the buffer.
8443 	 */
8444 	if (cb->l2rcb_abd != NULL) {
8445 		ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
8446 		if (zio->io_error == 0) {
8447 			if (using_rdata) {
8448 				abd_copy(hdr->b_crypt_hdr.b_rabd,
8449 				    cb->l2rcb_abd, arc_hdr_size(hdr));
8450 			} else {
8451 				abd_copy(hdr->b_l1hdr.b_pabd,
8452 				    cb->l2rcb_abd, arc_hdr_size(hdr));
8453 			}
8454 		}
8455 
8456 		/*
8457 		 * The following must be done regardless of whether
8458 		 * there was an error:
8459 		 * - free the temporary buffer
8460 		 * - point zio to the real ARC buffer
8461 		 * - set zio size accordingly
8462 		 * These are required because zio is either re-used for
8463 		 * an I/O of the block in the case of the error
8464 		 * or the zio is passed to arc_read_done() and it
8465 		 * needs real data.
8466 		 */
8467 		abd_free(cb->l2rcb_abd);
8468 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
8469 
8470 		if (using_rdata) {
8471 			ASSERT(HDR_HAS_RABD(hdr));
8472 			zio->io_abd = zio->io_orig_abd =
8473 			    hdr->b_crypt_hdr.b_rabd;
8474 		} else {
8475 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8476 			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
8477 		}
8478 	}
8479 
8480 	ASSERT3P(zio->io_abd, !=, NULL);
8481 
8482 	/*
8483 	 * Check this survived the L2ARC journey.
8484 	 */
8485 	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
8486 	    (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
8487 	zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
8488 	zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
8489 	zio->io_prop.zp_complevel = hdr->b_complevel;
8490 
8491 	valid_cksum = arc_cksum_is_equal(hdr, zio);
8492 
8493 	/*
8494 	 * b_rabd will always match the data as it exists on disk if it is
8495 	 * being used. Therefore if we are reading into b_rabd we do not
8496 	 * attempt to untransform the data.
8497 	 */
8498 	if (valid_cksum && !using_rdata)
8499 		tfm_error = l2arc_untransform(zio, cb);
8500 
8501 	if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
8502 	    !HDR_L2_EVICTED(hdr)) {
8503 		mutex_exit(hash_lock);
8504 		zio->io_private = hdr;
8505 		arc_read_done(zio);
8506 	} else {
8507 		/*
8508 		 * Buffer didn't survive caching.  Increment stats and
8509 		 * reissue to the original storage device.
8510 		 */
8511 		if (zio->io_error != 0) {
8512 			ARCSTAT_BUMP(arcstat_l2_io_error);
8513 		} else {
8514 			zio->io_error = SET_ERROR(EIO);
8515 		}
8516 		if (!valid_cksum || tfm_error != 0)
8517 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
8518 
8519 		/*
8520 		 * If there's no waiter, issue an async i/o to the primary
8521 		 * storage now.  If there *is* a waiter, the caller must
8522 		 * issue the i/o in a context where it's OK to block.
8523 		 */
8524 		if (zio->io_waiter == NULL) {
8525 			zio_t *pio = zio_unique_parent(zio);
8526 			void *abd = (using_rdata) ?
8527 			    hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
8528 
8529 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
8530 
8531 			zio = zio_read(pio, zio->io_spa, zio->io_bp,
8532 			    abd, zio->io_size, arc_read_done,
8533 			    hdr, zio->io_priority, cb->l2rcb_flags,
8534 			    &cb->l2rcb_zb);
8535 
8536 			/*
8537 			 * Original ZIO will be freed, so we need to update
8538 			 * ARC header with the new ZIO pointer to be used
8539 			 * by zio_change_priority() in arc_read().
8540 			 */
8541 			for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
8542 			    acb != NULL; acb = acb->acb_next)
8543 				acb->acb_zio_head = zio;
8544 
8545 			mutex_exit(hash_lock);
8546 			zio_nowait(zio);
8547 		} else {
8548 			mutex_exit(hash_lock);
8549 		}
8550 	}
8551 
8552 	kmem_free(cb, sizeof (l2arc_read_callback_t));
8553 }
8554 
8555 /*
8556  * This is the list priority from which the L2ARC will search for pages to
8557  * cache.  This is used within loops (0..3) to cycle through lists in the
8558  * desired order.  This order can have a significant effect on cache
8559  * performance.
8560  *
8561  * Currently the metadata lists are hit first, MFU then MRU, followed by
8562  * the data lists.  This function returns a locked list, and also returns
8563  * the lock pointer.
8564  */
8565 static multilist_sublist_t *
8566 l2arc_sublist_lock(int list_num)
8567 {
8568 	multilist_t *ml = NULL;
8569 	unsigned int idx;
8570 
8571 	ASSERT(list_num >= 0 && list_num < L2ARC_FEED_TYPES);
8572 
8573 	switch (list_num) {
8574 	case 0:
8575 		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
8576 		break;
8577 	case 1:
8578 		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
8579 		break;
8580 	case 2:
8581 		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
8582 		break;
8583 	case 3:
8584 		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
8585 		break;
8586 	default:
8587 		return (NULL);
8588 	}
8589 
8590 	/*
8591 	 * Return a randomly-selected sublist. This is acceptable
8592 	 * because the caller feeds only a little bit of data for each
8593 	 * call (8MB). Subsequent calls will result in different
8594 	 * sublists being selected.
8595 	 */
8596 	idx = multilist_get_random_index(ml);
8597 	return (multilist_sublist_lock(ml, idx));
8598 }
8599 
8600 /*
8601  * Calculates the maximum overhead of L2ARC metadata log blocks for a given
8602  * L2ARC write size. l2arc_evict and l2arc_write_size need to include this
8603  * overhead in processing to make sure there is enough headroom available
8604  * when writing buffers.
8605  */
8606 static inline uint64_t
8607 l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev)
8608 {
8609 	if (dev->l2ad_log_entries == 0) {
8610 		return (0);
8611 	} else {
8612 		uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT;
8613 
8614 		uint64_t log_blocks = (log_entries +
8615 		    dev->l2ad_log_entries - 1) /
8616 		    dev->l2ad_log_entries;
8617 
8618 		return (vdev_psize_to_asize(dev->l2ad_vdev,
8619 		    sizeof (l2arc_log_blk_phys_t)) * log_blocks);
8620 	}
8621 }
8622 
8623 /*
8624  * Evict buffers from the device write hand to the distance specified in
8625  * bytes. This distance may span populated buffers, it may span nothing.
8626  * This is clearing a region on the L2ARC device ready for writing.
8627  * If the 'all' boolean is set, every buffer is evicted.
8628  */
8629 static void
8630 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
8631 {
8632 	list_t *buflist;
8633 	arc_buf_hdr_t *hdr, *hdr_prev;
8634 	kmutex_t *hash_lock;
8635 	uint64_t taddr;
8636 	l2arc_lb_ptr_buf_t *lb_ptr_buf, *lb_ptr_buf_prev;
8637 	vdev_t *vd = dev->l2ad_vdev;
8638 	boolean_t rerun;
8639 
8640 	buflist = &dev->l2ad_buflist;
8641 
8642 top:
8643 	rerun = B_FALSE;
8644 	if (dev->l2ad_hand + distance > dev->l2ad_end) {
8645 		/*
8646 		 * When there is no space to accommodate upcoming writes,
8647 		 * evict to the end. Then bump the write and evict hands
8648 		 * to the start and iterate. This iteration does not
8649 		 * happen indefinitely as we make sure in
8650 		 * l2arc_write_size() that when the write hand is reset,
8651 		 * the write size does not exceed the end of the device.
8652 		 */
8653 		rerun = B_TRUE;
8654 		taddr = dev->l2ad_end;
8655 	} else {
8656 		taddr = dev->l2ad_hand + distance;
8657 	}
8658 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
8659 	    uint64_t, taddr, boolean_t, all);
8660 
8661 	if (!all) {
8662 		/*
8663 		 * This check has to be placed after deciding whether to
8664 		 * iterate (rerun).
8665 		 */
8666 		if (dev->l2ad_first) {
8667 			/*
8668 			 * This is the first sweep through the device. There is
8669 			 * nothing to evict. We have already trimmmed the
8670 			 * whole device.
8671 			 */
8672 			goto out;
8673 		} else {
8674 			/*
8675 			 * Trim the space to be evicted.
8676 			 */
8677 			if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
8678 			    l2arc_trim_ahead > 0) {
8679 				/*
8680 				 * We have to drop the spa_config lock because
8681 				 * vdev_trim_range() will acquire it.
8682 				 * l2ad_evict already accounts for the label
8683 				 * size. To prevent vdev_trim_ranges() from
8684 				 * adding it again, we subtract it from
8685 				 * l2ad_evict.
8686 				 */
8687 				spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
8688 				vdev_trim_simple(vd,
8689 				    dev->l2ad_evict - VDEV_LABEL_START_SIZE,
8690 				    taddr - dev->l2ad_evict);
8691 				spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
8692 				    RW_READER);
8693 			}
8694 
8695 			/*
8696 			 * When rebuilding L2ARC we retrieve the evict hand
8697 			 * from the header of the device. Of note, l2arc_evict()
8698 			 * does not actually delete buffers from the cache
8699 			 * device, but trimming may do so depending on the
8700 			 * hardware implementation. Thus keeping track of the
8701 			 * evict hand is useful.
8702 			 */
8703 			dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
8704 		}
8705 	}
8706 
8707 retry:
8708 	mutex_enter(&dev->l2ad_mtx);
8709 	/*
8710 	 * We have to account for evicted log blocks. Run vdev_space_update()
8711 	 * on log blocks whose offset (in bytes) is before the evicted offset
8712 	 * (in bytes) by searching in the list of pointers to log blocks
8713 	 * present in the L2ARC device.
8714 	 */
8715 	for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
8716 	    lb_ptr_buf = lb_ptr_buf_prev) {
8717 
8718 		lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);
8719 
8720 		/* L2BLK_GET_PSIZE returns aligned size for log blocks */
8721 		uint64_t asize = L2BLK_GET_PSIZE(
8722 		    (lb_ptr_buf->lb_ptr)->lbp_prop);
8723 
8724 		/*
8725 		 * We don't worry about log blocks left behind (ie
8726 		 * lbp_payload_start < l2ad_hand) because l2arc_write_buffers()
8727 		 * will never write more than l2arc_evict() evicts.
8728 		 */
8729 		if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
8730 			break;
8731 		} else {
8732 			vdev_space_update(vd, -asize, 0, 0);
8733 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
8734 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
8735 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
8736 			    lb_ptr_buf);
8737 			zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
8738 			list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
8739 			kmem_free(lb_ptr_buf->lb_ptr,
8740 			    sizeof (l2arc_log_blkptr_t));
8741 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
8742 		}
8743 	}
8744 
8745 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
8746 		hdr_prev = list_prev(buflist, hdr);
8747 
8748 		ASSERT(!HDR_EMPTY(hdr));
8749 		hash_lock = HDR_LOCK(hdr);
8750 
8751 		/*
8752 		 * We cannot use mutex_enter or else we can deadlock
8753 		 * with l2arc_write_buffers (due to swapping the order
8754 		 * the hash lock and l2ad_mtx are taken).
8755 		 */
8756 		if (!mutex_tryenter(hash_lock)) {
8757 			/*
8758 			 * Missed the hash lock.  Retry.
8759 			 */
8760 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
8761 			mutex_exit(&dev->l2ad_mtx);
8762 			mutex_enter(hash_lock);
8763 			mutex_exit(hash_lock);
8764 			goto retry;
8765 		}
8766 
8767 		/*
8768 		 * A header can't be on this list if it doesn't have L2 header.
8769 		 */
8770 		ASSERT(HDR_HAS_L2HDR(hdr));
8771 
8772 		/* Ensure this header has finished being written. */
8773 		ASSERT(!HDR_L2_WRITING(hdr));
8774 		ASSERT(!HDR_L2_WRITE_HEAD(hdr));
8775 
8776 		if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
8777 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
8778 			/*
8779 			 * We've evicted to the target address,
8780 			 * or the end of the device.
8781 			 */
8782 			mutex_exit(hash_lock);
8783 			break;
8784 		}
8785 
8786 		if (!HDR_HAS_L1HDR(hdr)) {
8787 			ASSERT(!HDR_L2_READING(hdr));
8788 			/*
8789 			 * This doesn't exist in the ARC.  Destroy.
8790 			 * arc_hdr_destroy() will call list_remove()
8791 			 * and decrement arcstat_l2_lsize.
8792 			 */
8793 			arc_change_state(arc_anon, hdr);
8794 			arc_hdr_destroy(hdr);
8795 		} else {
8796 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
8797 			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
8798 			/*
8799 			 * Invalidate issued or about to be issued
8800 			 * reads, since we may be about to write
8801 			 * over this location.
8802 			 */
8803 			if (HDR_L2_READING(hdr)) {
8804 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
8805 				arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
8806 			}
8807 
8808 			arc_hdr_l2hdr_destroy(hdr);
8809 		}
8810 		mutex_exit(hash_lock);
8811 	}
8812 	mutex_exit(&dev->l2ad_mtx);
8813 
8814 out:
8815 	/*
8816 	 * We need to check if we evict all buffers, otherwise we may iterate
8817 	 * unnecessarily.
8818 	 */
8819 	if (!all && rerun) {
8820 		/*
8821 		 * Bump device hand to the device start if it is approaching the
8822 		 * end. l2arc_evict() has already evicted ahead for this case.
8823 		 */
8824 		dev->l2ad_hand = dev->l2ad_start;
8825 		dev->l2ad_evict = dev->l2ad_start;
8826 		dev->l2ad_first = B_FALSE;
8827 		goto top;
8828 	}
8829 
8830 	if (!all) {
8831 		/*
8832 		 * In case of cache device removal (all) the following
8833 		 * assertions may be violated without functional consequences
8834 		 * as the device is about to be removed.
8835 		 */
8836 		ASSERT3U(dev->l2ad_hand + distance, <, dev->l2ad_end);
8837 		if (!dev->l2ad_first)
8838 			ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
8839 	}
8840 }
8841 
8842 /*
8843  * Handle any abd transforms that might be required for writing to the L2ARC.
8844  * If successful, this function will always return an abd with the data
8845  * transformed as it is on disk in a new abd of asize bytes.
8846  */
8847 static int
8848 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
8849     abd_t **abd_out)
8850 {
8851 	int ret;
8852 	void *tmp = NULL;
8853 	abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
8854 	enum zio_compress compress = HDR_GET_COMPRESS(hdr);
8855 	uint64_t psize = HDR_GET_PSIZE(hdr);
8856 	uint64_t size = arc_hdr_size(hdr);
8857 	boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
8858 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
8859 	dsl_crypto_key_t *dck = NULL;
8860 	uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
8861 	boolean_t no_crypt = B_FALSE;
8862 
8863 	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
8864 	    !HDR_COMPRESSION_ENABLED(hdr)) ||
8865 	    HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);
8866 	ASSERT3U(psize, <=, asize);
8867 
8868 	/*
8869 	 * If this data simply needs its own buffer, we simply allocate it
8870 	 * and copy the data. This may be done to eliminate a dependency on a
8871 	 * shared buffer or to reallocate the buffer to match asize.
8872 	 */
8873 	if (HDR_HAS_RABD(hdr) && asize != psize) {
8874 		ASSERT3U(asize, >=, psize);
8875 		to_write = abd_alloc_for_io(asize, ismd);
8876 		abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
8877 		if (psize != asize)
8878 			abd_zero_off(to_write, psize, asize - psize);
8879 		goto out;
8880 	}
8881 
8882 	if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
8883 	    !HDR_ENCRYPTED(hdr)) {
8884 		ASSERT3U(size, ==, psize);
8885 		to_write = abd_alloc_for_io(asize, ismd);
8886 		abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
8887 		if (size != asize)
8888 			abd_zero_off(to_write, size, asize - size);
8889 		goto out;
8890 	}
8891 
8892 	if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
8893 		/*
8894 		 * In some cases, we can wind up with size > asize, so
8895 		 * we need to opt for the larger allocation option here.
8896 		 *
8897 		 * (We also need abd_return_buf_copy in all cases because
8898 		 * it's an ASSERT() to modify the buffer before returning it
8899 		 * with arc_return_buf(), and all the compressors
8900 		 * write things before deciding to fail compression in nearly
8901 		 * every case.)
8902 		 */
8903 		uint64_t bufsize = MAX(size, asize);
8904 		cabd = abd_alloc_for_io(bufsize, ismd);
8905 		tmp = abd_borrow_buf(cabd, bufsize);
8906 
8907 		psize = zio_compress_data(compress, to_write, &tmp, size,
8908 		    hdr->b_complevel);
8909 
8910 		if (psize >= asize) {
8911 			psize = HDR_GET_PSIZE(hdr);
8912 			abd_return_buf_copy(cabd, tmp, bufsize);
8913 			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
8914 			to_write = cabd;
8915 			abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize);
8916 			if (psize != asize)
8917 				abd_zero_off(to_write, psize, asize - psize);
8918 			goto encrypt;
8919 		}
8920 		ASSERT3U(psize, <=, HDR_GET_PSIZE(hdr));
8921 		if (psize < asize)
8922 			memset((char *)tmp + psize, 0, bufsize - psize);
8923 		psize = HDR_GET_PSIZE(hdr);
8924 		abd_return_buf_copy(cabd, tmp, bufsize);
8925 		to_write = cabd;
8926 	}
8927 
8928 encrypt:
8929 	if (HDR_ENCRYPTED(hdr)) {
8930 		eabd = abd_alloc_for_io(asize, ismd);
8931 
8932 		/*
8933 		 * If the dataset was disowned before the buffer
8934 		 * made it to this point, the key to re-encrypt
8935 		 * it won't be available. In this case we simply
8936 		 * won't write the buffer to the L2ARC.
8937 		 */
8938 		ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
8939 		    FTAG, &dck);
8940 		if (ret != 0)
8941 			goto error;
8942 
8943 		ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
8944 		    hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
8945 		    hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
8946 		    &no_crypt);
8947 		if (ret != 0)
8948 			goto error;
8949 
8950 		if (no_crypt)
8951 			abd_copy(eabd, to_write, psize);
8952 
8953 		if (psize != asize)
8954 			abd_zero_off(eabd, psize, asize - psize);
8955 
8956 		/* assert that the MAC we got here matches the one we saved */
8957 		ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
8958 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
8959 
8960 		if (to_write == cabd)
8961 			abd_free(cabd);
8962 
8963 		to_write = eabd;
8964 	}
8965 
8966 out:
8967 	ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
8968 	*abd_out = to_write;
8969 	return (0);
8970 
8971 error:
8972 	if (dck != NULL)
8973 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
8974 	if (cabd != NULL)
8975 		abd_free(cabd);
8976 	if (eabd != NULL)
8977 		abd_free(eabd);
8978 
8979 	*abd_out = NULL;
8980 	return (ret);
8981 }
8982 
8983 static void
8984 l2arc_blk_fetch_done(zio_t *zio)
8985 {
8986 	l2arc_read_callback_t *cb;
8987 
8988 	cb = zio->io_private;
8989 	if (cb->l2rcb_abd != NULL)
8990 		abd_free(cb->l2rcb_abd);
8991 	kmem_free(cb, sizeof (l2arc_read_callback_t));
8992 }
8993 
8994 /*
8995  * Find and write ARC buffers to the L2ARC device.
8996  *
8997  * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
8998  * for reading until they have completed writing.
8999  * The headroom_boost is an in-out parameter used to maintain headroom boost
9000  * state between calls to this function.
9001  *
9002  * Returns the number of bytes actually written (which may be smaller than
9003  * the delta by which the device hand has changed due to alignment and the
9004  * writing of log blocks).
9005  */
9006 static uint64_t
9007 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
9008 {
9009 	arc_buf_hdr_t 		*hdr, *hdr_prev, *head;
9010 	uint64_t 		write_asize, write_psize, write_lsize, headroom;
9011 	boolean_t		full;
9012 	l2arc_write_callback_t	*cb = NULL;
9013 	zio_t 			*pio, *wzio;
9014 	uint64_t 		guid = spa_load_guid(spa);
9015 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
9016 
9017 	ASSERT3P(dev->l2ad_vdev, !=, NULL);
9018 
9019 	pio = NULL;
9020 	write_lsize = write_asize = write_psize = 0;
9021 	full = B_FALSE;
9022 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
9023 	arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
9024 
9025 	/*
9026 	 * Copy buffers for L2ARC writing.
9027 	 */
9028 	for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) {
9029 		/*
9030 		 * If pass == 1 or 3, we cache MRU metadata and data
9031 		 * respectively.
9032 		 */
9033 		if (l2arc_mfuonly) {
9034 			if (pass == 1 || pass == 3)
9035 				continue;
9036 		}
9037 
9038 		multilist_sublist_t *mls = l2arc_sublist_lock(pass);
9039 		uint64_t passed_sz = 0;
9040 
9041 		VERIFY3P(mls, !=, NULL);
9042 
9043 		/*
9044 		 * L2ARC fast warmup.
9045 		 *
9046 		 * Until the ARC is warm and starts to evict, read from the
9047 		 * head of the ARC lists rather than the tail.
9048 		 */
9049 		if (arc_warm == B_FALSE)
9050 			hdr = multilist_sublist_head(mls);
9051 		else
9052 			hdr = multilist_sublist_tail(mls);
9053 
9054 		headroom = target_sz * l2arc_headroom;
9055 		if (zfs_compressed_arc_enabled)
9056 			headroom = (headroom * l2arc_headroom_boost) / 100;
9057 
9058 		for (; hdr; hdr = hdr_prev) {
9059 			kmutex_t *hash_lock;
9060 			abd_t *to_write = NULL;
9061 
9062 			if (arc_warm == B_FALSE)
9063 				hdr_prev = multilist_sublist_next(mls, hdr);
9064 			else
9065 				hdr_prev = multilist_sublist_prev(mls, hdr);
9066 
9067 			hash_lock = HDR_LOCK(hdr);
9068 			if (!mutex_tryenter(hash_lock)) {
9069 				/*
9070 				 * Skip this buffer rather than waiting.
9071 				 */
9072 				continue;
9073 			}
9074 
9075 			passed_sz += HDR_GET_LSIZE(hdr);
9076 			if (l2arc_headroom != 0 && passed_sz > headroom) {
9077 				/*
9078 				 * Searched too far.
9079 				 */
9080 				mutex_exit(hash_lock);
9081 				break;
9082 			}
9083 
9084 			if (!l2arc_write_eligible(guid, hdr)) {
9085 				mutex_exit(hash_lock);
9086 				continue;
9087 			}
9088 
9089 			ASSERT(HDR_HAS_L1HDR(hdr));
9090 
9091 			ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
9092 			ASSERT3U(arc_hdr_size(hdr), >, 0);
9093 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
9094 			    HDR_HAS_RABD(hdr));
9095 			uint64_t psize = HDR_GET_PSIZE(hdr);
9096 			uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
9097 			    psize);
9098 
9099 			/*
9100 			 * If the allocated size of this buffer plus the max
9101 			 * size for the pending log block exceeds the evicted
9102 			 * target size, terminate writing buffers for this run.
9103 			 */
9104 			if (write_asize + asize +
9105 			    sizeof (l2arc_log_blk_phys_t) > target_sz) {
9106 				full = B_TRUE;
9107 				mutex_exit(hash_lock);
9108 				break;
9109 			}
9110 
9111 			/*
9112 			 * We rely on the L1 portion of the header below, so
9113 			 * it's invalid for this header to have been evicted out
9114 			 * of the ghost cache, prior to being written out. The
9115 			 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
9116 			 */
9117 			arc_hdr_set_flags(hdr, ARC_FLAG_L2_WRITING);
9118 
9119 			/*
9120 			 * If this header has b_rabd, we can use this since it
9121 			 * must always match the data exactly as it exists on
9122 			 * disk. Otherwise, the L2ARC can normally use the
9123 			 * hdr's data, but if we're sharing data between the
9124 			 * hdr and one of its bufs, L2ARC needs its own copy of
9125 			 * the data so that the ZIO below can't race with the
9126 			 * buf consumer. To ensure that this copy will be
9127 			 * available for the lifetime of the ZIO and be cleaned
9128 			 * up afterwards, we add it to the l2arc_free_on_write
9129 			 * queue. If we need to apply any transforms to the
9130 			 * data (compression, encryption) we will also need the
9131 			 * extra buffer.
9132 			 */
9133 			if (HDR_HAS_RABD(hdr) && psize == asize) {
9134 				to_write = hdr->b_crypt_hdr.b_rabd;
9135 			} else if ((HDR_COMPRESSION_ENABLED(hdr) ||
9136 			    HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
9137 			    !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
9138 			    psize == asize) {
9139 				to_write = hdr->b_l1hdr.b_pabd;
9140 			} else {
9141 				int ret;
9142 				arc_buf_contents_t type = arc_buf_type(hdr);
9143 
9144 				ret = l2arc_apply_transforms(spa, hdr, asize,
9145 				    &to_write);
9146 				if (ret != 0) {
9147 					arc_hdr_clear_flags(hdr,
9148 					    ARC_FLAG_L2_WRITING);
9149 					mutex_exit(hash_lock);
9150 					continue;
9151 				}
9152 
9153 				l2arc_free_abd_on_write(to_write, asize, type);
9154 			}
9155 
9156 			if (pio == NULL) {
9157 				/*
9158 				 * Insert a dummy header on the buflist so
9159 				 * l2arc_write_done() can find where the
9160 				 * write buffers begin without searching.
9161 				 */
9162 				mutex_enter(&dev->l2ad_mtx);
9163 				list_insert_head(&dev->l2ad_buflist, head);
9164 				mutex_exit(&dev->l2ad_mtx);
9165 
9166 				cb = kmem_alloc(
9167 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
9168 				cb->l2wcb_dev = dev;
9169 				cb->l2wcb_head = head;
9170 				/*
9171 				 * Create a list to save allocated abd buffers
9172 				 * for l2arc_log_blk_commit().
9173 				 */
9174 				list_create(&cb->l2wcb_abd_list,
9175 				    sizeof (l2arc_lb_abd_buf_t),
9176 				    offsetof(l2arc_lb_abd_buf_t, node));
9177 				pio = zio_root(spa, l2arc_write_done, cb,
9178 				    ZIO_FLAG_CANFAIL);
9179 			}
9180 
9181 			hdr->b_l2hdr.b_dev = dev;
9182 			hdr->b_l2hdr.b_hits = 0;
9183 
9184 			hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
9185 			hdr->b_l2hdr.b_arcs_state =
9186 			    hdr->b_l1hdr.b_state->arcs_state;
9187 			arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR);
9188 
9189 			mutex_enter(&dev->l2ad_mtx);
9190 			list_insert_head(&dev->l2ad_buflist, hdr);
9191 			mutex_exit(&dev->l2ad_mtx);
9192 
9193 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
9194 			    arc_hdr_size(hdr), hdr);
9195 
9196 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
9197 			    hdr->b_l2hdr.b_daddr, asize, to_write,
9198 			    ZIO_CHECKSUM_OFF, NULL, hdr,
9199 			    ZIO_PRIORITY_ASYNC_WRITE,
9200 			    ZIO_FLAG_CANFAIL, B_FALSE);
9201 
9202 			write_lsize += HDR_GET_LSIZE(hdr);
9203 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
9204 			    zio_t *, wzio);
9205 
9206 			write_psize += psize;
9207 			write_asize += asize;
9208 			dev->l2ad_hand += asize;
9209 			l2arc_hdr_arcstats_increment(hdr);
9210 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
9211 
9212 			mutex_exit(hash_lock);
9213 
9214 			/*
9215 			 * Append buf info to current log and commit if full.
9216 			 * arcstat_l2_{size,asize} kstats are updated
9217 			 * internally.
9218 			 */
9219 			if (l2arc_log_blk_insert(dev, hdr)) {
9220 				/*
9221 				 * l2ad_hand will be adjusted in
9222 				 * l2arc_log_blk_commit().
9223 				 */
9224 				write_asize +=
9225 				    l2arc_log_blk_commit(dev, pio, cb);
9226 			}
9227 
9228 			zio_nowait(wzio);
9229 		}
9230 
9231 		multilist_sublist_unlock(mls);
9232 
9233 		if (full == B_TRUE)
9234 			break;
9235 	}
9236 
9237 	/* No buffers selected for writing? */
9238 	if (pio == NULL) {
9239 		ASSERT0(write_lsize);
9240 		ASSERT(!HDR_HAS_L1HDR(head));
9241 		kmem_cache_free(hdr_l2only_cache, head);
9242 
9243 		/*
9244 		 * Although we did not write any buffers l2ad_evict may
9245 		 * have advanced.
9246 		 */
9247 		if (dev->l2ad_evict != l2dhdr->dh_evict)
9248 			l2arc_dev_hdr_update(dev);
9249 
9250 		return (0);
9251 	}
9252 
9253 	if (!dev->l2ad_first)
9254 		ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9255 
9256 	ASSERT3U(write_asize, <=, target_sz);
9257 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
9258 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);
9259 
9260 	dev->l2ad_writing = B_TRUE;
9261 	(void) zio_wait(pio);
9262 	dev->l2ad_writing = B_FALSE;
9263 
9264 	/*
9265 	 * Update the device header after the zio completes as
9266 	 * l2arc_write_done() may have updated the memory holding the log block
9267 	 * pointers in the device header.
9268 	 */
9269 	l2arc_dev_hdr_update(dev);
9270 
9271 	return (write_asize);
9272 }
9273 
9274 static boolean_t
9275 l2arc_hdr_limit_reached(void)
9276 {
9277 	int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);
9278 
9279 	return (arc_reclaim_needed() ||
9280 	    (s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
9281 }
9282 
9283 /*
9284  * This thread feeds the L2ARC at regular intervals.  This is the beating
9285  * heart of the L2ARC.
9286  */
9287 static  __attribute__((noreturn)) void
9288 l2arc_feed_thread(void *unused)
9289 {
9290 	(void) unused;
9291 	callb_cpr_t cpr;
9292 	l2arc_dev_t *dev;
9293 	spa_t *spa;
9294 	uint64_t size, wrote;
9295 	clock_t begin, next = ddi_get_lbolt();
9296 	fstrans_cookie_t cookie;
9297 
9298 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
9299 
9300 	mutex_enter(&l2arc_feed_thr_lock);
9301 
9302 	cookie = spl_fstrans_mark();
9303 	while (l2arc_thread_exit == 0) {
9304 		CALLB_CPR_SAFE_BEGIN(&cpr);
9305 		(void) cv_timedwait_idle(&l2arc_feed_thr_cv,
9306 		    &l2arc_feed_thr_lock, next);
9307 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
9308 		next = ddi_get_lbolt() + hz;
9309 
9310 		/*
9311 		 * Quick check for L2ARC devices.
9312 		 */
9313 		mutex_enter(&l2arc_dev_mtx);
9314 		if (l2arc_ndev == 0) {
9315 			mutex_exit(&l2arc_dev_mtx);
9316 			continue;
9317 		}
9318 		mutex_exit(&l2arc_dev_mtx);
9319 		begin = ddi_get_lbolt();
9320 
9321 		/*
9322 		 * This selects the next l2arc device to write to, and in
9323 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
9324 		 * will return NULL if there are now no l2arc devices or if
9325 		 * they are all faulted.
9326 		 *
9327 		 * If a device is returned, its spa's config lock is also
9328 		 * held to prevent device removal.  l2arc_dev_get_next()
9329 		 * will grab and release l2arc_dev_mtx.
9330 		 */
9331 		if ((dev = l2arc_dev_get_next()) == NULL)
9332 			continue;
9333 
9334 		spa = dev->l2ad_spa;
9335 		ASSERT3P(spa, !=, NULL);
9336 
9337 		/*
9338 		 * If the pool is read-only then force the feed thread to
9339 		 * sleep a little longer.
9340 		 */
9341 		if (!spa_writeable(spa)) {
9342 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
9343 			spa_config_exit(spa, SCL_L2ARC, dev);
9344 			continue;
9345 		}
9346 
9347 		/*
9348 		 * Avoid contributing to memory pressure.
9349 		 */
9350 		if (l2arc_hdr_limit_reached()) {
9351 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
9352 			spa_config_exit(spa, SCL_L2ARC, dev);
9353 			continue;
9354 		}
9355 
9356 		ARCSTAT_BUMP(arcstat_l2_feeds);
9357 
9358 		size = l2arc_write_size(dev);
9359 
9360 		/*
9361 		 * Evict L2ARC buffers that will be overwritten.
9362 		 */
9363 		l2arc_evict(dev, size, B_FALSE);
9364 
9365 		/*
9366 		 * Write ARC buffers.
9367 		 */
9368 		wrote = l2arc_write_buffers(spa, dev, size);
9369 
9370 		/*
9371 		 * Calculate interval between writes.
9372 		 */
9373 		next = l2arc_write_interval(begin, size, wrote);
9374 		spa_config_exit(spa, SCL_L2ARC, dev);
9375 	}
9376 	spl_fstrans_unmark(cookie);
9377 
9378 	l2arc_thread_exit = 0;
9379 	cv_broadcast(&l2arc_feed_thr_cv);
9380 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
9381 	thread_exit();
9382 }
9383 
9384 boolean_t
9385 l2arc_vdev_present(vdev_t *vd)
9386 {
9387 	return (l2arc_vdev_get(vd) != NULL);
9388 }
9389 
9390 /*
9391  * Returns the l2arc_dev_t associated with a particular vdev_t or NULL if
9392  * the vdev_t isn't an L2ARC device.
9393  */
9394 l2arc_dev_t *
9395 l2arc_vdev_get(vdev_t *vd)
9396 {
9397 	l2arc_dev_t	*dev;
9398 
9399 	mutex_enter(&l2arc_dev_mtx);
9400 	for (dev = list_head(l2arc_dev_list); dev != NULL;
9401 	    dev = list_next(l2arc_dev_list, dev)) {
9402 		if (dev->l2ad_vdev == vd)
9403 			break;
9404 	}
9405 	mutex_exit(&l2arc_dev_mtx);
9406 
9407 	return (dev);
9408 }
9409 
9410 static void
9411 l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
9412 {
9413 	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9414 	uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
9415 	spa_t *spa = dev->l2ad_spa;
9416 
9417 	/*
9418 	 * The L2ARC has to hold at least the payload of one log block for
9419 	 * them to be restored (persistent L2ARC). The payload of a log block
9420 	 * depends on the amount of its log entries. We always write log blocks
9421 	 * with 1022 entries. How many of them are committed or restored depends
9422 	 * on the size of the L2ARC device. Thus the maximum payload of
9423 	 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
9424 	 * is less than that, we reduce the amount of committed and restored
9425 	 * log entries per block so as to enable persistence.
9426 	 */
9427 	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
9428 		dev->l2ad_log_entries = 0;
9429 	} else {
9430 		dev->l2ad_log_entries = MIN((dev->l2ad_end -
9431 		    dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
9432 		    L2ARC_LOG_BLK_MAX_ENTRIES);
9433 	}
9434 
9435 	/*
9436 	 * Read the device header, if an error is returned do not rebuild L2ARC.
9437 	 */
9438 	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
9439 		/*
9440 		 * If we are onlining a cache device (vdev_reopen) that was
9441 		 * still present (l2arc_vdev_present()) and rebuild is enabled,
9442 		 * we should evict all ARC buffers and pointers to log blocks
9443 		 * and reclaim their space before restoring its contents to
9444 		 * L2ARC.
9445 		 */
9446 		if (reopen) {
9447 			if (!l2arc_rebuild_enabled) {
9448 				return;
9449 			} else {
9450 				l2arc_evict(dev, 0, B_TRUE);
9451 				/* start a new log block */
9452 				dev->l2ad_log_ent_idx = 0;
9453 				dev->l2ad_log_blk_payload_asize = 0;
9454 				dev->l2ad_log_blk_payload_start = 0;
9455 			}
9456 		}
9457 		/*
9458 		 * Just mark the device as pending for a rebuild. We won't
9459 		 * be starting a rebuild in line here as it would block pool
9460 		 * import. Instead spa_load_impl will hand that off to an
9461 		 * async task which will call l2arc_spa_rebuild_start.
9462 		 */
9463 		dev->l2ad_rebuild = B_TRUE;
9464 	} else if (spa_writeable(spa)) {
9465 		/*
9466 		 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
9467 		 * otherwise create a new header. We zero out the memory holding
9468 		 * the header to reset dh_start_lbps. If we TRIM the whole
9469 		 * device the new header will be written by
9470 		 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
9471 		 * trim_state in the header too. When reading the header, if
9472 		 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
9473 		 * we opt to TRIM the whole device again.
9474 		 */
9475 		if (l2arc_trim_ahead > 0) {
9476 			dev->l2ad_trim_all = B_TRUE;
9477 		} else {
9478 			memset(l2dhdr, 0, l2dhdr_asize);
9479 			l2arc_dev_hdr_update(dev);
9480 		}
9481 	}
9482 }
9483 
9484 /*
9485  * Add a vdev for use by the L2ARC.  By this point the spa has already
9486  * validated the vdev and opened it.
9487  */
9488 void
9489 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
9490 {
9491 	l2arc_dev_t		*adddev;
9492 	uint64_t		l2dhdr_asize;
9493 
9494 	ASSERT(!l2arc_vdev_present(vd));
9495 
9496 	/*
9497 	 * Create a new l2arc device entry.
9498 	 */
9499 	adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
9500 	adddev->l2ad_spa = spa;
9501 	adddev->l2ad_vdev = vd;
9502 	/* leave extra size for an l2arc device header */
9503 	l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
9504 	    MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
9505 	adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
9506 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
9507 	ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
9508 	adddev->l2ad_hand = adddev->l2ad_start;
9509 	adddev->l2ad_evict = adddev->l2ad_start;
9510 	adddev->l2ad_first = B_TRUE;
9511 	adddev->l2ad_writing = B_FALSE;
9512 	adddev->l2ad_trim_all = B_FALSE;
9513 	list_link_init(&adddev->l2ad_node);
9514 	adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);
9515 
9516 	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
9517 	/*
9518 	 * This is a list of all ARC buffers that are still valid on the
9519 	 * device.
9520 	 */
9521 	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
9522 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
9523 
9524 	/*
9525 	 * This is a list of pointers to log blocks that are still present
9526 	 * on the device.
9527 	 */
9528 	list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
9529 	    offsetof(l2arc_lb_ptr_buf_t, node));
9530 
9531 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
9532 	zfs_refcount_create(&adddev->l2ad_alloc);
9533 	zfs_refcount_create(&adddev->l2ad_lb_asize);
9534 	zfs_refcount_create(&adddev->l2ad_lb_count);
9535 
9536 	/*
9537 	 * Decide if dev is eligible for L2ARC rebuild or whole device
9538 	 * trimming. This has to happen before the device is added in the
9539 	 * cache device list and l2arc_dev_mtx is released. Otherwise
9540 	 * l2arc_feed_thread() might already start writing on the
9541 	 * device.
9542 	 */
9543 	l2arc_rebuild_dev(adddev, B_FALSE);
9544 
9545 	/*
9546 	 * Add device to global list
9547 	 */
9548 	mutex_enter(&l2arc_dev_mtx);
9549 	list_insert_head(l2arc_dev_list, adddev);
9550 	atomic_inc_64(&l2arc_ndev);
9551 	mutex_exit(&l2arc_dev_mtx);
9552 }
9553 
9554 /*
9555  * Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
9556  * in case of onlining a cache device.
9557  */
9558 void
9559 l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
9560 {
9561 	l2arc_dev_t		*dev = NULL;
9562 
9563 	dev = l2arc_vdev_get(vd);
9564 	ASSERT3P(dev, !=, NULL);
9565 
9566 	/*
9567 	 * In contrast to l2arc_add_vdev() we do not have to worry about
9568 	 * l2arc_feed_thread() invalidating previous content when onlining a
9569 	 * cache device. The device parameters (l2ad*) are not cleared when
9570 	 * offlining the device and writing new buffers will not invalidate
9571 	 * all previous content. In worst case only buffers that have not had
9572 	 * their log block written to the device will be lost.
9573 	 * When onlining the cache device (ie offline->online without exporting
9574 	 * the pool in between) this happens:
9575 	 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
9576 	 * 			|			|
9577 	 * 		vdev_is_dead() = B_FALSE	l2ad_rebuild = B_TRUE
9578 	 * During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
9579 	 * is set to B_TRUE we might write additional buffers to the device.
9580 	 */
9581 	l2arc_rebuild_dev(dev, reopen);
9582 }
9583 
9584 /*
9585  * Remove a vdev from the L2ARC.
9586  */
9587 void
9588 l2arc_remove_vdev(vdev_t *vd)
9589 {
9590 	l2arc_dev_t *remdev = NULL;
9591 
9592 	/*
9593 	 * Find the device by vdev
9594 	 */
9595 	remdev = l2arc_vdev_get(vd);
9596 	ASSERT3P(remdev, !=, NULL);
9597 
9598 	/*
9599 	 * Cancel any ongoing or scheduled rebuild.
9600 	 */
9601 	mutex_enter(&l2arc_rebuild_thr_lock);
9602 	if (remdev->l2ad_rebuild_began == B_TRUE) {
9603 		remdev->l2ad_rebuild_cancel = B_TRUE;
9604 		while (remdev->l2ad_rebuild == B_TRUE)
9605 			cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
9606 	}
9607 	mutex_exit(&l2arc_rebuild_thr_lock);
9608 
9609 	/*
9610 	 * Remove device from global list
9611 	 */
9612 	mutex_enter(&l2arc_dev_mtx);
9613 	list_remove(l2arc_dev_list, remdev);
9614 	l2arc_dev_last = NULL;		/* may have been invalidated */
9615 	atomic_dec_64(&l2arc_ndev);
9616 	mutex_exit(&l2arc_dev_mtx);
9617 
9618 	/*
9619 	 * Clear all buflists and ARC references.  L2ARC device flush.
9620 	 */
9621 	l2arc_evict(remdev, 0, B_TRUE);
9622 	list_destroy(&remdev->l2ad_buflist);
9623 	ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
9624 	list_destroy(&remdev->l2ad_lbptr_list);
9625 	mutex_destroy(&remdev->l2ad_mtx);
9626 	zfs_refcount_destroy(&remdev->l2ad_alloc);
9627 	zfs_refcount_destroy(&remdev->l2ad_lb_asize);
9628 	zfs_refcount_destroy(&remdev->l2ad_lb_count);
9629 	kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
9630 	vmem_free(remdev, sizeof (l2arc_dev_t));
9631 }
9632 
9633 void
9634 l2arc_init(void)
9635 {
9636 	l2arc_thread_exit = 0;
9637 	l2arc_ndev = 0;
9638 
9639 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
9640 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
9641 	mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL);
9642 	cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL);
9643 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
9644 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
9645 
9646 	l2arc_dev_list = &L2ARC_dev_list;
9647 	l2arc_free_on_write = &L2ARC_free_on_write;
9648 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
9649 	    offsetof(l2arc_dev_t, l2ad_node));
9650 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
9651 	    offsetof(l2arc_data_free_t, l2df_list_node));
9652 }
9653 
9654 void
9655 l2arc_fini(void)
9656 {
9657 	mutex_destroy(&l2arc_feed_thr_lock);
9658 	cv_destroy(&l2arc_feed_thr_cv);
9659 	mutex_destroy(&l2arc_rebuild_thr_lock);
9660 	cv_destroy(&l2arc_rebuild_thr_cv);
9661 	mutex_destroy(&l2arc_dev_mtx);
9662 	mutex_destroy(&l2arc_free_on_write_mtx);
9663 
9664 	list_destroy(l2arc_dev_list);
9665 	list_destroy(l2arc_free_on_write);
9666 }
9667 
9668 void
9669 l2arc_start(void)
9670 {
9671 	if (!(spa_mode_global & SPA_MODE_WRITE))
9672 		return;
9673 
9674 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
9675 	    TS_RUN, defclsyspri);
9676 }
9677 
9678 void
9679 l2arc_stop(void)
9680 {
9681 	if (!(spa_mode_global & SPA_MODE_WRITE))
9682 		return;
9683 
9684 	mutex_enter(&l2arc_feed_thr_lock);
9685 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
9686 	l2arc_thread_exit = 1;
9687 	while (l2arc_thread_exit != 0)
9688 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
9689 	mutex_exit(&l2arc_feed_thr_lock);
9690 }
9691 
9692 /*
9693  * Punches out rebuild threads for the L2ARC devices in a spa. This should
9694  * be called after pool import from the spa async thread, since starting
9695  * these threads directly from spa_import() will make them part of the
9696  * "zpool import" context and delay process exit (and thus pool import).
9697  */
9698 void
9699 l2arc_spa_rebuild_start(spa_t *spa)
9700 {
9701 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
9702 
9703 	/*
9704 	 * Locate the spa's l2arc devices and kick off rebuild threads.
9705 	 */
9706 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
9707 		l2arc_dev_t *dev =
9708 		    l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
9709 		if (dev == NULL) {
9710 			/* Don't attempt a rebuild if the vdev is UNAVAIL */
9711 			continue;
9712 		}
9713 		mutex_enter(&l2arc_rebuild_thr_lock);
9714 		if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
9715 			dev->l2ad_rebuild_began = B_TRUE;
9716 			(void) thread_create(NULL, 0, l2arc_dev_rebuild_thread,
9717 			    dev, 0, &p0, TS_RUN, minclsyspri);
9718 		}
9719 		mutex_exit(&l2arc_rebuild_thr_lock);
9720 	}
9721 }
9722 
9723 /*
9724  * Main entry point for L2ARC rebuilding.
9725  */
9726 static __attribute__((noreturn)) void
9727 l2arc_dev_rebuild_thread(void *arg)
9728 {
9729 	l2arc_dev_t *dev = arg;
9730 
9731 	VERIFY(!dev->l2ad_rebuild_cancel);
9732 	VERIFY(dev->l2ad_rebuild);
9733 	(void) l2arc_rebuild(dev);
9734 	mutex_enter(&l2arc_rebuild_thr_lock);
9735 	dev->l2ad_rebuild_began = B_FALSE;
9736 	dev->l2ad_rebuild = B_FALSE;
9737 	mutex_exit(&l2arc_rebuild_thr_lock);
9738 
9739 	thread_exit();
9740 }
9741 
9742 /*
9743  * This function implements the actual L2ARC metadata rebuild. It:
9744  * starts reading the log block chain and restores each block's contents
9745  * to memory (reconstructing arc_buf_hdr_t's).
9746  *
9747  * Operation stops under any of the following conditions:
9748  *
9749  * 1) We reach the end of the log block chain.
9750  * 2) We encounter *any* error condition (cksum errors, io errors)
9751  */
9752 static int
9753 l2arc_rebuild(l2arc_dev_t *dev)
9754 {
9755 	vdev_t			*vd = dev->l2ad_vdev;
9756 	spa_t			*spa = vd->vdev_spa;
9757 	int			err = 0;
9758 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
9759 	l2arc_log_blk_phys_t	*this_lb, *next_lb;
9760 	zio_t			*this_io = NULL, *next_io = NULL;
9761 	l2arc_log_blkptr_t	lbps[2];
9762 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
9763 	boolean_t		lock_held;
9764 
9765 	this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP);
9766 	next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP);
9767 
9768 	/*
9769 	 * We prevent device removal while issuing reads to the device,
9770 	 * then during the rebuilding phases we drop this lock again so
9771 	 * that a spa_unload or device remove can be initiated - this is
9772 	 * safe, because the spa will signal us to stop before removing
9773 	 * our device and wait for us to stop.
9774 	 */
9775 	spa_config_enter(spa, SCL_L2ARC, vd, RW_READER);
9776 	lock_held = B_TRUE;
9777 
9778 	/*
9779 	 * Retrieve the persistent L2ARC device state.
9780 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
9781 	 */
9782 	dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
9783 	dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
9784 	    L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
9785 	    dev->l2ad_start);
9786 	dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);
9787 
9788 	vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
9789 	vd->vdev_trim_state = l2dhdr->dh_trim_state;
9790 
9791 	/*
9792 	 * In case the zfs module parameter l2arc_rebuild_enabled is false
9793 	 * we do not start the rebuild process.
9794 	 */
9795 	if (!l2arc_rebuild_enabled)
9796 		goto out;
9797 
9798 	/* Prepare the rebuild process */
9799 	memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));
9800 
9801 	/* Start the rebuild process */
9802 	for (;;) {
9803 		if (!l2arc_log_blkptr_valid(dev, &lbps[0]))
9804 			break;
9805 
9806 		if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1],
9807 		    this_lb, next_lb, this_io, &next_io)) != 0)
9808 			goto out;
9809 
9810 		/*
9811 		 * Our memory pressure valve. If the system is running low
9812 		 * on memory, rather than swamping memory with new ARC buf
9813 		 * hdrs, we opt not to rebuild the L2ARC. At this point,
9814 		 * however, we have already set up our L2ARC dev to chain in
9815 		 * new metadata log blocks, so the user may choose to offline/
9816 		 * online the L2ARC dev at a later time (or re-import the pool)
9817 		 * to reconstruct it (when there's less memory pressure).
9818 		 */
9819 		if (l2arc_hdr_limit_reached()) {
9820 			ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem);
9821 			cmn_err(CE_NOTE, "System running low on memory, "
9822 			    "aborting L2ARC rebuild.");
9823 			err = SET_ERROR(ENOMEM);
9824 			goto out;
9825 		}
9826 
9827 		spa_config_exit(spa, SCL_L2ARC, vd);
9828 		lock_held = B_FALSE;
9829 
9830 		/*
9831 		 * Now that we know that the next_lb checks out alright, we
9832 		 * can start reconstruction from this log block.
9833 		 * L2BLK_GET_PSIZE returns aligned size for log blocks.
9834 		 */
9835 		uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
9836 		l2arc_log_blk_restore(dev, this_lb, asize);
9837 
9838 		/*
9839 		 * log block restored, include its pointer in the list of
9840 		 * pointers to log blocks present in the L2ARC device.
9841 		 */
9842 		lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
9843 		lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
9844 		    KM_SLEEP);
9845 		memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
9846 		    sizeof (l2arc_log_blkptr_t));
9847 		mutex_enter(&dev->l2ad_mtx);
9848 		list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
9849 		ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
9850 		ARCSTAT_BUMP(arcstat_l2_log_blk_count);
9851 		zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
9852 		zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
9853 		mutex_exit(&dev->l2ad_mtx);
9854 		vdev_space_update(vd, asize, 0, 0);
9855 
9856 		/*
9857 		 * Protection against loops of log blocks:
9858 		 *
9859 		 *				       l2ad_hand  l2ad_evict
9860 		 *                                         V	      V
9861 		 * l2ad_start |=======================================| l2ad_end
9862 		 *             -----|||----|||---|||----|||
9863 		 *                  (3)    (2)   (1)    (0)
9864 		 *             ---|||---|||----|||---|||
9865 		 *		  (7)   (6)    (5)   (4)
9866 		 *
9867 		 * In this situation the pointer of log block (4) passes
9868 		 * l2arc_log_blkptr_valid() but the log block should not be
9869 		 * restored as it is overwritten by the payload of log block
9870 		 * (0). Only log blocks (0)-(3) should be restored. We check
9871 		 * whether l2ad_evict lies in between the payload starting
9872 		 * offset of the next log block (lbps[1].lbp_payload_start)
9873 		 * and the payload starting offset of the present log block
9874 		 * (lbps[0].lbp_payload_start). If true and this isn't the
9875 		 * first pass, we are looping from the beginning and we should
9876 		 * stop.
9877 		 */
9878 		if (l2arc_range_check_overlap(lbps[1].lbp_payload_start,
9879 		    lbps[0].lbp_payload_start, dev->l2ad_evict) &&
9880 		    !dev->l2ad_first)
9881 			goto out;
9882 
9883 		kpreempt(KPREEMPT_SYNC);
9884 		for (;;) {
9885 			mutex_enter(&l2arc_rebuild_thr_lock);
9886 			if (dev->l2ad_rebuild_cancel) {
9887 				dev->l2ad_rebuild = B_FALSE;
9888 				cv_signal(&l2arc_rebuild_thr_cv);
9889 				mutex_exit(&l2arc_rebuild_thr_lock);
9890 				err = SET_ERROR(ECANCELED);
9891 				goto out;
9892 			}
9893 			mutex_exit(&l2arc_rebuild_thr_lock);
9894 			if (spa_config_tryenter(spa, SCL_L2ARC, vd,
9895 			    RW_READER)) {
9896 				lock_held = B_TRUE;
9897 				break;
9898 			}
9899 			/*
9900 			 * L2ARC config lock held by somebody in writer,
9901 			 * possibly due to them trying to remove us. They'll
9902 			 * likely to want us to shut down, so after a little
9903 			 * delay, we check l2ad_rebuild_cancel and retry
9904 			 * the lock again.
9905 			 */
9906 			delay(1);
9907 		}
9908 
9909 		/*
9910 		 * Continue with the next log block.
9911 		 */
9912 		lbps[0] = lbps[1];
9913 		lbps[1] = this_lb->lb_prev_lbp;
9914 		PTR_SWAP(this_lb, next_lb);
9915 		this_io = next_io;
9916 		next_io = NULL;
9917 	}
9918 
9919 	if (this_io != NULL)
9920 		l2arc_log_blk_fetch_abort(this_io);
9921 out:
9922 	if (next_io != NULL)
9923 		l2arc_log_blk_fetch_abort(next_io);
9924 	vmem_free(this_lb, sizeof (*this_lb));
9925 	vmem_free(next_lb, sizeof (*next_lb));
9926 
9927 	if (!l2arc_rebuild_enabled) {
9928 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
9929 		    "disabled");
9930 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
9931 		ARCSTAT_BUMP(arcstat_l2_rebuild_success);
9932 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
9933 		    "successful, restored %llu blocks",
9934 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
9935 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
9936 		/*
9937 		 * No error but also nothing restored, meaning the lbps array
9938 		 * in the device header points to invalid/non-present log
9939 		 * blocks. Reset the header.
9940 		 */
9941 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
9942 		    "no valid log blocks");
9943 		memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
9944 		l2arc_dev_hdr_update(dev);
9945 	} else if (err == ECANCELED) {
9946 		/*
9947 		 * In case the rebuild was canceled do not log to spa history
9948 		 * log as the pool may be in the process of being removed.
9949 		 */
9950 		zfs_dbgmsg("L2ARC rebuild aborted, restored %llu blocks",
9951 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
9952 	} else if (err != 0) {
9953 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
9954 		    "aborted, restored %llu blocks",
9955 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
9956 	}
9957 
9958 	if (lock_held)
9959 		spa_config_exit(spa, SCL_L2ARC, vd);
9960 
9961 	return (err);
9962 }
9963 
9964 /*
9965  * Attempts to read the device header on the provided L2ARC device and writes
9966  * it to `hdr'. On success, this function returns 0, otherwise the appropriate
9967  * error code is returned.
9968  */
9969 static int
9970 l2arc_dev_hdr_read(l2arc_dev_t *dev)
9971 {
9972 	int			err;
9973 	uint64_t		guid;
9974 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
9975 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
9976 	abd_t 			*abd;
9977 
9978 	guid = spa_guid(dev->l2ad_vdev->vdev_spa);
9979 
9980 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
9981 
9982 	err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
9983 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd,
9984 	    ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
9985 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY |
9986 	    ZIO_FLAG_SPECULATIVE, B_FALSE));
9987 
9988 	abd_free(abd);
9989 
9990 	if (err != 0) {
9991 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors);
9992 		zfs_dbgmsg("L2ARC IO error (%d) while reading device header, "
9993 		    "vdev guid: %llu", err,
9994 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
9995 		return (err);
9996 	}
9997 
9998 	if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
9999 		byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr));
10000 
10001 	if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC ||
10002 	    l2dhdr->dh_spa_guid != guid ||
10003 	    l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid ||
10004 	    l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION ||
10005 	    l2dhdr->dh_log_entries != dev->l2ad_log_entries ||
10006 	    l2dhdr->dh_end != dev->l2ad_end ||
10007 	    !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
10008 	    l2dhdr->dh_evict) ||
10009 	    (l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
10010 	    l2arc_trim_ahead > 0)) {
10011 		/*
10012 		 * Attempt to rebuild a device containing no actual dev hdr
10013 		 * or containing a header from some other pool or from another
10014 		 * version of persistent L2ARC.
10015 		 */
10016 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported);
10017 		return (SET_ERROR(ENOTSUP));
10018 	}
10019 
10020 	return (0);
10021 }
10022 
10023 /*
10024  * Reads L2ARC log blocks from storage and validates their contents.
10025  *
10026  * This function implements a simple fetcher to make sure that while
10027  * we're processing one buffer the L2ARC is already fetching the next
10028  * one in the chain.
10029  *
10030  * The arguments this_lp and next_lp point to the current and next log block
10031  * address in the block chain. Similarly, this_lb and next_lb hold the
10032  * l2arc_log_blk_phys_t's of the current and next L2ARC blk.
10033  *
10034  * The `this_io' and `next_io' arguments are used for block fetching.
10035  * When issuing the first blk IO during rebuild, you should pass NULL for
10036  * `this_io'. This function will then issue a sync IO to read the block and
10037  * also issue an async IO to fetch the next block in the block chain. The
10038  * fetched IO is returned in `next_io'. On subsequent calls to this
10039  * function, pass the value returned in `next_io' from the previous call
10040  * as `this_io' and a fresh `next_io' pointer to hold the next fetch IO.
10041  * Prior to the call, you should initialize your `next_io' pointer to be
10042  * NULL. If no fetch IO was issued, the pointer is left set at NULL.
10043  *
10044  * On success, this function returns 0, otherwise it returns an appropriate
10045  * error code. On error the fetching IO is aborted and cleared before
10046  * returning from this function. Therefore, if we return `success', the
10047  * caller can assume that we have taken care of cleanup of fetch IOs.
10048  */
10049 static int
10050 l2arc_log_blk_read(l2arc_dev_t *dev,
10051     const l2arc_log_blkptr_t *this_lbp, const l2arc_log_blkptr_t *next_lbp,
10052     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
10053     zio_t *this_io, zio_t **next_io)
10054 {
10055 	int		err = 0;
10056 	zio_cksum_t	cksum;
10057 	abd_t		*abd = NULL;
10058 	uint64_t	asize;
10059 
10060 	ASSERT(this_lbp != NULL && next_lbp != NULL);
10061 	ASSERT(this_lb != NULL && next_lb != NULL);
10062 	ASSERT(next_io != NULL && *next_io == NULL);
10063 	ASSERT(l2arc_log_blkptr_valid(dev, this_lbp));
10064 
10065 	/*
10066 	 * Check to see if we have issued the IO for this log block in a
10067 	 * previous run. If not, this is the first call, so issue it now.
10068 	 */
10069 	if (this_io == NULL) {
10070 		this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
10071 		    this_lb);
10072 	}
10073 
10074 	/*
10075 	 * Peek to see if we can start issuing the next IO immediately.
10076 	 */
10077 	if (l2arc_log_blkptr_valid(dev, next_lbp)) {
10078 		/*
10079 		 * Start issuing IO for the next log block early - this
10080 		 * should help keep the L2ARC device busy while we
10081 		 * decompress and restore this log block.
10082 		 */
10083 		*next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
10084 		    next_lb);
10085 	}
10086 
10087 	/* Wait for the IO to read this log block to complete */
10088 	if ((err = zio_wait(this_io)) != 0) {
10089 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors);
10090 		zfs_dbgmsg("L2ARC IO error (%d) while reading log block, "
10091 		    "offset: %llu, vdev guid: %llu", err,
10092 		    (u_longlong_t)this_lbp->lbp_daddr,
10093 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10094 		goto cleanup;
10095 	}
10096 
10097 	/*
10098 	 * Make sure the buffer checks out.
10099 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10100 	 */
10101 	asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
10102 	fletcher_4_native(this_lb, asize, NULL, &cksum);
10103 	if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
10104 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors);
10105 		zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, "
10106 		    "vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu",
10107 		    (u_longlong_t)this_lbp->lbp_daddr,
10108 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid,
10109 		    (u_longlong_t)dev->l2ad_hand,
10110 		    (u_longlong_t)dev->l2ad_evict);
10111 		err = SET_ERROR(ECKSUM);
10112 		goto cleanup;
10113 	}
10114 
10115 	/* Now we can take our time decoding this buffer */
10116 	switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
10117 	case ZIO_COMPRESS_OFF:
10118 		break;
10119 	case ZIO_COMPRESS_LZ4:
10120 		abd = abd_alloc_for_io(asize, B_TRUE);
10121 		abd_copy_from_buf_off(abd, this_lb, 0, asize);
10122 		if ((err = zio_decompress_data(
10123 		    L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
10124 		    abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
10125 			err = SET_ERROR(EINVAL);
10126 			goto cleanup;
10127 		}
10128 		break;
10129 	default:
10130 		err = SET_ERROR(EINVAL);
10131 		goto cleanup;
10132 	}
10133 	if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
10134 		byteswap_uint64_array(this_lb, sizeof (*this_lb));
10135 	if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
10136 		err = SET_ERROR(EINVAL);
10137 		goto cleanup;
10138 	}
10139 cleanup:
10140 	/* Abort an in-flight fetch I/O in case of error */
10141 	if (err != 0 && *next_io != NULL) {
10142 		l2arc_log_blk_fetch_abort(*next_io);
10143 		*next_io = NULL;
10144 	}
10145 	if (abd != NULL)
10146 		abd_free(abd);
10147 	return (err);
10148 }
10149 
10150 /*
10151  * Restores the payload of a log block to ARC. This creates empty ARC hdr
10152  * entries which only contain an l2arc hdr, essentially restoring the
10153  * buffers to their L2ARC evicted state. This function also updates space
10154  * usage on the L2ARC vdev to make sure it tracks restored buffers.
10155  */
10156 static void
10157 l2arc_log_blk_restore(l2arc_dev_t *dev, const l2arc_log_blk_phys_t *lb,
10158     uint64_t lb_asize)
10159 {
10160 	uint64_t	size = 0, asize = 0;
10161 	uint64_t	log_entries = dev->l2ad_log_entries;
10162 
10163 	/*
10164 	 * Usually arc_adapt() is called only for data, not headers, but
10165 	 * since we may allocate significant amount of memory here, let ARC
10166 	 * grow its arc_c.
10167 	 */
10168 	arc_adapt(log_entries * HDR_L2ONLY_SIZE);
10169 
10170 	for (int i = log_entries - 1; i >= 0; i--) {
10171 		/*
10172 		 * Restore goes in the reverse temporal direction to preserve
10173 		 * correct temporal ordering of buffers in the l2ad_buflist.
10174 		 * l2arc_hdr_restore also does a list_insert_tail instead of
10175 		 * list_insert_head on the l2ad_buflist:
10176 		 *
10177 		 *		LIST	l2ad_buflist		LIST
10178 		 *		HEAD  <------ (time) ------	TAIL
10179 		 * direction	+-----+-----+-----+-----+-----+    direction
10180 		 * of l2arc <== | buf | buf | buf | buf | buf | ===> of rebuild
10181 		 * fill		+-----+-----+-----+-----+-----+
10182 		 *		^				^
10183 		 *		|				|
10184 		 *		|				|
10185 		 *	l2arc_feed_thread		l2arc_rebuild
10186 		 *	will place new bufs here	restores bufs here
10187 		 *
10188 		 * During l2arc_rebuild() the device is not used by
10189 		 * l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
10190 		 */
10191 		size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
10192 		asize += vdev_psize_to_asize(dev->l2ad_vdev,
10193 		    L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
10194 		l2arc_hdr_restore(&lb->lb_entries[i], dev);
10195 	}
10196 
10197 	/*
10198 	 * Record rebuild stats:
10199 	 *	size		Logical size of restored buffers in the L2ARC
10200 	 *	asize		Aligned size of restored buffers in the L2ARC
10201 	 */
10202 	ARCSTAT_INCR(arcstat_l2_rebuild_size, size);
10203 	ARCSTAT_INCR(arcstat_l2_rebuild_asize, asize);
10204 	ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries);
10205 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, lb_asize);
10206 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, asize / lb_asize);
10207 	ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks);
10208 }
10209 
10210 /*
10211  * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
10212  * into a state indicating that it has been evicted to L2ARC.
10213  */
10214 static void
10215 l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, l2arc_dev_t *dev)
10216 {
10217 	arc_buf_hdr_t		*hdr, *exists;
10218 	kmutex_t		*hash_lock;
10219 	arc_buf_contents_t	type = L2BLK_GET_TYPE((le)->le_prop);
10220 	uint64_t		asize;
10221 
10222 	/*
10223 	 * Do all the allocation before grabbing any locks, this lets us
10224 	 * sleep if memory is full and we don't have to deal with failed
10225 	 * allocations.
10226 	 */
10227 	hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
10228 	    dev, le->le_dva, le->le_daddr,
10229 	    L2BLK_GET_PSIZE((le)->le_prop), le->le_birth,
10230 	    L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
10231 	    L2BLK_GET_PROTECTED((le)->le_prop),
10232 	    L2BLK_GET_PREFETCH((le)->le_prop),
10233 	    L2BLK_GET_STATE((le)->le_prop));
10234 	asize = vdev_psize_to_asize(dev->l2ad_vdev,
10235 	    L2BLK_GET_PSIZE((le)->le_prop));
10236 
10237 	/*
10238 	 * vdev_space_update() has to be called before arc_hdr_destroy() to
10239 	 * avoid underflow since the latter also calls vdev_space_update().
10240 	 */
10241 	l2arc_hdr_arcstats_increment(hdr);
10242 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10243 
10244 	mutex_enter(&dev->l2ad_mtx);
10245 	list_insert_tail(&dev->l2ad_buflist, hdr);
10246 	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
10247 	mutex_exit(&dev->l2ad_mtx);
10248 
10249 	exists = buf_hash_insert(hdr, &hash_lock);
10250 	if (exists) {
10251 		/* Buffer was already cached, no need to restore it. */
10252 		arc_hdr_destroy(hdr);
10253 		/*
10254 		 * If the buffer is already cached, check whether it has
10255 		 * L2ARC metadata. If not, enter them and update the flag.
10256 		 * This is important is case of onlining a cache device, since
10257 		 * we previously evicted all L2ARC metadata from ARC.
10258 		 */
10259 		if (!HDR_HAS_L2HDR(exists)) {
10260 			arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR);
10261 			exists->b_l2hdr.b_dev = dev;
10262 			exists->b_l2hdr.b_daddr = le->le_daddr;
10263 			exists->b_l2hdr.b_arcs_state =
10264 			    L2BLK_GET_STATE((le)->le_prop);
10265 			mutex_enter(&dev->l2ad_mtx);
10266 			list_insert_tail(&dev->l2ad_buflist, exists);
10267 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
10268 			    arc_hdr_size(exists), exists);
10269 			mutex_exit(&dev->l2ad_mtx);
10270 			l2arc_hdr_arcstats_increment(exists);
10271 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10272 		}
10273 		ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached);
10274 	}
10275 
10276 	mutex_exit(hash_lock);
10277 }
10278 
10279 /*
10280  * Starts an asynchronous read IO to read a log block. This is used in log
10281  * block reconstruction to start reading the next block before we are done
10282  * decoding and reconstructing the current block, to keep the l2arc device
10283  * nice and hot with read IO to process.
10284  * The returned zio will contain a newly allocated memory buffers for the IO
10285  * data which should then be freed by the caller once the zio is no longer
10286  * needed (i.e. due to it having completed). If you wish to abort this
10287  * zio, you should do so using l2arc_log_blk_fetch_abort, which takes
10288  * care of disposing of the allocated buffers correctly.
10289  */
10290 static zio_t *
10291 l2arc_log_blk_fetch(vdev_t *vd, const l2arc_log_blkptr_t *lbp,
10292     l2arc_log_blk_phys_t *lb)
10293 {
10294 	uint32_t		asize;
10295 	zio_t			*pio;
10296 	l2arc_read_callback_t	*cb;
10297 
10298 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
10299 	asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10300 	ASSERT(asize <= sizeof (l2arc_log_blk_phys_t));
10301 
10302 	cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP);
10303 	cb->l2rcb_abd = abd_get_from_buf(lb, asize);
10304 	pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
10305 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY);
10306 	(void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
10307 	    cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10308 	    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL |
10309 	    ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE));
10310 
10311 	return (pio);
10312 }
10313 
10314 /*
10315  * Aborts a zio returned from l2arc_log_blk_fetch and frees the data
10316  * buffers allocated for it.
10317  */
10318 static void
10319 l2arc_log_blk_fetch_abort(zio_t *zio)
10320 {
10321 	(void) zio_wait(zio);
10322 }
10323 
10324 /*
10325  * Creates a zio to update the device header on an l2arc device.
10326  */
10327 void
10328 l2arc_dev_hdr_update(l2arc_dev_t *dev)
10329 {
10330 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10331 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10332 	abd_t			*abd;
10333 	int			err;
10334 
10335 	VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));
10336 
10337 	l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
10338 	l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
10339 	l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10340 	l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
10341 	l2dhdr->dh_log_entries = dev->l2ad_log_entries;
10342 	l2dhdr->dh_evict = dev->l2ad_evict;
10343 	l2dhdr->dh_start = dev->l2ad_start;
10344 	l2dhdr->dh_end = dev->l2ad_end;
10345 	l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
10346 	l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
10347 	l2dhdr->dh_flags = 0;
10348 	l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
10349 	l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
10350 	if (dev->l2ad_first)
10351 		l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST;
10352 
10353 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
10354 
10355 	err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
10356 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL,
10357 	    NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE));
10358 
10359 	abd_free(abd);
10360 
10361 	if (err != 0) {
10362 		zfs_dbgmsg("L2ARC IO error (%d) while writing device header, "
10363 		    "vdev guid: %llu", err,
10364 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10365 	}
10366 }
10367 
10368 /*
10369  * Commits a log block to the L2ARC device. This routine is invoked from
10370  * l2arc_write_buffers when the log block fills up.
10371  * This function allocates some memory to temporarily hold the serialized
10372  * buffer to be written. This is then released in l2arc_write_done.
10373  */
10374 static uint64_t
10375 l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
10376 {
10377 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
10378 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10379 	uint64_t		psize, asize;
10380 	zio_t			*wzio;
10381 	l2arc_lb_abd_buf_t	*abd_buf;
10382 	uint8_t			*tmpbuf = NULL;
10383 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
10384 
10385 	VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
10386 
10387 	abd_buf = zio_buf_alloc(sizeof (*abd_buf));
10388 	abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
10389 	lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
10390 	lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);
10391 
10392 	/* link the buffer into the block chain */
10393 	lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
10394 	lb->lb_magic = L2ARC_LOG_BLK_MAGIC;
10395 
10396 	/*
10397 	 * l2arc_log_blk_commit() may be called multiple times during a single
10398 	 * l2arc_write_buffers() call. Save the allocated abd buffers in a list
10399 	 * so we can free them in l2arc_write_done() later on.
10400 	 */
10401 	list_insert_tail(&cb->l2wcb_abd_list, abd_buf);
10402 
10403 	/* try to compress the buffer */
10404 	psize = zio_compress_data(ZIO_COMPRESS_LZ4,
10405 	    abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0);
10406 
10407 	/* a log block is never entirely zero */
10408 	ASSERT(psize != 0);
10409 	asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
10410 	ASSERT(asize <= sizeof (*lb));
10411 
10412 	/*
10413 	 * Update the start log block pointer in the device header to point
10414 	 * to the log block we're about to write.
10415 	 */
10416 	l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
10417 	l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
10418 	l2dhdr->dh_start_lbps[0].lbp_payload_asize =
10419 	    dev->l2ad_log_blk_payload_asize;
10420 	l2dhdr->dh_start_lbps[0].lbp_payload_start =
10421 	    dev->l2ad_log_blk_payload_start;
10422 	L2BLK_SET_LSIZE(
10423 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
10424 	L2BLK_SET_PSIZE(
10425 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
10426 	L2BLK_SET_CHECKSUM(
10427 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10428 	    ZIO_CHECKSUM_FLETCHER_4);
10429 	if (asize < sizeof (*lb)) {
10430 		/* compression succeeded */
10431 		memset(tmpbuf + psize, 0, asize - psize);
10432 		L2BLK_SET_COMPRESS(
10433 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10434 		    ZIO_COMPRESS_LZ4);
10435 	} else {
10436 		/* compression failed */
10437 		memcpy(tmpbuf, lb, sizeof (*lb));
10438 		L2BLK_SET_COMPRESS(
10439 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10440 		    ZIO_COMPRESS_OFF);
10441 	}
10442 
10443 	/* checksum what we're about to write */
10444 	fletcher_4_native(tmpbuf, asize, NULL,
10445 	    &l2dhdr->dh_start_lbps[0].lbp_cksum);
10446 
10447 	abd_free(abd_buf->abd);
10448 
10449 	/* perform the write itself */
10450 	abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb));
10451 	abd_take_ownership_of_buf(abd_buf->abd, B_TRUE);
10452 	wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
10453 	    asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10454 	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
10455 	DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio);
10456 	(void) zio_nowait(wzio);
10457 
10458 	dev->l2ad_hand += asize;
10459 	/*
10460 	 * Include the committed log block's pointer  in the list of pointers
10461 	 * to log blocks present in the L2ARC device.
10462 	 */
10463 	memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
10464 	    sizeof (l2arc_log_blkptr_t));
10465 	mutex_enter(&dev->l2ad_mtx);
10466 	list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
10467 	ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
10468 	ARCSTAT_BUMP(arcstat_l2_log_blk_count);
10469 	zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10470 	zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10471 	mutex_exit(&dev->l2ad_mtx);
10472 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10473 
10474 	/* bump the kstats */
10475 	ARCSTAT_INCR(arcstat_l2_write_bytes, asize);
10476 	ARCSTAT_BUMP(arcstat_l2_log_blk_writes);
10477 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, asize);
10478 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio,
10479 	    dev->l2ad_log_blk_payload_asize / asize);
10480 
10481 	/* start a new log block */
10482 	dev->l2ad_log_ent_idx = 0;
10483 	dev->l2ad_log_blk_payload_asize = 0;
10484 	dev->l2ad_log_blk_payload_start = 0;
10485 
10486 	return (asize);
10487 }
10488 
10489 /*
10490  * Validates an L2ARC log block address to make sure that it can be read
10491  * from the provided L2ARC device.
10492  */
10493 boolean_t
10494 l2arc_log_blkptr_valid(l2arc_dev_t *dev, const l2arc_log_blkptr_t *lbp)
10495 {
10496 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
10497 	uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10498 	uint64_t end = lbp->lbp_daddr + asize - 1;
10499 	uint64_t start = lbp->lbp_payload_start;
10500 	boolean_t evicted = B_FALSE;
10501 
10502 	/*
10503 	 * A log block is valid if all of the following conditions are true:
10504 	 * - it fits entirely (including its payload) between l2ad_start and
10505 	 *   l2ad_end
10506 	 * - it has a valid size
10507 	 * - neither the log block itself nor part of its payload was evicted
10508 	 *   by l2arc_evict():
10509 	 *
10510 	 *		l2ad_hand          l2ad_evict
10511 	 *		|			 |	lbp_daddr
10512 	 *		|     start		 |	|  end
10513 	 *		|     |			 |	|  |
10514 	 *		V     V		         V	V  V
10515 	 *   l2ad_start ============================================ l2ad_end
10516 	 *                    --------------------------||||
10517 	 *				^		 ^
10518 	 *				|		log block
10519 	 *				payload
10520 	 */
10521 
10522 	evicted =
10523 	    l2arc_range_check_overlap(start, end, dev->l2ad_hand) ||
10524 	    l2arc_range_check_overlap(start, end, dev->l2ad_evict) ||
10525 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) ||
10526 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);
10527 
10528 	return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
10529 	    asize > 0 && asize <= sizeof (l2arc_log_blk_phys_t) &&
10530 	    (!evicted || dev->l2ad_first));
10531 }
10532 
10533 /*
10534  * Inserts ARC buffer header `hdr' into the current L2ARC log block on
10535  * the device. The buffer being inserted must be present in L2ARC.
10536  * Returns B_TRUE if the L2ARC log block is full and needs to be committed
10537  * to L2ARC, or B_FALSE if it still has room for more ARC buffers.
10538  */
10539 static boolean_t
10540 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)
10541 {
10542 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
10543 	l2arc_log_ent_phys_t	*le;
10544 
10545 	if (dev->l2ad_log_entries == 0)
10546 		return (B_FALSE);
10547 
10548 	int index = dev->l2ad_log_ent_idx++;
10549 
10550 	ASSERT3S(index, <, dev->l2ad_log_entries);
10551 	ASSERT(HDR_HAS_L2HDR(hdr));
10552 
10553 	le = &lb->lb_entries[index];
10554 	memset(le, 0, sizeof (*le));
10555 	le->le_dva = hdr->b_dva;
10556 	le->le_birth = hdr->b_birth;
10557 	le->le_daddr = hdr->b_l2hdr.b_daddr;
10558 	if (index == 0)
10559 		dev->l2ad_log_blk_payload_start = le->le_daddr;
10560 	L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
10561 	L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
10562 	L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
10563 	le->le_complevel = hdr->b_complevel;
10564 	L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
10565 	L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
10566 	L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
10567 	L2BLK_SET_STATE((le)->le_prop, hdr->b_l1hdr.b_state->arcs_state);
10568 
10569 	dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
10570 	    HDR_GET_PSIZE(hdr));
10571 
10572 	return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
10573 }
10574 
10575 /*
10576  * Checks whether a given L2ARC device address sits in a time-sequential
10577  * range. The trick here is that the L2ARC is a rotary buffer, so we can't
10578  * just do a range comparison, we need to handle the situation in which the
10579  * range wraps around the end of the L2ARC device. Arguments:
10580  *	bottom -- Lower end of the range to check (written to earlier).
10581  *	top    -- Upper end of the range to check (written to later).
10582  *	check  -- The address for which we want to determine if it sits in
10583  *		  between the top and bottom.
10584  *
10585  * The 3-way conditional below represents the following cases:
10586  *
10587  *	bottom < top : Sequentially ordered case:
10588  *	  <check>--------+-------------------+
10589  *	                 |  (overlap here?)  |
10590  *	 L2ARC dev       V                   V
10591  *	 |---------------<bottom>============<top>--------------|
10592  *
10593  *	bottom > top: Looped-around case:
10594  *	                      <check>--------+------------------+
10595  *	                                     |  (overlap here?) |
10596  *	 L2ARC dev                           V                  V
10597  *	 |===============<top>---------------<bottom>===========|
10598  *	 ^               ^
10599  *	 |  (or here?)   |
10600  *	 +---------------+---------<check>
10601  *
10602  *	top == bottom : Just a single address comparison.
10603  */
10604 boolean_t
10605 l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check)
10606 {
10607 	if (bottom < top)
10608 		return (bottom <= check && check <= top);
10609 	else if (bottom > top)
10610 		return (check <= top || bottom <= check);
10611 	else
10612 		return (check == top);
10613 }
10614 
10615 EXPORT_SYMBOL(arc_buf_size);
10616 EXPORT_SYMBOL(arc_write);
10617 EXPORT_SYMBOL(arc_read);
10618 EXPORT_SYMBOL(arc_buf_info);
10619 EXPORT_SYMBOL(arc_getbuf_func);
10620 EXPORT_SYMBOL(arc_add_prune_callback);
10621 EXPORT_SYMBOL(arc_remove_prune_callback);
10622 
10623 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
10624 	spl_param_get_u64, ZMOD_RW, "Minimum ARC size in bytes");
10625 
10626 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
10627 	spl_param_get_u64, ZMOD_RW, "Maximum ARC size in bytes");
10628 
10629 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, meta_balance, UINT, ZMOD_RW,
10630 	"Balance between metadata and data on ghost hits.");
10631 
10632 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, grow_retry, param_set_arc_int,
10633 	param_get_uint, ZMOD_RW, "Seconds before growing ARC size");
10634 
10635 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, shrink_shift, param_set_arc_int,
10636 	param_get_uint, ZMOD_RW, "log2(fraction of ARC to reclaim)");
10637 
10638 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, pc_percent, UINT, ZMOD_RW,
10639 	"Percent of pagecache to reclaim ARC to");
10640 
10641 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, average_blocksize, UINT, ZMOD_RD,
10642 	"Target average block size");
10643 
10644 ZFS_MODULE_PARAM(zfs, zfs_, compressed_arc_enabled, INT, ZMOD_RW,
10645 	"Disable compressed ARC buffers");
10646 
10647 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prefetch_ms, param_set_arc_int,
10648 	param_get_uint, ZMOD_RW, "Min life of prefetch block in ms");
10649 
10650 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prescient_prefetch_ms,
10651     param_set_arc_int, param_get_uint, ZMOD_RW,
10652 	"Min life of prescient prefetched block in ms");
10653 
10654 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_max, U64, ZMOD_RW,
10655 	"Max write bytes per interval");
10656 
10657 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_boost, U64, ZMOD_RW,
10658 	"Extra write bytes during device warmup");
10659 
10660 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom, U64, ZMOD_RW,
10661 	"Number of max device writes to precache");
10662 
10663 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom_boost, U64, ZMOD_RW,
10664 	"Compressed l2arc_headroom multiplier");
10665 
10666 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, trim_ahead, U64, ZMOD_RW,
10667 	"TRIM ahead L2ARC write size multiplier");
10668 
10669 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_secs, U64, ZMOD_RW,
10670 	"Seconds between L2ARC writing");
10671 
10672 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_min_ms, U64, ZMOD_RW,
10673 	"Min feed interval in milliseconds");
10674 
10675 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, noprefetch, INT, ZMOD_RW,
10676 	"Skip caching prefetched buffers");
10677 
10678 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW,
10679 	"Turbo L2ARC warmup");
10680 
10681 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW,
10682 	"No reads during writes");
10683 
10684 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, UINT, ZMOD_RW,
10685 	"Percent of ARC size allowed for L2ARC-only headers");
10686 
10687 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
10688 	"Rebuild the L2ARC when importing a pool");
10689 
10690 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, U64, ZMOD_RW,
10691 	"Min size in bytes to write rebuild log blocks in L2ARC");
10692 
10693 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, mfuonly, INT, ZMOD_RW,
10694 	"Cache only MFU data from ARC into L2ARC");
10695 
10696 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, exclude_special, INT, ZMOD_RW,
10697 	"Exclude dbufs on special vdevs from being cached to L2ARC if set.");
10698 
10699 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int,
10700 	param_get_uint, ZMOD_RW, "System free memory I/O throttle in bytes");
10701 
10702 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, sys_free, param_set_arc_u64,
10703 	spl_param_get_u64, ZMOD_RW, "System free memory target size in bytes");
10704 
10705 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit, param_set_arc_u64,
10706 	spl_param_get_u64, ZMOD_RW, "Minimum bytes of dnodes in ARC");
10707 
10708 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit_percent,
10709     param_set_arc_int, param_get_uint, ZMOD_RW,
10710 	"Percent of ARC meta buffers for dnodes");
10711 
10712 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, dnode_reduce_percent, UINT, ZMOD_RW,
10713 	"Percentage of excess dnodes to try to unpin");
10714 
10715 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, UINT, ZMOD_RW,
10716 	"When full, ARC allocation waits for eviction of this % of alloc size");
10717 
10718 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, UINT, ZMOD_RW,
10719 	"The number of headers to evict per sublist before moving to the next");
10720 
10721 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
10722 	"Number of arc_prune threads");
10723