xref: /freebsd/sys/contrib/openzfs/module/zfs/arc.c (revision b1c1ee4429fcca8f69873a8be66184e68e1b19d7)
1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3  * CDDL HEADER START
4  *
5  * The contents of this file are subject to the terms of the
6  * Common Development and Distribution License (the "License").
7  * You may not use this file except in compliance with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or https://opensource.org/licenses/CDDL-1.0.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2018, Joyent, Inc.
25  * Copyright (c) 2011, 2020, Delphix. All rights reserved.
26  * Copyright (c) 2014, Saso Kiselkov. All rights reserved.
27  * Copyright (c) 2017, Nexenta Systems, Inc.  All rights reserved.
28  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
29  * Copyright (c) 2020, George Amanakis. All rights reserved.
30  * Copyright (c) 2019, 2024, 2025, Klara, Inc.
31  * Copyright (c) 2019, Allan Jude
32  * Copyright (c) 2020, The FreeBSD Foundation [1]
33  * Copyright (c) 2021, 2024 by George Melikov. All rights reserved.
34  *
35  * [1] Portions of this software were developed by Allan Jude
36  *     under sponsorship from the FreeBSD Foundation.
37  */
38 
39 /*
40  * DVA-based Adjustable Replacement Cache
41  *
42  * While much of the theory of operation used here is
43  * based on the self-tuning, low overhead replacement cache
44  * presented by Megiddo and Modha at FAST 2003, there are some
45  * significant differences:
46  *
47  * 1. The Megiddo and Modha model assumes any page is evictable.
48  * Pages in its cache cannot be "locked" into memory.  This makes
49  * the eviction algorithm simple: evict the last page in the list.
50  * This also make the performance characteristics easy to reason
51  * about.  Our cache is not so simple.  At any given moment, some
52  * subset of the blocks in the cache are un-evictable because we
53  * have handed out a reference to them.  Blocks are only evictable
54  * when there are no external references active.  This makes
55  * eviction far more problematic:  we choose to evict the evictable
56  * blocks that are the "lowest" in the list.
57  *
58  * There are times when it is not possible to evict the requested
59  * space.  In these circumstances we are unable to adjust the cache
60  * size.  To prevent the cache growing unbounded at these times we
61  * implement a "cache throttle" that slows the flow of new data
62  * into the cache until we can make space available.
63  *
64  * 2. The Megiddo and Modha model assumes a fixed cache size.
65  * Pages are evicted when the cache is full and there is a cache
66  * miss.  Our model has a variable sized cache.  It grows with
67  * high use, but also tries to react to memory pressure from the
68  * operating system: decreasing its size when system memory is
69  * tight.
70  *
71  * 3. The Megiddo and Modha model assumes a fixed page size. All
72  * elements of the cache are therefore exactly the same size.  So
73  * when adjusting the cache size following a cache miss, its simply
74  * a matter of choosing a single page to evict.  In our model, we
75  * have variable sized cache blocks (ranging from 512 bytes to
76  * 128K bytes).  We therefore choose a set of blocks to evict to make
77  * space for a cache miss that approximates as closely as possible
78  * the space used by the new block.
79  *
80  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
81  * by N. Megiddo & D. Modha, FAST 2003
82  */
83 
84 /*
85  * The locking model:
86  *
87  * A new reference to a cache buffer can be obtained in two
88  * ways: 1) via a hash table lookup using the DVA as a key,
89  * or 2) via one of the ARC lists.  The arc_read() interface
90  * uses method 1, while the internal ARC algorithms for
91  * adjusting the cache use method 2.  We therefore provide two
92  * types of locks: 1) the hash table lock array, and 2) the
93  * ARC list locks.
94  *
95  * Buffers do not have their own mutexes, rather they rely on the
96  * hash table mutexes for the bulk of their protection (i.e. most
97  * fields in the arc_buf_hdr_t are protected by these mutexes).
98  *
99  * buf_hash_find() returns the appropriate mutex (held) when it
100  * locates the requested buffer in the hash table.  It returns
101  * NULL for the mutex if the buffer was not in the table.
102  *
103  * buf_hash_remove() expects the appropriate hash mutex to be
104  * already held before it is invoked.
105  *
106  * Each ARC state also has a mutex which is used to protect the
107  * buffer list associated with the state.  When attempting to
108  * obtain a hash table lock while holding an ARC list lock you
109  * must use: mutex_tryenter() to avoid deadlock.  Also note that
110  * the active state mutex must be held before the ghost state mutex.
111  *
112  * It as also possible to register a callback which is run when the
113  * metadata limit is reached and no buffers can be safely evicted.  In
114  * this case the arc user should drop a reference on some arc buffers so
115  * they can be reclaimed.  For example, when using the ZPL each dentry
116  * holds a references on a znode.  These dentries must be pruned before
117  * the arc buffer holding the znode can be safely evicted.
118  *
119  * Note that the majority of the performance stats are manipulated
120  * with atomic operations.
121  *
122  * The L2ARC uses the l2ad_mtx on each vdev for the following:
123  *
124  *	- L2ARC buflist creation
125  *	- L2ARC buflist eviction
126  *	- L2ARC write completion, which walks L2ARC buflists
127  *	- ARC header destruction, as it removes from L2ARC buflists
128  *	- ARC header release, as it removes from L2ARC buflists
129  */
130 
131 /*
132  * ARC operation:
133  *
134  * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
135  * This structure can point either to a block that is still in the cache or to
136  * one that is only accessible in an L2 ARC device, or it can provide
137  * information about a block that was recently evicted. If a block is
138  * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
139  * information to retrieve it from the L2ARC device. This information is
140  * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
141  * that is in this state cannot access the data directly.
142  *
143  * Blocks that are actively being referenced or have not been evicted
144  * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
145  * the arc_buf_hdr_t that will point to the data block in memory. A block can
146  * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
147  * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
148  * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
149  *
150  * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
151  * ability to store the physical data (b_pabd) associated with the DVA of the
152  * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
153  * it will match its on-disk compression characteristics. This behavior can be
154  * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
155  * compressed ARC functionality is disabled, the b_pabd will point to an
156  * uncompressed version of the on-disk data.
157  *
158  * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
159  * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
160  * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
161  * consumer. The ARC will provide references to this data and will keep it
162  * cached until it is no longer in use. The ARC caches only the L1ARC's physical
163  * data block and will evict any arc_buf_t that is no longer referenced. The
164  * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
165  * "overhead_size" kstat.
166  *
167  * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
168  * compressed form. The typical case is that consumers will want uncompressed
169  * data, and when that happens a new data buffer is allocated where the data is
170  * decompressed for them to use. Currently the only consumer who wants
171  * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
172  * exists on disk. When this happens, the arc_buf_t's data buffer is shared
173  * with the arc_buf_hdr_t.
174  *
175  * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
176  * first one is owned by a compressed send consumer (and therefore references
177  * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
178  * used by any other consumer (and has its own uncompressed copy of the data
179  * buffer).
180  *
181  *   arc_buf_hdr_t
182  *   +-----------+
183  *   | fields    |
184  *   | common to |
185  *   | L1- and   |
186  *   | L2ARC     |
187  *   +-----------+
188  *   | l2arc_buf_hdr_t
189  *   |           |
190  *   +-----------+
191  *   | l1arc_buf_hdr_t
192  *   |           |              arc_buf_t
193  *   | b_buf     +------------>+-----------+      arc_buf_t
194  *   | b_pabd    +-+           |b_next     +---->+-----------+
195  *   +-----------+ |           |-----------|     |b_next     +-->NULL
196  *                 |           |b_comp = T |     +-----------+
197  *                 |           |b_data     +-+   |b_comp = F |
198  *                 |           +-----------+ |   |b_data     +-+
199  *                 +->+------+               |   +-----------+ |
200  *        compressed  |      |               |                 |
201  *           data     |      |<--------------+                 | uncompressed
202  *                    +------+          compressed,            |     data
203  *                                        shared               +-->+------+
204  *                                         data                    |      |
205  *                                                                 |      |
206  *                                                                 +------+
207  *
208  * When a consumer reads a block, the ARC must first look to see if the
209  * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
210  * arc_buf_t and either copies uncompressed data into a new data buffer from an
211  * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
212  * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
213  * hdr is compressed and the desired compression characteristics of the
214  * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
215  * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
216  * the last buffer in the hdr's b_buf list, however a shared compressed buf can
217  * be anywhere in the hdr's list.
218  *
219  * The diagram below shows an example of an uncompressed ARC hdr that is
220  * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
221  * the last element in the buf list):
222  *
223  *                arc_buf_hdr_t
224  *                +-----------+
225  *                |           |
226  *                |           |
227  *                |           |
228  *                +-----------+
229  * l2arc_buf_hdr_t|           |
230  *                |           |
231  *                +-----------+
232  * l1arc_buf_hdr_t|           |
233  *                |           |                 arc_buf_t    (shared)
234  *                |    b_buf  +------------>+---------+      arc_buf_t
235  *                |           |             |b_next   +---->+---------+
236  *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
237  *                +-----------+ |           |         |     +---------+
238  *                              |           |b_data   +-+   |         |
239  *                              |           +---------+ |   |b_data   +-+
240  *                              +->+------+             |   +---------+ |
241  *                                 |      |             |               |
242  *                   uncompressed  |      |             |               |
243  *                        data     +------+             |               |
244  *                                    ^                 +->+------+     |
245  *                                    |       uncompressed |      |     |
246  *                                    |           data     |      |     |
247  *                                    |                    +------+     |
248  *                                    +---------------------------------+
249  *
250  * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
251  * since the physical block is about to be rewritten. The new data contents
252  * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
253  * it may compress the data before writing it to disk. The ARC will be called
254  * with the transformed data and will memcpy the transformed on-disk block into
255  * a newly allocated b_pabd. Writes are always done into buffers which have
256  * either been loaned (and hence are new and don't have other readers) or
257  * buffers which have been released (and hence have their own hdr, if there
258  * were originally other readers of the buf's original hdr). This ensures that
259  * the ARC only needs to update a single buf and its hdr after a write occurs.
260  *
261  * When the L2ARC is in use, it will also take advantage of the b_pabd. The
262  * L2ARC will always write the contents of b_pabd to the L2ARC. This means
263  * that when compressed ARC is enabled that the L2ARC blocks are identical
264  * to the on-disk block in the main data pool. This provides a significant
265  * advantage since the ARC can leverage the bp's checksum when reading from the
266  * L2ARC to determine if the contents are valid. However, if the compressed
267  * ARC is disabled, then the L2ARC's block must be transformed to look
268  * like the physical block in the main data pool before comparing the
269  * checksum and determining its validity.
270  *
271  * The L1ARC has a slightly different system for storing encrypted data.
272  * Raw (encrypted + possibly compressed) data has a few subtle differences from
273  * data that is just compressed. The biggest difference is that it is not
274  * possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
275  * The other difference is that encryption cannot be treated as a suggestion.
276  * If a caller would prefer compressed data, but they actually wind up with
277  * uncompressed data the worst thing that could happen is there might be a
278  * performance hit. If the caller requests encrypted data, however, we must be
279  * sure they actually get it or else secret information could be leaked. Raw
280  * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
281  * may have both an encrypted version and a decrypted version of its data at
282  * once. When a caller needs a raw arc_buf_t, it is allocated and the data is
283  * copied out of this header. To avoid complications with b_pabd, raw buffers
284  * cannot be shared.
285  */
286 
287 #include <sys/spa.h>
288 #include <sys/zio.h>
289 #include <sys/spa_impl.h>
290 #include <sys/zio_compress.h>
291 #include <sys/zio_checksum.h>
292 #include <sys/zfs_context.h>
293 #include <sys/arc.h>
294 #include <sys/zfs_refcount.h>
295 #include <sys/vdev.h>
296 #include <sys/vdev_impl.h>
297 #include <sys/dsl_pool.h>
298 #include <sys/multilist.h>
299 #include <sys/abd.h>
300 #include <sys/dbuf.h>
301 #include <sys/zil.h>
302 #include <sys/fm/fs/zfs.h>
303 #include <sys/callb.h>
304 #include <sys/kstat.h>
305 #include <sys/zthr.h>
306 #include <zfs_fletcher.h>
307 #include <sys/arc_impl.h>
308 #include <sys/trace_zfs.h>
309 #include <sys/aggsum.h>
310 #include <sys/wmsum.h>
311 #include <cityhash.h>
312 #include <sys/vdev_trim.h>
313 #include <sys/zfs_racct.h>
314 #include <sys/zstd/zstd.h>
315 
316 #ifndef _KERNEL
317 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
318 boolean_t arc_watch = B_FALSE;
319 #endif
320 
321 /*
322  * This thread's job is to keep enough free memory in the system, by
323  * calling arc_kmem_reap_soon() plus arc_reduce_target_size(), which improves
324  * arc_available_memory().
325  */
326 static zthr_t *arc_reap_zthr;
327 
328 /*
329  * This thread's job is to keep arc_size under arc_c, by calling
330  * arc_evict(), which improves arc_is_overflowing().
331  */
332 static zthr_t *arc_evict_zthr;
333 static arc_buf_hdr_t **arc_state_evict_markers;
334 static int arc_state_evict_marker_count;
335 
336 static kmutex_t arc_evict_lock;
337 static boolean_t arc_evict_needed = B_FALSE;
338 static clock_t arc_last_uncached_flush;
339 
340 static taskq_t *arc_evict_taskq;
341 static struct evict_arg *arc_evict_arg;
342 
343 /*
344  * Count of bytes evicted since boot.
345  */
346 static uint64_t arc_evict_count;
347 
348 /*
349  * List of arc_evict_waiter_t's, representing threads waiting for the
350  * arc_evict_count to reach specific values.
351  */
352 static list_t arc_evict_waiters;
353 
354 /*
355  * When arc_is_overflowing(), arc_get_data_impl() waits for this percent of
356  * the requested amount of data to be evicted.  For example, by default for
357  * every 2KB that's evicted, 1KB of it may be "reused" by a new allocation.
358  * Since this is above 100%, it ensures that progress is made towards getting
359  * arc_size under arc_c.  Since this is finite, it ensures that allocations
360  * can still happen, even during the potentially long time that arc_size is
361  * more than arc_c.
362  */
363 static uint_t zfs_arc_eviction_pct = 200;
364 
365 /*
366  * The number of headers to evict in arc_evict_state_impl() before
367  * dropping the sublist lock and evicting from another sublist. A lower
368  * value means we're more likely to evict the "correct" header (i.e. the
369  * oldest header in the arc state), but comes with higher overhead
370  * (i.e. more invocations of arc_evict_state_impl()).
371  */
372 static uint_t zfs_arc_evict_batch_limit = 10;
373 
374 /* number of seconds before growing cache again */
375 uint_t arc_grow_retry = 5;
376 
377 /*
378  * Minimum time between calls to arc_kmem_reap_soon().
379  */
380 static const int arc_kmem_cache_reap_retry_ms = 1000;
381 
382 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */
383 static int zfs_arc_overflow_shift = 8;
384 
385 /* log2(fraction of arc to reclaim) */
386 uint_t arc_shrink_shift = 7;
387 
388 #ifdef _KERNEL
389 /* percent of pagecache to reclaim arc to */
390 uint_t zfs_arc_pc_percent = 0;
391 #endif
392 
393 /*
394  * log2(fraction of ARC which must be free to allow growing).
395  * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
396  * when reading a new block into the ARC, we will evict an equal-sized block
397  * from the ARC.
398  *
399  * This must be less than arc_shrink_shift, so that when we shrink the ARC,
400  * we will still not allow it to grow.
401  */
402 uint_t		arc_no_grow_shift = 5;
403 
404 
405 /*
406  * minimum lifespan of a prefetch block in clock ticks
407  * (initialized in arc_init())
408  */
409 static uint_t		arc_min_prefetch_ms;
410 static uint_t		arc_min_prescient_prefetch_ms;
411 
412 /*
413  * If this percent of memory is free, don't throttle.
414  */
415 uint_t arc_lotsfree_percent = 10;
416 
417 /*
418  * The arc has filled available memory and has now warmed up.
419  */
420 boolean_t arc_warm;
421 
422 /*
423  * These tunables are for performance analysis.
424  */
425 uint64_t zfs_arc_max = 0;
426 uint64_t zfs_arc_min = 0;
427 static uint64_t zfs_arc_dnode_limit = 0;
428 static uint_t zfs_arc_dnode_reduce_percent = 10;
429 static uint_t zfs_arc_grow_retry = 0;
430 static uint_t zfs_arc_shrink_shift = 0;
431 uint_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
432 
433 /*
434  * ARC dirty data constraints for arc_tempreserve_space() throttle:
435  * * total dirty data limit
436  * * anon block dirty limit
437  * * each pool's anon allowance
438  */
439 static const unsigned long zfs_arc_dirty_limit_percent = 50;
440 static const unsigned long zfs_arc_anon_limit_percent = 25;
441 static const unsigned long zfs_arc_pool_dirty_percent = 20;
442 
443 /*
444  * Enable or disable compressed arc buffers.
445  */
446 int zfs_compressed_arc_enabled = B_TRUE;
447 
448 /*
449  * Balance between metadata and data on ghost hits.  Values above 100
450  * increase metadata caching by proportionally reducing effect of ghost
451  * data hits on target data/metadata rate.
452  */
453 static uint_t zfs_arc_meta_balance = 500;
454 
455 /*
456  * Percentage that can be consumed by dnodes of ARC meta buffers.
457  */
458 static uint_t zfs_arc_dnode_limit_percent = 10;
459 
460 /*
461  * These tunables are Linux-specific
462  */
463 static uint64_t zfs_arc_sys_free = 0;
464 static uint_t zfs_arc_min_prefetch_ms = 0;
465 static uint_t zfs_arc_min_prescient_prefetch_ms = 0;
466 static uint_t zfs_arc_lotsfree_percent = 10;
467 
468 /*
469  * Number of arc_prune threads
470  */
471 static int zfs_arc_prune_task_threads = 1;
472 
473 /* Used by spa_export/spa_destroy to flush the arc asynchronously */
474 static taskq_t *arc_flush_taskq;
475 
476 /*
477  * Controls the number of ARC eviction threads to dispatch sublists to.
478  *
479  * Possible values:
480  * 0  (auto) compute the number of threads using a logarithmic formula.
481  * 1  (disabled) one thread - parallel eviction is disabled.
482  * 2+ (manual) set the number manually.
483  *
484  * See arc_evict_thread_init() for how "auto" is computed.
485  */
486 static uint_t zfs_arc_evict_threads = 0;
487 
488 /* The 7 states: */
489 arc_state_t ARC_anon;
490 arc_state_t ARC_mru;
491 arc_state_t ARC_mru_ghost;
492 arc_state_t ARC_mfu;
493 arc_state_t ARC_mfu_ghost;
494 arc_state_t ARC_l2c_only;
495 arc_state_t ARC_uncached;
496 
497 arc_stats_t arc_stats = {
498 	{ "hits",			KSTAT_DATA_UINT64 },
499 	{ "iohits",			KSTAT_DATA_UINT64 },
500 	{ "misses",			KSTAT_DATA_UINT64 },
501 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
502 	{ "demand_data_iohits",		KSTAT_DATA_UINT64 },
503 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
504 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
505 	{ "demand_metadata_iohits",	KSTAT_DATA_UINT64 },
506 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
507 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
508 	{ "prefetch_data_iohits",	KSTAT_DATA_UINT64 },
509 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
510 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
511 	{ "prefetch_metadata_iohits",	KSTAT_DATA_UINT64 },
512 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
513 	{ "mru_hits",			KSTAT_DATA_UINT64 },
514 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
515 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
516 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
517 	{ "uncached_hits",		KSTAT_DATA_UINT64 },
518 	{ "deleted",			KSTAT_DATA_UINT64 },
519 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
520 	{ "access_skip",		KSTAT_DATA_UINT64 },
521 	{ "evict_skip",			KSTAT_DATA_UINT64 },
522 	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
523 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
524 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
525 	{ "evict_l2_eligible_mfu",	KSTAT_DATA_UINT64 },
526 	{ "evict_l2_eligible_mru",	KSTAT_DATA_UINT64 },
527 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
528 	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
529 	{ "hash_elements",		KSTAT_DATA_UINT64 },
530 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
531 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
532 	{ "hash_chains",		KSTAT_DATA_UINT64 },
533 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
534 	{ "meta",			KSTAT_DATA_UINT64 },
535 	{ "pd",				KSTAT_DATA_UINT64 },
536 	{ "pm",				KSTAT_DATA_UINT64 },
537 	{ "c",				KSTAT_DATA_UINT64 },
538 	{ "c_min",			KSTAT_DATA_UINT64 },
539 	{ "c_max",			KSTAT_DATA_UINT64 },
540 	{ "size",			KSTAT_DATA_UINT64 },
541 	{ "compressed_size",		KSTAT_DATA_UINT64 },
542 	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
543 	{ "overhead_size",		KSTAT_DATA_UINT64 },
544 	{ "hdr_size",			KSTAT_DATA_UINT64 },
545 	{ "data_size",			KSTAT_DATA_UINT64 },
546 	{ "metadata_size",		KSTAT_DATA_UINT64 },
547 	{ "dbuf_size",			KSTAT_DATA_UINT64 },
548 	{ "dnode_size",			KSTAT_DATA_UINT64 },
549 	{ "bonus_size",			KSTAT_DATA_UINT64 },
550 #if defined(COMPAT_FREEBSD11)
551 	{ "other_size",			KSTAT_DATA_UINT64 },
552 #endif
553 	{ "anon_size",			KSTAT_DATA_UINT64 },
554 	{ "anon_data",			KSTAT_DATA_UINT64 },
555 	{ "anon_metadata",		KSTAT_DATA_UINT64 },
556 	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
557 	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
558 	{ "mru_size",			KSTAT_DATA_UINT64 },
559 	{ "mru_data",			KSTAT_DATA_UINT64 },
560 	{ "mru_metadata",		KSTAT_DATA_UINT64 },
561 	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
562 	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
563 	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
564 	{ "mru_ghost_data",		KSTAT_DATA_UINT64 },
565 	{ "mru_ghost_metadata",		KSTAT_DATA_UINT64 },
566 	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
567 	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
568 	{ "mfu_size",			KSTAT_DATA_UINT64 },
569 	{ "mfu_data",			KSTAT_DATA_UINT64 },
570 	{ "mfu_metadata",		KSTAT_DATA_UINT64 },
571 	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
572 	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
573 	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
574 	{ "mfu_ghost_data",		KSTAT_DATA_UINT64 },
575 	{ "mfu_ghost_metadata",		KSTAT_DATA_UINT64 },
576 	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
577 	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
578 	{ "uncached_size",		KSTAT_DATA_UINT64 },
579 	{ "uncached_data",		KSTAT_DATA_UINT64 },
580 	{ "uncached_metadata",		KSTAT_DATA_UINT64 },
581 	{ "uncached_evictable_data",	KSTAT_DATA_UINT64 },
582 	{ "uncached_evictable_metadata", KSTAT_DATA_UINT64 },
583 	{ "l2_hits",			KSTAT_DATA_UINT64 },
584 	{ "l2_misses",			KSTAT_DATA_UINT64 },
585 	{ "l2_prefetch_asize",		KSTAT_DATA_UINT64 },
586 	{ "l2_mru_asize",		KSTAT_DATA_UINT64 },
587 	{ "l2_mfu_asize",		KSTAT_DATA_UINT64 },
588 	{ "l2_bufc_data_asize",		KSTAT_DATA_UINT64 },
589 	{ "l2_bufc_metadata_asize",	KSTAT_DATA_UINT64 },
590 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
591 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
592 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
593 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
594 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
595 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
596 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
597 	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
598 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
599 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
600 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
601 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
602 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
603 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
604 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
605 	{ "l2_size",			KSTAT_DATA_UINT64 },
606 	{ "l2_asize",			KSTAT_DATA_UINT64 },
607 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
608 	{ "l2_log_blk_writes",		KSTAT_DATA_UINT64 },
609 	{ "l2_log_blk_avg_asize",	KSTAT_DATA_UINT64 },
610 	{ "l2_log_blk_asize",		KSTAT_DATA_UINT64 },
611 	{ "l2_log_blk_count",		KSTAT_DATA_UINT64 },
612 	{ "l2_data_to_meta_ratio",	KSTAT_DATA_UINT64 },
613 	{ "l2_rebuild_success",		KSTAT_DATA_UINT64 },
614 	{ "l2_rebuild_unsupported",	KSTAT_DATA_UINT64 },
615 	{ "l2_rebuild_io_errors",	KSTAT_DATA_UINT64 },
616 	{ "l2_rebuild_dh_errors",	KSTAT_DATA_UINT64 },
617 	{ "l2_rebuild_cksum_lb_errors",	KSTAT_DATA_UINT64 },
618 	{ "l2_rebuild_lowmem",		KSTAT_DATA_UINT64 },
619 	{ "l2_rebuild_size",		KSTAT_DATA_UINT64 },
620 	{ "l2_rebuild_asize",		KSTAT_DATA_UINT64 },
621 	{ "l2_rebuild_bufs",		KSTAT_DATA_UINT64 },
622 	{ "l2_rebuild_bufs_precached",	KSTAT_DATA_UINT64 },
623 	{ "l2_rebuild_log_blks",	KSTAT_DATA_UINT64 },
624 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
625 	{ "memory_direct_count",	KSTAT_DATA_UINT64 },
626 	{ "memory_indirect_count",	KSTAT_DATA_UINT64 },
627 	{ "memory_all_bytes",		KSTAT_DATA_UINT64 },
628 	{ "memory_free_bytes",		KSTAT_DATA_UINT64 },
629 	{ "memory_available_bytes",	KSTAT_DATA_INT64 },
630 	{ "arc_no_grow",		KSTAT_DATA_UINT64 },
631 	{ "arc_tempreserve",		KSTAT_DATA_UINT64 },
632 	{ "arc_loaned_bytes",		KSTAT_DATA_UINT64 },
633 	{ "arc_prune",			KSTAT_DATA_UINT64 },
634 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
635 	{ "arc_dnode_limit",		KSTAT_DATA_UINT64 },
636 	{ "async_upgrade_sync",		KSTAT_DATA_UINT64 },
637 	{ "predictive_prefetch", KSTAT_DATA_UINT64 },
638 	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
639 	{ "demand_iohit_predictive_prefetch", KSTAT_DATA_UINT64 },
640 	{ "prescient_prefetch", KSTAT_DATA_UINT64 },
641 	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
642 	{ "demand_iohit_prescient_prefetch", KSTAT_DATA_UINT64 },
643 	{ "arc_need_free",		KSTAT_DATA_UINT64 },
644 	{ "arc_sys_free",		KSTAT_DATA_UINT64 },
645 	{ "arc_raw_size",		KSTAT_DATA_UINT64 },
646 	{ "cached_only_in_progress",	KSTAT_DATA_UINT64 },
647 	{ "abd_chunk_waste_size",	KSTAT_DATA_UINT64 },
648 };
649 
650 arc_sums_t arc_sums;
651 
652 #define	ARCSTAT_MAX(stat, val) {					\
653 	uint64_t m;							\
654 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
655 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
656 		continue;						\
657 }
658 
659 /*
660  * We define a macro to allow ARC hits/misses to be easily broken down by
661  * two separate conditions, giving a total of four different subtypes for
662  * each of hits and misses (so eight statistics total).
663  */
664 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
665 	if (cond1) {							\
666 		if (cond2) {						\
667 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
668 		} else {						\
669 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
670 		}							\
671 	} else {							\
672 		if (cond2) {						\
673 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
674 		} else {						\
675 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
676 		}							\
677 	}
678 
679 /*
680  * This macro allows us to use kstats as floating averages. Each time we
681  * update this kstat, we first factor it and the update value by
682  * ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall
683  * average. This macro assumes that integer loads and stores are atomic, but
684  * is not safe for multiple writers updating the kstat in parallel (only the
685  * last writer's update will remain).
686  */
687 #define	ARCSTAT_F_AVG_FACTOR	3
688 #define	ARCSTAT_F_AVG(stat, value) \
689 	do { \
690 		uint64_t x = ARCSTAT(stat); \
691 		x = x - x / ARCSTAT_F_AVG_FACTOR + \
692 		    (value) / ARCSTAT_F_AVG_FACTOR; \
693 		ARCSTAT(stat) = x; \
694 	} while (0)
695 
696 static kstat_t			*arc_ksp;
697 
698 /*
699  * There are several ARC variables that are critical to export as kstats --
700  * but we don't want to have to grovel around in the kstat whenever we wish to
701  * manipulate them.  For these variables, we therefore define them to be in
702  * terms of the statistic variable.  This assures that we are not introducing
703  * the possibility of inconsistency by having shadow copies of the variables,
704  * while still allowing the code to be readable.
705  */
706 #define	arc_tempreserve	ARCSTAT(arcstat_tempreserve)
707 #define	arc_loaned_bytes	ARCSTAT(arcstat_loaned_bytes)
708 #define	arc_dnode_limit	ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
709 #define	arc_need_free	ARCSTAT(arcstat_need_free) /* waiting to be evicted */
710 
711 hrtime_t arc_growtime;
712 list_t arc_prune_list;
713 kmutex_t arc_prune_mtx;
714 taskq_t *arc_prune_taskq;
715 
716 #define	GHOST_STATE(state)	\
717 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
718 	(state) == arc_l2c_only)
719 
720 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
721 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
722 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
723 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
724 #define	HDR_PRESCIENT_PREFETCH(hdr)	\
725 	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
726 #define	HDR_COMPRESSION_ENABLED(hdr)	\
727 	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
728 
729 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
730 #define	HDR_UNCACHED(hdr)	((hdr)->b_flags & ARC_FLAG_UNCACHED)
731 #define	HDR_L2_READING(hdr)	\
732 	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
733 	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
734 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
735 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
736 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
737 #define	HDR_PROTECTED(hdr)	((hdr)->b_flags & ARC_FLAG_PROTECTED)
738 #define	HDR_NOAUTH(hdr)		((hdr)->b_flags & ARC_FLAG_NOAUTH)
739 #define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
740 
741 #define	HDR_ISTYPE_METADATA(hdr)	\
742 	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
743 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
744 
745 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
746 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
747 #define	HDR_HAS_RABD(hdr)	\
748 	(HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) &&	\
749 	(hdr)->b_crypt_hdr.b_rabd != NULL)
750 #define	HDR_ENCRYPTED(hdr)	\
751 	(HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
752 #define	HDR_AUTHENTICATED(hdr)	\
753 	(HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
754 
755 /* For storing compression mode in b_flags */
756 #define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
757 
758 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
759 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
760 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
761 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
762 
763 #define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
764 #define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
765 #define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
766 #define	ARC_BUF_ENCRYPTED(buf)	((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)
767 
768 /*
769  * Other sizes
770  */
771 
772 #define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
773 #define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
774 
775 /*
776  * Hash table routines
777  */
778 
779 #define	BUF_LOCKS 2048
780 typedef struct buf_hash_table {
781 	uint64_t ht_mask;
782 	arc_buf_hdr_t **ht_table;
783 	kmutex_t ht_locks[BUF_LOCKS] ____cacheline_aligned;
784 } buf_hash_table_t;
785 
786 static buf_hash_table_t buf_hash_table;
787 
788 #define	BUF_HASH_INDEX(spa, dva, birth) \
789 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
790 #define	BUF_HASH_LOCK(idx)	(&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
791 #define	HDR_LOCK(hdr) \
792 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
793 
794 uint64_t zfs_crc64_table[256];
795 
796 /*
797  * Asynchronous ARC flush
798  *
799  * We track these in a list for arc_async_flush_guid_inuse().
800  * Used for both L1 and L2 async teardown.
801  */
802 static list_t arc_async_flush_list;
803 static kmutex_t	arc_async_flush_lock;
804 
805 typedef struct arc_async_flush {
806 	uint64_t	af_spa_guid;
807 	taskq_ent_t	af_tqent;
808 	uint_t		af_cache_level;	/* 1 or 2 to differentiate node */
809 	list_node_t	af_node;
810 } arc_async_flush_t;
811 
812 
813 /*
814  * Level 2 ARC
815  */
816 
817 #define	L2ARC_WRITE_SIZE	(32 * 1024 * 1024)	/* initial write max */
818 #define	L2ARC_HEADROOM		8			/* num of writes */
819 
820 /*
821  * If we discover during ARC scan any buffers to be compressed, we boost
822  * our headroom for the next scanning cycle by this percentage multiple.
823  */
824 #define	L2ARC_HEADROOM_BOOST	200
825 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
826 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
827 
828 /*
829  * We can feed L2ARC from two states of ARC buffers, mru and mfu,
830  * and each of the state has two types: data and metadata.
831  */
832 #define	L2ARC_FEED_TYPES	4
833 
834 /* L2ARC Performance Tunables */
835 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* def max write size */
836 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra warmup write */
837 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* # of dev writes */
838 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
839 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
840 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval msecs */
841 int l2arc_noprefetch = B_TRUE;			/* don't cache prefetch bufs */
842 int l2arc_feed_again = B_TRUE;			/* turbo warmup */
843 int l2arc_norw = B_FALSE;			/* no reads during writes */
844 static uint_t l2arc_meta_percent = 33;	/* limit on headers size */
845 
846 /*
847  * L2ARC Internals
848  */
849 static list_t L2ARC_dev_list;			/* device list */
850 static list_t *l2arc_dev_list;			/* device list pointer */
851 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
852 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
853 static list_t L2ARC_free_on_write;		/* free after write buf list */
854 static list_t *l2arc_free_on_write;		/* free after write list ptr */
855 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
856 static uint64_t l2arc_ndev;			/* number of devices */
857 
858 typedef struct l2arc_read_callback {
859 	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
860 	blkptr_t		l2rcb_bp;		/* original blkptr */
861 	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
862 	int			l2rcb_flags;		/* original flags */
863 	abd_t			*l2rcb_abd;		/* temporary buffer */
864 } l2arc_read_callback_t;
865 
866 typedef struct l2arc_data_free {
867 	/* protected by l2arc_free_on_write_mtx */
868 	abd_t		*l2df_abd;
869 	size_t		l2df_size;
870 	arc_buf_contents_t l2df_type;
871 	list_node_t	l2df_list_node;
872 } l2arc_data_free_t;
873 
874 typedef enum arc_fill_flags {
875 	ARC_FILL_LOCKED		= 1 << 0, /* hdr lock is held */
876 	ARC_FILL_COMPRESSED	= 1 << 1, /* fill with compressed data */
877 	ARC_FILL_ENCRYPTED	= 1 << 2, /* fill with encrypted data */
878 	ARC_FILL_NOAUTH		= 1 << 3, /* don't attempt to authenticate */
879 	ARC_FILL_IN_PLACE	= 1 << 4  /* fill in place (special case) */
880 } arc_fill_flags_t;
881 
882 typedef enum arc_ovf_level {
883 	ARC_OVF_NONE,			/* ARC within target size. */
884 	ARC_OVF_SOME,			/* ARC is slightly overflowed. */
885 	ARC_OVF_SEVERE			/* ARC is severely overflowed. */
886 } arc_ovf_level_t;
887 
888 static kmutex_t l2arc_feed_thr_lock;
889 static kcondvar_t l2arc_feed_thr_cv;
890 static uint8_t l2arc_thread_exit;
891 
892 static kmutex_t l2arc_rebuild_thr_lock;
893 static kcondvar_t l2arc_rebuild_thr_cv;
894 
895 enum arc_hdr_alloc_flags {
896 	ARC_HDR_ALLOC_RDATA = 0x1,
897 	ARC_HDR_USE_RESERVE = 0x4,
898 	ARC_HDR_ALLOC_LINEAR = 0x8,
899 };
900 
901 
902 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, const void *, int);
903 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, const void *);
904 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, const void *, int);
905 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, const void *);
906 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, const void *);
907 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
908     const void *tag);
909 static void arc_hdr_free_abd(arc_buf_hdr_t *, boolean_t);
910 static void arc_hdr_alloc_abd(arc_buf_hdr_t *, int);
911 static void arc_hdr_destroy(arc_buf_hdr_t *);
912 static void arc_access(arc_buf_hdr_t *, arc_flags_t, boolean_t);
913 static void arc_buf_watch(arc_buf_t *);
914 static void arc_change_state(arc_state_t *, arc_buf_hdr_t *);
915 
916 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
917 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
918 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
919 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
920 
921 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
922 static void l2arc_read_done(zio_t *);
923 static void l2arc_do_free_on_write(void);
924 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
925     boolean_t state_only);
926 
927 static void arc_prune_async(uint64_t adjust);
928 
929 #define	l2arc_hdr_arcstats_increment(hdr) \
930 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
931 #define	l2arc_hdr_arcstats_decrement(hdr) \
932 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
933 #define	l2arc_hdr_arcstats_increment_state(hdr) \
934 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
935 #define	l2arc_hdr_arcstats_decrement_state(hdr) \
936 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
937 
938 /*
939  * l2arc_exclude_special : A zfs module parameter that controls whether buffers
940  * 		present on special vdevs are eligibile for caching in L2ARC. If
941  * 		set to 1, exclude dbufs on special vdevs from being cached to
942  * 		L2ARC.
943  */
944 int l2arc_exclude_special = 0;
945 
946 /*
947  * l2arc_mfuonly : A ZFS module parameter that controls whether only MFU
948  * 		metadata and data are cached from ARC into L2ARC.
949  */
950 static int l2arc_mfuonly = 0;
951 
952 /*
953  * L2ARC TRIM
954  * l2arc_trim_ahead : A ZFS module parameter that controls how much ahead of
955  * 		the current write size (l2arc_write_max) we should TRIM if we
956  * 		have filled the device. It is defined as a percentage of the
957  * 		write size. If set to 100 we trim twice the space required to
958  * 		accommodate upcoming writes. A minimum of 64MB will be trimmed.
959  * 		It also enables TRIM of the whole L2ARC device upon creation or
960  * 		addition to an existing pool or if the header of the device is
961  * 		invalid upon importing a pool or onlining a cache device. The
962  * 		default is 0, which disables TRIM on L2ARC altogether as it can
963  * 		put significant stress on the underlying storage devices. This
964  * 		will vary depending of how well the specific device handles
965  * 		these commands.
966  */
967 static uint64_t l2arc_trim_ahead = 0;
968 
969 /*
970  * Performance tuning of L2ARC persistence:
971  *
972  * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
973  * 		an L2ARC device (either at pool import or later) will attempt
974  * 		to rebuild L2ARC buffer contents.
975  * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
976  * 		whether log blocks are written to the L2ARC device. If the L2ARC
977  * 		device is less than 1GB, the amount of data l2arc_evict()
978  * 		evicts is significant compared to the amount of restored L2ARC
979  * 		data. In this case do not write log blocks in L2ARC in order
980  * 		not to waste space.
981  */
982 static int l2arc_rebuild_enabled = B_TRUE;
983 static uint64_t l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
984 
985 /* L2ARC persistence rebuild control routines. */
986 void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
987 static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
988 static int l2arc_rebuild(l2arc_dev_t *dev);
989 
990 /* L2ARC persistence read I/O routines. */
991 static int l2arc_dev_hdr_read(l2arc_dev_t *dev);
992 static int l2arc_log_blk_read(l2arc_dev_t *dev,
993     const l2arc_log_blkptr_t *this_lp, const l2arc_log_blkptr_t *next_lp,
994     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
995     zio_t *this_io, zio_t **next_io);
996 static zio_t *l2arc_log_blk_fetch(vdev_t *vd,
997     const l2arc_log_blkptr_t *lp, l2arc_log_blk_phys_t *lb);
998 static void l2arc_log_blk_fetch_abort(zio_t *zio);
999 
1000 /* L2ARC persistence block restoration routines. */
1001 static void l2arc_log_blk_restore(l2arc_dev_t *dev,
1002     const l2arc_log_blk_phys_t *lb, uint64_t lb_asize);
1003 static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le,
1004     l2arc_dev_t *dev);
1005 
1006 /* L2ARC persistence write I/O routines. */
1007 static uint64_t l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio,
1008     l2arc_write_callback_t *cb);
1009 
1010 /* L2ARC persistence auxiliary routines. */
1011 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
1012     const l2arc_log_blkptr_t *lbp);
1013 static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev,
1014     const arc_buf_hdr_t *ab);
1015 boolean_t l2arc_range_check_overlap(uint64_t bottom,
1016     uint64_t top, uint64_t check);
1017 static void l2arc_blk_fetch_done(zio_t *zio);
1018 static inline uint64_t
1019     l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev);
1020 
1021 /*
1022  * We use Cityhash for this. It's fast, and has good hash properties without
1023  * requiring any large static buffers.
1024  */
1025 static uint64_t
buf_hash(uint64_t spa,const dva_t * dva,uint64_t birth)1026 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1027 {
1028 	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
1029 }
1030 
1031 #define	HDR_EMPTY(hdr)						\
1032 	((hdr)->b_dva.dva_word[0] == 0 &&			\
1033 	(hdr)->b_dva.dva_word[1] == 0)
1034 
1035 #define	HDR_EMPTY_OR_LOCKED(hdr)				\
1036 	(HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
1037 
1038 #define	HDR_EQUAL(spa, dva, birth, hdr)				\
1039 	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
1040 	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
1041 	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1042 
1043 static void
buf_discard_identity(arc_buf_hdr_t * hdr)1044 buf_discard_identity(arc_buf_hdr_t *hdr)
1045 {
1046 	hdr->b_dva.dva_word[0] = 0;
1047 	hdr->b_dva.dva_word[1] = 0;
1048 	hdr->b_birth = 0;
1049 }
1050 
1051 static arc_buf_hdr_t *
buf_hash_find(uint64_t spa,const blkptr_t * bp,kmutex_t ** lockp)1052 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1053 {
1054 	const dva_t *dva = BP_IDENTITY(bp);
1055 	uint64_t birth = BP_GET_BIRTH(bp);
1056 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1057 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1058 	arc_buf_hdr_t *hdr;
1059 
1060 	mutex_enter(hash_lock);
1061 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1062 	    hdr = hdr->b_hash_next) {
1063 		if (HDR_EQUAL(spa, dva, birth, hdr)) {
1064 			*lockp = hash_lock;
1065 			return (hdr);
1066 		}
1067 	}
1068 	mutex_exit(hash_lock);
1069 	*lockp = NULL;
1070 	return (NULL);
1071 }
1072 
1073 /*
1074  * Insert an entry into the hash table.  If there is already an element
1075  * equal to elem in the hash table, then the already existing element
1076  * will be returned and the new element will not be inserted.
1077  * Otherwise returns NULL.
1078  * If lockp == NULL, the caller is assumed to already hold the hash lock.
1079  */
1080 static arc_buf_hdr_t *
buf_hash_insert(arc_buf_hdr_t * hdr,kmutex_t ** lockp)1081 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1082 {
1083 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1084 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1085 	arc_buf_hdr_t *fhdr;
1086 	uint32_t i;
1087 
1088 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1089 	ASSERT(hdr->b_birth != 0);
1090 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1091 
1092 	if (lockp != NULL) {
1093 		*lockp = hash_lock;
1094 		mutex_enter(hash_lock);
1095 	} else {
1096 		ASSERT(MUTEX_HELD(hash_lock));
1097 	}
1098 
1099 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1100 	    fhdr = fhdr->b_hash_next, i++) {
1101 		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1102 			return (fhdr);
1103 	}
1104 
1105 	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1106 	buf_hash_table.ht_table[idx] = hdr;
1107 	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1108 
1109 	/* collect some hash table performance data */
1110 	if (i > 0) {
1111 		ARCSTAT_BUMP(arcstat_hash_collisions);
1112 		if (i == 1)
1113 			ARCSTAT_BUMP(arcstat_hash_chains);
1114 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1115 	}
1116 	ARCSTAT_BUMP(arcstat_hash_elements);
1117 
1118 	return (NULL);
1119 }
1120 
1121 static void
buf_hash_remove(arc_buf_hdr_t * hdr)1122 buf_hash_remove(arc_buf_hdr_t *hdr)
1123 {
1124 	arc_buf_hdr_t *fhdr, **hdrp;
1125 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1126 
1127 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1128 	ASSERT(HDR_IN_HASH_TABLE(hdr));
1129 
1130 	hdrp = &buf_hash_table.ht_table[idx];
1131 	while ((fhdr = *hdrp) != hdr) {
1132 		ASSERT3P(fhdr, !=, NULL);
1133 		hdrp = &fhdr->b_hash_next;
1134 	}
1135 	*hdrp = hdr->b_hash_next;
1136 	hdr->b_hash_next = NULL;
1137 	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1138 
1139 	/* collect some hash table performance data */
1140 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1141 	if (buf_hash_table.ht_table[idx] &&
1142 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1143 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1144 }
1145 
1146 /*
1147  * Global data structures and functions for the buf kmem cache.
1148  */
1149 
1150 static kmem_cache_t *hdr_full_cache;
1151 static kmem_cache_t *hdr_l2only_cache;
1152 static kmem_cache_t *buf_cache;
1153 
1154 static void
buf_fini(void)1155 buf_fini(void)
1156 {
1157 #if defined(_KERNEL)
1158 	/*
1159 	 * Large allocations which do not require contiguous pages
1160 	 * should be using vmem_free() in the linux kernel\
1161 	 */
1162 	vmem_free(buf_hash_table.ht_table,
1163 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1164 #else
1165 	kmem_free(buf_hash_table.ht_table,
1166 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1167 #endif
1168 	for (int i = 0; i < BUF_LOCKS; i++)
1169 		mutex_destroy(BUF_HASH_LOCK(i));
1170 	kmem_cache_destroy(hdr_full_cache);
1171 	kmem_cache_destroy(hdr_l2only_cache);
1172 	kmem_cache_destroy(buf_cache);
1173 }
1174 
1175 /*
1176  * Constructor callback - called when the cache is empty
1177  * and a new buf is requested.
1178  */
1179 static int
hdr_full_cons(void * vbuf,void * unused,int kmflag)1180 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1181 {
1182 	(void) unused, (void) kmflag;
1183 	arc_buf_hdr_t *hdr = vbuf;
1184 
1185 	memset(hdr, 0, HDR_FULL_SIZE);
1186 	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
1187 	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1188 #ifdef ZFS_DEBUG
1189 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1190 #endif
1191 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1192 	list_link_init(&hdr->b_l2hdr.b_l2node);
1193 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1194 
1195 	return (0);
1196 }
1197 
1198 static int
hdr_l2only_cons(void * vbuf,void * unused,int kmflag)1199 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1200 {
1201 	(void) unused, (void) kmflag;
1202 	arc_buf_hdr_t *hdr = vbuf;
1203 
1204 	memset(hdr, 0, HDR_L2ONLY_SIZE);
1205 	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1206 
1207 	return (0);
1208 }
1209 
1210 static int
buf_cons(void * vbuf,void * unused,int kmflag)1211 buf_cons(void *vbuf, void *unused, int kmflag)
1212 {
1213 	(void) unused, (void) kmflag;
1214 	arc_buf_t *buf = vbuf;
1215 
1216 	memset(buf, 0, sizeof (arc_buf_t));
1217 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1218 
1219 	return (0);
1220 }
1221 
1222 /*
1223  * Destructor callback - called when a cached buf is
1224  * no longer required.
1225  */
1226 static void
hdr_full_dest(void * vbuf,void * unused)1227 hdr_full_dest(void *vbuf, void *unused)
1228 {
1229 	(void) unused;
1230 	arc_buf_hdr_t *hdr = vbuf;
1231 
1232 	ASSERT(HDR_EMPTY(hdr));
1233 	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1234 #ifdef ZFS_DEBUG
1235 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1236 #endif
1237 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1238 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1239 }
1240 
1241 static void
hdr_l2only_dest(void * vbuf,void * unused)1242 hdr_l2only_dest(void *vbuf, void *unused)
1243 {
1244 	(void) unused;
1245 	arc_buf_hdr_t *hdr = vbuf;
1246 
1247 	ASSERT(HDR_EMPTY(hdr));
1248 	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1249 }
1250 
1251 static void
buf_dest(void * vbuf,void * unused)1252 buf_dest(void *vbuf, void *unused)
1253 {
1254 	(void) unused;
1255 	(void) vbuf;
1256 
1257 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1258 }
1259 
1260 static void
buf_init(void)1261 buf_init(void)
1262 {
1263 	uint64_t *ct = NULL;
1264 	uint64_t hsize = 1ULL << 12;
1265 	int i, j;
1266 
1267 	/*
1268 	 * The hash table is big enough to fill all of physical memory
1269 	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1270 	 * By default, the table will take up
1271 	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1272 	 */
1273 	while (hsize * zfs_arc_average_blocksize < arc_all_memory())
1274 		hsize <<= 1;
1275 retry:
1276 	buf_hash_table.ht_mask = hsize - 1;
1277 #if defined(_KERNEL)
1278 	/*
1279 	 * Large allocations which do not require contiguous pages
1280 	 * should be using vmem_alloc() in the linux kernel
1281 	 */
1282 	buf_hash_table.ht_table =
1283 	    vmem_zalloc(hsize * sizeof (void*), KM_SLEEP);
1284 #else
1285 	buf_hash_table.ht_table =
1286 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1287 #endif
1288 	if (buf_hash_table.ht_table == NULL) {
1289 		ASSERT(hsize > (1ULL << 8));
1290 		hsize >>= 1;
1291 		goto retry;
1292 	}
1293 
1294 	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1295 	    0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
1296 	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1297 	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL,
1298 	    NULL, NULL, 0);
1299 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1300 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1301 
1302 	for (i = 0; i < 256; i++)
1303 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1304 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1305 
1306 	for (i = 0; i < BUF_LOCKS; i++)
1307 		mutex_init(BUF_HASH_LOCK(i), NULL, MUTEX_DEFAULT, NULL);
1308 }
1309 
1310 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
1311 
1312 /*
1313  * This is the size that the buf occupies in memory. If the buf is compressed,
1314  * it will correspond to the compressed size. You should use this method of
1315  * getting the buf size unless you explicitly need the logical size.
1316  */
1317 uint64_t
arc_buf_size(arc_buf_t * buf)1318 arc_buf_size(arc_buf_t *buf)
1319 {
1320 	return (ARC_BUF_COMPRESSED(buf) ?
1321 	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1322 }
1323 
1324 uint64_t
arc_buf_lsize(arc_buf_t * buf)1325 arc_buf_lsize(arc_buf_t *buf)
1326 {
1327 	return (HDR_GET_LSIZE(buf->b_hdr));
1328 }
1329 
1330 /*
1331  * This function will return B_TRUE if the buffer is encrypted in memory.
1332  * This buffer can be decrypted by calling arc_untransform().
1333  */
1334 boolean_t
arc_is_encrypted(arc_buf_t * buf)1335 arc_is_encrypted(arc_buf_t *buf)
1336 {
1337 	return (ARC_BUF_ENCRYPTED(buf) != 0);
1338 }
1339 
1340 /*
1341  * Returns B_TRUE if the buffer represents data that has not had its MAC
1342  * verified yet.
1343  */
1344 boolean_t
arc_is_unauthenticated(arc_buf_t * buf)1345 arc_is_unauthenticated(arc_buf_t *buf)
1346 {
1347 	return (HDR_NOAUTH(buf->b_hdr) != 0);
1348 }
1349 
1350 void
arc_get_raw_params(arc_buf_t * buf,boolean_t * byteorder,uint8_t * salt,uint8_t * iv,uint8_t * mac)1351 arc_get_raw_params(arc_buf_t *buf, boolean_t *byteorder, uint8_t *salt,
1352     uint8_t *iv, uint8_t *mac)
1353 {
1354 	arc_buf_hdr_t *hdr = buf->b_hdr;
1355 
1356 	ASSERT(HDR_PROTECTED(hdr));
1357 
1358 	memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
1359 	memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
1360 	memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
1361 	*byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
1362 	    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
1363 }
1364 
1365 /*
1366  * Indicates how this buffer is compressed in memory. If it is not compressed
1367  * the value will be ZIO_COMPRESS_OFF. It can be made normally readable with
1368  * arc_untransform() as long as it is also unencrypted.
1369  */
1370 enum zio_compress
arc_get_compression(arc_buf_t * buf)1371 arc_get_compression(arc_buf_t *buf)
1372 {
1373 	return (ARC_BUF_COMPRESSED(buf) ?
1374 	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1375 }
1376 
1377 /*
1378  * Return the compression algorithm used to store this data in the ARC. If ARC
1379  * compression is enabled or this is an encrypted block, this will be the same
1380  * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
1381  */
1382 static inline enum zio_compress
arc_hdr_get_compress(arc_buf_hdr_t * hdr)1383 arc_hdr_get_compress(arc_buf_hdr_t *hdr)
1384 {
1385 	return (HDR_COMPRESSION_ENABLED(hdr) ?
1386 	    HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
1387 }
1388 
1389 uint8_t
arc_get_complevel(arc_buf_t * buf)1390 arc_get_complevel(arc_buf_t *buf)
1391 {
1392 	return (buf->b_hdr->b_complevel);
1393 }
1394 
1395 static inline boolean_t
arc_buf_is_shared(arc_buf_t * buf)1396 arc_buf_is_shared(arc_buf_t *buf)
1397 {
1398 	boolean_t shared = (buf->b_data != NULL &&
1399 	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1400 	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1401 	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1402 	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1403 	EQUIV(shared, ARC_BUF_SHARED(buf));
1404 	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1405 
1406 	/*
1407 	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1408 	 * already being shared" requirement prevents us from doing that.
1409 	 */
1410 
1411 	return (shared);
1412 }
1413 
1414 /*
1415  * Free the checksum associated with this header. If there is no checksum, this
1416  * is a no-op.
1417  */
1418 static inline void
arc_cksum_free(arc_buf_hdr_t * hdr)1419 arc_cksum_free(arc_buf_hdr_t *hdr)
1420 {
1421 #ifdef ZFS_DEBUG
1422 	ASSERT(HDR_HAS_L1HDR(hdr));
1423 
1424 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1425 	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1426 		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1427 		hdr->b_l1hdr.b_freeze_cksum = NULL;
1428 	}
1429 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1430 #endif
1431 }
1432 
1433 /*
1434  * Return true iff at least one of the bufs on hdr is not compressed.
1435  * Encrypted buffers count as compressed.
1436  */
1437 static boolean_t
arc_hdr_has_uncompressed_buf(arc_buf_hdr_t * hdr)1438 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1439 {
1440 	ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
1441 
1442 	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1443 		if (!ARC_BUF_COMPRESSED(b)) {
1444 			return (B_TRUE);
1445 		}
1446 	}
1447 	return (B_FALSE);
1448 }
1449 
1450 
1451 /*
1452  * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1453  * matches the checksum that is stored in the hdr. If there is no checksum,
1454  * or if the buf is compressed, this is a no-op.
1455  */
1456 static void
arc_cksum_verify(arc_buf_t * buf)1457 arc_cksum_verify(arc_buf_t *buf)
1458 {
1459 #ifdef ZFS_DEBUG
1460 	arc_buf_hdr_t *hdr = buf->b_hdr;
1461 	zio_cksum_t zc;
1462 
1463 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1464 		return;
1465 
1466 	if (ARC_BUF_COMPRESSED(buf))
1467 		return;
1468 
1469 	ASSERT(HDR_HAS_L1HDR(hdr));
1470 
1471 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1472 
1473 	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1474 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1475 		return;
1476 	}
1477 
1478 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1479 	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1480 		panic("buffer modified while frozen!");
1481 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1482 #endif
1483 }
1484 
1485 /*
1486  * This function makes the assumption that data stored in the L2ARC
1487  * will be transformed exactly as it is in the main pool. Because of
1488  * this we can verify the checksum against the reading process's bp.
1489  */
1490 static boolean_t
arc_cksum_is_equal(arc_buf_hdr_t * hdr,zio_t * zio)1491 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1492 {
1493 	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1494 	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1495 
1496 	/*
1497 	 * Block pointers always store the checksum for the logical data.
1498 	 * If the block pointer has the gang bit set, then the checksum
1499 	 * it represents is for the reconstituted data and not for an
1500 	 * individual gang member. The zio pipeline, however, must be able to
1501 	 * determine the checksum of each of the gang constituents so it
1502 	 * treats the checksum comparison differently than what we need
1503 	 * for l2arc blocks. This prevents us from using the
1504 	 * zio_checksum_error() interface directly. Instead we must call the
1505 	 * zio_checksum_error_impl() so that we can ensure the checksum is
1506 	 * generated using the correct checksum algorithm and accounts for the
1507 	 * logical I/O size and not just a gang fragment.
1508 	 */
1509 	return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1510 	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1511 	    zio->io_offset, NULL) == 0);
1512 }
1513 
1514 /*
1515  * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
1516  * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1517  * isn't modified later on. If buf is compressed or there is already a checksum
1518  * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1519  */
1520 static void
arc_cksum_compute(arc_buf_t * buf)1521 arc_cksum_compute(arc_buf_t *buf)
1522 {
1523 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1524 		return;
1525 
1526 #ifdef ZFS_DEBUG
1527 	arc_buf_hdr_t *hdr = buf->b_hdr;
1528 	ASSERT(HDR_HAS_L1HDR(hdr));
1529 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1530 	if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
1531 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1532 		return;
1533 	}
1534 
1535 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
1536 	ASSERT(!ARC_BUF_COMPRESSED(buf));
1537 	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1538 	    KM_SLEEP);
1539 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1540 	    hdr->b_l1hdr.b_freeze_cksum);
1541 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1542 #endif
1543 	arc_buf_watch(buf);
1544 }
1545 
1546 #ifndef _KERNEL
1547 void
arc_buf_sigsegv(int sig,siginfo_t * si,void * unused)1548 arc_buf_sigsegv(int sig, siginfo_t *si, void *unused)
1549 {
1550 	(void) sig, (void) unused;
1551 	panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
1552 }
1553 #endif
1554 
1555 static void
arc_buf_unwatch(arc_buf_t * buf)1556 arc_buf_unwatch(arc_buf_t *buf)
1557 {
1558 #ifndef _KERNEL
1559 	if (arc_watch) {
1560 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1561 		    PROT_READ | PROT_WRITE));
1562 	}
1563 #else
1564 	(void) buf;
1565 #endif
1566 }
1567 
1568 static void
arc_buf_watch(arc_buf_t * buf)1569 arc_buf_watch(arc_buf_t *buf)
1570 {
1571 #ifndef _KERNEL
1572 	if (arc_watch)
1573 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1574 		    PROT_READ));
1575 #else
1576 	(void) buf;
1577 #endif
1578 }
1579 
1580 static arc_buf_contents_t
arc_buf_type(arc_buf_hdr_t * hdr)1581 arc_buf_type(arc_buf_hdr_t *hdr)
1582 {
1583 	arc_buf_contents_t type;
1584 	if (HDR_ISTYPE_METADATA(hdr)) {
1585 		type = ARC_BUFC_METADATA;
1586 	} else {
1587 		type = ARC_BUFC_DATA;
1588 	}
1589 	VERIFY3U(hdr->b_type, ==, type);
1590 	return (type);
1591 }
1592 
1593 boolean_t
arc_is_metadata(arc_buf_t * buf)1594 arc_is_metadata(arc_buf_t *buf)
1595 {
1596 	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
1597 }
1598 
1599 static uint32_t
arc_bufc_to_flags(arc_buf_contents_t type)1600 arc_bufc_to_flags(arc_buf_contents_t type)
1601 {
1602 	switch (type) {
1603 	case ARC_BUFC_DATA:
1604 		/* metadata field is 0 if buffer contains normal data */
1605 		return (0);
1606 	case ARC_BUFC_METADATA:
1607 		return (ARC_FLAG_BUFC_METADATA);
1608 	default:
1609 		break;
1610 	}
1611 	panic("undefined ARC buffer type!");
1612 	return ((uint32_t)-1);
1613 }
1614 
1615 void
arc_buf_thaw(arc_buf_t * buf)1616 arc_buf_thaw(arc_buf_t *buf)
1617 {
1618 	arc_buf_hdr_t *hdr = buf->b_hdr;
1619 
1620 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1621 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1622 
1623 	arc_cksum_verify(buf);
1624 
1625 	/*
1626 	 * Compressed buffers do not manipulate the b_freeze_cksum.
1627 	 */
1628 	if (ARC_BUF_COMPRESSED(buf))
1629 		return;
1630 
1631 	ASSERT(HDR_HAS_L1HDR(hdr));
1632 	arc_cksum_free(hdr);
1633 	arc_buf_unwatch(buf);
1634 }
1635 
1636 void
arc_buf_freeze(arc_buf_t * buf)1637 arc_buf_freeze(arc_buf_t *buf)
1638 {
1639 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1640 		return;
1641 
1642 	if (ARC_BUF_COMPRESSED(buf))
1643 		return;
1644 
1645 	ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
1646 	arc_cksum_compute(buf);
1647 }
1648 
1649 /*
1650  * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
1651  * the following functions should be used to ensure that the flags are
1652  * updated in a thread-safe way. When manipulating the flags either
1653  * the hash_lock must be held or the hdr must be undiscoverable. This
1654  * ensures that we're not racing with any other threads when updating
1655  * the flags.
1656  */
1657 static inline void
arc_hdr_set_flags(arc_buf_hdr_t * hdr,arc_flags_t flags)1658 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1659 {
1660 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1661 	hdr->b_flags |= flags;
1662 }
1663 
1664 static inline void
arc_hdr_clear_flags(arc_buf_hdr_t * hdr,arc_flags_t flags)1665 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1666 {
1667 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1668 	hdr->b_flags &= ~flags;
1669 }
1670 
1671 /*
1672  * Setting the compression bits in the arc_buf_hdr_t's b_flags is
1673  * done in a special way since we have to clear and set bits
1674  * at the same time. Consumers that wish to set the compression bits
1675  * must use this function to ensure that the flags are updated in
1676  * thread-safe manner.
1677  */
1678 static void
arc_hdr_set_compress(arc_buf_hdr_t * hdr,enum zio_compress cmp)1679 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
1680 {
1681 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1682 
1683 	/*
1684 	 * Holes and embedded blocks will always have a psize = 0 so
1685 	 * we ignore the compression of the blkptr and set the
1686 	 * want to uncompress them. Mark them as uncompressed.
1687 	 */
1688 	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
1689 		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1690 		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
1691 	} else {
1692 		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1693 		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
1694 	}
1695 
1696 	HDR_SET_COMPRESS(hdr, cmp);
1697 	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
1698 }
1699 
1700 /*
1701  * Looks for another buf on the same hdr which has the data decompressed, copies
1702  * from it, and returns true. If no such buf exists, returns false.
1703  */
1704 static boolean_t
arc_buf_try_copy_decompressed_data(arc_buf_t * buf)1705 arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
1706 {
1707 	arc_buf_hdr_t *hdr = buf->b_hdr;
1708 	boolean_t copied = B_FALSE;
1709 
1710 	ASSERT(HDR_HAS_L1HDR(hdr));
1711 	ASSERT3P(buf->b_data, !=, NULL);
1712 	ASSERT(!ARC_BUF_COMPRESSED(buf));
1713 
1714 	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1715 	    from = from->b_next) {
1716 		/* can't use our own data buffer */
1717 		if (from == buf) {
1718 			continue;
1719 		}
1720 
1721 		if (!ARC_BUF_COMPRESSED(from)) {
1722 			memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
1723 			copied = B_TRUE;
1724 			break;
1725 		}
1726 	}
1727 
1728 #ifdef ZFS_DEBUG
1729 	/*
1730 	 * There were no decompressed bufs, so there should not be a
1731 	 * checksum on the hdr either.
1732 	 */
1733 	if (zfs_flags & ZFS_DEBUG_MODIFY)
1734 		EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1735 #endif
1736 
1737 	return (copied);
1738 }
1739 
1740 /*
1741  * Allocates an ARC buf header that's in an evicted & L2-cached state.
1742  * This is used during l2arc reconstruction to make empty ARC buffers
1743  * which circumvent the regular disk->arc->l2arc path and instead come
1744  * into being in the reverse order, i.e. l2arc->arc.
1745  */
1746 static arc_buf_hdr_t *
arc_buf_alloc_l2only(size_t size,arc_buf_contents_t type,l2arc_dev_t * dev,dva_t dva,uint64_t daddr,int32_t psize,uint64_t asize,uint64_t birth,enum zio_compress compress,uint8_t complevel,boolean_t protected,boolean_t prefetch,arc_state_type_t arcs_state)1747 arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
1748     dva_t dva, uint64_t daddr, int32_t psize, uint64_t asize, uint64_t birth,
1749     enum zio_compress compress, uint8_t complevel, boolean_t protected,
1750     boolean_t prefetch, arc_state_type_t arcs_state)
1751 {
1752 	arc_buf_hdr_t	*hdr;
1753 
1754 	ASSERT(size != 0);
1755 	ASSERT(dev->l2ad_vdev != NULL);
1756 
1757 	hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
1758 	hdr->b_birth = birth;
1759 	hdr->b_type = type;
1760 	hdr->b_flags = 0;
1761 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);
1762 	HDR_SET_LSIZE(hdr, size);
1763 	HDR_SET_PSIZE(hdr, psize);
1764 	HDR_SET_L2SIZE(hdr, asize);
1765 	arc_hdr_set_compress(hdr, compress);
1766 	hdr->b_complevel = complevel;
1767 	if (protected)
1768 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
1769 	if (prefetch)
1770 		arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
1771 	hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
1772 
1773 	hdr->b_dva = dva;
1774 
1775 	hdr->b_l2hdr.b_dev = dev;
1776 	hdr->b_l2hdr.b_daddr = daddr;
1777 	hdr->b_l2hdr.b_arcs_state = arcs_state;
1778 
1779 	return (hdr);
1780 }
1781 
1782 /*
1783  * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
1784  */
1785 static uint64_t
arc_hdr_size(arc_buf_hdr_t * hdr)1786 arc_hdr_size(arc_buf_hdr_t *hdr)
1787 {
1788 	uint64_t size;
1789 
1790 	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
1791 	    HDR_GET_PSIZE(hdr) > 0) {
1792 		size = HDR_GET_PSIZE(hdr);
1793 	} else {
1794 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
1795 		size = HDR_GET_LSIZE(hdr);
1796 	}
1797 	return (size);
1798 }
1799 
1800 static int
arc_hdr_authenticate(arc_buf_hdr_t * hdr,spa_t * spa,uint64_t dsobj)1801 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
1802 {
1803 	int ret;
1804 	uint64_t csize;
1805 	uint64_t lsize = HDR_GET_LSIZE(hdr);
1806 	uint64_t psize = HDR_GET_PSIZE(hdr);
1807 	abd_t *abd = hdr->b_l1hdr.b_pabd;
1808 	boolean_t free_abd = B_FALSE;
1809 
1810 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1811 	ASSERT(HDR_AUTHENTICATED(hdr));
1812 	ASSERT3P(abd, !=, NULL);
1813 
1814 	/*
1815 	 * The MAC is calculated on the compressed data that is stored on disk.
1816 	 * However, if compressed arc is disabled we will only have the
1817 	 * decompressed data available to us now. Compress it into a temporary
1818 	 * abd so we can verify the MAC. The performance overhead of this will
1819 	 * be relatively low, since most objects in an encrypted objset will
1820 	 * be encrypted (instead of authenticated) anyway.
1821 	 */
1822 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1823 	    !HDR_COMPRESSION_ENABLED(hdr)) {
1824 		abd = NULL;
1825 		csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
1826 		    hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize),
1827 		    hdr->b_complevel);
1828 		if (csize >= lsize || csize > psize) {
1829 			ret = SET_ERROR(EIO);
1830 			return (ret);
1831 		}
1832 		ASSERT3P(abd, !=, NULL);
1833 		abd_zero_off(abd, csize, psize - csize);
1834 		free_abd = B_TRUE;
1835 	}
1836 
1837 	/*
1838 	 * Authentication is best effort. We authenticate whenever the key is
1839 	 * available. If we succeed we clear ARC_FLAG_NOAUTH.
1840 	 */
1841 	if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
1842 		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1843 		ASSERT3U(lsize, ==, psize);
1844 		ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd,
1845 		    psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1846 	} else {
1847 		ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize,
1848 		    hdr->b_crypt_hdr.b_mac);
1849 	}
1850 
1851 	if (ret == 0)
1852 		arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
1853 	else if (ret == ENOENT)
1854 		ret = 0;
1855 
1856 	if (free_abd)
1857 		abd_free(abd);
1858 
1859 	return (ret);
1860 }
1861 
1862 /*
1863  * This function will take a header that only has raw encrypted data in
1864  * b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in
1865  * b_l1hdr.b_pabd. If designated in the header flags, this function will
1866  * also decompress the data.
1867  */
1868 static int
arc_hdr_decrypt(arc_buf_hdr_t * hdr,spa_t * spa,const zbookmark_phys_t * zb)1869 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
1870 {
1871 	int ret;
1872 	abd_t *cabd = NULL;
1873 	boolean_t no_crypt = B_FALSE;
1874 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1875 
1876 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1877 	ASSERT(HDR_ENCRYPTED(hdr));
1878 
1879 	arc_hdr_alloc_abd(hdr, 0);
1880 
1881 	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
1882 	    B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
1883 	    hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
1884 	    hdr->b_crypt_hdr.b_rabd, &no_crypt);
1885 	if (ret != 0)
1886 		goto error;
1887 
1888 	if (no_crypt) {
1889 		abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
1890 		    HDR_GET_PSIZE(hdr));
1891 	}
1892 
1893 	/*
1894 	 * If this header has disabled arc compression but the b_pabd is
1895 	 * compressed after decrypting it, we need to decompress the newly
1896 	 * decrypted data.
1897 	 */
1898 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1899 	    !HDR_COMPRESSION_ENABLED(hdr)) {
1900 		/*
1901 		 * We want to make sure that we are correctly honoring the
1902 		 * zfs_abd_scatter_enabled setting, so we allocate an abd here
1903 		 * and then loan a buffer from it, rather than allocating a
1904 		 * linear buffer and wrapping it in an abd later.
1905 		 */
1906 		cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
1907 
1908 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
1909 		    hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
1910 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
1911 		if (ret != 0) {
1912 			goto error;
1913 		}
1914 
1915 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
1916 		    arc_hdr_size(hdr), hdr);
1917 		hdr->b_l1hdr.b_pabd = cabd;
1918 	}
1919 
1920 	return (0);
1921 
1922 error:
1923 	arc_hdr_free_abd(hdr, B_FALSE);
1924 	if (cabd != NULL)
1925 		arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
1926 
1927 	return (ret);
1928 }
1929 
1930 /*
1931  * This function is called during arc_buf_fill() to prepare the header's
1932  * abd plaintext pointer for use. This involves authenticated protected
1933  * data and decrypting encrypted data into the plaintext abd.
1934  */
1935 static int
arc_fill_hdr_crypt(arc_buf_hdr_t * hdr,kmutex_t * hash_lock,spa_t * spa,const zbookmark_phys_t * zb,boolean_t noauth)1936 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
1937     const zbookmark_phys_t *zb, boolean_t noauth)
1938 {
1939 	int ret;
1940 
1941 	ASSERT(HDR_PROTECTED(hdr));
1942 
1943 	if (hash_lock != NULL)
1944 		mutex_enter(hash_lock);
1945 
1946 	if (HDR_NOAUTH(hdr) && !noauth) {
1947 		/*
1948 		 * The caller requested authenticated data but our data has
1949 		 * not been authenticated yet. Verify the MAC now if we can.
1950 		 */
1951 		ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
1952 		if (ret != 0)
1953 			goto error;
1954 	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
1955 		/*
1956 		 * If we only have the encrypted version of the data, but the
1957 		 * unencrypted version was requested we take this opportunity
1958 		 * to store the decrypted version in the header for future use.
1959 		 */
1960 		ret = arc_hdr_decrypt(hdr, spa, zb);
1961 		if (ret != 0)
1962 			goto error;
1963 	}
1964 
1965 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1966 
1967 	if (hash_lock != NULL)
1968 		mutex_exit(hash_lock);
1969 
1970 	return (0);
1971 
1972 error:
1973 	if (hash_lock != NULL)
1974 		mutex_exit(hash_lock);
1975 
1976 	return (ret);
1977 }
1978 
1979 /*
1980  * This function is used by the dbuf code to decrypt bonus buffers in place.
1981  * The dbuf code itself doesn't have any locking for decrypting a shared dnode
1982  * block, so we use the hash lock here to protect against concurrent calls to
1983  * arc_buf_fill().
1984  */
1985 static void
arc_buf_untransform_in_place(arc_buf_t * buf)1986 arc_buf_untransform_in_place(arc_buf_t *buf)
1987 {
1988 	arc_buf_hdr_t *hdr = buf->b_hdr;
1989 
1990 	ASSERT(HDR_ENCRYPTED(hdr));
1991 	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
1992 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1993 	ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);
1994 
1995 	zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
1996 	    arc_buf_size(buf));
1997 	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
1998 	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
1999 }
2000 
2001 /*
2002  * Given a buf that has a data buffer attached to it, this function will
2003  * efficiently fill the buf with data of the specified compression setting from
2004  * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
2005  * are already sharing a data buf, no copy is performed.
2006  *
2007  * If the buf is marked as compressed but uncompressed data was requested, this
2008  * will allocate a new data buffer for the buf, remove that flag, and fill the
2009  * buf with uncompressed data. You can't request a compressed buf on a hdr with
2010  * uncompressed data, and (since we haven't added support for it yet) if you
2011  * want compressed data your buf must already be marked as compressed and have
2012  * the correct-sized data buffer.
2013  */
2014 static int
arc_buf_fill(arc_buf_t * buf,spa_t * spa,const zbookmark_phys_t * zb,arc_fill_flags_t flags)2015 arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
2016     arc_fill_flags_t flags)
2017 {
2018 	int error = 0;
2019 	arc_buf_hdr_t *hdr = buf->b_hdr;
2020 	boolean_t hdr_compressed =
2021 	    (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
2022 	boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0;
2023 	boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0;
2024 	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2025 	kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);
2026 
2027 	ASSERT3P(buf->b_data, !=, NULL);
2028 	IMPLY(compressed, hdr_compressed || ARC_BUF_ENCRYPTED(buf));
2029 	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2030 	IMPLY(encrypted, HDR_ENCRYPTED(hdr));
2031 	IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf));
2032 	IMPLY(encrypted, ARC_BUF_COMPRESSED(buf));
2033 	IMPLY(encrypted, !arc_buf_is_shared(buf));
2034 
2035 	/*
2036 	 * If the caller wanted encrypted data we just need to copy it from
2037 	 * b_rabd and potentially byteswap it. We won't be able to do any
2038 	 * further transforms on it.
2039 	 */
2040 	if (encrypted) {
2041 		ASSERT(HDR_HAS_RABD(hdr));
2042 		abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
2043 		    HDR_GET_PSIZE(hdr));
2044 		goto byteswap;
2045 	}
2046 
2047 	/*
2048 	 * Adjust encrypted and authenticated headers to accommodate
2049 	 * the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are
2050 	 * allowed to fail decryption due to keys not being loaded
2051 	 * without being marked as an IO error.
2052 	 */
2053 	if (HDR_PROTECTED(hdr)) {
2054 		error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
2055 		    zb, !!(flags & ARC_FILL_NOAUTH));
2056 		if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) {
2057 			return (error);
2058 		} else if (error != 0) {
2059 			if (hash_lock != NULL)
2060 				mutex_enter(hash_lock);
2061 			arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2062 			if (hash_lock != NULL)
2063 				mutex_exit(hash_lock);
2064 			return (error);
2065 		}
2066 	}
2067 
2068 	/*
2069 	 * There is a special case here for dnode blocks which are
2070 	 * decrypting their bonus buffers. These blocks may request to
2071 	 * be decrypted in-place. This is necessary because there may
2072 	 * be many dnodes pointing into this buffer and there is
2073 	 * currently no method to synchronize replacing the backing
2074 	 * b_data buffer and updating all of the pointers. Here we use
2075 	 * the hash lock to ensure there are no races. If the need
2076 	 * arises for other types to be decrypted in-place, they must
2077 	 * add handling here as well.
2078 	 */
2079 	if ((flags & ARC_FILL_IN_PLACE) != 0) {
2080 		ASSERT(!hdr_compressed);
2081 		ASSERT(!compressed);
2082 		ASSERT(!encrypted);
2083 
2084 		if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
2085 			ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
2086 
2087 			if (hash_lock != NULL)
2088 				mutex_enter(hash_lock);
2089 			arc_buf_untransform_in_place(buf);
2090 			if (hash_lock != NULL)
2091 				mutex_exit(hash_lock);
2092 
2093 			/* Compute the hdr's checksum if necessary */
2094 			arc_cksum_compute(buf);
2095 		}
2096 
2097 		return (0);
2098 	}
2099 
2100 	if (hdr_compressed == compressed) {
2101 		if (ARC_BUF_SHARED(buf)) {
2102 			ASSERT(arc_buf_is_shared(buf));
2103 		} else {
2104 			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2105 			    arc_buf_size(buf));
2106 		}
2107 	} else {
2108 		ASSERT(hdr_compressed);
2109 		ASSERT(!compressed);
2110 
2111 		/*
2112 		 * If the buf is sharing its data with the hdr, unlink it and
2113 		 * allocate a new data buffer for the buf.
2114 		 */
2115 		if (ARC_BUF_SHARED(buf)) {
2116 			ASSERTF(ARC_BUF_COMPRESSED(buf),
2117 			"buf %p was uncompressed", buf);
2118 
2119 			/* We need to give the buf its own b_data */
2120 			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2121 			buf->b_data =
2122 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2123 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2124 
2125 			/* Previously overhead was 0; just add new overhead */
2126 			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2127 		} else if (ARC_BUF_COMPRESSED(buf)) {
2128 			ASSERT(!arc_buf_is_shared(buf));
2129 
2130 			/* We need to reallocate the buf's b_data */
2131 			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2132 			    buf);
2133 			buf->b_data =
2134 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2135 
2136 			/* We increased the size of b_data; update overhead */
2137 			ARCSTAT_INCR(arcstat_overhead_size,
2138 			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2139 		}
2140 
2141 		/*
2142 		 * Regardless of the buf's previous compression settings, it
2143 		 * should not be compressed at the end of this function.
2144 		 */
2145 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2146 
2147 		/*
2148 		 * Try copying the data from another buf which already has a
2149 		 * decompressed version. If that's not possible, it's time to
2150 		 * bite the bullet and decompress the data from the hdr.
2151 		 */
2152 		if (arc_buf_try_copy_decompressed_data(buf)) {
2153 			/* Skip byteswapping and checksumming (already done) */
2154 			return (0);
2155 		} else {
2156 			abd_t dabd;
2157 			abd_get_from_buf_struct(&dabd, buf->b_data,
2158 			    HDR_GET_LSIZE(hdr));
2159 			error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2160 			    hdr->b_l1hdr.b_pabd, &dabd,
2161 			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
2162 			    &hdr->b_complevel);
2163 			abd_free(&dabd);
2164 
2165 			/*
2166 			 * Absent hardware errors or software bugs, this should
2167 			 * be impossible, but log it anyway so we can debug it.
2168 			 */
2169 			if (error != 0) {
2170 				zfs_dbgmsg(
2171 				    "hdr %px, compress %d, psize %d, lsize %d",
2172 				    hdr, arc_hdr_get_compress(hdr),
2173 				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2174 				if (hash_lock != NULL)
2175 					mutex_enter(hash_lock);
2176 				arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2177 				if (hash_lock != NULL)
2178 					mutex_exit(hash_lock);
2179 				return (SET_ERROR(EIO));
2180 			}
2181 		}
2182 	}
2183 
2184 byteswap:
2185 	/* Byteswap the buf's data if necessary */
2186 	if (bswap != DMU_BSWAP_NUMFUNCS) {
2187 		ASSERT(!HDR_SHARED_DATA(hdr));
2188 		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2189 		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2190 	}
2191 
2192 	/* Compute the hdr's checksum if necessary */
2193 	arc_cksum_compute(buf);
2194 
2195 	return (0);
2196 }
2197 
2198 /*
2199  * If this function is being called to decrypt an encrypted buffer or verify an
2200  * authenticated one, the key must be loaded and a mapping must be made
2201  * available in the keystore via spa_keystore_create_mapping() or one of its
2202  * callers.
2203  */
2204 int
arc_untransform(arc_buf_t * buf,spa_t * spa,const zbookmark_phys_t * zb,boolean_t in_place)2205 arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
2206     boolean_t in_place)
2207 {
2208 	int ret;
2209 	arc_fill_flags_t flags = 0;
2210 
2211 	if (in_place)
2212 		flags |= ARC_FILL_IN_PLACE;
2213 
2214 	ret = arc_buf_fill(buf, spa, zb, flags);
2215 	if (ret == ECKSUM) {
2216 		/*
2217 		 * Convert authentication and decryption errors to EIO
2218 		 * (and generate an ereport) before leaving the ARC.
2219 		 */
2220 		ret = SET_ERROR(EIO);
2221 		spa_log_error(spa, zb, buf->b_hdr->b_birth);
2222 		(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
2223 		    spa, NULL, zb, NULL, 0);
2224 	}
2225 
2226 	return (ret);
2227 }
2228 
2229 /*
2230  * Increment the amount of evictable space in the arc_state_t's refcount.
2231  * We account for the space used by the hdr and the arc buf individually
2232  * so that we can add and remove them from the refcount individually.
2233  */
2234 static void
arc_evictable_space_increment(arc_buf_hdr_t * hdr,arc_state_t * state)2235 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2236 {
2237 	arc_buf_contents_t type = arc_buf_type(hdr);
2238 
2239 	ASSERT(HDR_HAS_L1HDR(hdr));
2240 
2241 	if (GHOST_STATE(state)) {
2242 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2243 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2244 		ASSERT(!HDR_HAS_RABD(hdr));
2245 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2246 		    HDR_GET_LSIZE(hdr), hdr);
2247 		return;
2248 	}
2249 
2250 	if (hdr->b_l1hdr.b_pabd != NULL) {
2251 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2252 		    arc_hdr_size(hdr), hdr);
2253 	}
2254 	if (HDR_HAS_RABD(hdr)) {
2255 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2256 		    HDR_GET_PSIZE(hdr), hdr);
2257 	}
2258 
2259 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2260 	    buf = buf->b_next) {
2261 		if (ARC_BUF_SHARED(buf))
2262 			continue;
2263 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
2264 		    arc_buf_size(buf), buf);
2265 	}
2266 }
2267 
2268 /*
2269  * Decrement the amount of evictable space in the arc_state_t's refcount.
2270  * We account for the space used by the hdr and the arc buf individually
2271  * so that we can add and remove them from the refcount individually.
2272  */
2273 static void
arc_evictable_space_decrement(arc_buf_hdr_t * hdr,arc_state_t * state)2274 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2275 {
2276 	arc_buf_contents_t type = arc_buf_type(hdr);
2277 
2278 	ASSERT(HDR_HAS_L1HDR(hdr));
2279 
2280 	if (GHOST_STATE(state)) {
2281 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2282 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2283 		ASSERT(!HDR_HAS_RABD(hdr));
2284 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2285 		    HDR_GET_LSIZE(hdr), hdr);
2286 		return;
2287 	}
2288 
2289 	if (hdr->b_l1hdr.b_pabd != NULL) {
2290 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2291 		    arc_hdr_size(hdr), hdr);
2292 	}
2293 	if (HDR_HAS_RABD(hdr)) {
2294 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2295 		    HDR_GET_PSIZE(hdr), hdr);
2296 	}
2297 
2298 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2299 	    buf = buf->b_next) {
2300 		if (ARC_BUF_SHARED(buf))
2301 			continue;
2302 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2303 		    arc_buf_size(buf), buf);
2304 	}
2305 }
2306 
2307 /*
2308  * Add a reference to this hdr indicating that someone is actively
2309  * referencing that memory. When the refcount transitions from 0 to 1,
2310  * we remove it from the respective arc_state_t list to indicate that
2311  * it is not evictable.
2312  */
2313 static void
add_reference(arc_buf_hdr_t * hdr,const void * tag)2314 add_reference(arc_buf_hdr_t *hdr, const void *tag)
2315 {
2316 	arc_state_t *state = hdr->b_l1hdr.b_state;
2317 
2318 	ASSERT(HDR_HAS_L1HDR(hdr));
2319 	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
2320 		ASSERT(state == arc_anon);
2321 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2322 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2323 	}
2324 
2325 	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2326 	    state != arc_anon && state != arc_l2c_only) {
2327 		/* We don't use the L2-only state list. */
2328 		multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
2329 		arc_evictable_space_decrement(hdr, state);
2330 	}
2331 }
2332 
2333 /*
2334  * Remove a reference from this hdr. When the reference transitions from
2335  * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2336  * list making it eligible for eviction.
2337  */
2338 static int
remove_reference(arc_buf_hdr_t * hdr,const void * tag)2339 remove_reference(arc_buf_hdr_t *hdr, const void *tag)
2340 {
2341 	int cnt;
2342 	arc_state_t *state = hdr->b_l1hdr.b_state;
2343 
2344 	ASSERT(HDR_HAS_L1HDR(hdr));
2345 	ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));
2346 	ASSERT(!GHOST_STATE(state));	/* arc_l2c_only counts as a ghost. */
2347 
2348 	if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
2349 		return (cnt);
2350 
2351 	if (state == arc_anon) {
2352 		arc_hdr_destroy(hdr);
2353 		return (0);
2354 	}
2355 	if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
2356 		arc_change_state(arc_anon, hdr);
2357 		arc_hdr_destroy(hdr);
2358 		return (0);
2359 	}
2360 	multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2361 	arc_evictable_space_increment(hdr, state);
2362 	return (0);
2363 }
2364 
2365 /*
2366  * Returns detailed information about a specific arc buffer.  When the
2367  * state_index argument is set the function will calculate the arc header
2368  * list position for its arc state.  Since this requires a linear traversal
2369  * callers are strongly encourage not to do this.  However, it can be helpful
2370  * for targeted analysis so the functionality is provided.
2371  */
2372 void
arc_buf_info(arc_buf_t * ab,arc_buf_info_t * abi,int state_index)2373 arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
2374 {
2375 	(void) state_index;
2376 	arc_buf_hdr_t *hdr = ab->b_hdr;
2377 	l1arc_buf_hdr_t *l1hdr = NULL;
2378 	l2arc_buf_hdr_t *l2hdr = NULL;
2379 	arc_state_t *state = NULL;
2380 
2381 	memset(abi, 0, sizeof (arc_buf_info_t));
2382 
2383 	if (hdr == NULL)
2384 		return;
2385 
2386 	abi->abi_flags = hdr->b_flags;
2387 
2388 	if (HDR_HAS_L1HDR(hdr)) {
2389 		l1hdr = &hdr->b_l1hdr;
2390 		state = l1hdr->b_state;
2391 	}
2392 	if (HDR_HAS_L2HDR(hdr))
2393 		l2hdr = &hdr->b_l2hdr;
2394 
2395 	if (l1hdr) {
2396 		abi->abi_bufcnt = 0;
2397 		for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
2398 			abi->abi_bufcnt++;
2399 		abi->abi_access = l1hdr->b_arc_access;
2400 		abi->abi_mru_hits = l1hdr->b_mru_hits;
2401 		abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2402 		abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2403 		abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2404 		abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
2405 	}
2406 
2407 	if (l2hdr) {
2408 		abi->abi_l2arc_dattr = l2hdr->b_daddr;
2409 		abi->abi_l2arc_hits = l2hdr->b_hits;
2410 	}
2411 
2412 	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2413 	abi->abi_state_contents = arc_buf_type(hdr);
2414 	abi->abi_size = arc_hdr_size(hdr);
2415 }
2416 
2417 /*
2418  * Move the supplied buffer to the indicated state. The hash lock
2419  * for the buffer must be held by the caller.
2420  */
2421 static void
arc_change_state(arc_state_t * new_state,arc_buf_hdr_t * hdr)2422 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
2423 {
2424 	arc_state_t *old_state;
2425 	int64_t refcnt;
2426 	boolean_t update_old, update_new;
2427 	arc_buf_contents_t type = arc_buf_type(hdr);
2428 
2429 	/*
2430 	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2431 	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
2432 	 * L1 hdr doesn't always exist when we change state to arc_anon before
2433 	 * destroying a header, in which case reallocating to add the L1 hdr is
2434 	 * pointless.
2435 	 */
2436 	if (HDR_HAS_L1HDR(hdr)) {
2437 		old_state = hdr->b_l1hdr.b_state;
2438 		refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2439 		update_old = (hdr->b_l1hdr.b_buf != NULL ||
2440 		    hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
2441 
2442 		IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
2443 		IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
2444 		IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
2445 		    ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
2446 	} else {
2447 		old_state = arc_l2c_only;
2448 		refcnt = 0;
2449 		update_old = B_FALSE;
2450 	}
2451 	update_new = update_old;
2452 	if (GHOST_STATE(old_state))
2453 		update_old = B_TRUE;
2454 	if (GHOST_STATE(new_state))
2455 		update_new = B_TRUE;
2456 
2457 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2458 	ASSERT3P(new_state, !=, old_state);
2459 
2460 	/*
2461 	 * If this buffer is evictable, transfer it from the
2462 	 * old state list to the new state list.
2463 	 */
2464 	if (refcnt == 0) {
2465 		if (old_state != arc_anon && old_state != arc_l2c_only) {
2466 			ASSERT(HDR_HAS_L1HDR(hdr));
2467 			/* remove_reference() saves on insert. */
2468 			if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2469 				multilist_remove(&old_state->arcs_list[type],
2470 				    hdr);
2471 				arc_evictable_space_decrement(hdr, old_state);
2472 			}
2473 		}
2474 		if (new_state != arc_anon && new_state != arc_l2c_only) {
2475 			/*
2476 			 * An L1 header always exists here, since if we're
2477 			 * moving to some L1-cached state (i.e. not l2c_only or
2478 			 * anonymous), we realloc the header to add an L1hdr
2479 			 * beforehand.
2480 			 */
2481 			ASSERT(HDR_HAS_L1HDR(hdr));
2482 			multilist_insert(&new_state->arcs_list[type], hdr);
2483 			arc_evictable_space_increment(hdr, new_state);
2484 		}
2485 	}
2486 
2487 	ASSERT(!HDR_EMPTY(hdr));
2488 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2489 		buf_hash_remove(hdr);
2490 
2491 	/* adjust state sizes (ignore arc_l2c_only) */
2492 
2493 	if (update_new && new_state != arc_l2c_only) {
2494 		ASSERT(HDR_HAS_L1HDR(hdr));
2495 		if (GHOST_STATE(new_state)) {
2496 
2497 			/*
2498 			 * When moving a header to a ghost state, we first
2499 			 * remove all arc buffers. Thus, we'll have no arc
2500 			 * buffer to use for the reference. As a result, we
2501 			 * use the arc header pointer for the reference.
2502 			 */
2503 			(void) zfs_refcount_add_many(
2504 			    &new_state->arcs_size[type],
2505 			    HDR_GET_LSIZE(hdr), hdr);
2506 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2507 			ASSERT(!HDR_HAS_RABD(hdr));
2508 		} else {
2509 
2510 			/*
2511 			 * Each individual buffer holds a unique reference,
2512 			 * thus we must remove each of these references one
2513 			 * at a time.
2514 			 */
2515 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2516 			    buf = buf->b_next) {
2517 
2518 				/*
2519 				 * When the arc_buf_t is sharing the data
2520 				 * block with the hdr, the owner of the
2521 				 * reference belongs to the hdr. Only
2522 				 * add to the refcount if the arc_buf_t is
2523 				 * not shared.
2524 				 */
2525 				if (ARC_BUF_SHARED(buf))
2526 					continue;
2527 
2528 				(void) zfs_refcount_add_many(
2529 				    &new_state->arcs_size[type],
2530 				    arc_buf_size(buf), buf);
2531 			}
2532 
2533 			if (hdr->b_l1hdr.b_pabd != NULL) {
2534 				(void) zfs_refcount_add_many(
2535 				    &new_state->arcs_size[type],
2536 				    arc_hdr_size(hdr), hdr);
2537 			}
2538 
2539 			if (HDR_HAS_RABD(hdr)) {
2540 				(void) zfs_refcount_add_many(
2541 				    &new_state->arcs_size[type],
2542 				    HDR_GET_PSIZE(hdr), hdr);
2543 			}
2544 		}
2545 	}
2546 
2547 	if (update_old && old_state != arc_l2c_only) {
2548 		ASSERT(HDR_HAS_L1HDR(hdr));
2549 		if (GHOST_STATE(old_state)) {
2550 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2551 			ASSERT(!HDR_HAS_RABD(hdr));
2552 
2553 			/*
2554 			 * When moving a header off of a ghost state,
2555 			 * the header will not contain any arc buffers.
2556 			 * We use the arc header pointer for the reference
2557 			 * which is exactly what we did when we put the
2558 			 * header on the ghost state.
2559 			 */
2560 
2561 			(void) zfs_refcount_remove_many(
2562 			    &old_state->arcs_size[type],
2563 			    HDR_GET_LSIZE(hdr), hdr);
2564 		} else {
2565 
2566 			/*
2567 			 * Each individual buffer holds a unique reference,
2568 			 * thus we must remove each of these references one
2569 			 * at a time.
2570 			 */
2571 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2572 			    buf = buf->b_next) {
2573 
2574 				/*
2575 				 * When the arc_buf_t is sharing the data
2576 				 * block with the hdr, the owner of the
2577 				 * reference belongs to the hdr. Only
2578 				 * add to the refcount if the arc_buf_t is
2579 				 * not shared.
2580 				 */
2581 				if (ARC_BUF_SHARED(buf))
2582 					continue;
2583 
2584 				(void) zfs_refcount_remove_many(
2585 				    &old_state->arcs_size[type],
2586 				    arc_buf_size(buf), buf);
2587 			}
2588 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
2589 			    HDR_HAS_RABD(hdr));
2590 
2591 			if (hdr->b_l1hdr.b_pabd != NULL) {
2592 				(void) zfs_refcount_remove_many(
2593 				    &old_state->arcs_size[type],
2594 				    arc_hdr_size(hdr), hdr);
2595 			}
2596 
2597 			if (HDR_HAS_RABD(hdr)) {
2598 				(void) zfs_refcount_remove_many(
2599 				    &old_state->arcs_size[type],
2600 				    HDR_GET_PSIZE(hdr), hdr);
2601 			}
2602 		}
2603 	}
2604 
2605 	if (HDR_HAS_L1HDR(hdr)) {
2606 		hdr->b_l1hdr.b_state = new_state;
2607 
2608 		if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
2609 			l2arc_hdr_arcstats_decrement_state(hdr);
2610 			hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
2611 			l2arc_hdr_arcstats_increment_state(hdr);
2612 		}
2613 	}
2614 }
2615 
2616 void
arc_space_consume(uint64_t space,arc_space_type_t type)2617 arc_space_consume(uint64_t space, arc_space_type_t type)
2618 {
2619 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2620 
2621 	switch (type) {
2622 	default:
2623 		break;
2624 	case ARC_SPACE_DATA:
2625 		ARCSTAT_INCR(arcstat_data_size, space);
2626 		break;
2627 	case ARC_SPACE_META:
2628 		ARCSTAT_INCR(arcstat_metadata_size, space);
2629 		break;
2630 	case ARC_SPACE_BONUS:
2631 		ARCSTAT_INCR(arcstat_bonus_size, space);
2632 		break;
2633 	case ARC_SPACE_DNODE:
2634 		ARCSTAT_INCR(arcstat_dnode_size, space);
2635 		break;
2636 	case ARC_SPACE_DBUF:
2637 		ARCSTAT_INCR(arcstat_dbuf_size, space);
2638 		break;
2639 	case ARC_SPACE_HDRS:
2640 		ARCSTAT_INCR(arcstat_hdr_size, space);
2641 		break;
2642 	case ARC_SPACE_L2HDRS:
2643 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
2644 		break;
2645 	case ARC_SPACE_ABD_CHUNK_WASTE:
2646 		/*
2647 		 * Note: this includes space wasted by all scatter ABD's, not
2648 		 * just those allocated by the ARC.  But the vast majority of
2649 		 * scatter ABD's come from the ARC, because other users are
2650 		 * very short-lived.
2651 		 */
2652 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
2653 		break;
2654 	}
2655 
2656 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2657 		ARCSTAT_INCR(arcstat_meta_used, space);
2658 
2659 	aggsum_add(&arc_sums.arcstat_size, space);
2660 }
2661 
2662 void
arc_space_return(uint64_t space,arc_space_type_t type)2663 arc_space_return(uint64_t space, arc_space_type_t type)
2664 {
2665 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2666 
2667 	switch (type) {
2668 	default:
2669 		break;
2670 	case ARC_SPACE_DATA:
2671 		ARCSTAT_INCR(arcstat_data_size, -space);
2672 		break;
2673 	case ARC_SPACE_META:
2674 		ARCSTAT_INCR(arcstat_metadata_size, -space);
2675 		break;
2676 	case ARC_SPACE_BONUS:
2677 		ARCSTAT_INCR(arcstat_bonus_size, -space);
2678 		break;
2679 	case ARC_SPACE_DNODE:
2680 		ARCSTAT_INCR(arcstat_dnode_size, -space);
2681 		break;
2682 	case ARC_SPACE_DBUF:
2683 		ARCSTAT_INCR(arcstat_dbuf_size, -space);
2684 		break;
2685 	case ARC_SPACE_HDRS:
2686 		ARCSTAT_INCR(arcstat_hdr_size, -space);
2687 		break;
2688 	case ARC_SPACE_L2HDRS:
2689 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
2690 		break;
2691 	case ARC_SPACE_ABD_CHUNK_WASTE:
2692 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
2693 		break;
2694 	}
2695 
2696 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2697 		ARCSTAT_INCR(arcstat_meta_used, -space);
2698 
2699 	ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
2700 	aggsum_add(&arc_sums.arcstat_size, -space);
2701 }
2702 
2703 /*
2704  * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2705  * with the hdr's b_pabd.
2706  */
2707 static boolean_t
arc_can_share(arc_buf_hdr_t * hdr,arc_buf_t * buf)2708 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2709 {
2710 	/*
2711 	 * The criteria for sharing a hdr's data are:
2712 	 * 1. the buffer is not encrypted
2713 	 * 2. the hdr's compression matches the buf's compression
2714 	 * 3. the hdr doesn't need to be byteswapped
2715 	 * 4. the hdr isn't already being shared
2716 	 * 5. the buf is either compressed or it is the last buf in the hdr list
2717 	 *
2718 	 * Criterion #5 maintains the invariant that shared uncompressed
2719 	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2720 	 * might ask, "if a compressed buf is allocated first, won't that be the
2721 	 * last thing in the list?", but in that case it's impossible to create
2722 	 * a shared uncompressed buf anyway (because the hdr must be compressed
2723 	 * to have the compressed buf). You might also think that #3 is
2724 	 * sufficient to make this guarantee, however it's possible
2725 	 * (specifically in the rare L2ARC write race mentioned in
2726 	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2727 	 * is shareable, but wasn't at the time of its allocation. Rather than
2728 	 * allow a new shared uncompressed buf to be created and then shuffle
2729 	 * the list around to make it the last element, this simply disallows
2730 	 * sharing if the new buf isn't the first to be added.
2731 	 */
2732 	ASSERT3P(buf->b_hdr, ==, hdr);
2733 	boolean_t hdr_compressed =
2734 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
2735 	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2736 	return (!ARC_BUF_ENCRYPTED(buf) &&
2737 	    buf_compressed == hdr_compressed &&
2738 	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2739 	    !HDR_SHARED_DATA(hdr) &&
2740 	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2741 }
2742 
2743 /*
2744  * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2745  * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2746  * copy was made successfully, or an error code otherwise.
2747  */
2748 static int
arc_buf_alloc_impl(arc_buf_hdr_t * hdr,spa_t * spa,const zbookmark_phys_t * zb,const void * tag,boolean_t encrypted,boolean_t compressed,boolean_t noauth,boolean_t fill,arc_buf_t ** ret)2749 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
2750     const void *tag, boolean_t encrypted, boolean_t compressed,
2751     boolean_t noauth, boolean_t fill, arc_buf_t **ret)
2752 {
2753 	arc_buf_t *buf;
2754 	arc_fill_flags_t flags = ARC_FILL_LOCKED;
2755 
2756 	ASSERT(HDR_HAS_L1HDR(hdr));
2757 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2758 	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2759 	    hdr->b_type == ARC_BUFC_METADATA);
2760 	ASSERT3P(ret, !=, NULL);
2761 	ASSERT3P(*ret, ==, NULL);
2762 	IMPLY(encrypted, compressed);
2763 
2764 	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2765 	buf->b_hdr = hdr;
2766 	buf->b_data = NULL;
2767 	buf->b_next = hdr->b_l1hdr.b_buf;
2768 	buf->b_flags = 0;
2769 
2770 	add_reference(hdr, tag);
2771 
2772 	/*
2773 	 * We're about to change the hdr's b_flags. We must either
2774 	 * hold the hash_lock or be undiscoverable.
2775 	 */
2776 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2777 
2778 	/*
2779 	 * Only honor requests for compressed bufs if the hdr is actually
2780 	 * compressed. This must be overridden if the buffer is encrypted since
2781 	 * encrypted buffers cannot be decompressed.
2782 	 */
2783 	if (encrypted) {
2784 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2785 		buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED;
2786 		flags |= ARC_FILL_COMPRESSED | ARC_FILL_ENCRYPTED;
2787 	} else if (compressed &&
2788 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
2789 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2790 		flags |= ARC_FILL_COMPRESSED;
2791 	}
2792 
2793 	if (noauth) {
2794 		ASSERT0(encrypted);
2795 		flags |= ARC_FILL_NOAUTH;
2796 	}
2797 
2798 	/*
2799 	 * If the hdr's data can be shared then we share the data buffer and
2800 	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2801 	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2802 	 * buffer to store the buf's data.
2803 	 *
2804 	 * There are two additional restrictions here because we're sharing
2805 	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2806 	 * actively involved in an L2ARC write, because if this buf is used by
2807 	 * an arc_write() then the hdr's data buffer will be released when the
2808 	 * write completes, even though the L2ARC write might still be using it.
2809 	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2810 	 * need to be ABD-aware.  It must be allocated via
2811 	 * zio_[data_]buf_alloc(), not as a page, because we need to be able
2812 	 * to abd_release_ownership_of_buf(), which isn't allowed on "linear
2813 	 * page" buffers because the ABD code needs to handle freeing them
2814 	 * specially.
2815 	 */
2816 	boolean_t can_share = arc_can_share(hdr, buf) &&
2817 	    !HDR_L2_WRITING(hdr) &&
2818 	    hdr->b_l1hdr.b_pabd != NULL &&
2819 	    abd_is_linear(hdr->b_l1hdr.b_pabd) &&
2820 	    !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
2821 
2822 	/* Set up b_data and sharing */
2823 	if (can_share) {
2824 		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2825 		buf->b_flags |= ARC_BUF_FLAG_SHARED;
2826 		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2827 	} else {
2828 		buf->b_data =
2829 		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2830 		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2831 	}
2832 	VERIFY3P(buf->b_data, !=, NULL);
2833 
2834 	hdr->b_l1hdr.b_buf = buf;
2835 
2836 	/*
2837 	 * If the user wants the data from the hdr, we need to either copy or
2838 	 * decompress the data.
2839 	 */
2840 	if (fill) {
2841 		ASSERT3P(zb, !=, NULL);
2842 		return (arc_buf_fill(buf, spa, zb, flags));
2843 	}
2844 
2845 	return (0);
2846 }
2847 
2848 static const char *arc_onloan_tag = "onloan";
2849 
2850 static inline void
arc_loaned_bytes_update(int64_t delta)2851 arc_loaned_bytes_update(int64_t delta)
2852 {
2853 	atomic_add_64(&arc_loaned_bytes, delta);
2854 
2855 	/* assert that it did not wrap around */
2856 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
2857 }
2858 
2859 /*
2860  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
2861  * flight data by arc_tempreserve_space() until they are "returned". Loaned
2862  * buffers must be returned to the arc before they can be used by the DMU or
2863  * freed.
2864  */
2865 arc_buf_t *
arc_loan_buf(spa_t * spa,boolean_t is_metadata,int size)2866 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
2867 {
2868 	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
2869 	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
2870 
2871 	arc_loaned_bytes_update(arc_buf_size(buf));
2872 
2873 	return (buf);
2874 }
2875 
2876 arc_buf_t *
arc_loan_compressed_buf(spa_t * spa,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)2877 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
2878     enum zio_compress compression_type, uint8_t complevel)
2879 {
2880 	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
2881 	    psize, lsize, compression_type, complevel);
2882 
2883 	arc_loaned_bytes_update(arc_buf_size(buf));
2884 
2885 	return (buf);
2886 }
2887 
2888 arc_buf_t *
arc_loan_raw_buf(spa_t * spa,uint64_t dsobj,boolean_t byteorder,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac,dmu_object_type_t ot,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)2889 arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
2890     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
2891     dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
2892     enum zio_compress compression_type, uint8_t complevel)
2893 {
2894 	arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
2895 	    byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
2896 	    complevel);
2897 
2898 	atomic_add_64(&arc_loaned_bytes, psize);
2899 	return (buf);
2900 }
2901 
2902 
2903 /*
2904  * Return a loaned arc buffer to the arc.
2905  */
2906 void
arc_return_buf(arc_buf_t * buf,const void * tag)2907 arc_return_buf(arc_buf_t *buf, const void *tag)
2908 {
2909 	arc_buf_hdr_t *hdr = buf->b_hdr;
2910 
2911 	ASSERT3P(buf->b_data, !=, NULL);
2912 	ASSERT(HDR_HAS_L1HDR(hdr));
2913 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2914 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2915 
2916 	arc_loaned_bytes_update(-arc_buf_size(buf));
2917 }
2918 
2919 /* Detach an arc_buf from a dbuf (tag) */
2920 void
arc_loan_inuse_buf(arc_buf_t * buf,const void * tag)2921 arc_loan_inuse_buf(arc_buf_t *buf, const void *tag)
2922 {
2923 	arc_buf_hdr_t *hdr = buf->b_hdr;
2924 
2925 	ASSERT3P(buf->b_data, !=, NULL);
2926 	ASSERT(HDR_HAS_L1HDR(hdr));
2927 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2928 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2929 
2930 	arc_loaned_bytes_update(arc_buf_size(buf));
2931 }
2932 
2933 static void
l2arc_free_abd_on_write(abd_t * abd,size_t size,arc_buf_contents_t type)2934 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
2935 {
2936 	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
2937 
2938 	df->l2df_abd = abd;
2939 	df->l2df_size = size;
2940 	df->l2df_type = type;
2941 	mutex_enter(&l2arc_free_on_write_mtx);
2942 	list_insert_head(l2arc_free_on_write, df);
2943 	mutex_exit(&l2arc_free_on_write_mtx);
2944 }
2945 
2946 static void
arc_hdr_free_on_write(arc_buf_hdr_t * hdr,boolean_t free_rdata)2947 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
2948 {
2949 	arc_state_t *state = hdr->b_l1hdr.b_state;
2950 	arc_buf_contents_t type = arc_buf_type(hdr);
2951 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
2952 
2953 	/* protected by hash lock, if in the hash table */
2954 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2955 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2956 		ASSERT(state != arc_anon && state != arc_l2c_only);
2957 
2958 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
2959 		    size, hdr);
2960 	}
2961 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
2962 	if (type == ARC_BUFC_METADATA) {
2963 		arc_space_return(size, ARC_SPACE_META);
2964 	} else {
2965 		ASSERT(type == ARC_BUFC_DATA);
2966 		arc_space_return(size, ARC_SPACE_DATA);
2967 	}
2968 
2969 	if (free_rdata) {
2970 		l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
2971 	} else {
2972 		l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2973 	}
2974 }
2975 
2976 /*
2977  * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2978  * data buffer, we transfer the refcount ownership to the hdr and update
2979  * the appropriate kstats.
2980  */
2981 static void
arc_share_buf(arc_buf_hdr_t * hdr,arc_buf_t * buf)2982 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2983 {
2984 	ASSERT(arc_can_share(hdr, buf));
2985 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2986 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
2987 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2988 
2989 	/*
2990 	 * Start sharing the data buffer. We transfer the
2991 	 * refcount ownership to the hdr since it always owns
2992 	 * the refcount whenever an arc_buf_t is shared.
2993 	 */
2994 	zfs_refcount_transfer_ownership_many(
2995 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2996 	    arc_hdr_size(hdr), buf, hdr);
2997 	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2998 	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2999 	    HDR_ISTYPE_METADATA(hdr));
3000 	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
3001 	buf->b_flags |= ARC_BUF_FLAG_SHARED;
3002 
3003 	/*
3004 	 * Since we've transferred ownership to the hdr we need
3005 	 * to increment its compressed and uncompressed kstats and
3006 	 * decrement the overhead size.
3007 	 */
3008 	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
3009 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3010 	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
3011 }
3012 
3013 static void
arc_unshare_buf(arc_buf_hdr_t * hdr,arc_buf_t * buf)3014 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3015 {
3016 	ASSERT(arc_buf_is_shared(buf));
3017 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3018 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3019 
3020 	/*
3021 	 * We are no longer sharing this buffer so we need
3022 	 * to transfer its ownership to the rightful owner.
3023 	 */
3024 	zfs_refcount_transfer_ownership_many(
3025 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
3026 	    arc_hdr_size(hdr), hdr, buf);
3027 	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3028 	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3029 	abd_free(hdr->b_l1hdr.b_pabd);
3030 	hdr->b_l1hdr.b_pabd = NULL;
3031 	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
3032 
3033 	/*
3034 	 * Since the buffer is no longer shared between
3035 	 * the arc buf and the hdr, count it as overhead.
3036 	 */
3037 	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3038 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3039 	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3040 }
3041 
3042 /*
3043  * Remove an arc_buf_t from the hdr's buf list and return the last
3044  * arc_buf_t on the list. If no buffers remain on the list then return
3045  * NULL.
3046  */
3047 static arc_buf_t *
arc_buf_remove(arc_buf_hdr_t * hdr,arc_buf_t * buf)3048 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3049 {
3050 	ASSERT(HDR_HAS_L1HDR(hdr));
3051 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3052 
3053 	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3054 	arc_buf_t *lastbuf = NULL;
3055 
3056 	/*
3057 	 * Remove the buf from the hdr list and locate the last
3058 	 * remaining buffer on the list.
3059 	 */
3060 	while (*bufp != NULL) {
3061 		if (*bufp == buf)
3062 			*bufp = buf->b_next;
3063 
3064 		/*
3065 		 * If we've removed a buffer in the middle of
3066 		 * the list then update the lastbuf and update
3067 		 * bufp.
3068 		 */
3069 		if (*bufp != NULL) {
3070 			lastbuf = *bufp;
3071 			bufp = &(*bufp)->b_next;
3072 		}
3073 	}
3074 	buf->b_next = NULL;
3075 	ASSERT3P(lastbuf, !=, buf);
3076 	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3077 
3078 	return (lastbuf);
3079 }
3080 
3081 /*
3082  * Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
3083  * list and free it.
3084  */
3085 static void
arc_buf_destroy_impl(arc_buf_t * buf)3086 arc_buf_destroy_impl(arc_buf_t *buf)
3087 {
3088 	arc_buf_hdr_t *hdr = buf->b_hdr;
3089 
3090 	/*
3091 	 * Free up the data associated with the buf but only if we're not
3092 	 * sharing this with the hdr. If we are sharing it with the hdr, the
3093 	 * hdr is responsible for doing the free.
3094 	 */
3095 	if (buf->b_data != NULL) {
3096 		/*
3097 		 * We're about to change the hdr's b_flags. We must either
3098 		 * hold the hash_lock or be undiscoverable.
3099 		 */
3100 		ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3101 
3102 		arc_cksum_verify(buf);
3103 		arc_buf_unwatch(buf);
3104 
3105 		if (ARC_BUF_SHARED(buf)) {
3106 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3107 		} else {
3108 			ASSERT(!arc_buf_is_shared(buf));
3109 			uint64_t size = arc_buf_size(buf);
3110 			arc_free_data_buf(hdr, buf->b_data, size, buf);
3111 			ARCSTAT_INCR(arcstat_overhead_size, -size);
3112 		}
3113 		buf->b_data = NULL;
3114 
3115 		/*
3116 		 * If we have no more encrypted buffers and we've already
3117 		 * gotten a copy of the decrypted data we can free b_rabd
3118 		 * to save some space.
3119 		 */
3120 		if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
3121 		    hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
3122 			arc_buf_t *b;
3123 			for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
3124 				if (b != buf && ARC_BUF_ENCRYPTED(b))
3125 					break;
3126 			}
3127 			if (b == NULL)
3128 				arc_hdr_free_abd(hdr, B_TRUE);
3129 		}
3130 	}
3131 
3132 	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3133 
3134 	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3135 		/*
3136 		 * If the current arc_buf_t is sharing its data buffer with the
3137 		 * hdr, then reassign the hdr's b_pabd to share it with the new
3138 		 * buffer at the end of the list. The shared buffer is always
3139 		 * the last one on the hdr's buffer list.
3140 		 *
3141 		 * There is an equivalent case for compressed bufs, but since
3142 		 * they aren't guaranteed to be the last buf in the list and
3143 		 * that is an exceedingly rare case, we just allow that space be
3144 		 * wasted temporarily. We must also be careful not to share
3145 		 * encrypted buffers, since they cannot be shared.
3146 		 */
3147 		if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) {
3148 			/* Only one buf can be shared at once */
3149 			ASSERT(!arc_buf_is_shared(lastbuf));
3150 			/* hdr is uncompressed so can't have compressed buf */
3151 			ASSERT(!ARC_BUF_COMPRESSED(lastbuf));
3152 
3153 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3154 			arc_hdr_free_abd(hdr, B_FALSE);
3155 
3156 			/*
3157 			 * We must setup a new shared block between the
3158 			 * last buffer and the hdr. The data would have
3159 			 * been allocated by the arc buf so we need to transfer
3160 			 * ownership to the hdr since it's now being shared.
3161 			 */
3162 			arc_share_buf(hdr, lastbuf);
3163 		}
3164 	} else if (HDR_SHARED_DATA(hdr)) {
3165 		/*
3166 		 * Uncompressed shared buffers are always at the end
3167 		 * of the list. Compressed buffers don't have the
3168 		 * same requirements. This makes it hard to
3169 		 * simply assert that the lastbuf is shared so
3170 		 * we rely on the hdr's compression flags to determine
3171 		 * if we have a compressed, shared buffer.
3172 		 */
3173 		ASSERT3P(lastbuf, !=, NULL);
3174 		ASSERT(arc_buf_is_shared(lastbuf) ||
3175 		    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
3176 	}
3177 
3178 	/*
3179 	 * Free the checksum if we're removing the last uncompressed buf from
3180 	 * this hdr.
3181 	 */
3182 	if (!arc_hdr_has_uncompressed_buf(hdr)) {
3183 		arc_cksum_free(hdr);
3184 	}
3185 
3186 	/* clean up the buf */
3187 	buf->b_hdr = NULL;
3188 	kmem_cache_free(buf_cache, buf);
3189 }
3190 
3191 static void
arc_hdr_alloc_abd(arc_buf_hdr_t * hdr,int alloc_flags)3192 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
3193 {
3194 	uint64_t size;
3195 	boolean_t alloc_rdata = ((alloc_flags & ARC_HDR_ALLOC_RDATA) != 0);
3196 
3197 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3198 	ASSERT(HDR_HAS_L1HDR(hdr));
3199 	ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);
3200 	IMPLY(alloc_rdata, HDR_PROTECTED(hdr));
3201 
3202 	if (alloc_rdata) {
3203 		size = HDR_GET_PSIZE(hdr);
3204 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
3205 		hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
3206 		    alloc_flags);
3207 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
3208 		ARCSTAT_INCR(arcstat_raw_size, size);
3209 	} else {
3210 		size = arc_hdr_size(hdr);
3211 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3212 		hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
3213 		    alloc_flags);
3214 		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3215 	}
3216 
3217 	ARCSTAT_INCR(arcstat_compressed_size, size);
3218 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3219 }
3220 
3221 static void
arc_hdr_free_abd(arc_buf_hdr_t * hdr,boolean_t free_rdata)3222 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
3223 {
3224 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
3225 
3226 	ASSERT(HDR_HAS_L1HDR(hdr));
3227 	ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
3228 	IMPLY(free_rdata, HDR_HAS_RABD(hdr));
3229 
3230 	/*
3231 	 * If the hdr is currently being written to the l2arc then
3232 	 * we defer freeing the data by adding it to the l2arc_free_on_write
3233 	 * list. The l2arc will free the data once it's finished
3234 	 * writing it to the l2arc device.
3235 	 */
3236 	if (HDR_L2_WRITING(hdr)) {
3237 		arc_hdr_free_on_write(hdr, free_rdata);
3238 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
3239 	} else if (free_rdata) {
3240 		arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
3241 	} else {
3242 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
3243 	}
3244 
3245 	if (free_rdata) {
3246 		hdr->b_crypt_hdr.b_rabd = NULL;
3247 		ARCSTAT_INCR(arcstat_raw_size, -size);
3248 	} else {
3249 		hdr->b_l1hdr.b_pabd = NULL;
3250 	}
3251 
3252 	if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
3253 		hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3254 
3255 	ARCSTAT_INCR(arcstat_compressed_size, -size);
3256 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3257 }
3258 
3259 /*
3260  * Allocate empty anonymous ARC header.  The header will get its identity
3261  * assigned and buffers attached later as part of read or write operations.
3262  *
3263  * In case of read arc_read() assigns header its identify (b_dva + b_birth),
3264  * inserts it into ARC hash to become globally visible and allocates physical
3265  * (b_pabd) or raw (b_rabd) ABD buffer to read into from disk.  On disk read
3266  * completion arc_read_done() allocates ARC buffer(s) as needed, potentially
3267  * sharing one of them with the physical ABD buffer.
3268  *
3269  * In case of write arc_alloc_buf() allocates ARC buffer to be filled with
3270  * data.  Then after compression and/or encryption arc_write_ready() allocates
3271  * and fills (or potentially shares) physical (b_pabd) or raw (b_rabd) ABD
3272  * buffer.  On disk write completion arc_write_done() assigns the header its
3273  * new identity (b_dva + b_birth) and inserts into ARC hash.
3274  *
3275  * In case of partial overwrite the old data is read first as described. Then
3276  * arc_release() either allocates new anonymous ARC header and moves the ARC
3277  * buffer to it, or reuses the old ARC header by discarding its identity and
3278  * removing it from ARC hash.  After buffer modification normal write process
3279  * follows as described.
3280  */
3281 static arc_buf_hdr_t *
arc_hdr_alloc(uint64_t spa,int32_t psize,int32_t lsize,boolean_t protected,enum zio_compress compression_type,uint8_t complevel,arc_buf_contents_t type)3282 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3283     boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
3284     arc_buf_contents_t type)
3285 {
3286 	arc_buf_hdr_t *hdr;
3287 
3288 	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3289 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3290 
3291 	ASSERT(HDR_EMPTY(hdr));
3292 #ifdef ZFS_DEBUG
3293 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3294 #endif
3295 	HDR_SET_PSIZE(hdr, psize);
3296 	HDR_SET_LSIZE(hdr, lsize);
3297 	hdr->b_spa = spa;
3298 	hdr->b_type = type;
3299 	hdr->b_flags = 0;
3300 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3301 	arc_hdr_set_compress(hdr, compression_type);
3302 	hdr->b_complevel = complevel;
3303 	if (protected)
3304 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3305 
3306 	hdr->b_l1hdr.b_state = arc_anon;
3307 	hdr->b_l1hdr.b_arc_access = 0;
3308 	hdr->b_l1hdr.b_mru_hits = 0;
3309 	hdr->b_l1hdr.b_mru_ghost_hits = 0;
3310 	hdr->b_l1hdr.b_mfu_hits = 0;
3311 	hdr->b_l1hdr.b_mfu_ghost_hits = 0;
3312 	hdr->b_l1hdr.b_buf = NULL;
3313 
3314 	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3315 
3316 	return (hdr);
3317 }
3318 
3319 /*
3320  * Transition between the two allocation states for the arc_buf_hdr struct.
3321  * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3322  * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3323  * version is used when a cache buffer is only in the L2ARC in order to reduce
3324  * memory usage.
3325  */
3326 static arc_buf_hdr_t *
arc_hdr_realloc(arc_buf_hdr_t * hdr,kmem_cache_t * old,kmem_cache_t * new)3327 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3328 {
3329 	ASSERT(HDR_HAS_L2HDR(hdr));
3330 
3331 	arc_buf_hdr_t *nhdr;
3332 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3333 
3334 	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3335 	    (old == hdr_l2only_cache && new == hdr_full_cache));
3336 
3337 	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3338 
3339 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3340 	buf_hash_remove(hdr);
3341 
3342 	memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);
3343 
3344 	if (new == hdr_full_cache) {
3345 		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3346 		/*
3347 		 * arc_access and arc_change_state need to be aware that a
3348 		 * header has just come out of L2ARC, so we set its state to
3349 		 * l2c_only even though it's about to change.
3350 		 */
3351 		nhdr->b_l1hdr.b_state = arc_l2c_only;
3352 
3353 		/* Verify previous threads set to NULL before freeing */
3354 		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3355 		ASSERT(!HDR_HAS_RABD(hdr));
3356 	} else {
3357 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3358 #ifdef ZFS_DEBUG
3359 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3360 #endif
3361 
3362 		/*
3363 		 * If we've reached here, We must have been called from
3364 		 * arc_evict_hdr(), as such we should have already been
3365 		 * removed from any ghost list we were previously on
3366 		 * (which protects us from racing with arc_evict_state),
3367 		 * thus no locking is needed during this check.
3368 		 */
3369 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3370 
3371 		/*
3372 		 * A buffer must not be moved into the arc_l2c_only
3373 		 * state if it's not finished being written out to the
3374 		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3375 		 * might try to be accessed, even though it was removed.
3376 		 */
3377 		VERIFY(!HDR_L2_WRITING(hdr));
3378 		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3379 		ASSERT(!HDR_HAS_RABD(hdr));
3380 
3381 		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3382 	}
3383 	/*
3384 	 * The header has been reallocated so we need to re-insert it into any
3385 	 * lists it was on.
3386 	 */
3387 	(void) buf_hash_insert(nhdr, NULL);
3388 
3389 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3390 
3391 	mutex_enter(&dev->l2ad_mtx);
3392 
3393 	/*
3394 	 * We must place the realloc'ed header back into the list at
3395 	 * the same spot. Otherwise, if it's placed earlier in the list,
3396 	 * l2arc_write_buffers() could find it during the function's
3397 	 * write phase, and try to write it out to the l2arc.
3398 	 */
3399 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3400 	list_remove(&dev->l2ad_buflist, hdr);
3401 
3402 	mutex_exit(&dev->l2ad_mtx);
3403 
3404 	/*
3405 	 * Since we're using the pointer address as the tag when
3406 	 * incrementing and decrementing the l2ad_alloc refcount, we
3407 	 * must remove the old pointer (that we're about to destroy) and
3408 	 * add the new pointer to the refcount. Otherwise we'd remove
3409 	 * the wrong pointer address when calling arc_hdr_destroy() later.
3410 	 */
3411 
3412 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
3413 	    arc_hdr_size(hdr), hdr);
3414 	(void) zfs_refcount_add_many(&dev->l2ad_alloc,
3415 	    arc_hdr_size(nhdr), nhdr);
3416 
3417 	buf_discard_identity(hdr);
3418 	kmem_cache_free(old, hdr);
3419 
3420 	return (nhdr);
3421 }
3422 
3423 /*
3424  * This function is used by the send / receive code to convert a newly
3425  * allocated arc_buf_t to one that is suitable for a raw encrypted write. It
3426  * is also used to allow the root objset block to be updated without altering
3427  * its embedded MACs. Both block types will always be uncompressed so we do not
3428  * have to worry about compression type or psize.
3429  */
3430 void
arc_convert_to_raw(arc_buf_t * buf,uint64_t dsobj,boolean_t byteorder,dmu_object_type_t ot,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac)3431 arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder,
3432     dmu_object_type_t ot, const uint8_t *salt, const uint8_t *iv,
3433     const uint8_t *mac)
3434 {
3435 	arc_buf_hdr_t *hdr = buf->b_hdr;
3436 
3437 	ASSERT(ot == DMU_OT_DNODE || ot == DMU_OT_OBJSET);
3438 	ASSERT(HDR_HAS_L1HDR(hdr));
3439 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3440 
3441 	buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED);
3442 	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3443 	hdr->b_crypt_hdr.b_dsobj = dsobj;
3444 	hdr->b_crypt_hdr.b_ot = ot;
3445 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3446 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3447 	if (!arc_hdr_has_uncompressed_buf(hdr))
3448 		arc_cksum_free(hdr);
3449 
3450 	if (salt != NULL)
3451 		memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3452 	if (iv != NULL)
3453 		memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3454 	if (mac != NULL)
3455 		memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3456 }
3457 
3458 /*
3459  * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3460  * The buf is returned thawed since we expect the consumer to modify it.
3461  */
3462 arc_buf_t *
arc_alloc_buf(spa_t * spa,const void * tag,arc_buf_contents_t type,int32_t size)3463 arc_alloc_buf(spa_t *spa, const void *tag, arc_buf_contents_t type,
3464     int32_t size)
3465 {
3466 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3467 	    B_FALSE, ZIO_COMPRESS_OFF, 0, type);
3468 
3469 	arc_buf_t *buf = NULL;
3470 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
3471 	    B_FALSE, B_FALSE, &buf));
3472 	arc_buf_thaw(buf);
3473 
3474 	return (buf);
3475 }
3476 
3477 /*
3478  * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3479  * for bufs containing metadata.
3480  */
3481 arc_buf_t *
arc_alloc_compressed_buf(spa_t * spa,const void * tag,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)3482 arc_alloc_compressed_buf(spa_t *spa, const void *tag, uint64_t psize,
3483     uint64_t lsize, enum zio_compress compression_type, uint8_t complevel)
3484 {
3485 	ASSERT3U(lsize, >, 0);
3486 	ASSERT3U(lsize, >=, psize);
3487 	ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF);
3488 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3489 
3490 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3491 	    B_FALSE, compression_type, complevel, ARC_BUFC_DATA);
3492 
3493 	arc_buf_t *buf = NULL;
3494 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
3495 	    B_TRUE, B_FALSE, B_FALSE, &buf));
3496 	arc_buf_thaw(buf);
3497 
3498 	/*
3499 	 * To ensure that the hdr has the correct data in it if we call
3500 	 * arc_untransform() on this buf before it's been written to disk,
3501 	 * it's easiest if we just set up sharing between the buf and the hdr.
3502 	 */
3503 	arc_share_buf(hdr, buf);
3504 
3505 	return (buf);
3506 }
3507 
3508 arc_buf_t *
arc_alloc_raw_buf(spa_t * spa,const void * tag,uint64_t dsobj,boolean_t byteorder,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac,dmu_object_type_t ot,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)3509 arc_alloc_raw_buf(spa_t *spa, const void *tag, uint64_t dsobj,
3510     boolean_t byteorder, const uint8_t *salt, const uint8_t *iv,
3511     const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
3512     enum zio_compress compression_type, uint8_t complevel)
3513 {
3514 	arc_buf_hdr_t *hdr;
3515 	arc_buf_t *buf;
3516 	arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ?
3517 	    ARC_BUFC_METADATA : ARC_BUFC_DATA;
3518 
3519 	ASSERT3U(lsize, >, 0);
3520 	ASSERT3U(lsize, >=, psize);
3521 	ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF);
3522 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3523 
3524 	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
3525 	    compression_type, complevel, type);
3526 
3527 	hdr->b_crypt_hdr.b_dsobj = dsobj;
3528 	hdr->b_crypt_hdr.b_ot = ot;
3529 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3530 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3531 	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3532 	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3533 	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3534 
3535 	/*
3536 	 * This buffer will be considered encrypted even if the ot is not an
3537 	 * encrypted type. It will become authenticated instead in
3538 	 * arc_write_ready().
3539 	 */
3540 	buf = NULL;
3541 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
3542 	    B_FALSE, B_FALSE, &buf));
3543 	arc_buf_thaw(buf);
3544 
3545 	return (buf);
3546 }
3547 
3548 static void
l2arc_hdr_arcstats_update(arc_buf_hdr_t * hdr,boolean_t incr,boolean_t state_only)3549 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
3550     boolean_t state_only)
3551 {
3552 	uint64_t lsize = HDR_GET_LSIZE(hdr);
3553 	uint64_t psize = HDR_GET_PSIZE(hdr);
3554 	uint64_t asize = HDR_GET_L2SIZE(hdr);
3555 	arc_buf_contents_t type = hdr->b_type;
3556 	int64_t lsize_s;
3557 	int64_t psize_s;
3558 	int64_t asize_s;
3559 
3560 	/* For L2 we expect the header's b_l2size to be valid */
3561 	ASSERT3U(asize, >=, psize);
3562 
3563 	if (incr) {
3564 		lsize_s = lsize;
3565 		psize_s = psize;
3566 		asize_s = asize;
3567 	} else {
3568 		lsize_s = -lsize;
3569 		psize_s = -psize;
3570 		asize_s = -asize;
3571 	}
3572 
3573 	/* If the buffer is a prefetch, count it as such. */
3574 	if (HDR_PREFETCH(hdr)) {
3575 		ARCSTAT_INCR(arcstat_l2_prefetch_asize, asize_s);
3576 	} else {
3577 		/*
3578 		 * We use the value stored in the L2 header upon initial
3579 		 * caching in L2ARC. This value will be updated in case
3580 		 * an MRU/MRU_ghost buffer transitions to MFU but the L2ARC
3581 		 * metadata (log entry) cannot currently be updated. Having
3582 		 * the ARC state in the L2 header solves the problem of a
3583 		 * possibly absent L1 header (apparent in buffers restored
3584 		 * from persistent L2ARC).
3585 		 */
3586 		switch (hdr->b_l2hdr.b_arcs_state) {
3587 			case ARC_STATE_MRU_GHOST:
3588 			case ARC_STATE_MRU:
3589 				ARCSTAT_INCR(arcstat_l2_mru_asize, asize_s);
3590 				break;
3591 			case ARC_STATE_MFU_GHOST:
3592 			case ARC_STATE_MFU:
3593 				ARCSTAT_INCR(arcstat_l2_mfu_asize, asize_s);
3594 				break;
3595 			default:
3596 				break;
3597 		}
3598 	}
3599 
3600 	if (state_only)
3601 		return;
3602 
3603 	ARCSTAT_INCR(arcstat_l2_psize, psize_s);
3604 	ARCSTAT_INCR(arcstat_l2_lsize, lsize_s);
3605 
3606 	switch (type) {
3607 		case ARC_BUFC_DATA:
3608 			ARCSTAT_INCR(arcstat_l2_bufc_data_asize, asize_s);
3609 			break;
3610 		case ARC_BUFC_METADATA:
3611 			ARCSTAT_INCR(arcstat_l2_bufc_metadata_asize, asize_s);
3612 			break;
3613 		default:
3614 			break;
3615 	}
3616 }
3617 
3618 
3619 static void
arc_hdr_l2hdr_destroy(arc_buf_hdr_t * hdr)3620 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3621 {
3622 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3623 	l2arc_dev_t *dev = l2hdr->b_dev;
3624 
3625 	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3626 	ASSERT(HDR_HAS_L2HDR(hdr));
3627 
3628 	list_remove(&dev->l2ad_buflist, hdr);
3629 
3630 	l2arc_hdr_arcstats_decrement(hdr);
3631 	if (dev->l2ad_vdev != NULL) {
3632 		uint64_t asize = HDR_GET_L2SIZE(hdr);
3633 		vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3634 	}
3635 
3636 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3637 	    hdr);
3638 	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3639 }
3640 
3641 static void
arc_hdr_destroy(arc_buf_hdr_t * hdr)3642 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3643 {
3644 	if (HDR_HAS_L1HDR(hdr)) {
3645 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3646 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3647 	}
3648 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3649 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
3650 
3651 	if (HDR_HAS_L2HDR(hdr)) {
3652 		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3653 		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3654 
3655 		if (!buflist_held)
3656 			mutex_enter(&dev->l2ad_mtx);
3657 
3658 		/*
3659 		 * Even though we checked this conditional above, we
3660 		 * need to check this again now that we have the
3661 		 * l2ad_mtx. This is because we could be racing with
3662 		 * another thread calling l2arc_evict() which might have
3663 		 * destroyed this header's L2 portion as we were waiting
3664 		 * to acquire the l2ad_mtx. If that happens, we don't
3665 		 * want to re-destroy the header's L2 portion.
3666 		 */
3667 		if (HDR_HAS_L2HDR(hdr)) {
3668 
3669 			if (!HDR_EMPTY(hdr))
3670 				buf_discard_identity(hdr);
3671 
3672 			arc_hdr_l2hdr_destroy(hdr);
3673 		}
3674 
3675 		if (!buflist_held)
3676 			mutex_exit(&dev->l2ad_mtx);
3677 	}
3678 
3679 	/*
3680 	 * The header's identify can only be safely discarded once it is no
3681 	 * longer discoverable.  This requires removing it from the hash table
3682 	 * and the l2arc header list.  After this point the hash lock can not
3683 	 * be used to protect the header.
3684 	 */
3685 	if (!HDR_EMPTY(hdr))
3686 		buf_discard_identity(hdr);
3687 
3688 	if (HDR_HAS_L1HDR(hdr)) {
3689 		arc_cksum_free(hdr);
3690 
3691 		while (hdr->b_l1hdr.b_buf != NULL)
3692 			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3693 
3694 		if (hdr->b_l1hdr.b_pabd != NULL)
3695 			arc_hdr_free_abd(hdr, B_FALSE);
3696 
3697 		if (HDR_HAS_RABD(hdr))
3698 			arc_hdr_free_abd(hdr, B_TRUE);
3699 	}
3700 
3701 	ASSERT3P(hdr->b_hash_next, ==, NULL);
3702 	if (HDR_HAS_L1HDR(hdr)) {
3703 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3704 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3705 #ifdef ZFS_DEBUG
3706 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3707 #endif
3708 		kmem_cache_free(hdr_full_cache, hdr);
3709 	} else {
3710 		kmem_cache_free(hdr_l2only_cache, hdr);
3711 	}
3712 }
3713 
3714 void
arc_buf_destroy(arc_buf_t * buf,const void * tag)3715 arc_buf_destroy(arc_buf_t *buf, const void *tag)
3716 {
3717 	arc_buf_hdr_t *hdr = buf->b_hdr;
3718 
3719 	if (hdr->b_l1hdr.b_state == arc_anon) {
3720 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
3721 		ASSERT(ARC_BUF_LAST(buf));
3722 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3723 		VERIFY0(remove_reference(hdr, tag));
3724 		return;
3725 	}
3726 
3727 	kmutex_t *hash_lock = HDR_LOCK(hdr);
3728 	mutex_enter(hash_lock);
3729 
3730 	ASSERT3P(hdr, ==, buf->b_hdr);
3731 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
3732 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3733 	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3734 	ASSERT3P(buf->b_data, !=, NULL);
3735 
3736 	arc_buf_destroy_impl(buf);
3737 	(void) remove_reference(hdr, tag);
3738 	mutex_exit(hash_lock);
3739 }
3740 
3741 /*
3742  * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3743  * state of the header is dependent on its state prior to entering this
3744  * function. The following transitions are possible:
3745  *
3746  *    - arc_mru -> arc_mru_ghost
3747  *    - arc_mfu -> arc_mfu_ghost
3748  *    - arc_mru_ghost -> arc_l2c_only
3749  *    - arc_mru_ghost -> deleted
3750  *    - arc_mfu_ghost -> arc_l2c_only
3751  *    - arc_mfu_ghost -> deleted
3752  *    - arc_uncached -> deleted
3753  *
3754  * Return total size of evicted data buffers for eviction progress tracking.
3755  * When evicting from ghost states return logical buffer size to make eviction
3756  * progress at the same (or at least comparable) rate as from non-ghost states.
3757  *
3758  * Return *real_evicted for actual ARC size reduction to wake up threads
3759  * waiting for it.  For non-ghost states it includes size of evicted data
3760  * buffers (the headers are not freed there).  For ghost states it includes
3761  * only the evicted headers size.
3762  */
3763 static int64_t
arc_evict_hdr(arc_buf_hdr_t * hdr,uint64_t * real_evicted)3764 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
3765 {
3766 	arc_state_t *evicted_state, *state;
3767 	int64_t bytes_evicted = 0;
3768 	uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3769 	    arc_min_prescient_prefetch_ms : arc_min_prefetch_ms;
3770 
3771 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3772 	ASSERT(HDR_HAS_L1HDR(hdr));
3773 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3774 	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3775 	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3776 
3777 	*real_evicted = 0;
3778 	state = hdr->b_l1hdr.b_state;
3779 	if (GHOST_STATE(state)) {
3780 
3781 		/*
3782 		 * l2arc_write_buffers() relies on a header's L1 portion
3783 		 * (i.e. its b_pabd field) during it's write phase.
3784 		 * Thus, we cannot push a header onto the arc_l2c_only
3785 		 * state (removing its L1 piece) until the header is
3786 		 * done being written to the l2arc.
3787 		 */
3788 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3789 			ARCSTAT_BUMP(arcstat_evict_l2_skip);
3790 			return (bytes_evicted);
3791 		}
3792 
3793 		ARCSTAT_BUMP(arcstat_deleted);
3794 		bytes_evicted += HDR_GET_LSIZE(hdr);
3795 
3796 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3797 
3798 		if (HDR_HAS_L2HDR(hdr)) {
3799 			ASSERT(hdr->b_l1hdr.b_pabd == NULL);
3800 			ASSERT(!HDR_HAS_RABD(hdr));
3801 			/*
3802 			 * This buffer is cached on the 2nd Level ARC;
3803 			 * don't destroy the header.
3804 			 */
3805 			arc_change_state(arc_l2c_only, hdr);
3806 			/*
3807 			 * dropping from L1+L2 cached to L2-only,
3808 			 * realloc to remove the L1 header.
3809 			 */
3810 			(void) arc_hdr_realloc(hdr, hdr_full_cache,
3811 			    hdr_l2only_cache);
3812 			*real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
3813 		} else {
3814 			arc_change_state(arc_anon, hdr);
3815 			arc_hdr_destroy(hdr);
3816 			*real_evicted += HDR_FULL_SIZE;
3817 		}
3818 		return (bytes_evicted);
3819 	}
3820 
3821 	ASSERT(state == arc_mru || state == arc_mfu || state == arc_uncached);
3822 	evicted_state = (state == arc_uncached) ? arc_anon :
3823 	    ((state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost);
3824 
3825 	/* prefetch buffers have a minimum lifespan */
3826 	if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3827 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3828 	    MSEC_TO_TICK(min_lifetime)) {
3829 		ARCSTAT_BUMP(arcstat_evict_skip);
3830 		return (bytes_evicted);
3831 	}
3832 
3833 	if (HDR_HAS_L2HDR(hdr)) {
3834 		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3835 	} else {
3836 		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3837 			ARCSTAT_INCR(arcstat_evict_l2_eligible,
3838 			    HDR_GET_LSIZE(hdr));
3839 
3840 			switch (state->arcs_state) {
3841 				case ARC_STATE_MRU:
3842 					ARCSTAT_INCR(
3843 					    arcstat_evict_l2_eligible_mru,
3844 					    HDR_GET_LSIZE(hdr));
3845 					break;
3846 				case ARC_STATE_MFU:
3847 					ARCSTAT_INCR(
3848 					    arcstat_evict_l2_eligible_mfu,
3849 					    HDR_GET_LSIZE(hdr));
3850 					break;
3851 				default:
3852 					break;
3853 			}
3854 		} else {
3855 			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3856 			    HDR_GET_LSIZE(hdr));
3857 		}
3858 	}
3859 
3860 	bytes_evicted += arc_hdr_size(hdr);
3861 	*real_evicted += arc_hdr_size(hdr);
3862 
3863 	/*
3864 	 * If this hdr is being evicted and has a compressed buffer then we
3865 	 * discard it here before we change states.  This ensures that the
3866 	 * accounting is updated correctly in arc_free_data_impl().
3867 	 */
3868 	if (hdr->b_l1hdr.b_pabd != NULL)
3869 		arc_hdr_free_abd(hdr, B_FALSE);
3870 
3871 	if (HDR_HAS_RABD(hdr))
3872 		arc_hdr_free_abd(hdr, B_TRUE);
3873 
3874 	arc_change_state(evicted_state, hdr);
3875 	DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3876 	if (evicted_state == arc_anon) {
3877 		arc_hdr_destroy(hdr);
3878 		*real_evicted += HDR_FULL_SIZE;
3879 	} else {
3880 		ASSERT(HDR_IN_HASH_TABLE(hdr));
3881 	}
3882 
3883 	return (bytes_evicted);
3884 }
3885 
3886 static void
arc_set_need_free(void)3887 arc_set_need_free(void)
3888 {
3889 	ASSERT(MUTEX_HELD(&arc_evict_lock));
3890 	int64_t remaining = arc_free_memory() - arc_sys_free / 2;
3891 	arc_evict_waiter_t *aw = list_tail(&arc_evict_waiters);
3892 	if (aw == NULL) {
3893 		arc_need_free = MAX(-remaining, 0);
3894 	} else {
3895 		arc_need_free =
3896 		    MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
3897 	}
3898 }
3899 
3900 static uint64_t
arc_evict_state_impl(multilist_t * ml,int idx,arc_buf_hdr_t * marker,uint64_t spa,uint64_t bytes)3901 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3902     uint64_t spa, uint64_t bytes)
3903 {
3904 	multilist_sublist_t *mls;
3905 	uint64_t bytes_evicted = 0, real_evicted = 0;
3906 	arc_buf_hdr_t *hdr;
3907 	kmutex_t *hash_lock;
3908 	uint_t evict_count = zfs_arc_evict_batch_limit;
3909 
3910 	ASSERT3P(marker, !=, NULL);
3911 
3912 	mls = multilist_sublist_lock_idx(ml, idx);
3913 
3914 	for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
3915 	    hdr = multilist_sublist_prev(mls, marker)) {
3916 		if ((evict_count == 0) || (bytes_evicted >= bytes))
3917 			break;
3918 
3919 		/*
3920 		 * To keep our iteration location, move the marker
3921 		 * forward. Since we're not holding hdr's hash lock, we
3922 		 * must be very careful and not remove 'hdr' from the
3923 		 * sublist. Otherwise, other consumers might mistake the
3924 		 * 'hdr' as not being on a sublist when they call the
3925 		 * multilist_link_active() function (they all rely on
3926 		 * the hash lock protecting concurrent insertions and
3927 		 * removals). multilist_sublist_move_forward() was
3928 		 * specifically implemented to ensure this is the case
3929 		 * (only 'marker' will be removed and re-inserted).
3930 		 */
3931 		multilist_sublist_move_forward(mls, marker);
3932 
3933 		/*
3934 		 * The only case where the b_spa field should ever be
3935 		 * zero, is the marker headers inserted by
3936 		 * arc_evict_state(). It's possible for multiple threads
3937 		 * to be calling arc_evict_state() concurrently (e.g.
3938 		 * dsl_pool_close() and zio_inject_fault()), so we must
3939 		 * skip any markers we see from these other threads.
3940 		 */
3941 		if (hdr->b_spa == 0)
3942 			continue;
3943 
3944 		/* we're only interested in evicting buffers of a certain spa */
3945 		if (spa != 0 && hdr->b_spa != spa) {
3946 			ARCSTAT_BUMP(arcstat_evict_skip);
3947 			continue;
3948 		}
3949 
3950 		hash_lock = HDR_LOCK(hdr);
3951 
3952 		/*
3953 		 * We aren't calling this function from any code path
3954 		 * that would already be holding a hash lock, so we're
3955 		 * asserting on this assumption to be defensive in case
3956 		 * this ever changes. Without this check, it would be
3957 		 * possible to incorrectly increment arcstat_mutex_miss
3958 		 * below (e.g. if the code changed such that we called
3959 		 * this function with a hash lock held).
3960 		 */
3961 		ASSERT(!MUTEX_HELD(hash_lock));
3962 
3963 		if (mutex_tryenter(hash_lock)) {
3964 			uint64_t revicted;
3965 			uint64_t evicted = arc_evict_hdr(hdr, &revicted);
3966 			mutex_exit(hash_lock);
3967 
3968 			bytes_evicted += evicted;
3969 			real_evicted += revicted;
3970 
3971 			/*
3972 			 * If evicted is zero, arc_evict_hdr() must have
3973 			 * decided to skip this header, don't increment
3974 			 * evict_count in this case.
3975 			 */
3976 			if (evicted != 0)
3977 				evict_count--;
3978 
3979 		} else {
3980 			ARCSTAT_BUMP(arcstat_mutex_miss);
3981 		}
3982 	}
3983 
3984 	multilist_sublist_unlock(mls);
3985 
3986 	/*
3987 	 * Increment the count of evicted bytes, and wake up any threads that
3988 	 * are waiting for the count to reach this value.  Since the list is
3989 	 * ordered by ascending aew_count, we pop off the beginning of the
3990 	 * list until we reach the end, or a waiter that's past the current
3991 	 * "count".  Doing this outside the loop reduces the number of times
3992 	 * we need to acquire the global arc_evict_lock.
3993 	 *
3994 	 * Only wake when there's sufficient free memory in the system
3995 	 * (specifically, arc_sys_free/2, which by default is a bit more than
3996 	 * 1/64th of RAM).  See the comments in arc_wait_for_eviction().
3997 	 */
3998 	mutex_enter(&arc_evict_lock);
3999 	arc_evict_count += real_evicted;
4000 
4001 	if (arc_free_memory() > arc_sys_free / 2) {
4002 		arc_evict_waiter_t *aw;
4003 		while ((aw = list_head(&arc_evict_waiters)) != NULL &&
4004 		    aw->aew_count <= arc_evict_count) {
4005 			list_remove(&arc_evict_waiters, aw);
4006 			cv_broadcast(&aw->aew_cv);
4007 		}
4008 	}
4009 	arc_set_need_free();
4010 	mutex_exit(&arc_evict_lock);
4011 
4012 	/*
4013 	 * If the ARC size is reduced from arc_c_max to arc_c_min (especially
4014 	 * if the average cached block is small), eviction can be on-CPU for
4015 	 * many seconds.  To ensure that other threads that may be bound to
4016 	 * this CPU are able to make progress, make a voluntary preemption
4017 	 * call here.
4018 	 */
4019 	kpreempt(KPREEMPT_SYNC);
4020 
4021 	return (bytes_evicted);
4022 }
4023 
4024 static arc_buf_hdr_t *
arc_state_alloc_marker(void)4025 arc_state_alloc_marker(void)
4026 {
4027 	arc_buf_hdr_t *marker = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
4028 
4029 	/*
4030 	 * A b_spa of 0 is used to indicate that this header is
4031 	 * a marker. This fact is used in arc_evict_state_impl().
4032 	 */
4033 	marker->b_spa = 0;
4034 
4035 	return (marker);
4036 }
4037 
4038 static void
arc_state_free_marker(arc_buf_hdr_t * marker)4039 arc_state_free_marker(arc_buf_hdr_t *marker)
4040 {
4041 	kmem_cache_free(hdr_full_cache, marker);
4042 }
4043 
4044 /*
4045  * Allocate an array of buffer headers used as placeholders during arc state
4046  * eviction.
4047  */
4048 static arc_buf_hdr_t **
arc_state_alloc_markers(int count)4049 arc_state_alloc_markers(int count)
4050 {
4051 	arc_buf_hdr_t **markers;
4052 
4053 	markers = kmem_zalloc(sizeof (*markers) * count, KM_SLEEP);
4054 	for (int i = 0; i < count; i++)
4055 		markers[i] = arc_state_alloc_marker();
4056 	return (markers);
4057 }
4058 
4059 static void
arc_state_free_markers(arc_buf_hdr_t ** markers,int count)4060 arc_state_free_markers(arc_buf_hdr_t **markers, int count)
4061 {
4062 	for (int i = 0; i < count; i++)
4063 		arc_state_free_marker(markers[i]);
4064 	kmem_free(markers, sizeof (*markers) * count);
4065 }
4066 
4067 typedef struct evict_arg {
4068 	taskq_ent_t		eva_tqent;
4069 	multilist_t		*eva_ml;
4070 	arc_buf_hdr_t		*eva_marker;
4071 	int			eva_idx;
4072 	uint64_t		eva_spa;
4073 	uint64_t		eva_bytes;
4074 	uint64_t		eva_evicted;
4075 } evict_arg_t;
4076 
4077 static void
arc_evict_task(void * arg)4078 arc_evict_task(void *arg)
4079 {
4080 	evict_arg_t *eva = arg;
4081 	eva->eva_evicted = arc_evict_state_impl(eva->eva_ml, eva->eva_idx,
4082 	    eva->eva_marker, eva->eva_spa, eva->eva_bytes);
4083 }
4084 
4085 static void
arc_evict_thread_init(void)4086 arc_evict_thread_init(void)
4087 {
4088 	if (zfs_arc_evict_threads == 0) {
4089 		/*
4090 		 * Compute number of threads we want to use for eviction.
4091 		 *
4092 		 * Normally, it's log2(ncpus) + ncpus/32, which gets us to the
4093 		 * default max of 16 threads at ~256 CPUs.
4094 		 *
4095 		 * However, that formula goes to two threads at 4 CPUs, which
4096 		 * is still rather to low to be really useful, so we just go
4097 		 * with 1 thread at fewer than 6 cores.
4098 		 */
4099 		if (max_ncpus < 6)
4100 			zfs_arc_evict_threads = 1;
4101 		else
4102 			zfs_arc_evict_threads =
4103 			    (highbit64(max_ncpus) - 1) + max_ncpus / 32;
4104 	} else if (zfs_arc_evict_threads > max_ncpus)
4105 		zfs_arc_evict_threads = max_ncpus;
4106 
4107 	if (zfs_arc_evict_threads > 1) {
4108 		arc_evict_taskq = taskq_create("arc_evict",
4109 		    zfs_arc_evict_threads, defclsyspri, 0, INT_MAX,
4110 		    TASKQ_PREPOPULATE);
4111 		arc_evict_arg = kmem_zalloc(
4112 		    sizeof (evict_arg_t) * zfs_arc_evict_threads, KM_SLEEP);
4113 	}
4114 }
4115 
4116 /*
4117  * The minimum number of bytes we can evict at once is a block size.
4118  * So, SPA_MAXBLOCKSIZE is a reasonable minimal value per an eviction task.
4119  * We use this value to compute a scaling factor for the eviction tasks.
4120  */
4121 #define	MIN_EVICT_SIZE	(SPA_MAXBLOCKSIZE)
4122 
4123 /*
4124  * Evict buffers from the given arc state, until we've removed the
4125  * specified number of bytes. Move the removed buffers to the
4126  * appropriate evict state.
4127  *
4128  * This function makes a "best effort". It skips over any buffers
4129  * it can't get a hash_lock on, and so, may not catch all candidates.
4130  * It may also return without evicting as much space as requested.
4131  *
4132  * If bytes is specified using the special value ARC_EVICT_ALL, this
4133  * will evict all available (i.e. unlocked and evictable) buffers from
4134  * the given arc state; which is used by arc_flush().
4135  */
4136 static uint64_t
arc_evict_state(arc_state_t * state,arc_buf_contents_t type,uint64_t spa,uint64_t bytes)4137 arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
4138     uint64_t bytes)
4139 {
4140 	uint64_t total_evicted = 0;
4141 	multilist_t *ml = &state->arcs_list[type];
4142 	int num_sublists;
4143 	arc_buf_hdr_t **markers;
4144 	evict_arg_t *eva = NULL;
4145 
4146 	num_sublists = multilist_get_num_sublists(ml);
4147 
4148 	boolean_t use_evcttq = zfs_arc_evict_threads > 1;
4149 
4150 	/*
4151 	 * If we've tried to evict from each sublist, made some
4152 	 * progress, but still have not hit the target number of bytes
4153 	 * to evict, we want to keep trying. The markers allow us to
4154 	 * pick up where we left off for each individual sublist, rather
4155 	 * than starting from the tail each time.
4156 	 */
4157 	if (zthr_iscurthread(arc_evict_zthr)) {
4158 		markers = arc_state_evict_markers;
4159 		ASSERT3S(num_sublists, <=, arc_state_evict_marker_count);
4160 	} else {
4161 		markers = arc_state_alloc_markers(num_sublists);
4162 	}
4163 	for (int i = 0; i < num_sublists; i++) {
4164 		multilist_sublist_t *mls;
4165 
4166 		mls = multilist_sublist_lock_idx(ml, i);
4167 		multilist_sublist_insert_tail(mls, markers[i]);
4168 		multilist_sublist_unlock(mls);
4169 	}
4170 
4171 	if (use_evcttq) {
4172 		if (zthr_iscurthread(arc_evict_zthr))
4173 			eva = arc_evict_arg;
4174 		else
4175 			eva = kmem_alloc(sizeof (evict_arg_t) *
4176 			    zfs_arc_evict_threads, KM_NOSLEEP);
4177 		if (eva) {
4178 			for (int i = 0; i < zfs_arc_evict_threads; i++) {
4179 				taskq_init_ent(&eva[i].eva_tqent);
4180 				eva[i].eva_ml = ml;
4181 				eva[i].eva_spa = spa;
4182 			}
4183 		} else {
4184 			/*
4185 			 * Fall back to the regular single evict if it is not
4186 			 * possible to allocate memory for the taskq entries.
4187 			 */
4188 			use_evcttq = B_FALSE;
4189 		}
4190 	}
4191 
4192 	/*
4193 	 * Start eviction using a randomly selected sublist, this is to try and
4194 	 * evenly balance eviction across all sublists. Always starting at the
4195 	 * same sublist (e.g. index 0) would cause evictions to favor certain
4196 	 * sublists over others.
4197 	 */
4198 	uint64_t scan_evicted = 0;
4199 	int sublists_left = num_sublists;
4200 	int sublist_idx = multilist_get_random_index(ml);
4201 
4202 	/*
4203 	 * While we haven't hit our target number of bytes to evict, or
4204 	 * we're evicting all available buffers.
4205 	 */
4206 	while (total_evicted < bytes) {
4207 		uint64_t evict = MIN_EVICT_SIZE;
4208 		uint_t ntasks = zfs_arc_evict_threads;
4209 
4210 		if (use_evcttq) {
4211 			if (sublists_left < ntasks)
4212 				ntasks = sublists_left;
4213 
4214 			if (ntasks < 2)
4215 				use_evcttq = B_FALSE;
4216 		}
4217 
4218 		if (use_evcttq) {
4219 			uint64_t left = bytes - total_evicted;
4220 
4221 			if (bytes == ARC_EVICT_ALL) {
4222 				evict = bytes;
4223 			} else if (left > ntasks * MIN_EVICT_SIZE) {
4224 				evict = DIV_ROUND_UP(left, ntasks);
4225 			} else {
4226 				ntasks = DIV_ROUND_UP(left, MIN_EVICT_SIZE);
4227 				if (ntasks == 1)
4228 					use_evcttq = B_FALSE;
4229 			}
4230 		}
4231 
4232 		for (int i = 0; sublists_left > 0; i++, sublist_idx++,
4233 		    sublists_left--) {
4234 			uint64_t bytes_remaining;
4235 			uint64_t bytes_evicted;
4236 
4237 			/* we've reached the end, wrap to the beginning */
4238 			if (sublist_idx >= num_sublists)
4239 				sublist_idx = 0;
4240 
4241 			if (use_evcttq) {
4242 				if (i == ntasks)
4243 					break;
4244 
4245 				eva[i].eva_marker = markers[sublist_idx];
4246 				eva[i].eva_idx = sublist_idx;
4247 				eva[i].eva_bytes = evict;
4248 
4249 				taskq_dispatch_ent(arc_evict_taskq,
4250 				    arc_evict_task, &eva[i], 0,
4251 				    &eva[i].eva_tqent);
4252 
4253 				continue;
4254 			}
4255 
4256 			if (total_evicted < bytes)
4257 				bytes_remaining = bytes - total_evicted;
4258 			else
4259 				break;
4260 
4261 			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
4262 			    markers[sublist_idx], spa, bytes_remaining);
4263 
4264 			scan_evicted += bytes_evicted;
4265 			total_evicted += bytes_evicted;
4266 		}
4267 
4268 		if (use_evcttq) {
4269 			taskq_wait(arc_evict_taskq);
4270 
4271 			for (int i = 0; i < ntasks; i++) {
4272 				scan_evicted += eva[i].eva_evicted;
4273 				total_evicted += eva[i].eva_evicted;
4274 			}
4275 		}
4276 
4277 		/*
4278 		 * If we scanned all sublists and didn't evict anything, we
4279 		 * have no reason to believe we'll evict more during another
4280 		 * scan, so break the loop.
4281 		 */
4282 		if (scan_evicted == 0 && sublists_left == 0) {
4283 			/* This isn't possible, let's make that obvious */
4284 			ASSERT3S(bytes, !=, 0);
4285 
4286 			/*
4287 			 * When bytes is ARC_EVICT_ALL, the only way to
4288 			 * break the loop is when scan_evicted is zero.
4289 			 * In that case, we actually have evicted enough,
4290 			 * so we don't want to increment the kstat.
4291 			 */
4292 			if (bytes != ARC_EVICT_ALL) {
4293 				ASSERT3S(total_evicted, <, bytes);
4294 				ARCSTAT_BUMP(arcstat_evict_not_enough);
4295 			}
4296 
4297 			break;
4298 		}
4299 
4300 		/*
4301 		 * If we scanned all sublists but still have more to do,
4302 		 * reset the counts so we can go around again.
4303 		 */
4304 		if (sublists_left == 0) {
4305 			sublists_left = num_sublists;
4306 			sublist_idx = multilist_get_random_index(ml);
4307 			scan_evicted = 0;
4308 
4309 			/*
4310 			 * Since we're about to reconsider all sublists,
4311 			 * re-enable use of the evict threads if available.
4312 			 */
4313 			use_evcttq = (zfs_arc_evict_threads > 1 && eva != NULL);
4314 		}
4315 	}
4316 
4317 	if (eva != NULL && eva != arc_evict_arg)
4318 		kmem_free(eva, sizeof (evict_arg_t) * zfs_arc_evict_threads);
4319 
4320 	for (int i = 0; i < num_sublists; i++) {
4321 		multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
4322 		multilist_sublist_remove(mls, markers[i]);
4323 		multilist_sublist_unlock(mls);
4324 	}
4325 
4326 	if (markers != arc_state_evict_markers)
4327 		arc_state_free_markers(markers, num_sublists);
4328 
4329 	return (total_evicted);
4330 }
4331 
4332 /*
4333  * Flush all "evictable" data of the given type from the arc state
4334  * specified. This will not evict any "active" buffers (i.e. referenced).
4335  *
4336  * When 'retry' is set to B_FALSE, the function will make a single pass
4337  * over the state and evict any buffers that it can. Since it doesn't
4338  * continually retry the eviction, it might end up leaving some buffers
4339  * in the ARC due to lock misses.
4340  *
4341  * When 'retry' is set to B_TRUE, the function will continually retry the
4342  * eviction until *all* evictable buffers have been removed from the
4343  * state. As a result, if concurrent insertions into the state are
4344  * allowed (e.g. if the ARC isn't shutting down), this function might
4345  * wind up in an infinite loop, continually trying to evict buffers.
4346  */
4347 static uint64_t
arc_flush_state(arc_state_t * state,uint64_t spa,arc_buf_contents_t type,boolean_t retry)4348 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
4349     boolean_t retry)
4350 {
4351 	uint64_t evicted = 0;
4352 
4353 	while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
4354 		evicted += arc_evict_state(state, type, spa, ARC_EVICT_ALL);
4355 
4356 		if (!retry)
4357 			break;
4358 	}
4359 
4360 	return (evicted);
4361 }
4362 
4363 /*
4364  * Evict the specified number of bytes from the state specified. This
4365  * function prevents us from trying to evict more from a state's list
4366  * than is "evictable", and to skip evicting altogether when passed a
4367  * negative value for "bytes". In contrast, arc_evict_state() will
4368  * evict everything it can, when passed a negative value for "bytes".
4369  */
4370 static uint64_t
arc_evict_impl(arc_state_t * state,arc_buf_contents_t type,int64_t bytes)4371 arc_evict_impl(arc_state_t *state, arc_buf_contents_t type, int64_t bytes)
4372 {
4373 	uint64_t delta;
4374 
4375 	if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
4376 		delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
4377 		    bytes);
4378 		return (arc_evict_state(state, type, 0, delta));
4379 	}
4380 
4381 	return (0);
4382 }
4383 
4384 /*
4385  * Adjust specified fraction, taking into account initial ghost state(s) size,
4386  * ghost hit bytes towards increasing the fraction, ghost hit bytes towards
4387  * decreasing it, plus a balance factor, controlling the decrease rate, used
4388  * to balance metadata vs data.
4389  */
4390 static uint64_t
arc_evict_adj(uint64_t frac,uint64_t total,uint64_t up,uint64_t down,uint_t balance)4391 arc_evict_adj(uint64_t frac, uint64_t total, uint64_t up, uint64_t down,
4392     uint_t balance)
4393 {
4394 	if (total < 32 || up + down == 0)
4395 		return (frac);
4396 
4397 	/*
4398 	 * We should not have more ghost hits than ghost size, but they may
4399 	 * get close.  To avoid overflows below up/down should not be bigger
4400 	 * than 1/5 of total.  But to limit maximum adjustment speed restrict
4401 	 * it some more.
4402 	 */
4403 	if (up + down >= total / 16) {
4404 		uint64_t scale = (up + down) / (total / 32);
4405 		up /= scale;
4406 		down /= scale;
4407 	}
4408 
4409 	/* Get maximal dynamic range by choosing optimal shifts. */
4410 	int s = highbit64(total);
4411 	s = MIN(64 - s, 32);
4412 
4413 	ASSERT3U(frac, <=, 1ULL << 32);
4414 	uint64_t ofrac = (1ULL << 32) - frac;
4415 
4416 	if (frac >= 4 * ofrac)
4417 		up /= frac / (2 * ofrac + 1);
4418 	up = (up << s) / (total >> (32 - s));
4419 	if (ofrac >= 4 * frac)
4420 		down /= ofrac / (2 * frac + 1);
4421 	down = (down << s) / (total >> (32 - s));
4422 	down = down * 100 / balance;
4423 
4424 	ASSERT3U(up, <=, (1ULL << 32) - frac);
4425 	ASSERT3U(down, <=, frac);
4426 	return (frac + up - down);
4427 }
4428 
4429 /*
4430  * Calculate (x * multiplier / divisor) without unnecesary overflows.
4431  */
4432 static uint64_t
arc_mf(uint64_t x,uint64_t multiplier,uint64_t divisor)4433 arc_mf(uint64_t x, uint64_t multiplier, uint64_t divisor)
4434 {
4435 	uint64_t q = (x / divisor);
4436 	uint64_t r = (x % divisor);
4437 
4438 	return ((q * multiplier) + ((r * multiplier) / divisor));
4439 }
4440 
4441 /*
4442  * Evict buffers from the cache, such that arcstat_size is capped by arc_c.
4443  */
4444 static uint64_t
arc_evict(void)4445 arc_evict(void)
4446 {
4447 	uint64_t bytes, total_evicted = 0;
4448 	int64_t e, mrud, mrum, mfud, mfum, w;
4449 	static uint64_t ogrd, ogrm, ogfd, ogfm;
4450 	static uint64_t gsrd, gsrm, gsfd, gsfm;
4451 	uint64_t ngrd, ngrm, ngfd, ngfm;
4452 
4453 	/* Get current size of ARC states we can evict from. */
4454 	mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
4455 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
4456 	mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
4457 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
4458 	mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
4459 	mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
4460 	uint64_t d = mrud + mfud;
4461 	uint64_t m = mrum + mfum;
4462 	uint64_t t = d + m;
4463 
4464 	/* Get ARC ghost hits since last eviction. */
4465 	ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
4466 	uint64_t grd = ngrd - ogrd;
4467 	ogrd = ngrd;
4468 	ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
4469 	uint64_t grm = ngrm - ogrm;
4470 	ogrm = ngrm;
4471 	ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
4472 	uint64_t gfd = ngfd - ogfd;
4473 	ogfd = ngfd;
4474 	ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
4475 	uint64_t gfm = ngfm - ogfm;
4476 	ogfm = ngfm;
4477 
4478 	/* Adjust ARC states balance based on ghost hits. */
4479 	arc_meta = arc_evict_adj(arc_meta, gsrd + gsrm + gsfd + gsfm,
4480 	    grm + gfm, grd + gfd, zfs_arc_meta_balance);
4481 	arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
4482 	arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);
4483 
4484 	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
4485 	uint64_t ac = arc_c;
4486 	int64_t wt = t - (asize - ac);
4487 
4488 	/*
4489 	 * Try to reduce pinned dnodes if more than 3/4 of wanted metadata
4490 	 * target is not evictable or if they go over arc_dnode_limit.
4491 	 */
4492 	int64_t prune = 0;
4493 	int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
4494 	int64_t nem = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA])
4495 	    + zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA])
4496 	    - zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA])
4497 	    - zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
4498 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
4499 	if (nem > w * 3 / 4) {
4500 		prune = dn / sizeof (dnode_t) *
4501 		    zfs_arc_dnode_reduce_percent / 100;
4502 		if (nem < w && w > 4)
4503 			prune = arc_mf(prune, nem - w * 3 / 4, w / 4);
4504 	}
4505 	if (dn > arc_dnode_limit) {
4506 		prune = MAX(prune, (dn - arc_dnode_limit) / sizeof (dnode_t) *
4507 		    zfs_arc_dnode_reduce_percent / 100);
4508 	}
4509 	if (prune > 0)
4510 		arc_prune_async(prune);
4511 
4512 	/* Evict MRU metadata. */
4513 	w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
4514 	e = MIN((int64_t)(asize - ac), (int64_t)(mrum - w));
4515 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
4516 	total_evicted += bytes;
4517 	mrum -= bytes;
4518 	asize -= bytes;
4519 
4520 	/* Evict MFU metadata. */
4521 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
4522 	e = MIN((int64_t)(asize - ac), (int64_t)(m - bytes - w));
4523 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
4524 	total_evicted += bytes;
4525 	mfum -= bytes;
4526 	asize -= bytes;
4527 
4528 	/* Evict MRU data. */
4529 	wt -= m - total_evicted;
4530 	w = wt * (int64_t)(arc_pd >> 16) >> 16;
4531 	e = MIN((int64_t)(asize - ac), (int64_t)(mrud - w));
4532 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
4533 	total_evicted += bytes;
4534 	mrud -= bytes;
4535 	asize -= bytes;
4536 
4537 	/* Evict MFU data. */
4538 	e = asize - ac;
4539 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
4540 	mfud -= bytes;
4541 	total_evicted += bytes;
4542 
4543 	/*
4544 	 * Evict ghost lists
4545 	 *
4546 	 * Size of each state's ghost list represents how much that state
4547 	 * may grow by shrinking the other states.  Would it need to shrink
4548 	 * other states to zero (that is unlikely), its ghost size would be
4549 	 * equal to sum of other three state sizes.  But excessive ghost
4550 	 * size may result in false ghost hits (too far back), that may
4551 	 * never result in real cache hits if several states are competing.
4552 	 * So choose some arbitraty point of 1/2 of other state sizes.
4553 	 */
4554 	gsrd = (mrum + mfud + mfum) / 2;
4555 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
4556 	    gsrd;
4557 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_DATA, e);
4558 
4559 	gsrm = (mrud + mfud + mfum) / 2;
4560 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
4561 	    gsrm;
4562 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_METADATA, e);
4563 
4564 	gsfd = (mrud + mrum + mfum) / 2;
4565 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
4566 	    gsfd;
4567 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_DATA, e);
4568 
4569 	gsfm = (mrud + mrum + mfud) / 2;
4570 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
4571 	    gsfm;
4572 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_METADATA, e);
4573 
4574 	return (total_evicted);
4575 }
4576 
4577 static void
arc_flush_impl(uint64_t guid,boolean_t retry)4578 arc_flush_impl(uint64_t guid, boolean_t retry)
4579 {
4580 	ASSERT(!retry || guid == 0);
4581 
4582 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4583 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4584 
4585 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4586 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4587 
4588 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4589 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4590 
4591 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4592 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4593 
4594 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_DATA, retry);
4595 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_METADATA, retry);
4596 }
4597 
4598 void
arc_flush(spa_t * spa,boolean_t retry)4599 arc_flush(spa_t *spa, boolean_t retry)
4600 {
4601 	/*
4602 	 * If retry is B_TRUE, a spa must not be specified since we have
4603 	 * no good way to determine if all of a spa's buffers have been
4604 	 * evicted from an arc state.
4605 	 */
4606 	ASSERT(!retry || spa == NULL);
4607 
4608 	arc_flush_impl(spa != NULL ? spa_load_guid(spa) : 0, retry);
4609 }
4610 
4611 static arc_async_flush_t *
arc_async_flush_add(uint64_t spa_guid,uint_t level)4612 arc_async_flush_add(uint64_t spa_guid, uint_t level)
4613 {
4614 	arc_async_flush_t *af = kmem_alloc(sizeof (*af), KM_SLEEP);
4615 	af->af_spa_guid = spa_guid;
4616 	af->af_cache_level = level;
4617 	taskq_init_ent(&af->af_tqent);
4618 	list_link_init(&af->af_node);
4619 
4620 	mutex_enter(&arc_async_flush_lock);
4621 	list_insert_tail(&arc_async_flush_list, af);
4622 	mutex_exit(&arc_async_flush_lock);
4623 
4624 	return (af);
4625 }
4626 
4627 static void
arc_async_flush_remove(uint64_t spa_guid,uint_t level)4628 arc_async_flush_remove(uint64_t spa_guid, uint_t level)
4629 {
4630 	mutex_enter(&arc_async_flush_lock);
4631 	for (arc_async_flush_t *af = list_head(&arc_async_flush_list);
4632 	    af != NULL; af = list_next(&arc_async_flush_list, af)) {
4633 		if (af->af_spa_guid == spa_guid &&
4634 		    af->af_cache_level == level) {
4635 			list_remove(&arc_async_flush_list, af);
4636 			kmem_free(af, sizeof (*af));
4637 			break;
4638 		}
4639 	}
4640 	mutex_exit(&arc_async_flush_lock);
4641 }
4642 
4643 static void
arc_flush_task(void * arg)4644 arc_flush_task(void *arg)
4645 {
4646 	arc_async_flush_t *af = arg;
4647 	hrtime_t start_time = gethrtime();
4648 	uint64_t spa_guid = af->af_spa_guid;
4649 
4650 	arc_flush_impl(spa_guid, B_FALSE);
4651 	arc_async_flush_remove(spa_guid, af->af_cache_level);
4652 
4653 	uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
4654 	if (elaspsed > 0) {
4655 		zfs_dbgmsg("spa %llu arc flushed in %llu ms",
4656 		    (u_longlong_t)spa_guid, (u_longlong_t)elaspsed);
4657 	}
4658 }
4659 
4660 /*
4661  * ARC buffers use the spa's load guid and can continue to exist after
4662  * the spa_t is gone (exported). The blocks are orphaned since each
4663  * spa import has a different load guid.
4664  *
4665  * It's OK if the spa is re-imported while this asynchronous flush is
4666  * still in progress. The new spa_load_guid will be different.
4667  *
4668  * Also, arc_fini will wait for any arc_flush_task to finish.
4669  */
4670 void
arc_flush_async(spa_t * spa)4671 arc_flush_async(spa_t *spa)
4672 {
4673 	uint64_t spa_guid = spa_load_guid(spa);
4674 	arc_async_flush_t *af = arc_async_flush_add(spa_guid, 1);
4675 
4676 	taskq_dispatch_ent(arc_flush_taskq, arc_flush_task,
4677 	    af, TQ_SLEEP, &af->af_tqent);
4678 }
4679 
4680 /*
4681  * Check if a guid is still in-use as part of an async teardown task
4682  */
4683 boolean_t
arc_async_flush_guid_inuse(uint64_t spa_guid)4684 arc_async_flush_guid_inuse(uint64_t spa_guid)
4685 {
4686 	mutex_enter(&arc_async_flush_lock);
4687 	for (arc_async_flush_t *af = list_head(&arc_async_flush_list);
4688 	    af != NULL; af = list_next(&arc_async_flush_list, af)) {
4689 		if (af->af_spa_guid == spa_guid) {
4690 			mutex_exit(&arc_async_flush_lock);
4691 			return (B_TRUE);
4692 		}
4693 	}
4694 	mutex_exit(&arc_async_flush_lock);
4695 	return (B_FALSE);
4696 }
4697 
4698 uint64_t
arc_reduce_target_size(uint64_t to_free)4699 arc_reduce_target_size(uint64_t to_free)
4700 {
4701 	/*
4702 	 * Get the actual arc size.  Even if we don't need it, this updates
4703 	 * the aggsum lower bound estimate for arc_is_overflowing().
4704 	 */
4705 	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
4706 
4707 	/*
4708 	 * All callers want the ARC to actually evict (at least) this much
4709 	 * memory.  Therefore we reduce from the lower of the current size and
4710 	 * the target size.  This way, even if arc_c is much higher than
4711 	 * arc_size (as can be the case after many calls to arc_freed(), we will
4712 	 * immediately have arc_c < arc_size and therefore the arc_evict_zthr
4713 	 * will evict.
4714 	 */
4715 	uint64_t c = arc_c;
4716 	if (c > arc_c_min) {
4717 		c = MIN(c, MAX(asize, arc_c_min));
4718 		to_free = MIN(to_free, c - arc_c_min);
4719 		arc_c = c - to_free;
4720 	} else {
4721 		to_free = 0;
4722 	}
4723 
4724 	/*
4725 	 * Since dbuf cache size is a fraction of target ARC size, we should
4726 	 * notify dbuf about the reduction, which might be significant,
4727 	 * especially if current ARC size was much smaller than the target.
4728 	 */
4729 	dbuf_cache_reduce_target_size();
4730 
4731 	/*
4732 	 * Whether or not we reduced the target size, request eviction if the
4733 	 * current size is over it now, since caller obviously wants some RAM.
4734 	 */
4735 	if (asize > arc_c) {
4736 		/* See comment in arc_evict_cb_check() on why lock+flag */
4737 		mutex_enter(&arc_evict_lock);
4738 		arc_evict_needed = B_TRUE;
4739 		mutex_exit(&arc_evict_lock);
4740 		zthr_wakeup(arc_evict_zthr);
4741 	}
4742 
4743 	return (to_free);
4744 }
4745 
4746 /*
4747  * Determine if the system is under memory pressure and is asking
4748  * to reclaim memory. A return value of B_TRUE indicates that the system
4749  * is under memory pressure and that the arc should adjust accordingly.
4750  */
4751 boolean_t
arc_reclaim_needed(void)4752 arc_reclaim_needed(void)
4753 {
4754 	return (arc_available_memory() < 0);
4755 }
4756 
4757 void
arc_kmem_reap_soon(void)4758 arc_kmem_reap_soon(void)
4759 {
4760 	size_t			i;
4761 	kmem_cache_t		*prev_cache = NULL;
4762 	kmem_cache_t		*prev_data_cache = NULL;
4763 
4764 #ifdef _KERNEL
4765 #if defined(_ILP32)
4766 	/*
4767 	 * Reclaim unused memory from all kmem caches.
4768 	 */
4769 	kmem_reap();
4770 #endif
4771 #endif
4772 
4773 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4774 #if defined(_ILP32)
4775 		/* reach upper limit of cache size on 32-bit */
4776 		if (zio_buf_cache[i] == NULL)
4777 			break;
4778 #endif
4779 		if (zio_buf_cache[i] != prev_cache) {
4780 			prev_cache = zio_buf_cache[i];
4781 			kmem_cache_reap_now(zio_buf_cache[i]);
4782 		}
4783 		if (zio_data_buf_cache[i] != prev_data_cache) {
4784 			prev_data_cache = zio_data_buf_cache[i];
4785 			kmem_cache_reap_now(zio_data_buf_cache[i]);
4786 		}
4787 	}
4788 	kmem_cache_reap_now(buf_cache);
4789 	kmem_cache_reap_now(hdr_full_cache);
4790 	kmem_cache_reap_now(hdr_l2only_cache);
4791 	kmem_cache_reap_now(zfs_btree_leaf_cache);
4792 	abd_cache_reap_now();
4793 }
4794 
4795 static boolean_t
arc_evict_cb_check(void * arg,zthr_t * zthr)4796 arc_evict_cb_check(void *arg, zthr_t *zthr)
4797 {
4798 	(void) arg, (void) zthr;
4799 
4800 #ifdef ZFS_DEBUG
4801 	/*
4802 	 * This is necessary in order to keep the kstat information
4803 	 * up to date for tools that display kstat data such as the
4804 	 * mdb ::arc dcmd and the Linux crash utility.  These tools
4805 	 * typically do not call kstat's update function, but simply
4806 	 * dump out stats from the most recent update.  Without
4807 	 * this call, these commands may show stale stats for the
4808 	 * anon, mru, mru_ghost, mfu, and mfu_ghost lists.  Even
4809 	 * with this call, the data might be out of date if the
4810 	 * evict thread hasn't been woken recently; but that should
4811 	 * suffice.  The arc_state_t structures can be queried
4812 	 * directly if more accurate information is needed.
4813 	 */
4814 	if (arc_ksp != NULL)
4815 		arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4816 #endif
4817 
4818 	/*
4819 	 * We have to rely on arc_wait_for_eviction() to tell us when to
4820 	 * evict, rather than checking if we are overflowing here, so that we
4821 	 * are sure to not leave arc_wait_for_eviction() waiting on aew_cv.
4822 	 * If we have become "not overflowing" since arc_wait_for_eviction()
4823 	 * checked, we need to wake it up.  We could broadcast the CV here,
4824 	 * but arc_wait_for_eviction() may have not yet gone to sleep.  We
4825 	 * would need to use a mutex to ensure that this function doesn't
4826 	 * broadcast until arc_wait_for_eviction() has gone to sleep (e.g.
4827 	 * the arc_evict_lock).  However, the lock ordering of such a lock
4828 	 * would necessarily be incorrect with respect to the zthr_lock,
4829 	 * which is held before this function is called, and is held by
4830 	 * arc_wait_for_eviction() when it calls zthr_wakeup().
4831 	 */
4832 	if (arc_evict_needed)
4833 		return (B_TRUE);
4834 
4835 	/*
4836 	 * If we have buffers in uncached state, evict them periodically.
4837 	 */
4838 	return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
4839 	    zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
4840 	    ddi_get_lbolt() - arc_last_uncached_flush >
4841 	    MSEC_TO_TICK(arc_min_prefetch_ms / 2)));
4842 }
4843 
4844 /*
4845  * Keep arc_size under arc_c by running arc_evict which evicts data
4846  * from the ARC.
4847  */
4848 static void
arc_evict_cb(void * arg,zthr_t * zthr)4849 arc_evict_cb(void *arg, zthr_t *zthr)
4850 {
4851 	(void) arg;
4852 
4853 	uint64_t evicted = 0;
4854 	fstrans_cookie_t cookie = spl_fstrans_mark();
4855 
4856 	/* Always try to evict from uncached state. */
4857 	arc_last_uncached_flush = ddi_get_lbolt();
4858 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_DATA, B_FALSE);
4859 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_METADATA, B_FALSE);
4860 
4861 	/* Evict from other states only if told to. */
4862 	if (arc_evict_needed)
4863 		evicted += arc_evict();
4864 
4865 	/*
4866 	 * If evicted is zero, we couldn't evict anything
4867 	 * via arc_evict(). This could be due to hash lock
4868 	 * collisions, but more likely due to the majority of
4869 	 * arc buffers being unevictable. Therefore, even if
4870 	 * arc_size is above arc_c, another pass is unlikely to
4871 	 * be helpful and could potentially cause us to enter an
4872 	 * infinite loop.  Additionally, zthr_iscancelled() is
4873 	 * checked here so that if the arc is shutting down, the
4874 	 * broadcast will wake any remaining arc evict waiters.
4875 	 *
4876 	 * Note we cancel using zthr instead of arc_evict_zthr
4877 	 * because the latter may not yet be initializd when the
4878 	 * callback is first invoked.
4879 	 */
4880 	mutex_enter(&arc_evict_lock);
4881 	arc_evict_needed = !zthr_iscancelled(zthr) &&
4882 	    evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
4883 	if (!arc_evict_needed) {
4884 		/*
4885 		 * We're either no longer overflowing, or we
4886 		 * can't evict anything more, so we should wake
4887 		 * arc_get_data_impl() sooner.
4888 		 */
4889 		arc_evict_waiter_t *aw;
4890 		while ((aw = list_remove_head(&arc_evict_waiters)) != NULL) {
4891 			cv_broadcast(&aw->aew_cv);
4892 		}
4893 		arc_set_need_free();
4894 	}
4895 	mutex_exit(&arc_evict_lock);
4896 	spl_fstrans_unmark(cookie);
4897 }
4898 
4899 static boolean_t
arc_reap_cb_check(void * arg,zthr_t * zthr)4900 arc_reap_cb_check(void *arg, zthr_t *zthr)
4901 {
4902 	(void) arg, (void) zthr;
4903 
4904 	int64_t free_memory = arc_available_memory();
4905 	static int reap_cb_check_counter = 0;
4906 
4907 	/*
4908 	 * If a kmem reap is already active, don't schedule more.  We must
4909 	 * check for this because kmem_cache_reap_soon() won't actually
4910 	 * block on the cache being reaped (this is to prevent callers from
4911 	 * becoming implicitly blocked by a system-wide kmem reap -- which,
4912 	 * on a system with many, many full magazines, can take minutes).
4913 	 */
4914 	if (!kmem_cache_reap_active() && free_memory < 0) {
4915 
4916 		arc_no_grow = B_TRUE;
4917 		arc_warm = B_TRUE;
4918 		/*
4919 		 * Wait at least zfs_grow_retry (default 5) seconds
4920 		 * before considering growing.
4921 		 */
4922 		arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4923 		return (B_TRUE);
4924 	} else if (free_memory < arc_c >> arc_no_grow_shift) {
4925 		arc_no_grow = B_TRUE;
4926 	} else if (gethrtime() >= arc_growtime) {
4927 		arc_no_grow = B_FALSE;
4928 	}
4929 
4930 	/*
4931 	 * Called unconditionally every 60 seconds to reclaim unused
4932 	 * zstd compression and decompression context. This is done
4933 	 * here to avoid the need for an independent thread.
4934 	 */
4935 	if (!((reap_cb_check_counter++) % 60))
4936 		zfs_zstd_cache_reap_now();
4937 
4938 	return (B_FALSE);
4939 }
4940 
4941 /*
4942  * Keep enough free memory in the system by reaping the ARC's kmem
4943  * caches.  To cause more slabs to be reapable, we may reduce the
4944  * target size of the cache (arc_c), causing the arc_evict_cb()
4945  * to free more buffers.
4946  */
4947 static void
arc_reap_cb(void * arg,zthr_t * zthr)4948 arc_reap_cb(void *arg, zthr_t *zthr)
4949 {
4950 	int64_t can_free, free_memory, to_free;
4951 
4952 	(void) arg, (void) zthr;
4953 	fstrans_cookie_t cookie = spl_fstrans_mark();
4954 
4955 	/*
4956 	 * Kick off asynchronous kmem_reap()'s of all our caches.
4957 	 */
4958 	arc_kmem_reap_soon();
4959 
4960 	/*
4961 	 * Wait at least arc_kmem_cache_reap_retry_ms between
4962 	 * arc_kmem_reap_soon() calls. Without this check it is possible to
4963 	 * end up in a situation where we spend lots of time reaping
4964 	 * caches, while we're near arc_c_min.  Waiting here also gives the
4965 	 * subsequent free memory check a chance of finding that the
4966 	 * asynchronous reap has already freed enough memory, and we don't
4967 	 * need to call arc_reduce_target_size().
4968 	 */
4969 	delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);
4970 
4971 	/*
4972 	 * Reduce the target size as needed to maintain the amount of free
4973 	 * memory in the system at a fraction of the arc_size (1/128th by
4974 	 * default).  If oversubscribed (free_memory < 0) then reduce the
4975 	 * target arc_size by the deficit amount plus the fractional
4976 	 * amount.  If free memory is positive but less than the fractional
4977 	 * amount, reduce by what is needed to hit the fractional amount.
4978 	 */
4979 	free_memory = arc_available_memory();
4980 	can_free = arc_c - arc_c_min;
4981 	to_free = (MAX(can_free, 0) >> arc_shrink_shift) - free_memory;
4982 	if (to_free > 0)
4983 		arc_reduce_target_size(to_free);
4984 	spl_fstrans_unmark(cookie);
4985 }
4986 
4987 #ifdef _KERNEL
4988 /*
4989  * Determine the amount of memory eligible for eviction contained in the
4990  * ARC. All clean data reported by the ghost lists can always be safely
4991  * evicted. Due to arc_c_min, the same does not hold for all clean data
4992  * contained by the regular mru and mfu lists.
4993  *
4994  * In the case of the regular mru and mfu lists, we need to report as
4995  * much clean data as possible, such that evicting that same reported
4996  * data will not bring arc_size below arc_c_min. Thus, in certain
4997  * circumstances, the total amount of clean data in the mru and mfu
4998  * lists might not actually be evictable.
4999  *
5000  * The following two distinct cases are accounted for:
5001  *
5002  * 1. The sum of the amount of dirty data contained by both the mru and
5003  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
5004  *    is greater than or equal to arc_c_min.
5005  *    (i.e. amount of dirty data >= arc_c_min)
5006  *
5007  *    This is the easy case; all clean data contained by the mru and mfu
5008  *    lists is evictable. Evicting all clean data can only drop arc_size
5009  *    to the amount of dirty data, which is greater than arc_c_min.
5010  *
5011  * 2. The sum of the amount of dirty data contained by both the mru and
5012  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
5013  *    is less than arc_c_min.
5014  *    (i.e. arc_c_min > amount of dirty data)
5015  *
5016  *    2.1. arc_size is greater than or equal arc_c_min.
5017  *         (i.e. arc_size >= arc_c_min > amount of dirty data)
5018  *
5019  *         In this case, not all clean data from the regular mru and mfu
5020  *         lists is actually evictable; we must leave enough clean data
5021  *         to keep arc_size above arc_c_min. Thus, the maximum amount of
5022  *         evictable data from the two lists combined, is exactly the
5023  *         difference between arc_size and arc_c_min.
5024  *
5025  *    2.2. arc_size is less than arc_c_min
5026  *         (i.e. arc_c_min > arc_size > amount of dirty data)
5027  *
5028  *         In this case, none of the data contained in the mru and mfu
5029  *         lists is evictable, even if it's clean. Since arc_size is
5030  *         already below arc_c_min, evicting any more would only
5031  *         increase this negative difference.
5032  */
5033 
5034 #endif /* _KERNEL */
5035 
5036 /*
5037  * Adapt arc info given the number of bytes we are trying to add and
5038  * the state that we are coming from.  This function is only called
5039  * when we are adding new content to the cache.
5040  */
5041 static void
arc_adapt(uint64_t bytes)5042 arc_adapt(uint64_t bytes)
5043 {
5044 	/*
5045 	 * Wake reap thread if we do not have any available memory
5046 	 */
5047 	if (arc_reclaim_needed()) {
5048 		zthr_wakeup(arc_reap_zthr);
5049 		return;
5050 	}
5051 
5052 	if (arc_no_grow)
5053 		return;
5054 
5055 	if (arc_c >= arc_c_max)
5056 		return;
5057 
5058 	/*
5059 	 * If we're within (2 * maxblocksize) bytes of the target
5060 	 * cache size, increment the target cache size
5061 	 */
5062 	if (aggsum_upper_bound(&arc_sums.arcstat_size) +
5063 	    2 * SPA_MAXBLOCKSIZE >= arc_c) {
5064 		uint64_t dc = MAX(bytes, SPA_OLD_MAXBLOCKSIZE);
5065 		if (atomic_add_64_nv(&arc_c, dc) > arc_c_max)
5066 			arc_c = arc_c_max;
5067 	}
5068 }
5069 
5070 /*
5071  * Check if ARC current size has grown past our upper thresholds.
5072  */
5073 static arc_ovf_level_t
arc_is_overflowing(boolean_t lax,boolean_t use_reserve)5074 arc_is_overflowing(boolean_t lax, boolean_t use_reserve)
5075 {
5076 	/*
5077 	 * We just compare the lower bound here for performance reasons. Our
5078 	 * primary goals are to make sure that the arc never grows without
5079 	 * bound, and that it can reach its maximum size. This check
5080 	 * accomplishes both goals. The maximum amount we could run over by is
5081 	 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
5082 	 * in the ARC. In practice, that's in the tens of MB, which is low
5083 	 * enough to be safe.
5084 	 */
5085 	int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) - arc_c -
5086 	    zfs_max_recordsize;
5087 
5088 	/* Always allow at least one block of overflow. */
5089 	if (over < 0)
5090 		return (ARC_OVF_NONE);
5091 
5092 	/* If we are under memory pressure, report severe overflow. */
5093 	if (!lax)
5094 		return (ARC_OVF_SEVERE);
5095 
5096 	/* We are not under pressure, so be more or less relaxed. */
5097 	int64_t overflow = (arc_c >> zfs_arc_overflow_shift) / 2;
5098 	if (use_reserve)
5099 		overflow *= 3;
5100 	return (over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
5101 }
5102 
5103 static abd_t *
arc_get_data_abd(arc_buf_hdr_t * hdr,uint64_t size,const void * tag,int alloc_flags)5104 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
5105     int alloc_flags)
5106 {
5107 	arc_buf_contents_t type = arc_buf_type(hdr);
5108 
5109 	arc_get_data_impl(hdr, size, tag, alloc_flags);
5110 	if (alloc_flags & ARC_HDR_ALLOC_LINEAR)
5111 		return (abd_alloc_linear(size, type == ARC_BUFC_METADATA));
5112 	else
5113 		return (abd_alloc(size, type == ARC_BUFC_METADATA));
5114 }
5115 
5116 static void *
arc_get_data_buf(arc_buf_hdr_t * hdr,uint64_t size,const void * tag)5117 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
5118 {
5119 	arc_buf_contents_t type = arc_buf_type(hdr);
5120 
5121 	arc_get_data_impl(hdr, size, tag, 0);
5122 	if (type == ARC_BUFC_METADATA) {
5123 		return (zio_buf_alloc(size));
5124 	} else {
5125 		ASSERT(type == ARC_BUFC_DATA);
5126 		return (zio_data_buf_alloc(size));
5127 	}
5128 }
5129 
5130 /*
5131  * Wait for the specified amount of data (in bytes) to be evicted from the
5132  * ARC, and for there to be sufficient free memory in the system.
5133  * The lax argument specifies that caller does not have a specific reason
5134  * to wait, not aware of any memory pressure.  Low memory handlers though
5135  * should set it to B_FALSE to wait for all required evictions to complete.
5136  * The use_reserve argument allows some callers to wait less than others
5137  * to not block critical code paths, possibly blocking other resources.
5138  */
5139 void
arc_wait_for_eviction(uint64_t amount,boolean_t lax,boolean_t use_reserve)5140 arc_wait_for_eviction(uint64_t amount, boolean_t lax, boolean_t use_reserve)
5141 {
5142 	switch (arc_is_overflowing(lax, use_reserve)) {
5143 	case ARC_OVF_NONE:
5144 		return;
5145 	case ARC_OVF_SOME:
5146 		/*
5147 		 * This is a bit racy without taking arc_evict_lock, but the
5148 		 * worst that can happen is we either call zthr_wakeup() extra
5149 		 * time due to race with other thread here, or the set flag
5150 		 * get cleared by arc_evict_cb(), which is unlikely due to
5151 		 * big hysteresis, but also not important since at this level
5152 		 * of overflow the eviction is purely advisory.  Same time
5153 		 * taking the global lock here every time without waiting for
5154 		 * the actual eviction creates a significant lock contention.
5155 		 */
5156 		if (!arc_evict_needed) {
5157 			arc_evict_needed = B_TRUE;
5158 			zthr_wakeup(arc_evict_zthr);
5159 		}
5160 		return;
5161 	case ARC_OVF_SEVERE:
5162 	default:
5163 	{
5164 		arc_evict_waiter_t aw;
5165 		list_link_init(&aw.aew_node);
5166 		cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL);
5167 
5168 		uint64_t last_count = 0;
5169 		mutex_enter(&arc_evict_lock);
5170 		if (!list_is_empty(&arc_evict_waiters)) {
5171 			arc_evict_waiter_t *last =
5172 			    list_tail(&arc_evict_waiters);
5173 			last_count = last->aew_count;
5174 		} else if (!arc_evict_needed) {
5175 			arc_evict_needed = B_TRUE;
5176 			zthr_wakeup(arc_evict_zthr);
5177 		}
5178 		/*
5179 		 * Note, the last waiter's count may be less than
5180 		 * arc_evict_count if we are low on memory in which
5181 		 * case arc_evict_state_impl() may have deferred
5182 		 * wakeups (but still incremented arc_evict_count).
5183 		 */
5184 		aw.aew_count = MAX(last_count, arc_evict_count) + amount;
5185 
5186 		list_insert_tail(&arc_evict_waiters, &aw);
5187 
5188 		arc_set_need_free();
5189 
5190 		DTRACE_PROBE3(arc__wait__for__eviction,
5191 		    uint64_t, amount,
5192 		    uint64_t, arc_evict_count,
5193 		    uint64_t, aw.aew_count);
5194 
5195 		/*
5196 		 * We will be woken up either when arc_evict_count reaches
5197 		 * aew_count, or when the ARC is no longer overflowing and
5198 		 * eviction completes.
5199 		 * In case of "false" wakeup, we will still be on the list.
5200 		 */
5201 		do {
5202 			cv_wait(&aw.aew_cv, &arc_evict_lock);
5203 		} while (list_link_active(&aw.aew_node));
5204 		mutex_exit(&arc_evict_lock);
5205 
5206 		cv_destroy(&aw.aew_cv);
5207 	}
5208 	}
5209 }
5210 
5211 /*
5212  * Allocate a block and return it to the caller. If we are hitting the
5213  * hard limit for the cache size, we must sleep, waiting for the eviction
5214  * thread to catch up. If we're past the target size but below the hard
5215  * limit, we'll only signal the reclaim thread and continue on.
5216  */
5217 static void
arc_get_data_impl(arc_buf_hdr_t * hdr,uint64_t size,const void * tag,int alloc_flags)5218 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
5219     int alloc_flags)
5220 {
5221 	arc_adapt(size);
5222 
5223 	/*
5224 	 * If arc_size is currently overflowing, we must be adding data
5225 	 * faster than we are evicting.  To ensure we don't compound the
5226 	 * problem by adding more data and forcing arc_size to grow even
5227 	 * further past it's target size, we wait for the eviction thread to
5228 	 * make some progress.  We also wait for there to be sufficient free
5229 	 * memory in the system, as measured by arc_free_memory().
5230 	 *
5231 	 * Specifically, we wait for zfs_arc_eviction_pct percent of the
5232 	 * requested size to be evicted.  This should be more than 100%, to
5233 	 * ensure that that progress is also made towards getting arc_size
5234 	 * under arc_c.  See the comment above zfs_arc_eviction_pct.
5235 	 */
5236 	arc_wait_for_eviction(size * zfs_arc_eviction_pct / 100,
5237 	    B_TRUE, alloc_flags & ARC_HDR_USE_RESERVE);
5238 
5239 	arc_buf_contents_t type = arc_buf_type(hdr);
5240 	if (type == ARC_BUFC_METADATA) {
5241 		arc_space_consume(size, ARC_SPACE_META);
5242 	} else {
5243 		arc_space_consume(size, ARC_SPACE_DATA);
5244 	}
5245 
5246 	/*
5247 	 * Update the state size.  Note that ghost states have a
5248 	 * "ghost size" and so don't need to be updated.
5249 	 */
5250 	arc_state_t *state = hdr->b_l1hdr.b_state;
5251 	if (!GHOST_STATE(state)) {
5252 
5253 		(void) zfs_refcount_add_many(&state->arcs_size[type], size,
5254 		    tag);
5255 
5256 		/*
5257 		 * If this is reached via arc_read, the link is
5258 		 * protected by the hash lock. If reached via
5259 		 * arc_buf_alloc, the header should not be accessed by
5260 		 * any other thread. And, if reached via arc_read_done,
5261 		 * the hash lock will protect it if it's found in the
5262 		 * hash table; otherwise no other thread should be
5263 		 * trying to [add|remove]_reference it.
5264 		 */
5265 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5266 			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5267 			(void) zfs_refcount_add_many(&state->arcs_esize[type],
5268 			    size, tag);
5269 		}
5270 	}
5271 }
5272 
5273 static void
arc_free_data_abd(arc_buf_hdr_t * hdr,abd_t * abd,uint64_t size,const void * tag)5274 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,
5275     const void *tag)
5276 {
5277 	arc_free_data_impl(hdr, size, tag);
5278 	abd_free(abd);
5279 }
5280 
5281 static void
arc_free_data_buf(arc_buf_hdr_t * hdr,void * buf,uint64_t size,const void * tag)5282 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)
5283 {
5284 	arc_buf_contents_t type = arc_buf_type(hdr);
5285 
5286 	arc_free_data_impl(hdr, size, tag);
5287 	if (type == ARC_BUFC_METADATA) {
5288 		zio_buf_free(buf, size);
5289 	} else {
5290 		ASSERT(type == ARC_BUFC_DATA);
5291 		zio_data_buf_free(buf, size);
5292 	}
5293 }
5294 
5295 /*
5296  * Free the arc data buffer.
5297  */
5298 static void
arc_free_data_impl(arc_buf_hdr_t * hdr,uint64_t size,const void * tag)5299 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
5300 {
5301 	arc_state_t *state = hdr->b_l1hdr.b_state;
5302 	arc_buf_contents_t type = arc_buf_type(hdr);
5303 
5304 	/* protected by hash lock, if in the hash table */
5305 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5306 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5307 		ASSERT(state != arc_anon && state != arc_l2c_only);
5308 
5309 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
5310 		    size, tag);
5311 	}
5312 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);
5313 
5314 	VERIFY3U(hdr->b_type, ==, type);
5315 	if (type == ARC_BUFC_METADATA) {
5316 		arc_space_return(size, ARC_SPACE_META);
5317 	} else {
5318 		ASSERT(type == ARC_BUFC_DATA);
5319 		arc_space_return(size, ARC_SPACE_DATA);
5320 	}
5321 }
5322 
5323 /*
5324  * This routine is called whenever a buffer is accessed.
5325  */
5326 static void
arc_access(arc_buf_hdr_t * hdr,arc_flags_t arc_flags,boolean_t hit)5327 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
5328 {
5329 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
5330 	ASSERT(HDR_HAS_L1HDR(hdr));
5331 
5332 	/*
5333 	 * Update buffer prefetch status.
5334 	 */
5335 	boolean_t was_prefetch = HDR_PREFETCH(hdr);
5336 	boolean_t now_prefetch = arc_flags & ARC_FLAG_PREFETCH;
5337 	if (was_prefetch != now_prefetch) {
5338 		if (was_prefetch) {
5339 			ARCSTAT_CONDSTAT(hit, demand_hit, demand_iohit,
5340 			    HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
5341 			    prefetch);
5342 		}
5343 		if (HDR_HAS_L2HDR(hdr))
5344 			l2arc_hdr_arcstats_decrement_state(hdr);
5345 		if (was_prefetch) {
5346 			arc_hdr_clear_flags(hdr,
5347 			    ARC_FLAG_PREFETCH | ARC_FLAG_PRESCIENT_PREFETCH);
5348 		} else {
5349 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5350 		}
5351 		if (HDR_HAS_L2HDR(hdr))
5352 			l2arc_hdr_arcstats_increment_state(hdr);
5353 	}
5354 	if (now_prefetch) {
5355 		if (arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
5356 			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5357 			ARCSTAT_BUMP(arcstat_prescient_prefetch);
5358 		} else {
5359 			ARCSTAT_BUMP(arcstat_predictive_prefetch);
5360 		}
5361 	}
5362 	if (arc_flags & ARC_FLAG_L2CACHE)
5363 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5364 
5365 	clock_t now = ddi_get_lbolt();
5366 	if (hdr->b_l1hdr.b_state == arc_anon) {
5367 		arc_state_t	*new_state;
5368 		/*
5369 		 * This buffer is not in the cache, and does not appear in
5370 		 * our "ghost" lists.  Add it to the MRU or uncached state.
5371 		 */
5372 		ASSERT0(hdr->b_l1hdr.b_arc_access);
5373 		hdr->b_l1hdr.b_arc_access = now;
5374 		if (HDR_UNCACHED(hdr)) {
5375 			new_state = arc_uncached;
5376 			DTRACE_PROBE1(new_state__uncached, arc_buf_hdr_t *,
5377 			    hdr);
5378 		} else {
5379 			new_state = arc_mru;
5380 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5381 		}
5382 		arc_change_state(new_state, hdr);
5383 	} else if (hdr->b_l1hdr.b_state == arc_mru) {
5384 		/*
5385 		 * This buffer has been accessed once recently and either
5386 		 * its read is still in progress or it is in the cache.
5387 		 */
5388 		if (HDR_IO_IN_PROGRESS(hdr)) {
5389 			hdr->b_l1hdr.b_arc_access = now;
5390 			return;
5391 		}
5392 		hdr->b_l1hdr.b_mru_hits++;
5393 		ARCSTAT_BUMP(arcstat_mru_hits);
5394 
5395 		/*
5396 		 * If the previous access was a prefetch, then it already
5397 		 * handled possible promotion, so nothing more to do for now.
5398 		 */
5399 		if (was_prefetch) {
5400 			hdr->b_l1hdr.b_arc_access = now;
5401 			return;
5402 		}
5403 
5404 		/*
5405 		 * If more than ARC_MINTIME have passed from the previous
5406 		 * hit, promote the buffer to the MFU state.
5407 		 */
5408 		if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
5409 		    ARC_MINTIME)) {
5410 			hdr->b_l1hdr.b_arc_access = now;
5411 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5412 			arc_change_state(arc_mfu, hdr);
5413 		}
5414 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5415 		arc_state_t	*new_state;
5416 		/*
5417 		 * This buffer has been accessed once recently, but was
5418 		 * evicted from the cache.  Would we have bigger MRU, it
5419 		 * would be an MRU hit, so handle it the same way, except
5420 		 * we don't need to check the previous access time.
5421 		 */
5422 		hdr->b_l1hdr.b_mru_ghost_hits++;
5423 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
5424 		hdr->b_l1hdr.b_arc_access = now;
5425 		wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
5426 		    arc_hdr_size(hdr));
5427 		if (was_prefetch) {
5428 			new_state = arc_mru;
5429 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5430 		} else {
5431 			new_state = arc_mfu;
5432 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5433 		}
5434 		arc_change_state(new_state, hdr);
5435 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
5436 		/*
5437 		 * This buffer has been accessed more than once and either
5438 		 * still in the cache or being restored from one of ghosts.
5439 		 */
5440 		if (!HDR_IO_IN_PROGRESS(hdr)) {
5441 			hdr->b_l1hdr.b_mfu_hits++;
5442 			ARCSTAT_BUMP(arcstat_mfu_hits);
5443 		}
5444 		hdr->b_l1hdr.b_arc_access = now;
5445 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5446 		/*
5447 		 * This buffer has been accessed more than once recently, but
5448 		 * has been evicted from the cache.  Would we have bigger MFU
5449 		 * it would stay in cache, so move it back to MFU state.
5450 		 */
5451 		hdr->b_l1hdr.b_mfu_ghost_hits++;
5452 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
5453 		hdr->b_l1hdr.b_arc_access = now;
5454 		wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
5455 		    arc_hdr_size(hdr));
5456 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5457 		arc_change_state(arc_mfu, hdr);
5458 	} else if (hdr->b_l1hdr.b_state == arc_uncached) {
5459 		/*
5460 		 * This buffer is uncacheable, but we got a hit.  Probably
5461 		 * a demand read after prefetch.  Nothing more to do here.
5462 		 */
5463 		if (!HDR_IO_IN_PROGRESS(hdr))
5464 			ARCSTAT_BUMP(arcstat_uncached_hits);
5465 		hdr->b_l1hdr.b_arc_access = now;
5466 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5467 		/*
5468 		 * This buffer is on the 2nd Level ARC and was not accessed
5469 		 * for a long time, so treat it as new and put into MRU.
5470 		 */
5471 		hdr->b_l1hdr.b_arc_access = now;
5472 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5473 		arc_change_state(arc_mru, hdr);
5474 	} else {
5475 		cmn_err(CE_PANIC, "invalid arc state 0x%p",
5476 		    hdr->b_l1hdr.b_state);
5477 	}
5478 }
5479 
5480 /*
5481  * This routine is called by dbuf_hold() to update the arc_access() state
5482  * which otherwise would be skipped for entries in the dbuf cache.
5483  */
5484 void
arc_buf_access(arc_buf_t * buf)5485 arc_buf_access(arc_buf_t *buf)
5486 {
5487 	arc_buf_hdr_t *hdr = buf->b_hdr;
5488 
5489 	/*
5490 	 * Avoid taking the hash_lock when possible as an optimization.
5491 	 * The header must be checked again under the hash_lock in order
5492 	 * to handle the case where it is concurrently being released.
5493 	 */
5494 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
5495 		return;
5496 
5497 	kmutex_t *hash_lock = HDR_LOCK(hdr);
5498 	mutex_enter(hash_lock);
5499 
5500 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5501 		mutex_exit(hash_lock);
5502 		ARCSTAT_BUMP(arcstat_access_skip);
5503 		return;
5504 	}
5505 
5506 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5507 	    hdr->b_l1hdr.b_state == arc_mfu ||
5508 	    hdr->b_l1hdr.b_state == arc_uncached);
5509 
5510 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5511 	arc_access(hdr, 0, B_TRUE);
5512 	mutex_exit(hash_lock);
5513 
5514 	ARCSTAT_BUMP(arcstat_hits);
5515 	ARCSTAT_CONDSTAT(B_TRUE /* demand */, demand, prefetch,
5516 	    !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5517 }
5518 
5519 /* a generic arc_read_done_func_t which you can use */
5520 void
arc_bcopy_func(zio_t * zio,const zbookmark_phys_t * zb,const blkptr_t * bp,arc_buf_t * buf,void * arg)5521 arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5522     arc_buf_t *buf, void *arg)
5523 {
5524 	(void) zio, (void) zb, (void) bp;
5525 
5526 	if (buf == NULL)
5527 		return;
5528 
5529 	memcpy(arg, buf->b_data, arc_buf_size(buf));
5530 	arc_buf_destroy(buf, arg);
5531 }
5532 
5533 /* a generic arc_read_done_func_t */
5534 void
arc_getbuf_func(zio_t * zio,const zbookmark_phys_t * zb,const blkptr_t * bp,arc_buf_t * buf,void * arg)5535 arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5536     arc_buf_t *buf, void *arg)
5537 {
5538 	(void) zb, (void) bp;
5539 	arc_buf_t **bufp = arg;
5540 
5541 	if (buf == NULL) {
5542 		ASSERT(zio == NULL || zio->io_error != 0);
5543 		*bufp = NULL;
5544 	} else {
5545 		ASSERT(zio == NULL || zio->io_error == 0);
5546 		*bufp = buf;
5547 		ASSERT(buf->b_data != NULL);
5548 	}
5549 }
5550 
5551 static void
arc_hdr_verify(arc_buf_hdr_t * hdr,blkptr_t * bp)5552 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5553 {
5554 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5555 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5556 		ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
5557 	} else {
5558 		if (HDR_COMPRESSION_ENABLED(hdr)) {
5559 			ASSERT3U(arc_hdr_get_compress(hdr), ==,
5560 			    BP_GET_COMPRESS(bp));
5561 		}
5562 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5563 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5564 		ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
5565 	}
5566 }
5567 
5568 static void
arc_read_done(zio_t * zio)5569 arc_read_done(zio_t *zio)
5570 {
5571 	blkptr_t 	*bp = zio->io_bp;
5572 	arc_buf_hdr_t	*hdr = zio->io_private;
5573 	kmutex_t	*hash_lock = NULL;
5574 	arc_callback_t	*callback_list;
5575 	arc_callback_t	*acb;
5576 
5577 	/*
5578 	 * The hdr was inserted into hash-table and removed from lists
5579 	 * prior to starting I/O.  We should find this header, since
5580 	 * it's in the hash table, and it should be legit since it's
5581 	 * not possible to evict it during the I/O.  The only possible
5582 	 * reason for it not to be found is if we were freed during the
5583 	 * read.
5584 	 */
5585 	if (HDR_IN_HASH_TABLE(hdr)) {
5586 		arc_buf_hdr_t *found;
5587 
5588 		ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp));
5589 		ASSERT3U(hdr->b_dva.dva_word[0], ==,
5590 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
5591 		ASSERT3U(hdr->b_dva.dva_word[1], ==,
5592 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
5593 
5594 		found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
5595 
5596 		ASSERT((found == hdr &&
5597 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5598 		    (found == hdr && HDR_L2_READING(hdr)));
5599 		ASSERT3P(hash_lock, !=, NULL);
5600 	}
5601 
5602 	if (BP_IS_PROTECTED(bp)) {
5603 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
5604 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
5605 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
5606 		    hdr->b_crypt_hdr.b_iv);
5607 
5608 		if (zio->io_error == 0) {
5609 			if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) {
5610 				void *tmpbuf;
5611 
5612 				tmpbuf = abd_borrow_buf_copy(zio->io_abd,
5613 				    sizeof (zil_chain_t));
5614 				zio_crypt_decode_mac_zil(tmpbuf,
5615 				    hdr->b_crypt_hdr.b_mac);
5616 				abd_return_buf(zio->io_abd, tmpbuf,
5617 				    sizeof (zil_chain_t));
5618 			} else {
5619 				zio_crypt_decode_mac_bp(bp,
5620 				    hdr->b_crypt_hdr.b_mac);
5621 			}
5622 		}
5623 	}
5624 
5625 	if (zio->io_error == 0) {
5626 		/* byteswap if necessary */
5627 		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5628 			if (BP_GET_LEVEL(zio->io_bp) > 0) {
5629 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5630 			} else {
5631 				hdr->b_l1hdr.b_byteswap =
5632 				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5633 			}
5634 		} else {
5635 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5636 		}
5637 		if (!HDR_L2_READING(hdr)) {
5638 			hdr->b_complevel = zio->io_prop.zp_complevel;
5639 		}
5640 	}
5641 
5642 	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5643 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5644 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5645 
5646 	callback_list = hdr->b_l1hdr.b_acb;
5647 	ASSERT3P(callback_list, !=, NULL);
5648 	hdr->b_l1hdr.b_acb = NULL;
5649 
5650 	/*
5651 	 * If a read request has a callback (i.e. acb_done is not NULL), then we
5652 	 * make a buf containing the data according to the parameters which were
5653 	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5654 	 * aren't needlessly decompressing the data multiple times.
5655 	 */
5656 	int callback_cnt = 0;
5657 	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5658 
5659 		/* We need the last one to call below in original order. */
5660 		callback_list = acb;
5661 
5662 		if (!acb->acb_done || acb->acb_nobuf)
5663 			continue;
5664 
5665 		callback_cnt++;
5666 
5667 		if (zio->io_error != 0)
5668 			continue;
5669 
5670 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,
5671 		    &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
5672 		    acb->acb_compressed, acb->acb_noauth, B_TRUE,
5673 		    &acb->acb_buf);
5674 
5675 		/*
5676 		 * Assert non-speculative zios didn't fail because an
5677 		 * encryption key wasn't loaded
5678 		 */
5679 		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
5680 		    error != EACCES);
5681 
5682 		/*
5683 		 * If we failed to decrypt, report an error now (as the zio
5684 		 * layer would have done if it had done the transforms).
5685 		 */
5686 		if (error == ECKSUM) {
5687 			ASSERT(BP_IS_PROTECTED(bp));
5688 			error = SET_ERROR(EIO);
5689 			if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5690 				spa_log_error(zio->io_spa, &acb->acb_zb,
5691 				    BP_GET_LOGICAL_BIRTH(zio->io_bp));
5692 				(void) zfs_ereport_post(
5693 				    FM_EREPORT_ZFS_AUTHENTICATION,
5694 				    zio->io_spa, NULL, &acb->acb_zb, zio, 0);
5695 			}
5696 		}
5697 
5698 		if (error != 0) {
5699 			/*
5700 			 * Decompression or decryption failed.  Set
5701 			 * io_error so that when we call acb_done
5702 			 * (below), we will indicate that the read
5703 			 * failed. Note that in the unusual case
5704 			 * where one callback is compressed and another
5705 			 * uncompressed, we will mark all of them
5706 			 * as failed, even though the uncompressed
5707 			 * one can't actually fail.  In this case,
5708 			 * the hdr will not be anonymous, because
5709 			 * if there are multiple callbacks, it's
5710 			 * because multiple threads found the same
5711 			 * arc buf in the hash table.
5712 			 */
5713 			zio->io_error = error;
5714 		}
5715 	}
5716 
5717 	/*
5718 	 * If there are multiple callbacks, we must have the hash lock,
5719 	 * because the only way for multiple threads to find this hdr is
5720 	 * in the hash table.  This ensures that if there are multiple
5721 	 * callbacks, the hdr is not anonymous.  If it were anonymous,
5722 	 * we couldn't use arc_buf_destroy() in the error case below.
5723 	 */
5724 	ASSERT(callback_cnt < 2 || hash_lock != NULL);
5725 
5726 	if (zio->io_error == 0) {
5727 		arc_hdr_verify(hdr, zio->io_bp);
5728 	} else {
5729 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5730 		if (hdr->b_l1hdr.b_state != arc_anon)
5731 			arc_change_state(arc_anon, hdr);
5732 		if (HDR_IN_HASH_TABLE(hdr))
5733 			buf_hash_remove(hdr);
5734 	}
5735 
5736 	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5737 	(void) remove_reference(hdr, hdr);
5738 
5739 	if (hash_lock != NULL)
5740 		mutex_exit(hash_lock);
5741 
5742 	/* execute each callback and free its structure */
5743 	while ((acb = callback_list) != NULL) {
5744 		if (acb->acb_done != NULL) {
5745 			if (zio->io_error != 0 && acb->acb_buf != NULL) {
5746 				/*
5747 				 * If arc_buf_alloc_impl() fails during
5748 				 * decompression, the buf will still be
5749 				 * allocated, and needs to be freed here.
5750 				 */
5751 				arc_buf_destroy(acb->acb_buf,
5752 				    acb->acb_private);
5753 				acb->acb_buf = NULL;
5754 			}
5755 			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5756 			    acb->acb_buf, acb->acb_private);
5757 		}
5758 
5759 		if (acb->acb_zio_dummy != NULL) {
5760 			acb->acb_zio_dummy->io_error = zio->io_error;
5761 			zio_nowait(acb->acb_zio_dummy);
5762 		}
5763 
5764 		callback_list = acb->acb_prev;
5765 		if (acb->acb_wait) {
5766 			mutex_enter(&acb->acb_wait_lock);
5767 			acb->acb_wait_error = zio->io_error;
5768 			acb->acb_wait = B_FALSE;
5769 			cv_signal(&acb->acb_wait_cv);
5770 			mutex_exit(&acb->acb_wait_lock);
5771 			/* acb will be freed by the waiting thread. */
5772 		} else {
5773 			kmem_free(acb, sizeof (arc_callback_t));
5774 		}
5775 	}
5776 }
5777 
5778 /*
5779  * Lookup the block at the specified DVA (in bp), and return the manner in
5780  * which the block is cached. A zero return indicates not cached.
5781  */
5782 int
arc_cached(spa_t * spa,const blkptr_t * bp)5783 arc_cached(spa_t *spa, const blkptr_t *bp)
5784 {
5785 	arc_buf_hdr_t *hdr = NULL;
5786 	kmutex_t *hash_lock = NULL;
5787 	uint64_t guid = spa_load_guid(spa);
5788 	int flags = 0;
5789 
5790 	if (BP_IS_EMBEDDED(bp))
5791 		return (ARC_CACHED_EMBEDDED);
5792 
5793 	hdr = buf_hash_find(guid, bp, &hash_lock);
5794 	if (hdr == NULL)
5795 		return (0);
5796 
5797 	if (HDR_HAS_L1HDR(hdr)) {
5798 		arc_state_t *state = hdr->b_l1hdr.b_state;
5799 		/*
5800 		 * We switch to ensure that any future arc_state_type_t
5801 		 * changes are handled. This is just a shift to promote
5802 		 * more compile-time checking.
5803 		 */
5804 		switch (state->arcs_state) {
5805 		case ARC_STATE_ANON:
5806 			break;
5807 		case ARC_STATE_MRU:
5808 			flags |= ARC_CACHED_IN_MRU | ARC_CACHED_IN_L1;
5809 			break;
5810 		case ARC_STATE_MFU:
5811 			flags |= ARC_CACHED_IN_MFU | ARC_CACHED_IN_L1;
5812 			break;
5813 		case ARC_STATE_UNCACHED:
5814 			/* The header is still in L1, probably not for long */
5815 			flags |= ARC_CACHED_IN_L1;
5816 			break;
5817 		default:
5818 			break;
5819 		}
5820 	}
5821 	if (HDR_HAS_L2HDR(hdr))
5822 		flags |= ARC_CACHED_IN_L2;
5823 
5824 	mutex_exit(hash_lock);
5825 
5826 	return (flags);
5827 }
5828 
5829 /*
5830  * "Read" the block at the specified DVA (in bp) via the
5831  * cache.  If the block is found in the cache, invoke the provided
5832  * callback immediately and return.  Note that the `zio' parameter
5833  * in the callback will be NULL in this case, since no IO was
5834  * required.  If the block is not in the cache pass the read request
5835  * on to the spa with a substitute callback function, so that the
5836  * requested block will be added to the cache.
5837  *
5838  * If a read request arrives for a block that has a read in-progress,
5839  * either wait for the in-progress read to complete (and return the
5840  * results); or, if this is a read with a "done" func, add a record
5841  * to the read to invoke the "done" func when the read completes,
5842  * and return; or just return.
5843  *
5844  * arc_read_done() will invoke all the requested "done" functions
5845  * for readers of this block.
5846  */
5847 int
arc_read(zio_t * pio,spa_t * spa,const blkptr_t * bp,arc_read_done_func_t * done,void * private,zio_priority_t priority,int zio_flags,arc_flags_t * arc_flags,const zbookmark_phys_t * zb)5848 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
5849     arc_read_done_func_t *done, void *private, zio_priority_t priority,
5850     int zio_flags, arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5851 {
5852 	arc_buf_hdr_t *hdr = NULL;
5853 	kmutex_t *hash_lock = NULL;
5854 	zio_t *rzio;
5855 	uint64_t guid = spa_load_guid(spa);
5856 	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0;
5857 	boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) &&
5858 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5859 	boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) &&
5860 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5861 	boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
5862 	boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
5863 	arc_buf_t *buf = NULL;
5864 	int rc = 0;
5865 	boolean_t bp_validation = B_FALSE;
5866 
5867 	ASSERT(!embedded_bp ||
5868 	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5869 	ASSERT(!BP_IS_HOLE(bp));
5870 	ASSERT(!BP_IS_REDACTED(bp));
5871 
5872 	/*
5873 	 * Normally SPL_FSTRANS will already be set since kernel threads which
5874 	 * expect to call the DMU interfaces will set it when created.  System
5875 	 * calls are similarly handled by setting/cleaning the bit in the
5876 	 * registered callback (module/os/.../zfs/zpl_*).
5877 	 *
5878 	 * External consumers such as Lustre which call the exported DMU
5879 	 * interfaces may not have set SPL_FSTRANS.  To avoid a deadlock
5880 	 * on the hash_lock always set and clear the bit.
5881 	 */
5882 	fstrans_cookie_t cookie = spl_fstrans_mark();
5883 top:
5884 	if (!embedded_bp) {
5885 		/*
5886 		 * Embedded BP's have no DVA and require no I/O to "read".
5887 		 * Create an anonymous arc buf to back it.
5888 		 */
5889 		hdr = buf_hash_find(guid, bp, &hash_lock);
5890 	}
5891 
5892 	/*
5893 	 * Determine if we have an L1 cache hit or a cache miss. For simplicity
5894 	 * we maintain encrypted data separately from compressed / uncompressed
5895 	 * data. If the user is requesting raw encrypted data and we don't have
5896 	 * that in the header we will read from disk to guarantee that we can
5897 	 * get it even if the encryption keys aren't loaded.
5898 	 */
5899 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
5900 	    (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
5901 		boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);
5902 
5903 		/*
5904 		 * Verify the block pointer contents are reasonable.  This
5905 		 * should always be the case since the blkptr is protected by
5906 		 * a checksum.
5907 		 */
5908 		if (zfs_blkptr_verify(spa, bp, BLK_CONFIG_SKIP,
5909 		    BLK_VERIFY_LOG)) {
5910 			mutex_exit(hash_lock);
5911 			rc = SET_ERROR(ECKSUM);
5912 			goto done;
5913 		}
5914 
5915 		if (HDR_IO_IN_PROGRESS(hdr)) {
5916 			if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
5917 				mutex_exit(hash_lock);
5918 				ARCSTAT_BUMP(arcstat_cached_only_in_progress);
5919 				rc = SET_ERROR(ENOENT);
5920 				goto done;
5921 			}
5922 
5923 			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5924 			ASSERT3P(head_zio, !=, NULL);
5925 			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5926 			    priority == ZIO_PRIORITY_SYNC_READ) {
5927 				/*
5928 				 * This is a sync read that needs to wait for
5929 				 * an in-flight async read. Request that the
5930 				 * zio have its priority upgraded.
5931 				 */
5932 				zio_change_priority(head_zio, priority);
5933 				DTRACE_PROBE1(arc__async__upgrade__sync,
5934 				    arc_buf_hdr_t *, hdr);
5935 				ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5936 			}
5937 
5938 			DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
5939 			arc_access(hdr, *arc_flags, B_FALSE);
5940 
5941 			/*
5942 			 * If there are multiple threads reading the same block
5943 			 * and that block is not yet in the ARC, then only one
5944 			 * thread will do the physical I/O and all other
5945 			 * threads will wait until that I/O completes.
5946 			 * Synchronous reads use the acb_wait_cv whereas nowait
5947 			 * reads register a callback. Both are signalled/called
5948 			 * in arc_read_done.
5949 			 *
5950 			 * Errors of the physical I/O may need to be propagated.
5951 			 * Synchronous read errors are returned here from
5952 			 * arc_read_done via acb_wait_error.  Nowait reads
5953 			 * attach the acb_zio_dummy zio to pio and
5954 			 * arc_read_done propagates the physical I/O's io_error
5955 			 * to acb_zio_dummy, and thereby to pio.
5956 			 */
5957 			arc_callback_t *acb = NULL;
5958 			if (done || pio || *arc_flags & ARC_FLAG_WAIT) {
5959 				acb = kmem_zalloc(sizeof (arc_callback_t),
5960 				    KM_SLEEP);
5961 				acb->acb_done = done;
5962 				acb->acb_private = private;
5963 				acb->acb_compressed = compressed_read;
5964 				acb->acb_encrypted = encrypted_read;
5965 				acb->acb_noauth = noauth_read;
5966 				acb->acb_nobuf = no_buf;
5967 				if (*arc_flags & ARC_FLAG_WAIT) {
5968 					acb->acb_wait = B_TRUE;
5969 					mutex_init(&acb->acb_wait_lock, NULL,
5970 					    MUTEX_DEFAULT, NULL);
5971 					cv_init(&acb->acb_wait_cv, NULL,
5972 					    CV_DEFAULT, NULL);
5973 				}
5974 				acb->acb_zb = *zb;
5975 				if (pio != NULL) {
5976 					acb->acb_zio_dummy = zio_null(pio,
5977 					    spa, NULL, NULL, NULL, zio_flags);
5978 				}
5979 				acb->acb_zio_head = head_zio;
5980 				acb->acb_next = hdr->b_l1hdr.b_acb;
5981 				hdr->b_l1hdr.b_acb->acb_prev = acb;
5982 				hdr->b_l1hdr.b_acb = acb;
5983 			}
5984 			mutex_exit(hash_lock);
5985 
5986 			ARCSTAT_BUMP(arcstat_iohits);
5987 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5988 			    demand, prefetch, is_data, data, metadata, iohits);
5989 
5990 			if (*arc_flags & ARC_FLAG_WAIT) {
5991 				mutex_enter(&acb->acb_wait_lock);
5992 				while (acb->acb_wait) {
5993 					cv_wait(&acb->acb_wait_cv,
5994 					    &acb->acb_wait_lock);
5995 				}
5996 				rc = acb->acb_wait_error;
5997 				mutex_exit(&acb->acb_wait_lock);
5998 				mutex_destroy(&acb->acb_wait_lock);
5999 				cv_destroy(&acb->acb_wait_cv);
6000 				kmem_free(acb, sizeof (arc_callback_t));
6001 			}
6002 			goto out;
6003 		}
6004 
6005 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
6006 		    hdr->b_l1hdr.b_state == arc_mfu ||
6007 		    hdr->b_l1hdr.b_state == arc_uncached);
6008 
6009 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
6010 		arc_access(hdr, *arc_flags, B_TRUE);
6011 
6012 		if (done && !no_buf) {
6013 			ASSERT(!embedded_bp || !BP_IS_HOLE(bp));
6014 
6015 			/* Get a buf with the desired data in it. */
6016 			rc = arc_buf_alloc_impl(hdr, spa, zb, private,
6017 			    encrypted_read, compressed_read, noauth_read,
6018 			    B_TRUE, &buf);
6019 			if (rc == ECKSUM) {
6020 				/*
6021 				 * Convert authentication and decryption errors
6022 				 * to EIO (and generate an ereport if needed)
6023 				 * before leaving the ARC.
6024 				 */
6025 				rc = SET_ERROR(EIO);
6026 				if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
6027 					spa_log_error(spa, zb, hdr->b_birth);
6028 					(void) zfs_ereport_post(
6029 					    FM_EREPORT_ZFS_AUTHENTICATION,
6030 					    spa, NULL, zb, NULL, 0);
6031 				}
6032 			}
6033 			if (rc != 0) {
6034 				arc_buf_destroy_impl(buf);
6035 				buf = NULL;
6036 				(void) remove_reference(hdr, private);
6037 			}
6038 
6039 			/* assert any errors weren't due to unloaded keys */
6040 			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
6041 			    rc != EACCES);
6042 		}
6043 		mutex_exit(hash_lock);
6044 		ARCSTAT_BUMP(arcstat_hits);
6045 		ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
6046 		    demand, prefetch, is_data, data, metadata, hits);
6047 		*arc_flags |= ARC_FLAG_CACHED;
6048 		goto done;
6049 	} else {
6050 		uint64_t lsize = BP_GET_LSIZE(bp);
6051 		uint64_t psize = BP_GET_PSIZE(bp);
6052 		arc_callback_t *acb;
6053 		vdev_t *vd = NULL;
6054 		uint64_t addr = 0;
6055 		boolean_t devw = B_FALSE;
6056 		uint64_t size;
6057 		abd_t *hdr_abd;
6058 		int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;
6059 		arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
6060 		int config_lock;
6061 		int error;
6062 
6063 		if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
6064 			if (hash_lock != NULL)
6065 				mutex_exit(hash_lock);
6066 			rc = SET_ERROR(ENOENT);
6067 			goto done;
6068 		}
6069 
6070 		if (zio_flags & ZIO_FLAG_CONFIG_WRITER) {
6071 			config_lock = BLK_CONFIG_HELD;
6072 		} else if (hash_lock != NULL) {
6073 			/*
6074 			 * Prevent lock order reversal
6075 			 */
6076 			config_lock = BLK_CONFIG_NEEDED_TRY;
6077 		} else {
6078 			config_lock = BLK_CONFIG_NEEDED;
6079 		}
6080 
6081 		/*
6082 		 * Verify the block pointer contents are reasonable.  This
6083 		 * should always be the case since the blkptr is protected by
6084 		 * a checksum.
6085 		 */
6086 		if (!bp_validation && (error = zfs_blkptr_verify(spa, bp,
6087 		    config_lock, BLK_VERIFY_LOG))) {
6088 			if (hash_lock != NULL)
6089 				mutex_exit(hash_lock);
6090 			if (error == EBUSY && !zfs_blkptr_verify(spa, bp,
6091 			    BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
6092 				bp_validation = B_TRUE;
6093 				goto top;
6094 			}
6095 			rc = SET_ERROR(ECKSUM);
6096 			goto done;
6097 		}
6098 
6099 		if (hdr == NULL) {
6100 			/*
6101 			 * This block is not in the cache or it has
6102 			 * embedded data.
6103 			 */
6104 			arc_buf_hdr_t *exists = NULL;
6105 			hdr = arc_hdr_alloc(guid, psize, lsize,
6106 			    BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type);
6107 
6108 			if (!embedded_bp) {
6109 				hdr->b_dva = *BP_IDENTITY(bp);
6110 				hdr->b_birth = BP_GET_BIRTH(bp);
6111 				exists = buf_hash_insert(hdr, &hash_lock);
6112 			}
6113 			if (exists != NULL) {
6114 				/* somebody beat us to the hash insert */
6115 				mutex_exit(hash_lock);
6116 				buf_discard_identity(hdr);
6117 				arc_hdr_destroy(hdr);
6118 				goto top; /* restart the IO request */
6119 			}
6120 		} else {
6121 			/*
6122 			 * This block is in the ghost cache or encrypted data
6123 			 * was requested and we didn't have it. If it was
6124 			 * L2-only (and thus didn't have an L1 hdr),
6125 			 * we realloc the header to add an L1 hdr.
6126 			 */
6127 			if (!HDR_HAS_L1HDR(hdr)) {
6128 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
6129 				    hdr_full_cache);
6130 			}
6131 
6132 			if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
6133 				ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6134 				ASSERT(!HDR_HAS_RABD(hdr));
6135 				ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6136 				ASSERT0(zfs_refcount_count(
6137 				    &hdr->b_l1hdr.b_refcnt));
6138 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
6139 #ifdef ZFS_DEBUG
6140 				ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
6141 #endif
6142 			} else if (HDR_IO_IN_PROGRESS(hdr)) {
6143 				/*
6144 				 * If this header already had an IO in progress
6145 				 * and we are performing another IO to fetch
6146 				 * encrypted data we must wait until the first
6147 				 * IO completes so as not to confuse
6148 				 * arc_read_done(). This should be very rare
6149 				 * and so the performance impact shouldn't
6150 				 * matter.
6151 				 */
6152 				arc_callback_t *acb = kmem_zalloc(
6153 				    sizeof (arc_callback_t), KM_SLEEP);
6154 				acb->acb_wait = B_TRUE;
6155 				mutex_init(&acb->acb_wait_lock, NULL,
6156 				    MUTEX_DEFAULT, NULL);
6157 				cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
6158 				    NULL);
6159 				acb->acb_zio_head =
6160 				    hdr->b_l1hdr.b_acb->acb_zio_head;
6161 				acb->acb_next = hdr->b_l1hdr.b_acb;
6162 				hdr->b_l1hdr.b_acb->acb_prev = acb;
6163 				hdr->b_l1hdr.b_acb = acb;
6164 				mutex_exit(hash_lock);
6165 				mutex_enter(&acb->acb_wait_lock);
6166 				while (acb->acb_wait) {
6167 					cv_wait(&acb->acb_wait_cv,
6168 					    &acb->acb_wait_lock);
6169 				}
6170 				mutex_exit(&acb->acb_wait_lock);
6171 				mutex_destroy(&acb->acb_wait_lock);
6172 				cv_destroy(&acb->acb_wait_cv);
6173 				kmem_free(acb, sizeof (arc_callback_t));
6174 				goto top;
6175 			}
6176 		}
6177 		if (*arc_flags & ARC_FLAG_UNCACHED) {
6178 			arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
6179 			if (!encrypted_read)
6180 				alloc_flags |= ARC_HDR_ALLOC_LINEAR;
6181 		}
6182 
6183 		/*
6184 		 * Take additional reference for IO_IN_PROGRESS.  It stops
6185 		 * arc_access() from putting this header without any buffers
6186 		 * and so other references but obviously nonevictable onto
6187 		 * the evictable list of MRU or MFU state.
6188 		 */
6189 		add_reference(hdr, hdr);
6190 		if (!embedded_bp)
6191 			arc_access(hdr, *arc_flags, B_FALSE);
6192 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6193 		arc_hdr_alloc_abd(hdr, alloc_flags);
6194 		if (encrypted_read) {
6195 			ASSERT(HDR_HAS_RABD(hdr));
6196 			size = HDR_GET_PSIZE(hdr);
6197 			hdr_abd = hdr->b_crypt_hdr.b_rabd;
6198 			zio_flags |= ZIO_FLAG_RAW;
6199 		} else {
6200 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6201 			size = arc_hdr_size(hdr);
6202 			hdr_abd = hdr->b_l1hdr.b_pabd;
6203 
6204 			if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
6205 				zio_flags |= ZIO_FLAG_RAW_COMPRESS;
6206 			}
6207 
6208 			/*
6209 			 * For authenticated bp's, we do not ask the ZIO layer
6210 			 * to authenticate them since this will cause the entire
6211 			 * IO to fail if the key isn't loaded. Instead, we
6212 			 * defer authentication until arc_buf_fill(), which will
6213 			 * verify the data when the key is available.
6214 			 */
6215 			if (BP_IS_AUTHENTICATED(bp))
6216 				zio_flags |= ZIO_FLAG_RAW_ENCRYPT;
6217 		}
6218 
6219 		if (BP_IS_AUTHENTICATED(bp))
6220 			arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6221 		if (BP_GET_LEVEL(bp) > 0)
6222 			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
6223 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
6224 
6225 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
6226 		acb->acb_done = done;
6227 		acb->acb_private = private;
6228 		acb->acb_compressed = compressed_read;
6229 		acb->acb_encrypted = encrypted_read;
6230 		acb->acb_noauth = noauth_read;
6231 		acb->acb_nobuf = no_buf;
6232 		acb->acb_zb = *zb;
6233 
6234 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6235 		hdr->b_l1hdr.b_acb = acb;
6236 
6237 		if (HDR_HAS_L2HDR(hdr) &&
6238 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
6239 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
6240 			addr = hdr->b_l2hdr.b_daddr;
6241 			/*
6242 			 * Lock out L2ARC device removal.
6243 			 */
6244 			if (vdev_is_dead(vd) ||
6245 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
6246 				vd = NULL;
6247 		}
6248 
6249 		/*
6250 		 * We count both async reads and scrub IOs as asynchronous so
6251 		 * that both can be upgraded in the event of a cache hit while
6252 		 * the read IO is still in-flight.
6253 		 */
6254 		if (priority == ZIO_PRIORITY_ASYNC_READ ||
6255 		    priority == ZIO_PRIORITY_SCRUB)
6256 			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6257 		else
6258 			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6259 
6260 		/*
6261 		 * At this point, we have a level 1 cache miss or a blkptr
6262 		 * with embedded data.  Try again in L2ARC if possible.
6263 		 */
6264 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
6265 
6266 		/*
6267 		 * Skip ARC stat bump for block pointers with embedded
6268 		 * data. The data are read from the blkptr itself via
6269 		 * decode_embedded_bp_compressed().
6270 		 */
6271 		if (!embedded_bp) {
6272 			DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
6273 			    blkptr_t *, bp, uint64_t, lsize,
6274 			    zbookmark_phys_t *, zb);
6275 			ARCSTAT_BUMP(arcstat_misses);
6276 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
6277 			    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
6278 			    metadata, misses);
6279 			zfs_racct_read(spa, size, 1,
6280 			    (*arc_flags & ARC_FLAG_UNCACHED) ?
6281 			    DMU_UNCACHEDIO : 0);
6282 		}
6283 
6284 		/* Check if the spa even has l2 configured */
6285 		const boolean_t spa_has_l2 = l2arc_ndev != 0 &&
6286 		    spa->spa_l2cache.sav_count > 0;
6287 
6288 		if (vd != NULL && spa_has_l2 && !(l2arc_norw && devw)) {
6289 			/*
6290 			 * Read from the L2ARC if the following are true:
6291 			 * 1. The L2ARC vdev was previously cached.
6292 			 * 2. This buffer still has L2ARC metadata.
6293 			 * 3. This buffer isn't currently writing to the L2ARC.
6294 			 * 4. The L2ARC entry wasn't evicted, which may
6295 			 *    also have invalidated the vdev.
6296 			 */
6297 			if (HDR_HAS_L2HDR(hdr) &&
6298 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
6299 				l2arc_read_callback_t *cb;
6300 				abd_t *abd;
6301 				uint64_t asize;
6302 
6303 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
6304 				ARCSTAT_BUMP(arcstat_l2_hits);
6305 				hdr->b_l2hdr.b_hits++;
6306 
6307 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
6308 				    KM_SLEEP);
6309 				cb->l2rcb_hdr = hdr;
6310 				cb->l2rcb_bp = *bp;
6311 				cb->l2rcb_zb = *zb;
6312 				cb->l2rcb_flags = zio_flags;
6313 
6314 				/*
6315 				 * When Compressed ARC is disabled, but the
6316 				 * L2ARC block is compressed, arc_hdr_size()
6317 				 * will have returned LSIZE rather than PSIZE.
6318 				 */
6319 				if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
6320 				    !HDR_COMPRESSION_ENABLED(hdr) &&
6321 				    HDR_GET_PSIZE(hdr) != 0) {
6322 					size = HDR_GET_PSIZE(hdr);
6323 				}
6324 
6325 				asize = vdev_psize_to_asize(vd, size);
6326 				if (asize != size) {
6327 					abd = abd_alloc_for_io(asize,
6328 					    HDR_ISTYPE_METADATA(hdr));
6329 					cb->l2rcb_abd = abd;
6330 				} else {
6331 					abd = hdr_abd;
6332 				}
6333 
6334 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
6335 				    addr + asize <= vd->vdev_psize -
6336 				    VDEV_LABEL_END_SIZE);
6337 
6338 				/*
6339 				 * l2arc read.  The SCL_L2ARC lock will be
6340 				 * released by l2arc_read_done().
6341 				 * Issue a null zio if the underlying buffer
6342 				 * was squashed to zero size by compression.
6343 				 */
6344 				ASSERT3U(arc_hdr_get_compress(hdr), !=,
6345 				    ZIO_COMPRESS_EMPTY);
6346 				rzio = zio_read_phys(pio, vd, addr,
6347 				    asize, abd,
6348 				    ZIO_CHECKSUM_OFF,
6349 				    l2arc_read_done, cb, priority,
6350 				    zio_flags | ZIO_FLAG_CANFAIL |
6351 				    ZIO_FLAG_DONT_PROPAGATE |
6352 				    ZIO_FLAG_DONT_RETRY, B_FALSE);
6353 				acb->acb_zio_head = rzio;
6354 
6355 				if (hash_lock != NULL)
6356 					mutex_exit(hash_lock);
6357 
6358 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
6359 				    zio_t *, rzio);
6360 				ARCSTAT_INCR(arcstat_l2_read_bytes,
6361 				    HDR_GET_PSIZE(hdr));
6362 
6363 				if (*arc_flags & ARC_FLAG_NOWAIT) {
6364 					zio_nowait(rzio);
6365 					goto out;
6366 				}
6367 
6368 				ASSERT(*arc_flags & ARC_FLAG_WAIT);
6369 				if (zio_wait(rzio) == 0)
6370 					goto out;
6371 
6372 				/* l2arc read error; goto zio_read() */
6373 				if (hash_lock != NULL)
6374 					mutex_enter(hash_lock);
6375 			} else {
6376 				DTRACE_PROBE1(l2arc__miss,
6377 				    arc_buf_hdr_t *, hdr);
6378 				ARCSTAT_BUMP(arcstat_l2_misses);
6379 				if (HDR_L2_WRITING(hdr))
6380 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
6381 				spa_config_exit(spa, SCL_L2ARC, vd);
6382 			}
6383 		} else {
6384 			if (vd != NULL)
6385 				spa_config_exit(spa, SCL_L2ARC, vd);
6386 
6387 			/*
6388 			 * Only a spa with l2 should contribute to l2
6389 			 * miss stats.  (Including the case of having a
6390 			 * faulted cache device - that's also a miss.)
6391 			 */
6392 			if (spa_has_l2) {
6393 				/*
6394 				 * Skip ARC stat bump for block pointers with
6395 				 * embedded data. The data are read from the
6396 				 * blkptr itself via
6397 				 * decode_embedded_bp_compressed().
6398 				 */
6399 				if (!embedded_bp) {
6400 					DTRACE_PROBE1(l2arc__miss,
6401 					    arc_buf_hdr_t *, hdr);
6402 					ARCSTAT_BUMP(arcstat_l2_misses);
6403 				}
6404 			}
6405 		}
6406 
6407 		rzio = zio_read(pio, spa, bp, hdr_abd, size,
6408 		    arc_read_done, hdr, priority, zio_flags, zb);
6409 		acb->acb_zio_head = rzio;
6410 
6411 		if (hash_lock != NULL)
6412 			mutex_exit(hash_lock);
6413 
6414 		if (*arc_flags & ARC_FLAG_WAIT) {
6415 			rc = zio_wait(rzio);
6416 			goto out;
6417 		}
6418 
6419 		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
6420 		zio_nowait(rzio);
6421 	}
6422 
6423 out:
6424 	/* embedded bps don't actually go to disk */
6425 	if (!embedded_bp)
6426 		spa_read_history_add(spa, zb, *arc_flags);
6427 	spl_fstrans_unmark(cookie);
6428 	return (rc);
6429 
6430 done:
6431 	if (done)
6432 		done(NULL, zb, bp, buf, private);
6433 	if (pio && rc != 0) {
6434 		zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
6435 		zio->io_error = rc;
6436 		zio_nowait(zio);
6437 	}
6438 	goto out;
6439 }
6440 
6441 arc_prune_t *
arc_add_prune_callback(arc_prune_func_t * func,void * private)6442 arc_add_prune_callback(arc_prune_func_t *func, void *private)
6443 {
6444 	arc_prune_t *p;
6445 
6446 	p = kmem_alloc(sizeof (*p), KM_SLEEP);
6447 	p->p_pfunc = func;
6448 	p->p_private = private;
6449 	list_link_init(&p->p_node);
6450 	zfs_refcount_create(&p->p_refcnt);
6451 
6452 	mutex_enter(&arc_prune_mtx);
6453 	zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
6454 	list_insert_head(&arc_prune_list, p);
6455 	mutex_exit(&arc_prune_mtx);
6456 
6457 	return (p);
6458 }
6459 
6460 void
arc_remove_prune_callback(arc_prune_t * p)6461 arc_remove_prune_callback(arc_prune_t *p)
6462 {
6463 	boolean_t wait = B_FALSE;
6464 	mutex_enter(&arc_prune_mtx);
6465 	list_remove(&arc_prune_list, p);
6466 	if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6467 		wait = B_TRUE;
6468 	mutex_exit(&arc_prune_mtx);
6469 
6470 	/* wait for arc_prune_task to finish */
6471 	if (wait)
6472 		taskq_wait_outstanding(arc_prune_taskq, 0);
6473 	ASSERT0(zfs_refcount_count(&p->p_refcnt));
6474 	zfs_refcount_destroy(&p->p_refcnt);
6475 	kmem_free(p, sizeof (*p));
6476 }
6477 
6478 /*
6479  * Helper function for arc_prune_async() it is responsible for safely
6480  * handling the execution of a registered arc_prune_func_t.
6481  */
6482 static void
arc_prune_task(void * ptr)6483 arc_prune_task(void *ptr)
6484 {
6485 	arc_prune_t *ap = (arc_prune_t *)ptr;
6486 	arc_prune_func_t *func = ap->p_pfunc;
6487 
6488 	if (func != NULL)
6489 		func(ap->p_adjust, ap->p_private);
6490 
6491 	(void) zfs_refcount_remove(&ap->p_refcnt, func);
6492 }
6493 
6494 /*
6495  * Notify registered consumers they must drop holds on a portion of the ARC
6496  * buffers they reference.  This provides a mechanism to ensure the ARC can
6497  * honor the metadata limit and reclaim otherwise pinned ARC buffers.
6498  *
6499  * This operation is performed asynchronously so it may be safely called
6500  * in the context of the arc_reclaim_thread().  A reference is taken here
6501  * for each registered arc_prune_t and the arc_prune_task() is responsible
6502  * for releasing it once the registered arc_prune_func_t has completed.
6503  */
6504 static void
arc_prune_async(uint64_t adjust)6505 arc_prune_async(uint64_t adjust)
6506 {
6507 	arc_prune_t *ap;
6508 
6509 	mutex_enter(&arc_prune_mtx);
6510 	for (ap = list_head(&arc_prune_list); ap != NULL;
6511 	    ap = list_next(&arc_prune_list, ap)) {
6512 
6513 		if (zfs_refcount_count(&ap->p_refcnt) >= 2)
6514 			continue;
6515 
6516 		zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
6517 		ap->p_adjust = adjust;
6518 		if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
6519 		    ap, TQ_SLEEP) == TASKQID_INVALID) {
6520 			(void) zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
6521 			continue;
6522 		}
6523 		ARCSTAT_BUMP(arcstat_prune);
6524 	}
6525 	mutex_exit(&arc_prune_mtx);
6526 }
6527 
6528 /*
6529  * Notify the arc that a block was freed, and thus will never be used again.
6530  */
6531 void
arc_freed(spa_t * spa,const blkptr_t * bp)6532 arc_freed(spa_t *spa, const blkptr_t *bp)
6533 {
6534 	arc_buf_hdr_t *hdr;
6535 	kmutex_t *hash_lock;
6536 	uint64_t guid = spa_load_guid(spa);
6537 
6538 	ASSERT(!BP_IS_EMBEDDED(bp));
6539 
6540 	hdr = buf_hash_find(guid, bp, &hash_lock);
6541 	if (hdr == NULL)
6542 		return;
6543 
6544 	/*
6545 	 * We might be trying to free a block that is still doing I/O
6546 	 * (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
6547 	 * dmu_sync-ed block). A block may also have a reference if it is
6548 	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6549 	 * have written the new block to its final resting place on disk but
6550 	 * without the dedup flag set. This would have left the hdr in the MRU
6551 	 * state and discoverable. When the txg finally syncs it detects that
6552 	 * the block was overridden in open context and issues an override I/O.
6553 	 * Since this is a dedup block, the override I/O will determine if the
6554 	 * block is already in the DDT. If so, then it will replace the io_bp
6555 	 * with the bp from the DDT and allow the I/O to finish. When the I/O
6556 	 * reaches the done callback, dbuf_write_override_done, it will
6557 	 * check to see if the io_bp and io_bp_override are identical.
6558 	 * If they are not, then it indicates that the bp was replaced with
6559 	 * the bp in the DDT and the override bp is freed. This allows
6560 	 * us to arrive here with a reference on a block that is being
6561 	 * freed. So if we have an I/O in progress, or a reference to
6562 	 * this hdr, then we don't destroy the hdr.
6563 	 */
6564 	if (!HDR_HAS_L1HDR(hdr) ||
6565 	    zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6566 		arc_change_state(arc_anon, hdr);
6567 		arc_hdr_destroy(hdr);
6568 		mutex_exit(hash_lock);
6569 	} else {
6570 		mutex_exit(hash_lock);
6571 	}
6572 
6573 }
6574 
6575 /*
6576  * Release this buffer from the cache, making it an anonymous buffer.  This
6577  * must be done after a read and prior to modifying the buffer contents.
6578  * If the buffer has more than one reference, we must make
6579  * a new hdr for the buffer.
6580  */
6581 void
arc_release(arc_buf_t * buf,const void * tag)6582 arc_release(arc_buf_t *buf, const void *tag)
6583 {
6584 	arc_buf_hdr_t *hdr = buf->b_hdr;
6585 
6586 	/*
6587 	 * It would be nice to assert that if its DMU metadata (level >
6588 	 * 0 || it's the dnode file), then it must be syncing context.
6589 	 * But we don't know that information at this level.
6590 	 */
6591 
6592 	ASSERT(HDR_HAS_L1HDR(hdr));
6593 
6594 	/*
6595 	 * We don't grab the hash lock prior to this check, because if
6596 	 * the buffer's header is in the arc_anon state, it won't be
6597 	 * linked into the hash table.
6598 	 */
6599 	if (hdr->b_l1hdr.b_state == arc_anon) {
6600 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6601 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
6602 		ASSERT(!HDR_HAS_L2HDR(hdr));
6603 
6604 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6605 		ASSERT(ARC_BUF_LAST(buf));
6606 		ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6607 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6608 
6609 		hdr->b_l1hdr.b_arc_access = 0;
6610 
6611 		/*
6612 		 * If the buf is being overridden then it may already
6613 		 * have a hdr that is not empty.
6614 		 */
6615 		buf_discard_identity(hdr);
6616 		arc_buf_thaw(buf);
6617 
6618 		return;
6619 	}
6620 
6621 	kmutex_t *hash_lock = HDR_LOCK(hdr);
6622 	mutex_enter(hash_lock);
6623 
6624 	/*
6625 	 * This assignment is only valid as long as the hash_lock is
6626 	 * held, we must be careful not to reference state or the
6627 	 * b_state field after dropping the lock.
6628 	 */
6629 	arc_state_t *state = hdr->b_l1hdr.b_state;
6630 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6631 	ASSERT3P(state, !=, arc_anon);
6632 	ASSERT3P(state, !=, arc_l2c_only);
6633 
6634 	/* this buffer is not on any list */
6635 	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6636 
6637 	/*
6638 	 * Do we have more than one buf?
6639 	 */
6640 	if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
6641 		arc_buf_hdr_t *nhdr;
6642 		uint64_t spa = hdr->b_spa;
6643 		uint64_t psize = HDR_GET_PSIZE(hdr);
6644 		uint64_t lsize = HDR_GET_LSIZE(hdr);
6645 		boolean_t protected = HDR_PROTECTED(hdr);
6646 		enum zio_compress compress = arc_hdr_get_compress(hdr);
6647 		arc_buf_contents_t type = arc_buf_type(hdr);
6648 
6649 		if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
6650 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6651 			ASSERT(ARC_BUF_LAST(buf));
6652 		}
6653 
6654 		/*
6655 		 * Pull the buffer off of this hdr and find the last buffer
6656 		 * in the hdr's buffer list.
6657 		 */
6658 		VERIFY3S(remove_reference(hdr, tag), >, 0);
6659 		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6660 		ASSERT3P(lastbuf, !=, NULL);
6661 
6662 		/*
6663 		 * If the current arc_buf_t and the hdr are sharing their data
6664 		 * buffer, then we must stop sharing that block.
6665 		 */
6666 		if (ARC_BUF_SHARED(buf)) {
6667 			ASSERT(!arc_buf_is_shared(lastbuf));
6668 
6669 			/*
6670 			 * First, sever the block sharing relationship between
6671 			 * buf and the arc_buf_hdr_t.
6672 			 */
6673 			arc_unshare_buf(hdr, buf);
6674 
6675 			/*
6676 			 * Now we need to recreate the hdr's b_pabd. Since we
6677 			 * have lastbuf handy, we try to share with it, but if
6678 			 * we can't then we allocate a new b_pabd and copy the
6679 			 * data from buf into it.
6680 			 */
6681 			if (arc_can_share(hdr, lastbuf)) {
6682 				arc_share_buf(hdr, lastbuf);
6683 			} else {
6684 				arc_hdr_alloc_abd(hdr, 0);
6685 				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6686 				    buf->b_data, psize);
6687 			}
6688 		} else if (HDR_SHARED_DATA(hdr)) {
6689 			/*
6690 			 * Uncompressed shared buffers are always at the end
6691 			 * of the list. Compressed buffers don't have the
6692 			 * same requirements. This makes it hard to
6693 			 * simply assert that the lastbuf is shared so
6694 			 * we rely on the hdr's compression flags to determine
6695 			 * if we have a compressed, shared buffer.
6696 			 */
6697 			ASSERT(arc_buf_is_shared(lastbuf) ||
6698 			    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
6699 			ASSERT(!arc_buf_is_shared(buf));
6700 		}
6701 
6702 		ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
6703 
6704 		(void) zfs_refcount_remove_many(&state->arcs_size[type],
6705 		    arc_buf_size(buf), buf);
6706 
6707 		arc_cksum_verify(buf);
6708 		arc_buf_unwatch(buf);
6709 
6710 		/* if this is the last uncompressed buf free the checksum */
6711 		if (!arc_hdr_has_uncompressed_buf(hdr))
6712 			arc_cksum_free(hdr);
6713 
6714 		mutex_exit(hash_lock);
6715 
6716 		nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
6717 		    compress, hdr->b_complevel, type);
6718 		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6719 		ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
6720 		VERIFY3U(nhdr->b_type, ==, type);
6721 		ASSERT(!HDR_SHARED_DATA(nhdr));
6722 
6723 		nhdr->b_l1hdr.b_buf = buf;
6724 		(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6725 		buf->b_hdr = nhdr;
6726 
6727 		(void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
6728 		    arc_buf_size(buf), buf);
6729 	} else {
6730 		ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6731 		/* protected by hash lock, or hdr is on arc_anon */
6732 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6733 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6734 
6735 		if (HDR_HAS_L2HDR(hdr)) {
6736 			mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6737 			/* Recheck to prevent race with l2arc_evict(). */
6738 			if (HDR_HAS_L2HDR(hdr))
6739 				arc_hdr_l2hdr_destroy(hdr);
6740 			mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6741 		}
6742 
6743 		hdr->b_l1hdr.b_mru_hits = 0;
6744 		hdr->b_l1hdr.b_mru_ghost_hits = 0;
6745 		hdr->b_l1hdr.b_mfu_hits = 0;
6746 		hdr->b_l1hdr.b_mfu_ghost_hits = 0;
6747 		arc_change_state(arc_anon, hdr);
6748 		hdr->b_l1hdr.b_arc_access = 0;
6749 
6750 		mutex_exit(hash_lock);
6751 		buf_discard_identity(hdr);
6752 		arc_buf_thaw(buf);
6753 	}
6754 }
6755 
6756 int
arc_released(arc_buf_t * buf)6757 arc_released(arc_buf_t *buf)
6758 {
6759 	return (buf->b_data != NULL &&
6760 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
6761 }
6762 
6763 #ifdef ZFS_DEBUG
6764 int
arc_referenced(arc_buf_t * buf)6765 arc_referenced(arc_buf_t *buf)
6766 {
6767 	return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6768 }
6769 #endif
6770 
6771 static void
arc_write_ready(zio_t * zio)6772 arc_write_ready(zio_t *zio)
6773 {
6774 	arc_write_callback_t *callback = zio->io_private;
6775 	arc_buf_t *buf = callback->awcb_buf;
6776 	arc_buf_hdr_t *hdr = buf->b_hdr;
6777 	blkptr_t *bp = zio->io_bp;
6778 	uint64_t psize = BP_IS_HOLE(bp) ? 0 : BP_GET_PSIZE(bp);
6779 	fstrans_cookie_t cookie = spl_fstrans_mark();
6780 
6781 	ASSERT(HDR_HAS_L1HDR(hdr));
6782 	ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6783 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6784 
6785 	/*
6786 	 * If we're reexecuting this zio because the pool suspended, then
6787 	 * cleanup any state that was previously set the first time the
6788 	 * callback was invoked.
6789 	 */
6790 	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6791 		arc_cksum_free(hdr);
6792 		arc_buf_unwatch(buf);
6793 		if (hdr->b_l1hdr.b_pabd != NULL) {
6794 			if (ARC_BUF_SHARED(buf)) {
6795 				arc_unshare_buf(hdr, buf);
6796 			} else {
6797 				ASSERT(!arc_buf_is_shared(buf));
6798 				arc_hdr_free_abd(hdr, B_FALSE);
6799 			}
6800 		}
6801 
6802 		if (HDR_HAS_RABD(hdr))
6803 			arc_hdr_free_abd(hdr, B_TRUE);
6804 	}
6805 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6806 	ASSERT(!HDR_HAS_RABD(hdr));
6807 	ASSERT(!HDR_SHARED_DATA(hdr));
6808 	ASSERT(!arc_buf_is_shared(buf));
6809 
6810 	callback->awcb_ready(zio, buf, callback->awcb_private);
6811 
6812 	if (HDR_IO_IN_PROGRESS(hdr)) {
6813 		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6814 	} else {
6815 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6816 		add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
6817 	}
6818 
6819 	if (BP_IS_PROTECTED(bp)) {
6820 		/* ZIL blocks are written through zio_rewrite */
6821 		ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
6822 
6823 		if (BP_SHOULD_BYTESWAP(bp)) {
6824 			if (BP_GET_LEVEL(bp) > 0) {
6825 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
6826 			} else {
6827 				hdr->b_l1hdr.b_byteswap =
6828 				    DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
6829 			}
6830 		} else {
6831 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
6832 		}
6833 
6834 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
6835 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
6836 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
6837 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
6838 		    hdr->b_crypt_hdr.b_iv);
6839 		zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
6840 	} else {
6841 		arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
6842 	}
6843 
6844 	/*
6845 	 * If this block was written for raw encryption but the zio layer
6846 	 * ended up only authenticating it, adjust the buffer flags now.
6847 	 */
6848 	if (BP_IS_AUTHENTICATED(bp) && ARC_BUF_ENCRYPTED(buf)) {
6849 		arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6850 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6851 		if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF)
6852 			buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6853 	} else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) {
6854 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6855 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6856 	}
6857 
6858 	/* this must be done after the buffer flags are adjusted */
6859 	arc_cksum_compute(buf);
6860 
6861 	enum zio_compress compress;
6862 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
6863 		compress = ZIO_COMPRESS_OFF;
6864 	} else {
6865 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
6866 		compress = BP_GET_COMPRESS(bp);
6867 	}
6868 	HDR_SET_PSIZE(hdr, psize);
6869 	arc_hdr_set_compress(hdr, compress);
6870 	hdr->b_complevel = zio->io_prop.zp_complevel;
6871 
6872 	if (zio->io_error != 0 || psize == 0)
6873 		goto out;
6874 
6875 	/*
6876 	 * Fill the hdr with data. If the buffer is encrypted we have no choice
6877 	 * but to copy the data into b_radb. If the hdr is compressed, the data
6878 	 * we want is available from the zio, otherwise we can take it from
6879 	 * the buf.
6880 	 *
6881 	 * We might be able to share the buf's data with the hdr here. However,
6882 	 * doing so would cause the ARC to be full of linear ABDs if we write a
6883 	 * lot of shareable data. As a compromise, we check whether scattered
6884 	 * ABDs are allowed, and assume that if they are then the user wants
6885 	 * the ARC to be primarily filled with them regardless of the data being
6886 	 * written. Therefore, if they're allowed then we allocate one and copy
6887 	 * the data into it; otherwise, we share the data directly if we can.
6888 	 */
6889 	if (ARC_BUF_ENCRYPTED(buf)) {
6890 		ASSERT3U(psize, >, 0);
6891 		ASSERT(ARC_BUF_COMPRESSED(buf));
6892 		arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6893 		    ARC_HDR_USE_RESERVE);
6894 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6895 	} else if (!(HDR_UNCACHED(hdr) ||
6896 	    abd_size_alloc_linear(arc_buf_size(buf))) ||
6897 	    !arc_can_share(hdr, buf)) {
6898 		/*
6899 		 * Ideally, we would always copy the io_abd into b_pabd, but the
6900 		 * user may have disabled compressed ARC, thus we must check the
6901 		 * hdr's compression setting rather than the io_bp's.
6902 		 */
6903 		if (BP_IS_ENCRYPTED(bp)) {
6904 			ASSERT3U(psize, >, 0);
6905 			arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6906 			    ARC_HDR_USE_RESERVE);
6907 			abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6908 		} else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
6909 		    !ARC_BUF_COMPRESSED(buf)) {
6910 			ASSERT3U(psize, >, 0);
6911 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6912 			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6913 		} else {
6914 			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6915 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6916 			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6917 			    arc_buf_size(buf));
6918 		}
6919 	} else {
6920 		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
6921 		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
6922 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6923 		ASSERT(ARC_BUF_LAST(buf));
6924 
6925 		arc_share_buf(hdr, buf);
6926 	}
6927 
6928 out:
6929 	arc_hdr_verify(hdr, bp);
6930 	spl_fstrans_unmark(cookie);
6931 }
6932 
6933 static void
arc_write_children_ready(zio_t * zio)6934 arc_write_children_ready(zio_t *zio)
6935 {
6936 	arc_write_callback_t *callback = zio->io_private;
6937 	arc_buf_t *buf = callback->awcb_buf;
6938 
6939 	callback->awcb_children_ready(zio, buf, callback->awcb_private);
6940 }
6941 
6942 static void
arc_write_done(zio_t * zio)6943 arc_write_done(zio_t *zio)
6944 {
6945 	arc_write_callback_t *callback = zio->io_private;
6946 	arc_buf_t *buf = callback->awcb_buf;
6947 	arc_buf_hdr_t *hdr = buf->b_hdr;
6948 
6949 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6950 
6951 	if (zio->io_error == 0) {
6952 		arc_hdr_verify(hdr, zio->io_bp);
6953 
6954 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6955 			buf_discard_identity(hdr);
6956 		} else {
6957 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
6958 			hdr->b_birth = BP_GET_BIRTH(zio->io_bp);
6959 		}
6960 	} else {
6961 		ASSERT(HDR_EMPTY(hdr));
6962 	}
6963 
6964 	/*
6965 	 * If the block to be written was all-zero or compressed enough to be
6966 	 * embedded in the BP, no write was performed so there will be no
6967 	 * dva/birth/checksum.  The buffer must therefore remain anonymous
6968 	 * (and uncached).
6969 	 */
6970 	if (!HDR_EMPTY(hdr)) {
6971 		arc_buf_hdr_t *exists;
6972 		kmutex_t *hash_lock;
6973 
6974 		ASSERT3U(zio->io_error, ==, 0);
6975 
6976 		arc_cksum_verify(buf);
6977 
6978 		exists = buf_hash_insert(hdr, &hash_lock);
6979 		if (exists != NULL) {
6980 			/*
6981 			 * This can only happen if we overwrite for
6982 			 * sync-to-convergence, because we remove
6983 			 * buffers from the hash table when we arc_free().
6984 			 */
6985 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
6986 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6987 					panic("bad overwrite, hdr=%p exists=%p",
6988 					    (void *)hdr, (void *)exists);
6989 				ASSERT(zfs_refcount_is_zero(
6990 				    &exists->b_l1hdr.b_refcnt));
6991 				arc_change_state(arc_anon, exists);
6992 				arc_hdr_destroy(exists);
6993 				mutex_exit(hash_lock);
6994 				exists = buf_hash_insert(hdr, &hash_lock);
6995 				ASSERT3P(exists, ==, NULL);
6996 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
6997 				/* nopwrite */
6998 				ASSERT(zio->io_prop.zp_nopwrite);
6999 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
7000 					panic("bad nopwrite, hdr=%p exists=%p",
7001 					    (void *)hdr, (void *)exists);
7002 			} else {
7003 				/* Dedup */
7004 				ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
7005 				ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
7006 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
7007 				ASSERT(BP_GET_DEDUP(zio->io_bp));
7008 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
7009 			}
7010 		}
7011 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
7012 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
7013 		/* if it's not anon, we are doing a scrub */
7014 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
7015 			arc_access(hdr, 0, B_FALSE);
7016 		mutex_exit(hash_lock);
7017 	} else {
7018 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
7019 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
7020 	}
7021 
7022 	callback->awcb_done(zio, buf, callback->awcb_private);
7023 
7024 	abd_free(zio->io_abd);
7025 	kmem_free(callback, sizeof (arc_write_callback_t));
7026 }
7027 
7028 zio_t *
arc_write(zio_t * pio,spa_t * spa,uint64_t txg,blkptr_t * bp,arc_buf_t * buf,boolean_t uncached,boolean_t l2arc,const zio_prop_t * zp,arc_write_done_func_t * ready,arc_write_done_func_t * children_ready,arc_write_done_func_t * done,void * private,zio_priority_t priority,int zio_flags,const zbookmark_phys_t * zb)7029 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
7030     blkptr_t *bp, arc_buf_t *buf, boolean_t uncached, boolean_t l2arc,
7031     const zio_prop_t *zp, arc_write_done_func_t *ready,
7032     arc_write_done_func_t *children_ready, arc_write_done_func_t *done,
7033     void *private, zio_priority_t priority, int zio_flags,
7034     const zbookmark_phys_t *zb)
7035 {
7036 	arc_buf_hdr_t *hdr = buf->b_hdr;
7037 	arc_write_callback_t *callback;
7038 	zio_t *zio;
7039 	zio_prop_t localprop = *zp;
7040 
7041 	ASSERT3P(ready, !=, NULL);
7042 	ASSERT3P(done, !=, NULL);
7043 	ASSERT(!HDR_IO_ERROR(hdr));
7044 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
7045 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
7046 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
7047 	if (uncached)
7048 		arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
7049 	else if (l2arc)
7050 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
7051 
7052 	if (ARC_BUF_ENCRYPTED(buf)) {
7053 		ASSERT(ARC_BUF_COMPRESSED(buf));
7054 		localprop.zp_encrypt = B_TRUE;
7055 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
7056 		localprop.zp_complevel = hdr->b_complevel;
7057 		localprop.zp_byteorder =
7058 		    (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
7059 		    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
7060 		memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
7061 		    ZIO_DATA_SALT_LEN);
7062 		memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
7063 		    ZIO_DATA_IV_LEN);
7064 		memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
7065 		    ZIO_DATA_MAC_LEN);
7066 		if (DMU_OT_IS_ENCRYPTED(localprop.zp_type)) {
7067 			localprop.zp_nopwrite = B_FALSE;
7068 			localprop.zp_copies =
7069 			    MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
7070 			localprop.zp_gang_copies =
7071 			    MIN(localprop.zp_gang_copies, SPA_DVAS_PER_BP - 1);
7072 		}
7073 		zio_flags |= ZIO_FLAG_RAW;
7074 	} else if (ARC_BUF_COMPRESSED(buf)) {
7075 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
7076 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
7077 		localprop.zp_complevel = hdr->b_complevel;
7078 		zio_flags |= ZIO_FLAG_RAW_COMPRESS;
7079 	}
7080 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
7081 	callback->awcb_ready = ready;
7082 	callback->awcb_children_ready = children_ready;
7083 	callback->awcb_done = done;
7084 	callback->awcb_private = private;
7085 	callback->awcb_buf = buf;
7086 
7087 	/*
7088 	 * The hdr's b_pabd is now stale, free it now. A new data block
7089 	 * will be allocated when the zio pipeline calls arc_write_ready().
7090 	 */
7091 	if (hdr->b_l1hdr.b_pabd != NULL) {
7092 		/*
7093 		 * If the buf is currently sharing the data block with
7094 		 * the hdr then we need to break that relationship here.
7095 		 * The hdr will remain with a NULL data pointer and the
7096 		 * buf will take sole ownership of the block.
7097 		 */
7098 		if (ARC_BUF_SHARED(buf)) {
7099 			arc_unshare_buf(hdr, buf);
7100 		} else {
7101 			ASSERT(!arc_buf_is_shared(buf));
7102 			arc_hdr_free_abd(hdr, B_FALSE);
7103 		}
7104 		VERIFY3P(buf->b_data, !=, NULL);
7105 	}
7106 
7107 	if (HDR_HAS_RABD(hdr))
7108 		arc_hdr_free_abd(hdr, B_TRUE);
7109 
7110 	if (!(zio_flags & ZIO_FLAG_RAW))
7111 		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
7112 
7113 	ASSERT(!arc_buf_is_shared(buf));
7114 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
7115 
7116 	zio = zio_write(pio, spa, txg, bp,
7117 	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
7118 	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
7119 	    (children_ready != NULL) ? arc_write_children_ready : NULL,
7120 	    arc_write_done, callback, priority, zio_flags, zb);
7121 
7122 	return (zio);
7123 }
7124 
7125 void
arc_tempreserve_clear(uint64_t reserve)7126 arc_tempreserve_clear(uint64_t reserve)
7127 {
7128 	atomic_add_64(&arc_tempreserve, -reserve);
7129 	ASSERT((int64_t)arc_tempreserve >= 0);
7130 }
7131 
7132 int
arc_tempreserve_space(spa_t * spa,uint64_t reserve,uint64_t txg)7133 arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg)
7134 {
7135 	int error;
7136 	uint64_t anon_size;
7137 
7138 	if (!arc_no_grow &&
7139 	    reserve > arc_c/4 &&
7140 	    reserve * 4 > (2ULL << SPA_MAXBLOCKSHIFT))
7141 		arc_c = MIN(arc_c_max, reserve * 4);
7142 
7143 	/*
7144 	 * Throttle when the calculated memory footprint for the TXG
7145 	 * exceeds the target ARC size.
7146 	 */
7147 	if (reserve > arc_c) {
7148 		DMU_TX_STAT_BUMP(dmu_tx_memory_reserve);
7149 		return (SET_ERROR(ERESTART));
7150 	}
7151 
7152 	/*
7153 	 * Don't count loaned bufs as in flight dirty data to prevent long
7154 	 * network delays from blocking transactions that are ready to be
7155 	 * assigned to a txg.
7156 	 */
7157 
7158 	/* assert that it has not wrapped around */
7159 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
7160 
7161 	anon_size = MAX((int64_t)
7162 	    (zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
7163 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
7164 	    arc_loaned_bytes), 0);
7165 
7166 	/*
7167 	 * Writes will, almost always, require additional memory allocations
7168 	 * in order to compress/encrypt/etc the data.  We therefore need to
7169 	 * make sure that there is sufficient available memory for this.
7170 	 */
7171 	error = arc_memory_throttle(spa, reserve, txg);
7172 	if (error != 0)
7173 		return (error);
7174 
7175 	/*
7176 	 * Throttle writes when the amount of dirty data in the cache
7177 	 * gets too large.  We try to keep the cache less than half full
7178 	 * of dirty blocks so that our sync times don't grow too large.
7179 	 *
7180 	 * In the case of one pool being built on another pool, we want
7181 	 * to make sure we don't end up throttling the lower (backing)
7182 	 * pool when the upper pool is the majority contributor to dirty
7183 	 * data. To insure we make forward progress during throttling, we
7184 	 * also check the current pool's net dirty data and only throttle
7185 	 * if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty
7186 	 * data in the cache.
7187 	 *
7188 	 * Note: if two requests come in concurrently, we might let them
7189 	 * both succeed, when one of them should fail.  Not a huge deal.
7190 	 */
7191 	uint64_t total_dirty = reserve + arc_tempreserve + anon_size;
7192 	uint64_t spa_dirty_anon = spa_dirty_data(spa);
7193 	uint64_t rarc_c = arc_warm ? arc_c : arc_c_max;
7194 	if (total_dirty > rarc_c * zfs_arc_dirty_limit_percent / 100 &&
7195 	    anon_size > rarc_c * zfs_arc_anon_limit_percent / 100 &&
7196 	    spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) {
7197 #ifdef ZFS_DEBUG
7198 		uint64_t meta_esize = zfs_refcount_count(
7199 		    &arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7200 		uint64_t data_esize =
7201 		    zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7202 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
7203 		    "anon_data=%lluK tempreserve=%lluK rarc_c=%lluK\n",
7204 		    (u_longlong_t)arc_tempreserve >> 10,
7205 		    (u_longlong_t)meta_esize >> 10,
7206 		    (u_longlong_t)data_esize >> 10,
7207 		    (u_longlong_t)reserve >> 10,
7208 		    (u_longlong_t)rarc_c >> 10);
7209 #endif
7210 		DMU_TX_STAT_BUMP(dmu_tx_dirty_throttle);
7211 		return (SET_ERROR(ERESTART));
7212 	}
7213 	atomic_add_64(&arc_tempreserve, reserve);
7214 	return (0);
7215 }
7216 
7217 static void
arc_kstat_update_state(arc_state_t * state,kstat_named_t * size,kstat_named_t * data,kstat_named_t * metadata,kstat_named_t * evict_data,kstat_named_t * evict_metadata)7218 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
7219     kstat_named_t *data, kstat_named_t *metadata,
7220     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
7221 {
7222 	data->value.ui64 =
7223 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
7224 	metadata->value.ui64 =
7225 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
7226 	size->value.ui64 = data->value.ui64 + metadata->value.ui64;
7227 	evict_data->value.ui64 =
7228 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
7229 	evict_metadata->value.ui64 =
7230 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
7231 }
7232 
7233 static int
arc_kstat_update(kstat_t * ksp,int rw)7234 arc_kstat_update(kstat_t *ksp, int rw)
7235 {
7236 	arc_stats_t *as = ksp->ks_data;
7237 
7238 	if (rw == KSTAT_WRITE)
7239 		return (SET_ERROR(EACCES));
7240 
7241 	as->arcstat_hits.value.ui64 =
7242 	    wmsum_value(&arc_sums.arcstat_hits);
7243 	as->arcstat_iohits.value.ui64 =
7244 	    wmsum_value(&arc_sums.arcstat_iohits);
7245 	as->arcstat_misses.value.ui64 =
7246 	    wmsum_value(&arc_sums.arcstat_misses);
7247 	as->arcstat_demand_data_hits.value.ui64 =
7248 	    wmsum_value(&arc_sums.arcstat_demand_data_hits);
7249 	as->arcstat_demand_data_iohits.value.ui64 =
7250 	    wmsum_value(&arc_sums.arcstat_demand_data_iohits);
7251 	as->arcstat_demand_data_misses.value.ui64 =
7252 	    wmsum_value(&arc_sums.arcstat_demand_data_misses);
7253 	as->arcstat_demand_metadata_hits.value.ui64 =
7254 	    wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
7255 	as->arcstat_demand_metadata_iohits.value.ui64 =
7256 	    wmsum_value(&arc_sums.arcstat_demand_metadata_iohits);
7257 	as->arcstat_demand_metadata_misses.value.ui64 =
7258 	    wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
7259 	as->arcstat_prefetch_data_hits.value.ui64 =
7260 	    wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
7261 	as->arcstat_prefetch_data_iohits.value.ui64 =
7262 	    wmsum_value(&arc_sums.arcstat_prefetch_data_iohits);
7263 	as->arcstat_prefetch_data_misses.value.ui64 =
7264 	    wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
7265 	as->arcstat_prefetch_metadata_hits.value.ui64 =
7266 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
7267 	as->arcstat_prefetch_metadata_iohits.value.ui64 =
7268 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_iohits);
7269 	as->arcstat_prefetch_metadata_misses.value.ui64 =
7270 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
7271 	as->arcstat_mru_hits.value.ui64 =
7272 	    wmsum_value(&arc_sums.arcstat_mru_hits);
7273 	as->arcstat_mru_ghost_hits.value.ui64 =
7274 	    wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
7275 	as->arcstat_mfu_hits.value.ui64 =
7276 	    wmsum_value(&arc_sums.arcstat_mfu_hits);
7277 	as->arcstat_mfu_ghost_hits.value.ui64 =
7278 	    wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
7279 	as->arcstat_uncached_hits.value.ui64 =
7280 	    wmsum_value(&arc_sums.arcstat_uncached_hits);
7281 	as->arcstat_deleted.value.ui64 =
7282 	    wmsum_value(&arc_sums.arcstat_deleted);
7283 	as->arcstat_mutex_miss.value.ui64 =
7284 	    wmsum_value(&arc_sums.arcstat_mutex_miss);
7285 	as->arcstat_access_skip.value.ui64 =
7286 	    wmsum_value(&arc_sums.arcstat_access_skip);
7287 	as->arcstat_evict_skip.value.ui64 =
7288 	    wmsum_value(&arc_sums.arcstat_evict_skip);
7289 	as->arcstat_evict_not_enough.value.ui64 =
7290 	    wmsum_value(&arc_sums.arcstat_evict_not_enough);
7291 	as->arcstat_evict_l2_cached.value.ui64 =
7292 	    wmsum_value(&arc_sums.arcstat_evict_l2_cached);
7293 	as->arcstat_evict_l2_eligible.value.ui64 =
7294 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
7295 	as->arcstat_evict_l2_eligible_mfu.value.ui64 =
7296 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
7297 	as->arcstat_evict_l2_eligible_mru.value.ui64 =
7298 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
7299 	as->arcstat_evict_l2_ineligible.value.ui64 =
7300 	    wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
7301 	as->arcstat_evict_l2_skip.value.ui64 =
7302 	    wmsum_value(&arc_sums.arcstat_evict_l2_skip);
7303 	as->arcstat_hash_elements.value.ui64 =
7304 	    as->arcstat_hash_elements_max.value.ui64 =
7305 	    wmsum_value(&arc_sums.arcstat_hash_elements);
7306 	as->arcstat_hash_collisions.value.ui64 =
7307 	    wmsum_value(&arc_sums.arcstat_hash_collisions);
7308 	as->arcstat_hash_chains.value.ui64 =
7309 	    wmsum_value(&arc_sums.arcstat_hash_chains);
7310 	as->arcstat_size.value.ui64 =
7311 	    aggsum_value(&arc_sums.arcstat_size);
7312 	as->arcstat_compressed_size.value.ui64 =
7313 	    wmsum_value(&arc_sums.arcstat_compressed_size);
7314 	as->arcstat_uncompressed_size.value.ui64 =
7315 	    wmsum_value(&arc_sums.arcstat_uncompressed_size);
7316 	as->arcstat_overhead_size.value.ui64 =
7317 	    wmsum_value(&arc_sums.arcstat_overhead_size);
7318 	as->arcstat_hdr_size.value.ui64 =
7319 	    wmsum_value(&arc_sums.arcstat_hdr_size);
7320 	as->arcstat_data_size.value.ui64 =
7321 	    wmsum_value(&arc_sums.arcstat_data_size);
7322 	as->arcstat_metadata_size.value.ui64 =
7323 	    wmsum_value(&arc_sums.arcstat_metadata_size);
7324 	as->arcstat_dbuf_size.value.ui64 =
7325 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
7326 #if defined(COMPAT_FREEBSD11)
7327 	as->arcstat_other_size.value.ui64 =
7328 	    wmsum_value(&arc_sums.arcstat_bonus_size) +
7329 	    wmsum_value(&arc_sums.arcstat_dnode_size) +
7330 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
7331 #endif
7332 
7333 	arc_kstat_update_state(arc_anon,
7334 	    &as->arcstat_anon_size,
7335 	    &as->arcstat_anon_data,
7336 	    &as->arcstat_anon_metadata,
7337 	    &as->arcstat_anon_evictable_data,
7338 	    &as->arcstat_anon_evictable_metadata);
7339 	arc_kstat_update_state(arc_mru,
7340 	    &as->arcstat_mru_size,
7341 	    &as->arcstat_mru_data,
7342 	    &as->arcstat_mru_metadata,
7343 	    &as->arcstat_mru_evictable_data,
7344 	    &as->arcstat_mru_evictable_metadata);
7345 	arc_kstat_update_state(arc_mru_ghost,
7346 	    &as->arcstat_mru_ghost_size,
7347 	    &as->arcstat_mru_ghost_data,
7348 	    &as->arcstat_mru_ghost_metadata,
7349 	    &as->arcstat_mru_ghost_evictable_data,
7350 	    &as->arcstat_mru_ghost_evictable_metadata);
7351 	arc_kstat_update_state(arc_mfu,
7352 	    &as->arcstat_mfu_size,
7353 	    &as->arcstat_mfu_data,
7354 	    &as->arcstat_mfu_metadata,
7355 	    &as->arcstat_mfu_evictable_data,
7356 	    &as->arcstat_mfu_evictable_metadata);
7357 	arc_kstat_update_state(arc_mfu_ghost,
7358 	    &as->arcstat_mfu_ghost_size,
7359 	    &as->arcstat_mfu_ghost_data,
7360 	    &as->arcstat_mfu_ghost_metadata,
7361 	    &as->arcstat_mfu_ghost_evictable_data,
7362 	    &as->arcstat_mfu_ghost_evictable_metadata);
7363 	arc_kstat_update_state(arc_uncached,
7364 	    &as->arcstat_uncached_size,
7365 	    &as->arcstat_uncached_data,
7366 	    &as->arcstat_uncached_metadata,
7367 	    &as->arcstat_uncached_evictable_data,
7368 	    &as->arcstat_uncached_evictable_metadata);
7369 
7370 	as->arcstat_dnode_size.value.ui64 =
7371 	    wmsum_value(&arc_sums.arcstat_dnode_size);
7372 	as->arcstat_bonus_size.value.ui64 =
7373 	    wmsum_value(&arc_sums.arcstat_bonus_size);
7374 	as->arcstat_l2_hits.value.ui64 =
7375 	    wmsum_value(&arc_sums.arcstat_l2_hits);
7376 	as->arcstat_l2_misses.value.ui64 =
7377 	    wmsum_value(&arc_sums.arcstat_l2_misses);
7378 	as->arcstat_l2_prefetch_asize.value.ui64 =
7379 	    wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
7380 	as->arcstat_l2_mru_asize.value.ui64 =
7381 	    wmsum_value(&arc_sums.arcstat_l2_mru_asize);
7382 	as->arcstat_l2_mfu_asize.value.ui64 =
7383 	    wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
7384 	as->arcstat_l2_bufc_data_asize.value.ui64 =
7385 	    wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
7386 	as->arcstat_l2_bufc_metadata_asize.value.ui64 =
7387 	    wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
7388 	as->arcstat_l2_feeds.value.ui64 =
7389 	    wmsum_value(&arc_sums.arcstat_l2_feeds);
7390 	as->arcstat_l2_rw_clash.value.ui64 =
7391 	    wmsum_value(&arc_sums.arcstat_l2_rw_clash);
7392 	as->arcstat_l2_read_bytes.value.ui64 =
7393 	    wmsum_value(&arc_sums.arcstat_l2_read_bytes);
7394 	as->arcstat_l2_write_bytes.value.ui64 =
7395 	    wmsum_value(&arc_sums.arcstat_l2_write_bytes);
7396 	as->arcstat_l2_writes_sent.value.ui64 =
7397 	    wmsum_value(&arc_sums.arcstat_l2_writes_sent);
7398 	as->arcstat_l2_writes_done.value.ui64 =
7399 	    wmsum_value(&arc_sums.arcstat_l2_writes_done);
7400 	as->arcstat_l2_writes_error.value.ui64 =
7401 	    wmsum_value(&arc_sums.arcstat_l2_writes_error);
7402 	as->arcstat_l2_writes_lock_retry.value.ui64 =
7403 	    wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
7404 	as->arcstat_l2_evict_lock_retry.value.ui64 =
7405 	    wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
7406 	as->arcstat_l2_evict_reading.value.ui64 =
7407 	    wmsum_value(&arc_sums.arcstat_l2_evict_reading);
7408 	as->arcstat_l2_evict_l1cached.value.ui64 =
7409 	    wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
7410 	as->arcstat_l2_free_on_write.value.ui64 =
7411 	    wmsum_value(&arc_sums.arcstat_l2_free_on_write);
7412 	as->arcstat_l2_abort_lowmem.value.ui64 =
7413 	    wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
7414 	as->arcstat_l2_cksum_bad.value.ui64 =
7415 	    wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
7416 	as->arcstat_l2_io_error.value.ui64 =
7417 	    wmsum_value(&arc_sums.arcstat_l2_io_error);
7418 	as->arcstat_l2_lsize.value.ui64 =
7419 	    wmsum_value(&arc_sums.arcstat_l2_lsize);
7420 	as->arcstat_l2_psize.value.ui64 =
7421 	    wmsum_value(&arc_sums.arcstat_l2_psize);
7422 	as->arcstat_l2_hdr_size.value.ui64 =
7423 	    aggsum_value(&arc_sums.arcstat_l2_hdr_size);
7424 	as->arcstat_l2_log_blk_writes.value.ui64 =
7425 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
7426 	as->arcstat_l2_log_blk_asize.value.ui64 =
7427 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
7428 	as->arcstat_l2_log_blk_count.value.ui64 =
7429 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
7430 	as->arcstat_l2_rebuild_success.value.ui64 =
7431 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
7432 	as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
7433 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
7434 	as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
7435 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
7436 	as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
7437 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
7438 	as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
7439 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
7440 	as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
7441 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
7442 	as->arcstat_l2_rebuild_size.value.ui64 =
7443 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
7444 	as->arcstat_l2_rebuild_asize.value.ui64 =
7445 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
7446 	as->arcstat_l2_rebuild_bufs.value.ui64 =
7447 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
7448 	as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
7449 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
7450 	as->arcstat_l2_rebuild_log_blks.value.ui64 =
7451 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
7452 	as->arcstat_memory_throttle_count.value.ui64 =
7453 	    wmsum_value(&arc_sums.arcstat_memory_throttle_count);
7454 	as->arcstat_memory_direct_count.value.ui64 =
7455 	    wmsum_value(&arc_sums.arcstat_memory_direct_count);
7456 	as->arcstat_memory_indirect_count.value.ui64 =
7457 	    wmsum_value(&arc_sums.arcstat_memory_indirect_count);
7458 
7459 	as->arcstat_memory_all_bytes.value.ui64 =
7460 	    arc_all_memory();
7461 	as->arcstat_memory_free_bytes.value.ui64 =
7462 	    arc_free_memory();
7463 	as->arcstat_memory_available_bytes.value.i64 =
7464 	    arc_available_memory();
7465 
7466 	as->arcstat_prune.value.ui64 =
7467 	    wmsum_value(&arc_sums.arcstat_prune);
7468 	as->arcstat_meta_used.value.ui64 =
7469 	    wmsum_value(&arc_sums.arcstat_meta_used);
7470 	as->arcstat_async_upgrade_sync.value.ui64 =
7471 	    wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
7472 	as->arcstat_predictive_prefetch.value.ui64 =
7473 	    wmsum_value(&arc_sums.arcstat_predictive_prefetch);
7474 	as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
7475 	    wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7476 	as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
7477 	    wmsum_value(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7478 	as->arcstat_prescient_prefetch.value.ui64 =
7479 	    wmsum_value(&arc_sums.arcstat_prescient_prefetch);
7480 	as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
7481 	    wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7482 	as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
7483 	    wmsum_value(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7484 	as->arcstat_raw_size.value.ui64 =
7485 	    wmsum_value(&arc_sums.arcstat_raw_size);
7486 	as->arcstat_cached_only_in_progress.value.ui64 =
7487 	    wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
7488 	as->arcstat_abd_chunk_waste_size.value.ui64 =
7489 	    wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);
7490 
7491 	return (0);
7492 }
7493 
7494 /*
7495  * This function *must* return indices evenly distributed between all
7496  * sublists of the multilist. This is needed due to how the ARC eviction
7497  * code is laid out; arc_evict_state() assumes ARC buffers are evenly
7498  * distributed between all sublists and uses this assumption when
7499  * deciding which sublist to evict from and how much to evict from it.
7500  */
7501 static unsigned int
arc_state_multilist_index_func(multilist_t * ml,void * obj)7502 arc_state_multilist_index_func(multilist_t *ml, void *obj)
7503 {
7504 	arc_buf_hdr_t *hdr = obj;
7505 
7506 	/*
7507 	 * We rely on b_dva to generate evenly distributed index
7508 	 * numbers using buf_hash below. So, as an added precaution,
7509 	 * let's make sure we never add empty buffers to the arc lists.
7510 	 */
7511 	ASSERT(!HDR_EMPTY(hdr));
7512 
7513 	/*
7514 	 * The assumption here, is the hash value for a given
7515 	 * arc_buf_hdr_t will remain constant throughout its lifetime
7516 	 * (i.e. its b_spa, b_dva, and b_birth fields don't change).
7517 	 * Thus, we don't need to store the header's sublist index
7518 	 * on insertion, as this index can be recalculated on removal.
7519 	 *
7520 	 * Also, the low order bits of the hash value are thought to be
7521 	 * distributed evenly. Otherwise, in the case that the multilist
7522 	 * has a power of two number of sublists, each sublists' usage
7523 	 * would not be evenly distributed. In this context full 64bit
7524 	 * division would be a waste of time, so limit it to 32 bits.
7525 	 */
7526 	return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
7527 	    multilist_get_num_sublists(ml));
7528 }
7529 
7530 static unsigned int
arc_state_l2c_multilist_index_func(multilist_t * ml,void * obj)7531 arc_state_l2c_multilist_index_func(multilist_t *ml, void *obj)
7532 {
7533 	panic("Header %p insert into arc_l2c_only %p", obj, ml);
7534 }
7535 
7536 #define	WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do {	\
7537 	if ((do_warn) && (tuning) && ((tuning) != (value))) {	\
7538 		cmn_err(CE_WARN,				\
7539 		    "ignoring tunable %s (using %llu instead)",	\
7540 		    (#tuning), (u_longlong_t)(value));	\
7541 	}							\
7542 } while (0)
7543 
7544 /*
7545  * Called during module initialization and periodically thereafter to
7546  * apply reasonable changes to the exposed performance tunings.  Can also be
7547  * called explicitly by param_set_arc_*() functions when ARC tunables are
7548  * updated manually.  Non-zero zfs_* values which differ from the currently set
7549  * values will be applied.
7550  */
7551 void
arc_tuning_update(boolean_t verbose)7552 arc_tuning_update(boolean_t verbose)
7553 {
7554 	uint64_t allmem = arc_all_memory();
7555 
7556 	/* Valid range: 32M - <arc_c_max> */
7557 	if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) &&
7558 	    (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) &&
7559 	    (zfs_arc_min <= arc_c_max)) {
7560 		arc_c_min = zfs_arc_min;
7561 		arc_c = MAX(arc_c, arc_c_min);
7562 	}
7563 	WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose);
7564 
7565 	/* Valid range: 64M - <all physical memory> */
7566 	if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
7567 	    (zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
7568 	    (zfs_arc_max > arc_c_min)) {
7569 		arc_c_max = zfs_arc_max;
7570 		arc_c = MIN(arc_c, arc_c_max);
7571 		if (arc_dnode_limit > arc_c_max)
7572 			arc_dnode_limit = arc_c_max;
7573 	}
7574 	WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose);
7575 
7576 	/* Valid range: 0 - <all physical memory> */
7577 	arc_dnode_limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit :
7578 	    MIN(zfs_arc_dnode_limit_percent, 100) * arc_c_max / 100;
7579 	WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_limit, verbose);
7580 
7581 	/* Valid range: 1 - N */
7582 	if (zfs_arc_grow_retry)
7583 		arc_grow_retry = zfs_arc_grow_retry;
7584 
7585 	/* Valid range: 1 - N */
7586 	if (zfs_arc_shrink_shift) {
7587 		arc_shrink_shift = zfs_arc_shrink_shift;
7588 		arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
7589 	}
7590 
7591 	/* Valid range: 1 - N ms */
7592 	if (zfs_arc_min_prefetch_ms)
7593 		arc_min_prefetch_ms = zfs_arc_min_prefetch_ms;
7594 
7595 	/* Valid range: 1 - N ms */
7596 	if (zfs_arc_min_prescient_prefetch_ms) {
7597 		arc_min_prescient_prefetch_ms =
7598 		    zfs_arc_min_prescient_prefetch_ms;
7599 	}
7600 
7601 	/* Valid range: 0 - 100 */
7602 	if (zfs_arc_lotsfree_percent <= 100)
7603 		arc_lotsfree_percent = zfs_arc_lotsfree_percent;
7604 	WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent,
7605 	    verbose);
7606 
7607 	/* Valid range: 0 - <all physical memory> */
7608 	if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free))
7609 		arc_sys_free = MIN(zfs_arc_sys_free, allmem);
7610 	WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose);
7611 }
7612 
7613 static void
arc_state_multilist_init(multilist_t * ml,multilist_sublist_index_func_t * index_func,int * maxcountp)7614 arc_state_multilist_init(multilist_t *ml,
7615     multilist_sublist_index_func_t *index_func, int *maxcountp)
7616 {
7617 	multilist_create(ml, sizeof (arc_buf_hdr_t),
7618 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func);
7619 	*maxcountp = MAX(*maxcountp, multilist_get_num_sublists(ml));
7620 }
7621 
7622 static void
arc_state_init(void)7623 arc_state_init(void)
7624 {
7625 	int num_sublists = 0;
7626 
7627 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
7628 	    arc_state_multilist_index_func, &num_sublists);
7629 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
7630 	    arc_state_multilist_index_func, &num_sublists);
7631 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
7632 	    arc_state_multilist_index_func, &num_sublists);
7633 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
7634 	    arc_state_multilist_index_func, &num_sublists);
7635 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
7636 	    arc_state_multilist_index_func, &num_sublists);
7637 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
7638 	    arc_state_multilist_index_func, &num_sublists);
7639 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
7640 	    arc_state_multilist_index_func, &num_sublists);
7641 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
7642 	    arc_state_multilist_index_func, &num_sublists);
7643 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
7644 	    arc_state_multilist_index_func, &num_sublists);
7645 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
7646 	    arc_state_multilist_index_func, &num_sublists);
7647 
7648 	/*
7649 	 * L2 headers should never be on the L2 state list since they don't
7650 	 * have L1 headers allocated.  Special index function asserts that.
7651 	 */
7652 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
7653 	    arc_state_l2c_multilist_index_func, &num_sublists);
7654 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
7655 	    arc_state_l2c_multilist_index_func, &num_sublists);
7656 
7657 	/*
7658 	 * Keep track of the number of markers needed to reclaim buffers from
7659 	 * any ARC state.  The markers will be pre-allocated so as to minimize
7660 	 * the number of memory allocations performed by the eviction thread.
7661 	 */
7662 	arc_state_evict_marker_count = num_sublists;
7663 
7664 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7665 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7666 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7667 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7668 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7669 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7670 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7671 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7672 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7673 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7674 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7675 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7676 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7677 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7678 
7679 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7680 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7681 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7682 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7683 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7684 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7685 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7686 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7687 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7688 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7689 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7690 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7691 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7692 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7693 
7694 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7695 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7696 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7697 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7698 
7699 	wmsum_init(&arc_sums.arcstat_hits, 0);
7700 	wmsum_init(&arc_sums.arcstat_iohits, 0);
7701 	wmsum_init(&arc_sums.arcstat_misses, 0);
7702 	wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
7703 	wmsum_init(&arc_sums.arcstat_demand_data_iohits, 0);
7704 	wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
7705 	wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
7706 	wmsum_init(&arc_sums.arcstat_demand_metadata_iohits, 0);
7707 	wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
7708 	wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
7709 	wmsum_init(&arc_sums.arcstat_prefetch_data_iohits, 0);
7710 	wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
7711 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
7712 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_iohits, 0);
7713 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
7714 	wmsum_init(&arc_sums.arcstat_mru_hits, 0);
7715 	wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
7716 	wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
7717 	wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
7718 	wmsum_init(&arc_sums.arcstat_uncached_hits, 0);
7719 	wmsum_init(&arc_sums.arcstat_deleted, 0);
7720 	wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
7721 	wmsum_init(&arc_sums.arcstat_access_skip, 0);
7722 	wmsum_init(&arc_sums.arcstat_evict_skip, 0);
7723 	wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
7724 	wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
7725 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
7726 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
7727 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
7728 	wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
7729 	wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
7730 	wmsum_init(&arc_sums.arcstat_hash_elements, 0);
7731 	wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
7732 	wmsum_init(&arc_sums.arcstat_hash_chains, 0);
7733 	aggsum_init(&arc_sums.arcstat_size, 0);
7734 	wmsum_init(&arc_sums.arcstat_compressed_size, 0);
7735 	wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
7736 	wmsum_init(&arc_sums.arcstat_overhead_size, 0);
7737 	wmsum_init(&arc_sums.arcstat_hdr_size, 0);
7738 	wmsum_init(&arc_sums.arcstat_data_size, 0);
7739 	wmsum_init(&arc_sums.arcstat_metadata_size, 0);
7740 	wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
7741 	wmsum_init(&arc_sums.arcstat_dnode_size, 0);
7742 	wmsum_init(&arc_sums.arcstat_bonus_size, 0);
7743 	wmsum_init(&arc_sums.arcstat_l2_hits, 0);
7744 	wmsum_init(&arc_sums.arcstat_l2_misses, 0);
7745 	wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
7746 	wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
7747 	wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
7748 	wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
7749 	wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
7750 	wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
7751 	wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
7752 	wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
7753 	wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
7754 	wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
7755 	wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
7756 	wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
7757 	wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
7758 	wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
7759 	wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
7760 	wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
7761 	wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
7762 	wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
7763 	wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
7764 	wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
7765 	wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
7766 	wmsum_init(&arc_sums.arcstat_l2_psize, 0);
7767 	aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
7768 	wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
7769 	wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
7770 	wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
7771 	wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
7772 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
7773 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
7774 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
7775 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
7776 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
7777 	wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
7778 	wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
7779 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
7780 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
7781 	wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
7782 	wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
7783 	wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
7784 	wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
7785 	wmsum_init(&arc_sums.arcstat_prune, 0);
7786 	wmsum_init(&arc_sums.arcstat_meta_used, 0);
7787 	wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
7788 	wmsum_init(&arc_sums.arcstat_predictive_prefetch, 0);
7789 	wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
7790 	wmsum_init(&arc_sums.arcstat_demand_iohit_predictive_prefetch, 0);
7791 	wmsum_init(&arc_sums.arcstat_prescient_prefetch, 0);
7792 	wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
7793 	wmsum_init(&arc_sums.arcstat_demand_iohit_prescient_prefetch, 0);
7794 	wmsum_init(&arc_sums.arcstat_raw_size, 0);
7795 	wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
7796 	wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);
7797 
7798 	arc_anon->arcs_state = ARC_STATE_ANON;
7799 	arc_mru->arcs_state = ARC_STATE_MRU;
7800 	arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
7801 	arc_mfu->arcs_state = ARC_STATE_MFU;
7802 	arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
7803 	arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
7804 	arc_uncached->arcs_state = ARC_STATE_UNCACHED;
7805 }
7806 
7807 static void
arc_state_fini(void)7808 arc_state_fini(void)
7809 {
7810 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7811 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7812 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7813 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7814 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7815 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7816 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7817 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7818 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7819 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7820 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7821 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7822 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7823 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7824 
7825 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7826 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7827 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7828 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7829 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7830 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7831 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7832 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7833 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7834 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7835 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7836 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7837 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7838 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7839 
7840 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
7841 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7842 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7843 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7844 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
7845 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7846 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
7847 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7848 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7849 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
7850 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
7851 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);
7852 
7853 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
7854 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
7855 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
7856 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
7857 
7858 	wmsum_fini(&arc_sums.arcstat_hits);
7859 	wmsum_fini(&arc_sums.arcstat_iohits);
7860 	wmsum_fini(&arc_sums.arcstat_misses);
7861 	wmsum_fini(&arc_sums.arcstat_demand_data_hits);
7862 	wmsum_fini(&arc_sums.arcstat_demand_data_iohits);
7863 	wmsum_fini(&arc_sums.arcstat_demand_data_misses);
7864 	wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
7865 	wmsum_fini(&arc_sums.arcstat_demand_metadata_iohits);
7866 	wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
7867 	wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
7868 	wmsum_fini(&arc_sums.arcstat_prefetch_data_iohits);
7869 	wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
7870 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
7871 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_iohits);
7872 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
7873 	wmsum_fini(&arc_sums.arcstat_mru_hits);
7874 	wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
7875 	wmsum_fini(&arc_sums.arcstat_mfu_hits);
7876 	wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
7877 	wmsum_fini(&arc_sums.arcstat_uncached_hits);
7878 	wmsum_fini(&arc_sums.arcstat_deleted);
7879 	wmsum_fini(&arc_sums.arcstat_mutex_miss);
7880 	wmsum_fini(&arc_sums.arcstat_access_skip);
7881 	wmsum_fini(&arc_sums.arcstat_evict_skip);
7882 	wmsum_fini(&arc_sums.arcstat_evict_not_enough);
7883 	wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
7884 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
7885 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
7886 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
7887 	wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
7888 	wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
7889 	wmsum_fini(&arc_sums.arcstat_hash_elements);
7890 	wmsum_fini(&arc_sums.arcstat_hash_collisions);
7891 	wmsum_fini(&arc_sums.arcstat_hash_chains);
7892 	aggsum_fini(&arc_sums.arcstat_size);
7893 	wmsum_fini(&arc_sums.arcstat_compressed_size);
7894 	wmsum_fini(&arc_sums.arcstat_uncompressed_size);
7895 	wmsum_fini(&arc_sums.arcstat_overhead_size);
7896 	wmsum_fini(&arc_sums.arcstat_hdr_size);
7897 	wmsum_fini(&arc_sums.arcstat_data_size);
7898 	wmsum_fini(&arc_sums.arcstat_metadata_size);
7899 	wmsum_fini(&arc_sums.arcstat_dbuf_size);
7900 	wmsum_fini(&arc_sums.arcstat_dnode_size);
7901 	wmsum_fini(&arc_sums.arcstat_bonus_size);
7902 	wmsum_fini(&arc_sums.arcstat_l2_hits);
7903 	wmsum_fini(&arc_sums.arcstat_l2_misses);
7904 	wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
7905 	wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
7906 	wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
7907 	wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
7908 	wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
7909 	wmsum_fini(&arc_sums.arcstat_l2_feeds);
7910 	wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
7911 	wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
7912 	wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
7913 	wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
7914 	wmsum_fini(&arc_sums.arcstat_l2_writes_done);
7915 	wmsum_fini(&arc_sums.arcstat_l2_writes_error);
7916 	wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
7917 	wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
7918 	wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
7919 	wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
7920 	wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
7921 	wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
7922 	wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
7923 	wmsum_fini(&arc_sums.arcstat_l2_io_error);
7924 	wmsum_fini(&arc_sums.arcstat_l2_lsize);
7925 	wmsum_fini(&arc_sums.arcstat_l2_psize);
7926 	aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
7927 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
7928 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
7929 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
7930 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
7931 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
7932 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
7933 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
7934 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
7935 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
7936 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
7937 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
7938 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
7939 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
7940 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
7941 	wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
7942 	wmsum_fini(&arc_sums.arcstat_memory_direct_count);
7943 	wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
7944 	wmsum_fini(&arc_sums.arcstat_prune);
7945 	wmsum_fini(&arc_sums.arcstat_meta_used);
7946 	wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
7947 	wmsum_fini(&arc_sums.arcstat_predictive_prefetch);
7948 	wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7949 	wmsum_fini(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7950 	wmsum_fini(&arc_sums.arcstat_prescient_prefetch);
7951 	wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7952 	wmsum_fini(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7953 	wmsum_fini(&arc_sums.arcstat_raw_size);
7954 	wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
7955 	wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
7956 }
7957 
7958 uint64_t
arc_target_bytes(void)7959 arc_target_bytes(void)
7960 {
7961 	return (arc_c);
7962 }
7963 
7964 void
arc_set_limits(uint64_t allmem)7965 arc_set_limits(uint64_t allmem)
7966 {
7967 	/* Set min cache to 1/32 of all memory, or 32MB, whichever is more. */
7968 	arc_c_min = MAX(allmem / 32, 2ULL << SPA_MAXBLOCKSHIFT);
7969 
7970 	/* How to set default max varies by platform. */
7971 	arc_c_max = arc_default_max(arc_c_min, allmem);
7972 }
7973 
7974 void
arc_init(void)7975 arc_init(void)
7976 {
7977 	uint64_t percent, allmem = arc_all_memory();
7978 	mutex_init(&arc_evict_lock, NULL, MUTEX_DEFAULT, NULL);
7979 	list_create(&arc_evict_waiters, sizeof (arc_evict_waiter_t),
7980 	    offsetof(arc_evict_waiter_t, aew_node));
7981 
7982 	arc_min_prefetch_ms = 1000;
7983 	arc_min_prescient_prefetch_ms = 6000;
7984 
7985 #if defined(_KERNEL)
7986 	arc_lowmem_init();
7987 #endif
7988 
7989 	arc_set_limits(allmem);
7990 
7991 #ifdef _KERNEL
7992 	/*
7993 	 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
7994 	 * environment before the module was loaded, don't block setting the
7995 	 * maximum because it is less than arc_c_min, instead, reset arc_c_min
7996 	 * to a lower value.
7997 	 * zfs_arc_min will be handled by arc_tuning_update().
7998 	 */
7999 	if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
8000 	    zfs_arc_max < allmem) {
8001 		arc_c_max = zfs_arc_max;
8002 		if (arc_c_min >= arc_c_max) {
8003 			arc_c_min = MAX(zfs_arc_max / 2,
8004 			    2ULL << SPA_MAXBLOCKSHIFT);
8005 		}
8006 	}
8007 #else
8008 	/*
8009 	 * In userland, there's only the memory pressure that we artificially
8010 	 * create (see arc_available_memory()).  Don't let arc_c get too
8011 	 * small, because it can cause transactions to be larger than
8012 	 * arc_c, causing arc_tempreserve_space() to fail.
8013 	 */
8014 	arc_c_min = MAX(arc_c_max / 2, 2ULL << SPA_MAXBLOCKSHIFT);
8015 #endif
8016 
8017 	arc_c = arc_c_min;
8018 	/*
8019 	 * 32-bit fixed point fractions of metadata from total ARC size,
8020 	 * MRU data from all data and MRU metadata from all metadata.
8021 	 */
8022 	arc_meta = (1ULL << 32) / 4;	/* Metadata is 25% of arc_c. */
8023 	arc_pd = (1ULL << 32) / 2;	/* Data MRU is 50% of data. */
8024 	arc_pm = (1ULL << 32) / 2;	/* Metadata MRU is 50% of metadata. */
8025 
8026 	percent = MIN(zfs_arc_dnode_limit_percent, 100);
8027 	arc_dnode_limit = arc_c_max * percent / 100;
8028 
8029 	/* Apply user specified tunings */
8030 	arc_tuning_update(B_TRUE);
8031 
8032 	/* if kmem_flags are set, lets try to use less memory */
8033 	if (kmem_debugging())
8034 		arc_c = arc_c / 2;
8035 	if (arc_c < arc_c_min)
8036 		arc_c = arc_c_min;
8037 
8038 	arc_register_hotplug();
8039 
8040 	arc_state_init();
8041 
8042 	buf_init();
8043 
8044 	list_create(&arc_prune_list, sizeof (arc_prune_t),
8045 	    offsetof(arc_prune_t, p_node));
8046 	mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL);
8047 
8048 	arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
8049 	    defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
8050 
8051 	arc_evict_thread_init();
8052 
8053 	list_create(&arc_async_flush_list, sizeof (arc_async_flush_t),
8054 	    offsetof(arc_async_flush_t, af_node));
8055 	mutex_init(&arc_async_flush_lock, NULL, MUTEX_DEFAULT, NULL);
8056 	arc_flush_taskq = taskq_create("arc_flush", MIN(boot_ncpus, 4),
8057 	    defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC);
8058 
8059 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
8060 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
8061 
8062 	if (arc_ksp != NULL) {
8063 		arc_ksp->ks_data = &arc_stats;
8064 		arc_ksp->ks_update = arc_kstat_update;
8065 		kstat_install(arc_ksp);
8066 	}
8067 
8068 	arc_state_evict_markers =
8069 	    arc_state_alloc_markers(arc_state_evict_marker_count);
8070 	arc_evict_zthr = zthr_create_timer("arc_evict",
8071 	    arc_evict_cb_check, arc_evict_cb, NULL, SEC2NSEC(1), defclsyspri);
8072 	arc_reap_zthr = zthr_create_timer("arc_reap",
8073 	    arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);
8074 
8075 	arc_warm = B_FALSE;
8076 
8077 	/*
8078 	 * Calculate maximum amount of dirty data per pool.
8079 	 *
8080 	 * If it has been set by a module parameter, take that.
8081 	 * Otherwise, use a percentage of physical memory defined by
8082 	 * zfs_dirty_data_max_percent (default 10%) with a cap at
8083 	 * zfs_dirty_data_max_max (default 4G or 25% of physical memory).
8084 	 */
8085 #ifdef __LP64__
8086 	if (zfs_dirty_data_max_max == 0)
8087 		zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
8088 		    allmem * zfs_dirty_data_max_max_percent / 100);
8089 #else
8090 	if (zfs_dirty_data_max_max == 0)
8091 		zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
8092 		    allmem * zfs_dirty_data_max_max_percent / 100);
8093 #endif
8094 
8095 	if (zfs_dirty_data_max == 0) {
8096 		zfs_dirty_data_max = allmem *
8097 		    zfs_dirty_data_max_percent / 100;
8098 		zfs_dirty_data_max = MIN(zfs_dirty_data_max,
8099 		    zfs_dirty_data_max_max);
8100 	}
8101 
8102 	if (zfs_wrlog_data_max == 0) {
8103 
8104 		/*
8105 		 * dp_wrlog_total is reduced for each txg at the end of
8106 		 * spa_sync(). However, dp_dirty_total is reduced every time
8107 		 * a block is written out. Thus under normal operation,
8108 		 * dp_wrlog_total could grow 2 times as big as
8109 		 * zfs_dirty_data_max.
8110 		 */
8111 		zfs_wrlog_data_max = zfs_dirty_data_max * 2;
8112 	}
8113 }
8114 
8115 void
arc_fini(void)8116 arc_fini(void)
8117 {
8118 	arc_prune_t *p;
8119 
8120 #ifdef _KERNEL
8121 	arc_lowmem_fini();
8122 #endif /* _KERNEL */
8123 
8124 	/* Wait for any background flushes */
8125 	taskq_wait(arc_flush_taskq);
8126 	taskq_destroy(arc_flush_taskq);
8127 
8128 	/* Use B_TRUE to ensure *all* buffers are evicted */
8129 	arc_flush(NULL, B_TRUE);
8130 
8131 	if (arc_ksp != NULL) {
8132 		kstat_delete(arc_ksp);
8133 		arc_ksp = NULL;
8134 	}
8135 
8136 	taskq_wait(arc_prune_taskq);
8137 	taskq_destroy(arc_prune_taskq);
8138 
8139 	list_destroy(&arc_async_flush_list);
8140 	mutex_destroy(&arc_async_flush_lock);
8141 
8142 	mutex_enter(&arc_prune_mtx);
8143 	while ((p = list_remove_head(&arc_prune_list)) != NULL) {
8144 		(void) zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
8145 		zfs_refcount_destroy(&p->p_refcnt);
8146 		kmem_free(p, sizeof (*p));
8147 	}
8148 	mutex_exit(&arc_prune_mtx);
8149 
8150 	list_destroy(&arc_prune_list);
8151 	mutex_destroy(&arc_prune_mtx);
8152 
8153 	if (arc_evict_taskq != NULL)
8154 		taskq_wait(arc_evict_taskq);
8155 
8156 	(void) zthr_cancel(arc_evict_zthr);
8157 	(void) zthr_cancel(arc_reap_zthr);
8158 	arc_state_free_markers(arc_state_evict_markers,
8159 	    arc_state_evict_marker_count);
8160 
8161 	if (arc_evict_taskq != NULL) {
8162 		taskq_destroy(arc_evict_taskq);
8163 		kmem_free(arc_evict_arg,
8164 		    sizeof (evict_arg_t) * zfs_arc_evict_threads);
8165 	}
8166 
8167 	mutex_destroy(&arc_evict_lock);
8168 	list_destroy(&arc_evict_waiters);
8169 
8170 	/*
8171 	 * Free any buffers that were tagged for destruction.  This needs
8172 	 * to occur before arc_state_fini() runs and destroys the aggsum
8173 	 * values which are updated when freeing scatter ABDs.
8174 	 */
8175 	l2arc_do_free_on_write();
8176 
8177 	/*
8178 	 * buf_fini() must proceed arc_state_fini() because buf_fin() may
8179 	 * trigger the release of kmem magazines, which can callback to
8180 	 * arc_space_return() which accesses aggsums freed in act_state_fini().
8181 	 */
8182 	buf_fini();
8183 	arc_state_fini();
8184 
8185 	arc_unregister_hotplug();
8186 
8187 	/*
8188 	 * We destroy the zthrs after all the ARC state has been
8189 	 * torn down to avoid the case of them receiving any
8190 	 * wakeup() signals after they are destroyed.
8191 	 */
8192 	zthr_destroy(arc_evict_zthr);
8193 	zthr_destroy(arc_reap_zthr);
8194 
8195 	ASSERT0(arc_loaned_bytes);
8196 }
8197 
8198 /*
8199  * Level 2 ARC
8200  *
8201  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
8202  * It uses dedicated storage devices to hold cached data, which are populated
8203  * using large infrequent writes.  The main role of this cache is to boost
8204  * the performance of random read workloads.  The intended L2ARC devices
8205  * include short-stroked disks, solid state disks, and other media with
8206  * substantially faster read latency than disk.
8207  *
8208  *                 +-----------------------+
8209  *                 |         ARC           |
8210  *                 +-----------------------+
8211  *                    |         ^     ^
8212  *                    |         |     |
8213  *      l2arc_feed_thread()    arc_read()
8214  *                    |         |     |
8215  *                    |  l2arc read   |
8216  *                    V         |     |
8217  *               +---------------+    |
8218  *               |     L2ARC     |    |
8219  *               +---------------+    |
8220  *                   |    ^           |
8221  *          l2arc_write() |           |
8222  *                   |    |           |
8223  *                   V    |           |
8224  *                 +-------+      +-------+
8225  *                 | vdev  |      | vdev  |
8226  *                 | cache |      | cache |
8227  *                 +-------+      +-------+
8228  *                 +=========+     .-----.
8229  *                 :  L2ARC  :    |-_____-|
8230  *                 : devices :    | Disks |
8231  *                 +=========+    `-_____-'
8232  *
8233  * Read requests are satisfied from the following sources, in order:
8234  *
8235  *	1) ARC
8236  *	2) vdev cache of L2ARC devices
8237  *	3) L2ARC devices
8238  *	4) vdev cache of disks
8239  *	5) disks
8240  *
8241  * Some L2ARC device types exhibit extremely slow write performance.
8242  * To accommodate for this there are some significant differences between
8243  * the L2ARC and traditional cache design:
8244  *
8245  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
8246  * the ARC behave as usual, freeing buffers and placing headers on ghost
8247  * lists.  The ARC does not send buffers to the L2ARC during eviction as
8248  * this would add inflated write latencies for all ARC memory pressure.
8249  *
8250  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
8251  * It does this by periodically scanning buffers from the eviction-end of
8252  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
8253  * not already there. It scans until a headroom of buffers is satisfied,
8254  * which itself is a buffer for ARC eviction. If a compressible buffer is
8255  * found during scanning and selected for writing to an L2ARC device, we
8256  * temporarily boost scanning headroom during the next scan cycle to make
8257  * sure we adapt to compression effects (which might significantly reduce
8258  * the data volume we write to L2ARC). The thread that does this is
8259  * l2arc_feed_thread(), illustrated below; example sizes are included to
8260  * provide a better sense of ratio than this diagram:
8261  *
8262  *	       head -->                        tail
8263  *	        +---------------------+----------+
8264  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
8265  *	        +---------------------+----------+   |   o L2ARC eligible
8266  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
8267  *	        +---------------------+----------+   |
8268  *	             15.9 Gbytes      ^ 32 Mbytes    |
8269  *	                           headroom          |
8270  *	                                      l2arc_feed_thread()
8271  *	                                             |
8272  *	                 l2arc write hand <--[oooo]--'
8273  *	                         |           8 Mbyte
8274  *	                         |          write max
8275  *	                         V
8276  *		  +==============================+
8277  *	L2ARC dev |####|#|###|###|    |####| ... |
8278  *	          +==============================+
8279  *	                     32 Gbytes
8280  *
8281  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
8282  * evicted, then the L2ARC has cached a buffer much sooner than it probably
8283  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
8284  * safe to say that this is an uncommon case, since buffers at the end of
8285  * the ARC lists have moved there due to inactivity.
8286  *
8287  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
8288  * then the L2ARC simply misses copying some buffers.  This serves as a
8289  * pressure valve to prevent heavy read workloads from both stalling the ARC
8290  * with waits and clogging the L2ARC with writes.  This also helps prevent
8291  * the potential for the L2ARC to churn if it attempts to cache content too
8292  * quickly, such as during backups of the entire pool.
8293  *
8294  * 5. After system boot and before the ARC has filled main memory, there are
8295  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
8296  * lists can remain mostly static.  Instead of searching from tail of these
8297  * lists as pictured, the l2arc_feed_thread() will search from the list heads
8298  * for eligible buffers, greatly increasing its chance of finding them.
8299  *
8300  * The L2ARC device write speed is also boosted during this time so that
8301  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
8302  * there are no L2ARC reads, and no fear of degrading read performance
8303  * through increased writes.
8304  *
8305  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
8306  * the vdev queue can aggregate them into larger and fewer writes.  Each
8307  * device is written to in a rotor fashion, sweeping writes through
8308  * available space then repeating.
8309  *
8310  * 7. The L2ARC does not store dirty content.  It never needs to flush
8311  * write buffers back to disk based storage.
8312  *
8313  * 8. If an ARC buffer is written (and dirtied) which also exists in the
8314  * L2ARC, the now stale L2ARC buffer is immediately dropped.
8315  *
8316  * The performance of the L2ARC can be tweaked by a number of tunables, which
8317  * may be necessary for different workloads:
8318  *
8319  *	l2arc_write_max		max write bytes per interval
8320  *	l2arc_write_boost	extra write bytes during device warmup
8321  *	l2arc_noprefetch	skip caching prefetched buffers
8322  *	l2arc_headroom		number of max device writes to precache
8323  *	l2arc_headroom_boost	when we find compressed buffers during ARC
8324  *				scanning, we multiply headroom by this
8325  *				percentage factor for the next scan cycle,
8326  *				since more compressed buffers are likely to
8327  *				be present
8328  *	l2arc_feed_secs		seconds between L2ARC writing
8329  *
8330  * Tunables may be removed or added as future performance improvements are
8331  * integrated, and also may become zpool properties.
8332  *
8333  * There are three key functions that control how the L2ARC warms up:
8334  *
8335  *	l2arc_write_eligible()	check if a buffer is eligible to cache
8336  *	l2arc_write_size()	calculate how much to write
8337  *	l2arc_write_interval()	calculate sleep delay between writes
8338  *
8339  * These three functions determine what to write, how much, and how quickly
8340  * to send writes.
8341  *
8342  * L2ARC persistence:
8343  *
8344  * When writing buffers to L2ARC, we periodically add some metadata to
8345  * make sure we can pick them up after reboot, thus dramatically reducing
8346  * the impact that any downtime has on the performance of storage systems
8347  * with large caches.
8348  *
8349  * The implementation works fairly simply by integrating the following two
8350  * modifications:
8351  *
8352  * *) When writing to the L2ARC, we occasionally write a "l2arc log block",
8353  *    which is an additional piece of metadata which describes what's been
8354  *    written. This allows us to rebuild the arc_buf_hdr_t structures of the
8355  *    main ARC buffers. There are 2 linked-lists of log blocks headed by
8356  *    dh_start_lbps[2]. We alternate which chain we append to, so they are
8357  *    time-wise and offset-wise interleaved, but that is an optimization rather
8358  *    than for correctness. The log block also includes a pointer to the
8359  *    previous block in its chain.
8360  *
8361  * *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device
8362  *    for our header bookkeeping purposes. This contains a device header,
8363  *    which contains our top-level reference structures. We update it each
8364  *    time we write a new log block, so that we're able to locate it in the
8365  *    L2ARC device. If this write results in an inconsistent device header
8366  *    (e.g. due to power failure), we detect this by verifying the header's
8367  *    checksum and simply fail to reconstruct the L2ARC after reboot.
8368  *
8369  * Implementation diagram:
8370  *
8371  * +=== L2ARC device (not to scale) ======================================+
8372  * |       ___two newest log block pointers__.__________                  |
8373  * |      /                                   \dh_start_lbps[1]           |
8374  * |	 /				       \         \dh_start_lbps[0]|
8375  * |.___/__.                                    V         V               |
8376  * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---|
8377  * ||   hdr|      ^         /^       /^        /         /                |
8378  * |+------+  ...--\-------/  \-----/--\------/         /                 |
8379  * |                \--------------/    \--------------/                  |
8380  * +======================================================================+
8381  *
8382  * As can be seen on the diagram, rather than using a simple linked list,
8383  * we use a pair of linked lists with alternating elements. This is a
8384  * performance enhancement due to the fact that we only find out the
8385  * address of the next log block access once the current block has been
8386  * completely read in. Obviously, this hurts performance, because we'd be
8387  * keeping the device's I/O queue at only a 1 operation deep, thus
8388  * incurring a large amount of I/O round-trip latency. Having two lists
8389  * allows us to fetch two log blocks ahead of where we are currently
8390  * rebuilding L2ARC buffers.
8391  *
8392  * On-device data structures:
8393  *
8394  * L2ARC device header:	l2arc_dev_hdr_phys_t
8395  * L2ARC log block:	l2arc_log_blk_phys_t
8396  *
8397  * L2ARC reconstruction:
8398  *
8399  * When writing data, we simply write in the standard rotary fashion,
8400  * evicting buffers as we go and simply writing new data over them (writing
8401  * a new log block every now and then). This obviously means that once we
8402  * loop around the end of the device, we will start cutting into an already
8403  * committed log block (and its referenced data buffers), like so:
8404  *
8405  *    current write head__       __old tail
8406  *                        \     /
8407  *                        V    V
8408  * <--|bufs |lb |bufs |lb |    |bufs |lb |bufs |lb |-->
8409  *                         ^    ^^^^^^^^^___________________________________
8410  *                         |                                                \
8411  *                   <<nextwrite>> may overwrite this blk and/or its bufs --'
8412  *
8413  * When importing the pool, we detect this situation and use it to stop
8414  * our scanning process (see l2arc_rebuild).
8415  *
8416  * There is one significant caveat to consider when rebuilding ARC contents
8417  * from an L2ARC device: what about invalidated buffers? Given the above
8418  * construction, we cannot update blocks which we've already written to amend
8419  * them to remove buffers which were invalidated. Thus, during reconstruction,
8420  * we might be populating the cache with buffers for data that's not on the
8421  * main pool anymore, or may have been overwritten!
8422  *
8423  * As it turns out, this isn't a problem. Every arc_read request includes
8424  * both the DVA and, crucially, the birth TXG of the BP the caller is
8425  * looking for. So even if the cache were populated by completely rotten
8426  * blocks for data that had been long deleted and/or overwritten, we'll
8427  * never actually return bad data from the cache, since the DVA with the
8428  * birth TXG uniquely identify a block in space and time - once created,
8429  * a block is immutable on disk. The worst thing we have done is wasted
8430  * some time and memory at l2arc rebuild to reconstruct outdated ARC
8431  * entries that will get dropped from the l2arc as it is being updated
8432  * with new blocks.
8433  *
8434  * L2ARC buffers that have been evicted by l2arc_evict() ahead of the write
8435  * hand are not restored. This is done by saving the offset (in bytes)
8436  * l2arc_evict() has evicted to in the L2ARC device header and taking it
8437  * into account when restoring buffers.
8438  */
8439 
8440 static boolean_t
l2arc_write_eligible(uint64_t spa_guid,arc_buf_hdr_t * hdr)8441 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
8442 {
8443 	/*
8444 	 * A buffer is *not* eligible for the L2ARC if it:
8445 	 * 1. belongs to a different spa.
8446 	 * 2. is already cached on the L2ARC.
8447 	 * 3. has an I/O in progress (it may be an incomplete read).
8448 	 * 4. is flagged not eligible (zfs property).
8449 	 */
8450 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
8451 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
8452 		return (B_FALSE);
8453 
8454 	return (B_TRUE);
8455 }
8456 
8457 static uint64_t
l2arc_write_size(l2arc_dev_t * dev)8458 l2arc_write_size(l2arc_dev_t *dev)
8459 {
8460 	uint64_t size;
8461 
8462 	/*
8463 	 * Make sure our globals have meaningful values in case the user
8464 	 * altered them.
8465 	 */
8466 	size = l2arc_write_max;
8467 	if (size == 0) {
8468 		cmn_err(CE_NOTE, "l2arc_write_max must be greater than zero, "
8469 		    "resetting it to the default (%d)", L2ARC_WRITE_SIZE);
8470 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
8471 	}
8472 
8473 	if (arc_warm == B_FALSE)
8474 		size += l2arc_write_boost;
8475 
8476 	/* We need to add in the worst case scenario of log block overhead. */
8477 	size += l2arc_log_blk_overhead(size, dev);
8478 	if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
8479 		/*
8480 		 * Trim ahead of the write size 64MB or (l2arc_trim_ahead/100)
8481 		 * times the writesize, whichever is greater.
8482 		 */
8483 		size += MAX(64 * 1024 * 1024,
8484 		    (size * l2arc_trim_ahead) / 100);
8485 	}
8486 
8487 	/*
8488 	 * Make sure the write size does not exceed the size of the cache
8489 	 * device. This is important in l2arc_evict(), otherwise infinite
8490 	 * iteration can occur.
8491 	 */
8492 	size = MIN(size, (dev->l2ad_end - dev->l2ad_start) / 4);
8493 
8494 	size = P2ROUNDUP(size, 1ULL << dev->l2ad_vdev->vdev_ashift);
8495 
8496 	return (size);
8497 
8498 }
8499 
8500 static clock_t
l2arc_write_interval(clock_t began,uint64_t wanted,uint64_t wrote)8501 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
8502 {
8503 	clock_t interval, next, now;
8504 
8505 	/*
8506 	 * If the ARC lists are busy, increase our write rate; if the
8507 	 * lists are stale, idle back.  This is achieved by checking
8508 	 * how much we previously wrote - if it was more than half of
8509 	 * what we wanted, schedule the next write much sooner.
8510 	 */
8511 	if (l2arc_feed_again && wrote > (wanted / 2))
8512 		interval = (hz * l2arc_feed_min_ms) / 1000;
8513 	else
8514 		interval = hz * l2arc_feed_secs;
8515 
8516 	now = ddi_get_lbolt();
8517 	next = MAX(now, MIN(now + interval, began + interval));
8518 
8519 	return (next);
8520 }
8521 
8522 static boolean_t
l2arc_dev_invalid(const l2arc_dev_t * dev)8523 l2arc_dev_invalid(const l2arc_dev_t *dev)
8524 {
8525 	/*
8526 	 * We want to skip devices that are being rebuilt, trimmed,
8527 	 * removed, or belong to a spa that is being exported.
8528 	 */
8529 	return (dev->l2ad_vdev == NULL || vdev_is_dead(dev->l2ad_vdev) ||
8530 	    dev->l2ad_rebuild || dev->l2ad_trim_all ||
8531 	    dev->l2ad_spa == NULL || dev->l2ad_spa->spa_is_exporting);
8532 }
8533 
8534 /*
8535  * Cycle through L2ARC devices.  This is how L2ARC load balances.
8536  * If a device is returned, this also returns holding the spa config lock.
8537  */
8538 static l2arc_dev_t *
l2arc_dev_get_next(void)8539 l2arc_dev_get_next(void)
8540 {
8541 	l2arc_dev_t *first, *next = NULL;
8542 
8543 	/*
8544 	 * Lock out the removal of spas (spa_namespace_lock), then removal
8545 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
8546 	 * both locks will be dropped and a spa config lock held instead.
8547 	 */
8548 	mutex_enter(&spa_namespace_lock);
8549 	mutex_enter(&l2arc_dev_mtx);
8550 
8551 	/* if there are no vdevs, there is nothing to do */
8552 	if (l2arc_ndev == 0)
8553 		goto out;
8554 
8555 	first = NULL;
8556 	next = l2arc_dev_last;
8557 	do {
8558 		/* loop around the list looking for a non-faulted vdev */
8559 		if (next == NULL) {
8560 			next = list_head(l2arc_dev_list);
8561 		} else {
8562 			next = list_next(l2arc_dev_list, next);
8563 			if (next == NULL)
8564 				next = list_head(l2arc_dev_list);
8565 		}
8566 
8567 		/* if we have come back to the start, bail out */
8568 		if (first == NULL)
8569 			first = next;
8570 		else if (next == first)
8571 			break;
8572 
8573 		ASSERT3P(next, !=, NULL);
8574 	} while (l2arc_dev_invalid(next));
8575 
8576 	/* if we were unable to find any usable vdevs, return NULL */
8577 	if (l2arc_dev_invalid(next))
8578 		next = NULL;
8579 
8580 	l2arc_dev_last = next;
8581 
8582 out:
8583 	mutex_exit(&l2arc_dev_mtx);
8584 
8585 	/*
8586 	 * Grab the config lock to prevent the 'next' device from being
8587 	 * removed while we are writing to it.
8588 	 */
8589 	if (next != NULL)
8590 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
8591 	mutex_exit(&spa_namespace_lock);
8592 
8593 	return (next);
8594 }
8595 
8596 /*
8597  * Free buffers that were tagged for destruction.
8598  */
8599 static void
l2arc_do_free_on_write(void)8600 l2arc_do_free_on_write(void)
8601 {
8602 	l2arc_data_free_t *df;
8603 
8604 	mutex_enter(&l2arc_free_on_write_mtx);
8605 	while ((df = list_remove_head(l2arc_free_on_write)) != NULL) {
8606 		ASSERT3P(df->l2df_abd, !=, NULL);
8607 		abd_free(df->l2df_abd);
8608 		kmem_free(df, sizeof (l2arc_data_free_t));
8609 	}
8610 	mutex_exit(&l2arc_free_on_write_mtx);
8611 }
8612 
8613 /*
8614  * A write to a cache device has completed.  Update all headers to allow
8615  * reads from these buffers to begin.
8616  */
8617 static void
l2arc_write_done(zio_t * zio)8618 l2arc_write_done(zio_t *zio)
8619 {
8620 	l2arc_write_callback_t	*cb;
8621 	l2arc_lb_abd_buf_t	*abd_buf;
8622 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
8623 	l2arc_dev_t		*dev;
8624 	l2arc_dev_hdr_phys_t	*l2dhdr;
8625 	list_t			*buflist;
8626 	arc_buf_hdr_t		*head, *hdr, *hdr_prev;
8627 	kmutex_t		*hash_lock;
8628 	int64_t			bytes_dropped = 0;
8629 
8630 	cb = zio->io_private;
8631 	ASSERT3P(cb, !=, NULL);
8632 	dev = cb->l2wcb_dev;
8633 	l2dhdr = dev->l2ad_dev_hdr;
8634 	ASSERT3P(dev, !=, NULL);
8635 	head = cb->l2wcb_head;
8636 	ASSERT3P(head, !=, NULL);
8637 	buflist = &dev->l2ad_buflist;
8638 	ASSERT3P(buflist, !=, NULL);
8639 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
8640 	    l2arc_write_callback_t *, cb);
8641 
8642 	/*
8643 	 * All writes completed, or an error was hit.
8644 	 */
8645 top:
8646 	mutex_enter(&dev->l2ad_mtx);
8647 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
8648 		hdr_prev = list_prev(buflist, hdr);
8649 
8650 		hash_lock = HDR_LOCK(hdr);
8651 
8652 		/*
8653 		 * We cannot use mutex_enter or else we can deadlock
8654 		 * with l2arc_write_buffers (due to swapping the order
8655 		 * the hash lock and l2ad_mtx are taken).
8656 		 */
8657 		if (!mutex_tryenter(hash_lock)) {
8658 			/*
8659 			 * Missed the hash lock. We must retry so we
8660 			 * don't leave the ARC_FLAG_L2_WRITING bit set.
8661 			 */
8662 			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
8663 
8664 			/*
8665 			 * We don't want to rescan the headers we've
8666 			 * already marked as having been written out, so
8667 			 * we reinsert the head node so we can pick up
8668 			 * where we left off.
8669 			 */
8670 			list_remove(buflist, head);
8671 			list_insert_after(buflist, hdr, head);
8672 
8673 			mutex_exit(&dev->l2ad_mtx);
8674 
8675 			/*
8676 			 * We wait for the hash lock to become available
8677 			 * to try and prevent busy waiting, and increase
8678 			 * the chance we'll be able to acquire the lock
8679 			 * the next time around.
8680 			 */
8681 			mutex_enter(hash_lock);
8682 			mutex_exit(hash_lock);
8683 			goto top;
8684 		}
8685 
8686 		/*
8687 		 * We could not have been moved into the arc_l2c_only
8688 		 * state while in-flight due to our ARC_FLAG_L2_WRITING
8689 		 * bit being set. Let's just ensure that's being enforced.
8690 		 */
8691 		ASSERT(HDR_HAS_L1HDR(hdr));
8692 
8693 		/*
8694 		 * Skipped - drop L2ARC entry and mark the header as no
8695 		 * longer L2 eligibile.
8696 		 */
8697 		if (zio->io_error != 0) {
8698 			/*
8699 			 * Error - drop L2ARC entry.
8700 			 */
8701 			list_remove(buflist, hdr);
8702 			arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
8703 
8704 			uint64_t psize = HDR_GET_PSIZE(hdr);
8705 			l2arc_hdr_arcstats_decrement(hdr);
8706 
8707 			ASSERT(dev->l2ad_vdev != NULL);
8708 
8709 			bytes_dropped +=
8710 			    vdev_psize_to_asize(dev->l2ad_vdev, psize);
8711 			(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
8712 			    arc_hdr_size(hdr), hdr);
8713 		}
8714 
8715 		/*
8716 		 * Allow ARC to begin reads and ghost list evictions to
8717 		 * this L2ARC entry.
8718 		 */
8719 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
8720 
8721 		mutex_exit(hash_lock);
8722 	}
8723 
8724 	/*
8725 	 * Free the allocated abd buffers for writing the log blocks.
8726 	 * If the zio failed reclaim the allocated space and remove the
8727 	 * pointers to these log blocks from the log block pointer list
8728 	 * of the L2ARC device.
8729 	 */
8730 	while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
8731 		abd_free(abd_buf->abd);
8732 		zio_buf_free(abd_buf, sizeof (*abd_buf));
8733 		if (zio->io_error != 0) {
8734 			lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
8735 			/*
8736 			 * L2BLK_GET_PSIZE returns aligned size for log
8737 			 * blocks.
8738 			 */
8739 			uint64_t asize =
8740 			    L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
8741 			bytes_dropped += asize;
8742 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
8743 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
8744 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
8745 			    lb_ptr_buf);
8746 			(void) zfs_refcount_remove(&dev->l2ad_lb_count,
8747 			    lb_ptr_buf);
8748 			kmem_free(lb_ptr_buf->lb_ptr,
8749 			    sizeof (l2arc_log_blkptr_t));
8750 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
8751 		}
8752 	}
8753 	list_destroy(&cb->l2wcb_abd_list);
8754 
8755 	if (zio->io_error != 0) {
8756 		ARCSTAT_BUMP(arcstat_l2_writes_error);
8757 
8758 		/*
8759 		 * Restore the lbps array in the header to its previous state.
8760 		 * If the list of log block pointers is empty, zero out the
8761 		 * log block pointers in the device header.
8762 		 */
8763 		lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
8764 		for (int i = 0; i < 2; i++) {
8765 			if (lb_ptr_buf == NULL) {
8766 				/*
8767 				 * If the list is empty zero out the device
8768 				 * header. Otherwise zero out the second log
8769 				 * block pointer in the header.
8770 				 */
8771 				if (i == 0) {
8772 					memset(l2dhdr, 0,
8773 					    dev->l2ad_dev_hdr_asize);
8774 				} else {
8775 					memset(&l2dhdr->dh_start_lbps[i], 0,
8776 					    sizeof (l2arc_log_blkptr_t));
8777 				}
8778 				break;
8779 			}
8780 			memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
8781 			    sizeof (l2arc_log_blkptr_t));
8782 			lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
8783 			    lb_ptr_buf);
8784 		}
8785 	}
8786 
8787 	ARCSTAT_BUMP(arcstat_l2_writes_done);
8788 	list_remove(buflist, head);
8789 	ASSERT(!HDR_HAS_L1HDR(head));
8790 	kmem_cache_free(hdr_l2only_cache, head);
8791 	mutex_exit(&dev->l2ad_mtx);
8792 
8793 	ASSERT(dev->l2ad_vdev != NULL);
8794 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
8795 
8796 	l2arc_do_free_on_write();
8797 
8798 	kmem_free(cb, sizeof (l2arc_write_callback_t));
8799 }
8800 
8801 static int
l2arc_untransform(zio_t * zio,l2arc_read_callback_t * cb)8802 l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
8803 {
8804 	int ret;
8805 	spa_t *spa = zio->io_spa;
8806 	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
8807 	blkptr_t *bp = zio->io_bp;
8808 	uint8_t salt[ZIO_DATA_SALT_LEN];
8809 	uint8_t iv[ZIO_DATA_IV_LEN];
8810 	uint8_t mac[ZIO_DATA_MAC_LEN];
8811 	boolean_t no_crypt = B_FALSE;
8812 
8813 	/*
8814 	 * ZIL data is never be written to the L2ARC, so we don't need
8815 	 * special handling for its unique MAC storage.
8816 	 */
8817 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
8818 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
8819 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8820 
8821 	/*
8822 	 * If the data was encrypted, decrypt it now. Note that
8823 	 * we must check the bp here and not the hdr, since the
8824 	 * hdr does not have its encryption parameters updated
8825 	 * until arc_read_done().
8826 	 */
8827 	if (BP_IS_ENCRYPTED(bp)) {
8828 		abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8829 		    ARC_HDR_USE_RESERVE);
8830 
8831 		zio_crypt_decode_params_bp(bp, salt, iv);
8832 		zio_crypt_decode_mac_bp(bp, mac);
8833 
8834 		ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
8835 		    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
8836 		    salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
8837 		    hdr->b_l1hdr.b_pabd, &no_crypt);
8838 		if (ret != 0) {
8839 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8840 			goto error;
8841 		}
8842 
8843 		/*
8844 		 * If we actually performed decryption, replace b_pabd
8845 		 * with the decrypted data. Otherwise we can just throw
8846 		 * our decryption buffer away.
8847 		 */
8848 		if (!no_crypt) {
8849 			arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8850 			    arc_hdr_size(hdr), hdr);
8851 			hdr->b_l1hdr.b_pabd = eabd;
8852 			zio->io_abd = eabd;
8853 		} else {
8854 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8855 		}
8856 	}
8857 
8858 	/*
8859 	 * If the L2ARC block was compressed, but ARC compression
8860 	 * is disabled we decompress the data into a new buffer and
8861 	 * replace the existing data.
8862 	 */
8863 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
8864 	    !HDR_COMPRESSION_ENABLED(hdr)) {
8865 		abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8866 		    ARC_HDR_USE_RESERVE);
8867 
8868 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
8869 		    hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
8870 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
8871 		if (ret != 0) {
8872 			arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
8873 			goto error;
8874 		}
8875 
8876 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8877 		    arc_hdr_size(hdr), hdr);
8878 		hdr->b_l1hdr.b_pabd = cabd;
8879 		zio->io_abd = cabd;
8880 		zio->io_size = HDR_GET_LSIZE(hdr);
8881 	}
8882 
8883 	return (0);
8884 
8885 error:
8886 	return (ret);
8887 }
8888 
8889 
8890 /*
8891  * A read to a cache device completed.  Validate buffer contents before
8892  * handing over to the regular ARC routines.
8893  */
8894 static void
l2arc_read_done(zio_t * zio)8895 l2arc_read_done(zio_t *zio)
8896 {
8897 	int tfm_error = 0;
8898 	l2arc_read_callback_t *cb = zio->io_private;
8899 	arc_buf_hdr_t *hdr;
8900 	kmutex_t *hash_lock;
8901 	boolean_t valid_cksum;
8902 	boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
8903 	    (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));
8904 
8905 	ASSERT3P(zio->io_vd, !=, NULL);
8906 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
8907 
8908 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
8909 
8910 	ASSERT3P(cb, !=, NULL);
8911 	hdr = cb->l2rcb_hdr;
8912 	ASSERT3P(hdr, !=, NULL);
8913 
8914 	hash_lock = HDR_LOCK(hdr);
8915 	mutex_enter(hash_lock);
8916 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
8917 
8918 	/*
8919 	 * If the data was read into a temporary buffer,
8920 	 * move it and free the buffer.
8921 	 */
8922 	if (cb->l2rcb_abd != NULL) {
8923 		ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
8924 		if (zio->io_error == 0) {
8925 			if (using_rdata) {
8926 				abd_copy(hdr->b_crypt_hdr.b_rabd,
8927 				    cb->l2rcb_abd, arc_hdr_size(hdr));
8928 			} else {
8929 				abd_copy(hdr->b_l1hdr.b_pabd,
8930 				    cb->l2rcb_abd, arc_hdr_size(hdr));
8931 			}
8932 		}
8933 
8934 		/*
8935 		 * The following must be done regardless of whether
8936 		 * there was an error:
8937 		 * - free the temporary buffer
8938 		 * - point zio to the real ARC buffer
8939 		 * - set zio size accordingly
8940 		 * These are required because zio is either re-used for
8941 		 * an I/O of the block in the case of the error
8942 		 * or the zio is passed to arc_read_done() and it
8943 		 * needs real data.
8944 		 */
8945 		abd_free(cb->l2rcb_abd);
8946 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
8947 
8948 		if (using_rdata) {
8949 			ASSERT(HDR_HAS_RABD(hdr));
8950 			zio->io_abd = zio->io_orig_abd =
8951 			    hdr->b_crypt_hdr.b_rabd;
8952 		} else {
8953 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8954 			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
8955 		}
8956 	}
8957 
8958 	ASSERT3P(zio->io_abd, !=, NULL);
8959 
8960 	/*
8961 	 * Check this survived the L2ARC journey.
8962 	 */
8963 	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
8964 	    (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
8965 	zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
8966 	zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
8967 	zio->io_prop.zp_complevel = hdr->b_complevel;
8968 
8969 	valid_cksum = arc_cksum_is_equal(hdr, zio);
8970 
8971 	/*
8972 	 * b_rabd will always match the data as it exists on disk if it is
8973 	 * being used. Therefore if we are reading into b_rabd we do not
8974 	 * attempt to untransform the data.
8975 	 */
8976 	if (valid_cksum && !using_rdata)
8977 		tfm_error = l2arc_untransform(zio, cb);
8978 
8979 	if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
8980 	    !HDR_L2_EVICTED(hdr)) {
8981 		mutex_exit(hash_lock);
8982 		zio->io_private = hdr;
8983 		arc_read_done(zio);
8984 	} else {
8985 		/*
8986 		 * Buffer didn't survive caching.  Increment stats and
8987 		 * reissue to the original storage device.
8988 		 */
8989 		if (zio->io_error != 0) {
8990 			ARCSTAT_BUMP(arcstat_l2_io_error);
8991 		} else {
8992 			zio->io_error = SET_ERROR(EIO);
8993 		}
8994 		if (!valid_cksum || tfm_error != 0)
8995 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
8996 
8997 		/*
8998 		 * If there's no waiter, issue an async i/o to the primary
8999 		 * storage now.  If there *is* a waiter, the caller must
9000 		 * issue the i/o in a context where it's OK to block.
9001 		 */
9002 		if (zio->io_waiter == NULL) {
9003 			zio_t *pio = zio_unique_parent(zio);
9004 			void *abd = (using_rdata) ?
9005 			    hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
9006 
9007 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
9008 
9009 			zio = zio_read(pio, zio->io_spa, zio->io_bp,
9010 			    abd, zio->io_size, arc_read_done,
9011 			    hdr, zio->io_priority, cb->l2rcb_flags,
9012 			    &cb->l2rcb_zb);
9013 
9014 			/*
9015 			 * Original ZIO will be freed, so we need to update
9016 			 * ARC header with the new ZIO pointer to be used
9017 			 * by zio_change_priority() in arc_read().
9018 			 */
9019 			for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
9020 			    acb != NULL; acb = acb->acb_next)
9021 				acb->acb_zio_head = zio;
9022 
9023 			mutex_exit(hash_lock);
9024 			zio_nowait(zio);
9025 		} else {
9026 			mutex_exit(hash_lock);
9027 		}
9028 	}
9029 
9030 	kmem_free(cb, sizeof (l2arc_read_callback_t));
9031 }
9032 
9033 /*
9034  * This is the list priority from which the L2ARC will search for pages to
9035  * cache.  This is used within loops (0..3) to cycle through lists in the
9036  * desired order.  This order can have a significant effect on cache
9037  * performance.
9038  *
9039  * Currently the metadata lists are hit first, MFU then MRU, followed by
9040  * the data lists.  This function returns a locked list, and also returns
9041  * the lock pointer.
9042  */
9043 static multilist_sublist_t *
l2arc_sublist_lock(int list_num)9044 l2arc_sublist_lock(int list_num)
9045 {
9046 	multilist_t *ml = NULL;
9047 	unsigned int idx;
9048 
9049 	ASSERT(list_num >= 0 && list_num < L2ARC_FEED_TYPES);
9050 
9051 	switch (list_num) {
9052 	case 0:
9053 		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
9054 		break;
9055 	case 1:
9056 		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
9057 		break;
9058 	case 2:
9059 		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
9060 		break;
9061 	case 3:
9062 		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
9063 		break;
9064 	default:
9065 		return (NULL);
9066 	}
9067 
9068 	/*
9069 	 * Return a randomly-selected sublist. This is acceptable
9070 	 * because the caller feeds only a little bit of data for each
9071 	 * call (8MB). Subsequent calls will result in different
9072 	 * sublists being selected.
9073 	 */
9074 	idx = multilist_get_random_index(ml);
9075 	return (multilist_sublist_lock_idx(ml, idx));
9076 }
9077 
9078 /*
9079  * Calculates the maximum overhead of L2ARC metadata log blocks for a given
9080  * L2ARC write size. l2arc_evict and l2arc_write_size need to include this
9081  * overhead in processing to make sure there is enough headroom available
9082  * when writing buffers.
9083  */
9084 static inline uint64_t
l2arc_log_blk_overhead(uint64_t write_sz,l2arc_dev_t * dev)9085 l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev)
9086 {
9087 	if (dev->l2ad_log_entries == 0) {
9088 		return (0);
9089 	} else {
9090 		ASSERT(dev->l2ad_vdev != NULL);
9091 
9092 		uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT;
9093 
9094 		uint64_t log_blocks = (log_entries +
9095 		    dev->l2ad_log_entries - 1) /
9096 		    dev->l2ad_log_entries;
9097 
9098 		return (vdev_psize_to_asize(dev->l2ad_vdev,
9099 		    sizeof (l2arc_log_blk_phys_t)) * log_blocks);
9100 	}
9101 }
9102 
9103 /*
9104  * Evict buffers from the device write hand to the distance specified in
9105  * bytes. This distance may span populated buffers, it may span nothing.
9106  * This is clearing a region on the L2ARC device ready for writing.
9107  * If the 'all' boolean is set, every buffer is evicted.
9108  */
9109 static void
l2arc_evict(l2arc_dev_t * dev,uint64_t distance,boolean_t all)9110 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
9111 {
9112 	list_t *buflist;
9113 	arc_buf_hdr_t *hdr, *hdr_prev;
9114 	kmutex_t *hash_lock;
9115 	uint64_t taddr;
9116 	l2arc_lb_ptr_buf_t *lb_ptr_buf, *lb_ptr_buf_prev;
9117 	vdev_t *vd = dev->l2ad_vdev;
9118 	boolean_t rerun;
9119 
9120 	ASSERT(vd != NULL || all);
9121 	ASSERT(dev->l2ad_spa != NULL || all);
9122 
9123 	buflist = &dev->l2ad_buflist;
9124 
9125 top:
9126 	rerun = B_FALSE;
9127 	if (dev->l2ad_hand + distance > dev->l2ad_end) {
9128 		/*
9129 		 * When there is no space to accommodate upcoming writes,
9130 		 * evict to the end. Then bump the write and evict hands
9131 		 * to the start and iterate. This iteration does not
9132 		 * happen indefinitely as we make sure in
9133 		 * l2arc_write_size() that when the write hand is reset,
9134 		 * the write size does not exceed the end of the device.
9135 		 */
9136 		rerun = B_TRUE;
9137 		taddr = dev->l2ad_end;
9138 	} else {
9139 		taddr = dev->l2ad_hand + distance;
9140 	}
9141 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
9142 	    uint64_t, taddr, boolean_t, all);
9143 
9144 	if (!all) {
9145 		/*
9146 		 * This check has to be placed after deciding whether to
9147 		 * iterate (rerun).
9148 		 */
9149 		if (dev->l2ad_first) {
9150 			/*
9151 			 * This is the first sweep through the device. There is
9152 			 * nothing to evict. We have already trimmmed the
9153 			 * whole device.
9154 			 */
9155 			goto out;
9156 		} else {
9157 			/*
9158 			 * Trim the space to be evicted.
9159 			 */
9160 			if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
9161 			    l2arc_trim_ahead > 0) {
9162 				/*
9163 				 * We have to drop the spa_config lock because
9164 				 * vdev_trim_range() will acquire it.
9165 				 * l2ad_evict already accounts for the label
9166 				 * size. To prevent vdev_trim_ranges() from
9167 				 * adding it again, we subtract it from
9168 				 * l2ad_evict.
9169 				 */
9170 				spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
9171 				vdev_trim_simple(vd,
9172 				    dev->l2ad_evict - VDEV_LABEL_START_SIZE,
9173 				    taddr - dev->l2ad_evict);
9174 				spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
9175 				    RW_READER);
9176 			}
9177 
9178 			/*
9179 			 * When rebuilding L2ARC we retrieve the evict hand
9180 			 * from the header of the device. Of note, l2arc_evict()
9181 			 * does not actually delete buffers from the cache
9182 			 * device, but trimming may do so depending on the
9183 			 * hardware implementation. Thus keeping track of the
9184 			 * evict hand is useful.
9185 			 */
9186 			dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
9187 		}
9188 	}
9189 
9190 retry:
9191 	mutex_enter(&dev->l2ad_mtx);
9192 	/*
9193 	 * We have to account for evicted log blocks. Run vdev_space_update()
9194 	 * on log blocks whose offset (in bytes) is before the evicted offset
9195 	 * (in bytes) by searching in the list of pointers to log blocks
9196 	 * present in the L2ARC device.
9197 	 */
9198 	for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
9199 	    lb_ptr_buf = lb_ptr_buf_prev) {
9200 
9201 		lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);
9202 
9203 		/* L2BLK_GET_PSIZE returns aligned size for log blocks */
9204 		uint64_t asize = L2BLK_GET_PSIZE(
9205 		    (lb_ptr_buf->lb_ptr)->lbp_prop);
9206 
9207 		/*
9208 		 * We don't worry about log blocks left behind (ie
9209 		 * lbp_payload_start < l2ad_hand) because l2arc_write_buffers()
9210 		 * will never write more than l2arc_evict() evicts.
9211 		 */
9212 		if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
9213 			break;
9214 		} else {
9215 			if (vd != NULL)
9216 				vdev_space_update(vd, -asize, 0, 0);
9217 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
9218 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
9219 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
9220 			    lb_ptr_buf);
9221 			(void) zfs_refcount_remove(&dev->l2ad_lb_count,
9222 			    lb_ptr_buf);
9223 			list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
9224 			kmem_free(lb_ptr_buf->lb_ptr,
9225 			    sizeof (l2arc_log_blkptr_t));
9226 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
9227 		}
9228 	}
9229 
9230 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
9231 		hdr_prev = list_prev(buflist, hdr);
9232 
9233 		ASSERT(!HDR_EMPTY(hdr));
9234 		hash_lock = HDR_LOCK(hdr);
9235 
9236 		/*
9237 		 * We cannot use mutex_enter or else we can deadlock
9238 		 * with l2arc_write_buffers (due to swapping the order
9239 		 * the hash lock and l2ad_mtx are taken).
9240 		 */
9241 		if (!mutex_tryenter(hash_lock)) {
9242 			/*
9243 			 * Missed the hash lock.  Retry.
9244 			 */
9245 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
9246 			mutex_exit(&dev->l2ad_mtx);
9247 			mutex_enter(hash_lock);
9248 			mutex_exit(hash_lock);
9249 			goto retry;
9250 		}
9251 
9252 		/*
9253 		 * A header can't be on this list if it doesn't have L2 header.
9254 		 */
9255 		ASSERT(HDR_HAS_L2HDR(hdr));
9256 
9257 		/* Ensure this header has finished being written. */
9258 		ASSERT(!HDR_L2_WRITING(hdr));
9259 		ASSERT(!HDR_L2_WRITE_HEAD(hdr));
9260 
9261 		if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
9262 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
9263 			/*
9264 			 * We've evicted to the target address,
9265 			 * or the end of the device.
9266 			 */
9267 			mutex_exit(hash_lock);
9268 			break;
9269 		}
9270 
9271 		if (!HDR_HAS_L1HDR(hdr)) {
9272 			ASSERT(!HDR_L2_READING(hdr));
9273 			/*
9274 			 * This doesn't exist in the ARC.  Destroy.
9275 			 * arc_hdr_destroy() will call list_remove()
9276 			 * and decrement arcstat_l2_lsize.
9277 			 */
9278 			arc_change_state(arc_anon, hdr);
9279 			arc_hdr_destroy(hdr);
9280 		} else {
9281 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
9282 			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
9283 			/*
9284 			 * Invalidate issued or about to be issued
9285 			 * reads, since we may be about to write
9286 			 * over this location.
9287 			 */
9288 			if (HDR_L2_READING(hdr)) {
9289 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
9290 				arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
9291 			}
9292 
9293 			arc_hdr_l2hdr_destroy(hdr);
9294 		}
9295 		mutex_exit(hash_lock);
9296 	}
9297 	mutex_exit(&dev->l2ad_mtx);
9298 
9299 out:
9300 	/*
9301 	 * We need to check if we evict all buffers, otherwise we may iterate
9302 	 * unnecessarily.
9303 	 */
9304 	if (!all && rerun) {
9305 		/*
9306 		 * Bump device hand to the device start if it is approaching the
9307 		 * end. l2arc_evict() has already evicted ahead for this case.
9308 		 */
9309 		dev->l2ad_hand = dev->l2ad_start;
9310 		dev->l2ad_evict = dev->l2ad_start;
9311 		dev->l2ad_first = B_FALSE;
9312 		goto top;
9313 	}
9314 
9315 	if (!all) {
9316 		/*
9317 		 * In case of cache device removal (all) the following
9318 		 * assertions may be violated without functional consequences
9319 		 * as the device is about to be removed.
9320 		 */
9321 		ASSERT3U(dev->l2ad_hand + distance, <=, dev->l2ad_end);
9322 		if (!dev->l2ad_first)
9323 			ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9324 	}
9325 }
9326 
9327 /*
9328  * Handle any abd transforms that might be required for writing to the L2ARC.
9329  * If successful, this function will always return an abd with the data
9330  * transformed as it is on disk in a new abd of asize bytes.
9331  */
9332 static int
l2arc_apply_transforms(spa_t * spa,arc_buf_hdr_t * hdr,uint64_t asize,abd_t ** abd_out)9333 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
9334     abd_t **abd_out)
9335 {
9336 	int ret;
9337 	abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
9338 	enum zio_compress compress = HDR_GET_COMPRESS(hdr);
9339 	uint64_t psize = HDR_GET_PSIZE(hdr);
9340 	uint64_t size = arc_hdr_size(hdr);
9341 	boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
9342 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
9343 	dsl_crypto_key_t *dck = NULL;
9344 	uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
9345 	boolean_t no_crypt = B_FALSE;
9346 
9347 	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
9348 	    !HDR_COMPRESSION_ENABLED(hdr)) ||
9349 	    HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);
9350 	ASSERT3U(psize, <=, asize);
9351 
9352 	/*
9353 	 * If this data simply needs its own buffer, we simply allocate it
9354 	 * and copy the data. This may be done to eliminate a dependency on a
9355 	 * shared buffer or to reallocate the buffer to match asize.
9356 	 */
9357 	if (HDR_HAS_RABD(hdr)) {
9358 		ASSERT3U(asize, >, psize);
9359 		to_write = abd_alloc_for_io(asize, ismd);
9360 		abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
9361 		abd_zero_off(to_write, psize, asize - psize);
9362 		goto out;
9363 	}
9364 
9365 	if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
9366 	    !HDR_ENCRYPTED(hdr)) {
9367 		ASSERT3U(size, ==, psize);
9368 		to_write = abd_alloc_for_io(asize, ismd);
9369 		abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
9370 		if (asize > size)
9371 			abd_zero_off(to_write, size, asize - size);
9372 		goto out;
9373 	}
9374 
9375 	if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
9376 		cabd = abd_alloc_for_io(MAX(size, asize), ismd);
9377 		uint64_t csize = zio_compress_data(compress, to_write, &cabd,
9378 		    size, MIN(size, psize), hdr->b_complevel);
9379 		if (csize >= size || csize > psize) {
9380 			/*
9381 			 * We can't re-compress the block into the original
9382 			 * psize.  Even if it fits into asize, it does not
9383 			 * matter, since checksum will never match on read.
9384 			 */
9385 			abd_free(cabd);
9386 			return (SET_ERROR(EIO));
9387 		}
9388 		if (asize > csize)
9389 			abd_zero_off(cabd, csize, asize - csize);
9390 		to_write = cabd;
9391 	}
9392 
9393 	if (HDR_ENCRYPTED(hdr)) {
9394 		eabd = abd_alloc_for_io(asize, ismd);
9395 
9396 		/*
9397 		 * If the dataset was disowned before the buffer
9398 		 * made it to this point, the key to re-encrypt
9399 		 * it won't be available. In this case we simply
9400 		 * won't write the buffer to the L2ARC.
9401 		 */
9402 		ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
9403 		    FTAG, &dck);
9404 		if (ret != 0)
9405 			goto error;
9406 
9407 		ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
9408 		    hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
9409 		    hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
9410 		    &no_crypt);
9411 		if (ret != 0)
9412 			goto error;
9413 
9414 		if (no_crypt)
9415 			abd_copy(eabd, to_write, psize);
9416 
9417 		if (psize != asize)
9418 			abd_zero_off(eabd, psize, asize - psize);
9419 
9420 		/* assert that the MAC we got here matches the one we saved */
9421 		ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
9422 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
9423 
9424 		if (to_write == cabd)
9425 			abd_free(cabd);
9426 
9427 		to_write = eabd;
9428 	}
9429 
9430 out:
9431 	ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
9432 	*abd_out = to_write;
9433 	return (0);
9434 
9435 error:
9436 	if (dck != NULL)
9437 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
9438 	if (cabd != NULL)
9439 		abd_free(cabd);
9440 	if (eabd != NULL)
9441 		abd_free(eabd);
9442 
9443 	*abd_out = NULL;
9444 	return (ret);
9445 }
9446 
9447 static void
l2arc_blk_fetch_done(zio_t * zio)9448 l2arc_blk_fetch_done(zio_t *zio)
9449 {
9450 	l2arc_read_callback_t *cb;
9451 
9452 	cb = zio->io_private;
9453 	if (cb->l2rcb_abd != NULL)
9454 		abd_free(cb->l2rcb_abd);
9455 	kmem_free(cb, sizeof (l2arc_read_callback_t));
9456 }
9457 
9458 /*
9459  * Find and write ARC buffers to the L2ARC device.
9460  *
9461  * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
9462  * for reading until they have completed writing.
9463  * The headroom_boost is an in-out parameter used to maintain headroom boost
9464  * state between calls to this function.
9465  *
9466  * Returns the number of bytes actually written (which may be smaller than
9467  * the delta by which the device hand has changed due to alignment and the
9468  * writing of log blocks).
9469  */
9470 static uint64_t
l2arc_write_buffers(spa_t * spa,l2arc_dev_t * dev,uint64_t target_sz)9471 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
9472 {
9473 	arc_buf_hdr_t 		*hdr, *head, *marker;
9474 	uint64_t 		write_asize, write_psize, headroom;
9475 	boolean_t		full, from_head = !arc_warm;
9476 	l2arc_write_callback_t	*cb = NULL;
9477 	zio_t 			*pio, *wzio;
9478 	uint64_t 		guid = spa_load_guid(spa);
9479 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
9480 
9481 	ASSERT3P(dev->l2ad_vdev, !=, NULL);
9482 
9483 	pio = NULL;
9484 	write_asize = write_psize = 0;
9485 	full = B_FALSE;
9486 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
9487 	arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
9488 	marker = arc_state_alloc_marker();
9489 
9490 	/*
9491 	 * Copy buffers for L2ARC writing.
9492 	 */
9493 	for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) {
9494 		/*
9495 		 * pass == 0: MFU meta
9496 		 * pass == 1: MRU meta
9497 		 * pass == 2: MFU data
9498 		 * pass == 3: MRU data
9499 		 */
9500 		if (l2arc_mfuonly == 1) {
9501 			if (pass == 1 || pass == 3)
9502 				continue;
9503 		} else if (l2arc_mfuonly > 1) {
9504 			if (pass == 3)
9505 				continue;
9506 		}
9507 
9508 		uint64_t passed_sz = 0;
9509 		headroom = target_sz * l2arc_headroom;
9510 		if (zfs_compressed_arc_enabled)
9511 			headroom = (headroom * l2arc_headroom_boost) / 100;
9512 
9513 		/*
9514 		 * Until the ARC is warm and starts to evict, read from the
9515 		 * head of the ARC lists rather than the tail.
9516 		 */
9517 		multilist_sublist_t *mls = l2arc_sublist_lock(pass);
9518 		ASSERT3P(mls, !=, NULL);
9519 		if (from_head)
9520 			hdr = multilist_sublist_head(mls);
9521 		else
9522 			hdr = multilist_sublist_tail(mls);
9523 
9524 		while (hdr != NULL) {
9525 			kmutex_t *hash_lock;
9526 			abd_t *to_write = NULL;
9527 
9528 			hash_lock = HDR_LOCK(hdr);
9529 			if (!mutex_tryenter(hash_lock)) {
9530 skip:
9531 				/* Skip this buffer rather than waiting. */
9532 				if (from_head)
9533 					hdr = multilist_sublist_next(mls, hdr);
9534 				else
9535 					hdr = multilist_sublist_prev(mls, hdr);
9536 				continue;
9537 			}
9538 
9539 			passed_sz += HDR_GET_LSIZE(hdr);
9540 			if (l2arc_headroom != 0 && passed_sz > headroom) {
9541 				/*
9542 				 * Searched too far.
9543 				 */
9544 				mutex_exit(hash_lock);
9545 				break;
9546 			}
9547 
9548 			if (!l2arc_write_eligible(guid, hdr)) {
9549 				mutex_exit(hash_lock);
9550 				goto skip;
9551 			}
9552 
9553 			ASSERT(HDR_HAS_L1HDR(hdr));
9554 			ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
9555 			ASSERT3U(arc_hdr_size(hdr), >, 0);
9556 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
9557 			    HDR_HAS_RABD(hdr));
9558 			uint64_t psize = HDR_GET_PSIZE(hdr);
9559 			uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
9560 			    psize);
9561 
9562 			/*
9563 			 * If the allocated size of this buffer plus the max
9564 			 * size for the pending log block exceeds the evicted
9565 			 * target size, terminate writing buffers for this run.
9566 			 */
9567 			if (write_asize + asize +
9568 			    sizeof (l2arc_log_blk_phys_t) > target_sz) {
9569 				full = B_TRUE;
9570 				mutex_exit(hash_lock);
9571 				break;
9572 			}
9573 
9574 			/*
9575 			 * We should not sleep with sublist lock held or it
9576 			 * may block ARC eviction.  Insert a marker to save
9577 			 * the position and drop the lock.
9578 			 */
9579 			if (from_head) {
9580 				multilist_sublist_insert_after(mls, hdr,
9581 				    marker);
9582 			} else {
9583 				multilist_sublist_insert_before(mls, hdr,
9584 				    marker);
9585 			}
9586 			multilist_sublist_unlock(mls);
9587 
9588 			/*
9589 			 * If this header has b_rabd, we can use this since it
9590 			 * must always match the data exactly as it exists on
9591 			 * disk. Otherwise, the L2ARC can normally use the
9592 			 * hdr's data, but if we're sharing data between the
9593 			 * hdr and one of its bufs, L2ARC needs its own copy of
9594 			 * the data so that the ZIO below can't race with the
9595 			 * buf consumer. To ensure that this copy will be
9596 			 * available for the lifetime of the ZIO and be cleaned
9597 			 * up afterwards, we add it to the l2arc_free_on_write
9598 			 * queue. If we need to apply any transforms to the
9599 			 * data (compression, encryption) we will also need the
9600 			 * extra buffer.
9601 			 */
9602 			if (HDR_HAS_RABD(hdr) && psize == asize) {
9603 				to_write = hdr->b_crypt_hdr.b_rabd;
9604 			} else if ((HDR_COMPRESSION_ENABLED(hdr) ||
9605 			    HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
9606 			    !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
9607 			    psize == asize) {
9608 				to_write = hdr->b_l1hdr.b_pabd;
9609 			} else {
9610 				int ret;
9611 				arc_buf_contents_t type = arc_buf_type(hdr);
9612 
9613 				ret = l2arc_apply_transforms(spa, hdr, asize,
9614 				    &to_write);
9615 				if (ret != 0) {
9616 					arc_hdr_clear_flags(hdr,
9617 					    ARC_FLAG_L2CACHE);
9618 					mutex_exit(hash_lock);
9619 					goto next;
9620 				}
9621 
9622 				l2arc_free_abd_on_write(to_write, asize, type);
9623 			}
9624 
9625 			hdr->b_l2hdr.b_dev = dev;
9626 			hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
9627 			hdr->b_l2hdr.b_hits = 0;
9628 			hdr->b_l2hdr.b_arcs_state =
9629 			    hdr->b_l1hdr.b_state->arcs_state;
9630 			/* l2arc_hdr_arcstats_update() expects a valid asize */
9631 			HDR_SET_L2SIZE(hdr, asize);
9632 			arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR |
9633 			    ARC_FLAG_L2_WRITING);
9634 
9635 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
9636 			    arc_hdr_size(hdr), hdr);
9637 			l2arc_hdr_arcstats_increment(hdr);
9638 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
9639 
9640 			mutex_enter(&dev->l2ad_mtx);
9641 			if (pio == NULL) {
9642 				/*
9643 				 * Insert a dummy header on the buflist so
9644 				 * l2arc_write_done() can find where the
9645 				 * write buffers begin without searching.
9646 				 */
9647 				list_insert_head(&dev->l2ad_buflist, head);
9648 			}
9649 			list_insert_head(&dev->l2ad_buflist, hdr);
9650 			mutex_exit(&dev->l2ad_mtx);
9651 
9652 			boolean_t commit = l2arc_log_blk_insert(dev, hdr);
9653 			mutex_exit(hash_lock);
9654 
9655 			if (pio == NULL) {
9656 				cb = kmem_alloc(
9657 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
9658 				cb->l2wcb_dev = dev;
9659 				cb->l2wcb_head = head;
9660 				list_create(&cb->l2wcb_abd_list,
9661 				    sizeof (l2arc_lb_abd_buf_t),
9662 				    offsetof(l2arc_lb_abd_buf_t, node));
9663 				pio = zio_root(spa, l2arc_write_done, cb,
9664 				    ZIO_FLAG_CANFAIL);
9665 			}
9666 
9667 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
9668 			    dev->l2ad_hand, asize, to_write,
9669 			    ZIO_CHECKSUM_OFF, NULL, hdr,
9670 			    ZIO_PRIORITY_ASYNC_WRITE,
9671 			    ZIO_FLAG_CANFAIL, B_FALSE);
9672 
9673 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
9674 			    zio_t *, wzio);
9675 			zio_nowait(wzio);
9676 
9677 			write_psize += psize;
9678 			write_asize += asize;
9679 			dev->l2ad_hand += asize;
9680 
9681 			if (commit) {
9682 				/* l2ad_hand will be adjusted inside. */
9683 				write_asize +=
9684 				    l2arc_log_blk_commit(dev, pio, cb);
9685 			}
9686 
9687 next:
9688 			multilist_sublist_lock(mls);
9689 			if (from_head)
9690 				hdr = multilist_sublist_next(mls, marker);
9691 			else
9692 				hdr = multilist_sublist_prev(mls, marker);
9693 			multilist_sublist_remove(mls, marker);
9694 		}
9695 
9696 		multilist_sublist_unlock(mls);
9697 
9698 		if (full == B_TRUE)
9699 			break;
9700 	}
9701 
9702 	arc_state_free_marker(marker);
9703 
9704 	/* No buffers selected for writing? */
9705 	if (pio == NULL) {
9706 		ASSERT0(write_psize);
9707 		ASSERT(!HDR_HAS_L1HDR(head));
9708 		kmem_cache_free(hdr_l2only_cache, head);
9709 
9710 		/*
9711 		 * Although we did not write any buffers l2ad_evict may
9712 		 * have advanced.
9713 		 */
9714 		if (dev->l2ad_evict != l2dhdr->dh_evict)
9715 			l2arc_dev_hdr_update(dev);
9716 
9717 		return (0);
9718 	}
9719 
9720 	if (!dev->l2ad_first)
9721 		ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9722 
9723 	ASSERT3U(write_asize, <=, target_sz);
9724 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
9725 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);
9726 
9727 	dev->l2ad_writing = B_TRUE;
9728 	(void) zio_wait(pio);
9729 	dev->l2ad_writing = B_FALSE;
9730 
9731 	/*
9732 	 * Update the device header after the zio completes as
9733 	 * l2arc_write_done() may have updated the memory holding the log block
9734 	 * pointers in the device header.
9735 	 */
9736 	l2arc_dev_hdr_update(dev);
9737 
9738 	return (write_asize);
9739 }
9740 
9741 static boolean_t
l2arc_hdr_limit_reached(void)9742 l2arc_hdr_limit_reached(void)
9743 {
9744 	int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);
9745 
9746 	return (arc_reclaim_needed() ||
9747 	    (s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
9748 }
9749 
9750 /*
9751  * This thread feeds the L2ARC at regular intervals.  This is the beating
9752  * heart of the L2ARC.
9753  */
9754 static  __attribute__((noreturn)) void
l2arc_feed_thread(void * unused)9755 l2arc_feed_thread(void *unused)
9756 {
9757 	(void) unused;
9758 	callb_cpr_t cpr;
9759 	l2arc_dev_t *dev;
9760 	spa_t *spa;
9761 	uint64_t size, wrote;
9762 	clock_t begin, next = ddi_get_lbolt();
9763 	fstrans_cookie_t cookie;
9764 
9765 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
9766 
9767 	mutex_enter(&l2arc_feed_thr_lock);
9768 
9769 	cookie = spl_fstrans_mark();
9770 	while (l2arc_thread_exit == 0) {
9771 		CALLB_CPR_SAFE_BEGIN(&cpr);
9772 		(void) cv_timedwait_idle(&l2arc_feed_thr_cv,
9773 		    &l2arc_feed_thr_lock, next);
9774 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
9775 		next = ddi_get_lbolt() + hz;
9776 
9777 		/*
9778 		 * Quick check for L2ARC devices.
9779 		 */
9780 		mutex_enter(&l2arc_dev_mtx);
9781 		if (l2arc_ndev == 0) {
9782 			mutex_exit(&l2arc_dev_mtx);
9783 			continue;
9784 		}
9785 		mutex_exit(&l2arc_dev_mtx);
9786 		begin = ddi_get_lbolt();
9787 
9788 		/*
9789 		 * This selects the next l2arc device to write to, and in
9790 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
9791 		 * will return NULL if there are now no l2arc devices or if
9792 		 * they are all faulted.
9793 		 *
9794 		 * If a device is returned, its spa's config lock is also
9795 		 * held to prevent device removal.  l2arc_dev_get_next()
9796 		 * will grab and release l2arc_dev_mtx.
9797 		 */
9798 		if ((dev = l2arc_dev_get_next()) == NULL)
9799 			continue;
9800 
9801 		spa = dev->l2ad_spa;
9802 		ASSERT3P(spa, !=, NULL);
9803 
9804 		/*
9805 		 * If the pool is read-only then force the feed thread to
9806 		 * sleep a little longer.
9807 		 */
9808 		if (!spa_writeable(spa)) {
9809 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
9810 			spa_config_exit(spa, SCL_L2ARC, dev);
9811 			continue;
9812 		}
9813 
9814 		/*
9815 		 * Avoid contributing to memory pressure.
9816 		 */
9817 		if (l2arc_hdr_limit_reached()) {
9818 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
9819 			spa_config_exit(spa, SCL_L2ARC, dev);
9820 			continue;
9821 		}
9822 
9823 		ARCSTAT_BUMP(arcstat_l2_feeds);
9824 
9825 		size = l2arc_write_size(dev);
9826 
9827 		/*
9828 		 * Evict L2ARC buffers that will be overwritten.
9829 		 */
9830 		l2arc_evict(dev, size, B_FALSE);
9831 
9832 		/*
9833 		 * Write ARC buffers.
9834 		 */
9835 		wrote = l2arc_write_buffers(spa, dev, size);
9836 
9837 		/*
9838 		 * Calculate interval between writes.
9839 		 */
9840 		next = l2arc_write_interval(begin, size, wrote);
9841 		spa_config_exit(spa, SCL_L2ARC, dev);
9842 	}
9843 	spl_fstrans_unmark(cookie);
9844 
9845 	l2arc_thread_exit = 0;
9846 	cv_broadcast(&l2arc_feed_thr_cv);
9847 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
9848 	thread_exit();
9849 }
9850 
9851 boolean_t
l2arc_vdev_present(vdev_t * vd)9852 l2arc_vdev_present(vdev_t *vd)
9853 {
9854 	return (l2arc_vdev_get(vd) != NULL);
9855 }
9856 
9857 /*
9858  * Returns the l2arc_dev_t associated with a particular vdev_t or NULL if
9859  * the vdev_t isn't an L2ARC device.
9860  */
9861 l2arc_dev_t *
l2arc_vdev_get(vdev_t * vd)9862 l2arc_vdev_get(vdev_t *vd)
9863 {
9864 	l2arc_dev_t	*dev;
9865 
9866 	mutex_enter(&l2arc_dev_mtx);
9867 	for (dev = list_head(l2arc_dev_list); dev != NULL;
9868 	    dev = list_next(l2arc_dev_list, dev)) {
9869 		if (dev->l2ad_vdev == vd)
9870 			break;
9871 	}
9872 	mutex_exit(&l2arc_dev_mtx);
9873 
9874 	return (dev);
9875 }
9876 
9877 static void
l2arc_rebuild_dev(l2arc_dev_t * dev,boolean_t reopen)9878 l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
9879 {
9880 	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9881 	uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
9882 	spa_t *spa = dev->l2ad_spa;
9883 
9884 	/*
9885 	 * After a l2arc_remove_vdev(), the spa_t will no longer be valid
9886 	 */
9887 	if (spa == NULL)
9888 		return;
9889 
9890 	/*
9891 	 * The L2ARC has to hold at least the payload of one log block for
9892 	 * them to be restored (persistent L2ARC). The payload of a log block
9893 	 * depends on the amount of its log entries. We always write log blocks
9894 	 * with 1022 entries. How many of them are committed or restored depends
9895 	 * on the size of the L2ARC device. Thus the maximum payload of
9896 	 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
9897 	 * is less than that, we reduce the amount of committed and restored
9898 	 * log entries per block so as to enable persistence.
9899 	 */
9900 	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
9901 		dev->l2ad_log_entries = 0;
9902 	} else {
9903 		dev->l2ad_log_entries = MIN((dev->l2ad_end -
9904 		    dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
9905 		    L2ARC_LOG_BLK_MAX_ENTRIES);
9906 	}
9907 
9908 	/*
9909 	 * Read the device header, if an error is returned do not rebuild L2ARC.
9910 	 */
9911 	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
9912 		/*
9913 		 * If we are onlining a cache device (vdev_reopen) that was
9914 		 * still present (l2arc_vdev_present()) and rebuild is enabled,
9915 		 * we should evict all ARC buffers and pointers to log blocks
9916 		 * and reclaim their space before restoring its contents to
9917 		 * L2ARC.
9918 		 */
9919 		if (reopen) {
9920 			if (!l2arc_rebuild_enabled) {
9921 				return;
9922 			} else {
9923 				l2arc_evict(dev, 0, B_TRUE);
9924 				/* start a new log block */
9925 				dev->l2ad_log_ent_idx = 0;
9926 				dev->l2ad_log_blk_payload_asize = 0;
9927 				dev->l2ad_log_blk_payload_start = 0;
9928 			}
9929 		}
9930 		/*
9931 		 * Just mark the device as pending for a rebuild. We won't
9932 		 * be starting a rebuild in line here as it would block pool
9933 		 * import. Instead spa_load_impl will hand that off to an
9934 		 * async task which will call l2arc_spa_rebuild_start.
9935 		 */
9936 		dev->l2ad_rebuild = B_TRUE;
9937 	} else if (spa_writeable(spa)) {
9938 		/*
9939 		 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
9940 		 * otherwise create a new header. We zero out the memory holding
9941 		 * the header to reset dh_start_lbps. If we TRIM the whole
9942 		 * device the new header will be written by
9943 		 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
9944 		 * trim_state in the header too. When reading the header, if
9945 		 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
9946 		 * we opt to TRIM the whole device again.
9947 		 */
9948 		if (l2arc_trim_ahead > 0) {
9949 			dev->l2ad_trim_all = B_TRUE;
9950 		} else {
9951 			memset(l2dhdr, 0, l2dhdr_asize);
9952 			l2arc_dev_hdr_update(dev);
9953 		}
9954 	}
9955 }
9956 
9957 /*
9958  * Add a vdev for use by the L2ARC.  By this point the spa has already
9959  * validated the vdev and opened it.
9960  */
9961 void
l2arc_add_vdev(spa_t * spa,vdev_t * vd)9962 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
9963 {
9964 	l2arc_dev_t		*adddev;
9965 	uint64_t		l2dhdr_asize;
9966 
9967 	ASSERT(!l2arc_vdev_present(vd));
9968 
9969 	/*
9970 	 * Create a new l2arc device entry.
9971 	 */
9972 	adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
9973 	adddev->l2ad_spa = spa;
9974 	adddev->l2ad_vdev = vd;
9975 	/* leave extra size for an l2arc device header */
9976 	l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
9977 	    MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
9978 	adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
9979 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
9980 	ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
9981 	adddev->l2ad_hand = adddev->l2ad_start;
9982 	adddev->l2ad_evict = adddev->l2ad_start;
9983 	adddev->l2ad_first = B_TRUE;
9984 	adddev->l2ad_writing = B_FALSE;
9985 	adddev->l2ad_trim_all = B_FALSE;
9986 	list_link_init(&adddev->l2ad_node);
9987 	adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);
9988 
9989 	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
9990 	/*
9991 	 * This is a list of all ARC buffers that are still valid on the
9992 	 * device.
9993 	 */
9994 	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
9995 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
9996 
9997 	/*
9998 	 * This is a list of pointers to log blocks that are still present
9999 	 * on the device.
10000 	 */
10001 	list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
10002 	    offsetof(l2arc_lb_ptr_buf_t, node));
10003 
10004 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
10005 	zfs_refcount_create(&adddev->l2ad_alloc);
10006 	zfs_refcount_create(&adddev->l2ad_lb_asize);
10007 	zfs_refcount_create(&adddev->l2ad_lb_count);
10008 
10009 	/*
10010 	 * Decide if dev is eligible for L2ARC rebuild or whole device
10011 	 * trimming. This has to happen before the device is added in the
10012 	 * cache device list and l2arc_dev_mtx is released. Otherwise
10013 	 * l2arc_feed_thread() might already start writing on the
10014 	 * device.
10015 	 */
10016 	l2arc_rebuild_dev(adddev, B_FALSE);
10017 
10018 	/*
10019 	 * Add device to global list
10020 	 */
10021 	mutex_enter(&l2arc_dev_mtx);
10022 	list_insert_head(l2arc_dev_list, adddev);
10023 	atomic_inc_64(&l2arc_ndev);
10024 	mutex_exit(&l2arc_dev_mtx);
10025 }
10026 
10027 /*
10028  * Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
10029  * in case of onlining a cache device.
10030  */
10031 void
l2arc_rebuild_vdev(vdev_t * vd,boolean_t reopen)10032 l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
10033 {
10034 	l2arc_dev_t		*dev = NULL;
10035 
10036 	dev = l2arc_vdev_get(vd);
10037 	ASSERT3P(dev, !=, NULL);
10038 
10039 	/*
10040 	 * In contrast to l2arc_add_vdev() we do not have to worry about
10041 	 * l2arc_feed_thread() invalidating previous content when onlining a
10042 	 * cache device. The device parameters (l2ad*) are not cleared when
10043 	 * offlining the device and writing new buffers will not invalidate
10044 	 * all previous content. In worst case only buffers that have not had
10045 	 * their log block written to the device will be lost.
10046 	 * When onlining the cache device (ie offline->online without exporting
10047 	 * the pool in between) this happens:
10048 	 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
10049 	 * 			|			|
10050 	 * 		vdev_is_dead() = B_FALSE	l2ad_rebuild = B_TRUE
10051 	 * During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
10052 	 * is set to B_TRUE we might write additional buffers to the device.
10053 	 */
10054 	l2arc_rebuild_dev(dev, reopen);
10055 }
10056 
10057 typedef struct {
10058 	l2arc_dev_t	*rva_l2arc_dev;
10059 	uint64_t	rva_spa_gid;
10060 	uint64_t	rva_vdev_gid;
10061 	boolean_t	rva_async;
10062 
10063 } remove_vdev_args_t;
10064 
10065 static void
l2arc_device_teardown(void * arg)10066 l2arc_device_teardown(void *arg)
10067 {
10068 	remove_vdev_args_t *rva = arg;
10069 	l2arc_dev_t *remdev = rva->rva_l2arc_dev;
10070 	hrtime_t start_time = gethrtime();
10071 
10072 	/*
10073 	 * Clear all buflists and ARC references.  L2ARC device flush.
10074 	 */
10075 	l2arc_evict(remdev, 0, B_TRUE);
10076 	list_destroy(&remdev->l2ad_buflist);
10077 	ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
10078 	list_destroy(&remdev->l2ad_lbptr_list);
10079 	mutex_destroy(&remdev->l2ad_mtx);
10080 	zfs_refcount_destroy(&remdev->l2ad_alloc);
10081 	zfs_refcount_destroy(&remdev->l2ad_lb_asize);
10082 	zfs_refcount_destroy(&remdev->l2ad_lb_count);
10083 	kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
10084 	vmem_free(remdev, sizeof (l2arc_dev_t));
10085 
10086 	uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
10087 	if (elaspsed > 0) {
10088 		zfs_dbgmsg("spa %llu, vdev %llu removed in %llu ms",
10089 		    (u_longlong_t)rva->rva_spa_gid,
10090 		    (u_longlong_t)rva->rva_vdev_gid,
10091 		    (u_longlong_t)elaspsed);
10092 	}
10093 
10094 	if (rva->rva_async)
10095 		arc_async_flush_remove(rva->rva_spa_gid, 2);
10096 	kmem_free(rva, sizeof (remove_vdev_args_t));
10097 }
10098 
10099 /*
10100  * Remove a vdev from the L2ARC.
10101  */
10102 void
l2arc_remove_vdev(vdev_t * vd)10103 l2arc_remove_vdev(vdev_t *vd)
10104 {
10105 	spa_t *spa = vd->vdev_spa;
10106 	boolean_t asynchronous = spa->spa_state == POOL_STATE_EXPORTED ||
10107 	    spa->spa_state == POOL_STATE_DESTROYED;
10108 
10109 	/*
10110 	 * Find the device by vdev
10111 	 */
10112 	l2arc_dev_t *remdev = l2arc_vdev_get(vd);
10113 	ASSERT3P(remdev, !=, NULL);
10114 
10115 	/*
10116 	 * Save info for final teardown
10117 	 */
10118 	remove_vdev_args_t *rva = kmem_alloc(sizeof (remove_vdev_args_t),
10119 	    KM_SLEEP);
10120 	rva->rva_l2arc_dev = remdev;
10121 	rva->rva_spa_gid = spa_load_guid(spa);
10122 	rva->rva_vdev_gid = remdev->l2ad_vdev->vdev_guid;
10123 
10124 	/*
10125 	 * Cancel any ongoing or scheduled rebuild.
10126 	 */
10127 	mutex_enter(&l2arc_rebuild_thr_lock);
10128 	remdev->l2ad_rebuild_cancel = B_TRUE;
10129 	if (remdev->l2ad_rebuild_began == B_TRUE) {
10130 		while (remdev->l2ad_rebuild == B_TRUE)
10131 			cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
10132 	}
10133 	mutex_exit(&l2arc_rebuild_thr_lock);
10134 	rva->rva_async = asynchronous;
10135 
10136 	/*
10137 	 * Remove device from global list
10138 	 */
10139 	ASSERT(spa_config_held(spa, SCL_L2ARC, RW_WRITER) & SCL_L2ARC);
10140 	mutex_enter(&l2arc_dev_mtx);
10141 	list_remove(l2arc_dev_list, remdev);
10142 	l2arc_dev_last = NULL;		/* may have been invalidated */
10143 	atomic_dec_64(&l2arc_ndev);
10144 
10145 	/* During a pool export spa & vdev will no longer be valid */
10146 	if (asynchronous) {
10147 		remdev->l2ad_spa = NULL;
10148 		remdev->l2ad_vdev = NULL;
10149 	}
10150 	mutex_exit(&l2arc_dev_mtx);
10151 
10152 	if (!asynchronous) {
10153 		l2arc_device_teardown(rva);
10154 		return;
10155 	}
10156 
10157 	arc_async_flush_t *af = arc_async_flush_add(rva->rva_spa_gid, 2);
10158 
10159 	taskq_dispatch_ent(arc_flush_taskq, l2arc_device_teardown, rva,
10160 	    TQ_SLEEP, &af->af_tqent);
10161 }
10162 
10163 void
l2arc_init(void)10164 l2arc_init(void)
10165 {
10166 	l2arc_thread_exit = 0;
10167 	l2arc_ndev = 0;
10168 
10169 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
10170 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
10171 	mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL);
10172 	cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL);
10173 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
10174 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
10175 
10176 	l2arc_dev_list = &L2ARC_dev_list;
10177 	l2arc_free_on_write = &L2ARC_free_on_write;
10178 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
10179 	    offsetof(l2arc_dev_t, l2ad_node));
10180 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
10181 	    offsetof(l2arc_data_free_t, l2df_list_node));
10182 }
10183 
10184 void
l2arc_fini(void)10185 l2arc_fini(void)
10186 {
10187 	mutex_destroy(&l2arc_feed_thr_lock);
10188 	cv_destroy(&l2arc_feed_thr_cv);
10189 	mutex_destroy(&l2arc_rebuild_thr_lock);
10190 	cv_destroy(&l2arc_rebuild_thr_cv);
10191 	mutex_destroy(&l2arc_dev_mtx);
10192 	mutex_destroy(&l2arc_free_on_write_mtx);
10193 
10194 	list_destroy(l2arc_dev_list);
10195 	list_destroy(l2arc_free_on_write);
10196 }
10197 
10198 void
l2arc_start(void)10199 l2arc_start(void)
10200 {
10201 	if (!(spa_mode_global & SPA_MODE_WRITE))
10202 		return;
10203 
10204 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
10205 	    TS_RUN, defclsyspri);
10206 }
10207 
10208 void
l2arc_stop(void)10209 l2arc_stop(void)
10210 {
10211 	if (!(spa_mode_global & SPA_MODE_WRITE))
10212 		return;
10213 
10214 	mutex_enter(&l2arc_feed_thr_lock);
10215 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
10216 	l2arc_thread_exit = 1;
10217 	while (l2arc_thread_exit != 0)
10218 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
10219 	mutex_exit(&l2arc_feed_thr_lock);
10220 }
10221 
10222 /*
10223  * Punches out rebuild threads for the L2ARC devices in a spa. This should
10224  * be called after pool import from the spa async thread, since starting
10225  * these threads directly from spa_import() will make them part of the
10226  * "zpool import" context and delay process exit (and thus pool import).
10227  */
10228 void
l2arc_spa_rebuild_start(spa_t * spa)10229 l2arc_spa_rebuild_start(spa_t *spa)
10230 {
10231 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
10232 
10233 	/*
10234 	 * Locate the spa's l2arc devices and kick off rebuild threads.
10235 	 */
10236 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10237 		l2arc_dev_t *dev =
10238 		    l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10239 		if (dev == NULL) {
10240 			/* Don't attempt a rebuild if the vdev is UNAVAIL */
10241 			continue;
10242 		}
10243 		mutex_enter(&l2arc_rebuild_thr_lock);
10244 		if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
10245 			dev->l2ad_rebuild_began = B_TRUE;
10246 			(void) thread_create(NULL, 0, l2arc_dev_rebuild_thread,
10247 			    dev, 0, &p0, TS_RUN, minclsyspri);
10248 		}
10249 		mutex_exit(&l2arc_rebuild_thr_lock);
10250 	}
10251 }
10252 
10253 void
l2arc_spa_rebuild_stop(spa_t * spa)10254 l2arc_spa_rebuild_stop(spa_t *spa)
10255 {
10256 	ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
10257 	    spa->spa_export_thread == curthread);
10258 
10259 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10260 		l2arc_dev_t *dev =
10261 		    l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10262 		if (dev == NULL)
10263 			continue;
10264 		mutex_enter(&l2arc_rebuild_thr_lock);
10265 		dev->l2ad_rebuild_cancel = B_TRUE;
10266 		mutex_exit(&l2arc_rebuild_thr_lock);
10267 	}
10268 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10269 		l2arc_dev_t *dev =
10270 		    l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10271 		if (dev == NULL)
10272 			continue;
10273 		mutex_enter(&l2arc_rebuild_thr_lock);
10274 		if (dev->l2ad_rebuild_began == B_TRUE) {
10275 			while (dev->l2ad_rebuild == B_TRUE) {
10276 				cv_wait(&l2arc_rebuild_thr_cv,
10277 				    &l2arc_rebuild_thr_lock);
10278 			}
10279 		}
10280 		mutex_exit(&l2arc_rebuild_thr_lock);
10281 	}
10282 }
10283 
10284 /*
10285  * Main entry point for L2ARC rebuilding.
10286  */
10287 static __attribute__((noreturn)) void
l2arc_dev_rebuild_thread(void * arg)10288 l2arc_dev_rebuild_thread(void *arg)
10289 {
10290 	l2arc_dev_t *dev = arg;
10291 
10292 	VERIFY(dev->l2ad_rebuild);
10293 	(void) l2arc_rebuild(dev);
10294 	mutex_enter(&l2arc_rebuild_thr_lock);
10295 	dev->l2ad_rebuild_began = B_FALSE;
10296 	dev->l2ad_rebuild = B_FALSE;
10297 	cv_signal(&l2arc_rebuild_thr_cv);
10298 	mutex_exit(&l2arc_rebuild_thr_lock);
10299 
10300 	thread_exit();
10301 }
10302 
10303 /*
10304  * This function implements the actual L2ARC metadata rebuild. It:
10305  * starts reading the log block chain and restores each block's contents
10306  * to memory (reconstructing arc_buf_hdr_t's).
10307  *
10308  * Operation stops under any of the following conditions:
10309  *
10310  * 1) We reach the end of the log block chain.
10311  * 2) We encounter *any* error condition (cksum errors, io errors)
10312  */
10313 static int
l2arc_rebuild(l2arc_dev_t * dev)10314 l2arc_rebuild(l2arc_dev_t *dev)
10315 {
10316 	vdev_t			*vd = dev->l2ad_vdev;
10317 	spa_t			*spa = vd->vdev_spa;
10318 	int			err = 0;
10319 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10320 	l2arc_log_blk_phys_t	*this_lb, *next_lb;
10321 	zio_t			*this_io = NULL, *next_io = NULL;
10322 	l2arc_log_blkptr_t	lbps[2];
10323 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
10324 	boolean_t		lock_held;
10325 
10326 	this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP);
10327 	next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP);
10328 
10329 	/*
10330 	 * We prevent device removal while issuing reads to the device,
10331 	 * then during the rebuilding phases we drop this lock again so
10332 	 * that a spa_unload or device remove can be initiated - this is
10333 	 * safe, because the spa will signal us to stop before removing
10334 	 * our device and wait for us to stop.
10335 	 */
10336 	spa_config_enter(spa, SCL_L2ARC, vd, RW_READER);
10337 	lock_held = B_TRUE;
10338 
10339 	/*
10340 	 * Retrieve the persistent L2ARC device state.
10341 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10342 	 */
10343 	dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
10344 	dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
10345 	    L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
10346 	    dev->l2ad_start);
10347 	dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);
10348 
10349 	vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
10350 	vd->vdev_trim_state = l2dhdr->dh_trim_state;
10351 
10352 	/*
10353 	 * In case the zfs module parameter l2arc_rebuild_enabled is false
10354 	 * we do not start the rebuild process.
10355 	 */
10356 	if (!l2arc_rebuild_enabled)
10357 		goto out;
10358 
10359 	/* Prepare the rebuild process */
10360 	memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));
10361 
10362 	/* Start the rebuild process */
10363 	for (;;) {
10364 		if (!l2arc_log_blkptr_valid(dev, &lbps[0]))
10365 			break;
10366 
10367 		if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1],
10368 		    this_lb, next_lb, this_io, &next_io)) != 0)
10369 			goto out;
10370 
10371 		/*
10372 		 * Our memory pressure valve. If the system is running low
10373 		 * on memory, rather than swamping memory with new ARC buf
10374 		 * hdrs, we opt not to rebuild the L2ARC. At this point,
10375 		 * however, we have already set up our L2ARC dev to chain in
10376 		 * new metadata log blocks, so the user may choose to offline/
10377 		 * online the L2ARC dev at a later time (or re-import the pool)
10378 		 * to reconstruct it (when there's less memory pressure).
10379 		 */
10380 		if (l2arc_hdr_limit_reached()) {
10381 			ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem);
10382 			cmn_err(CE_NOTE, "System running low on memory, "
10383 			    "aborting L2ARC rebuild.");
10384 			err = SET_ERROR(ENOMEM);
10385 			goto out;
10386 		}
10387 
10388 		spa_config_exit(spa, SCL_L2ARC, vd);
10389 		lock_held = B_FALSE;
10390 
10391 		/*
10392 		 * Now that we know that the next_lb checks out alright, we
10393 		 * can start reconstruction from this log block.
10394 		 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10395 		 */
10396 		uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
10397 		l2arc_log_blk_restore(dev, this_lb, asize);
10398 
10399 		/*
10400 		 * log block restored, include its pointer in the list of
10401 		 * pointers to log blocks present in the L2ARC device.
10402 		 */
10403 		lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
10404 		lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
10405 		    KM_SLEEP);
10406 		memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
10407 		    sizeof (l2arc_log_blkptr_t));
10408 		mutex_enter(&dev->l2ad_mtx);
10409 		list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
10410 		ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
10411 		ARCSTAT_BUMP(arcstat_l2_log_blk_count);
10412 		zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10413 		zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10414 		mutex_exit(&dev->l2ad_mtx);
10415 		vdev_space_update(vd, asize, 0, 0);
10416 
10417 		/*
10418 		 * Protection against loops of log blocks:
10419 		 *
10420 		 *				       l2ad_hand  l2ad_evict
10421 		 *                                         V	      V
10422 		 * l2ad_start |=======================================| l2ad_end
10423 		 *             -----|||----|||---|||----|||
10424 		 *                  (3)    (2)   (1)    (0)
10425 		 *             ---|||---|||----|||---|||
10426 		 *		  (7)   (6)    (5)   (4)
10427 		 *
10428 		 * In this situation the pointer of log block (4) passes
10429 		 * l2arc_log_blkptr_valid() but the log block should not be
10430 		 * restored as it is overwritten by the payload of log block
10431 		 * (0). Only log blocks (0)-(3) should be restored. We check
10432 		 * whether l2ad_evict lies in between the payload starting
10433 		 * offset of the next log block (lbps[1].lbp_payload_start)
10434 		 * and the payload starting offset of the present log block
10435 		 * (lbps[0].lbp_payload_start). If true and this isn't the
10436 		 * first pass, we are looping from the beginning and we should
10437 		 * stop.
10438 		 */
10439 		if (l2arc_range_check_overlap(lbps[1].lbp_payload_start,
10440 		    lbps[0].lbp_payload_start, dev->l2ad_evict) &&
10441 		    !dev->l2ad_first)
10442 			goto out;
10443 
10444 		kpreempt(KPREEMPT_SYNC);
10445 		for (;;) {
10446 			mutex_enter(&l2arc_rebuild_thr_lock);
10447 			if (dev->l2ad_rebuild_cancel) {
10448 				mutex_exit(&l2arc_rebuild_thr_lock);
10449 				err = SET_ERROR(ECANCELED);
10450 				goto out;
10451 			}
10452 			mutex_exit(&l2arc_rebuild_thr_lock);
10453 			if (spa_config_tryenter(spa, SCL_L2ARC, vd,
10454 			    RW_READER)) {
10455 				lock_held = B_TRUE;
10456 				break;
10457 			}
10458 			/*
10459 			 * L2ARC config lock held by somebody in writer,
10460 			 * possibly due to them trying to remove us. They'll
10461 			 * likely to want us to shut down, so after a little
10462 			 * delay, we check l2ad_rebuild_cancel and retry
10463 			 * the lock again.
10464 			 */
10465 			delay(1);
10466 		}
10467 
10468 		/*
10469 		 * Continue with the next log block.
10470 		 */
10471 		lbps[0] = lbps[1];
10472 		lbps[1] = this_lb->lb_prev_lbp;
10473 		PTR_SWAP(this_lb, next_lb);
10474 		this_io = next_io;
10475 		next_io = NULL;
10476 	}
10477 
10478 	if (this_io != NULL)
10479 		l2arc_log_blk_fetch_abort(this_io);
10480 out:
10481 	if (next_io != NULL)
10482 		l2arc_log_blk_fetch_abort(next_io);
10483 	vmem_free(this_lb, sizeof (*this_lb));
10484 	vmem_free(next_lb, sizeof (*next_lb));
10485 
10486 	if (err == ECANCELED) {
10487 		/*
10488 		 * In case the rebuild was canceled do not log to spa history
10489 		 * log as the pool may be in the process of being removed.
10490 		 */
10491 		zfs_dbgmsg("L2ARC rebuild aborted, restored %llu blocks",
10492 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10493 		return (err);
10494 	} else if (!l2arc_rebuild_enabled) {
10495 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10496 		    "disabled");
10497 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
10498 		ARCSTAT_BUMP(arcstat_l2_rebuild_success);
10499 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10500 		    "successful, restored %llu blocks",
10501 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10502 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
10503 		/*
10504 		 * No error but also nothing restored, meaning the lbps array
10505 		 * in the device header points to invalid/non-present log
10506 		 * blocks. Reset the header.
10507 		 */
10508 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10509 		    "no valid log blocks");
10510 		memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
10511 		l2arc_dev_hdr_update(dev);
10512 	} else if (err != 0) {
10513 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10514 		    "aborted, restored %llu blocks",
10515 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10516 	}
10517 
10518 	if (lock_held)
10519 		spa_config_exit(spa, SCL_L2ARC, vd);
10520 
10521 	return (err);
10522 }
10523 
10524 /*
10525  * Attempts to read the device header on the provided L2ARC device and writes
10526  * it to `hdr'. On success, this function returns 0, otherwise the appropriate
10527  * error code is returned.
10528  */
10529 static int
l2arc_dev_hdr_read(l2arc_dev_t * dev)10530 l2arc_dev_hdr_read(l2arc_dev_t *dev)
10531 {
10532 	int			err;
10533 	uint64_t		guid;
10534 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10535 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10536 	abd_t 			*abd;
10537 
10538 	guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10539 
10540 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
10541 
10542 	err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
10543 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd,
10544 	    ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
10545 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY |
10546 	    ZIO_FLAG_SPECULATIVE, B_FALSE));
10547 
10548 	abd_free(abd);
10549 
10550 	if (err != 0) {
10551 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors);
10552 		zfs_dbgmsg("L2ARC IO error (%d) while reading device header, "
10553 		    "vdev guid: %llu", err,
10554 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10555 		return (err);
10556 	}
10557 
10558 	if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
10559 		byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr));
10560 
10561 	if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC ||
10562 	    l2dhdr->dh_spa_guid != guid ||
10563 	    l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid ||
10564 	    l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION ||
10565 	    l2dhdr->dh_log_entries != dev->l2ad_log_entries ||
10566 	    l2dhdr->dh_end != dev->l2ad_end ||
10567 	    !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
10568 	    l2dhdr->dh_evict) ||
10569 	    (l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
10570 	    l2arc_trim_ahead > 0)) {
10571 		/*
10572 		 * Attempt to rebuild a device containing no actual dev hdr
10573 		 * or containing a header from some other pool or from another
10574 		 * version of persistent L2ARC.
10575 		 */
10576 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported);
10577 		return (SET_ERROR(ENOTSUP));
10578 	}
10579 
10580 	return (0);
10581 }
10582 
10583 /*
10584  * Reads L2ARC log blocks from storage and validates their contents.
10585  *
10586  * This function implements a simple fetcher to make sure that while
10587  * we're processing one buffer the L2ARC is already fetching the next
10588  * one in the chain.
10589  *
10590  * The arguments this_lp and next_lp point to the current and next log block
10591  * address in the block chain. Similarly, this_lb and next_lb hold the
10592  * l2arc_log_blk_phys_t's of the current and next L2ARC blk.
10593  *
10594  * The `this_io' and `next_io' arguments are used for block fetching.
10595  * When issuing the first blk IO during rebuild, you should pass NULL for
10596  * `this_io'. This function will then issue a sync IO to read the block and
10597  * also issue an async IO to fetch the next block in the block chain. The
10598  * fetched IO is returned in `next_io'. On subsequent calls to this
10599  * function, pass the value returned in `next_io' from the previous call
10600  * as `this_io' and a fresh `next_io' pointer to hold the next fetch IO.
10601  * Prior to the call, you should initialize your `next_io' pointer to be
10602  * NULL. If no fetch IO was issued, the pointer is left set at NULL.
10603  *
10604  * On success, this function returns 0, otherwise it returns an appropriate
10605  * error code. On error the fetching IO is aborted and cleared before
10606  * returning from this function. Therefore, if we return `success', the
10607  * caller can assume that we have taken care of cleanup of fetch IOs.
10608  */
10609 static int
l2arc_log_blk_read(l2arc_dev_t * dev,const l2arc_log_blkptr_t * this_lbp,const l2arc_log_blkptr_t * next_lbp,l2arc_log_blk_phys_t * this_lb,l2arc_log_blk_phys_t * next_lb,zio_t * this_io,zio_t ** next_io)10610 l2arc_log_blk_read(l2arc_dev_t *dev,
10611     const l2arc_log_blkptr_t *this_lbp, const l2arc_log_blkptr_t *next_lbp,
10612     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
10613     zio_t *this_io, zio_t **next_io)
10614 {
10615 	int		err = 0;
10616 	zio_cksum_t	cksum;
10617 	uint64_t	asize;
10618 
10619 	ASSERT(this_lbp != NULL && next_lbp != NULL);
10620 	ASSERT(this_lb != NULL && next_lb != NULL);
10621 	ASSERT(next_io != NULL && *next_io == NULL);
10622 	ASSERT(l2arc_log_blkptr_valid(dev, this_lbp));
10623 
10624 	/*
10625 	 * Check to see if we have issued the IO for this log block in a
10626 	 * previous run. If not, this is the first call, so issue it now.
10627 	 */
10628 	if (this_io == NULL) {
10629 		this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
10630 		    this_lb);
10631 	}
10632 
10633 	/*
10634 	 * Peek to see if we can start issuing the next IO immediately.
10635 	 */
10636 	if (l2arc_log_blkptr_valid(dev, next_lbp)) {
10637 		/*
10638 		 * Start issuing IO for the next log block early - this
10639 		 * should help keep the L2ARC device busy while we
10640 		 * decompress and restore this log block.
10641 		 */
10642 		*next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
10643 		    next_lb);
10644 	}
10645 
10646 	/* Wait for the IO to read this log block to complete */
10647 	if ((err = zio_wait(this_io)) != 0) {
10648 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors);
10649 		zfs_dbgmsg("L2ARC IO error (%d) while reading log block, "
10650 		    "offset: %llu, vdev guid: %llu", err,
10651 		    (u_longlong_t)this_lbp->lbp_daddr,
10652 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10653 		goto cleanup;
10654 	}
10655 
10656 	/*
10657 	 * Make sure the buffer checks out.
10658 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10659 	 */
10660 	asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
10661 	fletcher_4_native(this_lb, asize, NULL, &cksum);
10662 	if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
10663 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors);
10664 		zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, "
10665 		    "vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu",
10666 		    (u_longlong_t)this_lbp->lbp_daddr,
10667 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid,
10668 		    (u_longlong_t)dev->l2ad_hand,
10669 		    (u_longlong_t)dev->l2ad_evict);
10670 		err = SET_ERROR(ECKSUM);
10671 		goto cleanup;
10672 	}
10673 
10674 	/* Now we can take our time decoding this buffer */
10675 	switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
10676 	case ZIO_COMPRESS_OFF:
10677 		break;
10678 	case ZIO_COMPRESS_LZ4: {
10679 		abd_t *abd = abd_alloc_linear(asize, B_TRUE);
10680 		abd_copy_from_buf_off(abd, this_lb, 0, asize);
10681 		abd_t dabd;
10682 		abd_get_from_buf_struct(&dabd, this_lb, sizeof (*this_lb));
10683 		err = zio_decompress_data(
10684 		    L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
10685 		    abd, &dabd, asize, sizeof (*this_lb), NULL);
10686 		abd_free(&dabd);
10687 		abd_free(abd);
10688 		if (err != 0) {
10689 			err = SET_ERROR(EINVAL);
10690 			goto cleanup;
10691 		}
10692 		break;
10693 	}
10694 	default:
10695 		err = SET_ERROR(EINVAL);
10696 		goto cleanup;
10697 	}
10698 	if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
10699 		byteswap_uint64_array(this_lb, sizeof (*this_lb));
10700 	if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
10701 		err = SET_ERROR(EINVAL);
10702 		goto cleanup;
10703 	}
10704 cleanup:
10705 	/* Abort an in-flight fetch I/O in case of error */
10706 	if (err != 0 && *next_io != NULL) {
10707 		l2arc_log_blk_fetch_abort(*next_io);
10708 		*next_io = NULL;
10709 	}
10710 	return (err);
10711 }
10712 
10713 /*
10714  * Restores the payload of a log block to ARC. This creates empty ARC hdr
10715  * entries which only contain an l2arc hdr, essentially restoring the
10716  * buffers to their L2ARC evicted state. This function also updates space
10717  * usage on the L2ARC vdev to make sure it tracks restored buffers.
10718  */
10719 static void
l2arc_log_blk_restore(l2arc_dev_t * dev,const l2arc_log_blk_phys_t * lb,uint64_t lb_asize)10720 l2arc_log_blk_restore(l2arc_dev_t *dev, const l2arc_log_blk_phys_t *lb,
10721     uint64_t lb_asize)
10722 {
10723 	uint64_t	size = 0, asize = 0;
10724 	uint64_t	log_entries = dev->l2ad_log_entries;
10725 
10726 	/*
10727 	 * Usually arc_adapt() is called only for data, not headers, but
10728 	 * since we may allocate significant amount of memory here, let ARC
10729 	 * grow its arc_c.
10730 	 */
10731 	arc_adapt(log_entries * HDR_L2ONLY_SIZE);
10732 
10733 	for (int i = log_entries - 1; i >= 0; i--) {
10734 		/*
10735 		 * Restore goes in the reverse temporal direction to preserve
10736 		 * correct temporal ordering of buffers in the l2ad_buflist.
10737 		 * l2arc_hdr_restore also does a list_insert_tail instead of
10738 		 * list_insert_head on the l2ad_buflist:
10739 		 *
10740 		 *		LIST	l2ad_buflist		LIST
10741 		 *		HEAD  <------ (time) ------	TAIL
10742 		 * direction	+-----+-----+-----+-----+-----+    direction
10743 		 * of l2arc <== | buf | buf | buf | buf | buf | ===> of rebuild
10744 		 * fill		+-----+-----+-----+-----+-----+
10745 		 *		^				^
10746 		 *		|				|
10747 		 *		|				|
10748 		 *	l2arc_feed_thread		l2arc_rebuild
10749 		 *	will place new bufs here	restores bufs here
10750 		 *
10751 		 * During l2arc_rebuild() the device is not used by
10752 		 * l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
10753 		 */
10754 		size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
10755 		asize += vdev_psize_to_asize(dev->l2ad_vdev,
10756 		    L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
10757 		l2arc_hdr_restore(&lb->lb_entries[i], dev);
10758 	}
10759 
10760 	/*
10761 	 * Record rebuild stats:
10762 	 *	size		Logical size of restored buffers in the L2ARC
10763 	 *	asize		Aligned size of restored buffers in the L2ARC
10764 	 */
10765 	ARCSTAT_INCR(arcstat_l2_rebuild_size, size);
10766 	ARCSTAT_INCR(arcstat_l2_rebuild_asize, asize);
10767 	ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries);
10768 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, lb_asize);
10769 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, asize / lb_asize);
10770 	ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks);
10771 }
10772 
10773 /*
10774  * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
10775  * into a state indicating that it has been evicted to L2ARC.
10776  */
10777 static void
l2arc_hdr_restore(const l2arc_log_ent_phys_t * le,l2arc_dev_t * dev)10778 l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, l2arc_dev_t *dev)
10779 {
10780 	arc_buf_hdr_t		*hdr, *exists;
10781 	kmutex_t		*hash_lock;
10782 	arc_buf_contents_t	type = L2BLK_GET_TYPE((le)->le_prop);
10783 	uint64_t		asize = vdev_psize_to_asize(dev->l2ad_vdev,
10784 	    L2BLK_GET_PSIZE((le)->le_prop));
10785 
10786 	/*
10787 	 * Do all the allocation before grabbing any locks, this lets us
10788 	 * sleep if memory is full and we don't have to deal with failed
10789 	 * allocations.
10790 	 */
10791 	hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
10792 	    dev, le->le_dva, le->le_daddr,
10793 	    L2BLK_GET_PSIZE((le)->le_prop), asize, le->le_birth,
10794 	    L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
10795 	    L2BLK_GET_PROTECTED((le)->le_prop),
10796 	    L2BLK_GET_PREFETCH((le)->le_prop),
10797 	    L2BLK_GET_STATE((le)->le_prop));
10798 
10799 	/*
10800 	 * vdev_space_update() has to be called before arc_hdr_destroy() to
10801 	 * avoid underflow since the latter also calls vdev_space_update().
10802 	 */
10803 	l2arc_hdr_arcstats_increment(hdr);
10804 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10805 
10806 	mutex_enter(&dev->l2ad_mtx);
10807 	list_insert_tail(&dev->l2ad_buflist, hdr);
10808 	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
10809 	mutex_exit(&dev->l2ad_mtx);
10810 
10811 	exists = buf_hash_insert(hdr, &hash_lock);
10812 	if (exists) {
10813 		/* Buffer was already cached, no need to restore it. */
10814 		arc_hdr_destroy(hdr);
10815 		/*
10816 		 * If the buffer is already cached, check whether it has
10817 		 * L2ARC metadata. If not, enter them and update the flag.
10818 		 * This is important is case of onlining a cache device, since
10819 		 * we previously evicted all L2ARC metadata from ARC.
10820 		 */
10821 		if (!HDR_HAS_L2HDR(exists)) {
10822 			arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR);
10823 			exists->b_l2hdr.b_dev = dev;
10824 			exists->b_l2hdr.b_daddr = le->le_daddr;
10825 			exists->b_l2hdr.b_arcs_state =
10826 			    L2BLK_GET_STATE((le)->le_prop);
10827 			/* l2arc_hdr_arcstats_update() expects a valid asize */
10828 			HDR_SET_L2SIZE(exists, asize);
10829 			mutex_enter(&dev->l2ad_mtx);
10830 			list_insert_tail(&dev->l2ad_buflist, exists);
10831 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
10832 			    arc_hdr_size(exists), exists);
10833 			mutex_exit(&dev->l2ad_mtx);
10834 			l2arc_hdr_arcstats_increment(exists);
10835 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10836 		}
10837 		ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached);
10838 	}
10839 
10840 	mutex_exit(hash_lock);
10841 }
10842 
10843 /*
10844  * Starts an asynchronous read IO to read a log block. This is used in log
10845  * block reconstruction to start reading the next block before we are done
10846  * decoding and reconstructing the current block, to keep the l2arc device
10847  * nice and hot with read IO to process.
10848  * The returned zio will contain a newly allocated memory buffers for the IO
10849  * data which should then be freed by the caller once the zio is no longer
10850  * needed (i.e. due to it having completed). If you wish to abort this
10851  * zio, you should do so using l2arc_log_blk_fetch_abort, which takes
10852  * care of disposing of the allocated buffers correctly.
10853  */
10854 static zio_t *
l2arc_log_blk_fetch(vdev_t * vd,const l2arc_log_blkptr_t * lbp,l2arc_log_blk_phys_t * lb)10855 l2arc_log_blk_fetch(vdev_t *vd, const l2arc_log_blkptr_t *lbp,
10856     l2arc_log_blk_phys_t *lb)
10857 {
10858 	uint32_t		asize;
10859 	zio_t			*pio;
10860 	l2arc_read_callback_t	*cb;
10861 
10862 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
10863 	asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10864 	ASSERT(asize <= sizeof (l2arc_log_blk_phys_t));
10865 
10866 	cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP);
10867 	cb->l2rcb_abd = abd_get_from_buf(lb, asize);
10868 	pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
10869 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY);
10870 	(void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
10871 	    cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10872 	    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL |
10873 	    ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE));
10874 
10875 	return (pio);
10876 }
10877 
10878 /*
10879  * Aborts a zio returned from l2arc_log_blk_fetch and frees the data
10880  * buffers allocated for it.
10881  */
10882 static void
l2arc_log_blk_fetch_abort(zio_t * zio)10883 l2arc_log_blk_fetch_abort(zio_t *zio)
10884 {
10885 	(void) zio_wait(zio);
10886 }
10887 
10888 /*
10889  * Creates a zio to update the device header on an l2arc device.
10890  */
10891 void
l2arc_dev_hdr_update(l2arc_dev_t * dev)10892 l2arc_dev_hdr_update(l2arc_dev_t *dev)
10893 {
10894 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10895 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10896 	abd_t			*abd;
10897 	int			err;
10898 
10899 	VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));
10900 
10901 	l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
10902 	l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
10903 	l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10904 	l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
10905 	l2dhdr->dh_log_entries = dev->l2ad_log_entries;
10906 	l2dhdr->dh_evict = dev->l2ad_evict;
10907 	l2dhdr->dh_start = dev->l2ad_start;
10908 	l2dhdr->dh_end = dev->l2ad_end;
10909 	l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
10910 	l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
10911 	l2dhdr->dh_flags = 0;
10912 	l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
10913 	l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
10914 	if (dev->l2ad_first)
10915 		l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST;
10916 
10917 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
10918 
10919 	err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
10920 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL,
10921 	    NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE));
10922 
10923 	abd_free(abd);
10924 
10925 	if (err != 0) {
10926 		zfs_dbgmsg("L2ARC IO error (%d) while writing device header, "
10927 		    "vdev guid: %llu", err,
10928 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10929 	}
10930 }
10931 
10932 /*
10933  * Commits a log block to the L2ARC device. This routine is invoked from
10934  * l2arc_write_buffers when the log block fills up.
10935  * This function allocates some memory to temporarily hold the serialized
10936  * buffer to be written. This is then released in l2arc_write_done.
10937  */
10938 static uint64_t
l2arc_log_blk_commit(l2arc_dev_t * dev,zio_t * pio,l2arc_write_callback_t * cb)10939 l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
10940 {
10941 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
10942 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
10943 	uint64_t		psize, asize;
10944 	zio_t			*wzio;
10945 	l2arc_lb_abd_buf_t	*abd_buf;
10946 	abd_t			*abd = NULL;
10947 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
10948 
10949 	VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
10950 
10951 	abd_buf = zio_buf_alloc(sizeof (*abd_buf));
10952 	abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
10953 	lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
10954 	lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);
10955 
10956 	/* link the buffer into the block chain */
10957 	lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
10958 	lb->lb_magic = L2ARC_LOG_BLK_MAGIC;
10959 
10960 	/*
10961 	 * l2arc_log_blk_commit() may be called multiple times during a single
10962 	 * l2arc_write_buffers() call. Save the allocated abd buffers in a list
10963 	 * so we can free them in l2arc_write_done() later on.
10964 	 */
10965 	list_insert_tail(&cb->l2wcb_abd_list, abd_buf);
10966 
10967 	/* try to compress the buffer, at least one sector to save */
10968 	psize = zio_compress_data(ZIO_COMPRESS_LZ4,
10969 	    abd_buf->abd, &abd, sizeof (*lb),
10970 	    zio_get_compression_max_size(ZIO_COMPRESS_LZ4,
10971 	    dev->l2ad_vdev->vdev_ashift,
10972 	    dev->l2ad_vdev->vdev_ashift, sizeof (*lb)), 0);
10973 
10974 	/* a log block is never entirely zero */
10975 	ASSERT(psize != 0);
10976 	asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
10977 	ASSERT(asize <= sizeof (*lb));
10978 
10979 	/*
10980 	 * Update the start log block pointer in the device header to point
10981 	 * to the log block we're about to write.
10982 	 */
10983 	l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
10984 	l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
10985 	l2dhdr->dh_start_lbps[0].lbp_payload_asize =
10986 	    dev->l2ad_log_blk_payload_asize;
10987 	l2dhdr->dh_start_lbps[0].lbp_payload_start =
10988 	    dev->l2ad_log_blk_payload_start;
10989 	L2BLK_SET_LSIZE(
10990 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
10991 	L2BLK_SET_PSIZE(
10992 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
10993 	L2BLK_SET_CHECKSUM(
10994 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10995 	    ZIO_CHECKSUM_FLETCHER_4);
10996 	if (asize < sizeof (*lb)) {
10997 		/* compression succeeded */
10998 		abd_zero_off(abd, psize, asize - psize);
10999 		L2BLK_SET_COMPRESS(
11000 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
11001 		    ZIO_COMPRESS_LZ4);
11002 	} else {
11003 		/* compression failed */
11004 		abd_copy_from_buf_off(abd, lb, 0, sizeof (*lb));
11005 		L2BLK_SET_COMPRESS(
11006 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
11007 		    ZIO_COMPRESS_OFF);
11008 	}
11009 
11010 	/* checksum what we're about to write */
11011 	abd_fletcher_4_native(abd, asize, NULL,
11012 	    &l2dhdr->dh_start_lbps[0].lbp_cksum);
11013 
11014 	abd_free(abd_buf->abd);
11015 
11016 	/* perform the write itself */
11017 	abd_buf->abd = abd;
11018 	wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
11019 	    asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
11020 	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
11021 	DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio);
11022 	(void) zio_nowait(wzio);
11023 
11024 	dev->l2ad_hand += asize;
11025 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
11026 
11027 	/*
11028 	 * Include the committed log block's pointer  in the list of pointers
11029 	 * to log blocks present in the L2ARC device.
11030 	 */
11031 	memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
11032 	    sizeof (l2arc_log_blkptr_t));
11033 	mutex_enter(&dev->l2ad_mtx);
11034 	list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
11035 	ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
11036 	ARCSTAT_BUMP(arcstat_l2_log_blk_count);
11037 	zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
11038 	zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
11039 	mutex_exit(&dev->l2ad_mtx);
11040 
11041 	/* bump the kstats */
11042 	ARCSTAT_INCR(arcstat_l2_write_bytes, asize);
11043 	ARCSTAT_BUMP(arcstat_l2_log_blk_writes);
11044 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, asize);
11045 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio,
11046 	    dev->l2ad_log_blk_payload_asize / asize);
11047 
11048 	/* start a new log block */
11049 	dev->l2ad_log_ent_idx = 0;
11050 	dev->l2ad_log_blk_payload_asize = 0;
11051 	dev->l2ad_log_blk_payload_start = 0;
11052 
11053 	return (asize);
11054 }
11055 
11056 /*
11057  * Validates an L2ARC log block address to make sure that it can be read
11058  * from the provided L2ARC device.
11059  */
11060 boolean_t
l2arc_log_blkptr_valid(l2arc_dev_t * dev,const l2arc_log_blkptr_t * lbp)11061 l2arc_log_blkptr_valid(l2arc_dev_t *dev, const l2arc_log_blkptr_t *lbp)
11062 {
11063 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
11064 	uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
11065 	uint64_t end = lbp->lbp_daddr + asize - 1;
11066 	uint64_t start = lbp->lbp_payload_start;
11067 	boolean_t evicted = B_FALSE;
11068 
11069 	/*
11070 	 * A log block is valid if all of the following conditions are true:
11071 	 * - it fits entirely (including its payload) between l2ad_start and
11072 	 *   l2ad_end
11073 	 * - it has a valid size
11074 	 * - neither the log block itself nor part of its payload was evicted
11075 	 *   by l2arc_evict():
11076 	 *
11077 	 *		l2ad_hand          l2ad_evict
11078 	 *		|			 |	lbp_daddr
11079 	 *		|     start		 |	|  end
11080 	 *		|     |			 |	|  |
11081 	 *		V     V		         V	V  V
11082 	 *   l2ad_start ============================================ l2ad_end
11083 	 *                    --------------------------||||
11084 	 *				^		 ^
11085 	 *				|		log block
11086 	 *				payload
11087 	 */
11088 
11089 	evicted =
11090 	    l2arc_range_check_overlap(start, end, dev->l2ad_hand) ||
11091 	    l2arc_range_check_overlap(start, end, dev->l2ad_evict) ||
11092 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) ||
11093 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);
11094 
11095 	return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
11096 	    asize > 0 && asize <= sizeof (l2arc_log_blk_phys_t) &&
11097 	    (!evicted || dev->l2ad_first));
11098 }
11099 
11100 /*
11101  * Inserts ARC buffer header `hdr' into the current L2ARC log block on
11102  * the device. The buffer being inserted must be present in L2ARC.
11103  * Returns B_TRUE if the L2ARC log block is full and needs to be committed
11104  * to L2ARC, or B_FALSE if it still has room for more ARC buffers.
11105  */
11106 static boolean_t
l2arc_log_blk_insert(l2arc_dev_t * dev,const arc_buf_hdr_t * hdr)11107 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)
11108 {
11109 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
11110 	l2arc_log_ent_phys_t	*le;
11111 
11112 	if (dev->l2ad_log_entries == 0)
11113 		return (B_FALSE);
11114 
11115 	int index = dev->l2ad_log_ent_idx++;
11116 
11117 	ASSERT3S(index, <, dev->l2ad_log_entries);
11118 	ASSERT(HDR_HAS_L2HDR(hdr));
11119 
11120 	le = &lb->lb_entries[index];
11121 	memset(le, 0, sizeof (*le));
11122 	le->le_dva = hdr->b_dva;
11123 	le->le_birth = hdr->b_birth;
11124 	le->le_daddr = hdr->b_l2hdr.b_daddr;
11125 	if (index == 0)
11126 		dev->l2ad_log_blk_payload_start = le->le_daddr;
11127 	L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
11128 	L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
11129 	L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
11130 	le->le_complevel = hdr->b_complevel;
11131 	L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
11132 	L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
11133 	L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
11134 	L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);
11135 
11136 	dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
11137 	    HDR_GET_PSIZE(hdr));
11138 
11139 	return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
11140 }
11141 
11142 /*
11143  * Checks whether a given L2ARC device address sits in a time-sequential
11144  * range. The trick here is that the L2ARC is a rotary buffer, so we can't
11145  * just do a range comparison, we need to handle the situation in which the
11146  * range wraps around the end of the L2ARC device. Arguments:
11147  *	bottom -- Lower end of the range to check (written to earlier).
11148  *	top    -- Upper end of the range to check (written to later).
11149  *	check  -- The address for which we want to determine if it sits in
11150  *		  between the top and bottom.
11151  *
11152  * The 3-way conditional below represents the following cases:
11153  *
11154  *	bottom < top : Sequentially ordered case:
11155  *	  <check>--------+-------------------+
11156  *	                 |  (overlap here?)  |
11157  *	 L2ARC dev       V                   V
11158  *	 |---------------<bottom>============<top>--------------|
11159  *
11160  *	bottom > top: Looped-around case:
11161  *	                      <check>--------+------------------+
11162  *	                                     |  (overlap here?) |
11163  *	 L2ARC dev                           V                  V
11164  *	 |===============<top>---------------<bottom>===========|
11165  *	 ^               ^
11166  *	 |  (or here?)   |
11167  *	 +---------------+---------<check>
11168  *
11169  *	top == bottom : Just a single address comparison.
11170  */
11171 boolean_t
l2arc_range_check_overlap(uint64_t bottom,uint64_t top,uint64_t check)11172 l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check)
11173 {
11174 	if (bottom < top)
11175 		return (bottom <= check && check <= top);
11176 	else if (bottom > top)
11177 		return (check <= top || bottom <= check);
11178 	else
11179 		return (check == top);
11180 }
11181 
11182 EXPORT_SYMBOL(arc_buf_size);
11183 EXPORT_SYMBOL(arc_write);
11184 EXPORT_SYMBOL(arc_read);
11185 EXPORT_SYMBOL(arc_buf_info);
11186 EXPORT_SYMBOL(arc_getbuf_func);
11187 EXPORT_SYMBOL(arc_add_prune_callback);
11188 EXPORT_SYMBOL(arc_remove_prune_callback);
11189 
11190 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
11191 	spl_param_get_u64, ZMOD_RW, "Minimum ARC size in bytes");
11192 
11193 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
11194 	spl_param_get_u64, ZMOD_RW, "Maximum ARC size in bytes");
11195 
11196 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, meta_balance, UINT, ZMOD_RW,
11197 	"Balance between metadata and data on ghost hits.");
11198 
11199 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, grow_retry, param_set_arc_int,
11200 	param_get_uint, ZMOD_RW, "Seconds before growing ARC size");
11201 
11202 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, shrink_shift, param_set_arc_int,
11203 	param_get_uint, ZMOD_RW, "log2(fraction of ARC to reclaim)");
11204 
11205 #ifdef _KERNEL
11206 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, pc_percent, UINT, ZMOD_RW,
11207 	"Percent of pagecache to reclaim ARC to");
11208 #endif
11209 
11210 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, average_blocksize, UINT, ZMOD_RD,
11211 	"Target average block size");
11212 
11213 ZFS_MODULE_PARAM(zfs, zfs_, compressed_arc_enabled, INT, ZMOD_RW,
11214 	"Disable compressed ARC buffers");
11215 
11216 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prefetch_ms, param_set_arc_int,
11217 	param_get_uint, ZMOD_RW, "Min life of prefetch block in ms");
11218 
11219 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prescient_prefetch_ms,
11220     param_set_arc_int, param_get_uint, ZMOD_RW,
11221 	"Min life of prescient prefetched block in ms");
11222 
11223 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_max, U64, ZMOD_RW,
11224 	"Max write bytes per interval");
11225 
11226 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_boost, U64, ZMOD_RW,
11227 	"Extra write bytes during device warmup");
11228 
11229 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom, U64, ZMOD_RW,
11230 	"Number of max device writes to precache");
11231 
11232 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom_boost, U64, ZMOD_RW,
11233 	"Compressed l2arc_headroom multiplier");
11234 
11235 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, trim_ahead, U64, ZMOD_RW,
11236 	"TRIM ahead L2ARC write size multiplier");
11237 
11238 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_secs, U64, ZMOD_RW,
11239 	"Seconds between L2ARC writing");
11240 
11241 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_min_ms, U64, ZMOD_RW,
11242 	"Min feed interval in milliseconds");
11243 
11244 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, noprefetch, INT, ZMOD_RW,
11245 	"Skip caching prefetched buffers");
11246 
11247 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW,
11248 	"Turbo L2ARC warmup");
11249 
11250 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW,
11251 	"No reads during writes");
11252 
11253 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, UINT, ZMOD_RW,
11254 	"Percent of ARC size allowed for L2ARC-only headers");
11255 
11256 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
11257 	"Rebuild the L2ARC when importing a pool");
11258 
11259 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, U64, ZMOD_RW,
11260 	"Min size in bytes to write rebuild log blocks in L2ARC");
11261 
11262 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, mfuonly, INT, ZMOD_RW,
11263 	"Cache only MFU data from ARC into L2ARC");
11264 
11265 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, exclude_special, INT, ZMOD_RW,
11266 	"Exclude dbufs on special vdevs from being cached to L2ARC if set.");
11267 
11268 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int,
11269 	param_get_uint, ZMOD_RW, "System free memory I/O throttle in bytes");
11270 
11271 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, sys_free, param_set_arc_u64,
11272 	spl_param_get_u64, ZMOD_RW, "System free memory target size in bytes");
11273 
11274 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit, param_set_arc_u64,
11275 	spl_param_get_u64, ZMOD_RW, "Minimum bytes of dnodes in ARC");
11276 
11277 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit_percent,
11278     param_set_arc_int, param_get_uint, ZMOD_RW,
11279 	"Percent of ARC meta buffers for dnodes");
11280 
11281 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, dnode_reduce_percent, UINT, ZMOD_RW,
11282 	"Percentage of excess dnodes to try to unpin");
11283 
11284 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, UINT, ZMOD_RW,
11285 	"When full, ARC allocation waits for eviction of this % of alloc size");
11286 
11287 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, UINT, ZMOD_RW,
11288 	"The number of headers to evict per sublist before moving to the next");
11289 
11290 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
11291 	"Number of arc_prune threads");
11292 
11293 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_threads, UINT, ZMOD_RD,
11294 	"Number of threads to use for ARC eviction.");
11295