xref: /titanic_41/usr/src/uts/common/fs/zfs/arc.c (revision 2a796556e7d6dfb15a985d4d879c73b491b7ae11)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
24  * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
25  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
27  */
28 
29 /*
30  * DVA-based Adjustable Replacement Cache
31  *
32  * While much of the theory of operation used here is
33  * based on the self-tuning, low overhead replacement cache
34  * presented by Megiddo and Modha at FAST 2003, there are some
35  * significant differences:
36  *
37  * 1. The Megiddo and Modha model assumes any page is evictable.
38  * Pages in its cache cannot be "locked" into memory.  This makes
39  * the eviction algorithm simple: evict the last page in the list.
40  * This also make the performance characteristics easy to reason
41  * about.  Our cache is not so simple.  At any given moment, some
42  * subset of the blocks in the cache are un-evictable because we
43  * have handed out a reference to them.  Blocks are only evictable
44  * when there are no external references active.  This makes
45  * eviction far more problematic:  we choose to evict the evictable
46  * blocks that are the "lowest" in the list.
47  *
48  * There are times when it is not possible to evict the requested
49  * space.  In these circumstances we are unable to adjust the cache
50  * size.  To prevent the cache growing unbounded at these times we
51  * implement a "cache throttle" that slows the flow of new data
52  * into the cache until we can make space available.
53  *
54  * 2. The Megiddo and Modha model assumes a fixed cache size.
55  * Pages are evicted when the cache is full and there is a cache
56  * miss.  Our model has a variable sized cache.  It grows with
57  * high use, but also tries to react to memory pressure from the
58  * operating system: decreasing its size when system memory is
59  * tight.
60  *
61  * 3. The Megiddo and Modha model assumes a fixed page size. All
62  * elements of the cache are therefore exactly the same size.  So
63  * when adjusting the cache size following a cache miss, its simply
64  * a matter of choosing a single page to evict.  In our model, we
65  * have variable sized cache blocks (rangeing from 512 bytes to
66  * 128K bytes).  We therefore choose a set of blocks to evict to make
67  * space for a cache miss that approximates as closely as possible
68  * the space used by the new block.
69  *
70  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71  * by N. Megiddo & D. Modha, FAST 2003
72  */
73 
74 /*
75  * The locking model:
76  *
77  * A new reference to a cache buffer can be obtained in two
78  * ways: 1) via a hash table lookup using the DVA as a key,
79  * or 2) via one of the ARC lists.  The arc_read() interface
80  * uses method 1, while the internal arc algorithms for
81  * adjusting the cache use method 2.  We therefore provide two
82  * types of locks: 1) the hash table lock array, and 2) the
83  * arc list locks.
84  *
85  * Buffers do not have their own mutexes, rather they rely on the
86  * hash table mutexes for the bulk of their protection (i.e. most
87  * fields in the arc_buf_hdr_t are protected by these mutexes).
88  *
89  * buf_hash_find() returns the appropriate mutex (held) when it
90  * locates the requested buffer in the hash table.  It returns
91  * NULL for the mutex if the buffer was not in the table.
92  *
93  * buf_hash_remove() expects the appropriate hash mutex to be
94  * already held before it is invoked.
95  *
96  * Each arc state also has a mutex which is used to protect the
97  * buffer list associated with the state.  When attempting to
98  * obtain a hash table lock while holding an arc list lock you
99  * must use: mutex_tryenter() to avoid deadlock.  Also note that
100  * the active state mutex must be held before the ghost state mutex.
101  *
102  * Arc buffers may have an associated eviction callback function.
103  * This function will be invoked prior to removing the buffer (e.g.
104  * in arc_do_user_evicts()).  Note however that the data associated
105  * with the buffer may be evicted prior to the callback.  The callback
106  * must be made with *no locks held* (to prevent deadlock).  Additionally,
107  * the users of callbacks must ensure that their private data is
108  * protected from simultaneous callbacks from arc_clear_callback()
109  * and arc_do_user_evicts().
110  *
111  * Note that the majority of the performance stats are manipulated
112  * with atomic operations.
113  *
114  * The L2ARC uses the l2ad_mtx on each vdev for the following:
115  *
116  *	- L2ARC buflist creation
117  *	- L2ARC buflist eviction
118  *	- L2ARC write completion, which walks L2ARC buflists
119  *	- ARC header destruction, as it removes from L2ARC buflists
120  *	- ARC header release, as it removes from L2ARC buflists
121  */
122 
123 #include <sys/spa.h>
124 #include <sys/zio.h>
125 #include <sys/zio_compress.h>
126 #include <sys/zfs_context.h>
127 #include <sys/arc.h>
128 #include <sys/refcount.h>
129 #include <sys/vdev.h>
130 #include <sys/vdev_impl.h>
131 #include <sys/dsl_pool.h>
132 #include <sys/multilist.h>
133 #ifdef _KERNEL
134 #include <sys/vmsystm.h>
135 #include <vm/anon.h>
136 #include <sys/fs/swapnode.h>
137 #include <sys/dnlc.h>
138 #endif
139 #include <sys/callb.h>
140 #include <sys/kstat.h>
141 #include <zfs_fletcher.h>
142 
143 #ifndef _KERNEL
144 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
145 boolean_t arc_watch = B_FALSE;
146 int arc_procfd;
147 #endif
148 
149 static kmutex_t		arc_reclaim_lock;
150 static kcondvar_t	arc_reclaim_thread_cv;
151 static boolean_t	arc_reclaim_thread_exit;
152 static kcondvar_t	arc_reclaim_waiters_cv;
153 
154 static kmutex_t		arc_user_evicts_lock;
155 static kcondvar_t	arc_user_evicts_cv;
156 static boolean_t	arc_user_evicts_thread_exit;
157 
158 uint_t arc_reduce_dnlc_percent = 3;
159 
160 /*
161  * The number of headers to evict in arc_evict_state_impl() before
162  * dropping the sublist lock and evicting from another sublist. A lower
163  * value means we're more likely to evict the "correct" header (i.e. the
164  * oldest header in the arc state), but comes with higher overhead
165  * (i.e. more invocations of arc_evict_state_impl()).
166  */
167 int zfs_arc_evict_batch_limit = 10;
168 
169 /*
170  * The number of sublists used for each of the arc state lists. If this
171  * is not set to a suitable value by the user, it will be configured to
172  * the number of CPUs on the system in arc_init().
173  */
174 int zfs_arc_num_sublists_per_state = 0;
175 
176 /* number of seconds before growing cache again */
177 static int		arc_grow_retry = 60;
178 
179 /* shift of arc_c for calculating overflow limit in arc_get_data_buf */
180 int		zfs_arc_overflow_shift = 8;
181 
182 /* shift of arc_c for calculating both min and max arc_p */
183 static int		arc_p_min_shift = 4;
184 
185 /* log2(fraction of arc to reclaim) */
186 static int		arc_shrink_shift = 7;
187 
188 /*
189  * log2(fraction of ARC which must be free to allow growing).
190  * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
191  * when reading a new block into the ARC, we will evict an equal-sized block
192  * from the ARC.
193  *
194  * This must be less than arc_shrink_shift, so that when we shrink the ARC,
195  * we will still not allow it to grow.
196  */
197 int			arc_no_grow_shift = 5;
198 
199 
200 /*
201  * minimum lifespan of a prefetch block in clock ticks
202  * (initialized in arc_init())
203  */
204 static int		arc_min_prefetch_lifespan;
205 
206 /*
207  * If this percent of memory is free, don't throttle.
208  */
209 int arc_lotsfree_percent = 10;
210 
211 static int arc_dead;
212 
213 /*
214  * The arc has filled available memory and has now warmed up.
215  */
216 static boolean_t arc_warm;
217 
218 /*
219  * These tunables are for performance analysis.
220  */
221 uint64_t zfs_arc_max;
222 uint64_t zfs_arc_min;
223 uint64_t zfs_arc_meta_limit = 0;
224 uint64_t zfs_arc_meta_min = 0;
225 int zfs_arc_grow_retry = 0;
226 int zfs_arc_shrink_shift = 0;
227 int zfs_arc_p_min_shift = 0;
228 int zfs_disable_dup_eviction = 0;
229 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
230 
231 /*
232  * Note that buffers can be in one of 6 states:
233  *	ARC_anon	- anonymous (discussed below)
234  *	ARC_mru		- recently used, currently cached
235  *	ARC_mru_ghost	- recentely used, no longer in cache
236  *	ARC_mfu		- frequently used, currently cached
237  *	ARC_mfu_ghost	- frequently used, no longer in cache
238  *	ARC_l2c_only	- exists in L2ARC but not other states
239  * When there are no active references to the buffer, they are
240  * are linked onto a list in one of these arc states.  These are
241  * the only buffers that can be evicted or deleted.  Within each
242  * state there are multiple lists, one for meta-data and one for
243  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
244  * etc.) is tracked separately so that it can be managed more
245  * explicitly: favored over data, limited explicitly.
246  *
247  * Anonymous buffers are buffers that are not associated with
248  * a DVA.  These are buffers that hold dirty block copies
249  * before they are written to stable storage.  By definition,
250  * they are "ref'd" and are considered part of arc_mru
251  * that cannot be freed.  Generally, they will aquire a DVA
252  * as they are written and migrate onto the arc_mru list.
253  *
254  * The ARC_l2c_only state is for buffers that are in the second
255  * level ARC but no longer in any of the ARC_m* lists.  The second
256  * level ARC itself may also contain buffers that are in any of
257  * the ARC_m* states - meaning that a buffer can exist in two
258  * places.  The reason for the ARC_l2c_only state is to keep the
259  * buffer header in the hash table, so that reads that hit the
260  * second level ARC benefit from these fast lookups.
261  */
262 
263 typedef struct arc_state {
264 	/*
265 	 * list of evictable buffers
266 	 */
267 	multilist_t arcs_list[ARC_BUFC_NUMTYPES];
268 	/*
269 	 * total amount of evictable data in this state
270 	 */
271 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
272 	/*
273 	 * total amount of data in this state; this includes: evictable,
274 	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
275 	 */
276 	refcount_t arcs_size;
277 } arc_state_t;
278 
279 /* The 6 states: */
280 static arc_state_t ARC_anon;
281 static arc_state_t ARC_mru;
282 static arc_state_t ARC_mru_ghost;
283 static arc_state_t ARC_mfu;
284 static arc_state_t ARC_mfu_ghost;
285 static arc_state_t ARC_l2c_only;
286 
287 typedef struct arc_stats {
288 	kstat_named_t arcstat_hits;
289 	kstat_named_t arcstat_misses;
290 	kstat_named_t arcstat_demand_data_hits;
291 	kstat_named_t arcstat_demand_data_misses;
292 	kstat_named_t arcstat_demand_metadata_hits;
293 	kstat_named_t arcstat_demand_metadata_misses;
294 	kstat_named_t arcstat_prefetch_data_hits;
295 	kstat_named_t arcstat_prefetch_data_misses;
296 	kstat_named_t arcstat_prefetch_metadata_hits;
297 	kstat_named_t arcstat_prefetch_metadata_misses;
298 	kstat_named_t arcstat_mru_hits;
299 	kstat_named_t arcstat_mru_ghost_hits;
300 	kstat_named_t arcstat_mfu_hits;
301 	kstat_named_t arcstat_mfu_ghost_hits;
302 	kstat_named_t arcstat_deleted;
303 	/*
304 	 * Number of buffers that could not be evicted because the hash lock
305 	 * was held by another thread.  The lock may not necessarily be held
306 	 * by something using the same buffer, since hash locks are shared
307 	 * by multiple buffers.
308 	 */
309 	kstat_named_t arcstat_mutex_miss;
310 	/*
311 	 * Number of buffers skipped because they have I/O in progress, are
312 	 * indrect prefetch buffers that have not lived long enough, or are
313 	 * not from the spa we're trying to evict from.
314 	 */
315 	kstat_named_t arcstat_evict_skip;
316 	/*
317 	 * Number of times arc_evict_state() was unable to evict enough
318 	 * buffers to reach it's target amount.
319 	 */
320 	kstat_named_t arcstat_evict_not_enough;
321 	kstat_named_t arcstat_evict_l2_cached;
322 	kstat_named_t arcstat_evict_l2_eligible;
323 	kstat_named_t arcstat_evict_l2_ineligible;
324 	kstat_named_t arcstat_evict_l2_skip;
325 	kstat_named_t arcstat_hash_elements;
326 	kstat_named_t arcstat_hash_elements_max;
327 	kstat_named_t arcstat_hash_collisions;
328 	kstat_named_t arcstat_hash_chains;
329 	kstat_named_t arcstat_hash_chain_max;
330 	kstat_named_t arcstat_p;
331 	kstat_named_t arcstat_c;
332 	kstat_named_t arcstat_c_min;
333 	kstat_named_t arcstat_c_max;
334 	kstat_named_t arcstat_size;
335 	/*
336 	 * Number of bytes consumed by internal ARC structures necessary
337 	 * for tracking purposes; these structures are not actually
338 	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
339 	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
340 	 * caches), and arc_buf_t structures (allocated via arc_buf_t
341 	 * cache).
342 	 */
343 	kstat_named_t arcstat_hdr_size;
344 	/*
345 	 * Number of bytes consumed by ARC buffers of type equal to
346 	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
347 	 * on disk user data (e.g. plain file contents).
348 	 */
349 	kstat_named_t arcstat_data_size;
350 	/*
351 	 * Number of bytes consumed by ARC buffers of type equal to
352 	 * ARC_BUFC_METADATA. This is generally consumed by buffers
353 	 * backing on disk data that is used for internal ZFS
354 	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
355 	 */
356 	kstat_named_t arcstat_metadata_size;
357 	/*
358 	 * Number of bytes consumed by various buffers and structures
359 	 * not actually backed with ARC buffers. This includes bonus
360 	 * buffers (allocated directly via zio_buf_* functions),
361 	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
362 	 * cache), and dnode_t structures (allocated via dnode_t cache).
363 	 */
364 	kstat_named_t arcstat_other_size;
365 	/*
366 	 * Total number of bytes consumed by ARC buffers residing in the
367 	 * arc_anon state. This includes *all* buffers in the arc_anon
368 	 * state; e.g. data, metadata, evictable, and unevictable buffers
369 	 * are all included in this value.
370 	 */
371 	kstat_named_t arcstat_anon_size;
372 	/*
373 	 * Number of bytes consumed by ARC buffers that meet the
374 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
375 	 * residing in the arc_anon state, and are eligible for eviction
376 	 * (e.g. have no outstanding holds on the buffer).
377 	 */
378 	kstat_named_t arcstat_anon_evictable_data;
379 	/*
380 	 * Number of bytes consumed by ARC buffers that meet the
381 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
382 	 * residing in the arc_anon state, and are eligible for eviction
383 	 * (e.g. have no outstanding holds on the buffer).
384 	 */
385 	kstat_named_t arcstat_anon_evictable_metadata;
386 	/*
387 	 * Total number of bytes consumed by ARC buffers residing in the
388 	 * arc_mru state. This includes *all* buffers in the arc_mru
389 	 * state; e.g. data, metadata, evictable, and unevictable buffers
390 	 * are all included in this value.
391 	 */
392 	kstat_named_t arcstat_mru_size;
393 	/*
394 	 * Number of bytes consumed by ARC buffers that meet the
395 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
396 	 * residing in the arc_mru state, and are eligible for eviction
397 	 * (e.g. have no outstanding holds on the buffer).
398 	 */
399 	kstat_named_t arcstat_mru_evictable_data;
400 	/*
401 	 * Number of bytes consumed by ARC buffers that meet the
402 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
403 	 * residing in the arc_mru state, and are eligible for eviction
404 	 * (e.g. have no outstanding holds on the buffer).
405 	 */
406 	kstat_named_t arcstat_mru_evictable_metadata;
407 	/*
408 	 * Total number of bytes that *would have been* consumed by ARC
409 	 * buffers in the arc_mru_ghost state. The key thing to note
410 	 * here, is the fact that this size doesn't actually indicate
411 	 * RAM consumption. The ghost lists only consist of headers and
412 	 * don't actually have ARC buffers linked off of these headers.
413 	 * Thus, *if* the headers had associated ARC buffers, these
414 	 * buffers *would have* consumed this number of bytes.
415 	 */
416 	kstat_named_t arcstat_mru_ghost_size;
417 	/*
418 	 * Number of bytes that *would have been* consumed by ARC
419 	 * buffers that are eligible for eviction, of type
420 	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
421 	 */
422 	kstat_named_t arcstat_mru_ghost_evictable_data;
423 	/*
424 	 * Number of bytes that *would have been* consumed by ARC
425 	 * buffers that are eligible for eviction, of type
426 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
427 	 */
428 	kstat_named_t arcstat_mru_ghost_evictable_metadata;
429 	/*
430 	 * Total number of bytes consumed by ARC buffers residing in the
431 	 * arc_mfu state. This includes *all* buffers in the arc_mfu
432 	 * state; e.g. data, metadata, evictable, and unevictable buffers
433 	 * are all included in this value.
434 	 */
435 	kstat_named_t arcstat_mfu_size;
436 	/*
437 	 * Number of bytes consumed by ARC buffers that are eligible for
438 	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
439 	 * state.
440 	 */
441 	kstat_named_t arcstat_mfu_evictable_data;
442 	/*
443 	 * Number of bytes consumed by ARC buffers that are eligible for
444 	 * eviction, of type ARC_BUFC_METADATA, and reside in the
445 	 * arc_mfu state.
446 	 */
447 	kstat_named_t arcstat_mfu_evictable_metadata;
448 	/*
449 	 * Total number of bytes that *would have been* consumed by ARC
450 	 * buffers in the arc_mfu_ghost state. See the comment above
451 	 * arcstat_mru_ghost_size for more details.
452 	 */
453 	kstat_named_t arcstat_mfu_ghost_size;
454 	/*
455 	 * Number of bytes that *would have been* consumed by ARC
456 	 * buffers that are eligible for eviction, of type
457 	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
458 	 */
459 	kstat_named_t arcstat_mfu_ghost_evictable_data;
460 	/*
461 	 * Number of bytes that *would have been* consumed by ARC
462 	 * buffers that are eligible for eviction, of type
463 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
464 	 */
465 	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
466 	kstat_named_t arcstat_l2_hits;
467 	kstat_named_t arcstat_l2_misses;
468 	kstat_named_t arcstat_l2_feeds;
469 	kstat_named_t arcstat_l2_rw_clash;
470 	kstat_named_t arcstat_l2_read_bytes;
471 	kstat_named_t arcstat_l2_write_bytes;
472 	kstat_named_t arcstat_l2_writes_sent;
473 	kstat_named_t arcstat_l2_writes_done;
474 	kstat_named_t arcstat_l2_writes_error;
475 	kstat_named_t arcstat_l2_writes_lock_retry;
476 	kstat_named_t arcstat_l2_evict_lock_retry;
477 	kstat_named_t arcstat_l2_evict_reading;
478 	kstat_named_t arcstat_l2_evict_l1cached;
479 	kstat_named_t arcstat_l2_free_on_write;
480 	kstat_named_t arcstat_l2_cdata_free_on_write;
481 	kstat_named_t arcstat_l2_abort_lowmem;
482 	kstat_named_t arcstat_l2_cksum_bad;
483 	kstat_named_t arcstat_l2_io_error;
484 	kstat_named_t arcstat_l2_size;
485 	kstat_named_t arcstat_l2_asize;
486 	kstat_named_t arcstat_l2_hdr_size;
487 	kstat_named_t arcstat_l2_compress_successes;
488 	kstat_named_t arcstat_l2_compress_zeros;
489 	kstat_named_t arcstat_l2_compress_failures;
490 	kstat_named_t arcstat_memory_throttle_count;
491 	kstat_named_t arcstat_duplicate_buffers;
492 	kstat_named_t arcstat_duplicate_buffers_size;
493 	kstat_named_t arcstat_duplicate_reads;
494 	kstat_named_t arcstat_meta_used;
495 	kstat_named_t arcstat_meta_limit;
496 	kstat_named_t arcstat_meta_max;
497 	kstat_named_t arcstat_meta_min;
498 } arc_stats_t;
499 
500 static arc_stats_t arc_stats = {
501 	{ "hits",			KSTAT_DATA_UINT64 },
502 	{ "misses",			KSTAT_DATA_UINT64 },
503 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
504 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
505 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
506 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
507 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
508 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
509 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
510 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
511 	{ "mru_hits",			KSTAT_DATA_UINT64 },
512 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
513 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
514 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
515 	{ "deleted",			KSTAT_DATA_UINT64 },
516 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
517 	{ "evict_skip",			KSTAT_DATA_UINT64 },
518 	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
519 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
520 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
521 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
522 	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
523 	{ "hash_elements",		KSTAT_DATA_UINT64 },
524 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
525 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
526 	{ "hash_chains",		KSTAT_DATA_UINT64 },
527 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
528 	{ "p",				KSTAT_DATA_UINT64 },
529 	{ "c",				KSTAT_DATA_UINT64 },
530 	{ "c_min",			KSTAT_DATA_UINT64 },
531 	{ "c_max",			KSTAT_DATA_UINT64 },
532 	{ "size",			KSTAT_DATA_UINT64 },
533 	{ "hdr_size",			KSTAT_DATA_UINT64 },
534 	{ "data_size",			KSTAT_DATA_UINT64 },
535 	{ "metadata_size",		KSTAT_DATA_UINT64 },
536 	{ "other_size",			KSTAT_DATA_UINT64 },
537 	{ "anon_size",			KSTAT_DATA_UINT64 },
538 	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
539 	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
540 	{ "mru_size",			KSTAT_DATA_UINT64 },
541 	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
542 	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
543 	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
544 	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
545 	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
546 	{ "mfu_size",			KSTAT_DATA_UINT64 },
547 	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
548 	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
549 	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
550 	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
551 	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
552 	{ "l2_hits",			KSTAT_DATA_UINT64 },
553 	{ "l2_misses",			KSTAT_DATA_UINT64 },
554 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
555 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
556 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
557 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
558 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
559 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
560 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
561 	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
562 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
563 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
564 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
565 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
566 	{ "l2_cdata_free_on_write",	KSTAT_DATA_UINT64 },
567 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
568 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
569 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
570 	{ "l2_size",			KSTAT_DATA_UINT64 },
571 	{ "l2_asize",			KSTAT_DATA_UINT64 },
572 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
573 	{ "l2_compress_successes",	KSTAT_DATA_UINT64 },
574 	{ "l2_compress_zeros",		KSTAT_DATA_UINT64 },
575 	{ "l2_compress_failures",	KSTAT_DATA_UINT64 },
576 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
577 	{ "duplicate_buffers",		KSTAT_DATA_UINT64 },
578 	{ "duplicate_buffers_size",	KSTAT_DATA_UINT64 },
579 	{ "duplicate_reads",		KSTAT_DATA_UINT64 },
580 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
581 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
582 	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
583 	{ "arc_meta_min",		KSTAT_DATA_UINT64 }
584 };
585 
586 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
587 
588 #define	ARCSTAT_INCR(stat, val) \
589 	atomic_add_64(&arc_stats.stat.value.ui64, (val))
590 
591 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
592 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
593 
594 #define	ARCSTAT_MAX(stat, val) {					\
595 	uint64_t m;							\
596 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
597 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
598 		continue;						\
599 }
600 
601 #define	ARCSTAT_MAXSTAT(stat) \
602 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
603 
604 /*
605  * We define a macro to allow ARC hits/misses to be easily broken down by
606  * two separate conditions, giving a total of four different subtypes for
607  * each of hits and misses (so eight statistics total).
608  */
609 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
610 	if (cond1) {							\
611 		if (cond2) {						\
612 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
613 		} else {						\
614 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
615 		}							\
616 	} else {							\
617 		if (cond2) {						\
618 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
619 		} else {						\
620 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
621 		}							\
622 	}
623 
624 kstat_t			*arc_ksp;
625 static arc_state_t	*arc_anon;
626 static arc_state_t	*arc_mru;
627 static arc_state_t	*arc_mru_ghost;
628 static arc_state_t	*arc_mfu;
629 static arc_state_t	*arc_mfu_ghost;
630 static arc_state_t	*arc_l2c_only;
631 
632 /*
633  * There are several ARC variables that are critical to export as kstats --
634  * but we don't want to have to grovel around in the kstat whenever we wish to
635  * manipulate them.  For these variables, we therefore define them to be in
636  * terms of the statistic variable.  This assures that we are not introducing
637  * the possibility of inconsistency by having shadow copies of the variables,
638  * while still allowing the code to be readable.
639  */
640 #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
641 #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
642 #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
643 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
644 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
645 #define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
646 #define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
647 #define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
648 #define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
649 
650 #define	L2ARC_IS_VALID_COMPRESS(_c_) \
651 	((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
652 
653 static int		arc_no_grow;	/* Don't try to grow cache size */
654 static uint64_t		arc_tempreserve;
655 static uint64_t		arc_loaned_bytes;
656 
657 typedef struct arc_callback arc_callback_t;
658 
659 struct arc_callback {
660 	void			*acb_private;
661 	arc_done_func_t		*acb_done;
662 	arc_buf_t		*acb_buf;
663 	zio_t			*acb_zio_dummy;
664 	arc_callback_t		*acb_next;
665 };
666 
667 typedef struct arc_write_callback arc_write_callback_t;
668 
669 struct arc_write_callback {
670 	void		*awcb_private;
671 	arc_done_func_t	*awcb_ready;
672 	arc_done_func_t	*awcb_physdone;
673 	arc_done_func_t	*awcb_done;
674 	arc_buf_t	*awcb_buf;
675 };
676 
677 /*
678  * ARC buffers are separated into multiple structs as a memory saving measure:
679  *   - Common fields struct, always defined, and embedded within it:
680  *       - L2-only fields, always allocated but undefined when not in L2ARC
681  *       - L1-only fields, only allocated when in L1ARC
682  *
683  *           Buffer in L1                     Buffer only in L2
684  *    +------------------------+          +------------------------+
685  *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
686  *    |                        |          |                        |
687  *    |                        |          |                        |
688  *    |                        |          |                        |
689  *    +------------------------+          +------------------------+
690  *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
691  *    | (undefined if L1-only) |          |                        |
692  *    +------------------------+          +------------------------+
693  *    | l1arc_buf_hdr_t        |
694  *    |                        |
695  *    |                        |
696  *    |                        |
697  *    |                        |
698  *    +------------------------+
699  *
700  * Because it's possible for the L2ARC to become extremely large, we can wind
701  * up eating a lot of memory in L2ARC buffer headers, so the size of a header
702  * is minimized by only allocating the fields necessary for an L1-cached buffer
703  * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
704  * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
705  * words in pointers. arc_hdr_realloc() is used to switch a header between
706  * these two allocation states.
707  */
708 typedef struct l1arc_buf_hdr {
709 	kmutex_t		b_freeze_lock;
710 #ifdef ZFS_DEBUG
711 	/*
712 	 * used for debugging wtih kmem_flags - by allocating and freeing
713 	 * b_thawed when the buffer is thawed, we get a record of the stack
714 	 * trace that thawed it.
715 	 */
716 	void			*b_thawed;
717 #endif
718 
719 	arc_buf_t		*b_buf;
720 	uint32_t		b_datacnt;
721 	/* for waiting on writes to complete */
722 	kcondvar_t		b_cv;
723 
724 	/* protected by arc state mutex */
725 	arc_state_t		*b_state;
726 	multilist_node_t	b_arc_node;
727 
728 	/* updated atomically */
729 	clock_t			b_arc_access;
730 
731 	/* self protecting */
732 	refcount_t		b_refcnt;
733 
734 	arc_callback_t		*b_acb;
735 	/* temporary buffer holder for in-flight compressed data */
736 	void			*b_tmp_cdata;
737 } l1arc_buf_hdr_t;
738 
739 typedef struct l2arc_dev l2arc_dev_t;
740 
741 typedef struct l2arc_buf_hdr {
742 	/* protected by arc_buf_hdr mutex */
743 	l2arc_dev_t		*b_dev;		/* L2ARC device */
744 	uint64_t		b_daddr;	/* disk address, offset byte */
745 	/* real alloc'd buffer size depending on b_compress applied */
746 	int32_t			b_asize;
747 
748 	list_node_t		b_l2node;
749 } l2arc_buf_hdr_t;
750 
751 struct arc_buf_hdr {
752 	/* protected by hash lock */
753 	dva_t			b_dva;
754 	uint64_t		b_birth;
755 	/*
756 	 * Even though this checksum is only set/verified when a buffer is in
757 	 * the L1 cache, it needs to be in the set of common fields because it
758 	 * must be preserved from the time before a buffer is written out to
759 	 * L2ARC until after it is read back in.
760 	 */
761 	zio_cksum_t		*b_freeze_cksum;
762 
763 	arc_buf_hdr_t		*b_hash_next;
764 	arc_flags_t		b_flags;
765 
766 	/* immutable */
767 	int32_t			b_size;
768 	uint64_t		b_spa;
769 
770 	/* L2ARC fields. Undefined when not in L2ARC. */
771 	l2arc_buf_hdr_t		b_l2hdr;
772 	/* L1ARC fields. Undefined when in l2arc_only state */
773 	l1arc_buf_hdr_t		b_l1hdr;
774 };
775 
776 static arc_buf_t *arc_eviction_list;
777 static arc_buf_hdr_t arc_eviction_hdr;
778 
779 #define	GHOST_STATE(state)	\
780 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
781 	(state) == arc_l2c_only)
782 
783 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
784 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
785 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
786 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
787 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
788 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
789 
790 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
791 #define	HDR_L2COMPRESS(hdr)	((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
792 #define	HDR_L2_READING(hdr)	\
793 	    (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
794 	    ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
795 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
796 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
797 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
798 
799 #define	HDR_ISTYPE_METADATA(hdr)	\
800 	    ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
801 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
802 
803 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
804 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
805 
806 /* For storing compression mode in b_flags */
807 #define	HDR_COMPRESS_OFFSET	24
808 #define	HDR_COMPRESS_NBITS	7
809 
810 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET(hdr->b_flags, \
811 	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS))
812 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET(hdr->b_flags, \
813 	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS, (cmp))
814 
815 /*
816  * Other sizes
817  */
818 
819 #define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
820 #define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
821 
822 /*
823  * Hash table routines
824  */
825 
826 #define	HT_LOCK_PAD	64
827 
828 struct ht_lock {
829 	kmutex_t	ht_lock;
830 #ifdef _KERNEL
831 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
832 #endif
833 };
834 
835 #define	BUF_LOCKS 256
836 typedef struct buf_hash_table {
837 	uint64_t ht_mask;
838 	arc_buf_hdr_t **ht_table;
839 	struct ht_lock ht_locks[BUF_LOCKS];
840 } buf_hash_table_t;
841 
842 static buf_hash_table_t buf_hash_table;
843 
844 #define	BUF_HASH_INDEX(spa, dva, birth) \
845 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
846 #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
847 #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
848 #define	HDR_LOCK(hdr) \
849 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
850 
851 uint64_t zfs_crc64_table[256];
852 
853 /*
854  * Level 2 ARC
855  */
856 
857 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
858 #define	L2ARC_HEADROOM		2			/* num of writes */
859 /*
860  * If we discover during ARC scan any buffers to be compressed, we boost
861  * our headroom for the next scanning cycle by this percentage multiple.
862  */
863 #define	L2ARC_HEADROOM_BOOST	200
864 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
865 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
866 
867 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
868 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
869 
870 /* L2ARC Performance Tunables */
871 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
872 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
873 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
874 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
875 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
876 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
877 boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
878 boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
879 boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
880 
881 /*
882  * L2ARC Internals
883  */
884 struct l2arc_dev {
885 	vdev_t			*l2ad_vdev;	/* vdev */
886 	spa_t			*l2ad_spa;	/* spa */
887 	uint64_t		l2ad_hand;	/* next write location */
888 	uint64_t		l2ad_start;	/* first addr on device */
889 	uint64_t		l2ad_end;	/* last addr on device */
890 	uint64_t		l2ad_evict;	/* last addr eviction reached */
891 	boolean_t		l2ad_first;	/* first sweep through */
892 	boolean_t		l2ad_writing;	/* currently writing */
893 	kmutex_t		l2ad_mtx;	/* lock for buffer list */
894 	list_t			l2ad_buflist;	/* buffer list */
895 	list_node_t		l2ad_node;	/* device list node */
896 };
897 
898 static list_t L2ARC_dev_list;			/* device list */
899 static list_t *l2arc_dev_list;			/* device list pointer */
900 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
901 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
902 static list_t L2ARC_free_on_write;		/* free after write buf list */
903 static list_t *l2arc_free_on_write;		/* free after write list ptr */
904 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
905 static uint64_t l2arc_ndev;			/* number of devices */
906 
907 typedef struct l2arc_read_callback {
908 	arc_buf_t		*l2rcb_buf;		/* read buffer */
909 	spa_t			*l2rcb_spa;		/* spa */
910 	blkptr_t		l2rcb_bp;		/* original blkptr */
911 	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
912 	int			l2rcb_flags;		/* original flags */
913 	enum zio_compress	l2rcb_compress;		/* applied compress */
914 } l2arc_read_callback_t;
915 
916 typedef struct l2arc_write_callback {
917 	l2arc_dev_t	*l2wcb_dev;		/* device info */
918 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
919 } l2arc_write_callback_t;
920 
921 typedef struct l2arc_data_free {
922 	/* protected by l2arc_free_on_write_mtx */
923 	void		*l2df_data;
924 	size_t		l2df_size;
925 	void		(*l2df_func)(void *, size_t);
926 	list_node_t	l2df_list_node;
927 } l2arc_data_free_t;
928 
929 static kmutex_t l2arc_feed_thr_lock;
930 static kcondvar_t l2arc_feed_thr_cv;
931 static uint8_t l2arc_thread_exit;
932 
933 static void arc_get_data_buf(arc_buf_t *);
934 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
935 static boolean_t arc_is_overflowing();
936 static void arc_buf_watch(arc_buf_t *);
937 
938 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
939 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
940 
941 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
942 static void l2arc_read_done(zio_t *);
943 
944 static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
945 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
946 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
947 
948 static uint64_t
949 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
950 {
951 	uint8_t *vdva = (uint8_t *)dva;
952 	uint64_t crc = -1ULL;
953 	int i;
954 
955 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
956 
957 	for (i = 0; i < sizeof (dva_t); i++)
958 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
959 
960 	crc ^= (spa>>8) ^ birth;
961 
962 	return (crc);
963 }
964 
965 #define	BUF_EMPTY(buf)						\
966 	((buf)->b_dva.dva_word[0] == 0 &&			\
967 	(buf)->b_dva.dva_word[1] == 0)
968 
969 #define	BUF_EQUAL(spa, dva, birth, buf)				\
970 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
971 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
972 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
973 
974 static void
975 buf_discard_identity(arc_buf_hdr_t *hdr)
976 {
977 	hdr->b_dva.dva_word[0] = 0;
978 	hdr->b_dva.dva_word[1] = 0;
979 	hdr->b_birth = 0;
980 }
981 
982 static arc_buf_hdr_t *
983 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
984 {
985 	const dva_t *dva = BP_IDENTITY(bp);
986 	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
987 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
988 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
989 	arc_buf_hdr_t *hdr;
990 
991 	mutex_enter(hash_lock);
992 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
993 	    hdr = hdr->b_hash_next) {
994 		if (BUF_EQUAL(spa, dva, birth, hdr)) {
995 			*lockp = hash_lock;
996 			return (hdr);
997 		}
998 	}
999 	mutex_exit(hash_lock);
1000 	*lockp = NULL;
1001 	return (NULL);
1002 }
1003 
1004 /*
1005  * Insert an entry into the hash table.  If there is already an element
1006  * equal to elem in the hash table, then the already existing element
1007  * will be returned and the new element will not be inserted.
1008  * Otherwise returns NULL.
1009  * If lockp == NULL, the caller is assumed to already hold the hash lock.
1010  */
1011 static arc_buf_hdr_t *
1012 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1013 {
1014 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1015 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1016 	arc_buf_hdr_t *fhdr;
1017 	uint32_t i;
1018 
1019 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1020 	ASSERT(hdr->b_birth != 0);
1021 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1022 
1023 	if (lockp != NULL) {
1024 		*lockp = hash_lock;
1025 		mutex_enter(hash_lock);
1026 	} else {
1027 		ASSERT(MUTEX_HELD(hash_lock));
1028 	}
1029 
1030 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1031 	    fhdr = fhdr->b_hash_next, i++) {
1032 		if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1033 			return (fhdr);
1034 	}
1035 
1036 	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1037 	buf_hash_table.ht_table[idx] = hdr;
1038 	hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
1039 
1040 	/* collect some hash table performance data */
1041 	if (i > 0) {
1042 		ARCSTAT_BUMP(arcstat_hash_collisions);
1043 		if (i == 1)
1044 			ARCSTAT_BUMP(arcstat_hash_chains);
1045 
1046 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1047 	}
1048 
1049 	ARCSTAT_BUMP(arcstat_hash_elements);
1050 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1051 
1052 	return (NULL);
1053 }
1054 
1055 static void
1056 buf_hash_remove(arc_buf_hdr_t *hdr)
1057 {
1058 	arc_buf_hdr_t *fhdr, **hdrp;
1059 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1060 
1061 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1062 	ASSERT(HDR_IN_HASH_TABLE(hdr));
1063 
1064 	hdrp = &buf_hash_table.ht_table[idx];
1065 	while ((fhdr = *hdrp) != hdr) {
1066 		ASSERT(fhdr != NULL);
1067 		hdrp = &fhdr->b_hash_next;
1068 	}
1069 	*hdrp = hdr->b_hash_next;
1070 	hdr->b_hash_next = NULL;
1071 	hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE;
1072 
1073 	/* collect some hash table performance data */
1074 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1075 
1076 	if (buf_hash_table.ht_table[idx] &&
1077 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1078 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1079 }
1080 
1081 /*
1082  * Global data structures and functions for the buf kmem cache.
1083  */
1084 static kmem_cache_t *hdr_full_cache;
1085 static kmem_cache_t *hdr_l2only_cache;
1086 static kmem_cache_t *buf_cache;
1087 
1088 static void
1089 buf_fini(void)
1090 {
1091 	int i;
1092 
1093 	kmem_free(buf_hash_table.ht_table,
1094 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1095 	for (i = 0; i < BUF_LOCKS; i++)
1096 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1097 	kmem_cache_destroy(hdr_full_cache);
1098 	kmem_cache_destroy(hdr_l2only_cache);
1099 	kmem_cache_destroy(buf_cache);
1100 }
1101 
1102 /*
1103  * Constructor callback - called when the cache is empty
1104  * and a new buf is requested.
1105  */
1106 /* ARGSUSED */
1107 static int
1108 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1109 {
1110 	arc_buf_hdr_t *hdr = vbuf;
1111 
1112 	bzero(hdr, HDR_FULL_SIZE);
1113 	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1114 	refcount_create(&hdr->b_l1hdr.b_refcnt);
1115 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1116 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1117 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1118 
1119 	return (0);
1120 }
1121 
1122 /* ARGSUSED */
1123 static int
1124 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1125 {
1126 	arc_buf_hdr_t *hdr = vbuf;
1127 
1128 	bzero(hdr, HDR_L2ONLY_SIZE);
1129 	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1130 
1131 	return (0);
1132 }
1133 
1134 /* ARGSUSED */
1135 static int
1136 buf_cons(void *vbuf, void *unused, int kmflag)
1137 {
1138 	arc_buf_t *buf = vbuf;
1139 
1140 	bzero(buf, sizeof (arc_buf_t));
1141 	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1142 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1143 
1144 	return (0);
1145 }
1146 
1147 /*
1148  * Destructor callback - called when a cached buf is
1149  * no longer required.
1150  */
1151 /* ARGSUSED */
1152 static void
1153 hdr_full_dest(void *vbuf, void *unused)
1154 {
1155 	arc_buf_hdr_t *hdr = vbuf;
1156 
1157 	ASSERT(BUF_EMPTY(hdr));
1158 	cv_destroy(&hdr->b_l1hdr.b_cv);
1159 	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1160 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1161 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1162 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1163 }
1164 
1165 /* ARGSUSED */
1166 static void
1167 hdr_l2only_dest(void *vbuf, void *unused)
1168 {
1169 	arc_buf_hdr_t *hdr = vbuf;
1170 
1171 	ASSERT(BUF_EMPTY(hdr));
1172 	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1173 }
1174 
1175 /* ARGSUSED */
1176 static void
1177 buf_dest(void *vbuf, void *unused)
1178 {
1179 	arc_buf_t *buf = vbuf;
1180 
1181 	mutex_destroy(&buf->b_evict_lock);
1182 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1183 }
1184 
1185 /*
1186  * Reclaim callback -- invoked when memory is low.
1187  */
1188 /* ARGSUSED */
1189 static void
1190 hdr_recl(void *unused)
1191 {
1192 	dprintf("hdr_recl called\n");
1193 	/*
1194 	 * umem calls the reclaim func when we destroy the buf cache,
1195 	 * which is after we do arc_fini().
1196 	 */
1197 	if (!arc_dead)
1198 		cv_signal(&arc_reclaim_thread_cv);
1199 }
1200 
1201 static void
1202 buf_init(void)
1203 {
1204 	uint64_t *ct;
1205 	uint64_t hsize = 1ULL << 12;
1206 	int i, j;
1207 
1208 	/*
1209 	 * The hash table is big enough to fill all of physical memory
1210 	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1211 	 * By default, the table will take up
1212 	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1213 	 */
1214 	while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
1215 		hsize <<= 1;
1216 retry:
1217 	buf_hash_table.ht_mask = hsize - 1;
1218 	buf_hash_table.ht_table =
1219 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1220 	if (buf_hash_table.ht_table == NULL) {
1221 		ASSERT(hsize > (1ULL << 8));
1222 		hsize >>= 1;
1223 		goto retry;
1224 	}
1225 
1226 	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1227 	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1228 	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1229 	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1230 	    NULL, NULL, 0);
1231 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1232 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1233 
1234 	for (i = 0; i < 256; i++)
1235 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1236 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1237 
1238 	for (i = 0; i < BUF_LOCKS; i++) {
1239 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1240 		    NULL, MUTEX_DEFAULT, NULL);
1241 	}
1242 }
1243 
1244 /*
1245  * Transition between the two allocation states for the arc_buf_hdr struct.
1246  * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
1247  * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
1248  * version is used when a cache buffer is only in the L2ARC in order to reduce
1249  * memory usage.
1250  */
1251 static arc_buf_hdr_t *
1252 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
1253 {
1254 	ASSERT(HDR_HAS_L2HDR(hdr));
1255 
1256 	arc_buf_hdr_t *nhdr;
1257 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1258 
1259 	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
1260 	    (old == hdr_l2only_cache && new == hdr_full_cache));
1261 
1262 	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
1263 
1264 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
1265 	buf_hash_remove(hdr);
1266 
1267 	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
1268 	if (new == hdr_full_cache) {
1269 		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1270 		/*
1271 		 * arc_access and arc_change_state need to be aware that a
1272 		 * header has just come out of L2ARC, so we set its state to
1273 		 * l2c_only even though it's about to change.
1274 		 */
1275 		nhdr->b_l1hdr.b_state = arc_l2c_only;
1276 
1277 		/* Verify previous threads set to NULL before freeing */
1278 		ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1279 	} else {
1280 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
1281 		ASSERT0(hdr->b_l1hdr.b_datacnt);
1282 
1283 		/*
1284 		 * If we've reached here, We must have been called from
1285 		 * arc_evict_hdr(), as such we should have already been
1286 		 * removed from any ghost list we were previously on
1287 		 * (which protects us from racing with arc_evict_state),
1288 		 * thus no locking is needed during this check.
1289 		 */
1290 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1291 
1292 		/*
1293 		 * A buffer must not be moved into the arc_l2c_only
1294 		 * state if it's not finished being written out to the
1295 		 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
1296 		 * might try to be accessed, even though it was removed.
1297 		 */
1298 		VERIFY(!HDR_L2_WRITING(hdr));
1299 		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1300 
1301 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
1302 	}
1303 	/*
1304 	 * The header has been reallocated so we need to re-insert it into any
1305 	 * lists it was on.
1306 	 */
1307 	(void) buf_hash_insert(nhdr, NULL);
1308 
1309 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
1310 
1311 	mutex_enter(&dev->l2ad_mtx);
1312 
1313 	/*
1314 	 * We must place the realloc'ed header back into the list at
1315 	 * the same spot. Otherwise, if it's placed earlier in the list,
1316 	 * l2arc_write_buffers() could find it during the function's
1317 	 * write phase, and try to write it out to the l2arc.
1318 	 */
1319 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
1320 	list_remove(&dev->l2ad_buflist, hdr);
1321 
1322 	mutex_exit(&dev->l2ad_mtx);
1323 
1324 	buf_discard_identity(hdr);
1325 	hdr->b_freeze_cksum = NULL;
1326 	kmem_cache_free(old, hdr);
1327 
1328 	return (nhdr);
1329 }
1330 
1331 
1332 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
1333 
1334 static void
1335 arc_cksum_verify(arc_buf_t *buf)
1336 {
1337 	zio_cksum_t zc;
1338 
1339 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1340 		return;
1341 
1342 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1343 	if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
1344 		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1345 		return;
1346 	}
1347 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1348 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
1349 		panic("buffer modified while frozen!");
1350 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1351 }
1352 
1353 static int
1354 arc_cksum_equal(arc_buf_t *buf)
1355 {
1356 	zio_cksum_t zc;
1357 	int equal;
1358 
1359 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1360 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1361 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
1362 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1363 
1364 	return (equal);
1365 }
1366 
1367 static void
1368 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1369 {
1370 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1371 		return;
1372 
1373 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1374 	if (buf->b_hdr->b_freeze_cksum != NULL) {
1375 		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1376 		return;
1377 	}
1378 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1379 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1380 	    buf->b_hdr->b_freeze_cksum);
1381 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1382 	arc_buf_watch(buf);
1383 }
1384 
1385 #ifndef _KERNEL
1386 typedef struct procctl {
1387 	long cmd;
1388 	prwatch_t prwatch;
1389 } procctl_t;
1390 #endif
1391 
1392 /* ARGSUSED */
1393 static void
1394 arc_buf_unwatch(arc_buf_t *buf)
1395 {
1396 #ifndef _KERNEL
1397 	if (arc_watch) {
1398 		int result;
1399 		procctl_t ctl;
1400 		ctl.cmd = PCWATCH;
1401 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1402 		ctl.prwatch.pr_size = 0;
1403 		ctl.prwatch.pr_wflags = 0;
1404 		result = write(arc_procfd, &ctl, sizeof (ctl));
1405 		ASSERT3U(result, ==, sizeof (ctl));
1406 	}
1407 #endif
1408 }
1409 
1410 /* ARGSUSED */
1411 static void
1412 arc_buf_watch(arc_buf_t *buf)
1413 {
1414 #ifndef _KERNEL
1415 	if (arc_watch) {
1416 		int result;
1417 		procctl_t ctl;
1418 		ctl.cmd = PCWATCH;
1419 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1420 		ctl.prwatch.pr_size = buf->b_hdr->b_size;
1421 		ctl.prwatch.pr_wflags = WA_WRITE;
1422 		result = write(arc_procfd, &ctl, sizeof (ctl));
1423 		ASSERT3U(result, ==, sizeof (ctl));
1424 	}
1425 #endif
1426 }
1427 
1428 static arc_buf_contents_t
1429 arc_buf_type(arc_buf_hdr_t *hdr)
1430 {
1431 	if (HDR_ISTYPE_METADATA(hdr)) {
1432 		return (ARC_BUFC_METADATA);
1433 	} else {
1434 		return (ARC_BUFC_DATA);
1435 	}
1436 }
1437 
1438 static uint32_t
1439 arc_bufc_to_flags(arc_buf_contents_t type)
1440 {
1441 	switch (type) {
1442 	case ARC_BUFC_DATA:
1443 		/* metadata field is 0 if buffer contains normal data */
1444 		return (0);
1445 	case ARC_BUFC_METADATA:
1446 		return (ARC_FLAG_BUFC_METADATA);
1447 	default:
1448 		break;
1449 	}
1450 	panic("undefined ARC buffer type!");
1451 	return ((uint32_t)-1);
1452 }
1453 
1454 void
1455 arc_buf_thaw(arc_buf_t *buf)
1456 {
1457 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1458 		if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
1459 			panic("modifying non-anon buffer!");
1460 		if (HDR_IO_IN_PROGRESS(buf->b_hdr))
1461 			panic("modifying buffer while i/o in progress!");
1462 		arc_cksum_verify(buf);
1463 	}
1464 
1465 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1466 	if (buf->b_hdr->b_freeze_cksum != NULL) {
1467 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1468 		buf->b_hdr->b_freeze_cksum = NULL;
1469 	}
1470 
1471 #ifdef ZFS_DEBUG
1472 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1473 		if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
1474 			kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
1475 		buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1476 	}
1477 #endif
1478 
1479 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1480 
1481 	arc_buf_unwatch(buf);
1482 }
1483 
1484 void
1485 arc_buf_freeze(arc_buf_t *buf)
1486 {
1487 	kmutex_t *hash_lock;
1488 
1489 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1490 		return;
1491 
1492 	hash_lock = HDR_LOCK(buf->b_hdr);
1493 	mutex_enter(hash_lock);
1494 
1495 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1496 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
1497 	arc_cksum_compute(buf, B_FALSE);
1498 	mutex_exit(hash_lock);
1499 
1500 }
1501 
1502 static void
1503 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1504 {
1505 	ASSERT(HDR_HAS_L1HDR(hdr));
1506 	ASSERT(MUTEX_HELD(hash_lock));
1507 	arc_state_t *state = hdr->b_l1hdr.b_state;
1508 
1509 	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
1510 	    (state != arc_anon)) {
1511 		/* We don't use the L2-only state list. */
1512 		if (state != arc_l2c_only) {
1513 			arc_buf_contents_t type = arc_buf_type(hdr);
1514 			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
1515 			multilist_t *list = &state->arcs_list[type];
1516 			uint64_t *size = &state->arcs_lsize[type];
1517 
1518 			multilist_remove(list, hdr);
1519 
1520 			if (GHOST_STATE(state)) {
1521 				ASSERT0(hdr->b_l1hdr.b_datacnt);
1522 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
1523 				delta = hdr->b_size;
1524 			}
1525 			ASSERT(delta > 0);
1526 			ASSERT3U(*size, >=, delta);
1527 			atomic_add_64(size, -delta);
1528 		}
1529 		/* remove the prefetch flag if we get a reference */
1530 		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
1531 	}
1532 }
1533 
1534 static int
1535 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1536 {
1537 	int cnt;
1538 	arc_state_t *state = hdr->b_l1hdr.b_state;
1539 
1540 	ASSERT(HDR_HAS_L1HDR(hdr));
1541 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1542 	ASSERT(!GHOST_STATE(state));
1543 
1544 	/*
1545 	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
1546 	 * check to prevent usage of the arc_l2c_only list.
1547 	 */
1548 	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
1549 	    (state != arc_anon)) {
1550 		arc_buf_contents_t type = arc_buf_type(hdr);
1551 		multilist_t *list = &state->arcs_list[type];
1552 		uint64_t *size = &state->arcs_lsize[type];
1553 
1554 		multilist_insert(list, hdr);
1555 
1556 		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
1557 		atomic_add_64(size, hdr->b_size *
1558 		    hdr->b_l1hdr.b_datacnt);
1559 	}
1560 	return (cnt);
1561 }
1562 
1563 /*
1564  * Move the supplied buffer to the indicated state. The hash lock
1565  * for the buffer must be held by the caller.
1566  */
1567 static void
1568 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
1569     kmutex_t *hash_lock)
1570 {
1571 	arc_state_t *old_state;
1572 	int64_t refcnt;
1573 	uint32_t datacnt;
1574 	uint64_t from_delta, to_delta;
1575 	arc_buf_contents_t buftype = arc_buf_type(hdr);
1576 
1577 	/*
1578 	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
1579 	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
1580 	 * L1 hdr doesn't always exist when we change state to arc_anon before
1581 	 * destroying a header, in which case reallocating to add the L1 hdr is
1582 	 * pointless.
1583 	 */
1584 	if (HDR_HAS_L1HDR(hdr)) {
1585 		old_state = hdr->b_l1hdr.b_state;
1586 		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
1587 		datacnt = hdr->b_l1hdr.b_datacnt;
1588 	} else {
1589 		old_state = arc_l2c_only;
1590 		refcnt = 0;
1591 		datacnt = 0;
1592 	}
1593 
1594 	ASSERT(MUTEX_HELD(hash_lock));
1595 	ASSERT3P(new_state, !=, old_state);
1596 	ASSERT(refcnt == 0 || datacnt > 0);
1597 	ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
1598 	ASSERT(old_state != arc_anon || datacnt <= 1);
1599 
1600 	from_delta = to_delta = datacnt * hdr->b_size;
1601 
1602 	/*
1603 	 * If this buffer is evictable, transfer it from the
1604 	 * old state list to the new state list.
1605 	 */
1606 	if (refcnt == 0) {
1607 		if (old_state != arc_anon && old_state != arc_l2c_only) {
1608 			uint64_t *size = &old_state->arcs_lsize[buftype];
1609 
1610 			ASSERT(HDR_HAS_L1HDR(hdr));
1611 			multilist_remove(&old_state->arcs_list[buftype], hdr);
1612 
1613 			/*
1614 			 * If prefetching out of the ghost cache,
1615 			 * we will have a non-zero datacnt.
1616 			 */
1617 			if (GHOST_STATE(old_state) && datacnt == 0) {
1618 				/* ghost elements have a ghost size */
1619 				ASSERT(hdr->b_l1hdr.b_buf == NULL);
1620 				from_delta = hdr->b_size;
1621 			}
1622 			ASSERT3U(*size, >=, from_delta);
1623 			atomic_add_64(size, -from_delta);
1624 		}
1625 		if (new_state != arc_anon && new_state != arc_l2c_only) {
1626 			uint64_t *size = &new_state->arcs_lsize[buftype];
1627 
1628 			/*
1629 			 * An L1 header always exists here, since if we're
1630 			 * moving to some L1-cached state (i.e. not l2c_only or
1631 			 * anonymous), we realloc the header to add an L1hdr
1632 			 * beforehand.
1633 			 */
1634 			ASSERT(HDR_HAS_L1HDR(hdr));
1635 			multilist_insert(&new_state->arcs_list[buftype], hdr);
1636 
1637 			/* ghost elements have a ghost size */
1638 			if (GHOST_STATE(new_state)) {
1639 				ASSERT0(datacnt);
1640 				ASSERT(hdr->b_l1hdr.b_buf == NULL);
1641 				to_delta = hdr->b_size;
1642 			}
1643 			atomic_add_64(size, to_delta);
1644 		}
1645 	}
1646 
1647 	ASSERT(!BUF_EMPTY(hdr));
1648 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
1649 		buf_hash_remove(hdr);
1650 
1651 	/* adjust state sizes (ignore arc_l2c_only) */
1652 
1653 	if (to_delta && new_state != arc_l2c_only) {
1654 		ASSERT(HDR_HAS_L1HDR(hdr));
1655 		if (GHOST_STATE(new_state)) {
1656 			ASSERT0(datacnt);
1657 
1658 			/*
1659 			 * We moving a header to a ghost state, we first
1660 			 * remove all arc buffers. Thus, we'll have a
1661 			 * datacnt of zero, and no arc buffer to use for
1662 			 * the reference. As a result, we use the arc
1663 			 * header pointer for the reference.
1664 			 */
1665 			(void) refcount_add_many(&new_state->arcs_size,
1666 			    hdr->b_size, hdr);
1667 		} else {
1668 			ASSERT3U(datacnt, !=, 0);
1669 
1670 			/*
1671 			 * Each individual buffer holds a unique reference,
1672 			 * thus we must remove each of these references one
1673 			 * at a time.
1674 			 */
1675 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1676 			    buf = buf->b_next) {
1677 				(void) refcount_add_many(&new_state->arcs_size,
1678 				    hdr->b_size, buf);
1679 			}
1680 		}
1681 	}
1682 
1683 	if (from_delta && old_state != arc_l2c_only) {
1684 		ASSERT(HDR_HAS_L1HDR(hdr));
1685 		if (GHOST_STATE(old_state)) {
1686 			/*
1687 			 * When moving a header off of a ghost state,
1688 			 * there's the possibility for datacnt to be
1689 			 * non-zero. This is because we first add the
1690 			 * arc buffer to the header prior to changing
1691 			 * the header's state. Since we used the header
1692 			 * for the reference when putting the header on
1693 			 * the ghost state, we must balance that and use
1694 			 * the header when removing off the ghost state
1695 			 * (even though datacnt is non zero).
1696 			 */
1697 
1698 			IMPLY(datacnt == 0, new_state == arc_anon ||
1699 			    new_state == arc_l2c_only);
1700 
1701 			(void) refcount_remove_many(&old_state->arcs_size,
1702 			    hdr->b_size, hdr);
1703 		} else {
1704 			ASSERT3P(datacnt, !=, 0);
1705 
1706 			/*
1707 			 * Each individual buffer holds a unique reference,
1708 			 * thus we must remove each of these references one
1709 			 * at a time.
1710 			 */
1711 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
1712 			    buf = buf->b_next) {
1713 				(void) refcount_remove_many(
1714 				    &old_state->arcs_size, hdr->b_size, buf);
1715 			}
1716 		}
1717 	}
1718 
1719 	if (HDR_HAS_L1HDR(hdr))
1720 		hdr->b_l1hdr.b_state = new_state;
1721 
1722 	/*
1723 	 * L2 headers should never be on the L2 state list since they don't
1724 	 * have L1 headers allocated.
1725 	 */
1726 	ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
1727 	    multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
1728 }
1729 
1730 void
1731 arc_space_consume(uint64_t space, arc_space_type_t type)
1732 {
1733 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1734 
1735 	switch (type) {
1736 	case ARC_SPACE_DATA:
1737 		ARCSTAT_INCR(arcstat_data_size, space);
1738 		break;
1739 	case ARC_SPACE_META:
1740 		ARCSTAT_INCR(arcstat_metadata_size, space);
1741 		break;
1742 	case ARC_SPACE_OTHER:
1743 		ARCSTAT_INCR(arcstat_other_size, space);
1744 		break;
1745 	case ARC_SPACE_HDRS:
1746 		ARCSTAT_INCR(arcstat_hdr_size, space);
1747 		break;
1748 	case ARC_SPACE_L2HDRS:
1749 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1750 		break;
1751 	}
1752 
1753 	if (type != ARC_SPACE_DATA)
1754 		ARCSTAT_INCR(arcstat_meta_used, space);
1755 
1756 	atomic_add_64(&arc_size, space);
1757 }
1758 
1759 void
1760 arc_space_return(uint64_t space, arc_space_type_t type)
1761 {
1762 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1763 
1764 	switch (type) {
1765 	case ARC_SPACE_DATA:
1766 		ARCSTAT_INCR(arcstat_data_size, -space);
1767 		break;
1768 	case ARC_SPACE_META:
1769 		ARCSTAT_INCR(arcstat_metadata_size, -space);
1770 		break;
1771 	case ARC_SPACE_OTHER:
1772 		ARCSTAT_INCR(arcstat_other_size, -space);
1773 		break;
1774 	case ARC_SPACE_HDRS:
1775 		ARCSTAT_INCR(arcstat_hdr_size, -space);
1776 		break;
1777 	case ARC_SPACE_L2HDRS:
1778 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1779 		break;
1780 	}
1781 
1782 	if (type != ARC_SPACE_DATA) {
1783 		ASSERT(arc_meta_used >= space);
1784 		if (arc_meta_max < arc_meta_used)
1785 			arc_meta_max = arc_meta_used;
1786 		ARCSTAT_INCR(arcstat_meta_used, -space);
1787 	}
1788 
1789 	ASSERT(arc_size >= space);
1790 	atomic_add_64(&arc_size, -space);
1791 }
1792 
1793 arc_buf_t *
1794 arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
1795 {
1796 	arc_buf_hdr_t *hdr;
1797 	arc_buf_t *buf;
1798 
1799 	ASSERT3U(size, >, 0);
1800 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
1801 	ASSERT(BUF_EMPTY(hdr));
1802 	ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
1803 	hdr->b_size = size;
1804 	hdr->b_spa = spa_load_guid(spa);
1805 
1806 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1807 	buf->b_hdr = hdr;
1808 	buf->b_data = NULL;
1809 	buf->b_efunc = NULL;
1810 	buf->b_private = NULL;
1811 	buf->b_next = NULL;
1812 
1813 	hdr->b_flags = arc_bufc_to_flags(type);
1814 	hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1815 
1816 	hdr->b_l1hdr.b_buf = buf;
1817 	hdr->b_l1hdr.b_state = arc_anon;
1818 	hdr->b_l1hdr.b_arc_access = 0;
1819 	hdr->b_l1hdr.b_datacnt = 1;
1820 	hdr->b_l1hdr.b_tmp_cdata = NULL;
1821 
1822 	arc_get_data_buf(buf);
1823 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
1824 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1825 
1826 	return (buf);
1827 }
1828 
1829 static char *arc_onloan_tag = "onloan";
1830 
1831 /*
1832  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1833  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1834  * buffers must be returned to the arc before they can be used by the DMU or
1835  * freed.
1836  */
1837 arc_buf_t *
1838 arc_loan_buf(spa_t *spa, int size)
1839 {
1840 	arc_buf_t *buf;
1841 
1842 	buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1843 
1844 	atomic_add_64(&arc_loaned_bytes, size);
1845 	return (buf);
1846 }
1847 
1848 /*
1849  * Return a loaned arc buffer to the arc.
1850  */
1851 void
1852 arc_return_buf(arc_buf_t *buf, void *tag)
1853 {
1854 	arc_buf_hdr_t *hdr = buf->b_hdr;
1855 
1856 	ASSERT(buf->b_data != NULL);
1857 	ASSERT(HDR_HAS_L1HDR(hdr));
1858 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1859 	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1860 
1861 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1862 }
1863 
1864 /* Detach an arc_buf from a dbuf (tag) */
1865 void
1866 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1867 {
1868 	arc_buf_hdr_t *hdr = buf->b_hdr;
1869 
1870 	ASSERT(buf->b_data != NULL);
1871 	ASSERT(HDR_HAS_L1HDR(hdr));
1872 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1873 	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
1874 	buf->b_efunc = NULL;
1875 	buf->b_private = NULL;
1876 
1877 	atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1878 }
1879 
1880 static arc_buf_t *
1881 arc_buf_clone(arc_buf_t *from)
1882 {
1883 	arc_buf_t *buf;
1884 	arc_buf_hdr_t *hdr = from->b_hdr;
1885 	uint64_t size = hdr->b_size;
1886 
1887 	ASSERT(HDR_HAS_L1HDR(hdr));
1888 	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
1889 
1890 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1891 	buf->b_hdr = hdr;
1892 	buf->b_data = NULL;
1893 	buf->b_efunc = NULL;
1894 	buf->b_private = NULL;
1895 	buf->b_next = hdr->b_l1hdr.b_buf;
1896 	hdr->b_l1hdr.b_buf = buf;
1897 	arc_get_data_buf(buf);
1898 	bcopy(from->b_data, buf->b_data, size);
1899 
1900 	/*
1901 	 * This buffer already exists in the arc so create a duplicate
1902 	 * copy for the caller.  If the buffer is associated with user data
1903 	 * then track the size and number of duplicates.  These stats will be
1904 	 * updated as duplicate buffers are created and destroyed.
1905 	 */
1906 	if (HDR_ISTYPE_DATA(hdr)) {
1907 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
1908 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1909 	}
1910 	hdr->b_l1hdr.b_datacnt += 1;
1911 	return (buf);
1912 }
1913 
1914 void
1915 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1916 {
1917 	arc_buf_hdr_t *hdr;
1918 	kmutex_t *hash_lock;
1919 
1920 	/*
1921 	 * Check to see if this buffer is evicted.  Callers
1922 	 * must verify b_data != NULL to know if the add_ref
1923 	 * was successful.
1924 	 */
1925 	mutex_enter(&buf->b_evict_lock);
1926 	if (buf->b_data == NULL) {
1927 		mutex_exit(&buf->b_evict_lock);
1928 		return;
1929 	}
1930 	hash_lock = HDR_LOCK(buf->b_hdr);
1931 	mutex_enter(hash_lock);
1932 	hdr = buf->b_hdr;
1933 	ASSERT(HDR_HAS_L1HDR(hdr));
1934 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1935 	mutex_exit(&buf->b_evict_lock);
1936 
1937 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
1938 	    hdr->b_l1hdr.b_state == arc_mfu);
1939 
1940 	add_reference(hdr, hash_lock, tag);
1941 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1942 	arc_access(hdr, hash_lock);
1943 	mutex_exit(hash_lock);
1944 	ARCSTAT_BUMP(arcstat_hits);
1945 	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
1946 	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
1947 	    data, metadata, hits);
1948 }
1949 
1950 static void
1951 arc_buf_free_on_write(void *data, size_t size,
1952     void (*free_func)(void *, size_t))
1953 {
1954 	l2arc_data_free_t *df;
1955 
1956 	df = kmem_alloc(sizeof (*df), KM_SLEEP);
1957 	df->l2df_data = data;
1958 	df->l2df_size = size;
1959 	df->l2df_func = free_func;
1960 	mutex_enter(&l2arc_free_on_write_mtx);
1961 	list_insert_head(l2arc_free_on_write, df);
1962 	mutex_exit(&l2arc_free_on_write_mtx);
1963 }
1964 
1965 /*
1966  * Free the arc data buffer.  If it is an l2arc write in progress,
1967  * the buffer is placed on l2arc_free_on_write to be freed later.
1968  */
1969 static void
1970 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1971 {
1972 	arc_buf_hdr_t *hdr = buf->b_hdr;
1973 
1974 	if (HDR_L2_WRITING(hdr)) {
1975 		arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
1976 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
1977 	} else {
1978 		free_func(buf->b_data, hdr->b_size);
1979 	}
1980 }
1981 
1982 static void
1983 arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
1984 {
1985 	ASSERT(HDR_HAS_L2HDR(hdr));
1986 	ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
1987 
1988 	/*
1989 	 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
1990 	 * that doesn't exist, the header is in the arc_l2c_only state,
1991 	 * and there isn't anything to free (it's already been freed).
1992 	 */
1993 	if (!HDR_HAS_L1HDR(hdr))
1994 		return;
1995 
1996 	/*
1997 	 * The header isn't being written to the l2arc device, thus it
1998 	 * shouldn't have a b_tmp_cdata to free.
1999 	 */
2000 	if (!HDR_L2_WRITING(hdr)) {
2001 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2002 		return;
2003 	}
2004 
2005 	/*
2006 	 * The header does not have compression enabled. This can be due
2007 	 * to the buffer not being compressible, or because we're
2008 	 * freeing the buffer before the second phase of
2009 	 * l2arc_write_buffer() has started (which does the compression
2010 	 * step). In either case, b_tmp_cdata does not point to a
2011 	 * separately compressed buffer, so there's nothing to free (it
2012 	 * points to the same buffer as the arc_buf_t's b_data field).
2013 	 */
2014 	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
2015 		hdr->b_l1hdr.b_tmp_cdata = NULL;
2016 		return;
2017 	}
2018 
2019 	/*
2020 	 * There's nothing to free since the buffer was all zero's and
2021 	 * compressed to a zero length buffer.
2022 	 */
2023 	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_EMPTY) {
2024 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
2025 		return;
2026 	}
2027 
2028 	ASSERT(L2ARC_IS_VALID_COMPRESS(HDR_GET_COMPRESS(hdr)));
2029 
2030 	arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
2031 	    hdr->b_size, zio_data_buf_free);
2032 
2033 	ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
2034 	hdr->b_l1hdr.b_tmp_cdata = NULL;
2035 }
2036 
2037 /*
2038  * Free up buf->b_data and if 'remove' is set, then pull the
2039  * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
2040  */
2041 static void
2042 arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
2043 {
2044 	arc_buf_t **bufp;
2045 
2046 	/* free up data associated with the buf */
2047 	if (buf->b_data != NULL) {
2048 		arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
2049 		uint64_t size = buf->b_hdr->b_size;
2050 		arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
2051 
2052 		arc_cksum_verify(buf);
2053 		arc_buf_unwatch(buf);
2054 
2055 		if (type == ARC_BUFC_METADATA) {
2056 			arc_buf_data_free(buf, zio_buf_free);
2057 			arc_space_return(size, ARC_SPACE_META);
2058 		} else {
2059 			ASSERT(type == ARC_BUFC_DATA);
2060 			arc_buf_data_free(buf, zio_data_buf_free);
2061 			arc_space_return(size, ARC_SPACE_DATA);
2062 		}
2063 
2064 		/* protected by hash lock, if in the hash table */
2065 		if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
2066 			uint64_t *cnt = &state->arcs_lsize[type];
2067 
2068 			ASSERT(refcount_is_zero(
2069 			    &buf->b_hdr->b_l1hdr.b_refcnt));
2070 			ASSERT(state != arc_anon && state != arc_l2c_only);
2071 
2072 			ASSERT3U(*cnt, >=, size);
2073 			atomic_add_64(cnt, -size);
2074 		}
2075 
2076 		(void) refcount_remove_many(&state->arcs_size, size, buf);
2077 		buf->b_data = NULL;
2078 
2079 		/*
2080 		 * If we're destroying a duplicate buffer make sure
2081 		 * that the appropriate statistics are updated.
2082 		 */
2083 		if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
2084 		    HDR_ISTYPE_DATA(buf->b_hdr)) {
2085 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
2086 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
2087 		}
2088 		ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
2089 		buf->b_hdr->b_l1hdr.b_datacnt -= 1;
2090 	}
2091 
2092 	/* only remove the buf if requested */
2093 	if (!remove)
2094 		return;
2095 
2096 	/* remove the buf from the hdr list */
2097 	for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
2098 	    bufp = &(*bufp)->b_next)
2099 		continue;
2100 	*bufp = buf->b_next;
2101 	buf->b_next = NULL;
2102 
2103 	ASSERT(buf->b_efunc == NULL);
2104 
2105 	/* clean up the buf */
2106 	buf->b_hdr = NULL;
2107 	kmem_cache_free(buf_cache, buf);
2108 }
2109 
2110 static void
2111 arc_hdr_destroy(arc_buf_hdr_t *hdr)
2112 {
2113 	if (HDR_HAS_L1HDR(hdr)) {
2114 		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
2115 		    hdr->b_l1hdr.b_datacnt > 0);
2116 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2117 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2118 	}
2119 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2120 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
2121 
2122 	if (HDR_HAS_L2HDR(hdr)) {
2123 		l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
2124 		boolean_t buflist_held = MUTEX_HELD(&l2hdr->b_dev->l2ad_mtx);
2125 
2126 		if (!buflist_held) {
2127 			mutex_enter(&l2hdr->b_dev->l2ad_mtx);
2128 			l2hdr = &hdr->b_l2hdr;
2129 		}
2130 
2131 		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
2132 
2133 		/*
2134 		 * We don't want to leak the b_tmp_cdata buffer that was
2135 		 * allocated in l2arc_write_buffers()
2136 		 */
2137 		arc_buf_l2_cdata_free(hdr);
2138 
2139 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
2140 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
2141 
2142 		if (!buflist_held)
2143 			mutex_exit(&l2hdr->b_dev->l2ad_mtx);
2144 
2145 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
2146 	}
2147 
2148 	if (!BUF_EMPTY(hdr))
2149 		buf_discard_identity(hdr);
2150 
2151 	if (hdr->b_freeze_cksum != NULL) {
2152 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
2153 		hdr->b_freeze_cksum = NULL;
2154 	}
2155 
2156 	if (HDR_HAS_L1HDR(hdr)) {
2157 		while (hdr->b_l1hdr.b_buf) {
2158 			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2159 
2160 			if (buf->b_efunc != NULL) {
2161 				mutex_enter(&arc_user_evicts_lock);
2162 				mutex_enter(&buf->b_evict_lock);
2163 				ASSERT(buf->b_hdr != NULL);
2164 				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
2165 				hdr->b_l1hdr.b_buf = buf->b_next;
2166 				buf->b_hdr = &arc_eviction_hdr;
2167 				buf->b_next = arc_eviction_list;
2168 				arc_eviction_list = buf;
2169 				mutex_exit(&buf->b_evict_lock);
2170 				cv_signal(&arc_user_evicts_cv);
2171 				mutex_exit(&arc_user_evicts_lock);
2172 			} else {
2173 				arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
2174 			}
2175 		}
2176 #ifdef ZFS_DEBUG
2177 		if (hdr->b_l1hdr.b_thawed != NULL) {
2178 			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2179 			hdr->b_l1hdr.b_thawed = NULL;
2180 		}
2181 #endif
2182 	}
2183 
2184 	ASSERT3P(hdr->b_hash_next, ==, NULL);
2185 	if (HDR_HAS_L1HDR(hdr)) {
2186 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2187 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
2188 		kmem_cache_free(hdr_full_cache, hdr);
2189 	} else {
2190 		kmem_cache_free(hdr_l2only_cache, hdr);
2191 	}
2192 }
2193 
2194 void
2195 arc_buf_free(arc_buf_t *buf, void *tag)
2196 {
2197 	arc_buf_hdr_t *hdr = buf->b_hdr;
2198 	int hashed = hdr->b_l1hdr.b_state != arc_anon;
2199 
2200 	ASSERT(buf->b_efunc == NULL);
2201 	ASSERT(buf->b_data != NULL);
2202 
2203 	if (hashed) {
2204 		kmutex_t *hash_lock = HDR_LOCK(hdr);
2205 
2206 		mutex_enter(hash_lock);
2207 		hdr = buf->b_hdr;
2208 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2209 
2210 		(void) remove_reference(hdr, hash_lock, tag);
2211 		if (hdr->b_l1hdr.b_datacnt > 1) {
2212 			arc_buf_destroy(buf, TRUE);
2213 		} else {
2214 			ASSERT(buf == hdr->b_l1hdr.b_buf);
2215 			ASSERT(buf->b_efunc == NULL);
2216 			hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2217 		}
2218 		mutex_exit(hash_lock);
2219 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
2220 		int destroy_hdr;
2221 		/*
2222 		 * We are in the middle of an async write.  Don't destroy
2223 		 * this buffer unless the write completes before we finish
2224 		 * decrementing the reference count.
2225 		 */
2226 		mutex_enter(&arc_user_evicts_lock);
2227 		(void) remove_reference(hdr, NULL, tag);
2228 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2229 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
2230 		mutex_exit(&arc_user_evicts_lock);
2231 		if (destroy_hdr)
2232 			arc_hdr_destroy(hdr);
2233 	} else {
2234 		if (remove_reference(hdr, NULL, tag) > 0)
2235 			arc_buf_destroy(buf, TRUE);
2236 		else
2237 			arc_hdr_destroy(hdr);
2238 	}
2239 }
2240 
2241 boolean_t
2242 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
2243 {
2244 	arc_buf_hdr_t *hdr = buf->b_hdr;
2245 	kmutex_t *hash_lock = HDR_LOCK(hdr);
2246 	boolean_t no_callback = (buf->b_efunc == NULL);
2247 
2248 	if (hdr->b_l1hdr.b_state == arc_anon) {
2249 		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
2250 		arc_buf_free(buf, tag);
2251 		return (no_callback);
2252 	}
2253 
2254 	mutex_enter(hash_lock);
2255 	hdr = buf->b_hdr;
2256 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
2257 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2258 	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
2259 	ASSERT(buf->b_data != NULL);
2260 
2261 	(void) remove_reference(hdr, hash_lock, tag);
2262 	if (hdr->b_l1hdr.b_datacnt > 1) {
2263 		if (no_callback)
2264 			arc_buf_destroy(buf, TRUE);
2265 	} else if (no_callback) {
2266 		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
2267 		ASSERT(buf->b_efunc == NULL);
2268 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2269 	}
2270 	ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
2271 	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2272 	mutex_exit(hash_lock);
2273 	return (no_callback);
2274 }
2275 
2276 int32_t
2277 arc_buf_size(arc_buf_t *buf)
2278 {
2279 	return (buf->b_hdr->b_size);
2280 }
2281 
2282 /*
2283  * Called from the DMU to determine if the current buffer should be
2284  * evicted. In order to ensure proper locking, the eviction must be initiated
2285  * from the DMU. Return true if the buffer is associated with user data and
2286  * duplicate buffers still exist.
2287  */
2288 boolean_t
2289 arc_buf_eviction_needed(arc_buf_t *buf)
2290 {
2291 	arc_buf_hdr_t *hdr;
2292 	boolean_t evict_needed = B_FALSE;
2293 
2294 	if (zfs_disable_dup_eviction)
2295 		return (B_FALSE);
2296 
2297 	mutex_enter(&buf->b_evict_lock);
2298 	hdr = buf->b_hdr;
2299 	if (hdr == NULL) {
2300 		/*
2301 		 * We are in arc_do_user_evicts(); let that function
2302 		 * perform the eviction.
2303 		 */
2304 		ASSERT(buf->b_data == NULL);
2305 		mutex_exit(&buf->b_evict_lock);
2306 		return (B_FALSE);
2307 	} else if (buf->b_data == NULL) {
2308 		/*
2309 		 * We have already been added to the arc eviction list;
2310 		 * recommend eviction.
2311 		 */
2312 		ASSERT3P(hdr, ==, &arc_eviction_hdr);
2313 		mutex_exit(&buf->b_evict_lock);
2314 		return (B_TRUE);
2315 	}
2316 
2317 	if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
2318 		evict_needed = B_TRUE;
2319 
2320 	mutex_exit(&buf->b_evict_lock);
2321 	return (evict_needed);
2322 }
2323 
2324 /*
2325  * Evict the arc_buf_hdr that is provided as a parameter. The resultant
2326  * state of the header is dependent on it's state prior to entering this
2327  * function. The following transitions are possible:
2328  *
2329  *    - arc_mru -> arc_mru_ghost
2330  *    - arc_mfu -> arc_mfu_ghost
2331  *    - arc_mru_ghost -> arc_l2c_only
2332  *    - arc_mru_ghost -> deleted
2333  *    - arc_mfu_ghost -> arc_l2c_only
2334  *    - arc_mfu_ghost -> deleted
2335  */
2336 static int64_t
2337 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
2338 {
2339 	arc_state_t *evicted_state, *state;
2340 	int64_t bytes_evicted = 0;
2341 
2342 	ASSERT(MUTEX_HELD(hash_lock));
2343 	ASSERT(HDR_HAS_L1HDR(hdr));
2344 
2345 	state = hdr->b_l1hdr.b_state;
2346 	if (GHOST_STATE(state)) {
2347 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2348 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
2349 
2350 		/*
2351 		 * l2arc_write_buffers() relies on a header's L1 portion
2352 		 * (i.e. it's b_tmp_cdata field) during it's write phase.
2353 		 * Thus, we cannot push a header onto the arc_l2c_only
2354 		 * state (removing it's L1 piece) until the header is
2355 		 * done being written to the l2arc.
2356 		 */
2357 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
2358 			ARCSTAT_BUMP(arcstat_evict_l2_skip);
2359 			return (bytes_evicted);
2360 		}
2361 
2362 		ARCSTAT_BUMP(arcstat_deleted);
2363 		bytes_evicted += hdr->b_size;
2364 
2365 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
2366 
2367 		if (HDR_HAS_L2HDR(hdr)) {
2368 			/*
2369 			 * This buffer is cached on the 2nd Level ARC;
2370 			 * don't destroy the header.
2371 			 */
2372 			arc_change_state(arc_l2c_only, hdr, hash_lock);
2373 			/*
2374 			 * dropping from L1+L2 cached to L2-only,
2375 			 * realloc to remove the L1 header.
2376 			 */
2377 			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
2378 			    hdr_l2only_cache);
2379 		} else {
2380 			arc_change_state(arc_anon, hdr, hash_lock);
2381 			arc_hdr_destroy(hdr);
2382 		}
2383 		return (bytes_evicted);
2384 	}
2385 
2386 	ASSERT(state == arc_mru || state == arc_mfu);
2387 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
2388 
2389 	/* prefetch buffers have a minimum lifespan */
2390 	if (HDR_IO_IN_PROGRESS(hdr) ||
2391 	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
2392 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
2393 	    arc_min_prefetch_lifespan)) {
2394 		ARCSTAT_BUMP(arcstat_evict_skip);
2395 		return (bytes_evicted);
2396 	}
2397 
2398 	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
2399 	ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
2400 	while (hdr->b_l1hdr.b_buf) {
2401 		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2402 		if (!mutex_tryenter(&buf->b_evict_lock)) {
2403 			ARCSTAT_BUMP(arcstat_mutex_miss);
2404 			break;
2405 		}
2406 		if (buf->b_data != NULL)
2407 			bytes_evicted += hdr->b_size;
2408 		if (buf->b_efunc != NULL) {
2409 			mutex_enter(&arc_user_evicts_lock);
2410 			arc_buf_destroy(buf, FALSE);
2411 			hdr->b_l1hdr.b_buf = buf->b_next;
2412 			buf->b_hdr = &arc_eviction_hdr;
2413 			buf->b_next = arc_eviction_list;
2414 			arc_eviction_list = buf;
2415 			cv_signal(&arc_user_evicts_cv);
2416 			mutex_exit(&arc_user_evicts_lock);
2417 			mutex_exit(&buf->b_evict_lock);
2418 		} else {
2419 			mutex_exit(&buf->b_evict_lock);
2420 			arc_buf_destroy(buf, TRUE);
2421 		}
2422 	}
2423 
2424 	if (HDR_HAS_L2HDR(hdr)) {
2425 		ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
2426 	} else {
2427 		if (l2arc_write_eligible(hdr->b_spa, hdr))
2428 			ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
2429 		else
2430 			ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
2431 	}
2432 
2433 	if (hdr->b_l1hdr.b_datacnt == 0) {
2434 		arc_change_state(evicted_state, hdr, hash_lock);
2435 		ASSERT(HDR_IN_HASH_TABLE(hdr));
2436 		hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
2437 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
2438 		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
2439 	}
2440 
2441 	return (bytes_evicted);
2442 }
2443 
2444 static uint64_t
2445 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
2446     uint64_t spa, int64_t bytes)
2447 {
2448 	multilist_sublist_t *mls;
2449 	uint64_t bytes_evicted = 0;
2450 	arc_buf_hdr_t *hdr;
2451 	kmutex_t *hash_lock;
2452 	int evict_count = 0;
2453 
2454 	ASSERT3P(marker, !=, NULL);
2455 	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2456 
2457 	mls = multilist_sublist_lock(ml, idx);
2458 
2459 	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
2460 	    hdr = multilist_sublist_prev(mls, marker)) {
2461 		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
2462 		    (evict_count >= zfs_arc_evict_batch_limit))
2463 			break;
2464 
2465 		/*
2466 		 * To keep our iteration location, move the marker
2467 		 * forward. Since we're not holding hdr's hash lock, we
2468 		 * must be very careful and not remove 'hdr' from the
2469 		 * sublist. Otherwise, other consumers might mistake the
2470 		 * 'hdr' as not being on a sublist when they call the
2471 		 * multilist_link_active() function (they all rely on
2472 		 * the hash lock protecting concurrent insertions and
2473 		 * removals). multilist_sublist_move_forward() was
2474 		 * specifically implemented to ensure this is the case
2475 		 * (only 'marker' will be removed and re-inserted).
2476 		 */
2477 		multilist_sublist_move_forward(mls, marker);
2478 
2479 		/*
2480 		 * The only case where the b_spa field should ever be
2481 		 * zero, is the marker headers inserted by
2482 		 * arc_evict_state(). It's possible for multiple threads
2483 		 * to be calling arc_evict_state() concurrently (e.g.
2484 		 * dsl_pool_close() and zio_inject_fault()), so we must
2485 		 * skip any markers we see from these other threads.
2486 		 */
2487 		if (hdr->b_spa == 0)
2488 			continue;
2489 
2490 		/* we're only interested in evicting buffers of a certain spa */
2491 		if (spa != 0 && hdr->b_spa != spa) {
2492 			ARCSTAT_BUMP(arcstat_evict_skip);
2493 			continue;
2494 		}
2495 
2496 		hash_lock = HDR_LOCK(hdr);
2497 
2498 		/*
2499 		 * We aren't calling this function from any code path
2500 		 * that would already be holding a hash lock, so we're
2501 		 * asserting on this assumption to be defensive in case
2502 		 * this ever changes. Without this check, it would be
2503 		 * possible to incorrectly increment arcstat_mutex_miss
2504 		 * below (e.g. if the code changed such that we called
2505 		 * this function with a hash lock held).
2506 		 */
2507 		ASSERT(!MUTEX_HELD(hash_lock));
2508 
2509 		if (mutex_tryenter(hash_lock)) {
2510 			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
2511 			mutex_exit(hash_lock);
2512 
2513 			bytes_evicted += evicted;
2514 
2515 			/*
2516 			 * If evicted is zero, arc_evict_hdr() must have
2517 			 * decided to skip this header, don't increment
2518 			 * evict_count in this case.
2519 			 */
2520 			if (evicted != 0)
2521 				evict_count++;
2522 
2523 			/*
2524 			 * If arc_size isn't overflowing, signal any
2525 			 * threads that might happen to be waiting.
2526 			 *
2527 			 * For each header evicted, we wake up a single
2528 			 * thread. If we used cv_broadcast, we could
2529 			 * wake up "too many" threads causing arc_size
2530 			 * to significantly overflow arc_c; since
2531 			 * arc_get_data_buf() doesn't check for overflow
2532 			 * when it's woken up (it doesn't because it's
2533 			 * possible for the ARC to be overflowing while
2534 			 * full of un-evictable buffers, and the
2535 			 * function should proceed in this case).
2536 			 *
2537 			 * If threads are left sleeping, due to not
2538 			 * using cv_broadcast, they will be woken up
2539 			 * just before arc_reclaim_thread() sleeps.
2540 			 */
2541 			mutex_enter(&arc_reclaim_lock);
2542 			if (!arc_is_overflowing())
2543 				cv_signal(&arc_reclaim_waiters_cv);
2544 			mutex_exit(&arc_reclaim_lock);
2545 		} else {
2546 			ARCSTAT_BUMP(arcstat_mutex_miss);
2547 		}
2548 	}
2549 
2550 	multilist_sublist_unlock(mls);
2551 
2552 	return (bytes_evicted);
2553 }
2554 
2555 /*
2556  * Evict buffers from the given arc state, until we've removed the
2557  * specified number of bytes. Move the removed buffers to the
2558  * appropriate evict state.
2559  *
2560  * This function makes a "best effort". It skips over any buffers
2561  * it can't get a hash_lock on, and so, may not catch all candidates.
2562  * It may also return without evicting as much space as requested.
2563  *
2564  * If bytes is specified using the special value ARC_EVICT_ALL, this
2565  * will evict all available (i.e. unlocked and evictable) buffers from
2566  * the given arc state; which is used by arc_flush().
2567  */
2568 static uint64_t
2569 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
2570     arc_buf_contents_t type)
2571 {
2572 	uint64_t total_evicted = 0;
2573 	multilist_t *ml = &state->arcs_list[type];
2574 	int num_sublists;
2575 	arc_buf_hdr_t **markers;
2576 
2577 	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2578 
2579 	num_sublists = multilist_get_num_sublists(ml);
2580 
2581 	/*
2582 	 * If we've tried to evict from each sublist, made some
2583 	 * progress, but still have not hit the target number of bytes
2584 	 * to evict, we want to keep trying. The markers allow us to
2585 	 * pick up where we left off for each individual sublist, rather
2586 	 * than starting from the tail each time.
2587 	 */
2588 	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
2589 	for (int i = 0; i < num_sublists; i++) {
2590 		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
2591 
2592 		/*
2593 		 * A b_spa of 0 is used to indicate that this header is
2594 		 * a marker. This fact is used in arc_adjust_type() and
2595 		 * arc_evict_state_impl().
2596 		 */
2597 		markers[i]->b_spa = 0;
2598 
2599 		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2600 		multilist_sublist_insert_tail(mls, markers[i]);
2601 		multilist_sublist_unlock(mls);
2602 	}
2603 
2604 	/*
2605 	 * While we haven't hit our target number of bytes to evict, or
2606 	 * we're evicting all available buffers.
2607 	 */
2608 	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
2609 		/*
2610 		 * Start eviction using a randomly selected sublist,
2611 		 * this is to try and evenly balance eviction across all
2612 		 * sublists. Always starting at the same sublist
2613 		 * (e.g. index 0) would cause evictions to favor certain
2614 		 * sublists over others.
2615 		 */
2616 		int sublist_idx = multilist_get_random_index(ml);
2617 		uint64_t scan_evicted = 0;
2618 
2619 		for (int i = 0; i < num_sublists; i++) {
2620 			uint64_t bytes_remaining;
2621 			uint64_t bytes_evicted;
2622 
2623 			if (bytes == ARC_EVICT_ALL)
2624 				bytes_remaining = ARC_EVICT_ALL;
2625 			else if (total_evicted < bytes)
2626 				bytes_remaining = bytes - total_evicted;
2627 			else
2628 				break;
2629 
2630 			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
2631 			    markers[sublist_idx], spa, bytes_remaining);
2632 
2633 			scan_evicted += bytes_evicted;
2634 			total_evicted += bytes_evicted;
2635 
2636 			/* we've reached the end, wrap to the beginning */
2637 			if (++sublist_idx >= num_sublists)
2638 				sublist_idx = 0;
2639 		}
2640 
2641 		/*
2642 		 * If we didn't evict anything during this scan, we have
2643 		 * no reason to believe we'll evict more during another
2644 		 * scan, so break the loop.
2645 		 */
2646 		if (scan_evicted == 0) {
2647 			/* This isn't possible, let's make that obvious */
2648 			ASSERT3S(bytes, !=, 0);
2649 
2650 			/*
2651 			 * When bytes is ARC_EVICT_ALL, the only way to
2652 			 * break the loop is when scan_evicted is zero.
2653 			 * In that case, we actually have evicted enough,
2654 			 * so we don't want to increment the kstat.
2655 			 */
2656 			if (bytes != ARC_EVICT_ALL) {
2657 				ASSERT3S(total_evicted, <, bytes);
2658 				ARCSTAT_BUMP(arcstat_evict_not_enough);
2659 			}
2660 
2661 			break;
2662 		}
2663 	}
2664 
2665 	for (int i = 0; i < num_sublists; i++) {
2666 		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2667 		multilist_sublist_remove(mls, markers[i]);
2668 		multilist_sublist_unlock(mls);
2669 
2670 		kmem_cache_free(hdr_full_cache, markers[i]);
2671 	}
2672 	kmem_free(markers, sizeof (*markers) * num_sublists);
2673 
2674 	return (total_evicted);
2675 }
2676 
2677 /*
2678  * Flush all "evictable" data of the given type from the arc state
2679  * specified. This will not evict any "active" buffers (i.e. referenced).
2680  *
2681  * When 'retry' is set to FALSE, the function will make a single pass
2682  * over the state and evict any buffers that it can. Since it doesn't
2683  * continually retry the eviction, it might end up leaving some buffers
2684  * in the ARC due to lock misses.
2685  *
2686  * When 'retry' is set to TRUE, the function will continually retry the
2687  * eviction until *all* evictable buffers have been removed from the
2688  * state. As a result, if concurrent insertions into the state are
2689  * allowed (e.g. if the ARC isn't shutting down), this function might
2690  * wind up in an infinite loop, continually trying to evict buffers.
2691  */
2692 static uint64_t
2693 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
2694     boolean_t retry)
2695 {
2696 	uint64_t evicted = 0;
2697 
2698 	while (state->arcs_lsize[type] != 0) {
2699 		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
2700 
2701 		if (!retry)
2702 			break;
2703 	}
2704 
2705 	return (evicted);
2706 }
2707 
2708 /*
2709  * Evict the specified number of bytes from the state specified,
2710  * restricting eviction to the spa and type given. This function
2711  * prevents us from trying to evict more from a state's list than
2712  * is "evictable", and to skip evicting altogether when passed a
2713  * negative value for "bytes". In contrast, arc_evict_state() will
2714  * evict everything it can, when passed a negative value for "bytes".
2715  */
2716 static uint64_t
2717 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
2718     arc_buf_contents_t type)
2719 {
2720 	int64_t delta;
2721 
2722 	if (bytes > 0 && state->arcs_lsize[type] > 0) {
2723 		delta = MIN(state->arcs_lsize[type], bytes);
2724 		return (arc_evict_state(state, spa, delta, type));
2725 	}
2726 
2727 	return (0);
2728 }
2729 
2730 /*
2731  * Evict metadata buffers from the cache, such that arc_meta_used is
2732  * capped by the arc_meta_limit tunable.
2733  */
2734 static uint64_t
2735 arc_adjust_meta(void)
2736 {
2737 	uint64_t total_evicted = 0;
2738 	int64_t target;
2739 
2740 	/*
2741 	 * If we're over the meta limit, we want to evict enough
2742 	 * metadata to get back under the meta limit. We don't want to
2743 	 * evict so much that we drop the MRU below arc_p, though. If
2744 	 * we're over the meta limit more than we're over arc_p, we
2745 	 * evict some from the MRU here, and some from the MFU below.
2746 	 */
2747 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2748 	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
2749 	    refcount_count(&arc_mru->arcs_size) - arc_p));
2750 
2751 	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2752 
2753 	/*
2754 	 * Similar to the above, we want to evict enough bytes to get us
2755 	 * below the meta limit, but not so much as to drop us below the
2756 	 * space alloted to the MFU (which is defined as arc_c - arc_p).
2757 	 */
2758 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2759 	    (int64_t)(refcount_count(&arc_mfu->arcs_size) - (arc_c - arc_p)));
2760 
2761 	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2762 
2763 	return (total_evicted);
2764 }
2765 
2766 /*
2767  * Return the type of the oldest buffer in the given arc state
2768  *
2769  * This function will select a random sublist of type ARC_BUFC_DATA and
2770  * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
2771  * is compared, and the type which contains the "older" buffer will be
2772  * returned.
2773  */
2774 static arc_buf_contents_t
2775 arc_adjust_type(arc_state_t *state)
2776 {
2777 	multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
2778 	multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
2779 	int data_idx = multilist_get_random_index(data_ml);
2780 	int meta_idx = multilist_get_random_index(meta_ml);
2781 	multilist_sublist_t *data_mls;
2782 	multilist_sublist_t *meta_mls;
2783 	arc_buf_contents_t type;
2784 	arc_buf_hdr_t *data_hdr;
2785 	arc_buf_hdr_t *meta_hdr;
2786 
2787 	/*
2788 	 * We keep the sublist lock until we're finished, to prevent
2789 	 * the headers from being destroyed via arc_evict_state().
2790 	 */
2791 	data_mls = multilist_sublist_lock(data_ml, data_idx);
2792 	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
2793 
2794 	/*
2795 	 * These two loops are to ensure we skip any markers that
2796 	 * might be at the tail of the lists due to arc_evict_state().
2797 	 */
2798 
2799 	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
2800 	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
2801 		if (data_hdr->b_spa != 0)
2802 			break;
2803 	}
2804 
2805 	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
2806 	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
2807 		if (meta_hdr->b_spa != 0)
2808 			break;
2809 	}
2810 
2811 	if (data_hdr == NULL && meta_hdr == NULL) {
2812 		type = ARC_BUFC_DATA;
2813 	} else if (data_hdr == NULL) {
2814 		ASSERT3P(meta_hdr, !=, NULL);
2815 		type = ARC_BUFC_METADATA;
2816 	} else if (meta_hdr == NULL) {
2817 		ASSERT3P(data_hdr, !=, NULL);
2818 		type = ARC_BUFC_DATA;
2819 	} else {
2820 		ASSERT3P(data_hdr, !=, NULL);
2821 		ASSERT3P(meta_hdr, !=, NULL);
2822 
2823 		/* The headers can't be on the sublist without an L1 header */
2824 		ASSERT(HDR_HAS_L1HDR(data_hdr));
2825 		ASSERT(HDR_HAS_L1HDR(meta_hdr));
2826 
2827 		if (data_hdr->b_l1hdr.b_arc_access <
2828 		    meta_hdr->b_l1hdr.b_arc_access) {
2829 			type = ARC_BUFC_DATA;
2830 		} else {
2831 			type = ARC_BUFC_METADATA;
2832 		}
2833 	}
2834 
2835 	multilist_sublist_unlock(meta_mls);
2836 	multilist_sublist_unlock(data_mls);
2837 
2838 	return (type);
2839 }
2840 
2841 /*
2842  * Evict buffers from the cache, such that arc_size is capped by arc_c.
2843  */
2844 static uint64_t
2845 arc_adjust(void)
2846 {
2847 	uint64_t total_evicted = 0;
2848 	uint64_t bytes;
2849 	int64_t target;
2850 
2851 	/*
2852 	 * If we're over arc_meta_limit, we want to correct that before
2853 	 * potentially evicting data buffers below.
2854 	 */
2855 	total_evicted += arc_adjust_meta();
2856 
2857 	/*
2858 	 * Adjust MRU size
2859 	 *
2860 	 * If we're over the target cache size, we want to evict enough
2861 	 * from the list to get back to our target size. We don't want
2862 	 * to evict too much from the MRU, such that it drops below
2863 	 * arc_p. So, if we're over our target cache size more than
2864 	 * the MRU is over arc_p, we'll evict enough to get back to
2865 	 * arc_p here, and then evict more from the MFU below.
2866 	 */
2867 	target = MIN((int64_t)(arc_size - arc_c),
2868 	    (int64_t)(refcount_count(&arc_anon->arcs_size) +
2869 	    refcount_count(&arc_mru->arcs_size) + arc_meta_used - arc_p));
2870 
2871 	/*
2872 	 * If we're below arc_meta_min, always prefer to evict data.
2873 	 * Otherwise, try to satisfy the requested number of bytes to
2874 	 * evict from the type which contains older buffers; in an
2875 	 * effort to keep newer buffers in the cache regardless of their
2876 	 * type. If we cannot satisfy the number of bytes from this
2877 	 * type, spill over into the next type.
2878 	 */
2879 	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2880 	    arc_meta_used > arc_meta_min) {
2881 		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2882 		total_evicted += bytes;
2883 
2884 		/*
2885 		 * If we couldn't evict our target number of bytes from
2886 		 * metadata, we try to get the rest from data.
2887 		 */
2888 		target -= bytes;
2889 
2890 		total_evicted +=
2891 		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2892 	} else {
2893 		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2894 		total_evicted += bytes;
2895 
2896 		/*
2897 		 * If we couldn't evict our target number of bytes from
2898 		 * data, we try to get the rest from metadata.
2899 		 */
2900 		target -= bytes;
2901 
2902 		total_evicted +=
2903 		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2904 	}
2905 
2906 	/*
2907 	 * Adjust MFU size
2908 	 *
2909 	 * Now that we've tried to evict enough from the MRU to get its
2910 	 * size back to arc_p, if we're still above the target cache
2911 	 * size, we evict the rest from the MFU.
2912 	 */
2913 	target = arc_size - arc_c;
2914 
2915 	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2916 	    arc_meta_used > arc_meta_min) {
2917 		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2918 		total_evicted += bytes;
2919 
2920 		/*
2921 		 * If we couldn't evict our target number of bytes from
2922 		 * metadata, we try to get the rest from data.
2923 		 */
2924 		target -= bytes;
2925 
2926 		total_evicted +=
2927 		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2928 	} else {
2929 		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2930 		total_evicted += bytes;
2931 
2932 		/*
2933 		 * If we couldn't evict our target number of bytes from
2934 		 * data, we try to get the rest from data.
2935 		 */
2936 		target -= bytes;
2937 
2938 		total_evicted +=
2939 		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2940 	}
2941 
2942 	/*
2943 	 * Adjust ghost lists
2944 	 *
2945 	 * In addition to the above, the ARC also defines target values
2946 	 * for the ghost lists. The sum of the mru list and mru ghost
2947 	 * list should never exceed the target size of the cache, and
2948 	 * the sum of the mru list, mfu list, mru ghost list, and mfu
2949 	 * ghost list should never exceed twice the target size of the
2950 	 * cache. The following logic enforces these limits on the ghost
2951 	 * caches, and evicts from them as needed.
2952 	 */
2953 	target = refcount_count(&arc_mru->arcs_size) +
2954 	    refcount_count(&arc_mru_ghost->arcs_size) - arc_c;
2955 
2956 	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
2957 	total_evicted += bytes;
2958 
2959 	target -= bytes;
2960 
2961 	total_evicted +=
2962 	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
2963 
2964 	/*
2965 	 * We assume the sum of the mru list and mfu list is less than
2966 	 * or equal to arc_c (we enforced this above), which means we
2967 	 * can use the simpler of the two equations below:
2968 	 *
2969 	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
2970 	 *		    mru ghost + mfu ghost <= arc_c
2971 	 */
2972 	target = refcount_count(&arc_mru_ghost->arcs_size) +
2973 	    refcount_count(&arc_mfu_ghost->arcs_size) - arc_c;
2974 
2975 	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
2976 	total_evicted += bytes;
2977 
2978 	target -= bytes;
2979 
2980 	total_evicted +=
2981 	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
2982 
2983 	return (total_evicted);
2984 }
2985 
2986 static void
2987 arc_do_user_evicts(void)
2988 {
2989 	mutex_enter(&arc_user_evicts_lock);
2990 	while (arc_eviction_list != NULL) {
2991 		arc_buf_t *buf = arc_eviction_list;
2992 		arc_eviction_list = buf->b_next;
2993 		mutex_enter(&buf->b_evict_lock);
2994 		buf->b_hdr = NULL;
2995 		mutex_exit(&buf->b_evict_lock);
2996 		mutex_exit(&arc_user_evicts_lock);
2997 
2998 		if (buf->b_efunc != NULL)
2999 			VERIFY0(buf->b_efunc(buf->b_private));
3000 
3001 		buf->b_efunc = NULL;
3002 		buf->b_private = NULL;
3003 		kmem_cache_free(buf_cache, buf);
3004 		mutex_enter(&arc_user_evicts_lock);
3005 	}
3006 	mutex_exit(&arc_user_evicts_lock);
3007 }
3008 
3009 void
3010 arc_flush(spa_t *spa, boolean_t retry)
3011 {
3012 	uint64_t guid = 0;
3013 
3014 	/*
3015 	 * If retry is TRUE, a spa must not be specified since we have
3016 	 * no good way to determine if all of a spa's buffers have been
3017 	 * evicted from an arc state.
3018 	 */
3019 	ASSERT(!retry || spa == 0);
3020 
3021 	if (spa != NULL)
3022 		guid = spa_load_guid(spa);
3023 
3024 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
3025 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
3026 
3027 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
3028 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
3029 
3030 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
3031 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
3032 
3033 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
3034 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
3035 
3036 	arc_do_user_evicts();
3037 	ASSERT(spa || arc_eviction_list == NULL);
3038 }
3039 
3040 void
3041 arc_shrink(int64_t to_free)
3042 {
3043 	if (arc_c > arc_c_min) {
3044 
3045 		if (arc_c > arc_c_min + to_free)
3046 			atomic_add_64(&arc_c, -to_free);
3047 		else
3048 			arc_c = arc_c_min;
3049 
3050 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
3051 		if (arc_c > arc_size)
3052 			arc_c = MAX(arc_size, arc_c_min);
3053 		if (arc_p > arc_c)
3054 			arc_p = (arc_c >> 1);
3055 		ASSERT(arc_c >= arc_c_min);
3056 		ASSERT((int64_t)arc_p >= 0);
3057 	}
3058 
3059 	if (arc_size > arc_c)
3060 		(void) arc_adjust();
3061 }
3062 
3063 typedef enum free_memory_reason_t {
3064 	FMR_UNKNOWN,
3065 	FMR_NEEDFREE,
3066 	FMR_LOTSFREE,
3067 	FMR_SWAPFS_MINFREE,
3068 	FMR_PAGES_PP_MAXIMUM,
3069 	FMR_HEAP_ARENA,
3070 	FMR_ZIO_ARENA,
3071 } free_memory_reason_t;
3072 
3073 int64_t last_free_memory;
3074 free_memory_reason_t last_free_reason;
3075 
3076 /*
3077  * Additional reserve of pages for pp_reserve.
3078  */
3079 int64_t arc_pages_pp_reserve = 64;
3080 
3081 /*
3082  * Additional reserve of pages for swapfs.
3083  */
3084 int64_t arc_swapfs_reserve = 64;
3085 
3086 /*
3087  * Return the amount of memory that can be consumed before reclaim will be
3088  * needed.  Positive if there is sufficient free memory, negative indicates
3089  * the amount of memory that needs to be freed up.
3090  */
3091 static int64_t
3092 arc_available_memory(void)
3093 {
3094 	int64_t lowest = INT64_MAX;
3095 	int64_t n;
3096 	free_memory_reason_t r = FMR_UNKNOWN;
3097 
3098 #ifdef _KERNEL
3099 	if (needfree > 0) {
3100 		n = PAGESIZE * (-needfree);
3101 		if (n < lowest) {
3102 			lowest = n;
3103 			r = FMR_NEEDFREE;
3104 		}
3105 	}
3106 
3107 	/*
3108 	 * check that we're out of range of the pageout scanner.  It starts to
3109 	 * schedule paging if freemem is less than lotsfree and needfree.
3110 	 * lotsfree is the high-water mark for pageout, and needfree is the
3111 	 * number of needed free pages.  We add extra pages here to make sure
3112 	 * the scanner doesn't start up while we're freeing memory.
3113 	 */
3114 	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
3115 	if (n < lowest) {
3116 		lowest = n;
3117 		r = FMR_LOTSFREE;
3118 	}
3119 
3120 	/*
3121 	 * check to make sure that swapfs has enough space so that anon
3122 	 * reservations can still succeed. anon_resvmem() checks that the
3123 	 * availrmem is greater than swapfs_minfree, and the number of reserved
3124 	 * swap pages.  We also add a bit of extra here just to prevent
3125 	 * circumstances from getting really dire.
3126 	 */
3127 	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
3128 	    desfree - arc_swapfs_reserve);
3129 	if (n < lowest) {
3130 		lowest = n;
3131 		r = FMR_SWAPFS_MINFREE;
3132 	}
3133 
3134 
3135 	/*
3136 	 * Check that we have enough availrmem that memory locking (e.g., via
3137 	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
3138 	 * stores the number of pages that cannot be locked; when availrmem
3139 	 * drops below pages_pp_maximum, page locking mechanisms such as
3140 	 * page_pp_lock() will fail.)
3141 	 */
3142 	n = PAGESIZE * (availrmem - pages_pp_maximum -
3143 	    arc_pages_pp_reserve);
3144 	if (n < lowest) {
3145 		lowest = n;
3146 		r = FMR_PAGES_PP_MAXIMUM;
3147 	}
3148 
3149 #if defined(__i386)
3150 	/*
3151 	 * If we're on an i386 platform, it's possible that we'll exhaust the
3152 	 * kernel heap space before we ever run out of available physical
3153 	 * memory.  Most checks of the size of the heap_area compare against
3154 	 * tune.t_minarmem, which is the minimum available real memory that we
3155 	 * can have in the system.  However, this is generally fixed at 25 pages
3156 	 * which is so low that it's useless.  In this comparison, we seek to
3157 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
3158 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
3159 	 * free)
3160 	 */
3161 	n = vmem_size(heap_arena, VMEM_FREE) -
3162 	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
3163 	if (n < lowest) {
3164 		lowest = n;
3165 		r = FMR_HEAP_ARENA;
3166 	}
3167 #endif
3168 
3169 	/*
3170 	 * If zio data pages are being allocated out of a separate heap segment,
3171 	 * then enforce that the size of available vmem for this arena remains
3172 	 * above about 1/16th free.
3173 	 *
3174 	 * Note: The 1/16th arena free requirement was put in place
3175 	 * to aggressively evict memory from the arc in order to avoid
3176 	 * memory fragmentation issues.
3177 	 */
3178 	if (zio_arena != NULL) {
3179 		n = vmem_size(zio_arena, VMEM_FREE) -
3180 		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
3181 		if (n < lowest) {
3182 			lowest = n;
3183 			r = FMR_ZIO_ARENA;
3184 		}
3185 	}
3186 #else
3187 	/* Every 100 calls, free a small amount */
3188 	if (spa_get_random(100) == 0)
3189 		lowest = -1024;
3190 #endif
3191 
3192 	last_free_memory = lowest;
3193 	last_free_reason = r;
3194 
3195 	return (lowest);
3196 }
3197 
3198 
3199 /*
3200  * Determine if the system is under memory pressure and is asking
3201  * to reclaim memory. A return value of TRUE indicates that the system
3202  * is under memory pressure and that the arc should adjust accordingly.
3203  */
3204 static boolean_t
3205 arc_reclaim_needed(void)
3206 {
3207 	return (arc_available_memory() < 0);
3208 }
3209 
3210 static void
3211 arc_kmem_reap_now(void)
3212 {
3213 	size_t			i;
3214 	kmem_cache_t		*prev_cache = NULL;
3215 	kmem_cache_t		*prev_data_cache = NULL;
3216 	extern kmem_cache_t	*zio_buf_cache[];
3217 	extern kmem_cache_t	*zio_data_buf_cache[];
3218 	extern kmem_cache_t	*range_seg_cache;
3219 
3220 #ifdef _KERNEL
3221 	if (arc_meta_used >= arc_meta_limit) {
3222 		/*
3223 		 * We are exceeding our meta-data cache limit.
3224 		 * Purge some DNLC entries to release holds on meta-data.
3225 		 */
3226 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
3227 	}
3228 #if defined(__i386)
3229 	/*
3230 	 * Reclaim unused memory from all kmem caches.
3231 	 */
3232 	kmem_reap();
3233 #endif
3234 #endif
3235 
3236 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
3237 		if (zio_buf_cache[i] != prev_cache) {
3238 			prev_cache = zio_buf_cache[i];
3239 			kmem_cache_reap_now(zio_buf_cache[i]);
3240 		}
3241 		if (zio_data_buf_cache[i] != prev_data_cache) {
3242 			prev_data_cache = zio_data_buf_cache[i];
3243 			kmem_cache_reap_now(zio_data_buf_cache[i]);
3244 		}
3245 	}
3246 	kmem_cache_reap_now(buf_cache);
3247 	kmem_cache_reap_now(hdr_full_cache);
3248 	kmem_cache_reap_now(hdr_l2only_cache);
3249 	kmem_cache_reap_now(range_seg_cache);
3250 
3251 	if (zio_arena != NULL) {
3252 		/*
3253 		 * Ask the vmem arena to reclaim unused memory from its
3254 		 * quantum caches.
3255 		 */
3256 		vmem_qcache_reap(zio_arena);
3257 	}
3258 }
3259 
3260 /*
3261  * Threads can block in arc_get_data_buf() waiting for this thread to evict
3262  * enough data and signal them to proceed. When this happens, the threads in
3263  * arc_get_data_buf() are sleeping while holding the hash lock for their
3264  * particular arc header. Thus, we must be careful to never sleep on a
3265  * hash lock in this thread. This is to prevent the following deadlock:
3266  *
3267  *  - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
3268  *    waiting for the reclaim thread to signal it.
3269  *
3270  *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
3271  *    fails, and goes to sleep forever.
3272  *
3273  * This possible deadlock is avoided by always acquiring a hash lock
3274  * using mutex_tryenter() from arc_reclaim_thread().
3275  */
3276 static void
3277 arc_reclaim_thread(void)
3278 {
3279 	clock_t			growtime = 0;
3280 	callb_cpr_t		cpr;
3281 
3282 	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
3283 
3284 	mutex_enter(&arc_reclaim_lock);
3285 	while (!arc_reclaim_thread_exit) {
3286 		int64_t free_memory = arc_available_memory();
3287 		uint64_t evicted = 0;
3288 
3289 		mutex_exit(&arc_reclaim_lock);
3290 
3291 		if (free_memory < 0) {
3292 
3293 			arc_no_grow = B_TRUE;
3294 			arc_warm = B_TRUE;
3295 
3296 			/*
3297 			 * Wait at least zfs_grow_retry (default 60) seconds
3298 			 * before considering growing.
3299 			 */
3300 			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
3301 
3302 			arc_kmem_reap_now();
3303 
3304 			/*
3305 			 * If we are still low on memory, shrink the ARC
3306 			 * so that we have arc_shrink_min free space.
3307 			 */
3308 			free_memory = arc_available_memory();
3309 
3310 			int64_t to_free =
3311 			    (arc_c >> arc_shrink_shift) - free_memory;
3312 			if (to_free > 0) {
3313 #ifdef _KERNEL
3314 				to_free = MAX(to_free, ptob(needfree));
3315 #endif
3316 				arc_shrink(to_free);
3317 			}
3318 		} else if (free_memory < arc_c >> arc_no_grow_shift) {
3319 			arc_no_grow = B_TRUE;
3320 		} else if (ddi_get_lbolt() >= growtime) {
3321 			arc_no_grow = B_FALSE;
3322 		}
3323 
3324 		evicted = arc_adjust();
3325 
3326 		mutex_enter(&arc_reclaim_lock);
3327 
3328 		/*
3329 		 * If evicted is zero, we couldn't evict anything via
3330 		 * arc_adjust(). This could be due to hash lock
3331 		 * collisions, but more likely due to the majority of
3332 		 * arc buffers being unevictable. Therefore, even if
3333 		 * arc_size is above arc_c, another pass is unlikely to
3334 		 * be helpful and could potentially cause us to enter an
3335 		 * infinite loop.
3336 		 */
3337 		if (arc_size <= arc_c || evicted == 0) {
3338 			/*
3339 			 * We're either no longer overflowing, or we
3340 			 * can't evict anything more, so we should wake
3341 			 * up any threads before we go to sleep.
3342 			 */
3343 			cv_broadcast(&arc_reclaim_waiters_cv);
3344 
3345 			/*
3346 			 * Block until signaled, or after one second (we
3347 			 * might need to perform arc_kmem_reap_now()
3348 			 * even if we aren't being signalled)
3349 			 */
3350 			CALLB_CPR_SAFE_BEGIN(&cpr);
3351 			(void) cv_timedwait(&arc_reclaim_thread_cv,
3352 			    &arc_reclaim_lock, ddi_get_lbolt() + hz);
3353 			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
3354 		}
3355 	}
3356 
3357 	arc_reclaim_thread_exit = FALSE;
3358 	cv_broadcast(&arc_reclaim_thread_cv);
3359 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
3360 	thread_exit();
3361 }
3362 
3363 static void
3364 arc_user_evicts_thread(void)
3365 {
3366 	callb_cpr_t cpr;
3367 
3368 	CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
3369 
3370 	mutex_enter(&arc_user_evicts_lock);
3371 	while (!arc_user_evicts_thread_exit) {
3372 		mutex_exit(&arc_user_evicts_lock);
3373 
3374 		arc_do_user_evicts();
3375 
3376 		/*
3377 		 * This is necessary in order for the mdb ::arc dcmd to
3378 		 * show up to date information. Since the ::arc command
3379 		 * does not call the kstat's update function, without
3380 		 * this call, the command may show stale stats for the
3381 		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
3382 		 * with this change, the data might be up to 1 second
3383 		 * out of date; but that should suffice. The arc_state_t
3384 		 * structures can be queried directly if more accurate
3385 		 * information is needed.
3386 		 */
3387 		if (arc_ksp != NULL)
3388 			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
3389 
3390 		mutex_enter(&arc_user_evicts_lock);
3391 
3392 		/*
3393 		 * Block until signaled, or after one second (we need to
3394 		 * call the arc's kstat update function regularly).
3395 		 */
3396 		CALLB_CPR_SAFE_BEGIN(&cpr);
3397 		(void) cv_timedwait(&arc_user_evicts_cv,
3398 		    &arc_user_evicts_lock, ddi_get_lbolt() + hz);
3399 		CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
3400 	}
3401 
3402 	arc_user_evicts_thread_exit = FALSE;
3403 	cv_broadcast(&arc_user_evicts_cv);
3404 	CALLB_CPR_EXIT(&cpr);		/* drops arc_user_evicts_lock */
3405 	thread_exit();
3406 }
3407 
3408 /*
3409  * Adapt arc info given the number of bytes we are trying to add and
3410  * the state that we are comming from.  This function is only called
3411  * when we are adding new content to the cache.
3412  */
3413 static void
3414 arc_adapt(int bytes, arc_state_t *state)
3415 {
3416 	int mult;
3417 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
3418 	int64_t mrug_size = refcount_count(&arc_mru_ghost->arcs_size);
3419 	int64_t mfug_size = refcount_count(&arc_mfu_ghost->arcs_size);
3420 
3421 	if (state == arc_l2c_only)
3422 		return;
3423 
3424 	ASSERT(bytes > 0);
3425 	/*
3426 	 * Adapt the target size of the MRU list:
3427 	 *	- if we just hit in the MRU ghost list, then increase
3428 	 *	  the target size of the MRU list.
3429 	 *	- if we just hit in the MFU ghost list, then increase
3430 	 *	  the target size of the MFU list by decreasing the
3431 	 *	  target size of the MRU list.
3432 	 */
3433 	if (state == arc_mru_ghost) {
3434 		mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size);
3435 		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
3436 
3437 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
3438 	} else if (state == arc_mfu_ghost) {
3439 		uint64_t delta;
3440 
3441 		mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size);
3442 		mult = MIN(mult, 10);
3443 
3444 		delta = MIN(bytes * mult, arc_p);
3445 		arc_p = MAX(arc_p_min, arc_p - delta);
3446 	}
3447 	ASSERT((int64_t)arc_p >= 0);
3448 
3449 	if (arc_reclaim_needed()) {
3450 		cv_signal(&arc_reclaim_thread_cv);
3451 		return;
3452 	}
3453 
3454 	if (arc_no_grow)
3455 		return;
3456 
3457 	if (arc_c >= arc_c_max)
3458 		return;
3459 
3460 	/*
3461 	 * If we're within (2 * maxblocksize) bytes of the target
3462 	 * cache size, increment the target cache size
3463 	 */
3464 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
3465 		atomic_add_64(&arc_c, (int64_t)bytes);
3466 		if (arc_c > arc_c_max)
3467 			arc_c = arc_c_max;
3468 		else if (state == arc_anon)
3469 			atomic_add_64(&arc_p, (int64_t)bytes);
3470 		if (arc_p > arc_c)
3471 			arc_p = arc_c;
3472 	}
3473 	ASSERT((int64_t)arc_p >= 0);
3474 }
3475 
3476 /*
3477  * Check if arc_size has grown past our upper threshold, determined by
3478  * zfs_arc_overflow_shift.
3479  */
3480 static boolean_t
3481 arc_is_overflowing(void)
3482 {
3483 	/* Always allow at least one block of overflow */
3484 	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
3485 	    arc_c >> zfs_arc_overflow_shift);
3486 
3487 	return (arc_size >= arc_c + overflow);
3488 }
3489 
3490 /*
3491  * The buffer, supplied as the first argument, needs a data block. If we
3492  * are hitting the hard limit for the cache size, we must sleep, waiting
3493  * for the eviction thread to catch up. If we're past the target size
3494  * but below the hard limit, we'll only signal the reclaim thread and
3495  * continue on.
3496  */
3497 static void
3498 arc_get_data_buf(arc_buf_t *buf)
3499 {
3500 	arc_state_t		*state = buf->b_hdr->b_l1hdr.b_state;
3501 	uint64_t		size = buf->b_hdr->b_size;
3502 	arc_buf_contents_t	type = arc_buf_type(buf->b_hdr);
3503 
3504 	arc_adapt(size, state);
3505 
3506 	/*
3507 	 * If arc_size is currently overflowing, and has grown past our
3508 	 * upper limit, we must be adding data faster than the evict
3509 	 * thread can evict. Thus, to ensure we don't compound the
3510 	 * problem by adding more data and forcing arc_size to grow even
3511 	 * further past it's target size, we halt and wait for the
3512 	 * eviction thread to catch up.
3513 	 *
3514 	 * It's also possible that the reclaim thread is unable to evict
3515 	 * enough buffers to get arc_size below the overflow limit (e.g.
3516 	 * due to buffers being un-evictable, or hash lock collisions).
3517 	 * In this case, we want to proceed regardless if we're
3518 	 * overflowing; thus we don't use a while loop here.
3519 	 */
3520 	if (arc_is_overflowing()) {
3521 		mutex_enter(&arc_reclaim_lock);
3522 
3523 		/*
3524 		 * Now that we've acquired the lock, we may no longer be
3525 		 * over the overflow limit, lets check.
3526 		 *
3527 		 * We're ignoring the case of spurious wake ups. If that
3528 		 * were to happen, it'd let this thread consume an ARC
3529 		 * buffer before it should have (i.e. before we're under
3530 		 * the overflow limit and were signalled by the reclaim
3531 		 * thread). As long as that is a rare occurrence, it
3532 		 * shouldn't cause any harm.
3533 		 */
3534 		if (arc_is_overflowing()) {
3535 			cv_signal(&arc_reclaim_thread_cv);
3536 			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
3537 		}
3538 
3539 		mutex_exit(&arc_reclaim_lock);
3540 	}
3541 
3542 	if (type == ARC_BUFC_METADATA) {
3543 		buf->b_data = zio_buf_alloc(size);
3544 		arc_space_consume(size, ARC_SPACE_META);
3545 	} else {
3546 		ASSERT(type == ARC_BUFC_DATA);
3547 		buf->b_data = zio_data_buf_alloc(size);
3548 		arc_space_consume(size, ARC_SPACE_DATA);
3549 	}
3550 
3551 	/*
3552 	 * Update the state size.  Note that ghost states have a
3553 	 * "ghost size" and so don't need to be updated.
3554 	 */
3555 	if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
3556 		arc_buf_hdr_t *hdr = buf->b_hdr;
3557 		arc_state_t *state = hdr->b_l1hdr.b_state;
3558 
3559 		(void) refcount_add_many(&state->arcs_size, size, buf);
3560 
3561 		/*
3562 		 * If this is reached via arc_read, the link is
3563 		 * protected by the hash lock. If reached via
3564 		 * arc_buf_alloc, the header should not be accessed by
3565 		 * any other thread. And, if reached via arc_read_done,
3566 		 * the hash lock will protect it if it's found in the
3567 		 * hash table; otherwise no other thread should be
3568 		 * trying to [add|remove]_reference it.
3569 		 */
3570 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3571 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3572 			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
3573 			    size);
3574 		}
3575 		/*
3576 		 * If we are growing the cache, and we are adding anonymous
3577 		 * data, and we have outgrown arc_p, update arc_p
3578 		 */
3579 		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
3580 		    (refcount_count(&arc_anon->arcs_size) +
3581 		    refcount_count(&arc_mru->arcs_size) > arc_p))
3582 			arc_p = MIN(arc_c, arc_p + size);
3583 	}
3584 }
3585 
3586 /*
3587  * This routine is called whenever a buffer is accessed.
3588  * NOTE: the hash lock is dropped in this function.
3589  */
3590 static void
3591 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3592 {
3593 	clock_t now;
3594 
3595 	ASSERT(MUTEX_HELD(hash_lock));
3596 	ASSERT(HDR_HAS_L1HDR(hdr));
3597 
3598 	if (hdr->b_l1hdr.b_state == arc_anon) {
3599 		/*
3600 		 * This buffer is not in the cache, and does not
3601 		 * appear in our "ghost" list.  Add the new buffer
3602 		 * to the MRU state.
3603 		 */
3604 
3605 		ASSERT0(hdr->b_l1hdr.b_arc_access);
3606 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3607 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3608 		arc_change_state(arc_mru, hdr, hash_lock);
3609 
3610 	} else if (hdr->b_l1hdr.b_state == arc_mru) {
3611 		now = ddi_get_lbolt();
3612 
3613 		/*
3614 		 * If this buffer is here because of a prefetch, then either:
3615 		 * - clear the flag if this is a "referencing" read
3616 		 *   (any subsequent access will bump this into the MFU state).
3617 		 * or
3618 		 * - move the buffer to the head of the list if this is
3619 		 *   another prefetch (to make it less likely to be evicted).
3620 		 */
3621 		if (HDR_PREFETCH(hdr)) {
3622 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3623 				/* link protected by hash lock */
3624 				ASSERT(multilist_link_active(
3625 				    &hdr->b_l1hdr.b_arc_node));
3626 			} else {
3627 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3628 				ARCSTAT_BUMP(arcstat_mru_hits);
3629 			}
3630 			hdr->b_l1hdr.b_arc_access = now;
3631 			return;
3632 		}
3633 
3634 		/*
3635 		 * This buffer has been "accessed" only once so far,
3636 		 * but it is still in the cache. Move it to the MFU
3637 		 * state.
3638 		 */
3639 		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
3640 			/*
3641 			 * More than 125ms have passed since we
3642 			 * instantiated this buffer.  Move it to the
3643 			 * most frequently used state.
3644 			 */
3645 			hdr->b_l1hdr.b_arc_access = now;
3646 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3647 			arc_change_state(arc_mfu, hdr, hash_lock);
3648 		}
3649 		ARCSTAT_BUMP(arcstat_mru_hits);
3650 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
3651 		arc_state_t	*new_state;
3652 		/*
3653 		 * This buffer has been "accessed" recently, but
3654 		 * was evicted from the cache.  Move it to the
3655 		 * MFU state.
3656 		 */
3657 
3658 		if (HDR_PREFETCH(hdr)) {
3659 			new_state = arc_mru;
3660 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
3661 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3662 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3663 		} else {
3664 			new_state = arc_mfu;
3665 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3666 		}
3667 
3668 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3669 		arc_change_state(new_state, hdr, hash_lock);
3670 
3671 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
3672 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
3673 		/*
3674 		 * This buffer has been accessed more than once and is
3675 		 * still in the cache.  Keep it in the MFU state.
3676 		 *
3677 		 * NOTE: an add_reference() that occurred when we did
3678 		 * the arc_read() will have kicked this off the list.
3679 		 * If it was a prefetch, we will explicitly move it to
3680 		 * the head of the list now.
3681 		 */
3682 		if ((HDR_PREFETCH(hdr)) != 0) {
3683 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3684 			/* link protected by hash_lock */
3685 			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3686 		}
3687 		ARCSTAT_BUMP(arcstat_mfu_hits);
3688 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3689 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
3690 		arc_state_t	*new_state = arc_mfu;
3691 		/*
3692 		 * This buffer has been accessed more than once but has
3693 		 * been evicted from the cache.  Move it back to the
3694 		 * MFU state.
3695 		 */
3696 
3697 		if (HDR_PREFETCH(hdr)) {
3698 			/*
3699 			 * This is a prefetch access...
3700 			 * move this block back to the MRU state.
3701 			 */
3702 			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3703 			new_state = arc_mru;
3704 		}
3705 
3706 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3707 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3708 		arc_change_state(new_state, hdr, hash_lock);
3709 
3710 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
3711 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
3712 		/*
3713 		 * This buffer is on the 2nd Level ARC.
3714 		 */
3715 
3716 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3717 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3718 		arc_change_state(arc_mfu, hdr, hash_lock);
3719 	} else {
3720 		ASSERT(!"invalid arc state");
3721 	}
3722 }
3723 
3724 /* a generic arc_done_func_t which you can use */
3725 /* ARGSUSED */
3726 void
3727 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
3728 {
3729 	if (zio == NULL || zio->io_error == 0)
3730 		bcopy(buf->b_data, arg, buf->b_hdr->b_size);
3731 	VERIFY(arc_buf_remove_ref(buf, arg));
3732 }
3733 
3734 /* a generic arc_done_func_t */
3735 void
3736 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
3737 {
3738 	arc_buf_t **bufp = arg;
3739 	if (zio && zio->io_error) {
3740 		VERIFY(arc_buf_remove_ref(buf, arg));
3741 		*bufp = NULL;
3742 	} else {
3743 		*bufp = buf;
3744 		ASSERT(buf->b_data);
3745 	}
3746 }
3747 
3748 static void
3749 arc_read_done(zio_t *zio)
3750 {
3751 	arc_buf_hdr_t	*hdr;
3752 	arc_buf_t	*buf;
3753 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
3754 	kmutex_t	*hash_lock = NULL;
3755 	arc_callback_t	*callback_list, *acb;
3756 	int		freeable = FALSE;
3757 
3758 	buf = zio->io_private;
3759 	hdr = buf->b_hdr;
3760 
3761 	/*
3762 	 * The hdr was inserted into hash-table and removed from lists
3763 	 * prior to starting I/O.  We should find this header, since
3764 	 * it's in the hash table, and it should be legit since it's
3765 	 * not possible to evict it during the I/O.  The only possible
3766 	 * reason for it not to be found is if we were freed during the
3767 	 * read.
3768 	 */
3769 	if (HDR_IN_HASH_TABLE(hdr)) {
3770 		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
3771 		ASSERT3U(hdr->b_dva.dva_word[0], ==,
3772 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
3773 		ASSERT3U(hdr->b_dva.dva_word[1], ==,
3774 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
3775 
3776 		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
3777 		    &hash_lock);
3778 
3779 		ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) &&
3780 		    hash_lock == NULL) ||
3781 		    (found == hdr &&
3782 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
3783 		    (found == hdr && HDR_L2_READING(hdr)));
3784 	}
3785 
3786 	hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
3787 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
3788 		hdr->b_flags &= ~ARC_FLAG_L2CACHE;
3789 
3790 	/* byteswap if necessary */
3791 	callback_list = hdr->b_l1hdr.b_acb;
3792 	ASSERT(callback_list != NULL);
3793 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
3794 		dmu_object_byteswap_t bswap =
3795 		    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
3796 		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
3797 		    byteswap_uint64_array :
3798 		    dmu_ot_byteswap[bswap].ob_func;
3799 		func(buf->b_data, hdr->b_size);
3800 	}
3801 
3802 	arc_cksum_compute(buf, B_FALSE);
3803 	arc_buf_watch(buf);
3804 
3805 	if (hash_lock && zio->io_error == 0 &&
3806 	    hdr->b_l1hdr.b_state == arc_anon) {
3807 		/*
3808 		 * Only call arc_access on anonymous buffers.  This is because
3809 		 * if we've issued an I/O for an evicted buffer, we've already
3810 		 * called arc_access (to prevent any simultaneous readers from
3811 		 * getting confused).
3812 		 */
3813 		arc_access(hdr, hash_lock);
3814 	}
3815 
3816 	/* create copies of the data buffer for the callers */
3817 	abuf = buf;
3818 	for (acb = callback_list; acb; acb = acb->acb_next) {
3819 		if (acb->acb_done) {
3820 			if (abuf == NULL) {
3821 				ARCSTAT_BUMP(arcstat_duplicate_reads);
3822 				abuf = arc_buf_clone(buf);
3823 			}
3824 			acb->acb_buf = abuf;
3825 			abuf = NULL;
3826 		}
3827 	}
3828 	hdr->b_l1hdr.b_acb = NULL;
3829 	hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
3830 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
3831 	if (abuf == buf) {
3832 		ASSERT(buf->b_efunc == NULL);
3833 		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
3834 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
3835 	}
3836 
3837 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
3838 	    callback_list != NULL);
3839 
3840 	if (zio->io_error != 0) {
3841 		hdr->b_flags |= ARC_FLAG_IO_ERROR;
3842 		if (hdr->b_l1hdr.b_state != arc_anon)
3843 			arc_change_state(arc_anon, hdr, hash_lock);
3844 		if (HDR_IN_HASH_TABLE(hdr))
3845 			buf_hash_remove(hdr);
3846 		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3847 	}
3848 
3849 	/*
3850 	 * Broadcast before we drop the hash_lock to avoid the possibility
3851 	 * that the hdr (and hence the cv) might be freed before we get to
3852 	 * the cv_broadcast().
3853 	 */
3854 	cv_broadcast(&hdr->b_l1hdr.b_cv);
3855 
3856 	if (hash_lock != NULL) {
3857 		mutex_exit(hash_lock);
3858 	} else {
3859 		/*
3860 		 * This block was freed while we waited for the read to
3861 		 * complete.  It has been removed from the hash table and
3862 		 * moved to the anonymous state (so that it won't show up
3863 		 * in the cache).
3864 		 */
3865 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3866 		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3867 	}
3868 
3869 	/* execute each callback and free its structure */
3870 	while ((acb = callback_list) != NULL) {
3871 		if (acb->acb_done)
3872 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
3873 
3874 		if (acb->acb_zio_dummy != NULL) {
3875 			acb->acb_zio_dummy->io_error = zio->io_error;
3876 			zio_nowait(acb->acb_zio_dummy);
3877 		}
3878 
3879 		callback_list = acb->acb_next;
3880 		kmem_free(acb, sizeof (arc_callback_t));
3881 	}
3882 
3883 	if (freeable)
3884 		arc_hdr_destroy(hdr);
3885 }
3886 
3887 /*
3888  * "Read" the block at the specified DVA (in bp) via the
3889  * cache.  If the block is found in the cache, invoke the provided
3890  * callback immediately and return.  Note that the `zio' parameter
3891  * in the callback will be NULL in this case, since no IO was
3892  * required.  If the block is not in the cache pass the read request
3893  * on to the spa with a substitute callback function, so that the
3894  * requested block will be added to the cache.
3895  *
3896  * If a read request arrives for a block that has a read in-progress,
3897  * either wait for the in-progress read to complete (and return the
3898  * results); or, if this is a read with a "done" func, add a record
3899  * to the read to invoke the "done" func when the read completes,
3900  * and return; or just return.
3901  *
3902  * arc_read_done() will invoke all the requested "done" functions
3903  * for readers of this block.
3904  */
3905 int
3906 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
3907     void *private, zio_priority_t priority, int zio_flags,
3908     arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
3909 {
3910 	arc_buf_hdr_t *hdr = NULL;
3911 	arc_buf_t *buf = NULL;
3912 	kmutex_t *hash_lock = NULL;
3913 	zio_t *rzio;
3914 	uint64_t guid = spa_load_guid(spa);
3915 
3916 	ASSERT(!BP_IS_EMBEDDED(bp) ||
3917 	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
3918 
3919 top:
3920 	if (!BP_IS_EMBEDDED(bp)) {
3921 		/*
3922 		 * Embedded BP's have no DVA and require no I/O to "read".
3923 		 * Create an anonymous arc buf to back it.
3924 		 */
3925 		hdr = buf_hash_find(guid, bp, &hash_lock);
3926 	}
3927 
3928 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
3929 
3930 		*arc_flags |= ARC_FLAG_CACHED;
3931 
3932 		if (HDR_IO_IN_PROGRESS(hdr)) {
3933 
3934 			if (*arc_flags & ARC_FLAG_WAIT) {
3935 				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
3936 				mutex_exit(hash_lock);
3937 				goto top;
3938 			}
3939 			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
3940 
3941 			if (done) {
3942 				arc_callback_t	*acb = NULL;
3943 
3944 				acb = kmem_zalloc(sizeof (arc_callback_t),
3945 				    KM_SLEEP);
3946 				acb->acb_done = done;
3947 				acb->acb_private = private;
3948 				if (pio != NULL)
3949 					acb->acb_zio_dummy = zio_null(pio,
3950 					    spa, NULL, NULL, NULL, zio_flags);
3951 
3952 				ASSERT(acb->acb_done != NULL);
3953 				acb->acb_next = hdr->b_l1hdr.b_acb;
3954 				hdr->b_l1hdr.b_acb = acb;
3955 				add_reference(hdr, hash_lock, private);
3956 				mutex_exit(hash_lock);
3957 				return (0);
3958 			}
3959 			mutex_exit(hash_lock);
3960 			return (0);
3961 		}
3962 
3963 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
3964 		    hdr->b_l1hdr.b_state == arc_mfu);
3965 
3966 		if (done) {
3967 			add_reference(hdr, hash_lock, private);
3968 			/*
3969 			 * If this block is already in use, create a new
3970 			 * copy of the data so that we will be guaranteed
3971 			 * that arc_release() will always succeed.
3972 			 */
3973 			buf = hdr->b_l1hdr.b_buf;
3974 			ASSERT(buf);
3975 			ASSERT(buf->b_data);
3976 			if (HDR_BUF_AVAILABLE(hdr)) {
3977 				ASSERT(buf->b_efunc == NULL);
3978 				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
3979 			} else {
3980 				buf = arc_buf_clone(buf);
3981 			}
3982 
3983 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
3984 		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3985 			hdr->b_flags |= ARC_FLAG_PREFETCH;
3986 		}
3987 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
3988 		arc_access(hdr, hash_lock);
3989 		if (*arc_flags & ARC_FLAG_L2CACHE)
3990 			hdr->b_flags |= ARC_FLAG_L2CACHE;
3991 		if (*arc_flags & ARC_FLAG_L2COMPRESS)
3992 			hdr->b_flags |= ARC_FLAG_L2COMPRESS;
3993 		mutex_exit(hash_lock);
3994 		ARCSTAT_BUMP(arcstat_hits);
3995 		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
3996 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
3997 		    data, metadata, hits);
3998 
3999 		if (done)
4000 			done(NULL, buf, private);
4001 	} else {
4002 		uint64_t size = BP_GET_LSIZE(bp);
4003 		arc_callback_t *acb;
4004 		vdev_t *vd = NULL;
4005 		uint64_t addr = 0;
4006 		boolean_t devw = B_FALSE;
4007 		enum zio_compress b_compress = ZIO_COMPRESS_OFF;
4008 		int32_t b_asize = 0;
4009 
4010 		if (hdr == NULL) {
4011 			/* this block is not in the cache */
4012 			arc_buf_hdr_t *exists = NULL;
4013 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
4014 			buf = arc_buf_alloc(spa, size, private, type);
4015 			hdr = buf->b_hdr;
4016 			if (!BP_IS_EMBEDDED(bp)) {
4017 				hdr->b_dva = *BP_IDENTITY(bp);
4018 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
4019 				exists = buf_hash_insert(hdr, &hash_lock);
4020 			}
4021 			if (exists != NULL) {
4022 				/* somebody beat us to the hash insert */
4023 				mutex_exit(hash_lock);
4024 				buf_discard_identity(hdr);
4025 				(void) arc_buf_remove_ref(buf, private);
4026 				goto top; /* restart the IO request */
4027 			}
4028 
4029 			/* if this is a prefetch, we don't have a reference */
4030 			if (*arc_flags & ARC_FLAG_PREFETCH) {
4031 				(void) remove_reference(hdr, hash_lock,
4032 				    private);
4033 				hdr->b_flags |= ARC_FLAG_PREFETCH;
4034 			}
4035 			if (*arc_flags & ARC_FLAG_L2CACHE)
4036 				hdr->b_flags |= ARC_FLAG_L2CACHE;
4037 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
4038 				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4039 			if (BP_GET_LEVEL(bp) > 0)
4040 				hdr->b_flags |= ARC_FLAG_INDIRECT;
4041 		} else {
4042 			/*
4043 			 * This block is in the ghost cache. If it was L2-only
4044 			 * (and thus didn't have an L1 hdr), we realloc the
4045 			 * header to add an L1 hdr.
4046 			 */
4047 			if (!HDR_HAS_L1HDR(hdr)) {
4048 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
4049 				    hdr_full_cache);
4050 			}
4051 
4052 			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
4053 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4054 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4055 			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
4056 
4057 			/* if this is a prefetch, we don't have a reference */
4058 			if (*arc_flags & ARC_FLAG_PREFETCH)
4059 				hdr->b_flags |= ARC_FLAG_PREFETCH;
4060 			else
4061 				add_reference(hdr, hash_lock, private);
4062 			if (*arc_flags & ARC_FLAG_L2CACHE)
4063 				hdr->b_flags |= ARC_FLAG_L2CACHE;
4064 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
4065 				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4066 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
4067 			buf->b_hdr = hdr;
4068 			buf->b_data = NULL;
4069 			buf->b_efunc = NULL;
4070 			buf->b_private = NULL;
4071 			buf->b_next = NULL;
4072 			hdr->b_l1hdr.b_buf = buf;
4073 			ASSERT0(hdr->b_l1hdr.b_datacnt);
4074 			hdr->b_l1hdr.b_datacnt = 1;
4075 			arc_get_data_buf(buf);
4076 			arc_access(hdr, hash_lock);
4077 		}
4078 
4079 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
4080 
4081 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
4082 		acb->acb_done = done;
4083 		acb->acb_private = private;
4084 
4085 		ASSERT(hdr->b_l1hdr.b_acb == NULL);
4086 		hdr->b_l1hdr.b_acb = acb;
4087 		hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4088 
4089 		if (HDR_HAS_L2HDR(hdr) &&
4090 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
4091 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
4092 			addr = hdr->b_l2hdr.b_daddr;
4093 			b_compress = HDR_GET_COMPRESS(hdr);
4094 			b_asize = hdr->b_l2hdr.b_asize;
4095 			/*
4096 			 * Lock out device removal.
4097 			 */
4098 			if (vdev_is_dead(vd) ||
4099 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
4100 				vd = NULL;
4101 		}
4102 
4103 		if (hash_lock != NULL)
4104 			mutex_exit(hash_lock);
4105 
4106 		/*
4107 		 * At this point, we have a level 1 cache miss.  Try again in
4108 		 * L2ARC if possible.
4109 		 */
4110 		ASSERT3U(hdr->b_size, ==, size);
4111 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
4112 		    uint64_t, size, zbookmark_phys_t *, zb);
4113 		ARCSTAT_BUMP(arcstat_misses);
4114 		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4115 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4116 		    data, metadata, misses);
4117 
4118 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
4119 			/*
4120 			 * Read from the L2ARC if the following are true:
4121 			 * 1. The L2ARC vdev was previously cached.
4122 			 * 2. This buffer still has L2ARC metadata.
4123 			 * 3. This buffer isn't currently writing to the L2ARC.
4124 			 * 4. The L2ARC entry wasn't evicted, which may
4125 			 *    also have invalidated the vdev.
4126 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
4127 			 */
4128 			if (HDR_HAS_L2HDR(hdr) &&
4129 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
4130 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
4131 				l2arc_read_callback_t *cb;
4132 
4133 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
4134 				ARCSTAT_BUMP(arcstat_l2_hits);
4135 
4136 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
4137 				    KM_SLEEP);
4138 				cb->l2rcb_buf = buf;
4139 				cb->l2rcb_spa = spa;
4140 				cb->l2rcb_bp = *bp;
4141 				cb->l2rcb_zb = *zb;
4142 				cb->l2rcb_flags = zio_flags;
4143 				cb->l2rcb_compress = b_compress;
4144 
4145 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
4146 				    addr + size < vd->vdev_psize -
4147 				    VDEV_LABEL_END_SIZE);
4148 
4149 				/*
4150 				 * l2arc read.  The SCL_L2ARC lock will be
4151 				 * released by l2arc_read_done().
4152 				 * Issue a null zio if the underlying buffer
4153 				 * was squashed to zero size by compression.
4154 				 */
4155 				if (b_compress == ZIO_COMPRESS_EMPTY) {
4156 					rzio = zio_null(pio, spa, vd,
4157 					    l2arc_read_done, cb,
4158 					    zio_flags | ZIO_FLAG_DONT_CACHE |
4159 					    ZIO_FLAG_CANFAIL |
4160 					    ZIO_FLAG_DONT_PROPAGATE |
4161 					    ZIO_FLAG_DONT_RETRY);
4162 				} else {
4163 					rzio = zio_read_phys(pio, vd, addr,
4164 					    b_asize, buf->b_data,
4165 					    ZIO_CHECKSUM_OFF,
4166 					    l2arc_read_done, cb, priority,
4167 					    zio_flags | ZIO_FLAG_DONT_CACHE |
4168 					    ZIO_FLAG_CANFAIL |
4169 					    ZIO_FLAG_DONT_PROPAGATE |
4170 					    ZIO_FLAG_DONT_RETRY, B_FALSE);
4171 				}
4172 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
4173 				    zio_t *, rzio);
4174 				ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
4175 
4176 				if (*arc_flags & ARC_FLAG_NOWAIT) {
4177 					zio_nowait(rzio);
4178 					return (0);
4179 				}
4180 
4181 				ASSERT(*arc_flags & ARC_FLAG_WAIT);
4182 				if (zio_wait(rzio) == 0)
4183 					return (0);
4184 
4185 				/* l2arc read error; goto zio_read() */
4186 			} else {
4187 				DTRACE_PROBE1(l2arc__miss,
4188 				    arc_buf_hdr_t *, hdr);
4189 				ARCSTAT_BUMP(arcstat_l2_misses);
4190 				if (HDR_L2_WRITING(hdr))
4191 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
4192 				spa_config_exit(spa, SCL_L2ARC, vd);
4193 			}
4194 		} else {
4195 			if (vd != NULL)
4196 				spa_config_exit(spa, SCL_L2ARC, vd);
4197 			if (l2arc_ndev != 0) {
4198 				DTRACE_PROBE1(l2arc__miss,
4199 				    arc_buf_hdr_t *, hdr);
4200 				ARCSTAT_BUMP(arcstat_l2_misses);
4201 			}
4202 		}
4203 
4204 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
4205 		    arc_read_done, buf, priority, zio_flags, zb);
4206 
4207 		if (*arc_flags & ARC_FLAG_WAIT)
4208 			return (zio_wait(rzio));
4209 
4210 		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4211 		zio_nowait(rzio);
4212 	}
4213 	return (0);
4214 }
4215 
4216 void
4217 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
4218 {
4219 	ASSERT(buf->b_hdr != NULL);
4220 	ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
4221 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
4222 	    func == NULL);
4223 	ASSERT(buf->b_efunc == NULL);
4224 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
4225 
4226 	buf->b_efunc = func;
4227 	buf->b_private = private;
4228 }
4229 
4230 /*
4231  * Notify the arc that a block was freed, and thus will never be used again.
4232  */
4233 void
4234 arc_freed(spa_t *spa, const blkptr_t *bp)
4235 {
4236 	arc_buf_hdr_t *hdr;
4237 	kmutex_t *hash_lock;
4238 	uint64_t guid = spa_load_guid(spa);
4239 
4240 	ASSERT(!BP_IS_EMBEDDED(bp));
4241 
4242 	hdr = buf_hash_find(guid, bp, &hash_lock);
4243 	if (hdr == NULL)
4244 		return;
4245 	if (HDR_BUF_AVAILABLE(hdr)) {
4246 		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
4247 		add_reference(hdr, hash_lock, FTAG);
4248 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4249 		mutex_exit(hash_lock);
4250 
4251 		arc_release(buf, FTAG);
4252 		(void) arc_buf_remove_ref(buf, FTAG);
4253 	} else {
4254 		mutex_exit(hash_lock);
4255 	}
4256 
4257 }
4258 
4259 /*
4260  * Clear the user eviction callback set by arc_set_callback(), first calling
4261  * it if it exists.  Because the presence of a callback keeps an arc_buf cached
4262  * clearing the callback may result in the arc_buf being destroyed.  However,
4263  * it will not result in the *last* arc_buf being destroyed, hence the data
4264  * will remain cached in the ARC. We make a copy of the arc buffer here so
4265  * that we can process the callback without holding any locks.
4266  *
4267  * It's possible that the callback is already in the process of being cleared
4268  * by another thread.  In this case we can not clear the callback.
4269  *
4270  * Returns B_TRUE if the callback was successfully called and cleared.
4271  */
4272 boolean_t
4273 arc_clear_callback(arc_buf_t *buf)
4274 {
4275 	arc_buf_hdr_t *hdr;
4276 	kmutex_t *hash_lock;
4277 	arc_evict_func_t *efunc = buf->b_efunc;
4278 	void *private = buf->b_private;
4279 
4280 	mutex_enter(&buf->b_evict_lock);
4281 	hdr = buf->b_hdr;
4282 	if (hdr == NULL) {
4283 		/*
4284 		 * We are in arc_do_user_evicts().
4285 		 */
4286 		ASSERT(buf->b_data == NULL);
4287 		mutex_exit(&buf->b_evict_lock);
4288 		return (B_FALSE);
4289 	} else if (buf->b_data == NULL) {
4290 		/*
4291 		 * We are on the eviction list; process this buffer now
4292 		 * but let arc_do_user_evicts() do the reaping.
4293 		 */
4294 		buf->b_efunc = NULL;
4295 		mutex_exit(&buf->b_evict_lock);
4296 		VERIFY0(efunc(private));
4297 		return (B_TRUE);
4298 	}
4299 	hash_lock = HDR_LOCK(hdr);
4300 	mutex_enter(hash_lock);
4301 	hdr = buf->b_hdr;
4302 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4303 
4304 	ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
4305 	    hdr->b_l1hdr.b_datacnt);
4306 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4307 	    hdr->b_l1hdr.b_state == arc_mfu);
4308 
4309 	buf->b_efunc = NULL;
4310 	buf->b_private = NULL;
4311 
4312 	if (hdr->b_l1hdr.b_datacnt > 1) {
4313 		mutex_exit(&buf->b_evict_lock);
4314 		arc_buf_destroy(buf, TRUE);
4315 	} else {
4316 		ASSERT(buf == hdr->b_l1hdr.b_buf);
4317 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
4318 		mutex_exit(&buf->b_evict_lock);
4319 	}
4320 
4321 	mutex_exit(hash_lock);
4322 	VERIFY0(efunc(private));
4323 	return (B_TRUE);
4324 }
4325 
4326 /*
4327  * Release this buffer from the cache, making it an anonymous buffer.  This
4328  * must be done after a read and prior to modifying the buffer contents.
4329  * If the buffer has more than one reference, we must make
4330  * a new hdr for the buffer.
4331  */
4332 void
4333 arc_release(arc_buf_t *buf, void *tag)
4334 {
4335 	arc_buf_hdr_t *hdr = buf->b_hdr;
4336 
4337 	/*
4338 	 * It would be nice to assert that if it's DMU metadata (level >
4339 	 * 0 || it's the dnode file), then it must be syncing context.
4340 	 * But we don't know that information at this level.
4341 	 */
4342 
4343 	mutex_enter(&buf->b_evict_lock);
4344 
4345 	ASSERT(HDR_HAS_L1HDR(hdr));
4346 
4347 	/*
4348 	 * We don't grab the hash lock prior to this check, because if
4349 	 * the buffer's header is in the arc_anon state, it won't be
4350 	 * linked into the hash table.
4351 	 */
4352 	if (hdr->b_l1hdr.b_state == arc_anon) {
4353 		mutex_exit(&buf->b_evict_lock);
4354 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4355 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
4356 		ASSERT(!HDR_HAS_L2HDR(hdr));
4357 		ASSERT(BUF_EMPTY(hdr));
4358 
4359 		ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
4360 		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
4361 		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
4362 
4363 		ASSERT3P(buf->b_efunc, ==, NULL);
4364 		ASSERT3P(buf->b_private, ==, NULL);
4365 
4366 		hdr->b_l1hdr.b_arc_access = 0;
4367 		arc_buf_thaw(buf);
4368 
4369 		return;
4370 	}
4371 
4372 	kmutex_t *hash_lock = HDR_LOCK(hdr);
4373 	mutex_enter(hash_lock);
4374 
4375 	/*
4376 	 * This assignment is only valid as long as the hash_lock is
4377 	 * held, we must be careful not to reference state or the
4378 	 * b_state field after dropping the lock.
4379 	 */
4380 	arc_state_t *state = hdr->b_l1hdr.b_state;
4381 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4382 	ASSERT3P(state, !=, arc_anon);
4383 
4384 	/* this buffer is not on any list */
4385 	ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
4386 
4387 	if (HDR_HAS_L2HDR(hdr)) {
4388 		ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
4389 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
4390 
4391 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4392 		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
4393 
4394 		/*
4395 		 * We don't want to leak the b_tmp_cdata buffer that was
4396 		 * allocated in l2arc_write_buffers()
4397 		 */
4398 		arc_buf_l2_cdata_free(hdr);
4399 
4400 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4401 
4402 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
4403 	}
4404 
4405 	/*
4406 	 * Do we have more than one buf?
4407 	 */
4408 	if (hdr->b_l1hdr.b_datacnt > 1) {
4409 		arc_buf_hdr_t *nhdr;
4410 		arc_buf_t **bufp;
4411 		uint64_t blksz = hdr->b_size;
4412 		uint64_t spa = hdr->b_spa;
4413 		arc_buf_contents_t type = arc_buf_type(hdr);
4414 		uint32_t flags = hdr->b_flags;
4415 
4416 		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
4417 		/*
4418 		 * Pull the data off of this hdr and attach it to
4419 		 * a new anonymous hdr.
4420 		 */
4421 		(void) remove_reference(hdr, hash_lock, tag);
4422 		bufp = &hdr->b_l1hdr.b_buf;
4423 		while (*bufp != buf)
4424 			bufp = &(*bufp)->b_next;
4425 		*bufp = buf->b_next;
4426 		buf->b_next = NULL;
4427 
4428 		ASSERT3P(state, !=, arc_l2c_only);
4429 
4430 		(void) refcount_remove_many(
4431 		    &state->arcs_size, hdr->b_size, buf);
4432 
4433 		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
4434 			ASSERT3P(state, !=, arc_l2c_only);
4435 			uint64_t *size = &state->arcs_lsize[type];
4436 			ASSERT3U(*size, >=, hdr->b_size);
4437 			atomic_add_64(size, -hdr->b_size);
4438 		}
4439 
4440 		/*
4441 		 * We're releasing a duplicate user data buffer, update
4442 		 * our statistics accordingly.
4443 		 */
4444 		if (HDR_ISTYPE_DATA(hdr)) {
4445 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
4446 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
4447 			    -hdr->b_size);
4448 		}
4449 		hdr->b_l1hdr.b_datacnt -= 1;
4450 		arc_cksum_verify(buf);
4451 		arc_buf_unwatch(buf);
4452 
4453 		mutex_exit(hash_lock);
4454 
4455 		nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
4456 		nhdr->b_size = blksz;
4457 		nhdr->b_spa = spa;
4458 
4459 		nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
4460 		nhdr->b_flags |= arc_bufc_to_flags(type);
4461 		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
4462 
4463 		nhdr->b_l1hdr.b_buf = buf;
4464 		nhdr->b_l1hdr.b_datacnt = 1;
4465 		nhdr->b_l1hdr.b_state = arc_anon;
4466 		nhdr->b_l1hdr.b_arc_access = 0;
4467 		nhdr->b_l1hdr.b_tmp_cdata = NULL;
4468 		nhdr->b_freeze_cksum = NULL;
4469 
4470 		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
4471 		buf->b_hdr = nhdr;
4472 		mutex_exit(&buf->b_evict_lock);
4473 		(void) refcount_add_many(&arc_anon->arcs_size, blksz, buf);
4474 	} else {
4475 		mutex_exit(&buf->b_evict_lock);
4476 		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
4477 		/* protected by hash lock, or hdr is on arc_anon */
4478 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4479 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4480 		arc_change_state(arc_anon, hdr, hash_lock);
4481 		hdr->b_l1hdr.b_arc_access = 0;
4482 		mutex_exit(hash_lock);
4483 
4484 		buf_discard_identity(hdr);
4485 		arc_buf_thaw(buf);
4486 	}
4487 	buf->b_efunc = NULL;
4488 	buf->b_private = NULL;
4489 }
4490 
4491 int
4492 arc_released(arc_buf_t *buf)
4493 {
4494 	int released;
4495 
4496 	mutex_enter(&buf->b_evict_lock);
4497 	released = (buf->b_data != NULL &&
4498 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
4499 	mutex_exit(&buf->b_evict_lock);
4500 	return (released);
4501 }
4502 
4503 #ifdef ZFS_DEBUG
4504 int
4505 arc_referenced(arc_buf_t *buf)
4506 {
4507 	int referenced;
4508 
4509 	mutex_enter(&buf->b_evict_lock);
4510 	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
4511 	mutex_exit(&buf->b_evict_lock);
4512 	return (referenced);
4513 }
4514 #endif
4515 
4516 static void
4517 arc_write_ready(zio_t *zio)
4518 {
4519 	arc_write_callback_t *callback = zio->io_private;
4520 	arc_buf_t *buf = callback->awcb_buf;
4521 	arc_buf_hdr_t *hdr = buf->b_hdr;
4522 
4523 	ASSERT(HDR_HAS_L1HDR(hdr));
4524 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
4525 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4526 	callback->awcb_ready(zio, buf, callback->awcb_private);
4527 
4528 	/*
4529 	 * If the IO is already in progress, then this is a re-write
4530 	 * attempt, so we need to thaw and re-compute the cksum.
4531 	 * It is the responsibility of the callback to handle the
4532 	 * accounting for any re-write attempt.
4533 	 */
4534 	if (HDR_IO_IN_PROGRESS(hdr)) {
4535 		mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
4536 		if (hdr->b_freeze_cksum != NULL) {
4537 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
4538 			hdr->b_freeze_cksum = NULL;
4539 		}
4540 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
4541 	}
4542 	arc_cksum_compute(buf, B_FALSE);
4543 	hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4544 }
4545 
4546 /*
4547  * The SPA calls this callback for each physical write that happens on behalf
4548  * of a logical write.  See the comment in dbuf_write_physdone() for details.
4549  */
4550 static void
4551 arc_write_physdone(zio_t *zio)
4552 {
4553 	arc_write_callback_t *cb = zio->io_private;
4554 	if (cb->awcb_physdone != NULL)
4555 		cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
4556 }
4557 
4558 static void
4559 arc_write_done(zio_t *zio)
4560 {
4561 	arc_write_callback_t *callback = zio->io_private;
4562 	arc_buf_t *buf = callback->awcb_buf;
4563 	arc_buf_hdr_t *hdr = buf->b_hdr;
4564 
4565 	ASSERT(hdr->b_l1hdr.b_acb == NULL);
4566 
4567 	if (zio->io_error == 0) {
4568 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
4569 			buf_discard_identity(hdr);
4570 		} else {
4571 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
4572 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
4573 		}
4574 	} else {
4575 		ASSERT(BUF_EMPTY(hdr));
4576 	}
4577 
4578 	/*
4579 	 * If the block to be written was all-zero or compressed enough to be
4580 	 * embedded in the BP, no write was performed so there will be no
4581 	 * dva/birth/checksum.  The buffer must therefore remain anonymous
4582 	 * (and uncached).
4583 	 */
4584 	if (!BUF_EMPTY(hdr)) {
4585 		arc_buf_hdr_t *exists;
4586 		kmutex_t *hash_lock;
4587 
4588 		ASSERT(zio->io_error == 0);
4589 
4590 		arc_cksum_verify(buf);
4591 
4592 		exists = buf_hash_insert(hdr, &hash_lock);
4593 		if (exists != NULL) {
4594 			/*
4595 			 * This can only happen if we overwrite for
4596 			 * sync-to-convergence, because we remove
4597 			 * buffers from the hash table when we arc_free().
4598 			 */
4599 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
4600 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4601 					panic("bad overwrite, hdr=%p exists=%p",
4602 					    (void *)hdr, (void *)exists);
4603 				ASSERT(refcount_is_zero(
4604 				    &exists->b_l1hdr.b_refcnt));
4605 				arc_change_state(arc_anon, exists, hash_lock);
4606 				mutex_exit(hash_lock);
4607 				arc_hdr_destroy(exists);
4608 				exists = buf_hash_insert(hdr, &hash_lock);
4609 				ASSERT3P(exists, ==, NULL);
4610 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
4611 				/* nopwrite */
4612 				ASSERT(zio->io_prop.zp_nopwrite);
4613 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4614 					panic("bad nopwrite, hdr=%p exists=%p",
4615 					    (void *)hdr, (void *)exists);
4616 			} else {
4617 				/* Dedup */
4618 				ASSERT(hdr->b_l1hdr.b_datacnt == 1);
4619 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
4620 				ASSERT(BP_GET_DEDUP(zio->io_bp));
4621 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
4622 			}
4623 		}
4624 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4625 		/* if it's not anon, we are doing a scrub */
4626 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
4627 			arc_access(hdr, hash_lock);
4628 		mutex_exit(hash_lock);
4629 	} else {
4630 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4631 	}
4632 
4633 	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4634 	callback->awcb_done(zio, buf, callback->awcb_private);
4635 
4636 	kmem_free(callback, sizeof (arc_write_callback_t));
4637 }
4638 
4639 zio_t *
4640 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
4641     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
4642     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
4643     arc_done_func_t *done, void *private, zio_priority_t priority,
4644     int zio_flags, const zbookmark_phys_t *zb)
4645 {
4646 	arc_buf_hdr_t *hdr = buf->b_hdr;
4647 	arc_write_callback_t *callback;
4648 	zio_t *zio;
4649 
4650 	ASSERT(ready != NULL);
4651 	ASSERT(done != NULL);
4652 	ASSERT(!HDR_IO_ERROR(hdr));
4653 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4654 	ASSERT(hdr->b_l1hdr.b_acb == NULL);
4655 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4656 	if (l2arc)
4657 		hdr->b_flags |= ARC_FLAG_L2CACHE;
4658 	if (l2arc_compress)
4659 		hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4660 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
4661 	callback->awcb_ready = ready;
4662 	callback->awcb_physdone = physdone;
4663 	callback->awcb_done = done;
4664 	callback->awcb_private = private;
4665 	callback->awcb_buf = buf;
4666 
4667 	zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
4668 	    arc_write_ready, arc_write_physdone, arc_write_done, callback,
4669 	    priority, zio_flags, zb);
4670 
4671 	return (zio);
4672 }
4673 
4674 static int
4675 arc_memory_throttle(uint64_t reserve, uint64_t txg)
4676 {
4677 #ifdef _KERNEL
4678 	uint64_t available_memory = ptob(freemem);
4679 	static uint64_t page_load = 0;
4680 	static uint64_t last_txg = 0;
4681 
4682 #if defined(__i386)
4683 	available_memory =
4684 	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
4685 #endif
4686 
4687 	if (freemem > physmem * arc_lotsfree_percent / 100)
4688 		return (0);
4689 
4690 	if (txg > last_txg) {
4691 		last_txg = txg;
4692 		page_load = 0;
4693 	}
4694 	/*
4695 	 * If we are in pageout, we know that memory is already tight,
4696 	 * the arc is already going to be evicting, so we just want to
4697 	 * continue to let page writes occur as quickly as possible.
4698 	 */
4699 	if (curproc == proc_pageout) {
4700 		if (page_load > MAX(ptob(minfree), available_memory) / 4)
4701 			return (SET_ERROR(ERESTART));
4702 		/* Note: reserve is inflated, so we deflate */
4703 		page_load += reserve / 8;
4704 		return (0);
4705 	} else if (page_load > 0 && arc_reclaim_needed()) {
4706 		/* memory is low, delay before restarting */
4707 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
4708 		return (SET_ERROR(EAGAIN));
4709 	}
4710 	page_load = 0;
4711 #endif
4712 	return (0);
4713 }
4714 
4715 void
4716 arc_tempreserve_clear(uint64_t reserve)
4717 {
4718 	atomic_add_64(&arc_tempreserve, -reserve);
4719 	ASSERT((int64_t)arc_tempreserve >= 0);
4720 }
4721 
4722 int
4723 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
4724 {
4725 	int error;
4726 	uint64_t anon_size;
4727 
4728 	if (reserve > arc_c/4 && !arc_no_grow)
4729 		arc_c = MIN(arc_c_max, reserve * 4);
4730 	if (reserve > arc_c)
4731 		return (SET_ERROR(ENOMEM));
4732 
4733 	/*
4734 	 * Don't count loaned bufs as in flight dirty data to prevent long
4735 	 * network delays from blocking transactions that are ready to be
4736 	 * assigned to a txg.
4737 	 */
4738 	anon_size = MAX((int64_t)(refcount_count(&arc_anon->arcs_size) -
4739 	    arc_loaned_bytes), 0);
4740 
4741 	/*
4742 	 * Writes will, almost always, require additional memory allocations
4743 	 * in order to compress/encrypt/etc the data.  We therefore need to
4744 	 * make sure that there is sufficient available memory for this.
4745 	 */
4746 	error = arc_memory_throttle(reserve, txg);
4747 	if (error != 0)
4748 		return (error);
4749 
4750 	/*
4751 	 * Throttle writes when the amount of dirty data in the cache
4752 	 * gets too large.  We try to keep the cache less than half full
4753 	 * of dirty blocks so that our sync times don't grow too large.
4754 	 * Note: if two requests come in concurrently, we might let them
4755 	 * both succeed, when one of them should fail.  Not a huge deal.
4756 	 */
4757 
4758 	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
4759 	    anon_size > arc_c / 4) {
4760 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
4761 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
4762 		    arc_tempreserve>>10,
4763 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
4764 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
4765 		    reserve>>10, arc_c>>10);
4766 		return (SET_ERROR(ERESTART));
4767 	}
4768 	atomic_add_64(&arc_tempreserve, reserve);
4769 	return (0);
4770 }
4771 
4772 static void
4773 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
4774     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
4775 {
4776 	size->value.ui64 = refcount_count(&state->arcs_size);
4777 	evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
4778 	evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
4779 }
4780 
4781 static int
4782 arc_kstat_update(kstat_t *ksp, int rw)
4783 {
4784 	arc_stats_t *as = ksp->ks_data;
4785 
4786 	if (rw == KSTAT_WRITE) {
4787 		return (EACCES);
4788 	} else {
4789 		arc_kstat_update_state(arc_anon,
4790 		    &as->arcstat_anon_size,
4791 		    &as->arcstat_anon_evictable_data,
4792 		    &as->arcstat_anon_evictable_metadata);
4793 		arc_kstat_update_state(arc_mru,
4794 		    &as->arcstat_mru_size,
4795 		    &as->arcstat_mru_evictable_data,
4796 		    &as->arcstat_mru_evictable_metadata);
4797 		arc_kstat_update_state(arc_mru_ghost,
4798 		    &as->arcstat_mru_ghost_size,
4799 		    &as->arcstat_mru_ghost_evictable_data,
4800 		    &as->arcstat_mru_ghost_evictable_metadata);
4801 		arc_kstat_update_state(arc_mfu,
4802 		    &as->arcstat_mfu_size,
4803 		    &as->arcstat_mfu_evictable_data,
4804 		    &as->arcstat_mfu_evictable_metadata);
4805 		arc_kstat_update_state(arc_mfu_ghost,
4806 		    &as->arcstat_mfu_ghost_size,
4807 		    &as->arcstat_mfu_ghost_evictable_data,
4808 		    &as->arcstat_mfu_ghost_evictable_metadata);
4809 	}
4810 
4811 	return (0);
4812 }
4813 
4814 /*
4815  * This function *must* return indices evenly distributed between all
4816  * sublists of the multilist. This is needed due to how the ARC eviction
4817  * code is laid out; arc_evict_state() assumes ARC buffers are evenly
4818  * distributed between all sublists and uses this assumption when
4819  * deciding which sublist to evict from and how much to evict from it.
4820  */
4821 unsigned int
4822 arc_state_multilist_index_func(multilist_t *ml, void *obj)
4823 {
4824 	arc_buf_hdr_t *hdr = obj;
4825 
4826 	/*
4827 	 * We rely on b_dva to generate evenly distributed index
4828 	 * numbers using buf_hash below. So, as an added precaution,
4829 	 * let's make sure we never add empty buffers to the arc lists.
4830 	 */
4831 	ASSERT(!BUF_EMPTY(hdr));
4832 
4833 	/*
4834 	 * The assumption here, is the hash value for a given
4835 	 * arc_buf_hdr_t will remain constant throughout it's lifetime
4836 	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
4837 	 * Thus, we don't need to store the header's sublist index
4838 	 * on insertion, as this index can be recalculated on removal.
4839 	 *
4840 	 * Also, the low order bits of the hash value are thought to be
4841 	 * distributed evenly. Otherwise, in the case that the multilist
4842 	 * has a power of two number of sublists, each sublists' usage
4843 	 * would not be evenly distributed.
4844 	 */
4845 	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
4846 	    multilist_get_num_sublists(ml));
4847 }
4848 
4849 void
4850 arc_init(void)
4851 {
4852 	/*
4853 	 * allmem is "all memory that we could possibly use".
4854 	 */
4855 #ifdef _KERNEL
4856 	uint64_t allmem = ptob(physmem - swapfs_minfree);
4857 #else
4858 	uint64_t allmem = (physmem * PAGESIZE) / 2;
4859 #endif
4860 
4861 	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
4862 	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
4863 	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
4864 
4865 	mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
4866 	cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
4867 
4868 	/* Convert seconds to clock ticks */
4869 	arc_min_prefetch_lifespan = 1 * hz;
4870 
4871 	/* Start out with 1/8 of all memory */
4872 	arc_c = allmem / 8;
4873 
4874 #ifdef _KERNEL
4875 	/*
4876 	 * On architectures where the physical memory can be larger
4877 	 * than the addressable space (intel in 32-bit mode), we may
4878 	 * need to limit the cache to 1/8 of VM size.
4879 	 */
4880 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
4881 #endif
4882 
4883 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
4884 	arc_c_min = MAX(allmem / 32, 64 << 20);
4885 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
4886 	if (allmem >= 1 << 30)
4887 		arc_c_max = allmem - (1 << 30);
4888 	else
4889 		arc_c_max = arc_c_min;
4890 	arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
4891 
4892 	/*
4893 	 * Allow the tunables to override our calculations if they are
4894 	 * reasonable (ie. over 64MB)
4895 	 */
4896 	if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
4897 		arc_c_max = zfs_arc_max;
4898 	if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
4899 		arc_c_min = zfs_arc_min;
4900 
4901 	arc_c = arc_c_max;
4902 	arc_p = (arc_c >> 1);
4903 
4904 	/* limit meta-data to 1/4 of the arc capacity */
4905 	arc_meta_limit = arc_c_max / 4;
4906 
4907 	/* Allow the tunable to override if it is reasonable */
4908 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
4909 		arc_meta_limit = zfs_arc_meta_limit;
4910 
4911 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
4912 		arc_c_min = arc_meta_limit / 2;
4913 
4914 	if (zfs_arc_meta_min > 0) {
4915 		arc_meta_min = zfs_arc_meta_min;
4916 	} else {
4917 		arc_meta_min = arc_c_min / 2;
4918 	}
4919 
4920 	if (zfs_arc_grow_retry > 0)
4921 		arc_grow_retry = zfs_arc_grow_retry;
4922 
4923 	if (zfs_arc_shrink_shift > 0)
4924 		arc_shrink_shift = zfs_arc_shrink_shift;
4925 
4926 	/*
4927 	 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
4928 	 */
4929 	if (arc_no_grow_shift >= arc_shrink_shift)
4930 		arc_no_grow_shift = arc_shrink_shift - 1;
4931 
4932 	if (zfs_arc_p_min_shift > 0)
4933 		arc_p_min_shift = zfs_arc_p_min_shift;
4934 
4935 	if (zfs_arc_num_sublists_per_state < 1)
4936 		zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
4937 
4938 	/* if kmem_flags are set, lets try to use less memory */
4939 	if (kmem_debugging())
4940 		arc_c = arc_c / 2;
4941 	if (arc_c < arc_c_min)
4942 		arc_c = arc_c_min;
4943 
4944 	arc_anon = &ARC_anon;
4945 	arc_mru = &ARC_mru;
4946 	arc_mru_ghost = &ARC_mru_ghost;
4947 	arc_mfu = &ARC_mfu;
4948 	arc_mfu_ghost = &ARC_mfu_ghost;
4949 	arc_l2c_only = &ARC_l2c_only;
4950 	arc_size = 0;
4951 
4952 	multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
4953 	    sizeof (arc_buf_hdr_t),
4954 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4955 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4956 	multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
4957 	    sizeof (arc_buf_hdr_t),
4958 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4959 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4960 	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
4961 	    sizeof (arc_buf_hdr_t),
4962 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4963 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4964 	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
4965 	    sizeof (arc_buf_hdr_t),
4966 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4967 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4968 	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
4969 	    sizeof (arc_buf_hdr_t),
4970 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4971 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4972 	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
4973 	    sizeof (arc_buf_hdr_t),
4974 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4975 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4976 	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
4977 	    sizeof (arc_buf_hdr_t),
4978 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4979 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4980 	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
4981 	    sizeof (arc_buf_hdr_t),
4982 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4983 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4984 	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
4985 	    sizeof (arc_buf_hdr_t),
4986 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4987 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4988 	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
4989 	    sizeof (arc_buf_hdr_t),
4990 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4991 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4992 
4993 	refcount_create(&arc_anon->arcs_size);
4994 	refcount_create(&arc_mru->arcs_size);
4995 	refcount_create(&arc_mru_ghost->arcs_size);
4996 	refcount_create(&arc_mfu->arcs_size);
4997 	refcount_create(&arc_mfu_ghost->arcs_size);
4998 	refcount_create(&arc_l2c_only->arcs_size);
4999 
5000 	buf_init();
5001 
5002 	arc_reclaim_thread_exit = FALSE;
5003 	arc_user_evicts_thread_exit = FALSE;
5004 	arc_eviction_list = NULL;
5005 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
5006 
5007 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
5008 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
5009 
5010 	if (arc_ksp != NULL) {
5011 		arc_ksp->ks_data = &arc_stats;
5012 		arc_ksp->ks_update = arc_kstat_update;
5013 		kstat_install(arc_ksp);
5014 	}
5015 
5016 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
5017 	    TS_RUN, minclsyspri);
5018 
5019 	(void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
5020 	    TS_RUN, minclsyspri);
5021 
5022 	arc_dead = FALSE;
5023 	arc_warm = B_FALSE;
5024 
5025 	/*
5026 	 * Calculate maximum amount of dirty data per pool.
5027 	 *
5028 	 * If it has been set by /etc/system, take that.
5029 	 * Otherwise, use a percentage of physical memory defined by
5030 	 * zfs_dirty_data_max_percent (default 10%) with a cap at
5031 	 * zfs_dirty_data_max_max (default 4GB).
5032 	 */
5033 	if (zfs_dirty_data_max == 0) {
5034 		zfs_dirty_data_max = physmem * PAGESIZE *
5035 		    zfs_dirty_data_max_percent / 100;
5036 		zfs_dirty_data_max = MIN(zfs_dirty_data_max,
5037 		    zfs_dirty_data_max_max);
5038 	}
5039 }
5040 
5041 void
5042 arc_fini(void)
5043 {
5044 	mutex_enter(&arc_reclaim_lock);
5045 	arc_reclaim_thread_exit = TRUE;
5046 	/*
5047 	 * The reclaim thread will set arc_reclaim_thread_exit back to
5048 	 * FALSE when it is finished exiting; we're waiting for that.
5049 	 */
5050 	while (arc_reclaim_thread_exit) {
5051 		cv_signal(&arc_reclaim_thread_cv);
5052 		cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
5053 	}
5054 	mutex_exit(&arc_reclaim_lock);
5055 
5056 	mutex_enter(&arc_user_evicts_lock);
5057 	arc_user_evicts_thread_exit = TRUE;
5058 	/*
5059 	 * The user evicts thread will set arc_user_evicts_thread_exit
5060 	 * to FALSE when it is finished exiting; we're waiting for that.
5061 	 */
5062 	while (arc_user_evicts_thread_exit) {
5063 		cv_signal(&arc_user_evicts_cv);
5064 		cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
5065 	}
5066 	mutex_exit(&arc_user_evicts_lock);
5067 
5068 	/* Use TRUE to ensure *all* buffers are evicted */
5069 	arc_flush(NULL, TRUE);
5070 
5071 	arc_dead = TRUE;
5072 
5073 	if (arc_ksp != NULL) {
5074 		kstat_delete(arc_ksp);
5075 		arc_ksp = NULL;
5076 	}
5077 
5078 	mutex_destroy(&arc_reclaim_lock);
5079 	cv_destroy(&arc_reclaim_thread_cv);
5080 	cv_destroy(&arc_reclaim_waiters_cv);
5081 
5082 	mutex_destroy(&arc_user_evicts_lock);
5083 	cv_destroy(&arc_user_evicts_cv);
5084 
5085 	refcount_destroy(&arc_anon->arcs_size);
5086 	refcount_destroy(&arc_mru->arcs_size);
5087 	refcount_destroy(&arc_mru_ghost->arcs_size);
5088 	refcount_destroy(&arc_mfu->arcs_size);
5089 	refcount_destroy(&arc_mfu_ghost->arcs_size);
5090 	refcount_destroy(&arc_l2c_only->arcs_size);
5091 
5092 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
5093 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
5094 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
5095 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
5096 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
5097 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
5098 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
5099 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
5100 
5101 	buf_fini();
5102 
5103 	ASSERT0(arc_loaned_bytes);
5104 }
5105 
5106 /*
5107  * Level 2 ARC
5108  *
5109  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
5110  * It uses dedicated storage devices to hold cached data, which are populated
5111  * using large infrequent writes.  The main role of this cache is to boost
5112  * the performance of random read workloads.  The intended L2ARC devices
5113  * include short-stroked disks, solid state disks, and other media with
5114  * substantially faster read latency than disk.
5115  *
5116  *                 +-----------------------+
5117  *                 |         ARC           |
5118  *                 +-----------------------+
5119  *                    |         ^     ^
5120  *                    |         |     |
5121  *      l2arc_feed_thread()    arc_read()
5122  *                    |         |     |
5123  *                    |  l2arc read   |
5124  *                    V         |     |
5125  *               +---------------+    |
5126  *               |     L2ARC     |    |
5127  *               +---------------+    |
5128  *                   |    ^           |
5129  *          l2arc_write() |           |
5130  *                   |    |           |
5131  *                   V    |           |
5132  *                 +-------+      +-------+
5133  *                 | vdev  |      | vdev  |
5134  *                 | cache |      | cache |
5135  *                 +-------+      +-------+
5136  *                 +=========+     .-----.
5137  *                 :  L2ARC  :    |-_____-|
5138  *                 : devices :    | Disks |
5139  *                 +=========+    `-_____-'
5140  *
5141  * Read requests are satisfied from the following sources, in order:
5142  *
5143  *	1) ARC
5144  *	2) vdev cache of L2ARC devices
5145  *	3) L2ARC devices
5146  *	4) vdev cache of disks
5147  *	5) disks
5148  *
5149  * Some L2ARC device types exhibit extremely slow write performance.
5150  * To accommodate for this there are some significant differences between
5151  * the L2ARC and traditional cache design:
5152  *
5153  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
5154  * the ARC behave as usual, freeing buffers and placing headers on ghost
5155  * lists.  The ARC does not send buffers to the L2ARC during eviction as
5156  * this would add inflated write latencies for all ARC memory pressure.
5157  *
5158  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
5159  * It does this by periodically scanning buffers from the eviction-end of
5160  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
5161  * not already there. It scans until a headroom of buffers is satisfied,
5162  * which itself is a buffer for ARC eviction. If a compressible buffer is
5163  * found during scanning and selected for writing to an L2ARC device, we
5164  * temporarily boost scanning headroom during the next scan cycle to make
5165  * sure we adapt to compression effects (which might significantly reduce
5166  * the data volume we write to L2ARC). The thread that does this is
5167  * l2arc_feed_thread(), illustrated below; example sizes are included to
5168  * provide a better sense of ratio than this diagram:
5169  *
5170  *	       head -->                        tail
5171  *	        +---------------------+----------+
5172  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
5173  *	        +---------------------+----------+   |   o L2ARC eligible
5174  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
5175  *	        +---------------------+----------+   |
5176  *	             15.9 Gbytes      ^ 32 Mbytes    |
5177  *	                           headroom          |
5178  *	                                      l2arc_feed_thread()
5179  *	                                             |
5180  *	                 l2arc write hand <--[oooo]--'
5181  *	                         |           8 Mbyte
5182  *	                         |          write max
5183  *	                         V
5184  *		  +==============================+
5185  *	L2ARC dev |####|#|###|###|    |####| ... |
5186  *	          +==============================+
5187  *	                     32 Gbytes
5188  *
5189  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
5190  * evicted, then the L2ARC has cached a buffer much sooner than it probably
5191  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
5192  * safe to say that this is an uncommon case, since buffers at the end of
5193  * the ARC lists have moved there due to inactivity.
5194  *
5195  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
5196  * then the L2ARC simply misses copying some buffers.  This serves as a
5197  * pressure valve to prevent heavy read workloads from both stalling the ARC
5198  * with waits and clogging the L2ARC with writes.  This also helps prevent
5199  * the potential for the L2ARC to churn if it attempts to cache content too
5200  * quickly, such as during backups of the entire pool.
5201  *
5202  * 5. After system boot and before the ARC has filled main memory, there are
5203  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
5204  * lists can remain mostly static.  Instead of searching from tail of these
5205  * lists as pictured, the l2arc_feed_thread() will search from the list heads
5206  * for eligible buffers, greatly increasing its chance of finding them.
5207  *
5208  * The L2ARC device write speed is also boosted during this time so that
5209  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
5210  * there are no L2ARC reads, and no fear of degrading read performance
5211  * through increased writes.
5212  *
5213  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
5214  * the vdev queue can aggregate them into larger and fewer writes.  Each
5215  * device is written to in a rotor fashion, sweeping writes through
5216  * available space then repeating.
5217  *
5218  * 7. The L2ARC does not store dirty content.  It never needs to flush
5219  * write buffers back to disk based storage.
5220  *
5221  * 8. If an ARC buffer is written (and dirtied) which also exists in the
5222  * L2ARC, the now stale L2ARC buffer is immediately dropped.
5223  *
5224  * The performance of the L2ARC can be tweaked by a number of tunables, which
5225  * may be necessary for different workloads:
5226  *
5227  *	l2arc_write_max		max write bytes per interval
5228  *	l2arc_write_boost	extra write bytes during device warmup
5229  *	l2arc_noprefetch	skip caching prefetched buffers
5230  *	l2arc_headroom		number of max device writes to precache
5231  *	l2arc_headroom_boost	when we find compressed buffers during ARC
5232  *				scanning, we multiply headroom by this
5233  *				percentage factor for the next scan cycle,
5234  *				since more compressed buffers are likely to
5235  *				be present
5236  *	l2arc_feed_secs		seconds between L2ARC writing
5237  *
5238  * Tunables may be removed or added as future performance improvements are
5239  * integrated, and also may become zpool properties.
5240  *
5241  * There are three key functions that control how the L2ARC warms up:
5242  *
5243  *	l2arc_write_eligible()	check if a buffer is eligible to cache
5244  *	l2arc_write_size()	calculate how much to write
5245  *	l2arc_write_interval()	calculate sleep delay between writes
5246  *
5247  * These three functions determine what to write, how much, and how quickly
5248  * to send writes.
5249  */
5250 
5251 static boolean_t
5252 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
5253 {
5254 	/*
5255 	 * A buffer is *not* eligible for the L2ARC if it:
5256 	 * 1. belongs to a different spa.
5257 	 * 2. is already cached on the L2ARC.
5258 	 * 3. has an I/O in progress (it may be an incomplete read).
5259 	 * 4. is flagged not eligible (zfs property).
5260 	 */
5261 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
5262 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
5263 		return (B_FALSE);
5264 
5265 	return (B_TRUE);
5266 }
5267 
5268 static uint64_t
5269 l2arc_write_size(void)
5270 {
5271 	uint64_t size;
5272 
5273 	/*
5274 	 * Make sure our globals have meaningful values in case the user
5275 	 * altered them.
5276 	 */
5277 	size = l2arc_write_max;
5278 	if (size == 0) {
5279 		cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
5280 		    "be greater than zero, resetting it to the default (%d)",
5281 		    L2ARC_WRITE_SIZE);
5282 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
5283 	}
5284 
5285 	if (arc_warm == B_FALSE)
5286 		size += l2arc_write_boost;
5287 
5288 	return (size);
5289 
5290 }
5291 
5292 static clock_t
5293 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
5294 {
5295 	clock_t interval, next, now;
5296 
5297 	/*
5298 	 * If the ARC lists are busy, increase our write rate; if the
5299 	 * lists are stale, idle back.  This is achieved by checking
5300 	 * how much we previously wrote - if it was more than half of
5301 	 * what we wanted, schedule the next write much sooner.
5302 	 */
5303 	if (l2arc_feed_again && wrote > (wanted / 2))
5304 		interval = (hz * l2arc_feed_min_ms) / 1000;
5305 	else
5306 		interval = hz * l2arc_feed_secs;
5307 
5308 	now = ddi_get_lbolt();
5309 	next = MAX(now, MIN(now + interval, began + interval));
5310 
5311 	return (next);
5312 }
5313 
5314 /*
5315  * Cycle through L2ARC devices.  This is how L2ARC load balances.
5316  * If a device is returned, this also returns holding the spa config lock.
5317  */
5318 static l2arc_dev_t *
5319 l2arc_dev_get_next(void)
5320 {
5321 	l2arc_dev_t *first, *next = NULL;
5322 
5323 	/*
5324 	 * Lock out the removal of spas (spa_namespace_lock), then removal
5325 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
5326 	 * both locks will be dropped and a spa config lock held instead.
5327 	 */
5328 	mutex_enter(&spa_namespace_lock);
5329 	mutex_enter(&l2arc_dev_mtx);
5330 
5331 	/* if there are no vdevs, there is nothing to do */
5332 	if (l2arc_ndev == 0)
5333 		goto out;
5334 
5335 	first = NULL;
5336 	next = l2arc_dev_last;
5337 	do {
5338 		/* loop around the list looking for a non-faulted vdev */
5339 		if (next == NULL) {
5340 			next = list_head(l2arc_dev_list);
5341 		} else {
5342 			next = list_next(l2arc_dev_list, next);
5343 			if (next == NULL)
5344 				next = list_head(l2arc_dev_list);
5345 		}
5346 
5347 		/* if we have come back to the start, bail out */
5348 		if (first == NULL)
5349 			first = next;
5350 		else if (next == first)
5351 			break;
5352 
5353 	} while (vdev_is_dead(next->l2ad_vdev));
5354 
5355 	/* if we were unable to find any usable vdevs, return NULL */
5356 	if (vdev_is_dead(next->l2ad_vdev))
5357 		next = NULL;
5358 
5359 	l2arc_dev_last = next;
5360 
5361 out:
5362 	mutex_exit(&l2arc_dev_mtx);
5363 
5364 	/*
5365 	 * Grab the config lock to prevent the 'next' device from being
5366 	 * removed while we are writing to it.
5367 	 */
5368 	if (next != NULL)
5369 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
5370 	mutex_exit(&spa_namespace_lock);
5371 
5372 	return (next);
5373 }
5374 
5375 /*
5376  * Free buffers that were tagged for destruction.
5377  */
5378 static void
5379 l2arc_do_free_on_write()
5380 {
5381 	list_t *buflist;
5382 	l2arc_data_free_t *df, *df_prev;
5383 
5384 	mutex_enter(&l2arc_free_on_write_mtx);
5385 	buflist = l2arc_free_on_write;
5386 
5387 	for (df = list_tail(buflist); df; df = df_prev) {
5388 		df_prev = list_prev(buflist, df);
5389 		ASSERT(df->l2df_data != NULL);
5390 		ASSERT(df->l2df_func != NULL);
5391 		df->l2df_func(df->l2df_data, df->l2df_size);
5392 		list_remove(buflist, df);
5393 		kmem_free(df, sizeof (l2arc_data_free_t));
5394 	}
5395 
5396 	mutex_exit(&l2arc_free_on_write_mtx);
5397 }
5398 
5399 /*
5400  * A write to a cache device has completed.  Update all headers to allow
5401  * reads from these buffers to begin.
5402  */
5403 static void
5404 l2arc_write_done(zio_t *zio)
5405 {
5406 	l2arc_write_callback_t *cb;
5407 	l2arc_dev_t *dev;
5408 	list_t *buflist;
5409 	arc_buf_hdr_t *head, *hdr, *hdr_prev;
5410 	kmutex_t *hash_lock;
5411 	int64_t bytes_dropped = 0;
5412 
5413 	cb = zio->io_private;
5414 	ASSERT(cb != NULL);
5415 	dev = cb->l2wcb_dev;
5416 	ASSERT(dev != NULL);
5417 	head = cb->l2wcb_head;
5418 	ASSERT(head != NULL);
5419 	buflist = &dev->l2ad_buflist;
5420 	ASSERT(buflist != NULL);
5421 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
5422 	    l2arc_write_callback_t *, cb);
5423 
5424 	if (zio->io_error != 0)
5425 		ARCSTAT_BUMP(arcstat_l2_writes_error);
5426 
5427 	/*
5428 	 * All writes completed, or an error was hit.
5429 	 */
5430 top:
5431 	mutex_enter(&dev->l2ad_mtx);
5432 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
5433 		hdr_prev = list_prev(buflist, hdr);
5434 
5435 		hash_lock = HDR_LOCK(hdr);
5436 
5437 		/*
5438 		 * We cannot use mutex_enter or else we can deadlock
5439 		 * with l2arc_write_buffers (due to swapping the order
5440 		 * the hash lock and l2ad_mtx are taken).
5441 		 */
5442 		if (!mutex_tryenter(hash_lock)) {
5443 			/*
5444 			 * Missed the hash lock. We must retry so we
5445 			 * don't leave the ARC_FLAG_L2_WRITING bit set.
5446 			 */
5447 			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
5448 
5449 			/*
5450 			 * We don't want to rescan the headers we've
5451 			 * already marked as having been written out, so
5452 			 * we reinsert the head node so we can pick up
5453 			 * where we left off.
5454 			 */
5455 			list_remove(buflist, head);
5456 			list_insert_after(buflist, hdr, head);
5457 
5458 			mutex_exit(&dev->l2ad_mtx);
5459 
5460 			/*
5461 			 * We wait for the hash lock to become available
5462 			 * to try and prevent busy waiting, and increase
5463 			 * the chance we'll be able to acquire the lock
5464 			 * the next time around.
5465 			 */
5466 			mutex_enter(hash_lock);
5467 			mutex_exit(hash_lock);
5468 			goto top;
5469 		}
5470 
5471 		/*
5472 		 * We could not have been moved into the arc_l2c_only
5473 		 * state while in-flight due to our ARC_FLAG_L2_WRITING
5474 		 * bit being set. Let's just ensure that's being enforced.
5475 		 */
5476 		ASSERT(HDR_HAS_L1HDR(hdr));
5477 
5478 		/*
5479 		 * We may have allocated a buffer for L2ARC compression,
5480 		 * we must release it to avoid leaking this data.
5481 		 */
5482 		l2arc_release_cdata_buf(hdr);
5483 
5484 		if (zio->io_error != 0) {
5485 			/*
5486 			 * Error - drop L2ARC entry.
5487 			 */
5488 			list_remove(buflist, hdr);
5489 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5490 
5491 			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5492 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5493 		}
5494 
5495 		/*
5496 		 * Allow ARC to begin reads and ghost list evictions to
5497 		 * this L2ARC entry.
5498 		 */
5499 		hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
5500 
5501 		mutex_exit(hash_lock);
5502 	}
5503 
5504 	atomic_inc_64(&l2arc_writes_done);
5505 	list_remove(buflist, head);
5506 	ASSERT(!HDR_HAS_L1HDR(head));
5507 	kmem_cache_free(hdr_l2only_cache, head);
5508 	mutex_exit(&dev->l2ad_mtx);
5509 
5510 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
5511 
5512 	l2arc_do_free_on_write();
5513 
5514 	kmem_free(cb, sizeof (l2arc_write_callback_t));
5515 }
5516 
5517 /*
5518  * A read to a cache device completed.  Validate buffer contents before
5519  * handing over to the regular ARC routines.
5520  */
5521 static void
5522 l2arc_read_done(zio_t *zio)
5523 {
5524 	l2arc_read_callback_t *cb;
5525 	arc_buf_hdr_t *hdr;
5526 	arc_buf_t *buf;
5527 	kmutex_t *hash_lock;
5528 	int equal;
5529 
5530 	ASSERT(zio->io_vd != NULL);
5531 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
5532 
5533 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
5534 
5535 	cb = zio->io_private;
5536 	ASSERT(cb != NULL);
5537 	buf = cb->l2rcb_buf;
5538 	ASSERT(buf != NULL);
5539 
5540 	hash_lock = HDR_LOCK(buf->b_hdr);
5541 	mutex_enter(hash_lock);
5542 	hdr = buf->b_hdr;
5543 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5544 
5545 	/*
5546 	 * If the buffer was compressed, decompress it first.
5547 	 */
5548 	if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
5549 		l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
5550 	ASSERT(zio->io_data != NULL);
5551 
5552 	/*
5553 	 * Check this survived the L2ARC journey.
5554 	 */
5555 	equal = arc_cksum_equal(buf);
5556 	if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
5557 		mutex_exit(hash_lock);
5558 		zio->io_private = buf;
5559 		zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
5560 		zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
5561 		arc_read_done(zio);
5562 	} else {
5563 		mutex_exit(hash_lock);
5564 		/*
5565 		 * Buffer didn't survive caching.  Increment stats and
5566 		 * reissue to the original storage device.
5567 		 */
5568 		if (zio->io_error != 0) {
5569 			ARCSTAT_BUMP(arcstat_l2_io_error);
5570 		} else {
5571 			zio->io_error = SET_ERROR(EIO);
5572 		}
5573 		if (!equal)
5574 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
5575 
5576 		/*
5577 		 * If there's no waiter, issue an async i/o to the primary
5578 		 * storage now.  If there *is* a waiter, the caller must
5579 		 * issue the i/o in a context where it's OK to block.
5580 		 */
5581 		if (zio->io_waiter == NULL) {
5582 			zio_t *pio = zio_unique_parent(zio);
5583 
5584 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
5585 
5586 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
5587 			    buf->b_data, zio->io_size, arc_read_done, buf,
5588 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
5589 		}
5590 	}
5591 
5592 	kmem_free(cb, sizeof (l2arc_read_callback_t));
5593 }
5594 
5595 /*
5596  * This is the list priority from which the L2ARC will search for pages to
5597  * cache.  This is used within loops (0..3) to cycle through lists in the
5598  * desired order.  This order can have a significant effect on cache
5599  * performance.
5600  *
5601  * Currently the metadata lists are hit first, MFU then MRU, followed by
5602  * the data lists.  This function returns a locked list, and also returns
5603  * the lock pointer.
5604  */
5605 static multilist_sublist_t *
5606 l2arc_sublist_lock(int list_num)
5607 {
5608 	multilist_t *ml = NULL;
5609 	unsigned int idx;
5610 
5611 	ASSERT(list_num >= 0 && list_num <= 3);
5612 
5613 	switch (list_num) {
5614 	case 0:
5615 		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
5616 		break;
5617 	case 1:
5618 		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
5619 		break;
5620 	case 2:
5621 		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
5622 		break;
5623 	case 3:
5624 		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
5625 		break;
5626 	}
5627 
5628 	/*
5629 	 * Return a randomly-selected sublist. This is acceptable
5630 	 * because the caller feeds only a little bit of data for each
5631 	 * call (8MB). Subsequent calls will result in different
5632 	 * sublists being selected.
5633 	 */
5634 	idx = multilist_get_random_index(ml);
5635 	return (multilist_sublist_lock(ml, idx));
5636 }
5637 
5638 /*
5639  * Evict buffers from the device write hand to the distance specified in
5640  * bytes.  This distance may span populated buffers, it may span nothing.
5641  * This is clearing a region on the L2ARC device ready for writing.
5642  * If the 'all' boolean is set, every buffer is evicted.
5643  */
5644 static void
5645 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
5646 {
5647 	list_t *buflist;
5648 	arc_buf_hdr_t *hdr, *hdr_prev;
5649 	kmutex_t *hash_lock;
5650 	uint64_t taddr;
5651 	int64_t bytes_evicted = 0;
5652 
5653 	buflist = &dev->l2ad_buflist;
5654 
5655 	if (!all && dev->l2ad_first) {
5656 		/*
5657 		 * This is the first sweep through the device.  There is
5658 		 * nothing to evict.
5659 		 */
5660 		return;
5661 	}
5662 
5663 	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
5664 		/*
5665 		 * When nearing the end of the device, evict to the end
5666 		 * before the device write hand jumps to the start.
5667 		 */
5668 		taddr = dev->l2ad_end;
5669 	} else {
5670 		taddr = dev->l2ad_hand + distance;
5671 	}
5672 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
5673 	    uint64_t, taddr, boolean_t, all);
5674 
5675 top:
5676 	mutex_enter(&dev->l2ad_mtx);
5677 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
5678 		hdr_prev = list_prev(buflist, hdr);
5679 
5680 		hash_lock = HDR_LOCK(hdr);
5681 
5682 		/*
5683 		 * We cannot use mutex_enter or else we can deadlock
5684 		 * with l2arc_write_buffers (due to swapping the order
5685 		 * the hash lock and l2ad_mtx are taken).
5686 		 */
5687 		if (!mutex_tryenter(hash_lock)) {
5688 			/*
5689 			 * Missed the hash lock.  Retry.
5690 			 */
5691 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
5692 			mutex_exit(&dev->l2ad_mtx);
5693 			mutex_enter(hash_lock);
5694 			mutex_exit(hash_lock);
5695 			goto top;
5696 		}
5697 
5698 		if (HDR_L2_WRITE_HEAD(hdr)) {
5699 			/*
5700 			 * We hit a write head node.  Leave it for
5701 			 * l2arc_write_done().
5702 			 */
5703 			list_remove(buflist, hdr);
5704 			mutex_exit(hash_lock);
5705 			continue;
5706 		}
5707 
5708 		if (!all && HDR_HAS_L2HDR(hdr) &&
5709 		    (hdr->b_l2hdr.b_daddr > taddr ||
5710 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
5711 			/*
5712 			 * We've evicted to the target address,
5713 			 * or the end of the device.
5714 			 */
5715 			mutex_exit(hash_lock);
5716 			break;
5717 		}
5718 
5719 		ASSERT(HDR_HAS_L2HDR(hdr));
5720 		if (!HDR_HAS_L1HDR(hdr)) {
5721 			ASSERT(!HDR_L2_READING(hdr));
5722 			/*
5723 			 * This doesn't exist in the ARC.  Destroy.
5724 			 * arc_hdr_destroy() will call list_remove()
5725 			 * and decrement arcstat_l2_size.
5726 			 */
5727 			arc_change_state(arc_anon, hdr, hash_lock);
5728 			arc_hdr_destroy(hdr);
5729 		} else {
5730 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
5731 			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
5732 			/*
5733 			 * Invalidate issued or about to be issued
5734 			 * reads, since we may be about to write
5735 			 * over this location.
5736 			 */
5737 			if (HDR_L2_READING(hdr)) {
5738 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
5739 				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
5740 			}
5741 
5742 			/* Tell ARC this no longer exists in L2ARC. */
5743 			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5744 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5745 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5746 			list_remove(buflist, hdr);
5747 
5748 			/* Ensure this header has finished being written */
5749 			ASSERT(!HDR_L2_WRITING(hdr));
5750 			ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
5751 		}
5752 		mutex_exit(hash_lock);
5753 	}
5754 	mutex_exit(&dev->l2ad_mtx);
5755 
5756 	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
5757 	dev->l2ad_evict = taddr;
5758 }
5759 
5760 /*
5761  * Find and write ARC buffers to the L2ARC device.
5762  *
5763  * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
5764  * for reading until they have completed writing.
5765  * The headroom_boost is an in-out parameter used to maintain headroom boost
5766  * state between calls to this function.
5767  *
5768  * Returns the number of bytes actually written (which may be smaller than
5769  * the delta by which the device hand has changed due to alignment).
5770  */
5771 static uint64_t
5772 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
5773     boolean_t *headroom_boost)
5774 {
5775 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
5776 	uint64_t write_asize, write_psize, write_sz, headroom,
5777 	    buf_compress_minsz;
5778 	void *buf_data;
5779 	boolean_t full;
5780 	l2arc_write_callback_t *cb;
5781 	zio_t *pio, *wzio;
5782 	uint64_t guid = spa_load_guid(spa);
5783 	const boolean_t do_headroom_boost = *headroom_boost;
5784 
5785 	ASSERT(dev->l2ad_vdev != NULL);
5786 
5787 	/* Lower the flag now, we might want to raise it again later. */
5788 	*headroom_boost = B_FALSE;
5789 
5790 	pio = NULL;
5791 	write_sz = write_asize = write_psize = 0;
5792 	full = B_FALSE;
5793 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
5794 	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
5795 	head->b_flags |= ARC_FLAG_HAS_L2HDR;
5796 
5797 	/*
5798 	 * We will want to try to compress buffers that are at least 2x the
5799 	 * device sector size.
5800 	 */
5801 	buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
5802 
5803 	/*
5804 	 * Copy buffers for L2ARC writing.
5805 	 */
5806 	for (int try = 0; try <= 3; try++) {
5807 		multilist_sublist_t *mls = l2arc_sublist_lock(try);
5808 		uint64_t passed_sz = 0;
5809 
5810 		/*
5811 		 * L2ARC fast warmup.
5812 		 *
5813 		 * Until the ARC is warm and starts to evict, read from the
5814 		 * head of the ARC lists rather than the tail.
5815 		 */
5816 		if (arc_warm == B_FALSE)
5817 			hdr = multilist_sublist_head(mls);
5818 		else
5819 			hdr = multilist_sublist_tail(mls);
5820 
5821 		headroom = target_sz * l2arc_headroom;
5822 		if (do_headroom_boost)
5823 			headroom = (headroom * l2arc_headroom_boost) / 100;
5824 
5825 		for (; hdr; hdr = hdr_prev) {
5826 			kmutex_t *hash_lock;
5827 			uint64_t buf_sz;
5828 
5829 			if (arc_warm == B_FALSE)
5830 				hdr_prev = multilist_sublist_next(mls, hdr);
5831 			else
5832 				hdr_prev = multilist_sublist_prev(mls, hdr);
5833 
5834 			hash_lock = HDR_LOCK(hdr);
5835 			if (!mutex_tryenter(hash_lock)) {
5836 				/*
5837 				 * Skip this buffer rather than waiting.
5838 				 */
5839 				continue;
5840 			}
5841 
5842 			passed_sz += hdr->b_size;
5843 			if (passed_sz > headroom) {
5844 				/*
5845 				 * Searched too far.
5846 				 */
5847 				mutex_exit(hash_lock);
5848 				break;
5849 			}
5850 
5851 			if (!l2arc_write_eligible(guid, hdr)) {
5852 				mutex_exit(hash_lock);
5853 				continue;
5854 			}
5855 
5856 			if ((write_sz + hdr->b_size) > target_sz) {
5857 				full = B_TRUE;
5858 				mutex_exit(hash_lock);
5859 				break;
5860 			}
5861 
5862 			if (pio == NULL) {
5863 				/*
5864 				 * Insert a dummy header on the buflist so
5865 				 * l2arc_write_done() can find where the
5866 				 * write buffers begin without searching.
5867 				 */
5868 				mutex_enter(&dev->l2ad_mtx);
5869 				list_insert_head(&dev->l2ad_buflist, head);
5870 				mutex_exit(&dev->l2ad_mtx);
5871 
5872 				cb = kmem_alloc(
5873 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
5874 				cb->l2wcb_dev = dev;
5875 				cb->l2wcb_head = head;
5876 				pio = zio_root(spa, l2arc_write_done, cb,
5877 				    ZIO_FLAG_CANFAIL);
5878 			}
5879 
5880 			/*
5881 			 * Create and add a new L2ARC header.
5882 			 */
5883 			hdr->b_l2hdr.b_dev = dev;
5884 			hdr->b_flags |= ARC_FLAG_L2_WRITING;
5885 			/*
5886 			 * Temporarily stash the data buffer in b_tmp_cdata.
5887 			 * The subsequent write step will pick it up from
5888 			 * there. This is because can't access b_l1hdr.b_buf
5889 			 * without holding the hash_lock, which we in turn
5890 			 * can't access without holding the ARC list locks
5891 			 * (which we want to avoid during compression/writing).
5892 			 */
5893 			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
5894 			hdr->b_l2hdr.b_asize = hdr->b_size;
5895 			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
5896 
5897 			buf_sz = hdr->b_size;
5898 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
5899 
5900 			mutex_enter(&dev->l2ad_mtx);
5901 			list_insert_head(&dev->l2ad_buflist, hdr);
5902 			mutex_exit(&dev->l2ad_mtx);
5903 
5904 			/*
5905 			 * Compute and store the buffer cksum before
5906 			 * writing.  On debug the cksum is verified first.
5907 			 */
5908 			arc_cksum_verify(hdr->b_l1hdr.b_buf);
5909 			arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
5910 
5911 			mutex_exit(hash_lock);
5912 
5913 			write_sz += buf_sz;
5914 		}
5915 
5916 		multilist_sublist_unlock(mls);
5917 
5918 		if (full == B_TRUE)
5919 			break;
5920 	}
5921 
5922 	/* No buffers selected for writing? */
5923 	if (pio == NULL) {
5924 		ASSERT0(write_sz);
5925 		ASSERT(!HDR_HAS_L1HDR(head));
5926 		kmem_cache_free(hdr_l2only_cache, head);
5927 		return (0);
5928 	}
5929 
5930 	mutex_enter(&dev->l2ad_mtx);
5931 
5932 	/*
5933 	 * Now start writing the buffers. We're starting at the write head
5934 	 * and work backwards, retracing the course of the buffer selector
5935 	 * loop above.
5936 	 */
5937 	for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
5938 	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
5939 		uint64_t buf_sz;
5940 
5941 		/*
5942 		 * We rely on the L1 portion of the header below, so
5943 		 * it's invalid for this header to have been evicted out
5944 		 * of the ghost cache, prior to being written out. The
5945 		 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
5946 		 */
5947 		ASSERT(HDR_HAS_L1HDR(hdr));
5948 
5949 		/*
5950 		 * We shouldn't need to lock the buffer here, since we flagged
5951 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
5952 		 * take care to only access its L2 cache parameters. In
5953 		 * particular, hdr->l1hdr.b_buf may be invalid by now due to
5954 		 * ARC eviction.
5955 		 */
5956 		hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
5957 
5958 		if ((HDR_L2COMPRESS(hdr)) &&
5959 		    hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
5960 			if (l2arc_compress_buf(hdr)) {
5961 				/*
5962 				 * If compression succeeded, enable headroom
5963 				 * boost on the next scan cycle.
5964 				 */
5965 				*headroom_boost = B_TRUE;
5966 			}
5967 		}
5968 
5969 		/*
5970 		 * Pick up the buffer data we had previously stashed away
5971 		 * (and now potentially also compressed).
5972 		 */
5973 		buf_data = hdr->b_l1hdr.b_tmp_cdata;
5974 		buf_sz = hdr->b_l2hdr.b_asize;
5975 
5976 		/* Compression may have squashed the buffer to zero length. */
5977 		if (buf_sz != 0) {
5978 			uint64_t buf_p_sz;
5979 
5980 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
5981 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
5982 			    NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
5983 			    ZIO_FLAG_CANFAIL, B_FALSE);
5984 
5985 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
5986 			    zio_t *, wzio);
5987 			(void) zio_nowait(wzio);
5988 
5989 			write_asize += buf_sz;
5990 			/*
5991 			 * Keep the clock hand suitably device-aligned.
5992 			 */
5993 			buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
5994 			write_psize += buf_p_sz;
5995 			dev->l2ad_hand += buf_p_sz;
5996 		}
5997 	}
5998 
5999 	mutex_exit(&dev->l2ad_mtx);
6000 
6001 	ASSERT3U(write_asize, <=, target_sz);
6002 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
6003 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
6004 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
6005 	ARCSTAT_INCR(arcstat_l2_asize, write_asize);
6006 	vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
6007 
6008 	/*
6009 	 * Bump device hand to the device start if it is approaching the end.
6010 	 * l2arc_evict() will already have evicted ahead for this case.
6011 	 */
6012 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
6013 		dev->l2ad_hand = dev->l2ad_start;
6014 		dev->l2ad_evict = dev->l2ad_start;
6015 		dev->l2ad_first = B_FALSE;
6016 	}
6017 
6018 	dev->l2ad_writing = B_TRUE;
6019 	(void) zio_wait(pio);
6020 	dev->l2ad_writing = B_FALSE;
6021 
6022 	return (write_asize);
6023 }
6024 
6025 /*
6026  * Compresses an L2ARC buffer.
6027  * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
6028  * size in l2hdr->b_asize. This routine tries to compress the data and
6029  * depending on the compression result there are three possible outcomes:
6030  * *) The buffer was incompressible. The original l2hdr contents were left
6031  *    untouched and are ready for writing to an L2 device.
6032  * *) The buffer was all-zeros, so there is no need to write it to an L2
6033  *    device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
6034  *    set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
6035  * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
6036  *    data buffer which holds the compressed data to be written, and b_asize
6037  *    tells us how much data there is. b_compress is set to the appropriate
6038  *    compression algorithm. Once writing is done, invoke
6039  *    l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
6040  *
6041  * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
6042  * buffer was incompressible).
6043  */
6044 static boolean_t
6045 l2arc_compress_buf(arc_buf_hdr_t *hdr)
6046 {
6047 	void *cdata;
6048 	size_t csize, len, rounded;
6049 	ASSERT(HDR_HAS_L2HDR(hdr));
6050 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
6051 
6052 	ASSERT(HDR_HAS_L1HDR(hdr));
6053 	ASSERT(HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF);
6054 	ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6055 
6056 	len = l2hdr->b_asize;
6057 	cdata = zio_data_buf_alloc(len);
6058 	ASSERT3P(cdata, !=, NULL);
6059 	csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
6060 	    cdata, l2hdr->b_asize);
6061 
6062 	rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
6063 	if (rounded > csize) {
6064 		bzero((char *)cdata + csize, rounded - csize);
6065 		csize = rounded;
6066 	}
6067 
6068 	if (csize == 0) {
6069 		/* zero block, indicate that there's nothing to write */
6070 		zio_data_buf_free(cdata, len);
6071 		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_EMPTY);
6072 		l2hdr->b_asize = 0;
6073 		hdr->b_l1hdr.b_tmp_cdata = NULL;
6074 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
6075 		return (B_TRUE);
6076 	} else if (csize > 0 && csize < len) {
6077 		/*
6078 		 * Compression succeeded, we'll keep the cdata around for
6079 		 * writing and release it afterwards.
6080 		 */
6081 		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_LZ4);
6082 		l2hdr->b_asize = csize;
6083 		hdr->b_l1hdr.b_tmp_cdata = cdata;
6084 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
6085 		return (B_TRUE);
6086 	} else {
6087 		/*
6088 		 * Compression failed, release the compressed buffer.
6089 		 * l2hdr will be left unmodified.
6090 		 */
6091 		zio_data_buf_free(cdata, len);
6092 		ARCSTAT_BUMP(arcstat_l2_compress_failures);
6093 		return (B_FALSE);
6094 	}
6095 }
6096 
6097 /*
6098  * Decompresses a zio read back from an l2arc device. On success, the
6099  * underlying zio's io_data buffer is overwritten by the uncompressed
6100  * version. On decompression error (corrupt compressed stream), the
6101  * zio->io_error value is set to signal an I/O error.
6102  *
6103  * Please note that the compressed data stream is not checksummed, so
6104  * if the underlying device is experiencing data corruption, we may feed
6105  * corrupt data to the decompressor, so the decompressor needs to be
6106  * able to handle this situation (LZ4 does).
6107  */
6108 static void
6109 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
6110 {
6111 	ASSERT(L2ARC_IS_VALID_COMPRESS(c));
6112 
6113 	if (zio->io_error != 0) {
6114 		/*
6115 		 * An io error has occured, just restore the original io
6116 		 * size in preparation for a main pool read.
6117 		 */
6118 		zio->io_orig_size = zio->io_size = hdr->b_size;
6119 		return;
6120 	}
6121 
6122 	if (c == ZIO_COMPRESS_EMPTY) {
6123 		/*
6124 		 * An empty buffer results in a null zio, which means we
6125 		 * need to fill its io_data after we're done restoring the
6126 		 * buffer's contents.
6127 		 */
6128 		ASSERT(hdr->b_l1hdr.b_buf != NULL);
6129 		bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
6130 		zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
6131 	} else {
6132 		ASSERT(zio->io_data != NULL);
6133 		/*
6134 		 * We copy the compressed data from the start of the arc buffer
6135 		 * (the zio_read will have pulled in only what we need, the
6136 		 * rest is garbage which we will overwrite at decompression)
6137 		 * and then decompress back to the ARC data buffer. This way we
6138 		 * can minimize copying by simply decompressing back over the
6139 		 * original compressed data (rather than decompressing to an
6140 		 * aux buffer and then copying back the uncompressed buffer,
6141 		 * which is likely to be much larger).
6142 		 */
6143 		uint64_t csize;
6144 		void *cdata;
6145 
6146 		csize = zio->io_size;
6147 		cdata = zio_data_buf_alloc(csize);
6148 		bcopy(zio->io_data, cdata, csize);
6149 		if (zio_decompress_data(c, cdata, zio->io_data, csize,
6150 		    hdr->b_size) != 0)
6151 			zio->io_error = EIO;
6152 		zio_data_buf_free(cdata, csize);
6153 	}
6154 
6155 	/* Restore the expected uncompressed IO size. */
6156 	zio->io_orig_size = zio->io_size = hdr->b_size;
6157 }
6158 
6159 /*
6160  * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
6161  * This buffer serves as a temporary holder of compressed data while
6162  * the buffer entry is being written to an l2arc device. Once that is
6163  * done, we can dispose of it.
6164  */
6165 static void
6166 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
6167 {
6168 	enum zio_compress comp = HDR_GET_COMPRESS(hdr);
6169 
6170 	ASSERT(HDR_HAS_L1HDR(hdr));
6171 	ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
6172 
6173 	if (comp == ZIO_COMPRESS_OFF) {
6174 		/*
6175 		 * In this case, b_tmp_cdata points to the same buffer
6176 		 * as the arc_buf_t's b_data field. We don't want to
6177 		 * free it, since the arc_buf_t will handle that.
6178 		 */
6179 		hdr->b_l1hdr.b_tmp_cdata = NULL;
6180 	} else if (comp == ZIO_COMPRESS_EMPTY) {
6181 		/*
6182 		 * In this case, b_tmp_cdata was compressed to an empty
6183 		 * buffer, thus there's nothing to free and b_tmp_cdata
6184 		 * should have been set to NULL in l2arc_write_buffers().
6185 		 */
6186 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
6187 	} else {
6188 		/*
6189 		 * If the data was compressed, then we've allocated a
6190 		 * temporary buffer for it, so now we need to release it.
6191 		 */
6192 		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6193 		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
6194 		    hdr->b_size);
6195 		hdr->b_l1hdr.b_tmp_cdata = NULL;
6196 	}
6197 
6198 }
6199 
6200 /*
6201  * This thread feeds the L2ARC at regular intervals.  This is the beating
6202  * heart of the L2ARC.
6203  */
6204 static void
6205 l2arc_feed_thread(void)
6206 {
6207 	callb_cpr_t cpr;
6208 	l2arc_dev_t *dev;
6209 	spa_t *spa;
6210 	uint64_t size, wrote;
6211 	clock_t begin, next = ddi_get_lbolt();
6212 	boolean_t headroom_boost = B_FALSE;
6213 
6214 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
6215 
6216 	mutex_enter(&l2arc_feed_thr_lock);
6217 
6218 	while (l2arc_thread_exit == 0) {
6219 		CALLB_CPR_SAFE_BEGIN(&cpr);
6220 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
6221 		    next);
6222 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
6223 		next = ddi_get_lbolt() + hz;
6224 
6225 		/*
6226 		 * Quick check for L2ARC devices.
6227 		 */
6228 		mutex_enter(&l2arc_dev_mtx);
6229 		if (l2arc_ndev == 0) {
6230 			mutex_exit(&l2arc_dev_mtx);
6231 			continue;
6232 		}
6233 		mutex_exit(&l2arc_dev_mtx);
6234 		begin = ddi_get_lbolt();
6235 
6236 		/*
6237 		 * This selects the next l2arc device to write to, and in
6238 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
6239 		 * will return NULL if there are now no l2arc devices or if
6240 		 * they are all faulted.
6241 		 *
6242 		 * If a device is returned, its spa's config lock is also
6243 		 * held to prevent device removal.  l2arc_dev_get_next()
6244 		 * will grab and release l2arc_dev_mtx.
6245 		 */
6246 		if ((dev = l2arc_dev_get_next()) == NULL)
6247 			continue;
6248 
6249 		spa = dev->l2ad_spa;
6250 		ASSERT(spa != NULL);
6251 
6252 		/*
6253 		 * If the pool is read-only then force the feed thread to
6254 		 * sleep a little longer.
6255 		 */
6256 		if (!spa_writeable(spa)) {
6257 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
6258 			spa_config_exit(spa, SCL_L2ARC, dev);
6259 			continue;
6260 		}
6261 
6262 		/*
6263 		 * Avoid contributing to memory pressure.
6264 		 */
6265 		if (arc_reclaim_needed()) {
6266 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
6267 			spa_config_exit(spa, SCL_L2ARC, dev);
6268 			continue;
6269 		}
6270 
6271 		ARCSTAT_BUMP(arcstat_l2_feeds);
6272 
6273 		size = l2arc_write_size();
6274 
6275 		/*
6276 		 * Evict L2ARC buffers that will be overwritten.
6277 		 */
6278 		l2arc_evict(dev, size, B_FALSE);
6279 
6280 		/*
6281 		 * Write ARC buffers.
6282 		 */
6283 		wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
6284 
6285 		/*
6286 		 * Calculate interval between writes.
6287 		 */
6288 		next = l2arc_write_interval(begin, size, wrote);
6289 		spa_config_exit(spa, SCL_L2ARC, dev);
6290 	}
6291 
6292 	l2arc_thread_exit = 0;
6293 	cv_broadcast(&l2arc_feed_thr_cv);
6294 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
6295 	thread_exit();
6296 }
6297 
6298 boolean_t
6299 l2arc_vdev_present(vdev_t *vd)
6300 {
6301 	l2arc_dev_t *dev;
6302 
6303 	mutex_enter(&l2arc_dev_mtx);
6304 	for (dev = list_head(l2arc_dev_list); dev != NULL;
6305 	    dev = list_next(l2arc_dev_list, dev)) {
6306 		if (dev->l2ad_vdev == vd)
6307 			break;
6308 	}
6309 	mutex_exit(&l2arc_dev_mtx);
6310 
6311 	return (dev != NULL);
6312 }
6313 
6314 /*
6315  * Add a vdev for use by the L2ARC.  By this point the spa has already
6316  * validated the vdev and opened it.
6317  */
6318 void
6319 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
6320 {
6321 	l2arc_dev_t *adddev;
6322 
6323 	ASSERT(!l2arc_vdev_present(vd));
6324 
6325 	/*
6326 	 * Create a new l2arc device entry.
6327 	 */
6328 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
6329 	adddev->l2ad_spa = spa;
6330 	adddev->l2ad_vdev = vd;
6331 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
6332 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
6333 	adddev->l2ad_hand = adddev->l2ad_start;
6334 	adddev->l2ad_evict = adddev->l2ad_start;
6335 	adddev->l2ad_first = B_TRUE;
6336 	adddev->l2ad_writing = B_FALSE;
6337 
6338 	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
6339 	/*
6340 	 * This is a list of all ARC buffers that are still valid on the
6341 	 * device.
6342 	 */
6343 	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
6344 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
6345 
6346 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
6347 
6348 	/*
6349 	 * Add device to global list
6350 	 */
6351 	mutex_enter(&l2arc_dev_mtx);
6352 	list_insert_head(l2arc_dev_list, adddev);
6353 	atomic_inc_64(&l2arc_ndev);
6354 	mutex_exit(&l2arc_dev_mtx);
6355 }
6356 
6357 /*
6358  * Remove a vdev from the L2ARC.
6359  */
6360 void
6361 l2arc_remove_vdev(vdev_t *vd)
6362 {
6363 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
6364 
6365 	/*
6366 	 * Find the device by vdev
6367 	 */
6368 	mutex_enter(&l2arc_dev_mtx);
6369 	for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
6370 		nextdev = list_next(l2arc_dev_list, dev);
6371 		if (vd == dev->l2ad_vdev) {
6372 			remdev = dev;
6373 			break;
6374 		}
6375 	}
6376 	ASSERT(remdev != NULL);
6377 
6378 	/*
6379 	 * Remove device from global list
6380 	 */
6381 	list_remove(l2arc_dev_list, remdev);
6382 	l2arc_dev_last = NULL;		/* may have been invalidated */
6383 	atomic_dec_64(&l2arc_ndev);
6384 	mutex_exit(&l2arc_dev_mtx);
6385 
6386 	/*
6387 	 * Clear all buflists and ARC references.  L2ARC device flush.
6388 	 */
6389 	l2arc_evict(remdev, 0, B_TRUE);
6390 	list_destroy(&remdev->l2ad_buflist);
6391 	mutex_destroy(&remdev->l2ad_mtx);
6392 	kmem_free(remdev, sizeof (l2arc_dev_t));
6393 }
6394 
6395 void
6396 l2arc_init(void)
6397 {
6398 	l2arc_thread_exit = 0;
6399 	l2arc_ndev = 0;
6400 	l2arc_writes_sent = 0;
6401 	l2arc_writes_done = 0;
6402 
6403 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
6404 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
6405 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
6406 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
6407 
6408 	l2arc_dev_list = &L2ARC_dev_list;
6409 	l2arc_free_on_write = &L2ARC_free_on_write;
6410 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
6411 	    offsetof(l2arc_dev_t, l2ad_node));
6412 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
6413 	    offsetof(l2arc_data_free_t, l2df_list_node));
6414 }
6415 
6416 void
6417 l2arc_fini(void)
6418 {
6419 	/*
6420 	 * This is called from dmu_fini(), which is called from spa_fini();
6421 	 * Because of this, we can assume that all l2arc devices have
6422 	 * already been removed when the pools themselves were removed.
6423 	 */
6424 
6425 	l2arc_do_free_on_write();
6426 
6427 	mutex_destroy(&l2arc_feed_thr_lock);
6428 	cv_destroy(&l2arc_feed_thr_cv);
6429 	mutex_destroy(&l2arc_dev_mtx);
6430 	mutex_destroy(&l2arc_free_on_write_mtx);
6431 
6432 	list_destroy(l2arc_dev_list);
6433 	list_destroy(l2arc_free_on_write);
6434 }
6435 
6436 void
6437 l2arc_start(void)
6438 {
6439 	if (!(spa_mode_global & FWRITE))
6440 		return;
6441 
6442 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
6443 	    TS_RUN, minclsyspri);
6444 }
6445 
6446 void
6447 l2arc_stop(void)
6448 {
6449 	if (!(spa_mode_global & FWRITE))
6450 		return;
6451 
6452 	mutex_enter(&l2arc_feed_thr_lock);
6453 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
6454 	l2arc_thread_exit = 1;
6455 	while (l2arc_thread_exit != 0)
6456 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
6457 	mutex_exit(&l2arc_feed_thr_lock);
6458 }
6459