xref: /illumos-gate/usr/src/uts/common/fs/zfs/arc.c (revision 2dea4eed7ad1c66ae4770263aa2911815a8b86eb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * DVA-based Adjustable Replacement Cache
28  *
29  * While much of the theory of operation used here is
30  * based on the self-tuning, low overhead replacement cache
31  * presented by Megiddo and Modha at FAST 2003, there are some
32  * significant differences:
33  *
34  * 1. The Megiddo and Modha model assumes any page is evictable.
35  * Pages in its cache cannot be "locked" into memory.  This makes
36  * the eviction algorithm simple: evict the last page in the list.
37  * This also make the performance characteristics easy to reason
38  * about.  Our cache is not so simple.  At any given moment, some
39  * subset of the blocks in the cache are un-evictable because we
40  * have handed out a reference to them.  Blocks are only evictable
41  * when there are no external references active.  This makes
42  * eviction far more problematic:  we choose to evict the evictable
43  * blocks that are the "lowest" in the list.
44  *
45  * There are times when it is not possible to evict the requested
46  * space.  In these circumstances we are unable to adjust the cache
47  * size.  To prevent the cache growing unbounded at these times we
48  * implement a "cache throttle" that slows the flow of new data
49  * into the cache until we can make space available.
50  *
51  * 2. The Megiddo and Modha model assumes a fixed cache size.
52  * Pages are evicted when the cache is full and there is a cache
53  * miss.  Our model has a variable sized cache.  It grows with
54  * high use, but also tries to react to memory pressure from the
55  * operating system: decreasing its size when system memory is
56  * tight.
57  *
58  * 3. The Megiddo and Modha model assumes a fixed page size. All
59  * elements of the cache are therefor exactly the same size.  So
60  * when adjusting the cache size following a cache miss, its simply
61  * a matter of choosing a single page to evict.  In our model, we
62  * have variable sized cache blocks (rangeing from 512 bytes to
63  * 128K bytes).  We therefor choose a set of blocks to evict to make
64  * space for a cache miss that approximates as closely as possible
65  * the space used by the new block.
66  *
67  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
68  * by N. Megiddo & D. Modha, FAST 2003
69  */
70 
71 /*
72  * The locking model:
73  *
74  * A new reference to a cache buffer can be obtained in two
75  * ways: 1) via a hash table lookup using the DVA as a key,
76  * or 2) via one of the ARC lists.  The arc_read() interface
77  * uses method 1, while the internal arc algorithms for
78  * adjusting the cache use method 2.  We therefor provide two
79  * types of locks: 1) the hash table lock array, and 2) the
80  * arc list locks.
81  *
82  * Buffers do not have their own mutexs, rather they rely on the
83  * hash table mutexs for the bulk of their protection (i.e. most
84  * fields in the arc_buf_hdr_t are protected by these mutexs).
85  *
86  * buf_hash_find() returns the appropriate mutex (held) when it
87  * locates the requested buffer in the hash table.  It returns
88  * NULL for the mutex if the buffer was not in the table.
89  *
90  * buf_hash_remove() expects the appropriate hash mutex to be
91  * already held before it is invoked.
92  *
93  * Each arc state also has a mutex which is used to protect the
94  * buffer list associated with the state.  When attempting to
95  * obtain a hash table lock while holding an arc list lock you
96  * must use: mutex_tryenter() to avoid deadlock.  Also note that
97  * the active state mutex must be held before the ghost state mutex.
98  *
99  * Arc buffers may have an associated eviction callback function.
100  * This function will be invoked prior to removing the buffer (e.g.
101  * in arc_do_user_evicts()).  Note however that the data associated
102  * with the buffer may be evicted prior to the callback.  The callback
103  * must be made with *no locks held* (to prevent deadlock).  Additionally,
104  * the users of callbacks must ensure that their private data is
105  * protected from simultaneous callbacks from arc_buf_evict()
106  * and arc_do_user_evicts().
107  *
108  * Note that the majority of the performance stats are manipulated
109  * with atomic operations.
110  *
111  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
112  *
113  *	- L2ARC buflist creation
114  *	- L2ARC buflist eviction
115  *	- L2ARC write completion, which walks L2ARC buflists
116  *	- ARC header destruction, as it removes from L2ARC buflists
117  *	- ARC header release, as it removes from L2ARC buflists
118  */
119 
120 #include <sys/spa.h>
121 #include <sys/zio.h>
122 #include <sys/zfs_context.h>
123 #include <sys/arc.h>
124 #include <sys/refcount.h>
125 #include <sys/vdev.h>
126 #include <sys/vdev_impl.h>
127 #ifdef _KERNEL
128 #include <sys/vmsystm.h>
129 #include <vm/anon.h>
130 #include <sys/fs/swapnode.h>
131 #include <sys/dnlc.h>
132 #endif
133 #include <sys/callb.h>
134 #include <sys/kstat.h>
135 #include <zfs_fletcher.h>
136 
137 static kmutex_t		arc_reclaim_thr_lock;
138 static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
139 static uint8_t		arc_thread_exit;
140 
141 extern int zfs_write_limit_shift;
142 extern uint64_t zfs_write_limit_max;
143 extern kmutex_t zfs_write_limit_lock;
144 
145 #define	ARC_REDUCE_DNLC_PERCENT	3
146 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
147 
148 typedef enum arc_reclaim_strategy {
149 	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
150 	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
151 } arc_reclaim_strategy_t;
152 
153 /* number of seconds before growing cache again */
154 static int		arc_grow_retry = 60;
155 
156 /* shift of arc_c for calculating both min and max arc_p */
157 static int		arc_p_min_shift = 4;
158 
159 /* log2(fraction of arc to reclaim) */
160 static int		arc_shrink_shift = 5;
161 
162 /*
163  * minimum lifespan of a prefetch block in clock ticks
164  * (initialized in arc_init())
165  */
166 static int		arc_min_prefetch_lifespan;
167 
168 static int arc_dead;
169 
170 /*
171  * The arc has filled available memory and has now warmed up.
172  */
173 static boolean_t arc_warm;
174 
175 /*
176  * These tunables are for performance analysis.
177  */
178 uint64_t zfs_arc_max;
179 uint64_t zfs_arc_min;
180 uint64_t zfs_arc_meta_limit = 0;
181 int zfs_arc_grow_retry = 0;
182 int zfs_arc_shrink_shift = 0;
183 int zfs_arc_p_min_shift = 0;
184 
185 /*
186  * Note that buffers can be in one of 6 states:
187  *	ARC_anon	- anonymous (discussed below)
188  *	ARC_mru		- recently used, currently cached
189  *	ARC_mru_ghost	- recentely used, no longer in cache
190  *	ARC_mfu		- frequently used, currently cached
191  *	ARC_mfu_ghost	- frequently used, no longer in cache
192  *	ARC_l2c_only	- exists in L2ARC but not other states
193  * When there are no active references to the buffer, they are
194  * are linked onto a list in one of these arc states.  These are
195  * the only buffers that can be evicted or deleted.  Within each
196  * state there are multiple lists, one for meta-data and one for
197  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
198  * etc.) is tracked separately so that it can be managed more
199  * explicitly: favored over data, limited explicitly.
200  *
201  * Anonymous buffers are buffers that are not associated with
202  * a DVA.  These are buffers that hold dirty block copies
203  * before they are written to stable storage.  By definition,
204  * they are "ref'd" and are considered part of arc_mru
205  * that cannot be freed.  Generally, they will aquire a DVA
206  * as they are written and migrate onto the arc_mru list.
207  *
208  * The ARC_l2c_only state is for buffers that are in the second
209  * level ARC but no longer in any of the ARC_m* lists.  The second
210  * level ARC itself may also contain buffers that are in any of
211  * the ARC_m* states - meaning that a buffer can exist in two
212  * places.  The reason for the ARC_l2c_only state is to keep the
213  * buffer header in the hash table, so that reads that hit the
214  * second level ARC benefit from these fast lookups.
215  */
216 
217 typedef struct arc_state {
218 	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
219 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
220 	uint64_t arcs_size;	/* total amount of data in this state */
221 	kmutex_t arcs_mtx;
222 } arc_state_t;
223 
224 /* The 6 states: */
225 static arc_state_t ARC_anon;
226 static arc_state_t ARC_mru;
227 static arc_state_t ARC_mru_ghost;
228 static arc_state_t ARC_mfu;
229 static arc_state_t ARC_mfu_ghost;
230 static arc_state_t ARC_l2c_only;
231 
232 typedef struct arc_stats {
233 	kstat_named_t arcstat_hits;
234 	kstat_named_t arcstat_misses;
235 	kstat_named_t arcstat_demand_data_hits;
236 	kstat_named_t arcstat_demand_data_misses;
237 	kstat_named_t arcstat_demand_metadata_hits;
238 	kstat_named_t arcstat_demand_metadata_misses;
239 	kstat_named_t arcstat_prefetch_data_hits;
240 	kstat_named_t arcstat_prefetch_data_misses;
241 	kstat_named_t arcstat_prefetch_metadata_hits;
242 	kstat_named_t arcstat_prefetch_metadata_misses;
243 	kstat_named_t arcstat_mru_hits;
244 	kstat_named_t arcstat_mru_ghost_hits;
245 	kstat_named_t arcstat_mfu_hits;
246 	kstat_named_t arcstat_mfu_ghost_hits;
247 	kstat_named_t arcstat_deleted;
248 	kstat_named_t arcstat_recycle_miss;
249 	kstat_named_t arcstat_mutex_miss;
250 	kstat_named_t arcstat_evict_skip;
251 	kstat_named_t arcstat_evict_l2_cached;
252 	kstat_named_t arcstat_evict_l2_eligible;
253 	kstat_named_t arcstat_evict_l2_ineligible;
254 	kstat_named_t arcstat_hash_elements;
255 	kstat_named_t arcstat_hash_elements_max;
256 	kstat_named_t arcstat_hash_collisions;
257 	kstat_named_t arcstat_hash_chains;
258 	kstat_named_t arcstat_hash_chain_max;
259 	kstat_named_t arcstat_p;
260 	kstat_named_t arcstat_c;
261 	kstat_named_t arcstat_c_min;
262 	kstat_named_t arcstat_c_max;
263 	kstat_named_t arcstat_size;
264 	kstat_named_t arcstat_hdr_size;
265 	kstat_named_t arcstat_data_size;
266 	kstat_named_t arcstat_other_size;
267 	kstat_named_t arcstat_l2_hits;
268 	kstat_named_t arcstat_l2_misses;
269 	kstat_named_t arcstat_l2_feeds;
270 	kstat_named_t arcstat_l2_rw_clash;
271 	kstat_named_t arcstat_l2_read_bytes;
272 	kstat_named_t arcstat_l2_write_bytes;
273 	kstat_named_t arcstat_l2_writes_sent;
274 	kstat_named_t arcstat_l2_writes_done;
275 	kstat_named_t arcstat_l2_writes_error;
276 	kstat_named_t arcstat_l2_writes_hdr_miss;
277 	kstat_named_t arcstat_l2_evict_lock_retry;
278 	kstat_named_t arcstat_l2_evict_reading;
279 	kstat_named_t arcstat_l2_free_on_write;
280 	kstat_named_t arcstat_l2_abort_lowmem;
281 	kstat_named_t arcstat_l2_cksum_bad;
282 	kstat_named_t arcstat_l2_io_error;
283 	kstat_named_t arcstat_l2_size;
284 	kstat_named_t arcstat_l2_hdr_size;
285 	kstat_named_t arcstat_memory_throttle_count;
286 } arc_stats_t;
287 
288 static arc_stats_t arc_stats = {
289 	{ "hits",			KSTAT_DATA_UINT64 },
290 	{ "misses",			KSTAT_DATA_UINT64 },
291 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
292 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
293 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
294 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
295 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
296 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
297 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
298 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
299 	{ "mru_hits",			KSTAT_DATA_UINT64 },
300 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
301 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
302 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
303 	{ "deleted",			KSTAT_DATA_UINT64 },
304 	{ "recycle_miss",		KSTAT_DATA_UINT64 },
305 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
306 	{ "evict_skip",			KSTAT_DATA_UINT64 },
307 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
308 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
309 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
310 	{ "hash_elements",		KSTAT_DATA_UINT64 },
311 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
312 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
313 	{ "hash_chains",		KSTAT_DATA_UINT64 },
314 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
315 	{ "p",				KSTAT_DATA_UINT64 },
316 	{ "c",				KSTAT_DATA_UINT64 },
317 	{ "c_min",			KSTAT_DATA_UINT64 },
318 	{ "c_max",			KSTAT_DATA_UINT64 },
319 	{ "size",			KSTAT_DATA_UINT64 },
320 	{ "hdr_size",			KSTAT_DATA_UINT64 },
321 	{ "data_size",			KSTAT_DATA_UINT64 },
322 	{ "other_size",			KSTAT_DATA_UINT64 },
323 	{ "l2_hits",			KSTAT_DATA_UINT64 },
324 	{ "l2_misses",			KSTAT_DATA_UINT64 },
325 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
326 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
327 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
328 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
329 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
330 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
331 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
332 	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
333 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
334 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
335 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
336 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
337 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
338 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
339 	{ "l2_size",			KSTAT_DATA_UINT64 },
340 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
341 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 }
342 };
343 
344 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
345 
346 #define	ARCSTAT_INCR(stat, val) \
347 	atomic_add_64(&arc_stats.stat.value.ui64, (val));
348 
349 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
350 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
351 
352 #define	ARCSTAT_MAX(stat, val) {					\
353 	uint64_t m;							\
354 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
355 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
356 		continue;						\
357 }
358 
359 #define	ARCSTAT_MAXSTAT(stat) \
360 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
361 
362 /*
363  * We define a macro to allow ARC hits/misses to be easily broken down by
364  * two separate conditions, giving a total of four different subtypes for
365  * each of hits and misses (so eight statistics total).
366  */
367 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
368 	if (cond1) {							\
369 		if (cond2) {						\
370 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
371 		} else {						\
372 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
373 		}							\
374 	} else {							\
375 		if (cond2) {						\
376 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
377 		} else {						\
378 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
379 		}							\
380 	}
381 
382 kstat_t			*arc_ksp;
383 static arc_state_t	*arc_anon;
384 static arc_state_t	*arc_mru;
385 static arc_state_t	*arc_mru_ghost;
386 static arc_state_t	*arc_mfu;
387 static arc_state_t	*arc_mfu_ghost;
388 static arc_state_t	*arc_l2c_only;
389 
390 /*
391  * There are several ARC variables that are critical to export as kstats --
392  * but we don't want to have to grovel around in the kstat whenever we wish to
393  * manipulate them.  For these variables, we therefore define them to be in
394  * terms of the statistic variable.  This assures that we are not introducing
395  * the possibility of inconsistency by having shadow copies of the variables,
396  * while still allowing the code to be readable.
397  */
398 #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
399 #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
400 #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
401 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
402 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
403 
404 static int		arc_no_grow;	/* Don't try to grow cache size */
405 static uint64_t		arc_tempreserve;
406 static uint64_t		arc_loaned_bytes;
407 static uint64_t		arc_meta_used;
408 static uint64_t		arc_meta_limit;
409 static uint64_t		arc_meta_max = 0;
410 
411 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
412 
413 typedef struct arc_callback arc_callback_t;
414 
415 struct arc_callback {
416 	void			*acb_private;
417 	arc_done_func_t		*acb_done;
418 	arc_buf_t		*acb_buf;
419 	zio_t			*acb_zio_dummy;
420 	arc_callback_t		*acb_next;
421 };
422 
423 typedef struct arc_write_callback arc_write_callback_t;
424 
425 struct arc_write_callback {
426 	void		*awcb_private;
427 	arc_done_func_t	*awcb_ready;
428 	arc_done_func_t	*awcb_done;
429 	arc_buf_t	*awcb_buf;
430 };
431 
432 struct arc_buf_hdr {
433 	/* protected by hash lock */
434 	dva_t			b_dva;
435 	uint64_t		b_birth;
436 	uint64_t		b_cksum0;
437 
438 	kmutex_t		b_freeze_lock;
439 	zio_cksum_t		*b_freeze_cksum;
440 
441 	arc_buf_hdr_t		*b_hash_next;
442 	arc_buf_t		*b_buf;
443 	uint32_t		b_flags;
444 	uint32_t		b_datacnt;
445 
446 	arc_callback_t		*b_acb;
447 	kcondvar_t		b_cv;
448 
449 	/* immutable */
450 	arc_buf_contents_t	b_type;
451 	uint64_t		b_size;
452 	uint64_t		b_spa;
453 
454 	/* protected by arc state mutex */
455 	arc_state_t		*b_state;
456 	list_node_t		b_arc_node;
457 
458 	/* updated atomically */
459 	clock_t			b_arc_access;
460 
461 	/* self protecting */
462 	refcount_t		b_refcnt;
463 
464 	l2arc_buf_hdr_t		*b_l2hdr;
465 	list_node_t		b_l2node;
466 };
467 
468 static arc_buf_t *arc_eviction_list;
469 static kmutex_t arc_eviction_mtx;
470 static arc_buf_hdr_t arc_eviction_hdr;
471 static void arc_get_data_buf(arc_buf_t *buf);
472 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
473 static int arc_evict_needed(arc_buf_contents_t type);
474 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
475 
476 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
477 
478 #define	GHOST_STATE(state)	\
479 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
480 	(state) == arc_l2c_only)
481 
482 /*
483  * Private ARC flags.  These flags are private ARC only flags that will show up
484  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
485  * be passed in as arc_flags in things like arc_read.  However, these flags
486  * should never be passed and should only be set by ARC code.  When adding new
487  * public flags, make sure not to smash the private ones.
488  */
489 
490 #define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
491 #define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
492 #define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
493 #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
494 #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
495 #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
496 #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
497 #define	ARC_L2_WRITING		(1 << 16)	/* L2ARC write in progress */
498 #define	ARC_L2_EVICTED		(1 << 17)	/* evicted during I/O */
499 #define	ARC_L2_WRITE_HEAD	(1 << 18)	/* head of write list */
500 
501 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
502 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
503 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
504 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_PREFETCH)
505 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
506 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
507 #define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
508 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_L2CACHE)
509 #define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
510 				    (hdr)->b_l2hdr != NULL)
511 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
512 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_L2_EVICTED)
513 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_L2_WRITE_HEAD)
514 
515 /*
516  * Other sizes
517  */
518 
519 #define	HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
520 #define	L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
521 
522 /*
523  * Hash table routines
524  */
525 
526 #define	HT_LOCK_PAD	64
527 
528 struct ht_lock {
529 	kmutex_t	ht_lock;
530 #ifdef _KERNEL
531 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
532 #endif
533 };
534 
535 #define	BUF_LOCKS 256
536 typedef struct buf_hash_table {
537 	uint64_t ht_mask;
538 	arc_buf_hdr_t **ht_table;
539 	struct ht_lock ht_locks[BUF_LOCKS];
540 } buf_hash_table_t;
541 
542 static buf_hash_table_t buf_hash_table;
543 
544 #define	BUF_HASH_INDEX(spa, dva, birth) \
545 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
546 #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
547 #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
548 #define	HDR_LOCK(buf) \
549 	(BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth)))
550 
551 uint64_t zfs_crc64_table[256];
552 
553 /*
554  * Level 2 ARC
555  */
556 
557 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
558 #define	L2ARC_HEADROOM		2		/* num of writes */
559 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
560 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
561 
562 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
563 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
564 
565 /*
566  * L2ARC Performance Tunables
567  */
568 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
569 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
570 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
571 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
572 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
573 boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
574 boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
575 boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
576 
577 /*
578  * L2ARC Internals
579  */
580 typedef struct l2arc_dev {
581 	vdev_t			*l2ad_vdev;	/* vdev */
582 	spa_t			*l2ad_spa;	/* spa */
583 	uint64_t		l2ad_hand;	/* next write location */
584 	uint64_t		l2ad_write;	/* desired write size, bytes */
585 	uint64_t		l2ad_boost;	/* warmup write boost, bytes */
586 	uint64_t		l2ad_start;	/* first addr on device */
587 	uint64_t		l2ad_end;	/* last addr on device */
588 	uint64_t		l2ad_evict;	/* last addr eviction reached */
589 	boolean_t		l2ad_first;	/* first sweep through */
590 	boolean_t		l2ad_writing;	/* currently writing */
591 	list_t			*l2ad_buflist;	/* buffer list */
592 	list_node_t		l2ad_node;	/* device list node */
593 } l2arc_dev_t;
594 
595 static list_t L2ARC_dev_list;			/* device list */
596 static list_t *l2arc_dev_list;			/* device list pointer */
597 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
598 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
599 static kmutex_t l2arc_buflist_mtx;		/* mutex for all buflists */
600 static list_t L2ARC_free_on_write;		/* free after write buf list */
601 static list_t *l2arc_free_on_write;		/* free after write list ptr */
602 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
603 static uint64_t l2arc_ndev;			/* number of devices */
604 
605 typedef struct l2arc_read_callback {
606 	arc_buf_t	*l2rcb_buf;		/* read buffer */
607 	spa_t		*l2rcb_spa;		/* spa */
608 	blkptr_t	l2rcb_bp;		/* original blkptr */
609 	zbookmark_t	l2rcb_zb;		/* original bookmark */
610 	int		l2rcb_flags;		/* original flags */
611 } l2arc_read_callback_t;
612 
613 typedef struct l2arc_write_callback {
614 	l2arc_dev_t	*l2wcb_dev;		/* device info */
615 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
616 } l2arc_write_callback_t;
617 
618 struct l2arc_buf_hdr {
619 	/* protected by arc_buf_hdr  mutex */
620 	l2arc_dev_t	*b_dev;			/* L2ARC device */
621 	uint64_t	b_daddr;		/* disk address, offset byte */
622 };
623 
624 typedef struct l2arc_data_free {
625 	/* protected by l2arc_free_on_write_mtx */
626 	void		*l2df_data;
627 	size_t		l2df_size;
628 	void		(*l2df_func)(void *, size_t);
629 	list_node_t	l2df_list_node;
630 } l2arc_data_free_t;
631 
632 static kmutex_t l2arc_feed_thr_lock;
633 static kcondvar_t l2arc_feed_thr_cv;
634 static uint8_t l2arc_thread_exit;
635 
636 static void l2arc_read_done(zio_t *zio);
637 static void l2arc_hdr_stat_add(void);
638 static void l2arc_hdr_stat_remove(void);
639 
640 static uint64_t
641 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
642 {
643 	uint8_t *vdva = (uint8_t *)dva;
644 	uint64_t crc = -1ULL;
645 	int i;
646 
647 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
648 
649 	for (i = 0; i < sizeof (dva_t); i++)
650 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
651 
652 	crc ^= (spa>>8) ^ birth;
653 
654 	return (crc);
655 }
656 
657 #define	BUF_EMPTY(buf)						\
658 	((buf)->b_dva.dva_word[0] == 0 &&			\
659 	(buf)->b_dva.dva_word[1] == 0 &&			\
660 	(buf)->b_birth == 0)
661 
662 #define	BUF_EQUAL(spa, dva, birth, buf)				\
663 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
664 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
665 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
666 
667 static arc_buf_hdr_t *
668 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
669 {
670 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
671 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
672 	arc_buf_hdr_t *buf;
673 
674 	mutex_enter(hash_lock);
675 	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
676 	    buf = buf->b_hash_next) {
677 		if (BUF_EQUAL(spa, dva, birth, buf)) {
678 			*lockp = hash_lock;
679 			return (buf);
680 		}
681 	}
682 	mutex_exit(hash_lock);
683 	*lockp = NULL;
684 	return (NULL);
685 }
686 
687 /*
688  * Insert an entry into the hash table.  If there is already an element
689  * equal to elem in the hash table, then the already existing element
690  * will be returned and the new element will not be inserted.
691  * Otherwise returns NULL.
692  */
693 static arc_buf_hdr_t *
694 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
695 {
696 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
697 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
698 	arc_buf_hdr_t *fbuf;
699 	uint32_t i;
700 
701 	ASSERT(!HDR_IN_HASH_TABLE(buf));
702 	*lockp = hash_lock;
703 	mutex_enter(hash_lock);
704 	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
705 	    fbuf = fbuf->b_hash_next, i++) {
706 		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
707 			return (fbuf);
708 	}
709 
710 	buf->b_hash_next = buf_hash_table.ht_table[idx];
711 	buf_hash_table.ht_table[idx] = buf;
712 	buf->b_flags |= ARC_IN_HASH_TABLE;
713 
714 	/* collect some hash table performance data */
715 	if (i > 0) {
716 		ARCSTAT_BUMP(arcstat_hash_collisions);
717 		if (i == 1)
718 			ARCSTAT_BUMP(arcstat_hash_chains);
719 
720 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
721 	}
722 
723 	ARCSTAT_BUMP(arcstat_hash_elements);
724 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
725 
726 	return (NULL);
727 }
728 
729 static void
730 buf_hash_remove(arc_buf_hdr_t *buf)
731 {
732 	arc_buf_hdr_t *fbuf, **bufp;
733 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
734 
735 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
736 	ASSERT(HDR_IN_HASH_TABLE(buf));
737 
738 	bufp = &buf_hash_table.ht_table[idx];
739 	while ((fbuf = *bufp) != buf) {
740 		ASSERT(fbuf != NULL);
741 		bufp = &fbuf->b_hash_next;
742 	}
743 	*bufp = buf->b_hash_next;
744 	buf->b_hash_next = NULL;
745 	buf->b_flags &= ~ARC_IN_HASH_TABLE;
746 
747 	/* collect some hash table performance data */
748 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
749 
750 	if (buf_hash_table.ht_table[idx] &&
751 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
752 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
753 }
754 
755 /*
756  * Global data structures and functions for the buf kmem cache.
757  */
758 static kmem_cache_t *hdr_cache;
759 static kmem_cache_t *buf_cache;
760 
761 static void
762 buf_fini(void)
763 {
764 	int i;
765 
766 	kmem_free(buf_hash_table.ht_table,
767 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
768 	for (i = 0; i < BUF_LOCKS; i++)
769 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
770 	kmem_cache_destroy(hdr_cache);
771 	kmem_cache_destroy(buf_cache);
772 }
773 
774 /*
775  * Constructor callback - called when the cache is empty
776  * and a new buf is requested.
777  */
778 /* ARGSUSED */
779 static int
780 hdr_cons(void *vbuf, void *unused, int kmflag)
781 {
782 	arc_buf_hdr_t *buf = vbuf;
783 
784 	bzero(buf, sizeof (arc_buf_hdr_t));
785 	refcount_create(&buf->b_refcnt);
786 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
787 	mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
788 	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
789 
790 	return (0);
791 }
792 
793 /* ARGSUSED */
794 static int
795 buf_cons(void *vbuf, void *unused, int kmflag)
796 {
797 	arc_buf_t *buf = vbuf;
798 
799 	bzero(buf, sizeof (arc_buf_t));
800 	rw_init(&buf->b_lock, NULL, RW_DEFAULT, NULL);
801 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
802 
803 	return (0);
804 }
805 
806 /*
807  * Destructor callback - called when a cached buf is
808  * no longer required.
809  */
810 /* ARGSUSED */
811 static void
812 hdr_dest(void *vbuf, void *unused)
813 {
814 	arc_buf_hdr_t *buf = vbuf;
815 
816 	ASSERT(BUF_EMPTY(buf));
817 	refcount_destroy(&buf->b_refcnt);
818 	cv_destroy(&buf->b_cv);
819 	mutex_destroy(&buf->b_freeze_lock);
820 	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
821 }
822 
823 /* ARGSUSED */
824 static void
825 buf_dest(void *vbuf, void *unused)
826 {
827 	arc_buf_t *buf = vbuf;
828 
829 	rw_destroy(&buf->b_lock);
830 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
831 }
832 
833 /*
834  * Reclaim callback -- invoked when memory is low.
835  */
836 /* ARGSUSED */
837 static void
838 hdr_recl(void *unused)
839 {
840 	dprintf("hdr_recl called\n");
841 	/*
842 	 * umem calls the reclaim func when we destroy the buf cache,
843 	 * which is after we do arc_fini().
844 	 */
845 	if (!arc_dead)
846 		cv_signal(&arc_reclaim_thr_cv);
847 }
848 
849 static void
850 buf_init(void)
851 {
852 	uint64_t *ct;
853 	uint64_t hsize = 1ULL << 12;
854 	int i, j;
855 
856 	/*
857 	 * The hash table is big enough to fill all of physical memory
858 	 * with an average 64K block size.  The table will take up
859 	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
860 	 */
861 	while (hsize * 65536 < physmem * PAGESIZE)
862 		hsize <<= 1;
863 retry:
864 	buf_hash_table.ht_mask = hsize - 1;
865 	buf_hash_table.ht_table =
866 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
867 	if (buf_hash_table.ht_table == NULL) {
868 		ASSERT(hsize > (1ULL << 8));
869 		hsize >>= 1;
870 		goto retry;
871 	}
872 
873 	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
874 	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
875 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
876 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
877 
878 	for (i = 0; i < 256; i++)
879 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
880 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
881 
882 	for (i = 0; i < BUF_LOCKS; i++) {
883 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
884 		    NULL, MUTEX_DEFAULT, NULL);
885 	}
886 }
887 
888 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
889 
890 static void
891 arc_cksum_verify(arc_buf_t *buf)
892 {
893 	zio_cksum_t zc;
894 
895 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
896 		return;
897 
898 	mutex_enter(&buf->b_hdr->b_freeze_lock);
899 	if (buf->b_hdr->b_freeze_cksum == NULL ||
900 	    (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
901 		mutex_exit(&buf->b_hdr->b_freeze_lock);
902 		return;
903 	}
904 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
905 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
906 		panic("buffer modified while frozen!");
907 	mutex_exit(&buf->b_hdr->b_freeze_lock);
908 }
909 
910 static int
911 arc_cksum_equal(arc_buf_t *buf)
912 {
913 	zio_cksum_t zc;
914 	int equal;
915 
916 	mutex_enter(&buf->b_hdr->b_freeze_lock);
917 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
918 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
919 	mutex_exit(&buf->b_hdr->b_freeze_lock);
920 
921 	return (equal);
922 }
923 
924 static void
925 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
926 {
927 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
928 		return;
929 
930 	mutex_enter(&buf->b_hdr->b_freeze_lock);
931 	if (buf->b_hdr->b_freeze_cksum != NULL) {
932 		mutex_exit(&buf->b_hdr->b_freeze_lock);
933 		return;
934 	}
935 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
936 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
937 	    buf->b_hdr->b_freeze_cksum);
938 	mutex_exit(&buf->b_hdr->b_freeze_lock);
939 }
940 
941 void
942 arc_buf_thaw(arc_buf_t *buf)
943 {
944 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
945 		if (buf->b_hdr->b_state != arc_anon)
946 			panic("modifying non-anon buffer!");
947 		if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
948 			panic("modifying buffer while i/o in progress!");
949 		arc_cksum_verify(buf);
950 	}
951 
952 	mutex_enter(&buf->b_hdr->b_freeze_lock);
953 	if (buf->b_hdr->b_freeze_cksum != NULL) {
954 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
955 		buf->b_hdr->b_freeze_cksum = NULL;
956 	}
957 	mutex_exit(&buf->b_hdr->b_freeze_lock);
958 }
959 
960 void
961 arc_buf_freeze(arc_buf_t *buf)
962 {
963 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
964 		return;
965 
966 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
967 	    buf->b_hdr->b_state == arc_anon);
968 	arc_cksum_compute(buf, B_FALSE);
969 }
970 
971 static void
972 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
973 {
974 	ASSERT(MUTEX_HELD(hash_lock));
975 
976 	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
977 	    (ab->b_state != arc_anon)) {
978 		uint64_t delta = ab->b_size * ab->b_datacnt;
979 		list_t *list = &ab->b_state->arcs_list[ab->b_type];
980 		uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
981 
982 		ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
983 		mutex_enter(&ab->b_state->arcs_mtx);
984 		ASSERT(list_link_active(&ab->b_arc_node));
985 		list_remove(list, ab);
986 		if (GHOST_STATE(ab->b_state)) {
987 			ASSERT3U(ab->b_datacnt, ==, 0);
988 			ASSERT3P(ab->b_buf, ==, NULL);
989 			delta = ab->b_size;
990 		}
991 		ASSERT(delta > 0);
992 		ASSERT3U(*size, >=, delta);
993 		atomic_add_64(size, -delta);
994 		mutex_exit(&ab->b_state->arcs_mtx);
995 		/* remove the prefetch flag if we get a reference */
996 		if (ab->b_flags & ARC_PREFETCH)
997 			ab->b_flags &= ~ARC_PREFETCH;
998 	}
999 }
1000 
1001 static int
1002 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1003 {
1004 	int cnt;
1005 	arc_state_t *state = ab->b_state;
1006 
1007 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1008 	ASSERT(!GHOST_STATE(state));
1009 
1010 	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
1011 	    (state != arc_anon)) {
1012 		uint64_t *size = &state->arcs_lsize[ab->b_type];
1013 
1014 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
1015 		mutex_enter(&state->arcs_mtx);
1016 		ASSERT(!list_link_active(&ab->b_arc_node));
1017 		list_insert_head(&state->arcs_list[ab->b_type], ab);
1018 		ASSERT(ab->b_datacnt > 0);
1019 		atomic_add_64(size, ab->b_size * ab->b_datacnt);
1020 		mutex_exit(&state->arcs_mtx);
1021 	}
1022 	return (cnt);
1023 }
1024 
1025 /*
1026  * Move the supplied buffer to the indicated state.  The mutex
1027  * for the buffer must be held by the caller.
1028  */
1029 static void
1030 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1031 {
1032 	arc_state_t *old_state = ab->b_state;
1033 	int64_t refcnt = refcount_count(&ab->b_refcnt);
1034 	uint64_t from_delta, to_delta;
1035 
1036 	ASSERT(MUTEX_HELD(hash_lock));
1037 	ASSERT(new_state != old_state);
1038 	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1039 	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1040 	ASSERT(ab->b_datacnt <= 1 || new_state != arc_anon);
1041 	ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1042 
1043 	from_delta = to_delta = ab->b_datacnt * ab->b_size;
1044 
1045 	/*
1046 	 * If this buffer is evictable, transfer it from the
1047 	 * old state list to the new state list.
1048 	 */
1049 	if (refcnt == 0) {
1050 		if (old_state != arc_anon) {
1051 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
1052 			uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1053 
1054 			if (use_mutex)
1055 				mutex_enter(&old_state->arcs_mtx);
1056 
1057 			ASSERT(list_link_active(&ab->b_arc_node));
1058 			list_remove(&old_state->arcs_list[ab->b_type], ab);
1059 
1060 			/*
1061 			 * If prefetching out of the ghost cache,
1062 			 * we will have a non-null datacnt.
1063 			 */
1064 			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
1065 				/* ghost elements have a ghost size */
1066 				ASSERT(ab->b_buf == NULL);
1067 				from_delta = ab->b_size;
1068 			}
1069 			ASSERT3U(*size, >=, from_delta);
1070 			atomic_add_64(size, -from_delta);
1071 
1072 			if (use_mutex)
1073 				mutex_exit(&old_state->arcs_mtx);
1074 		}
1075 		if (new_state != arc_anon) {
1076 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
1077 			uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1078 
1079 			if (use_mutex)
1080 				mutex_enter(&new_state->arcs_mtx);
1081 
1082 			list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1083 
1084 			/* ghost elements have a ghost size */
1085 			if (GHOST_STATE(new_state)) {
1086 				ASSERT(ab->b_datacnt == 0);
1087 				ASSERT(ab->b_buf == NULL);
1088 				to_delta = ab->b_size;
1089 			}
1090 			atomic_add_64(size, to_delta);
1091 
1092 			if (use_mutex)
1093 				mutex_exit(&new_state->arcs_mtx);
1094 		}
1095 	}
1096 
1097 	ASSERT(!BUF_EMPTY(ab));
1098 	if (new_state == arc_anon) {
1099 		buf_hash_remove(ab);
1100 	}
1101 
1102 	/* adjust state sizes */
1103 	if (to_delta)
1104 		atomic_add_64(&new_state->arcs_size, to_delta);
1105 	if (from_delta) {
1106 		ASSERT3U(old_state->arcs_size, >=, from_delta);
1107 		atomic_add_64(&old_state->arcs_size, -from_delta);
1108 	}
1109 	ab->b_state = new_state;
1110 
1111 	/* adjust l2arc hdr stats */
1112 	if (new_state == arc_l2c_only)
1113 		l2arc_hdr_stat_add();
1114 	else if (old_state == arc_l2c_only)
1115 		l2arc_hdr_stat_remove();
1116 }
1117 
1118 void
1119 arc_space_consume(uint64_t space, arc_space_type_t type)
1120 {
1121 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1122 
1123 	switch (type) {
1124 	case ARC_SPACE_DATA:
1125 		ARCSTAT_INCR(arcstat_data_size, space);
1126 		break;
1127 	case ARC_SPACE_OTHER:
1128 		ARCSTAT_INCR(arcstat_other_size, space);
1129 		break;
1130 	case ARC_SPACE_HDRS:
1131 		ARCSTAT_INCR(arcstat_hdr_size, space);
1132 		break;
1133 	case ARC_SPACE_L2HDRS:
1134 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1135 		break;
1136 	}
1137 
1138 	atomic_add_64(&arc_meta_used, space);
1139 	atomic_add_64(&arc_size, space);
1140 }
1141 
1142 void
1143 arc_space_return(uint64_t space, arc_space_type_t type)
1144 {
1145 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1146 
1147 	switch (type) {
1148 	case ARC_SPACE_DATA:
1149 		ARCSTAT_INCR(arcstat_data_size, -space);
1150 		break;
1151 	case ARC_SPACE_OTHER:
1152 		ARCSTAT_INCR(arcstat_other_size, -space);
1153 		break;
1154 	case ARC_SPACE_HDRS:
1155 		ARCSTAT_INCR(arcstat_hdr_size, -space);
1156 		break;
1157 	case ARC_SPACE_L2HDRS:
1158 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1159 		break;
1160 	}
1161 
1162 	ASSERT(arc_meta_used >= space);
1163 	if (arc_meta_max < arc_meta_used)
1164 		arc_meta_max = arc_meta_used;
1165 	atomic_add_64(&arc_meta_used, -space);
1166 	ASSERT(arc_size >= space);
1167 	atomic_add_64(&arc_size, -space);
1168 }
1169 
1170 void *
1171 arc_data_buf_alloc(uint64_t size)
1172 {
1173 	if (arc_evict_needed(ARC_BUFC_DATA))
1174 		cv_signal(&arc_reclaim_thr_cv);
1175 	atomic_add_64(&arc_size, size);
1176 	return (zio_data_buf_alloc(size));
1177 }
1178 
1179 void
1180 arc_data_buf_free(void *buf, uint64_t size)
1181 {
1182 	zio_data_buf_free(buf, size);
1183 	ASSERT(arc_size >= size);
1184 	atomic_add_64(&arc_size, -size);
1185 }
1186 
1187 arc_buf_t *
1188 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1189 {
1190 	arc_buf_hdr_t *hdr;
1191 	arc_buf_t *buf;
1192 
1193 	ASSERT3U(size, >, 0);
1194 	hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1195 	ASSERT(BUF_EMPTY(hdr));
1196 	hdr->b_size = size;
1197 	hdr->b_type = type;
1198 	hdr->b_spa = spa_guid(spa);
1199 	hdr->b_state = arc_anon;
1200 	hdr->b_arc_access = 0;
1201 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1202 	buf->b_hdr = hdr;
1203 	buf->b_data = NULL;
1204 	buf->b_efunc = NULL;
1205 	buf->b_private = NULL;
1206 	buf->b_next = NULL;
1207 	hdr->b_buf = buf;
1208 	arc_get_data_buf(buf);
1209 	hdr->b_datacnt = 1;
1210 	hdr->b_flags = 0;
1211 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
1212 	(void) refcount_add(&hdr->b_refcnt, tag);
1213 
1214 	return (buf);
1215 }
1216 
1217 static char *arc_onloan_tag = "onloan";
1218 
1219 /*
1220  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1221  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1222  * buffers must be returned to the arc before they can be used by the DMU or
1223  * freed.
1224  */
1225 arc_buf_t *
1226 arc_loan_buf(spa_t *spa, int size)
1227 {
1228 	arc_buf_t *buf;
1229 
1230 	buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1231 
1232 	atomic_add_64(&arc_loaned_bytes, size);
1233 	return (buf);
1234 }
1235 
1236 /*
1237  * Return a loaned arc buffer to the arc.
1238  */
1239 void
1240 arc_return_buf(arc_buf_t *buf, void *tag)
1241 {
1242 	arc_buf_hdr_t *hdr = buf->b_hdr;
1243 
1244 	ASSERT(buf->b_data != NULL);
1245 	(void) refcount_add(&hdr->b_refcnt, tag);
1246 	(void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1247 
1248 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1249 }
1250 
1251 /* Detach an arc_buf from a dbuf (tag) */
1252 void
1253 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1254 {
1255 	arc_buf_hdr_t *hdr;
1256 
1257 	rw_enter(&buf->b_lock, RW_WRITER);
1258 	ASSERT(buf->b_data != NULL);
1259 	hdr = buf->b_hdr;
1260 	(void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1261 	(void) refcount_remove(&hdr->b_refcnt, tag);
1262 	buf->b_efunc = NULL;
1263 	buf->b_private = NULL;
1264 
1265 	atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1266 	rw_exit(&buf->b_lock);
1267 }
1268 
1269 static arc_buf_t *
1270 arc_buf_clone(arc_buf_t *from)
1271 {
1272 	arc_buf_t *buf;
1273 	arc_buf_hdr_t *hdr = from->b_hdr;
1274 	uint64_t size = hdr->b_size;
1275 
1276 	ASSERT(hdr->b_state != arc_anon);
1277 
1278 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1279 	buf->b_hdr = hdr;
1280 	buf->b_data = NULL;
1281 	buf->b_efunc = NULL;
1282 	buf->b_private = NULL;
1283 	buf->b_next = hdr->b_buf;
1284 	hdr->b_buf = buf;
1285 	arc_get_data_buf(buf);
1286 	bcopy(from->b_data, buf->b_data, size);
1287 	hdr->b_datacnt += 1;
1288 	return (buf);
1289 }
1290 
1291 void
1292 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1293 {
1294 	arc_buf_hdr_t *hdr;
1295 	kmutex_t *hash_lock;
1296 
1297 	/*
1298 	 * Check to see if this buffer is evicted.  Callers
1299 	 * must verify b_data != NULL to know if the add_ref
1300 	 * was successful.
1301 	 */
1302 	rw_enter(&buf->b_lock, RW_READER);
1303 	if (buf->b_data == NULL) {
1304 		rw_exit(&buf->b_lock);
1305 		return;
1306 	}
1307 	hdr = buf->b_hdr;
1308 	ASSERT(hdr != NULL);
1309 	hash_lock = HDR_LOCK(hdr);
1310 	mutex_enter(hash_lock);
1311 	rw_exit(&buf->b_lock);
1312 
1313 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1314 	add_reference(hdr, hash_lock, tag);
1315 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1316 	arc_access(hdr, hash_lock);
1317 	mutex_exit(hash_lock);
1318 	ARCSTAT_BUMP(arcstat_hits);
1319 	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1320 	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1321 	    data, metadata, hits);
1322 }
1323 
1324 /*
1325  * Free the arc data buffer.  If it is an l2arc write in progress,
1326  * the buffer is placed on l2arc_free_on_write to be freed later.
1327  */
1328 static void
1329 arc_buf_data_free(arc_buf_hdr_t *hdr, void (*free_func)(void *, size_t),
1330     void *data, size_t size)
1331 {
1332 	if (HDR_L2_WRITING(hdr)) {
1333 		l2arc_data_free_t *df;
1334 		df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1335 		df->l2df_data = data;
1336 		df->l2df_size = size;
1337 		df->l2df_func = free_func;
1338 		mutex_enter(&l2arc_free_on_write_mtx);
1339 		list_insert_head(l2arc_free_on_write, df);
1340 		mutex_exit(&l2arc_free_on_write_mtx);
1341 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
1342 	} else {
1343 		free_func(data, size);
1344 	}
1345 }
1346 
1347 static void
1348 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1349 {
1350 	arc_buf_t **bufp;
1351 
1352 	/* free up data associated with the buf */
1353 	if (buf->b_data) {
1354 		arc_state_t *state = buf->b_hdr->b_state;
1355 		uint64_t size = buf->b_hdr->b_size;
1356 		arc_buf_contents_t type = buf->b_hdr->b_type;
1357 
1358 		arc_cksum_verify(buf);
1359 
1360 		if (!recycle) {
1361 			if (type == ARC_BUFC_METADATA) {
1362 				arc_buf_data_free(buf->b_hdr, zio_buf_free,
1363 				    buf->b_data, size);
1364 				arc_space_return(size, ARC_SPACE_DATA);
1365 			} else {
1366 				ASSERT(type == ARC_BUFC_DATA);
1367 				arc_buf_data_free(buf->b_hdr,
1368 				    zio_data_buf_free, buf->b_data, size);
1369 				ARCSTAT_INCR(arcstat_data_size, -size);
1370 				atomic_add_64(&arc_size, -size);
1371 			}
1372 		}
1373 		if (list_link_active(&buf->b_hdr->b_arc_node)) {
1374 			uint64_t *cnt = &state->arcs_lsize[type];
1375 
1376 			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
1377 			ASSERT(state != arc_anon);
1378 
1379 			ASSERT3U(*cnt, >=, size);
1380 			atomic_add_64(cnt, -size);
1381 		}
1382 		ASSERT3U(state->arcs_size, >=, size);
1383 		atomic_add_64(&state->arcs_size, -size);
1384 		buf->b_data = NULL;
1385 		ASSERT(buf->b_hdr->b_datacnt > 0);
1386 		buf->b_hdr->b_datacnt -= 1;
1387 	}
1388 
1389 	/* only remove the buf if requested */
1390 	if (!all)
1391 		return;
1392 
1393 	/* remove the buf from the hdr list */
1394 	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1395 		continue;
1396 	*bufp = buf->b_next;
1397 
1398 	ASSERT(buf->b_efunc == NULL);
1399 
1400 	/* clean up the buf */
1401 	buf->b_hdr = NULL;
1402 	kmem_cache_free(buf_cache, buf);
1403 }
1404 
1405 static void
1406 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1407 {
1408 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
1409 	ASSERT3P(hdr->b_state, ==, arc_anon);
1410 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1411 	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1412 
1413 	if (l2hdr != NULL) {
1414 		boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1415 		/*
1416 		 * To prevent arc_free() and l2arc_evict() from
1417 		 * attempting to free the same buffer at the same time,
1418 		 * a FREE_IN_PROGRESS flag is given to arc_free() to
1419 		 * give it priority.  l2arc_evict() can't destroy this
1420 		 * header while we are waiting on l2arc_buflist_mtx.
1421 		 *
1422 		 * The hdr may be removed from l2ad_buflist before we
1423 		 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1424 		 */
1425 		if (!buflist_held) {
1426 			mutex_enter(&l2arc_buflist_mtx);
1427 			l2hdr = hdr->b_l2hdr;
1428 		}
1429 
1430 		if (l2hdr != NULL) {
1431 			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1432 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1433 			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1434 			if (hdr->b_state == arc_l2c_only)
1435 				l2arc_hdr_stat_remove();
1436 			hdr->b_l2hdr = NULL;
1437 		}
1438 
1439 		if (!buflist_held)
1440 			mutex_exit(&l2arc_buflist_mtx);
1441 	}
1442 
1443 	if (!BUF_EMPTY(hdr)) {
1444 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
1445 		bzero(&hdr->b_dva, sizeof (dva_t));
1446 		hdr->b_birth = 0;
1447 		hdr->b_cksum0 = 0;
1448 	}
1449 	while (hdr->b_buf) {
1450 		arc_buf_t *buf = hdr->b_buf;
1451 
1452 		if (buf->b_efunc) {
1453 			mutex_enter(&arc_eviction_mtx);
1454 			rw_enter(&buf->b_lock, RW_WRITER);
1455 			ASSERT(buf->b_hdr != NULL);
1456 			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1457 			hdr->b_buf = buf->b_next;
1458 			buf->b_hdr = &arc_eviction_hdr;
1459 			buf->b_next = arc_eviction_list;
1460 			arc_eviction_list = buf;
1461 			rw_exit(&buf->b_lock);
1462 			mutex_exit(&arc_eviction_mtx);
1463 		} else {
1464 			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1465 		}
1466 	}
1467 	if (hdr->b_freeze_cksum != NULL) {
1468 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1469 		hdr->b_freeze_cksum = NULL;
1470 	}
1471 
1472 	ASSERT(!list_link_active(&hdr->b_arc_node));
1473 	ASSERT3P(hdr->b_hash_next, ==, NULL);
1474 	ASSERT3P(hdr->b_acb, ==, NULL);
1475 	kmem_cache_free(hdr_cache, hdr);
1476 }
1477 
1478 void
1479 arc_buf_free(arc_buf_t *buf, void *tag)
1480 {
1481 	arc_buf_hdr_t *hdr = buf->b_hdr;
1482 	int hashed = hdr->b_state != arc_anon;
1483 
1484 	ASSERT(buf->b_efunc == NULL);
1485 	ASSERT(buf->b_data != NULL);
1486 
1487 	if (hashed) {
1488 		kmutex_t *hash_lock = HDR_LOCK(hdr);
1489 
1490 		mutex_enter(hash_lock);
1491 		(void) remove_reference(hdr, hash_lock, tag);
1492 		if (hdr->b_datacnt > 1) {
1493 			arc_buf_destroy(buf, FALSE, TRUE);
1494 		} else {
1495 			ASSERT(buf == hdr->b_buf);
1496 			ASSERT(buf->b_efunc == NULL);
1497 			hdr->b_flags |= ARC_BUF_AVAILABLE;
1498 		}
1499 		mutex_exit(hash_lock);
1500 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
1501 		int destroy_hdr;
1502 		/*
1503 		 * We are in the middle of an async write.  Don't destroy
1504 		 * this buffer unless the write completes before we finish
1505 		 * decrementing the reference count.
1506 		 */
1507 		mutex_enter(&arc_eviction_mtx);
1508 		(void) remove_reference(hdr, NULL, tag);
1509 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
1510 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1511 		mutex_exit(&arc_eviction_mtx);
1512 		if (destroy_hdr)
1513 			arc_hdr_destroy(hdr);
1514 	} else {
1515 		if (remove_reference(hdr, NULL, tag) > 0) {
1516 			ASSERT(HDR_IO_ERROR(hdr));
1517 			arc_buf_destroy(buf, FALSE, TRUE);
1518 		} else {
1519 			arc_hdr_destroy(hdr);
1520 		}
1521 	}
1522 }
1523 
1524 int
1525 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1526 {
1527 	arc_buf_hdr_t *hdr = buf->b_hdr;
1528 	kmutex_t *hash_lock = HDR_LOCK(hdr);
1529 	int no_callback = (buf->b_efunc == NULL);
1530 
1531 	if (hdr->b_state == arc_anon) {
1532 		ASSERT(hdr->b_datacnt == 1);
1533 		arc_buf_free(buf, tag);
1534 		return (no_callback);
1535 	}
1536 
1537 	mutex_enter(hash_lock);
1538 	ASSERT(hdr->b_state != arc_anon);
1539 	ASSERT(buf->b_data != NULL);
1540 
1541 	(void) remove_reference(hdr, hash_lock, tag);
1542 	if (hdr->b_datacnt > 1) {
1543 		if (no_callback)
1544 			arc_buf_destroy(buf, FALSE, TRUE);
1545 	} else if (no_callback) {
1546 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1547 		ASSERT(buf->b_efunc == NULL);
1548 		hdr->b_flags |= ARC_BUF_AVAILABLE;
1549 	}
1550 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
1551 	    refcount_is_zero(&hdr->b_refcnt));
1552 	mutex_exit(hash_lock);
1553 	return (no_callback);
1554 }
1555 
1556 int
1557 arc_buf_size(arc_buf_t *buf)
1558 {
1559 	return (buf->b_hdr->b_size);
1560 }
1561 
1562 /*
1563  * Evict buffers from list until we've removed the specified number of
1564  * bytes.  Move the removed buffers to the appropriate evict state.
1565  * If the recycle flag is set, then attempt to "recycle" a buffer:
1566  * - look for a buffer to evict that is `bytes' long.
1567  * - return the data block from this buffer rather than freeing it.
1568  * This flag is used by callers that are trying to make space for a
1569  * new buffer in a full arc cache.
1570  *
1571  * This function makes a "best effort".  It skips over any buffers
1572  * it can't get a hash_lock on, and so may not catch all candidates.
1573  * It may also return without evicting as much space as requested.
1574  */
1575 static void *
1576 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1577     arc_buf_contents_t type)
1578 {
1579 	arc_state_t *evicted_state;
1580 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1581 	arc_buf_hdr_t *ab, *ab_prev = NULL;
1582 	list_t *list = &state->arcs_list[type];
1583 	kmutex_t *hash_lock;
1584 	boolean_t have_lock;
1585 	void *stolen = NULL;
1586 
1587 	ASSERT(state == arc_mru || state == arc_mfu);
1588 
1589 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1590 
1591 	mutex_enter(&state->arcs_mtx);
1592 	mutex_enter(&evicted_state->arcs_mtx);
1593 
1594 	for (ab = list_tail(list); ab; ab = ab_prev) {
1595 		ab_prev = list_prev(list, ab);
1596 		/* prefetch buffers have a minimum lifespan */
1597 		if (HDR_IO_IN_PROGRESS(ab) ||
1598 		    (spa && ab->b_spa != spa) ||
1599 		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1600 		    ddi_get_lbolt() - ab->b_arc_access <
1601 		    arc_min_prefetch_lifespan)) {
1602 			skipped++;
1603 			continue;
1604 		}
1605 		/* "lookahead" for better eviction candidate */
1606 		if (recycle && ab->b_size != bytes &&
1607 		    ab_prev && ab_prev->b_size == bytes)
1608 			continue;
1609 		hash_lock = HDR_LOCK(ab);
1610 		have_lock = MUTEX_HELD(hash_lock);
1611 		if (have_lock || mutex_tryenter(hash_lock)) {
1612 			ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0);
1613 			ASSERT(ab->b_datacnt > 0);
1614 			while (ab->b_buf) {
1615 				arc_buf_t *buf = ab->b_buf;
1616 				if (!rw_tryenter(&buf->b_lock, RW_WRITER)) {
1617 					missed += 1;
1618 					break;
1619 				}
1620 				if (buf->b_data) {
1621 					bytes_evicted += ab->b_size;
1622 					if (recycle && ab->b_type == type &&
1623 					    ab->b_size == bytes &&
1624 					    !HDR_L2_WRITING(ab)) {
1625 						stolen = buf->b_data;
1626 						recycle = FALSE;
1627 					}
1628 				}
1629 				if (buf->b_efunc) {
1630 					mutex_enter(&arc_eviction_mtx);
1631 					arc_buf_destroy(buf,
1632 					    buf->b_data == stolen, FALSE);
1633 					ab->b_buf = buf->b_next;
1634 					buf->b_hdr = &arc_eviction_hdr;
1635 					buf->b_next = arc_eviction_list;
1636 					arc_eviction_list = buf;
1637 					mutex_exit(&arc_eviction_mtx);
1638 					rw_exit(&buf->b_lock);
1639 				} else {
1640 					rw_exit(&buf->b_lock);
1641 					arc_buf_destroy(buf,
1642 					    buf->b_data == stolen, TRUE);
1643 				}
1644 			}
1645 
1646 			if (ab->b_l2hdr) {
1647 				ARCSTAT_INCR(arcstat_evict_l2_cached,
1648 				    ab->b_size);
1649 			} else {
1650 				if (l2arc_write_eligible(ab->b_spa, ab)) {
1651 					ARCSTAT_INCR(arcstat_evict_l2_eligible,
1652 					    ab->b_size);
1653 				} else {
1654 					ARCSTAT_INCR(
1655 					    arcstat_evict_l2_ineligible,
1656 					    ab->b_size);
1657 				}
1658 			}
1659 
1660 			if (ab->b_datacnt == 0) {
1661 				arc_change_state(evicted_state, ab, hash_lock);
1662 				ASSERT(HDR_IN_HASH_TABLE(ab));
1663 				ab->b_flags |= ARC_IN_HASH_TABLE;
1664 				ab->b_flags &= ~ARC_BUF_AVAILABLE;
1665 				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
1666 			}
1667 			if (!have_lock)
1668 				mutex_exit(hash_lock);
1669 			if (bytes >= 0 && bytes_evicted >= bytes)
1670 				break;
1671 		} else {
1672 			missed += 1;
1673 		}
1674 	}
1675 
1676 	mutex_exit(&evicted_state->arcs_mtx);
1677 	mutex_exit(&state->arcs_mtx);
1678 
1679 	if (bytes_evicted < bytes)
1680 		dprintf("only evicted %lld bytes from %x",
1681 		    (longlong_t)bytes_evicted, state);
1682 
1683 	if (skipped)
1684 		ARCSTAT_INCR(arcstat_evict_skip, skipped);
1685 
1686 	if (missed)
1687 		ARCSTAT_INCR(arcstat_mutex_miss, missed);
1688 
1689 	/*
1690 	 * We have just evicted some date into the ghost state, make
1691 	 * sure we also adjust the ghost state size if necessary.
1692 	 */
1693 	if (arc_no_grow &&
1694 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1695 		int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1696 		    arc_mru_ghost->arcs_size - arc_c;
1697 
1698 		if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1699 			int64_t todelete =
1700 			    MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1701 			arc_evict_ghost(arc_mru_ghost, NULL, todelete);
1702 		} else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1703 			int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1704 			    arc_mru_ghost->arcs_size +
1705 			    arc_mfu_ghost->arcs_size - arc_c);
1706 			arc_evict_ghost(arc_mfu_ghost, NULL, todelete);
1707 		}
1708 	}
1709 
1710 	return (stolen);
1711 }
1712 
1713 /*
1714  * Remove buffers from list until we've removed the specified number of
1715  * bytes.  Destroy the buffers that are removed.
1716  */
1717 static void
1718 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1719 {
1720 	arc_buf_hdr_t *ab, *ab_prev;
1721 	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1722 	kmutex_t *hash_lock;
1723 	uint64_t bytes_deleted = 0;
1724 	uint64_t bufs_skipped = 0;
1725 
1726 	ASSERT(GHOST_STATE(state));
1727 top:
1728 	mutex_enter(&state->arcs_mtx);
1729 	for (ab = list_tail(list); ab; ab = ab_prev) {
1730 		ab_prev = list_prev(list, ab);
1731 		if (spa && ab->b_spa != spa)
1732 			continue;
1733 		hash_lock = HDR_LOCK(ab);
1734 		if (mutex_tryenter(hash_lock)) {
1735 			ASSERT(!HDR_IO_IN_PROGRESS(ab));
1736 			ASSERT(ab->b_buf == NULL);
1737 			ARCSTAT_BUMP(arcstat_deleted);
1738 			bytes_deleted += ab->b_size;
1739 
1740 			if (ab->b_l2hdr != NULL) {
1741 				/*
1742 				 * This buffer is cached on the 2nd Level ARC;
1743 				 * don't destroy the header.
1744 				 */
1745 				arc_change_state(arc_l2c_only, ab, hash_lock);
1746 				mutex_exit(hash_lock);
1747 			} else {
1748 				arc_change_state(arc_anon, ab, hash_lock);
1749 				mutex_exit(hash_lock);
1750 				arc_hdr_destroy(ab);
1751 			}
1752 
1753 			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1754 			if (bytes >= 0 && bytes_deleted >= bytes)
1755 				break;
1756 		} else {
1757 			if (bytes < 0) {
1758 				mutex_exit(&state->arcs_mtx);
1759 				mutex_enter(hash_lock);
1760 				mutex_exit(hash_lock);
1761 				goto top;
1762 			}
1763 			bufs_skipped += 1;
1764 		}
1765 	}
1766 	mutex_exit(&state->arcs_mtx);
1767 
1768 	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
1769 	    (bytes < 0 || bytes_deleted < bytes)) {
1770 		list = &state->arcs_list[ARC_BUFC_METADATA];
1771 		goto top;
1772 	}
1773 
1774 	if (bufs_skipped) {
1775 		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
1776 		ASSERT(bytes >= 0);
1777 	}
1778 
1779 	if (bytes_deleted < bytes)
1780 		dprintf("only deleted %lld bytes from %p",
1781 		    (longlong_t)bytes_deleted, state);
1782 }
1783 
1784 static void
1785 arc_adjust(void)
1786 {
1787 	int64_t adjustment, delta;
1788 
1789 	/*
1790 	 * Adjust MRU size
1791 	 */
1792 
1793 	adjustment = MIN(arc_size - arc_c,
1794 	    arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used - arc_p);
1795 
1796 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
1797 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
1798 		(void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
1799 		adjustment -= delta;
1800 	}
1801 
1802 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1803 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
1804 		(void) arc_evict(arc_mru, NULL, delta, FALSE,
1805 		    ARC_BUFC_METADATA);
1806 	}
1807 
1808 	/*
1809 	 * Adjust MFU size
1810 	 */
1811 
1812 	adjustment = arc_size - arc_c;
1813 
1814 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
1815 		delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
1816 		(void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
1817 		adjustment -= delta;
1818 	}
1819 
1820 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1821 		int64_t delta = MIN(adjustment,
1822 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
1823 		(void) arc_evict(arc_mfu, NULL, delta, FALSE,
1824 		    ARC_BUFC_METADATA);
1825 	}
1826 
1827 	/*
1828 	 * Adjust ghost lists
1829 	 */
1830 
1831 	adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
1832 
1833 	if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
1834 		delta = MIN(arc_mru_ghost->arcs_size, adjustment);
1835 		arc_evict_ghost(arc_mru_ghost, NULL, delta);
1836 	}
1837 
1838 	adjustment =
1839 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
1840 
1841 	if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
1842 		delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
1843 		arc_evict_ghost(arc_mfu_ghost, NULL, delta);
1844 	}
1845 }
1846 
1847 static void
1848 arc_do_user_evicts(void)
1849 {
1850 	mutex_enter(&arc_eviction_mtx);
1851 	while (arc_eviction_list != NULL) {
1852 		arc_buf_t *buf = arc_eviction_list;
1853 		arc_eviction_list = buf->b_next;
1854 		rw_enter(&buf->b_lock, RW_WRITER);
1855 		buf->b_hdr = NULL;
1856 		rw_exit(&buf->b_lock);
1857 		mutex_exit(&arc_eviction_mtx);
1858 
1859 		if (buf->b_efunc != NULL)
1860 			VERIFY(buf->b_efunc(buf) == 0);
1861 
1862 		buf->b_efunc = NULL;
1863 		buf->b_private = NULL;
1864 		kmem_cache_free(buf_cache, buf);
1865 		mutex_enter(&arc_eviction_mtx);
1866 	}
1867 	mutex_exit(&arc_eviction_mtx);
1868 }
1869 
1870 /*
1871  * Flush all *evictable* data from the cache for the given spa.
1872  * NOTE: this will not touch "active" (i.e. referenced) data.
1873  */
1874 void
1875 arc_flush(spa_t *spa)
1876 {
1877 	uint64_t guid = 0;
1878 
1879 	if (spa)
1880 		guid = spa_guid(spa);
1881 
1882 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
1883 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
1884 		if (spa)
1885 			break;
1886 	}
1887 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
1888 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
1889 		if (spa)
1890 			break;
1891 	}
1892 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
1893 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
1894 		if (spa)
1895 			break;
1896 	}
1897 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
1898 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
1899 		if (spa)
1900 			break;
1901 	}
1902 
1903 	arc_evict_ghost(arc_mru_ghost, guid, -1);
1904 	arc_evict_ghost(arc_mfu_ghost, guid, -1);
1905 
1906 	mutex_enter(&arc_reclaim_thr_lock);
1907 	arc_do_user_evicts();
1908 	mutex_exit(&arc_reclaim_thr_lock);
1909 	ASSERT(spa || arc_eviction_list == NULL);
1910 }
1911 
1912 void
1913 arc_shrink(void)
1914 {
1915 	if (arc_c > arc_c_min) {
1916 		uint64_t to_free;
1917 
1918 #ifdef _KERNEL
1919 		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
1920 #else
1921 		to_free = arc_c >> arc_shrink_shift;
1922 #endif
1923 		if (arc_c > arc_c_min + to_free)
1924 			atomic_add_64(&arc_c, -to_free);
1925 		else
1926 			arc_c = arc_c_min;
1927 
1928 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
1929 		if (arc_c > arc_size)
1930 			arc_c = MAX(arc_size, arc_c_min);
1931 		if (arc_p > arc_c)
1932 			arc_p = (arc_c >> 1);
1933 		ASSERT(arc_c >= arc_c_min);
1934 		ASSERT((int64_t)arc_p >= 0);
1935 	}
1936 
1937 	if (arc_size > arc_c)
1938 		arc_adjust();
1939 }
1940 
1941 static int
1942 arc_reclaim_needed(void)
1943 {
1944 	uint64_t extra;
1945 
1946 #ifdef _KERNEL
1947 
1948 	if (needfree)
1949 		return (1);
1950 
1951 	/*
1952 	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
1953 	 */
1954 	extra = desfree;
1955 
1956 	/*
1957 	 * check that we're out of range of the pageout scanner.  It starts to
1958 	 * schedule paging if freemem is less than lotsfree and needfree.
1959 	 * lotsfree is the high-water mark for pageout, and needfree is the
1960 	 * number of needed free pages.  We add extra pages here to make sure
1961 	 * the scanner doesn't start up while we're freeing memory.
1962 	 */
1963 	if (freemem < lotsfree + needfree + extra)
1964 		return (1);
1965 
1966 	/*
1967 	 * check to make sure that swapfs has enough space so that anon
1968 	 * reservations can still succeed. anon_resvmem() checks that the
1969 	 * availrmem is greater than swapfs_minfree, and the number of reserved
1970 	 * swap pages.  We also add a bit of extra here just to prevent
1971 	 * circumstances from getting really dire.
1972 	 */
1973 	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
1974 		return (1);
1975 
1976 #if defined(__i386)
1977 	/*
1978 	 * If we're on an i386 platform, it's possible that we'll exhaust the
1979 	 * kernel heap space before we ever run out of available physical
1980 	 * memory.  Most checks of the size of the heap_area compare against
1981 	 * tune.t_minarmem, which is the minimum available real memory that we
1982 	 * can have in the system.  However, this is generally fixed at 25 pages
1983 	 * which is so low that it's useless.  In this comparison, we seek to
1984 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
1985 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
1986 	 * free)
1987 	 */
1988 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
1989 	    (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2))
1990 		return (1);
1991 #endif
1992 
1993 #else
1994 	if (spa_get_random(100) == 0)
1995 		return (1);
1996 #endif
1997 	return (0);
1998 }
1999 
2000 static void
2001 arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2002 {
2003 	size_t			i;
2004 	kmem_cache_t		*prev_cache = NULL;
2005 	kmem_cache_t		*prev_data_cache = NULL;
2006 	extern kmem_cache_t	*zio_buf_cache[];
2007 	extern kmem_cache_t	*zio_data_buf_cache[];
2008 
2009 #ifdef _KERNEL
2010 	if (arc_meta_used >= arc_meta_limit) {
2011 		/*
2012 		 * We are exceeding our meta-data cache limit.
2013 		 * Purge some DNLC entries to release holds on meta-data.
2014 		 */
2015 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
2016 	}
2017 #if defined(__i386)
2018 	/*
2019 	 * Reclaim unused memory from all kmem caches.
2020 	 */
2021 	kmem_reap();
2022 #endif
2023 #endif
2024 
2025 	/*
2026 	 * An aggressive reclamation will shrink the cache size as well as
2027 	 * reap free buffers from the arc kmem caches.
2028 	 */
2029 	if (strat == ARC_RECLAIM_AGGR)
2030 		arc_shrink();
2031 
2032 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2033 		if (zio_buf_cache[i] != prev_cache) {
2034 			prev_cache = zio_buf_cache[i];
2035 			kmem_cache_reap_now(zio_buf_cache[i]);
2036 		}
2037 		if (zio_data_buf_cache[i] != prev_data_cache) {
2038 			prev_data_cache = zio_data_buf_cache[i];
2039 			kmem_cache_reap_now(zio_data_buf_cache[i]);
2040 		}
2041 	}
2042 	kmem_cache_reap_now(buf_cache);
2043 	kmem_cache_reap_now(hdr_cache);
2044 }
2045 
2046 static void
2047 arc_reclaim_thread(void)
2048 {
2049 	clock_t			growtime = 0;
2050 	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
2051 	callb_cpr_t		cpr;
2052 
2053 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2054 
2055 	mutex_enter(&arc_reclaim_thr_lock);
2056 	while (arc_thread_exit == 0) {
2057 		if (arc_reclaim_needed()) {
2058 
2059 			if (arc_no_grow) {
2060 				if (last_reclaim == ARC_RECLAIM_CONS) {
2061 					last_reclaim = ARC_RECLAIM_AGGR;
2062 				} else {
2063 					last_reclaim = ARC_RECLAIM_CONS;
2064 				}
2065 			} else {
2066 				arc_no_grow = TRUE;
2067 				last_reclaim = ARC_RECLAIM_AGGR;
2068 				membar_producer();
2069 			}
2070 
2071 			/* reset the growth delay for every reclaim */
2072 			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2073 
2074 			arc_kmem_reap_now(last_reclaim);
2075 			arc_warm = B_TRUE;
2076 
2077 		} else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
2078 			arc_no_grow = FALSE;
2079 		}
2080 
2081 		if (2 * arc_c < arc_size +
2082 		    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size)
2083 			arc_adjust();
2084 
2085 		if (arc_eviction_list != NULL)
2086 			arc_do_user_evicts();
2087 
2088 		/* block until needed, or one second, whichever is shorter */
2089 		CALLB_CPR_SAFE_BEGIN(&cpr);
2090 		(void) cv_timedwait(&arc_reclaim_thr_cv,
2091 		    &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2092 		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2093 	}
2094 
2095 	arc_thread_exit = 0;
2096 	cv_broadcast(&arc_reclaim_thr_cv);
2097 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
2098 	thread_exit();
2099 }
2100 
2101 /*
2102  * Adapt arc info given the number of bytes we are trying to add and
2103  * the state that we are comming from.  This function is only called
2104  * when we are adding new content to the cache.
2105  */
2106 static void
2107 arc_adapt(int bytes, arc_state_t *state)
2108 {
2109 	int mult;
2110 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2111 
2112 	if (state == arc_l2c_only)
2113 		return;
2114 
2115 	ASSERT(bytes > 0);
2116 	/*
2117 	 * Adapt the target size of the MRU list:
2118 	 *	- if we just hit in the MRU ghost list, then increase
2119 	 *	  the target size of the MRU list.
2120 	 *	- if we just hit in the MFU ghost list, then increase
2121 	 *	  the target size of the MFU list by decreasing the
2122 	 *	  target size of the MRU list.
2123 	 */
2124 	if (state == arc_mru_ghost) {
2125 		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
2126 		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
2127 
2128 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
2129 	} else if (state == arc_mfu_ghost) {
2130 		uint64_t delta;
2131 
2132 		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
2133 		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
2134 
2135 		delta = MIN(bytes * mult, arc_p);
2136 		arc_p = MAX(arc_p_min, arc_p - delta);
2137 	}
2138 	ASSERT((int64_t)arc_p >= 0);
2139 
2140 	if (arc_reclaim_needed()) {
2141 		cv_signal(&arc_reclaim_thr_cv);
2142 		return;
2143 	}
2144 
2145 	if (arc_no_grow)
2146 		return;
2147 
2148 	if (arc_c >= arc_c_max)
2149 		return;
2150 
2151 	/*
2152 	 * If we're within (2 * maxblocksize) bytes of the target
2153 	 * cache size, increment the target cache size
2154 	 */
2155 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
2156 		atomic_add_64(&arc_c, (int64_t)bytes);
2157 		if (arc_c > arc_c_max)
2158 			arc_c = arc_c_max;
2159 		else if (state == arc_anon)
2160 			atomic_add_64(&arc_p, (int64_t)bytes);
2161 		if (arc_p > arc_c)
2162 			arc_p = arc_c;
2163 	}
2164 	ASSERT((int64_t)arc_p >= 0);
2165 }
2166 
2167 /*
2168  * Check if the cache has reached its limits and eviction is required
2169  * prior to insert.
2170  */
2171 static int
2172 arc_evict_needed(arc_buf_contents_t type)
2173 {
2174 	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2175 		return (1);
2176 
2177 #ifdef _KERNEL
2178 	/*
2179 	 * If zio data pages are being allocated out of a separate heap segment,
2180 	 * then enforce that the size of available vmem for this area remains
2181 	 * above about 1/32nd free.
2182 	 */
2183 	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
2184 	    vmem_size(zio_arena, VMEM_FREE) <
2185 	    (vmem_size(zio_arena, VMEM_ALLOC) >> 5))
2186 		return (1);
2187 #endif
2188 
2189 	if (arc_reclaim_needed())
2190 		return (1);
2191 
2192 	return (arc_size > arc_c);
2193 }
2194 
2195 /*
2196  * The buffer, supplied as the first argument, needs a data block.
2197  * So, if we are at cache max, determine which cache should be victimized.
2198  * We have the following cases:
2199  *
2200  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2201  * In this situation if we're out of space, but the resident size of the MFU is
2202  * under the limit, victimize the MFU cache to satisfy this insertion request.
2203  *
2204  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2205  * Here, we've used up all of the available space for the MRU, so we need to
2206  * evict from our own cache instead.  Evict from the set of resident MRU
2207  * entries.
2208  *
2209  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2210  * c minus p represents the MFU space in the cache, since p is the size of the
2211  * cache that is dedicated to the MRU.  In this situation there's still space on
2212  * the MFU side, so the MRU side needs to be victimized.
2213  *
2214  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2215  * MFU's resident set is consuming more space than it has been allotted.  In
2216  * this situation, we must victimize our own cache, the MFU, for this insertion.
2217  */
2218 static void
2219 arc_get_data_buf(arc_buf_t *buf)
2220 {
2221 	arc_state_t		*state = buf->b_hdr->b_state;
2222 	uint64_t		size = buf->b_hdr->b_size;
2223 	arc_buf_contents_t	type = buf->b_hdr->b_type;
2224 
2225 	arc_adapt(size, state);
2226 
2227 	/*
2228 	 * We have not yet reached cache maximum size,
2229 	 * just allocate a new buffer.
2230 	 */
2231 	if (!arc_evict_needed(type)) {
2232 		if (type == ARC_BUFC_METADATA) {
2233 			buf->b_data = zio_buf_alloc(size);
2234 			arc_space_consume(size, ARC_SPACE_DATA);
2235 		} else {
2236 			ASSERT(type == ARC_BUFC_DATA);
2237 			buf->b_data = zio_data_buf_alloc(size);
2238 			ARCSTAT_INCR(arcstat_data_size, size);
2239 			atomic_add_64(&arc_size, size);
2240 		}
2241 		goto out;
2242 	}
2243 
2244 	/*
2245 	 * If we are prefetching from the mfu ghost list, this buffer
2246 	 * will end up on the mru list; so steal space from there.
2247 	 */
2248 	if (state == arc_mfu_ghost)
2249 		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
2250 	else if (state == arc_mru_ghost)
2251 		state = arc_mru;
2252 
2253 	if (state == arc_mru || state == arc_anon) {
2254 		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
2255 		state = (arc_mfu->arcs_lsize[type] >= size &&
2256 		    arc_p > mru_used) ? arc_mfu : arc_mru;
2257 	} else {
2258 		/* MFU cases */
2259 		uint64_t mfu_space = arc_c - arc_p;
2260 		state =  (arc_mru->arcs_lsize[type] >= size &&
2261 		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2262 	}
2263 	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2264 		if (type == ARC_BUFC_METADATA) {
2265 			buf->b_data = zio_buf_alloc(size);
2266 			arc_space_consume(size, ARC_SPACE_DATA);
2267 		} else {
2268 			ASSERT(type == ARC_BUFC_DATA);
2269 			buf->b_data = zio_data_buf_alloc(size);
2270 			ARCSTAT_INCR(arcstat_data_size, size);
2271 			atomic_add_64(&arc_size, size);
2272 		}
2273 		ARCSTAT_BUMP(arcstat_recycle_miss);
2274 	}
2275 	ASSERT(buf->b_data != NULL);
2276 out:
2277 	/*
2278 	 * Update the state size.  Note that ghost states have a
2279 	 * "ghost size" and so don't need to be updated.
2280 	 */
2281 	if (!GHOST_STATE(buf->b_hdr->b_state)) {
2282 		arc_buf_hdr_t *hdr = buf->b_hdr;
2283 
2284 		atomic_add_64(&hdr->b_state->arcs_size, size);
2285 		if (list_link_active(&hdr->b_arc_node)) {
2286 			ASSERT(refcount_is_zero(&hdr->b_refcnt));
2287 			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2288 		}
2289 		/*
2290 		 * If we are growing the cache, and we are adding anonymous
2291 		 * data, and we have outgrown arc_p, update arc_p
2292 		 */
2293 		if (arc_size < arc_c && hdr->b_state == arc_anon &&
2294 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
2295 			arc_p = MIN(arc_c, arc_p + size);
2296 	}
2297 }
2298 
2299 /*
2300  * This routine is called whenever a buffer is accessed.
2301  * NOTE: the hash lock is dropped in this function.
2302  */
2303 static void
2304 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2305 {
2306 	clock_t now;
2307 
2308 	ASSERT(MUTEX_HELD(hash_lock));
2309 
2310 	if (buf->b_state == arc_anon) {
2311 		/*
2312 		 * This buffer is not in the cache, and does not
2313 		 * appear in our "ghost" list.  Add the new buffer
2314 		 * to the MRU state.
2315 		 */
2316 
2317 		ASSERT(buf->b_arc_access == 0);
2318 		buf->b_arc_access = ddi_get_lbolt();
2319 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2320 		arc_change_state(arc_mru, buf, hash_lock);
2321 
2322 	} else if (buf->b_state == arc_mru) {
2323 		now = ddi_get_lbolt();
2324 
2325 		/*
2326 		 * If this buffer is here because of a prefetch, then either:
2327 		 * - clear the flag if this is a "referencing" read
2328 		 *   (any subsequent access will bump this into the MFU state).
2329 		 * or
2330 		 * - move the buffer to the head of the list if this is
2331 		 *   another prefetch (to make it less likely to be evicted).
2332 		 */
2333 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
2334 			if (refcount_count(&buf->b_refcnt) == 0) {
2335 				ASSERT(list_link_active(&buf->b_arc_node));
2336 			} else {
2337 				buf->b_flags &= ~ARC_PREFETCH;
2338 				ARCSTAT_BUMP(arcstat_mru_hits);
2339 			}
2340 			buf->b_arc_access = now;
2341 			return;
2342 		}
2343 
2344 		/*
2345 		 * This buffer has been "accessed" only once so far,
2346 		 * but it is still in the cache. Move it to the MFU
2347 		 * state.
2348 		 */
2349 		if (now > buf->b_arc_access + ARC_MINTIME) {
2350 			/*
2351 			 * More than 125ms have passed since we
2352 			 * instantiated this buffer.  Move it to the
2353 			 * most frequently used state.
2354 			 */
2355 			buf->b_arc_access = now;
2356 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2357 			arc_change_state(arc_mfu, buf, hash_lock);
2358 		}
2359 		ARCSTAT_BUMP(arcstat_mru_hits);
2360 	} else if (buf->b_state == arc_mru_ghost) {
2361 		arc_state_t	*new_state;
2362 		/*
2363 		 * This buffer has been "accessed" recently, but
2364 		 * was evicted from the cache.  Move it to the
2365 		 * MFU state.
2366 		 */
2367 
2368 		if (buf->b_flags & ARC_PREFETCH) {
2369 			new_state = arc_mru;
2370 			if (refcount_count(&buf->b_refcnt) > 0)
2371 				buf->b_flags &= ~ARC_PREFETCH;
2372 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2373 		} else {
2374 			new_state = arc_mfu;
2375 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2376 		}
2377 
2378 		buf->b_arc_access = ddi_get_lbolt();
2379 		arc_change_state(new_state, buf, hash_lock);
2380 
2381 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
2382 	} else if (buf->b_state == arc_mfu) {
2383 		/*
2384 		 * This buffer has been accessed more than once and is
2385 		 * still in the cache.  Keep it in the MFU state.
2386 		 *
2387 		 * NOTE: an add_reference() that occurred when we did
2388 		 * the arc_read() will have kicked this off the list.
2389 		 * If it was a prefetch, we will explicitly move it to
2390 		 * the head of the list now.
2391 		 */
2392 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
2393 			ASSERT(refcount_count(&buf->b_refcnt) == 0);
2394 			ASSERT(list_link_active(&buf->b_arc_node));
2395 		}
2396 		ARCSTAT_BUMP(arcstat_mfu_hits);
2397 		buf->b_arc_access = ddi_get_lbolt();
2398 	} else if (buf->b_state == arc_mfu_ghost) {
2399 		arc_state_t	*new_state = arc_mfu;
2400 		/*
2401 		 * This buffer has been accessed more than once but has
2402 		 * been evicted from the cache.  Move it back to the
2403 		 * MFU state.
2404 		 */
2405 
2406 		if (buf->b_flags & ARC_PREFETCH) {
2407 			/*
2408 			 * This is a prefetch access...
2409 			 * move this block back to the MRU state.
2410 			 */
2411 			ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0);
2412 			new_state = arc_mru;
2413 		}
2414 
2415 		buf->b_arc_access = ddi_get_lbolt();
2416 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2417 		arc_change_state(new_state, buf, hash_lock);
2418 
2419 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2420 	} else if (buf->b_state == arc_l2c_only) {
2421 		/*
2422 		 * This buffer is on the 2nd Level ARC.
2423 		 */
2424 
2425 		buf->b_arc_access = ddi_get_lbolt();
2426 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2427 		arc_change_state(arc_mfu, buf, hash_lock);
2428 	} else {
2429 		ASSERT(!"invalid arc state");
2430 	}
2431 }
2432 
2433 /* a generic arc_done_func_t which you can use */
2434 /* ARGSUSED */
2435 void
2436 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2437 {
2438 	bcopy(buf->b_data, arg, buf->b_hdr->b_size);
2439 	VERIFY(arc_buf_remove_ref(buf, arg) == 1);
2440 }
2441 
2442 /* a generic arc_done_func_t */
2443 void
2444 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2445 {
2446 	arc_buf_t **bufp = arg;
2447 	if (zio && zio->io_error) {
2448 		VERIFY(arc_buf_remove_ref(buf, arg) == 1);
2449 		*bufp = NULL;
2450 	} else {
2451 		*bufp = buf;
2452 	}
2453 }
2454 
2455 static void
2456 arc_read_done(zio_t *zio)
2457 {
2458 	arc_buf_hdr_t	*hdr, *found;
2459 	arc_buf_t	*buf;
2460 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
2461 	kmutex_t	*hash_lock;
2462 	arc_callback_t	*callback_list, *acb;
2463 	int		freeable = FALSE;
2464 
2465 	buf = zio->io_private;
2466 	hdr = buf->b_hdr;
2467 
2468 	/*
2469 	 * The hdr was inserted into hash-table and removed from lists
2470 	 * prior to starting I/O.  We should find this header, since
2471 	 * it's in the hash table, and it should be legit since it's
2472 	 * not possible to evict it during the I/O.  The only possible
2473 	 * reason for it not to be found is if we were freed during the
2474 	 * read.
2475 	 */
2476 	found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2477 	    &hash_lock);
2478 
2479 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2480 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2481 	    (found == hdr && HDR_L2_READING(hdr)));
2482 
2483 	hdr->b_flags &= ~ARC_L2_EVICTED;
2484 	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2485 		hdr->b_flags &= ~ARC_L2CACHE;
2486 
2487 	/* byteswap if necessary */
2488 	callback_list = hdr->b_acb;
2489 	ASSERT(callback_list != NULL);
2490 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2491 		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2492 		    byteswap_uint64_array :
2493 		    dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap;
2494 		func(buf->b_data, hdr->b_size);
2495 	}
2496 
2497 	arc_cksum_compute(buf, B_FALSE);
2498 
2499 	if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2500 		/*
2501 		 * Only call arc_access on anonymous buffers.  This is because
2502 		 * if we've issued an I/O for an evicted buffer, we've already
2503 		 * called arc_access (to prevent any simultaneous readers from
2504 		 * getting confused).
2505 		 */
2506 		arc_access(hdr, hash_lock);
2507 	}
2508 
2509 	/* create copies of the data buffer for the callers */
2510 	abuf = buf;
2511 	for (acb = callback_list; acb; acb = acb->acb_next) {
2512 		if (acb->acb_done) {
2513 			if (abuf == NULL)
2514 				abuf = arc_buf_clone(buf);
2515 			acb->acb_buf = abuf;
2516 			abuf = NULL;
2517 		}
2518 	}
2519 	hdr->b_acb = NULL;
2520 	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2521 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
2522 	if (abuf == buf) {
2523 		ASSERT(buf->b_efunc == NULL);
2524 		ASSERT(hdr->b_datacnt == 1);
2525 		hdr->b_flags |= ARC_BUF_AVAILABLE;
2526 	}
2527 
2528 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2529 
2530 	if (zio->io_error != 0) {
2531 		hdr->b_flags |= ARC_IO_ERROR;
2532 		if (hdr->b_state != arc_anon)
2533 			arc_change_state(arc_anon, hdr, hash_lock);
2534 		if (HDR_IN_HASH_TABLE(hdr))
2535 			buf_hash_remove(hdr);
2536 		freeable = refcount_is_zero(&hdr->b_refcnt);
2537 	}
2538 
2539 	/*
2540 	 * Broadcast before we drop the hash_lock to avoid the possibility
2541 	 * that the hdr (and hence the cv) might be freed before we get to
2542 	 * the cv_broadcast().
2543 	 */
2544 	cv_broadcast(&hdr->b_cv);
2545 
2546 	if (hash_lock) {
2547 		mutex_exit(hash_lock);
2548 	} else {
2549 		/*
2550 		 * This block was freed while we waited for the read to
2551 		 * complete.  It has been removed from the hash table and
2552 		 * moved to the anonymous state (so that it won't show up
2553 		 * in the cache).
2554 		 */
2555 		ASSERT3P(hdr->b_state, ==, arc_anon);
2556 		freeable = refcount_is_zero(&hdr->b_refcnt);
2557 	}
2558 
2559 	/* execute each callback and free its structure */
2560 	while ((acb = callback_list) != NULL) {
2561 		if (acb->acb_done)
2562 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2563 
2564 		if (acb->acb_zio_dummy != NULL) {
2565 			acb->acb_zio_dummy->io_error = zio->io_error;
2566 			zio_nowait(acb->acb_zio_dummy);
2567 		}
2568 
2569 		callback_list = acb->acb_next;
2570 		kmem_free(acb, sizeof (arc_callback_t));
2571 	}
2572 
2573 	if (freeable)
2574 		arc_hdr_destroy(hdr);
2575 }
2576 
2577 /*
2578  * "Read" the block block at the specified DVA (in bp) via the
2579  * cache.  If the block is found in the cache, invoke the provided
2580  * callback immediately and return.  Note that the `zio' parameter
2581  * in the callback will be NULL in this case, since no IO was
2582  * required.  If the block is not in the cache pass the read request
2583  * on to the spa with a substitute callback function, so that the
2584  * requested block will be added to the cache.
2585  *
2586  * If a read request arrives for a block that has a read in-progress,
2587  * either wait for the in-progress read to complete (and return the
2588  * results); or, if this is a read with a "done" func, add a record
2589  * to the read to invoke the "done" func when the read completes,
2590  * and return; or just return.
2591  *
2592  * arc_read_done() will invoke all the requested "done" functions
2593  * for readers of this block.
2594  *
2595  * Normal callers should use arc_read and pass the arc buffer and offset
2596  * for the bp.  But if you know you don't need locking, you can use
2597  * arc_read_bp.
2598  */
2599 int
2600 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_buf_t *pbuf,
2601     arc_done_func_t *done, void *private, int priority, int zio_flags,
2602     uint32_t *arc_flags, const zbookmark_t *zb)
2603 {
2604 	int err;
2605 
2606 	ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt));
2607 	ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size);
2608 	rw_enter(&pbuf->b_lock, RW_READER);
2609 
2610 	err = arc_read_nolock(pio, spa, bp, done, private, priority,
2611 	    zio_flags, arc_flags, zb);
2612 	rw_exit(&pbuf->b_lock);
2613 
2614 	return (err);
2615 }
2616 
2617 int
2618 arc_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bp,
2619     arc_done_func_t *done, void *private, int priority, int zio_flags,
2620     uint32_t *arc_flags, const zbookmark_t *zb)
2621 {
2622 	arc_buf_hdr_t *hdr;
2623 	arc_buf_t *buf;
2624 	kmutex_t *hash_lock;
2625 	zio_t *rzio;
2626 	uint64_t guid = spa_guid(spa);
2627 
2628 top:
2629 	hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2630 	    &hash_lock);
2631 	if (hdr && hdr->b_datacnt > 0) {
2632 
2633 		*arc_flags |= ARC_CACHED;
2634 
2635 		if (HDR_IO_IN_PROGRESS(hdr)) {
2636 
2637 			if (*arc_flags & ARC_WAIT) {
2638 				cv_wait(&hdr->b_cv, hash_lock);
2639 				mutex_exit(hash_lock);
2640 				goto top;
2641 			}
2642 			ASSERT(*arc_flags & ARC_NOWAIT);
2643 
2644 			if (done) {
2645 				arc_callback_t	*acb = NULL;
2646 
2647 				acb = kmem_zalloc(sizeof (arc_callback_t),
2648 				    KM_SLEEP);
2649 				acb->acb_done = done;
2650 				acb->acb_private = private;
2651 				if (pio != NULL)
2652 					acb->acb_zio_dummy = zio_null(pio,
2653 					    spa, NULL, NULL, NULL, zio_flags);
2654 
2655 				ASSERT(acb->acb_done != NULL);
2656 				acb->acb_next = hdr->b_acb;
2657 				hdr->b_acb = acb;
2658 				add_reference(hdr, hash_lock, private);
2659 				mutex_exit(hash_lock);
2660 				return (0);
2661 			}
2662 			mutex_exit(hash_lock);
2663 			return (0);
2664 		}
2665 
2666 		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2667 
2668 		if (done) {
2669 			add_reference(hdr, hash_lock, private);
2670 			/*
2671 			 * If this block is already in use, create a new
2672 			 * copy of the data so that we will be guaranteed
2673 			 * that arc_release() will always succeed.
2674 			 */
2675 			buf = hdr->b_buf;
2676 			ASSERT(buf);
2677 			ASSERT(buf->b_data);
2678 			if (HDR_BUF_AVAILABLE(hdr)) {
2679 				ASSERT(buf->b_efunc == NULL);
2680 				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2681 			} else {
2682 				buf = arc_buf_clone(buf);
2683 			}
2684 
2685 		} else if (*arc_flags & ARC_PREFETCH &&
2686 		    refcount_count(&hdr->b_refcnt) == 0) {
2687 			hdr->b_flags |= ARC_PREFETCH;
2688 		}
2689 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
2690 		arc_access(hdr, hash_lock);
2691 		if (*arc_flags & ARC_L2CACHE)
2692 			hdr->b_flags |= ARC_L2CACHE;
2693 		mutex_exit(hash_lock);
2694 		ARCSTAT_BUMP(arcstat_hits);
2695 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2696 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2697 		    data, metadata, hits);
2698 
2699 		if (done)
2700 			done(NULL, buf, private);
2701 	} else {
2702 		uint64_t size = BP_GET_LSIZE(bp);
2703 		arc_callback_t	*acb;
2704 		vdev_t *vd = NULL;
2705 		uint64_t addr;
2706 		boolean_t devw = B_FALSE;
2707 
2708 		if (hdr == NULL) {
2709 			/* this block is not in the cache */
2710 			arc_buf_hdr_t	*exists;
2711 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2712 			buf = arc_buf_alloc(spa, size, private, type);
2713 			hdr = buf->b_hdr;
2714 			hdr->b_dva = *BP_IDENTITY(bp);
2715 			hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2716 			hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2717 			exists = buf_hash_insert(hdr, &hash_lock);
2718 			if (exists) {
2719 				/* somebody beat us to the hash insert */
2720 				mutex_exit(hash_lock);
2721 				bzero(&hdr->b_dva, sizeof (dva_t));
2722 				hdr->b_birth = 0;
2723 				hdr->b_cksum0 = 0;
2724 				(void) arc_buf_remove_ref(buf, private);
2725 				goto top; /* restart the IO request */
2726 			}
2727 			/* if this is a prefetch, we don't have a reference */
2728 			if (*arc_flags & ARC_PREFETCH) {
2729 				(void) remove_reference(hdr, hash_lock,
2730 				    private);
2731 				hdr->b_flags |= ARC_PREFETCH;
2732 			}
2733 			if (*arc_flags & ARC_L2CACHE)
2734 				hdr->b_flags |= ARC_L2CACHE;
2735 			if (BP_GET_LEVEL(bp) > 0)
2736 				hdr->b_flags |= ARC_INDIRECT;
2737 		} else {
2738 			/* this block is in the ghost cache */
2739 			ASSERT(GHOST_STATE(hdr->b_state));
2740 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2741 			ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0);
2742 			ASSERT(hdr->b_buf == NULL);
2743 
2744 			/* if this is a prefetch, we don't have a reference */
2745 			if (*arc_flags & ARC_PREFETCH)
2746 				hdr->b_flags |= ARC_PREFETCH;
2747 			else
2748 				add_reference(hdr, hash_lock, private);
2749 			if (*arc_flags & ARC_L2CACHE)
2750 				hdr->b_flags |= ARC_L2CACHE;
2751 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2752 			buf->b_hdr = hdr;
2753 			buf->b_data = NULL;
2754 			buf->b_efunc = NULL;
2755 			buf->b_private = NULL;
2756 			buf->b_next = NULL;
2757 			hdr->b_buf = buf;
2758 			arc_get_data_buf(buf);
2759 			ASSERT(hdr->b_datacnt == 0);
2760 			hdr->b_datacnt = 1;
2761 		}
2762 
2763 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2764 		acb->acb_done = done;
2765 		acb->acb_private = private;
2766 
2767 		ASSERT(hdr->b_acb == NULL);
2768 		hdr->b_acb = acb;
2769 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
2770 
2771 		/*
2772 		 * If the buffer has been evicted, migrate it to a present state
2773 		 * before issuing the I/O.  Once we drop the hash-table lock,
2774 		 * the header will be marked as I/O in progress and have an
2775 		 * attached buffer.  At this point, anybody who finds this
2776 		 * buffer ought to notice that it's legit but has a pending I/O.
2777 		 */
2778 
2779 		if (GHOST_STATE(hdr->b_state))
2780 			arc_access(hdr, hash_lock);
2781 
2782 		if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL &&
2783 		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
2784 			devw = hdr->b_l2hdr->b_dev->l2ad_writing;
2785 			addr = hdr->b_l2hdr->b_daddr;
2786 			/*
2787 			 * Lock out device removal.
2788 			 */
2789 			if (vdev_is_dead(vd) ||
2790 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
2791 				vd = NULL;
2792 		}
2793 
2794 		mutex_exit(hash_lock);
2795 
2796 		ASSERT3U(hdr->b_size, ==, size);
2797 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
2798 		    uint64_t, size, zbookmark_t *, zb);
2799 		ARCSTAT_BUMP(arcstat_misses);
2800 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2801 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2802 		    data, metadata, misses);
2803 
2804 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
2805 			/*
2806 			 * Read from the L2ARC if the following are true:
2807 			 * 1. The L2ARC vdev was previously cached.
2808 			 * 2. This buffer still has L2ARC metadata.
2809 			 * 3. This buffer isn't currently writing to the L2ARC.
2810 			 * 4. The L2ARC entry wasn't evicted, which may
2811 			 *    also have invalidated the vdev.
2812 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
2813 			 */
2814 			if (hdr->b_l2hdr != NULL &&
2815 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
2816 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
2817 				l2arc_read_callback_t *cb;
2818 
2819 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
2820 				ARCSTAT_BUMP(arcstat_l2_hits);
2821 
2822 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
2823 				    KM_SLEEP);
2824 				cb->l2rcb_buf = buf;
2825 				cb->l2rcb_spa = spa;
2826 				cb->l2rcb_bp = *bp;
2827 				cb->l2rcb_zb = *zb;
2828 				cb->l2rcb_flags = zio_flags;
2829 
2830 				/*
2831 				 * l2arc read.  The SCL_L2ARC lock will be
2832 				 * released by l2arc_read_done().
2833 				 */
2834 				rzio = zio_read_phys(pio, vd, addr, size,
2835 				    buf->b_data, ZIO_CHECKSUM_OFF,
2836 				    l2arc_read_done, cb, priority, zio_flags |
2837 				    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL |
2838 				    ZIO_FLAG_DONT_PROPAGATE |
2839 				    ZIO_FLAG_DONT_RETRY, B_FALSE);
2840 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
2841 				    zio_t *, rzio);
2842 				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
2843 
2844 				if (*arc_flags & ARC_NOWAIT) {
2845 					zio_nowait(rzio);
2846 					return (0);
2847 				}
2848 
2849 				ASSERT(*arc_flags & ARC_WAIT);
2850 				if (zio_wait(rzio) == 0)
2851 					return (0);
2852 
2853 				/* l2arc read error; goto zio_read() */
2854 			} else {
2855 				DTRACE_PROBE1(l2arc__miss,
2856 				    arc_buf_hdr_t *, hdr);
2857 				ARCSTAT_BUMP(arcstat_l2_misses);
2858 				if (HDR_L2_WRITING(hdr))
2859 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
2860 				spa_config_exit(spa, SCL_L2ARC, vd);
2861 			}
2862 		} else {
2863 			if (vd != NULL)
2864 				spa_config_exit(spa, SCL_L2ARC, vd);
2865 			if (l2arc_ndev != 0) {
2866 				DTRACE_PROBE1(l2arc__miss,
2867 				    arc_buf_hdr_t *, hdr);
2868 				ARCSTAT_BUMP(arcstat_l2_misses);
2869 			}
2870 		}
2871 
2872 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
2873 		    arc_read_done, buf, priority, zio_flags, zb);
2874 
2875 		if (*arc_flags & ARC_WAIT)
2876 			return (zio_wait(rzio));
2877 
2878 		ASSERT(*arc_flags & ARC_NOWAIT);
2879 		zio_nowait(rzio);
2880 	}
2881 	return (0);
2882 }
2883 
2884 void
2885 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
2886 {
2887 	ASSERT(buf->b_hdr != NULL);
2888 	ASSERT(buf->b_hdr->b_state != arc_anon);
2889 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
2890 	ASSERT(buf->b_efunc == NULL);
2891 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
2892 
2893 	buf->b_efunc = func;
2894 	buf->b_private = private;
2895 }
2896 
2897 /*
2898  * This is used by the DMU to let the ARC know that a buffer is
2899  * being evicted, so the ARC should clean up.  If this arc buf
2900  * is not yet in the evicted state, it will be put there.
2901  */
2902 int
2903 arc_buf_evict(arc_buf_t *buf)
2904 {
2905 	arc_buf_hdr_t *hdr;
2906 	kmutex_t *hash_lock;
2907 	arc_buf_t **bufp;
2908 
2909 	rw_enter(&buf->b_lock, RW_WRITER);
2910 	hdr = buf->b_hdr;
2911 	if (hdr == NULL) {
2912 		/*
2913 		 * We are in arc_do_user_evicts().
2914 		 */
2915 		ASSERT(buf->b_data == NULL);
2916 		rw_exit(&buf->b_lock);
2917 		return (0);
2918 	} else if (buf->b_data == NULL) {
2919 		arc_buf_t copy = *buf; /* structure assignment */
2920 		/*
2921 		 * We are on the eviction list; process this buffer now
2922 		 * but let arc_do_user_evicts() do the reaping.
2923 		 */
2924 		buf->b_efunc = NULL;
2925 		rw_exit(&buf->b_lock);
2926 		VERIFY(copy.b_efunc(&copy) == 0);
2927 		return (1);
2928 	}
2929 	hash_lock = HDR_LOCK(hdr);
2930 	mutex_enter(hash_lock);
2931 
2932 	ASSERT(buf->b_hdr == hdr);
2933 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
2934 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2935 
2936 	/*
2937 	 * Pull this buffer off of the hdr
2938 	 */
2939 	bufp = &hdr->b_buf;
2940 	while (*bufp != buf)
2941 		bufp = &(*bufp)->b_next;
2942 	*bufp = buf->b_next;
2943 
2944 	ASSERT(buf->b_data != NULL);
2945 	arc_buf_destroy(buf, FALSE, FALSE);
2946 
2947 	if (hdr->b_datacnt == 0) {
2948 		arc_state_t *old_state = hdr->b_state;
2949 		arc_state_t *evicted_state;
2950 
2951 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
2952 
2953 		evicted_state =
2954 		    (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
2955 
2956 		mutex_enter(&old_state->arcs_mtx);
2957 		mutex_enter(&evicted_state->arcs_mtx);
2958 
2959 		arc_change_state(evicted_state, hdr, hash_lock);
2960 		ASSERT(HDR_IN_HASH_TABLE(hdr));
2961 		hdr->b_flags |= ARC_IN_HASH_TABLE;
2962 		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2963 
2964 		mutex_exit(&evicted_state->arcs_mtx);
2965 		mutex_exit(&old_state->arcs_mtx);
2966 	}
2967 	mutex_exit(hash_lock);
2968 	rw_exit(&buf->b_lock);
2969 
2970 	VERIFY(buf->b_efunc(buf) == 0);
2971 	buf->b_efunc = NULL;
2972 	buf->b_private = NULL;
2973 	buf->b_hdr = NULL;
2974 	kmem_cache_free(buf_cache, buf);
2975 	return (1);
2976 }
2977 
2978 /*
2979  * Release this buffer from the cache.  This must be done
2980  * after a read and prior to modifying the buffer contents.
2981  * If the buffer has more than one reference, we must make
2982  * a new hdr for the buffer.
2983  */
2984 void
2985 arc_release(arc_buf_t *buf, void *tag)
2986 {
2987 	arc_buf_hdr_t *hdr;
2988 	kmutex_t *hash_lock;
2989 	l2arc_buf_hdr_t *l2hdr;
2990 	uint64_t buf_size;
2991 	boolean_t released = B_FALSE;
2992 
2993 	rw_enter(&buf->b_lock, RW_WRITER);
2994 	hdr = buf->b_hdr;
2995 
2996 	/* this buffer is not on any list */
2997 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
2998 
2999 	if (hdr->b_state == arc_anon) {
3000 		/* this buffer is already released */
3001 		ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1);
3002 		ASSERT(BUF_EMPTY(hdr));
3003 		ASSERT(buf->b_efunc == NULL);
3004 		arc_buf_thaw(buf);
3005 		rw_exit(&buf->b_lock);
3006 		released = B_TRUE;
3007 	} else {
3008 		hash_lock = HDR_LOCK(hdr);
3009 		mutex_enter(hash_lock);
3010 	}
3011 
3012 	l2hdr = hdr->b_l2hdr;
3013 	if (l2hdr) {
3014 		mutex_enter(&l2arc_buflist_mtx);
3015 		hdr->b_l2hdr = NULL;
3016 		buf_size = hdr->b_size;
3017 	}
3018 
3019 	if (released)
3020 		goto out;
3021 
3022 	/*
3023 	 * Do we have more than one buf?
3024 	 */
3025 	if (hdr->b_datacnt > 1) {
3026 		arc_buf_hdr_t *nhdr;
3027 		arc_buf_t **bufp;
3028 		uint64_t blksz = hdr->b_size;
3029 		uint64_t spa = hdr->b_spa;
3030 		arc_buf_contents_t type = hdr->b_type;
3031 		uint32_t flags = hdr->b_flags;
3032 
3033 		ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3034 		/*
3035 		 * Pull the data off of this buf and attach it to
3036 		 * a new anonymous buf.
3037 		 */
3038 		(void) remove_reference(hdr, hash_lock, tag);
3039 		bufp = &hdr->b_buf;
3040 		while (*bufp != buf)
3041 			bufp = &(*bufp)->b_next;
3042 		*bufp = (*bufp)->b_next;
3043 		buf->b_next = NULL;
3044 
3045 		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3046 		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3047 		if (refcount_is_zero(&hdr->b_refcnt)) {
3048 			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3049 			ASSERT3U(*size, >=, hdr->b_size);
3050 			atomic_add_64(size, -hdr->b_size);
3051 		}
3052 		hdr->b_datacnt -= 1;
3053 		arc_cksum_verify(buf);
3054 
3055 		mutex_exit(hash_lock);
3056 
3057 		nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3058 		nhdr->b_size = blksz;
3059 		nhdr->b_spa = spa;
3060 		nhdr->b_type = type;
3061 		nhdr->b_buf = buf;
3062 		nhdr->b_state = arc_anon;
3063 		nhdr->b_arc_access = 0;
3064 		nhdr->b_flags = flags & ARC_L2_WRITING;
3065 		nhdr->b_l2hdr = NULL;
3066 		nhdr->b_datacnt = 1;
3067 		nhdr->b_freeze_cksum = NULL;
3068 		(void) refcount_add(&nhdr->b_refcnt, tag);
3069 		buf->b_hdr = nhdr;
3070 		rw_exit(&buf->b_lock);
3071 		atomic_add_64(&arc_anon->arcs_size, blksz);
3072 	} else {
3073 		rw_exit(&buf->b_lock);
3074 		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3075 		ASSERT(!list_link_active(&hdr->b_arc_node));
3076 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3077 		arc_change_state(arc_anon, hdr, hash_lock);
3078 		hdr->b_arc_access = 0;
3079 		mutex_exit(hash_lock);
3080 
3081 		bzero(&hdr->b_dva, sizeof (dva_t));
3082 		hdr->b_birth = 0;
3083 		hdr->b_cksum0 = 0;
3084 		arc_buf_thaw(buf);
3085 	}
3086 	buf->b_efunc = NULL;
3087 	buf->b_private = NULL;
3088 
3089 out:
3090 	if (l2hdr) {
3091 		list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3092 		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3093 		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3094 		mutex_exit(&l2arc_buflist_mtx);
3095 	}
3096 }
3097 
3098 int
3099 arc_released(arc_buf_t *buf)
3100 {
3101 	int released;
3102 
3103 	rw_enter(&buf->b_lock, RW_READER);
3104 	released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
3105 	rw_exit(&buf->b_lock);
3106 	return (released);
3107 }
3108 
3109 int
3110 arc_has_callback(arc_buf_t *buf)
3111 {
3112 	int callback;
3113 
3114 	rw_enter(&buf->b_lock, RW_READER);
3115 	callback = (buf->b_efunc != NULL);
3116 	rw_exit(&buf->b_lock);
3117 	return (callback);
3118 }
3119 
3120 #ifdef ZFS_DEBUG
3121 int
3122 arc_referenced(arc_buf_t *buf)
3123 {
3124 	int referenced;
3125 
3126 	rw_enter(&buf->b_lock, RW_READER);
3127 	referenced = (refcount_count(&buf->b_hdr->b_refcnt));
3128 	rw_exit(&buf->b_lock);
3129 	return (referenced);
3130 }
3131 #endif
3132 
3133 static void
3134 arc_write_ready(zio_t *zio)
3135 {
3136 	arc_write_callback_t *callback = zio->io_private;
3137 	arc_buf_t *buf = callback->awcb_buf;
3138 	arc_buf_hdr_t *hdr = buf->b_hdr;
3139 
3140 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3141 	callback->awcb_ready(zio, buf, callback->awcb_private);
3142 
3143 	/*
3144 	 * If the IO is already in progress, then this is a re-write
3145 	 * attempt, so we need to thaw and re-compute the cksum.
3146 	 * It is the responsibility of the callback to handle the
3147 	 * accounting for any re-write attempt.
3148 	 */
3149 	if (HDR_IO_IN_PROGRESS(hdr)) {
3150 		mutex_enter(&hdr->b_freeze_lock);
3151 		if (hdr->b_freeze_cksum != NULL) {
3152 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3153 			hdr->b_freeze_cksum = NULL;
3154 		}
3155 		mutex_exit(&hdr->b_freeze_lock);
3156 	}
3157 	arc_cksum_compute(buf, B_FALSE);
3158 	hdr->b_flags |= ARC_IO_IN_PROGRESS;
3159 }
3160 
3161 static void
3162 arc_write_done(zio_t *zio)
3163 {
3164 	arc_write_callback_t *callback = zio->io_private;
3165 	arc_buf_t *buf = callback->awcb_buf;
3166 	arc_buf_hdr_t *hdr = buf->b_hdr;
3167 
3168 	ASSERT(hdr->b_acb == NULL);
3169 
3170 	if (zio->io_error == 0) {
3171 		hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3172 		hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3173 		hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3174 	} else {
3175 		ASSERT(BUF_EMPTY(hdr));
3176 	}
3177 
3178 	/*
3179 	 * If the block to be written was all-zero, we may have
3180 	 * compressed it away.  In this case no write was performed
3181 	 * so there will be no dva/birth-date/checksum.  The buffer
3182 	 * must therefor remain anonymous (and uncached).
3183 	 */
3184 	if (!BUF_EMPTY(hdr)) {
3185 		arc_buf_hdr_t *exists;
3186 		kmutex_t *hash_lock;
3187 
3188 		ASSERT(zio->io_error == 0);
3189 
3190 		arc_cksum_verify(buf);
3191 
3192 		exists = buf_hash_insert(hdr, &hash_lock);
3193 		if (exists) {
3194 			/*
3195 			 * This can only happen if we overwrite for
3196 			 * sync-to-convergence, because we remove
3197 			 * buffers from the hash table when we arc_free().
3198 			 */
3199 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3200 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3201 					panic("bad overwrite, hdr=%p exists=%p",
3202 					    (void *)hdr, (void *)exists);
3203 				ASSERT(refcount_is_zero(&exists->b_refcnt));
3204 				arc_change_state(arc_anon, exists, hash_lock);
3205 				mutex_exit(hash_lock);
3206 				arc_hdr_destroy(exists);
3207 				exists = buf_hash_insert(hdr, &hash_lock);
3208 				ASSERT3P(exists, ==, NULL);
3209 			} else {
3210 				/* Dedup */
3211 				ASSERT(hdr->b_datacnt == 1);
3212 				ASSERT(hdr->b_state == arc_anon);
3213 				ASSERT(BP_GET_DEDUP(zio->io_bp));
3214 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3215 			}
3216 		}
3217 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3218 		/* if it's not anon, we are doing a scrub */
3219 		if (!exists && hdr->b_state == arc_anon)
3220 			arc_access(hdr, hash_lock);
3221 		mutex_exit(hash_lock);
3222 	} else {
3223 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3224 	}
3225 
3226 	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3227 	callback->awcb_done(zio, buf, callback->awcb_private);
3228 
3229 	kmem_free(callback, sizeof (arc_write_callback_t));
3230 }
3231 
3232 zio_t *
3233 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3234     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp,
3235     arc_done_func_t *ready, arc_done_func_t *done, void *private,
3236     int priority, int zio_flags, const zbookmark_t *zb)
3237 {
3238 	arc_buf_hdr_t *hdr = buf->b_hdr;
3239 	arc_write_callback_t *callback;
3240 	zio_t *zio;
3241 
3242 	ASSERT(ready != NULL);
3243 	ASSERT(done != NULL);
3244 	ASSERT(!HDR_IO_ERROR(hdr));
3245 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3246 	ASSERT(hdr->b_acb == NULL);
3247 	if (l2arc)
3248 		hdr->b_flags |= ARC_L2CACHE;
3249 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3250 	callback->awcb_ready = ready;
3251 	callback->awcb_done = done;
3252 	callback->awcb_private = private;
3253 	callback->awcb_buf = buf;
3254 
3255 	zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3256 	    arc_write_ready, arc_write_done, callback, priority, zio_flags, zb);
3257 
3258 	return (zio);
3259 }
3260 
3261 void
3262 arc_free(spa_t *spa, const blkptr_t *bp)
3263 {
3264 	arc_buf_hdr_t *ab;
3265 	kmutex_t *hash_lock;
3266 	uint64_t guid = spa_guid(spa);
3267 
3268 	/*
3269 	 * If this buffer is in the cache, release it, so it can be re-used.
3270 	 */
3271 	ab = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
3272 	    &hash_lock);
3273 	if (ab != NULL) {
3274 		if (ab->b_state != arc_anon)
3275 			arc_change_state(arc_anon, ab, hash_lock);
3276 		if (HDR_IO_IN_PROGRESS(ab)) {
3277 			/*
3278 			 * This should only happen when we prefetch.
3279 			 */
3280 			ASSERT(ab->b_flags & ARC_PREFETCH);
3281 			ASSERT3U(ab->b_datacnt, ==, 1);
3282 			ab->b_flags |= ARC_FREED_IN_READ;
3283 			if (HDR_IN_HASH_TABLE(ab))
3284 				buf_hash_remove(ab);
3285 			ab->b_arc_access = 0;
3286 			bzero(&ab->b_dva, sizeof (dva_t));
3287 			ab->b_birth = 0;
3288 			ab->b_cksum0 = 0;
3289 			ab->b_buf->b_efunc = NULL;
3290 			ab->b_buf->b_private = NULL;
3291 			mutex_exit(hash_lock);
3292 		} else {
3293 			ASSERT(refcount_is_zero(&ab->b_refcnt));
3294 			ab->b_flags |= ARC_FREE_IN_PROGRESS;
3295 			mutex_exit(hash_lock);
3296 			arc_hdr_destroy(ab);
3297 			ARCSTAT_BUMP(arcstat_deleted);
3298 		}
3299 	}
3300 }
3301 
3302 static int
3303 arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg)
3304 {
3305 #ifdef _KERNEL
3306 	uint64_t available_memory = ptob(freemem);
3307 	static uint64_t page_load = 0;
3308 	static uint64_t last_txg = 0;
3309 
3310 #if defined(__i386)
3311 	available_memory =
3312 	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3313 #endif
3314 	if (available_memory >= zfs_write_limit_max)
3315 		return (0);
3316 
3317 	if (txg > last_txg) {
3318 		last_txg = txg;
3319 		page_load = 0;
3320 	}
3321 	/*
3322 	 * If we are in pageout, we know that memory is already tight,
3323 	 * the arc is already going to be evicting, so we just want to
3324 	 * continue to let page writes occur as quickly as possible.
3325 	 */
3326 	if (curproc == proc_pageout) {
3327 		if (page_load > MAX(ptob(minfree), available_memory) / 4)
3328 			return (ERESTART);
3329 		/* Note: reserve is inflated, so we deflate */
3330 		page_load += reserve / 8;
3331 		return (0);
3332 	} else if (page_load > 0 && arc_reclaim_needed()) {
3333 		/* memory is low, delay before restarting */
3334 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3335 		return (EAGAIN);
3336 	}
3337 	page_load = 0;
3338 
3339 	if (arc_size > arc_c_min) {
3340 		uint64_t evictable_memory =
3341 		    arc_mru->arcs_lsize[ARC_BUFC_DATA] +
3342 		    arc_mru->arcs_lsize[ARC_BUFC_METADATA] +
3343 		    arc_mfu->arcs_lsize[ARC_BUFC_DATA] +
3344 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA];
3345 		available_memory += MIN(evictable_memory, arc_size - arc_c_min);
3346 	}
3347 
3348 	if (inflight_data > available_memory / 4) {
3349 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3350 		return (ERESTART);
3351 	}
3352 #endif
3353 	return (0);
3354 }
3355 
3356 void
3357 arc_tempreserve_clear(uint64_t reserve)
3358 {
3359 	atomic_add_64(&arc_tempreserve, -reserve);
3360 	ASSERT((int64_t)arc_tempreserve >= 0);
3361 }
3362 
3363 int
3364 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3365 {
3366 	int error;
3367 	uint64_t anon_size;
3368 
3369 #ifdef ZFS_DEBUG
3370 	/*
3371 	 * Once in a while, fail for no reason.  Everything should cope.
3372 	 */
3373 	if (spa_get_random(10000) == 0) {
3374 		dprintf("forcing random failure\n");
3375 		return (ERESTART);
3376 	}
3377 #endif
3378 	if (reserve > arc_c/4 && !arc_no_grow)
3379 		arc_c = MIN(arc_c_max, reserve * 4);
3380 	if (reserve > arc_c)
3381 		return (ENOMEM);
3382 
3383 	/*
3384 	 * Don't count loaned bufs as in flight dirty data to prevent long
3385 	 * network delays from blocking transactions that are ready to be
3386 	 * assigned to a txg.
3387 	 */
3388 	anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3389 
3390 	/*
3391 	 * Writes will, almost always, require additional memory allocations
3392 	 * in order to compress/encrypt/etc the data.  We therefor need to
3393 	 * make sure that there is sufficient available memory for this.
3394 	 */
3395 	if (error = arc_memory_throttle(reserve, anon_size, txg))
3396 		return (error);
3397 
3398 	/*
3399 	 * Throttle writes when the amount of dirty data in the cache
3400 	 * gets too large.  We try to keep the cache less than half full
3401 	 * of dirty blocks so that our sync times don't grow too large.
3402 	 * Note: if two requests come in concurrently, we might let them
3403 	 * both succeed, when one of them should fail.  Not a huge deal.
3404 	 */
3405 
3406 	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3407 	    anon_size > arc_c / 4) {
3408 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3409 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3410 		    arc_tempreserve>>10,
3411 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3412 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3413 		    reserve>>10, arc_c>>10);
3414 		return (ERESTART);
3415 	}
3416 	atomic_add_64(&arc_tempreserve, reserve);
3417 	return (0);
3418 }
3419 
3420 void
3421 arc_init(void)
3422 {
3423 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3424 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3425 
3426 	/* Convert seconds to clock ticks */
3427 	arc_min_prefetch_lifespan = 1 * hz;
3428 
3429 	/* Start out with 1/8 of all memory */
3430 	arc_c = physmem * PAGESIZE / 8;
3431 
3432 #ifdef _KERNEL
3433 	/*
3434 	 * On architectures where the physical memory can be larger
3435 	 * than the addressable space (intel in 32-bit mode), we may
3436 	 * need to limit the cache to 1/8 of VM size.
3437 	 */
3438 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3439 #endif
3440 
3441 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
3442 	arc_c_min = MAX(arc_c / 4, 64<<20);
3443 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
3444 	if (arc_c * 8 >= 1<<30)
3445 		arc_c_max = (arc_c * 8) - (1<<30);
3446 	else
3447 		arc_c_max = arc_c_min;
3448 	arc_c_max = MAX(arc_c * 6, arc_c_max);
3449 
3450 	/*
3451 	 * Allow the tunables to override our calculations if they are
3452 	 * reasonable (ie. over 64MB)
3453 	 */
3454 	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
3455 		arc_c_max = zfs_arc_max;
3456 	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
3457 		arc_c_min = zfs_arc_min;
3458 
3459 	arc_c = arc_c_max;
3460 	arc_p = (arc_c >> 1);
3461 
3462 	/* limit meta-data to 1/4 of the arc capacity */
3463 	arc_meta_limit = arc_c_max / 4;
3464 
3465 	/* Allow the tunable to override if it is reasonable */
3466 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
3467 		arc_meta_limit = zfs_arc_meta_limit;
3468 
3469 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
3470 		arc_c_min = arc_meta_limit / 2;
3471 
3472 	if (zfs_arc_grow_retry > 0)
3473 		arc_grow_retry = zfs_arc_grow_retry;
3474 
3475 	if (zfs_arc_shrink_shift > 0)
3476 		arc_shrink_shift = zfs_arc_shrink_shift;
3477 
3478 	if (zfs_arc_p_min_shift > 0)
3479 		arc_p_min_shift = zfs_arc_p_min_shift;
3480 
3481 	/* if kmem_flags are set, lets try to use less memory */
3482 	if (kmem_debugging())
3483 		arc_c = arc_c / 2;
3484 	if (arc_c < arc_c_min)
3485 		arc_c = arc_c_min;
3486 
3487 	arc_anon = &ARC_anon;
3488 	arc_mru = &ARC_mru;
3489 	arc_mru_ghost = &ARC_mru_ghost;
3490 	arc_mfu = &ARC_mfu;
3491 	arc_mfu_ghost = &ARC_mfu_ghost;
3492 	arc_l2c_only = &ARC_l2c_only;
3493 	arc_size = 0;
3494 
3495 	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3496 	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3497 	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3498 	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3499 	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3500 	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3501 
3502 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
3503 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3504 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
3505 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3506 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
3507 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3508 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
3509 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3510 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
3511 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3512 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
3513 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3514 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
3515 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3516 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
3517 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3518 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3519 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3520 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3521 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3522 
3523 	buf_init();
3524 
3525 	arc_thread_exit = 0;
3526 	arc_eviction_list = NULL;
3527 	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
3528 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3529 
3530 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
3531 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
3532 
3533 	if (arc_ksp != NULL) {
3534 		arc_ksp->ks_data = &arc_stats;
3535 		kstat_install(arc_ksp);
3536 	}
3537 
3538 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3539 	    TS_RUN, minclsyspri);
3540 
3541 	arc_dead = FALSE;
3542 	arc_warm = B_FALSE;
3543 
3544 	if (zfs_write_limit_max == 0)
3545 		zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
3546 	else
3547 		zfs_write_limit_shift = 0;
3548 	mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL);
3549 }
3550 
3551 void
3552 arc_fini(void)
3553 {
3554 	mutex_enter(&arc_reclaim_thr_lock);
3555 	arc_thread_exit = 1;
3556 	while (arc_thread_exit != 0)
3557 		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3558 	mutex_exit(&arc_reclaim_thr_lock);
3559 
3560 	arc_flush(NULL);
3561 
3562 	arc_dead = TRUE;
3563 
3564 	if (arc_ksp != NULL) {
3565 		kstat_delete(arc_ksp);
3566 		arc_ksp = NULL;
3567 	}
3568 
3569 	mutex_destroy(&arc_eviction_mtx);
3570 	mutex_destroy(&arc_reclaim_thr_lock);
3571 	cv_destroy(&arc_reclaim_thr_cv);
3572 
3573 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
3574 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
3575 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
3576 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
3577 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
3578 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
3579 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
3580 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3581 
3582 	mutex_destroy(&arc_anon->arcs_mtx);
3583 	mutex_destroy(&arc_mru->arcs_mtx);
3584 	mutex_destroy(&arc_mru_ghost->arcs_mtx);
3585 	mutex_destroy(&arc_mfu->arcs_mtx);
3586 	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3587 	mutex_destroy(&arc_l2c_only->arcs_mtx);
3588 
3589 	mutex_destroy(&zfs_write_limit_lock);
3590 
3591 	buf_fini();
3592 
3593 	ASSERT(arc_loaned_bytes == 0);
3594 }
3595 
3596 /*
3597  * Level 2 ARC
3598  *
3599  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3600  * It uses dedicated storage devices to hold cached data, which are populated
3601  * using large infrequent writes.  The main role of this cache is to boost
3602  * the performance of random read workloads.  The intended L2ARC devices
3603  * include short-stroked disks, solid state disks, and other media with
3604  * substantially faster read latency than disk.
3605  *
3606  *                 +-----------------------+
3607  *                 |         ARC           |
3608  *                 +-----------------------+
3609  *                    |         ^     ^
3610  *                    |         |     |
3611  *      l2arc_feed_thread()    arc_read()
3612  *                    |         |     |
3613  *                    |  l2arc read   |
3614  *                    V         |     |
3615  *               +---------------+    |
3616  *               |     L2ARC     |    |
3617  *               +---------------+    |
3618  *                   |    ^           |
3619  *          l2arc_write() |           |
3620  *                   |    |           |
3621  *                   V    |           |
3622  *                 +-------+      +-------+
3623  *                 | vdev  |      | vdev  |
3624  *                 | cache |      | cache |
3625  *                 +-------+      +-------+
3626  *                 +=========+     .-----.
3627  *                 :  L2ARC  :    |-_____-|
3628  *                 : devices :    | Disks |
3629  *                 +=========+    `-_____-'
3630  *
3631  * Read requests are satisfied from the following sources, in order:
3632  *
3633  *	1) ARC
3634  *	2) vdev cache of L2ARC devices
3635  *	3) L2ARC devices
3636  *	4) vdev cache of disks
3637  *	5) disks
3638  *
3639  * Some L2ARC device types exhibit extremely slow write performance.
3640  * To accommodate for this there are some significant differences between
3641  * the L2ARC and traditional cache design:
3642  *
3643  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3644  * the ARC behave as usual, freeing buffers and placing headers on ghost
3645  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3646  * this would add inflated write latencies for all ARC memory pressure.
3647  *
3648  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3649  * It does this by periodically scanning buffers from the eviction-end of
3650  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3651  * not already there.  It scans until a headroom of buffers is satisfied,
3652  * which itself is a buffer for ARC eviction.  The thread that does this is
3653  * l2arc_feed_thread(), illustrated below; example sizes are included to
3654  * provide a better sense of ratio than this diagram:
3655  *
3656  *	       head -->                        tail
3657  *	        +---------------------+----------+
3658  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3659  *	        +---------------------+----------+   |   o L2ARC eligible
3660  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3661  *	        +---------------------+----------+   |
3662  *	             15.9 Gbytes      ^ 32 Mbytes    |
3663  *	                           headroom          |
3664  *	                                      l2arc_feed_thread()
3665  *	                                             |
3666  *	                 l2arc write hand <--[oooo]--'
3667  *	                         |           8 Mbyte
3668  *	                         |          write max
3669  *	                         V
3670  *		  +==============================+
3671  *	L2ARC dev |####|#|###|###|    |####| ... |
3672  *	          +==============================+
3673  *	                     32 Gbytes
3674  *
3675  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3676  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3677  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3678  * safe to say that this is an uncommon case, since buffers at the end of
3679  * the ARC lists have moved there due to inactivity.
3680  *
3681  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3682  * then the L2ARC simply misses copying some buffers.  This serves as a
3683  * pressure valve to prevent heavy read workloads from both stalling the ARC
3684  * with waits and clogging the L2ARC with writes.  This also helps prevent
3685  * the potential for the L2ARC to churn if it attempts to cache content too
3686  * quickly, such as during backups of the entire pool.
3687  *
3688  * 5. After system boot and before the ARC has filled main memory, there are
3689  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
3690  * lists can remain mostly static.  Instead of searching from tail of these
3691  * lists as pictured, the l2arc_feed_thread() will search from the list heads
3692  * for eligible buffers, greatly increasing its chance of finding them.
3693  *
3694  * The L2ARC device write speed is also boosted during this time so that
3695  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
3696  * there are no L2ARC reads, and no fear of degrading read performance
3697  * through increased writes.
3698  *
3699  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
3700  * the vdev queue can aggregate them into larger and fewer writes.  Each
3701  * device is written to in a rotor fashion, sweeping writes through
3702  * available space then repeating.
3703  *
3704  * 7. The L2ARC does not store dirty content.  It never needs to flush
3705  * write buffers back to disk based storage.
3706  *
3707  * 8. If an ARC buffer is written (and dirtied) which also exists in the
3708  * L2ARC, the now stale L2ARC buffer is immediately dropped.
3709  *
3710  * The performance of the L2ARC can be tweaked by a number of tunables, which
3711  * may be necessary for different workloads:
3712  *
3713  *	l2arc_write_max		max write bytes per interval
3714  *	l2arc_write_boost	extra write bytes during device warmup
3715  *	l2arc_noprefetch	skip caching prefetched buffers
3716  *	l2arc_headroom		number of max device writes to precache
3717  *	l2arc_feed_secs		seconds between L2ARC writing
3718  *
3719  * Tunables may be removed or added as future performance improvements are
3720  * integrated, and also may become zpool properties.
3721  *
3722  * There are three key functions that control how the L2ARC warms up:
3723  *
3724  *	l2arc_write_eligible()	check if a buffer is eligible to cache
3725  *	l2arc_write_size()	calculate how much to write
3726  *	l2arc_write_interval()	calculate sleep delay between writes
3727  *
3728  * These three functions determine what to write, how much, and how quickly
3729  * to send writes.
3730  */
3731 
3732 static boolean_t
3733 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
3734 {
3735 	/*
3736 	 * A buffer is *not* eligible for the L2ARC if it:
3737 	 * 1. belongs to a different spa.
3738 	 * 2. is already cached on the L2ARC.
3739 	 * 3. has an I/O in progress (it may be an incomplete read).
3740 	 * 4. is flagged not eligible (zfs property).
3741 	 */
3742 	if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
3743 	    HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
3744 		return (B_FALSE);
3745 
3746 	return (B_TRUE);
3747 }
3748 
3749 static uint64_t
3750 l2arc_write_size(l2arc_dev_t *dev)
3751 {
3752 	uint64_t size;
3753 
3754 	size = dev->l2ad_write;
3755 
3756 	if (arc_warm == B_FALSE)
3757 		size += dev->l2ad_boost;
3758 
3759 	return (size);
3760 
3761 }
3762 
3763 static clock_t
3764 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
3765 {
3766 	clock_t interval, next, now;
3767 
3768 	/*
3769 	 * If the ARC lists are busy, increase our write rate; if the
3770 	 * lists are stale, idle back.  This is achieved by checking
3771 	 * how much we previously wrote - if it was more than half of
3772 	 * what we wanted, schedule the next write much sooner.
3773 	 */
3774 	if (l2arc_feed_again && wrote > (wanted / 2))
3775 		interval = (hz * l2arc_feed_min_ms) / 1000;
3776 	else
3777 		interval = hz * l2arc_feed_secs;
3778 
3779 	now = ddi_get_lbolt();
3780 	next = MAX(now, MIN(now + interval, began + interval));
3781 
3782 	return (next);
3783 }
3784 
3785 static void
3786 l2arc_hdr_stat_add(void)
3787 {
3788 	ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
3789 	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
3790 }
3791 
3792 static void
3793 l2arc_hdr_stat_remove(void)
3794 {
3795 	ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
3796 	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
3797 }
3798 
3799 /*
3800  * Cycle through L2ARC devices.  This is how L2ARC load balances.
3801  * If a device is returned, this also returns holding the spa config lock.
3802  */
3803 static l2arc_dev_t *
3804 l2arc_dev_get_next(void)
3805 {
3806 	l2arc_dev_t *first, *next = NULL;
3807 
3808 	/*
3809 	 * Lock out the removal of spas (spa_namespace_lock), then removal
3810 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
3811 	 * both locks will be dropped and a spa config lock held instead.
3812 	 */
3813 	mutex_enter(&spa_namespace_lock);
3814 	mutex_enter(&l2arc_dev_mtx);
3815 
3816 	/* if there are no vdevs, there is nothing to do */
3817 	if (l2arc_ndev == 0)
3818 		goto out;
3819 
3820 	first = NULL;
3821 	next = l2arc_dev_last;
3822 	do {
3823 		/* loop around the list looking for a non-faulted vdev */
3824 		if (next == NULL) {
3825 			next = list_head(l2arc_dev_list);
3826 		} else {
3827 			next = list_next(l2arc_dev_list, next);
3828 			if (next == NULL)
3829 				next = list_head(l2arc_dev_list);
3830 		}
3831 
3832 		/* if we have come back to the start, bail out */
3833 		if (first == NULL)
3834 			first = next;
3835 		else if (next == first)
3836 			break;
3837 
3838 	} while (vdev_is_dead(next->l2ad_vdev));
3839 
3840 	/* if we were unable to find any usable vdevs, return NULL */
3841 	if (vdev_is_dead(next->l2ad_vdev))
3842 		next = NULL;
3843 
3844 	l2arc_dev_last = next;
3845 
3846 out:
3847 	mutex_exit(&l2arc_dev_mtx);
3848 
3849 	/*
3850 	 * Grab the config lock to prevent the 'next' device from being
3851 	 * removed while we are writing to it.
3852 	 */
3853 	if (next != NULL)
3854 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
3855 	mutex_exit(&spa_namespace_lock);
3856 
3857 	return (next);
3858 }
3859 
3860 /*
3861  * Free buffers that were tagged for destruction.
3862  */
3863 static void
3864 l2arc_do_free_on_write()
3865 {
3866 	list_t *buflist;
3867 	l2arc_data_free_t *df, *df_prev;
3868 
3869 	mutex_enter(&l2arc_free_on_write_mtx);
3870 	buflist = l2arc_free_on_write;
3871 
3872 	for (df = list_tail(buflist); df; df = df_prev) {
3873 		df_prev = list_prev(buflist, df);
3874 		ASSERT(df->l2df_data != NULL);
3875 		ASSERT(df->l2df_func != NULL);
3876 		df->l2df_func(df->l2df_data, df->l2df_size);
3877 		list_remove(buflist, df);
3878 		kmem_free(df, sizeof (l2arc_data_free_t));
3879 	}
3880 
3881 	mutex_exit(&l2arc_free_on_write_mtx);
3882 }
3883 
3884 /*
3885  * A write to a cache device has completed.  Update all headers to allow
3886  * reads from these buffers to begin.
3887  */
3888 static void
3889 l2arc_write_done(zio_t *zio)
3890 {
3891 	l2arc_write_callback_t *cb;
3892 	l2arc_dev_t *dev;
3893 	list_t *buflist;
3894 	arc_buf_hdr_t *head, *ab, *ab_prev;
3895 	l2arc_buf_hdr_t *abl2;
3896 	kmutex_t *hash_lock;
3897 
3898 	cb = zio->io_private;
3899 	ASSERT(cb != NULL);
3900 	dev = cb->l2wcb_dev;
3901 	ASSERT(dev != NULL);
3902 	head = cb->l2wcb_head;
3903 	ASSERT(head != NULL);
3904 	buflist = dev->l2ad_buflist;
3905 	ASSERT(buflist != NULL);
3906 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
3907 	    l2arc_write_callback_t *, cb);
3908 
3909 	if (zio->io_error != 0)
3910 		ARCSTAT_BUMP(arcstat_l2_writes_error);
3911 
3912 	mutex_enter(&l2arc_buflist_mtx);
3913 
3914 	/*
3915 	 * All writes completed, or an error was hit.
3916 	 */
3917 	for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
3918 		ab_prev = list_prev(buflist, ab);
3919 
3920 		hash_lock = HDR_LOCK(ab);
3921 		if (!mutex_tryenter(hash_lock)) {
3922 			/*
3923 			 * This buffer misses out.  It may be in a stage
3924 			 * of eviction.  Its ARC_L2_WRITING flag will be
3925 			 * left set, denying reads to this buffer.
3926 			 */
3927 			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
3928 			continue;
3929 		}
3930 
3931 		if (zio->io_error != 0) {
3932 			/*
3933 			 * Error - drop L2ARC entry.
3934 			 */
3935 			list_remove(buflist, ab);
3936 			abl2 = ab->b_l2hdr;
3937 			ab->b_l2hdr = NULL;
3938 			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
3939 			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
3940 		}
3941 
3942 		/*
3943 		 * Allow ARC to begin reads to this L2ARC entry.
3944 		 */
3945 		ab->b_flags &= ~ARC_L2_WRITING;
3946 
3947 		mutex_exit(hash_lock);
3948 	}
3949 
3950 	atomic_inc_64(&l2arc_writes_done);
3951 	list_remove(buflist, head);
3952 	kmem_cache_free(hdr_cache, head);
3953 	mutex_exit(&l2arc_buflist_mtx);
3954 
3955 	l2arc_do_free_on_write();
3956 
3957 	kmem_free(cb, sizeof (l2arc_write_callback_t));
3958 }
3959 
3960 /*
3961  * A read to a cache device completed.  Validate buffer contents before
3962  * handing over to the regular ARC routines.
3963  */
3964 static void
3965 l2arc_read_done(zio_t *zio)
3966 {
3967 	l2arc_read_callback_t *cb;
3968 	arc_buf_hdr_t *hdr;
3969 	arc_buf_t *buf;
3970 	kmutex_t *hash_lock;
3971 	int equal;
3972 
3973 	ASSERT(zio->io_vd != NULL);
3974 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
3975 
3976 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
3977 
3978 	cb = zio->io_private;
3979 	ASSERT(cb != NULL);
3980 	buf = cb->l2rcb_buf;
3981 	ASSERT(buf != NULL);
3982 	hdr = buf->b_hdr;
3983 	ASSERT(hdr != NULL);
3984 
3985 	hash_lock = HDR_LOCK(hdr);
3986 	mutex_enter(hash_lock);
3987 
3988 	/*
3989 	 * Check this survived the L2ARC journey.
3990 	 */
3991 	equal = arc_cksum_equal(buf);
3992 	if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
3993 		mutex_exit(hash_lock);
3994 		zio->io_private = buf;
3995 		zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
3996 		zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
3997 		arc_read_done(zio);
3998 	} else {
3999 		mutex_exit(hash_lock);
4000 		/*
4001 		 * Buffer didn't survive caching.  Increment stats and
4002 		 * reissue to the original storage device.
4003 		 */
4004 		if (zio->io_error != 0) {
4005 			ARCSTAT_BUMP(arcstat_l2_io_error);
4006 		} else {
4007 			zio->io_error = EIO;
4008 		}
4009 		if (!equal)
4010 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4011 
4012 		/*
4013 		 * If there's no waiter, issue an async i/o to the primary
4014 		 * storage now.  If there *is* a waiter, the caller must
4015 		 * issue the i/o in a context where it's OK to block.
4016 		 */
4017 		if (zio->io_waiter == NULL) {
4018 			zio_t *pio = zio_unique_parent(zio);
4019 
4020 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4021 
4022 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4023 			    buf->b_data, zio->io_size, arc_read_done, buf,
4024 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4025 		}
4026 	}
4027 
4028 	kmem_free(cb, sizeof (l2arc_read_callback_t));
4029 }
4030 
4031 /*
4032  * This is the list priority from which the L2ARC will search for pages to
4033  * cache.  This is used within loops (0..3) to cycle through lists in the
4034  * desired order.  This order can have a significant effect on cache
4035  * performance.
4036  *
4037  * Currently the metadata lists are hit first, MFU then MRU, followed by
4038  * the data lists.  This function returns a locked list, and also returns
4039  * the lock pointer.
4040  */
4041 static list_t *
4042 l2arc_list_locked(int list_num, kmutex_t **lock)
4043 {
4044 	list_t *list;
4045 
4046 	ASSERT(list_num >= 0 && list_num <= 3);
4047 
4048 	switch (list_num) {
4049 	case 0:
4050 		list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4051 		*lock = &arc_mfu->arcs_mtx;
4052 		break;
4053 	case 1:
4054 		list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4055 		*lock = &arc_mru->arcs_mtx;
4056 		break;
4057 	case 2:
4058 		list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4059 		*lock = &arc_mfu->arcs_mtx;
4060 		break;
4061 	case 3:
4062 		list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4063 		*lock = &arc_mru->arcs_mtx;
4064 		break;
4065 	}
4066 
4067 	ASSERT(!(MUTEX_HELD(*lock)));
4068 	mutex_enter(*lock);
4069 	return (list);
4070 }
4071 
4072 /*
4073  * Evict buffers from the device write hand to the distance specified in
4074  * bytes.  This distance may span populated buffers, it may span nothing.
4075  * This is clearing a region on the L2ARC device ready for writing.
4076  * If the 'all' boolean is set, every buffer is evicted.
4077  */
4078 static void
4079 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4080 {
4081 	list_t *buflist;
4082 	l2arc_buf_hdr_t *abl2;
4083 	arc_buf_hdr_t *ab, *ab_prev;
4084 	kmutex_t *hash_lock;
4085 	uint64_t taddr;
4086 
4087 	buflist = dev->l2ad_buflist;
4088 
4089 	if (buflist == NULL)
4090 		return;
4091 
4092 	if (!all && dev->l2ad_first) {
4093 		/*
4094 		 * This is the first sweep through the device.  There is
4095 		 * nothing to evict.
4096 		 */
4097 		return;
4098 	}
4099 
4100 	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4101 		/*
4102 		 * When nearing the end of the device, evict to the end
4103 		 * before the device write hand jumps to the start.
4104 		 */
4105 		taddr = dev->l2ad_end;
4106 	} else {
4107 		taddr = dev->l2ad_hand + distance;
4108 	}
4109 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4110 	    uint64_t, taddr, boolean_t, all);
4111 
4112 top:
4113 	mutex_enter(&l2arc_buflist_mtx);
4114 	for (ab = list_tail(buflist); ab; ab = ab_prev) {
4115 		ab_prev = list_prev(buflist, ab);
4116 
4117 		hash_lock = HDR_LOCK(ab);
4118 		if (!mutex_tryenter(hash_lock)) {
4119 			/*
4120 			 * Missed the hash lock.  Retry.
4121 			 */
4122 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4123 			mutex_exit(&l2arc_buflist_mtx);
4124 			mutex_enter(hash_lock);
4125 			mutex_exit(hash_lock);
4126 			goto top;
4127 		}
4128 
4129 		if (HDR_L2_WRITE_HEAD(ab)) {
4130 			/*
4131 			 * We hit a write head node.  Leave it for
4132 			 * l2arc_write_done().
4133 			 */
4134 			list_remove(buflist, ab);
4135 			mutex_exit(hash_lock);
4136 			continue;
4137 		}
4138 
4139 		if (!all && ab->b_l2hdr != NULL &&
4140 		    (ab->b_l2hdr->b_daddr > taddr ||
4141 		    ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4142 			/*
4143 			 * We've evicted to the target address,
4144 			 * or the end of the device.
4145 			 */
4146 			mutex_exit(hash_lock);
4147 			break;
4148 		}
4149 
4150 		if (HDR_FREE_IN_PROGRESS(ab)) {
4151 			/*
4152 			 * Already on the path to destruction.
4153 			 */
4154 			mutex_exit(hash_lock);
4155 			continue;
4156 		}
4157 
4158 		if (ab->b_state == arc_l2c_only) {
4159 			ASSERT(!HDR_L2_READING(ab));
4160 			/*
4161 			 * This doesn't exist in the ARC.  Destroy.
4162 			 * arc_hdr_destroy() will call list_remove()
4163 			 * and decrement arcstat_l2_size.
4164 			 */
4165 			arc_change_state(arc_anon, ab, hash_lock);
4166 			arc_hdr_destroy(ab);
4167 		} else {
4168 			/*
4169 			 * Invalidate issued or about to be issued
4170 			 * reads, since we may be about to write
4171 			 * over this location.
4172 			 */
4173 			if (HDR_L2_READING(ab)) {
4174 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
4175 				ab->b_flags |= ARC_L2_EVICTED;
4176 			}
4177 
4178 			/*
4179 			 * Tell ARC this no longer exists in L2ARC.
4180 			 */
4181 			if (ab->b_l2hdr != NULL) {
4182 				abl2 = ab->b_l2hdr;
4183 				ab->b_l2hdr = NULL;
4184 				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4185 				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4186 			}
4187 			list_remove(buflist, ab);
4188 
4189 			/*
4190 			 * This may have been leftover after a
4191 			 * failed write.
4192 			 */
4193 			ab->b_flags &= ~ARC_L2_WRITING;
4194 		}
4195 		mutex_exit(hash_lock);
4196 	}
4197 	mutex_exit(&l2arc_buflist_mtx);
4198 
4199 	vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4200 	dev->l2ad_evict = taddr;
4201 }
4202 
4203 /*
4204  * Find and write ARC buffers to the L2ARC device.
4205  *
4206  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4207  * for reading until they have completed writing.
4208  */
4209 static uint64_t
4210 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
4211 {
4212 	arc_buf_hdr_t *ab, *ab_prev, *head;
4213 	l2arc_buf_hdr_t *hdrl2;
4214 	list_t *list;
4215 	uint64_t passed_sz, write_sz, buf_sz, headroom;
4216 	void *buf_data;
4217 	kmutex_t *hash_lock, *list_lock;
4218 	boolean_t have_lock, full;
4219 	l2arc_write_callback_t *cb;
4220 	zio_t *pio, *wzio;
4221 	uint64_t guid = spa_guid(spa);
4222 
4223 	ASSERT(dev->l2ad_vdev != NULL);
4224 
4225 	pio = NULL;
4226 	write_sz = 0;
4227 	full = B_FALSE;
4228 	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4229 	head->b_flags |= ARC_L2_WRITE_HEAD;
4230 
4231 	/*
4232 	 * Copy buffers for L2ARC writing.
4233 	 */
4234 	mutex_enter(&l2arc_buflist_mtx);
4235 	for (int try = 0; try <= 3; try++) {
4236 		list = l2arc_list_locked(try, &list_lock);
4237 		passed_sz = 0;
4238 
4239 		/*
4240 		 * L2ARC fast warmup.
4241 		 *
4242 		 * Until the ARC is warm and starts to evict, read from the
4243 		 * head of the ARC lists rather than the tail.
4244 		 */
4245 		headroom = target_sz * l2arc_headroom;
4246 		if (arc_warm == B_FALSE)
4247 			ab = list_head(list);
4248 		else
4249 			ab = list_tail(list);
4250 
4251 		for (; ab; ab = ab_prev) {
4252 			if (arc_warm == B_FALSE)
4253 				ab_prev = list_next(list, ab);
4254 			else
4255 				ab_prev = list_prev(list, ab);
4256 
4257 			hash_lock = HDR_LOCK(ab);
4258 			have_lock = MUTEX_HELD(hash_lock);
4259 			if (!have_lock && !mutex_tryenter(hash_lock)) {
4260 				/*
4261 				 * Skip this buffer rather than waiting.
4262 				 */
4263 				continue;
4264 			}
4265 
4266 			passed_sz += ab->b_size;
4267 			if (passed_sz > headroom) {
4268 				/*
4269 				 * Searched too far.
4270 				 */
4271 				mutex_exit(hash_lock);
4272 				break;
4273 			}
4274 
4275 			if (!l2arc_write_eligible(guid, ab)) {
4276 				mutex_exit(hash_lock);
4277 				continue;
4278 			}
4279 
4280 			if ((write_sz + ab->b_size) > target_sz) {
4281 				full = B_TRUE;
4282 				mutex_exit(hash_lock);
4283 				break;
4284 			}
4285 
4286 			if (pio == NULL) {
4287 				/*
4288 				 * Insert a dummy header on the buflist so
4289 				 * l2arc_write_done() can find where the
4290 				 * write buffers begin without searching.
4291 				 */
4292 				list_insert_head(dev->l2ad_buflist, head);
4293 
4294 				cb = kmem_alloc(
4295 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
4296 				cb->l2wcb_dev = dev;
4297 				cb->l2wcb_head = head;
4298 				pio = zio_root(spa, l2arc_write_done, cb,
4299 				    ZIO_FLAG_CANFAIL);
4300 			}
4301 
4302 			/*
4303 			 * Create and add a new L2ARC header.
4304 			 */
4305 			hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4306 			hdrl2->b_dev = dev;
4307 			hdrl2->b_daddr = dev->l2ad_hand;
4308 
4309 			ab->b_flags |= ARC_L2_WRITING;
4310 			ab->b_l2hdr = hdrl2;
4311 			list_insert_head(dev->l2ad_buflist, ab);
4312 			buf_data = ab->b_buf->b_data;
4313 			buf_sz = ab->b_size;
4314 
4315 			/*
4316 			 * Compute and store the buffer cksum before
4317 			 * writing.  On debug the cksum is verified first.
4318 			 */
4319 			arc_cksum_verify(ab->b_buf);
4320 			arc_cksum_compute(ab->b_buf, B_TRUE);
4321 
4322 			mutex_exit(hash_lock);
4323 
4324 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
4325 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4326 			    NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4327 			    ZIO_FLAG_CANFAIL, B_FALSE);
4328 
4329 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4330 			    zio_t *, wzio);
4331 			(void) zio_nowait(wzio);
4332 
4333 			/*
4334 			 * Keep the clock hand suitably device-aligned.
4335 			 */
4336 			buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4337 
4338 			write_sz += buf_sz;
4339 			dev->l2ad_hand += buf_sz;
4340 		}
4341 
4342 		mutex_exit(list_lock);
4343 
4344 		if (full == B_TRUE)
4345 			break;
4346 	}
4347 	mutex_exit(&l2arc_buflist_mtx);
4348 
4349 	if (pio == NULL) {
4350 		ASSERT3U(write_sz, ==, 0);
4351 		kmem_cache_free(hdr_cache, head);
4352 		return (0);
4353 	}
4354 
4355 	ASSERT3U(write_sz, <=, target_sz);
4356 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
4357 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4358 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
4359 	vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0);
4360 
4361 	/*
4362 	 * Bump device hand to the device start if it is approaching the end.
4363 	 * l2arc_evict() will already have evicted ahead for this case.
4364 	 */
4365 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4366 		vdev_space_update(dev->l2ad_vdev,
4367 		    dev->l2ad_end - dev->l2ad_hand, 0, 0);
4368 		dev->l2ad_hand = dev->l2ad_start;
4369 		dev->l2ad_evict = dev->l2ad_start;
4370 		dev->l2ad_first = B_FALSE;
4371 	}
4372 
4373 	dev->l2ad_writing = B_TRUE;
4374 	(void) zio_wait(pio);
4375 	dev->l2ad_writing = B_FALSE;
4376 
4377 	return (write_sz);
4378 }
4379 
4380 /*
4381  * This thread feeds the L2ARC at regular intervals.  This is the beating
4382  * heart of the L2ARC.
4383  */
4384 static void
4385 l2arc_feed_thread(void)
4386 {
4387 	callb_cpr_t cpr;
4388 	l2arc_dev_t *dev;
4389 	spa_t *spa;
4390 	uint64_t size, wrote;
4391 	clock_t begin, next = ddi_get_lbolt();
4392 
4393 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4394 
4395 	mutex_enter(&l2arc_feed_thr_lock);
4396 
4397 	while (l2arc_thread_exit == 0) {
4398 		CALLB_CPR_SAFE_BEGIN(&cpr);
4399 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
4400 		    next);
4401 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4402 		next = ddi_get_lbolt() + hz;
4403 
4404 		/*
4405 		 * Quick check for L2ARC devices.
4406 		 */
4407 		mutex_enter(&l2arc_dev_mtx);
4408 		if (l2arc_ndev == 0) {
4409 			mutex_exit(&l2arc_dev_mtx);
4410 			continue;
4411 		}
4412 		mutex_exit(&l2arc_dev_mtx);
4413 		begin = ddi_get_lbolt();
4414 
4415 		/*
4416 		 * This selects the next l2arc device to write to, and in
4417 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
4418 		 * will return NULL if there are now no l2arc devices or if
4419 		 * they are all faulted.
4420 		 *
4421 		 * If a device is returned, its spa's config lock is also
4422 		 * held to prevent device removal.  l2arc_dev_get_next()
4423 		 * will grab and release l2arc_dev_mtx.
4424 		 */
4425 		if ((dev = l2arc_dev_get_next()) == NULL)
4426 			continue;
4427 
4428 		spa = dev->l2ad_spa;
4429 		ASSERT(spa != NULL);
4430 
4431 		/*
4432 		 * Avoid contributing to memory pressure.
4433 		 */
4434 		if (arc_reclaim_needed()) {
4435 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
4436 			spa_config_exit(spa, SCL_L2ARC, dev);
4437 			continue;
4438 		}
4439 
4440 		ARCSTAT_BUMP(arcstat_l2_feeds);
4441 
4442 		size = l2arc_write_size(dev);
4443 
4444 		/*
4445 		 * Evict L2ARC buffers that will be overwritten.
4446 		 */
4447 		l2arc_evict(dev, size, B_FALSE);
4448 
4449 		/*
4450 		 * Write ARC buffers.
4451 		 */
4452 		wrote = l2arc_write_buffers(spa, dev, size);
4453 
4454 		/*
4455 		 * Calculate interval between writes.
4456 		 */
4457 		next = l2arc_write_interval(begin, size, wrote);
4458 		spa_config_exit(spa, SCL_L2ARC, dev);
4459 	}
4460 
4461 	l2arc_thread_exit = 0;
4462 	cv_broadcast(&l2arc_feed_thr_cv);
4463 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
4464 	thread_exit();
4465 }
4466 
4467 boolean_t
4468 l2arc_vdev_present(vdev_t *vd)
4469 {
4470 	l2arc_dev_t *dev;
4471 
4472 	mutex_enter(&l2arc_dev_mtx);
4473 	for (dev = list_head(l2arc_dev_list); dev != NULL;
4474 	    dev = list_next(l2arc_dev_list, dev)) {
4475 		if (dev->l2ad_vdev == vd)
4476 			break;
4477 	}
4478 	mutex_exit(&l2arc_dev_mtx);
4479 
4480 	return (dev != NULL);
4481 }
4482 
4483 /*
4484  * Add a vdev for use by the L2ARC.  By this point the spa has already
4485  * validated the vdev and opened it.
4486  */
4487 void
4488 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
4489 {
4490 	l2arc_dev_t *adddev;
4491 
4492 	ASSERT(!l2arc_vdev_present(vd));
4493 
4494 	/*
4495 	 * Create a new l2arc device entry.
4496 	 */
4497 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
4498 	adddev->l2ad_spa = spa;
4499 	adddev->l2ad_vdev = vd;
4500 	adddev->l2ad_write = l2arc_write_max;
4501 	adddev->l2ad_boost = l2arc_write_boost;
4502 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
4503 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
4504 	adddev->l2ad_hand = adddev->l2ad_start;
4505 	adddev->l2ad_evict = adddev->l2ad_start;
4506 	adddev->l2ad_first = B_TRUE;
4507 	adddev->l2ad_writing = B_FALSE;
4508 	ASSERT3U(adddev->l2ad_write, >, 0);
4509 
4510 	/*
4511 	 * This is a list of all ARC buffers that are still valid on the
4512 	 * device.
4513 	 */
4514 	adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
4515 	list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
4516 	    offsetof(arc_buf_hdr_t, b_l2node));
4517 
4518 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
4519 
4520 	/*
4521 	 * Add device to global list
4522 	 */
4523 	mutex_enter(&l2arc_dev_mtx);
4524 	list_insert_head(l2arc_dev_list, adddev);
4525 	atomic_inc_64(&l2arc_ndev);
4526 	mutex_exit(&l2arc_dev_mtx);
4527 }
4528 
4529 /*
4530  * Remove a vdev from the L2ARC.
4531  */
4532 void
4533 l2arc_remove_vdev(vdev_t *vd)
4534 {
4535 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
4536 
4537 	/*
4538 	 * Find the device by vdev
4539 	 */
4540 	mutex_enter(&l2arc_dev_mtx);
4541 	for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
4542 		nextdev = list_next(l2arc_dev_list, dev);
4543 		if (vd == dev->l2ad_vdev) {
4544 			remdev = dev;
4545 			break;
4546 		}
4547 	}
4548 	ASSERT(remdev != NULL);
4549 
4550 	/*
4551 	 * Remove device from global list
4552 	 */
4553 	list_remove(l2arc_dev_list, remdev);
4554 	l2arc_dev_last = NULL;		/* may have been invalidated */
4555 	atomic_dec_64(&l2arc_ndev);
4556 	mutex_exit(&l2arc_dev_mtx);
4557 
4558 	/*
4559 	 * Clear all buflists and ARC references.  L2ARC device flush.
4560 	 */
4561 	l2arc_evict(remdev, 0, B_TRUE);
4562 	list_destroy(remdev->l2ad_buflist);
4563 	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
4564 	kmem_free(remdev, sizeof (l2arc_dev_t));
4565 }
4566 
4567 void
4568 l2arc_init(void)
4569 {
4570 	l2arc_thread_exit = 0;
4571 	l2arc_ndev = 0;
4572 	l2arc_writes_sent = 0;
4573 	l2arc_writes_done = 0;
4574 
4575 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
4576 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
4577 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
4578 	mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
4579 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
4580 
4581 	l2arc_dev_list = &L2ARC_dev_list;
4582 	l2arc_free_on_write = &L2ARC_free_on_write;
4583 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
4584 	    offsetof(l2arc_dev_t, l2ad_node));
4585 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
4586 	    offsetof(l2arc_data_free_t, l2df_list_node));
4587 }
4588 
4589 void
4590 l2arc_fini(void)
4591 {
4592 	/*
4593 	 * This is called from dmu_fini(), which is called from spa_fini();
4594 	 * Because of this, we can assume that all l2arc devices have
4595 	 * already been removed when the pools themselves were removed.
4596 	 */
4597 
4598 	l2arc_do_free_on_write();
4599 
4600 	mutex_destroy(&l2arc_feed_thr_lock);
4601 	cv_destroy(&l2arc_feed_thr_cv);
4602 	mutex_destroy(&l2arc_dev_mtx);
4603 	mutex_destroy(&l2arc_buflist_mtx);
4604 	mutex_destroy(&l2arc_free_on_write_mtx);
4605 
4606 	list_destroy(l2arc_dev_list);
4607 	list_destroy(l2arc_free_on_write);
4608 }
4609 
4610 void
4611 l2arc_start(void)
4612 {
4613 	if (!(spa_mode_global & FWRITE))
4614 		return;
4615 
4616 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
4617 	    TS_RUN, minclsyspri);
4618 }
4619 
4620 void
4621 l2arc_stop(void)
4622 {
4623 	if (!(spa_mode_global & FWRITE))
4624 		return;
4625 
4626 	mutex_enter(&l2arc_feed_thr_lock);
4627 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
4628 	l2arc_thread_exit = 1;
4629 	while (l2arc_thread_exit != 0)
4630 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
4631 	mutex_exit(&l2arc_feed_thr_lock);
4632 }
4633