xref: /titanic_44/usr/src/uts/common/fs/zfs/arc.c (revision 4c6ae46166796718de10c1839ffe0569daa48250)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24  * Copyright (c) 2012 by Delphix. All rights reserved.
25  */
26 
27 /*
28  * DVA-based Adjustable Replacement Cache
29  *
30  * While much of the theory of operation used here is
31  * based on the self-tuning, low overhead replacement cache
32  * presented by Megiddo and Modha at FAST 2003, there are some
33  * significant differences:
34  *
35  * 1. The Megiddo and Modha model assumes any page is evictable.
36  * Pages in its cache cannot be "locked" into memory.  This makes
37  * the eviction algorithm simple: evict the last page in the list.
38  * This also make the performance characteristics easy to reason
39  * about.  Our cache is not so simple.  At any given moment, some
40  * subset of the blocks in the cache are un-evictable because we
41  * have handed out a reference to them.  Blocks are only evictable
42  * when there are no external references active.  This makes
43  * eviction far more problematic:  we choose to evict the evictable
44  * blocks that are the "lowest" in the list.
45  *
46  * There are times when it is not possible to evict the requested
47  * space.  In these circumstances we are unable to adjust the cache
48  * size.  To prevent the cache growing unbounded at these times we
49  * implement a "cache throttle" that slows the flow of new data
50  * into the cache until we can make space available.
51  *
52  * 2. The Megiddo and Modha model assumes a fixed cache size.
53  * Pages are evicted when the cache is full and there is a cache
54  * miss.  Our model has a variable sized cache.  It grows with
55  * high use, but also tries to react to memory pressure from the
56  * operating system: decreasing its size when system memory is
57  * tight.
58  *
59  * 3. The Megiddo and Modha model assumes a fixed page size. All
60  * elements of the cache are therefor exactly the same size.  So
61  * when adjusting the cache size following a cache miss, its simply
62  * a matter of choosing a single page to evict.  In our model, we
63  * have variable sized cache blocks (rangeing from 512 bytes to
64  * 128K bytes).  We therefor choose a set of blocks to evict to make
65  * space for a cache miss that approximates as closely as possible
66  * the space used by the new block.
67  *
68  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
69  * by N. Megiddo & D. Modha, FAST 2003
70  */
71 
72 /*
73  * The locking model:
74  *
75  * A new reference to a cache buffer can be obtained in two
76  * ways: 1) via a hash table lookup using the DVA as a key,
77  * or 2) via one of the ARC lists.  The arc_read() interface
78  * uses method 1, while the internal arc algorithms for
79  * adjusting the cache use method 2.  We therefor provide two
80  * types of locks: 1) the hash table lock array, and 2) the
81  * arc list locks.
82  *
83  * Buffers do not have their own mutexes, rather they rely on the
84  * hash table mutexes for the bulk of their protection (i.e. most
85  * fields in the arc_buf_hdr_t are protected by these mutexes).
86  *
87  * buf_hash_find() returns the appropriate mutex (held) when it
88  * locates the requested buffer in the hash table.  It returns
89  * NULL for the mutex if the buffer was not in the table.
90  *
91  * buf_hash_remove() expects the appropriate hash mutex to be
92  * already held before it is invoked.
93  *
94  * Each arc state also has a mutex which is used to protect the
95  * buffer list associated with the state.  When attempting to
96  * obtain a hash table lock while holding an arc list lock you
97  * must use: mutex_tryenter() to avoid deadlock.  Also note that
98  * the active state mutex must be held before the ghost state mutex.
99  *
100  * Arc buffers may have an associated eviction callback function.
101  * This function will be invoked prior to removing the buffer (e.g.
102  * in arc_do_user_evicts()).  Note however that the data associated
103  * with the buffer may be evicted prior to the callback.  The callback
104  * must be made with *no locks held* (to prevent deadlock).  Additionally,
105  * the users of callbacks must ensure that their private data is
106  * protected from simultaneous callbacks from arc_buf_evict()
107  * and arc_do_user_evicts().
108  *
109  * Note that the majority of the performance stats are manipulated
110  * with atomic operations.
111  *
112  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
113  *
114  *	- L2ARC buflist creation
115  *	- L2ARC buflist eviction
116  *	- L2ARC write completion, which walks L2ARC buflists
117  *	- ARC header destruction, as it removes from L2ARC buflists
118  *	- ARC header release, as it removes from L2ARC buflists
119  */
120 
121 #include <sys/spa.h>
122 #include <sys/zio.h>
123 #include <sys/zfs_context.h>
124 #include <sys/arc.h>
125 #include <sys/refcount.h>
126 #include <sys/vdev.h>
127 #include <sys/vdev_impl.h>
128 #ifdef _KERNEL
129 #include <sys/vmsystm.h>
130 #include <vm/anon.h>
131 #include <sys/fs/swapnode.h>
132 #include <sys/dnlc.h>
133 #endif
134 #include <sys/callb.h>
135 #include <sys/kstat.h>
136 #include <zfs_fletcher.h>
137 
138 #ifndef _KERNEL
139 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
140 boolean_t arc_watch = B_FALSE;
141 int arc_procfd;
142 #endif
143 
144 static kmutex_t		arc_reclaim_thr_lock;
145 static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
146 static uint8_t		arc_thread_exit;
147 
148 extern int zfs_write_limit_shift;
149 extern uint64_t zfs_write_limit_max;
150 extern kmutex_t zfs_write_limit_lock;
151 
152 #define	ARC_REDUCE_DNLC_PERCENT	3
153 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
154 
155 typedef enum arc_reclaim_strategy {
156 	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
157 	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
158 } arc_reclaim_strategy_t;
159 
160 /* number of seconds before growing cache again */
161 static int		arc_grow_retry = 60;
162 
163 /* shift of arc_c for calculating both min and max arc_p */
164 static int		arc_p_min_shift = 4;
165 
166 /* log2(fraction of arc to reclaim) */
167 static int		arc_shrink_shift = 5;
168 
169 /*
170  * minimum lifespan of a prefetch block in clock ticks
171  * (initialized in arc_init())
172  */
173 static int		arc_min_prefetch_lifespan;
174 
175 static int arc_dead;
176 
177 /*
178  * The arc has filled available memory and has now warmed up.
179  */
180 static boolean_t arc_warm;
181 
182 /*
183  * These tunables are for performance analysis.
184  */
185 uint64_t zfs_arc_max;
186 uint64_t zfs_arc_min;
187 uint64_t zfs_arc_meta_limit = 0;
188 int zfs_arc_grow_retry = 0;
189 int zfs_arc_shrink_shift = 0;
190 int zfs_arc_p_min_shift = 0;
191 int zfs_disable_dup_eviction = 0;
192 
193 /*
194  * Note that buffers can be in one of 6 states:
195  *	ARC_anon	- anonymous (discussed below)
196  *	ARC_mru		- recently used, currently cached
197  *	ARC_mru_ghost	- recentely used, no longer in cache
198  *	ARC_mfu		- frequently used, currently cached
199  *	ARC_mfu_ghost	- frequently used, no longer in cache
200  *	ARC_l2c_only	- exists in L2ARC but not other states
201  * When there are no active references to the buffer, they are
202  * are linked onto a list in one of these arc states.  These are
203  * the only buffers that can be evicted or deleted.  Within each
204  * state there are multiple lists, one for meta-data and one for
205  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
206  * etc.) is tracked separately so that it can be managed more
207  * explicitly: favored over data, limited explicitly.
208  *
209  * Anonymous buffers are buffers that are not associated with
210  * a DVA.  These are buffers that hold dirty block copies
211  * before they are written to stable storage.  By definition,
212  * they are "ref'd" and are considered part of arc_mru
213  * that cannot be freed.  Generally, they will aquire a DVA
214  * as they are written and migrate onto the arc_mru list.
215  *
216  * The ARC_l2c_only state is for buffers that are in the second
217  * level ARC but no longer in any of the ARC_m* lists.  The second
218  * level ARC itself may also contain buffers that are in any of
219  * the ARC_m* states - meaning that a buffer can exist in two
220  * places.  The reason for the ARC_l2c_only state is to keep the
221  * buffer header in the hash table, so that reads that hit the
222  * second level ARC benefit from these fast lookups.
223  */
224 
225 typedef struct arc_state {
226 	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
227 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
228 	uint64_t arcs_size;	/* total amount of data in this state */
229 	kmutex_t arcs_mtx;
230 } arc_state_t;
231 
232 /* The 6 states: */
233 static arc_state_t ARC_anon;
234 static arc_state_t ARC_mru;
235 static arc_state_t ARC_mru_ghost;
236 static arc_state_t ARC_mfu;
237 static arc_state_t ARC_mfu_ghost;
238 static arc_state_t ARC_l2c_only;
239 
240 typedef struct arc_stats {
241 	kstat_named_t arcstat_hits;
242 	kstat_named_t arcstat_misses;
243 	kstat_named_t arcstat_demand_data_hits;
244 	kstat_named_t arcstat_demand_data_misses;
245 	kstat_named_t arcstat_demand_metadata_hits;
246 	kstat_named_t arcstat_demand_metadata_misses;
247 	kstat_named_t arcstat_prefetch_data_hits;
248 	kstat_named_t arcstat_prefetch_data_misses;
249 	kstat_named_t arcstat_prefetch_metadata_hits;
250 	kstat_named_t arcstat_prefetch_metadata_misses;
251 	kstat_named_t arcstat_mru_hits;
252 	kstat_named_t arcstat_mru_ghost_hits;
253 	kstat_named_t arcstat_mfu_hits;
254 	kstat_named_t arcstat_mfu_ghost_hits;
255 	kstat_named_t arcstat_deleted;
256 	kstat_named_t arcstat_recycle_miss;
257 	kstat_named_t arcstat_mutex_miss;
258 	kstat_named_t arcstat_evict_skip;
259 	kstat_named_t arcstat_evict_l2_cached;
260 	kstat_named_t arcstat_evict_l2_eligible;
261 	kstat_named_t arcstat_evict_l2_ineligible;
262 	kstat_named_t arcstat_hash_elements;
263 	kstat_named_t arcstat_hash_elements_max;
264 	kstat_named_t arcstat_hash_collisions;
265 	kstat_named_t arcstat_hash_chains;
266 	kstat_named_t arcstat_hash_chain_max;
267 	kstat_named_t arcstat_p;
268 	kstat_named_t arcstat_c;
269 	kstat_named_t arcstat_c_min;
270 	kstat_named_t arcstat_c_max;
271 	kstat_named_t arcstat_size;
272 	kstat_named_t arcstat_hdr_size;
273 	kstat_named_t arcstat_data_size;
274 	kstat_named_t arcstat_other_size;
275 	kstat_named_t arcstat_l2_hits;
276 	kstat_named_t arcstat_l2_misses;
277 	kstat_named_t arcstat_l2_feeds;
278 	kstat_named_t arcstat_l2_rw_clash;
279 	kstat_named_t arcstat_l2_read_bytes;
280 	kstat_named_t arcstat_l2_write_bytes;
281 	kstat_named_t arcstat_l2_writes_sent;
282 	kstat_named_t arcstat_l2_writes_done;
283 	kstat_named_t arcstat_l2_writes_error;
284 	kstat_named_t arcstat_l2_writes_hdr_miss;
285 	kstat_named_t arcstat_l2_evict_lock_retry;
286 	kstat_named_t arcstat_l2_evict_reading;
287 	kstat_named_t arcstat_l2_free_on_write;
288 	kstat_named_t arcstat_l2_abort_lowmem;
289 	kstat_named_t arcstat_l2_cksum_bad;
290 	kstat_named_t arcstat_l2_io_error;
291 	kstat_named_t arcstat_l2_size;
292 	kstat_named_t arcstat_l2_hdr_size;
293 	kstat_named_t arcstat_memory_throttle_count;
294 	kstat_named_t arcstat_duplicate_buffers;
295 	kstat_named_t arcstat_duplicate_buffers_size;
296 	kstat_named_t arcstat_duplicate_reads;
297 } arc_stats_t;
298 
299 static arc_stats_t arc_stats = {
300 	{ "hits",			KSTAT_DATA_UINT64 },
301 	{ "misses",			KSTAT_DATA_UINT64 },
302 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
303 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
304 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
305 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
306 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
307 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
308 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
309 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
310 	{ "mru_hits",			KSTAT_DATA_UINT64 },
311 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
312 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
313 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
314 	{ "deleted",			KSTAT_DATA_UINT64 },
315 	{ "recycle_miss",		KSTAT_DATA_UINT64 },
316 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
317 	{ "evict_skip",			KSTAT_DATA_UINT64 },
318 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
319 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
320 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
321 	{ "hash_elements",		KSTAT_DATA_UINT64 },
322 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
323 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
324 	{ "hash_chains",		KSTAT_DATA_UINT64 },
325 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
326 	{ "p",				KSTAT_DATA_UINT64 },
327 	{ "c",				KSTAT_DATA_UINT64 },
328 	{ "c_min",			KSTAT_DATA_UINT64 },
329 	{ "c_max",			KSTAT_DATA_UINT64 },
330 	{ "size",			KSTAT_DATA_UINT64 },
331 	{ "hdr_size",			KSTAT_DATA_UINT64 },
332 	{ "data_size",			KSTAT_DATA_UINT64 },
333 	{ "other_size",			KSTAT_DATA_UINT64 },
334 	{ "l2_hits",			KSTAT_DATA_UINT64 },
335 	{ "l2_misses",			KSTAT_DATA_UINT64 },
336 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
337 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
338 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
339 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
340 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
341 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
342 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
343 	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
344 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
345 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
346 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
347 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
348 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
349 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
350 	{ "l2_size",			KSTAT_DATA_UINT64 },
351 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
352 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
353 	{ "duplicate_buffers",		KSTAT_DATA_UINT64 },
354 	{ "duplicate_buffers_size",	KSTAT_DATA_UINT64 },
355 	{ "duplicate_reads",		KSTAT_DATA_UINT64 }
356 };
357 
358 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
359 
360 #define	ARCSTAT_INCR(stat, val) \
361 	atomic_add_64(&arc_stats.stat.value.ui64, (val));
362 
363 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
364 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
365 
366 #define	ARCSTAT_MAX(stat, val) {					\
367 	uint64_t m;							\
368 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
369 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
370 		continue;						\
371 }
372 
373 #define	ARCSTAT_MAXSTAT(stat) \
374 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
375 
376 /*
377  * We define a macro to allow ARC hits/misses to be easily broken down by
378  * two separate conditions, giving a total of four different subtypes for
379  * each of hits and misses (so eight statistics total).
380  */
381 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
382 	if (cond1) {							\
383 		if (cond2) {						\
384 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
385 		} else {						\
386 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
387 		}							\
388 	} else {							\
389 		if (cond2) {						\
390 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
391 		} else {						\
392 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
393 		}							\
394 	}
395 
396 kstat_t			*arc_ksp;
397 static arc_state_t	*arc_anon;
398 static arc_state_t	*arc_mru;
399 static arc_state_t	*arc_mru_ghost;
400 static arc_state_t	*arc_mfu;
401 static arc_state_t	*arc_mfu_ghost;
402 static arc_state_t	*arc_l2c_only;
403 
404 /*
405  * There are several ARC variables that are critical to export as kstats --
406  * but we don't want to have to grovel around in the kstat whenever we wish to
407  * manipulate them.  For these variables, we therefore define them to be in
408  * terms of the statistic variable.  This assures that we are not introducing
409  * the possibility of inconsistency by having shadow copies of the variables,
410  * while still allowing the code to be readable.
411  */
412 #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
413 #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
414 #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
415 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
416 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
417 
418 static int		arc_no_grow;	/* Don't try to grow cache size */
419 static uint64_t		arc_tempreserve;
420 static uint64_t		arc_loaned_bytes;
421 static uint64_t		arc_meta_used;
422 static uint64_t		arc_meta_limit;
423 static uint64_t		arc_meta_max = 0;
424 
425 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
426 
427 typedef struct arc_callback arc_callback_t;
428 
429 struct arc_callback {
430 	void			*acb_private;
431 	arc_done_func_t		*acb_done;
432 	arc_buf_t		*acb_buf;
433 	zio_t			*acb_zio_dummy;
434 	arc_callback_t		*acb_next;
435 };
436 
437 typedef struct arc_write_callback arc_write_callback_t;
438 
439 struct arc_write_callback {
440 	void		*awcb_private;
441 	arc_done_func_t	*awcb_ready;
442 	arc_done_func_t	*awcb_done;
443 	arc_buf_t	*awcb_buf;
444 };
445 
446 struct arc_buf_hdr {
447 	/* protected by hash lock */
448 	dva_t			b_dva;
449 	uint64_t		b_birth;
450 	uint64_t		b_cksum0;
451 
452 	kmutex_t		b_freeze_lock;
453 	zio_cksum_t		*b_freeze_cksum;
454 	void			*b_thawed;
455 
456 	arc_buf_hdr_t		*b_hash_next;
457 	arc_buf_t		*b_buf;
458 	uint32_t		b_flags;
459 	uint32_t		b_datacnt;
460 
461 	arc_callback_t		*b_acb;
462 	kcondvar_t		b_cv;
463 
464 	/* immutable */
465 	arc_buf_contents_t	b_type;
466 	uint64_t		b_size;
467 	uint64_t		b_spa;
468 
469 	/* protected by arc state mutex */
470 	arc_state_t		*b_state;
471 	list_node_t		b_arc_node;
472 
473 	/* updated atomically */
474 	clock_t			b_arc_access;
475 
476 	/* self protecting */
477 	refcount_t		b_refcnt;
478 
479 	l2arc_buf_hdr_t		*b_l2hdr;
480 	list_node_t		b_l2node;
481 };
482 
483 static arc_buf_t *arc_eviction_list;
484 static kmutex_t arc_eviction_mtx;
485 static arc_buf_hdr_t arc_eviction_hdr;
486 static void arc_get_data_buf(arc_buf_t *buf);
487 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
488 static int arc_evict_needed(arc_buf_contents_t type);
489 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
490 static void arc_buf_watch(arc_buf_t *buf);
491 
492 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
493 
494 #define	GHOST_STATE(state)	\
495 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
496 	(state) == arc_l2c_only)
497 
498 /*
499  * Private ARC flags.  These flags are private ARC only flags that will show up
500  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
501  * be passed in as arc_flags in things like arc_read.  However, these flags
502  * should never be passed and should only be set by ARC code.  When adding new
503  * public flags, make sure not to smash the private ones.
504  */
505 
506 #define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
507 #define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
508 #define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
509 #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
510 #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
511 #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
512 #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
513 #define	ARC_L2_WRITING		(1 << 16)	/* L2ARC write in progress */
514 #define	ARC_L2_EVICTED		(1 << 17)	/* evicted during I/O */
515 #define	ARC_L2_WRITE_HEAD	(1 << 18)	/* head of write list */
516 
517 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
518 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
519 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
520 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_PREFETCH)
521 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
522 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
523 #define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
524 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_L2CACHE)
525 #define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
526 				    (hdr)->b_l2hdr != NULL)
527 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
528 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_L2_EVICTED)
529 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_L2_WRITE_HEAD)
530 
531 /*
532  * Other sizes
533  */
534 
535 #define	HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
536 #define	L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
537 
538 /*
539  * Hash table routines
540  */
541 
542 #define	HT_LOCK_PAD	64
543 
544 struct ht_lock {
545 	kmutex_t	ht_lock;
546 #ifdef _KERNEL
547 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
548 #endif
549 };
550 
551 #define	BUF_LOCKS 256
552 typedef struct buf_hash_table {
553 	uint64_t ht_mask;
554 	arc_buf_hdr_t **ht_table;
555 	struct ht_lock ht_locks[BUF_LOCKS];
556 } buf_hash_table_t;
557 
558 static buf_hash_table_t buf_hash_table;
559 
560 #define	BUF_HASH_INDEX(spa, dva, birth) \
561 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
562 #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
563 #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
564 #define	HDR_LOCK(hdr) \
565 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
566 
567 uint64_t zfs_crc64_table[256];
568 
569 /*
570  * Level 2 ARC
571  */
572 
573 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
574 #define	L2ARC_HEADROOM		2		/* num of writes */
575 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
576 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
577 
578 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
579 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
580 
581 /*
582  * L2ARC Performance Tunables
583  */
584 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
585 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
586 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
587 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
588 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
589 boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
590 boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
591 boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
592 
593 /*
594  * L2ARC Internals
595  */
596 typedef struct l2arc_dev {
597 	vdev_t			*l2ad_vdev;	/* vdev */
598 	spa_t			*l2ad_spa;	/* spa */
599 	uint64_t		l2ad_hand;	/* next write location */
600 	uint64_t		l2ad_write;	/* desired write size, bytes */
601 	uint64_t		l2ad_boost;	/* warmup write boost, bytes */
602 	uint64_t		l2ad_start;	/* first addr on device */
603 	uint64_t		l2ad_end;	/* last addr on device */
604 	uint64_t		l2ad_evict;	/* last addr eviction reached */
605 	boolean_t		l2ad_first;	/* first sweep through */
606 	boolean_t		l2ad_writing;	/* currently writing */
607 	list_t			*l2ad_buflist;	/* buffer list */
608 	list_node_t		l2ad_node;	/* device list node */
609 } l2arc_dev_t;
610 
611 static list_t L2ARC_dev_list;			/* device list */
612 static list_t *l2arc_dev_list;			/* device list pointer */
613 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
614 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
615 static kmutex_t l2arc_buflist_mtx;		/* mutex for all buflists */
616 static list_t L2ARC_free_on_write;		/* free after write buf list */
617 static list_t *l2arc_free_on_write;		/* free after write list ptr */
618 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
619 static uint64_t l2arc_ndev;			/* number of devices */
620 
621 typedef struct l2arc_read_callback {
622 	arc_buf_t	*l2rcb_buf;		/* read buffer */
623 	spa_t		*l2rcb_spa;		/* spa */
624 	blkptr_t	l2rcb_bp;		/* original blkptr */
625 	zbookmark_t	l2rcb_zb;		/* original bookmark */
626 	int		l2rcb_flags;		/* original flags */
627 } l2arc_read_callback_t;
628 
629 typedef struct l2arc_write_callback {
630 	l2arc_dev_t	*l2wcb_dev;		/* device info */
631 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
632 } l2arc_write_callback_t;
633 
634 struct l2arc_buf_hdr {
635 	/* protected by arc_buf_hdr  mutex */
636 	l2arc_dev_t	*b_dev;			/* L2ARC device */
637 	uint64_t	b_daddr;		/* disk address, offset byte */
638 };
639 
640 typedef struct l2arc_data_free {
641 	/* protected by l2arc_free_on_write_mtx */
642 	void		*l2df_data;
643 	size_t		l2df_size;
644 	void		(*l2df_func)(void *, size_t);
645 	list_node_t	l2df_list_node;
646 } l2arc_data_free_t;
647 
648 static kmutex_t l2arc_feed_thr_lock;
649 static kcondvar_t l2arc_feed_thr_cv;
650 static uint8_t l2arc_thread_exit;
651 
652 static void l2arc_read_done(zio_t *zio);
653 static void l2arc_hdr_stat_add(void);
654 static void l2arc_hdr_stat_remove(void);
655 
656 static uint64_t
657 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
658 {
659 	uint8_t *vdva = (uint8_t *)dva;
660 	uint64_t crc = -1ULL;
661 	int i;
662 
663 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
664 
665 	for (i = 0; i < sizeof (dva_t); i++)
666 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
667 
668 	crc ^= (spa>>8) ^ birth;
669 
670 	return (crc);
671 }
672 
673 #define	BUF_EMPTY(buf)						\
674 	((buf)->b_dva.dva_word[0] == 0 &&			\
675 	(buf)->b_dva.dva_word[1] == 0 &&			\
676 	(buf)->b_birth == 0)
677 
678 #define	BUF_EQUAL(spa, dva, birth, buf)				\
679 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
680 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
681 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
682 
683 static void
684 buf_discard_identity(arc_buf_hdr_t *hdr)
685 {
686 	hdr->b_dva.dva_word[0] = 0;
687 	hdr->b_dva.dva_word[1] = 0;
688 	hdr->b_birth = 0;
689 	hdr->b_cksum0 = 0;
690 }
691 
692 static arc_buf_hdr_t *
693 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
694 {
695 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
696 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
697 	arc_buf_hdr_t *buf;
698 
699 	mutex_enter(hash_lock);
700 	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
701 	    buf = buf->b_hash_next) {
702 		if (BUF_EQUAL(spa, dva, birth, buf)) {
703 			*lockp = hash_lock;
704 			return (buf);
705 		}
706 	}
707 	mutex_exit(hash_lock);
708 	*lockp = NULL;
709 	return (NULL);
710 }
711 
712 /*
713  * Insert an entry into the hash table.  If there is already an element
714  * equal to elem in the hash table, then the already existing element
715  * will be returned and the new element will not be inserted.
716  * Otherwise returns NULL.
717  */
718 static arc_buf_hdr_t *
719 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
720 {
721 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
722 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
723 	arc_buf_hdr_t *fbuf;
724 	uint32_t i;
725 
726 	ASSERT(!HDR_IN_HASH_TABLE(buf));
727 	*lockp = hash_lock;
728 	mutex_enter(hash_lock);
729 	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
730 	    fbuf = fbuf->b_hash_next, i++) {
731 		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
732 			return (fbuf);
733 	}
734 
735 	buf->b_hash_next = buf_hash_table.ht_table[idx];
736 	buf_hash_table.ht_table[idx] = buf;
737 	buf->b_flags |= ARC_IN_HASH_TABLE;
738 
739 	/* collect some hash table performance data */
740 	if (i > 0) {
741 		ARCSTAT_BUMP(arcstat_hash_collisions);
742 		if (i == 1)
743 			ARCSTAT_BUMP(arcstat_hash_chains);
744 
745 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
746 	}
747 
748 	ARCSTAT_BUMP(arcstat_hash_elements);
749 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
750 
751 	return (NULL);
752 }
753 
754 static void
755 buf_hash_remove(arc_buf_hdr_t *buf)
756 {
757 	arc_buf_hdr_t *fbuf, **bufp;
758 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
759 
760 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
761 	ASSERT(HDR_IN_HASH_TABLE(buf));
762 
763 	bufp = &buf_hash_table.ht_table[idx];
764 	while ((fbuf = *bufp) != buf) {
765 		ASSERT(fbuf != NULL);
766 		bufp = &fbuf->b_hash_next;
767 	}
768 	*bufp = buf->b_hash_next;
769 	buf->b_hash_next = NULL;
770 	buf->b_flags &= ~ARC_IN_HASH_TABLE;
771 
772 	/* collect some hash table performance data */
773 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
774 
775 	if (buf_hash_table.ht_table[idx] &&
776 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
777 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
778 }
779 
780 /*
781  * Global data structures and functions for the buf kmem cache.
782  */
783 static kmem_cache_t *hdr_cache;
784 static kmem_cache_t *buf_cache;
785 
786 static void
787 buf_fini(void)
788 {
789 	int i;
790 
791 	kmem_free(buf_hash_table.ht_table,
792 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
793 	for (i = 0; i < BUF_LOCKS; i++)
794 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
795 	kmem_cache_destroy(hdr_cache);
796 	kmem_cache_destroy(buf_cache);
797 }
798 
799 /*
800  * Constructor callback - called when the cache is empty
801  * and a new buf is requested.
802  */
803 /* ARGSUSED */
804 static int
805 hdr_cons(void *vbuf, void *unused, int kmflag)
806 {
807 	arc_buf_hdr_t *buf = vbuf;
808 
809 	bzero(buf, sizeof (arc_buf_hdr_t));
810 	refcount_create(&buf->b_refcnt);
811 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
812 	mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
813 	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
814 
815 	return (0);
816 }
817 
818 /* ARGSUSED */
819 static int
820 buf_cons(void *vbuf, void *unused, int kmflag)
821 {
822 	arc_buf_t *buf = vbuf;
823 
824 	bzero(buf, sizeof (arc_buf_t));
825 	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
826 	rw_init(&buf->b_data_lock, NULL, RW_DEFAULT, NULL);
827 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
828 
829 	return (0);
830 }
831 
832 /*
833  * Destructor callback - called when a cached buf is
834  * no longer required.
835  */
836 /* ARGSUSED */
837 static void
838 hdr_dest(void *vbuf, void *unused)
839 {
840 	arc_buf_hdr_t *buf = vbuf;
841 
842 	ASSERT(BUF_EMPTY(buf));
843 	refcount_destroy(&buf->b_refcnt);
844 	cv_destroy(&buf->b_cv);
845 	mutex_destroy(&buf->b_freeze_lock);
846 	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
847 }
848 
849 /* ARGSUSED */
850 static void
851 buf_dest(void *vbuf, void *unused)
852 {
853 	arc_buf_t *buf = vbuf;
854 
855 	mutex_destroy(&buf->b_evict_lock);
856 	rw_destroy(&buf->b_data_lock);
857 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
858 }
859 
860 /*
861  * Reclaim callback -- invoked when memory is low.
862  */
863 /* ARGSUSED */
864 static void
865 hdr_recl(void *unused)
866 {
867 	dprintf("hdr_recl called\n");
868 	/*
869 	 * umem calls the reclaim func when we destroy the buf cache,
870 	 * which is after we do arc_fini().
871 	 */
872 	if (!arc_dead)
873 		cv_signal(&arc_reclaim_thr_cv);
874 }
875 
876 static void
877 buf_init(void)
878 {
879 	uint64_t *ct;
880 	uint64_t hsize = 1ULL << 12;
881 	int i, j;
882 
883 	/*
884 	 * The hash table is big enough to fill all of physical memory
885 	 * with an average 64K block size.  The table will take up
886 	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
887 	 */
888 	while (hsize * 65536 < physmem * PAGESIZE)
889 		hsize <<= 1;
890 retry:
891 	buf_hash_table.ht_mask = hsize - 1;
892 	buf_hash_table.ht_table =
893 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
894 	if (buf_hash_table.ht_table == NULL) {
895 		ASSERT(hsize > (1ULL << 8));
896 		hsize >>= 1;
897 		goto retry;
898 	}
899 
900 	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
901 	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
902 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
903 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
904 
905 	for (i = 0; i < 256; i++)
906 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
907 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
908 
909 	for (i = 0; i < BUF_LOCKS; i++) {
910 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
911 		    NULL, MUTEX_DEFAULT, NULL);
912 	}
913 }
914 
915 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
916 
917 static void
918 arc_cksum_verify(arc_buf_t *buf)
919 {
920 	zio_cksum_t zc;
921 
922 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
923 		return;
924 
925 	mutex_enter(&buf->b_hdr->b_freeze_lock);
926 	if (buf->b_hdr->b_freeze_cksum == NULL ||
927 	    (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
928 		mutex_exit(&buf->b_hdr->b_freeze_lock);
929 		return;
930 	}
931 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
932 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
933 		panic("buffer modified while frozen!");
934 	mutex_exit(&buf->b_hdr->b_freeze_lock);
935 }
936 
937 static int
938 arc_cksum_equal(arc_buf_t *buf)
939 {
940 	zio_cksum_t zc;
941 	int equal;
942 
943 	mutex_enter(&buf->b_hdr->b_freeze_lock);
944 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
945 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
946 	mutex_exit(&buf->b_hdr->b_freeze_lock);
947 
948 	return (equal);
949 }
950 
951 static void
952 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
953 {
954 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
955 		return;
956 
957 	mutex_enter(&buf->b_hdr->b_freeze_lock);
958 	if (buf->b_hdr->b_freeze_cksum != NULL) {
959 		mutex_exit(&buf->b_hdr->b_freeze_lock);
960 		return;
961 	}
962 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
963 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
964 	    buf->b_hdr->b_freeze_cksum);
965 	mutex_exit(&buf->b_hdr->b_freeze_lock);
966 	arc_buf_watch(buf);
967 }
968 
969 #ifndef _KERNEL
970 typedef struct procctl {
971 	long cmd;
972 	prwatch_t prwatch;
973 } procctl_t;
974 #endif
975 
976 /* ARGSUSED */
977 static void
978 arc_buf_unwatch(arc_buf_t *buf)
979 {
980 #ifndef _KERNEL
981 	if (arc_watch) {
982 		int result;
983 		procctl_t ctl;
984 		ctl.cmd = PCWATCH;
985 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
986 		ctl.prwatch.pr_size = 0;
987 		ctl.prwatch.pr_wflags = 0;
988 		result = write(arc_procfd, &ctl, sizeof (ctl));
989 		ASSERT3U(result, ==, sizeof (ctl));
990 	}
991 #endif
992 }
993 
994 /* ARGSUSED */
995 static void
996 arc_buf_watch(arc_buf_t *buf)
997 {
998 #ifndef _KERNEL
999 	if (arc_watch) {
1000 		int result;
1001 		procctl_t ctl;
1002 		ctl.cmd = PCWATCH;
1003 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1004 		ctl.prwatch.pr_size = buf->b_hdr->b_size;
1005 		ctl.prwatch.pr_wflags = WA_WRITE;
1006 		result = write(arc_procfd, &ctl, sizeof (ctl));
1007 		ASSERT3U(result, ==, sizeof (ctl));
1008 	}
1009 #endif
1010 }
1011 
1012 void
1013 arc_buf_thaw(arc_buf_t *buf)
1014 {
1015 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1016 		if (buf->b_hdr->b_state != arc_anon)
1017 			panic("modifying non-anon buffer!");
1018 		if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1019 			panic("modifying buffer while i/o in progress!");
1020 		arc_cksum_verify(buf);
1021 	}
1022 
1023 	mutex_enter(&buf->b_hdr->b_freeze_lock);
1024 	if (buf->b_hdr->b_freeze_cksum != NULL) {
1025 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1026 		buf->b_hdr->b_freeze_cksum = NULL;
1027 	}
1028 
1029 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1030 		if (buf->b_hdr->b_thawed)
1031 			kmem_free(buf->b_hdr->b_thawed, 1);
1032 		buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1033 	}
1034 
1035 	mutex_exit(&buf->b_hdr->b_freeze_lock);
1036 
1037 	arc_buf_unwatch(buf);
1038 }
1039 
1040 void
1041 arc_buf_freeze(arc_buf_t *buf)
1042 {
1043 	kmutex_t *hash_lock;
1044 
1045 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1046 		return;
1047 
1048 	hash_lock = HDR_LOCK(buf->b_hdr);
1049 	mutex_enter(hash_lock);
1050 
1051 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1052 	    buf->b_hdr->b_state == arc_anon);
1053 	arc_cksum_compute(buf, B_FALSE);
1054 	mutex_exit(hash_lock);
1055 
1056 }
1057 
1058 static void
1059 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1060 {
1061 	ASSERT(MUTEX_HELD(hash_lock));
1062 
1063 	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
1064 	    (ab->b_state != arc_anon)) {
1065 		uint64_t delta = ab->b_size * ab->b_datacnt;
1066 		list_t *list = &ab->b_state->arcs_list[ab->b_type];
1067 		uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
1068 
1069 		ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
1070 		mutex_enter(&ab->b_state->arcs_mtx);
1071 		ASSERT(list_link_active(&ab->b_arc_node));
1072 		list_remove(list, ab);
1073 		if (GHOST_STATE(ab->b_state)) {
1074 			ASSERT0(ab->b_datacnt);
1075 			ASSERT3P(ab->b_buf, ==, NULL);
1076 			delta = ab->b_size;
1077 		}
1078 		ASSERT(delta > 0);
1079 		ASSERT3U(*size, >=, delta);
1080 		atomic_add_64(size, -delta);
1081 		mutex_exit(&ab->b_state->arcs_mtx);
1082 		/* remove the prefetch flag if we get a reference */
1083 		if (ab->b_flags & ARC_PREFETCH)
1084 			ab->b_flags &= ~ARC_PREFETCH;
1085 	}
1086 }
1087 
1088 static int
1089 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1090 {
1091 	int cnt;
1092 	arc_state_t *state = ab->b_state;
1093 
1094 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1095 	ASSERT(!GHOST_STATE(state));
1096 
1097 	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
1098 	    (state != arc_anon)) {
1099 		uint64_t *size = &state->arcs_lsize[ab->b_type];
1100 
1101 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
1102 		mutex_enter(&state->arcs_mtx);
1103 		ASSERT(!list_link_active(&ab->b_arc_node));
1104 		list_insert_head(&state->arcs_list[ab->b_type], ab);
1105 		ASSERT(ab->b_datacnt > 0);
1106 		atomic_add_64(size, ab->b_size * ab->b_datacnt);
1107 		mutex_exit(&state->arcs_mtx);
1108 	}
1109 	return (cnt);
1110 }
1111 
1112 /*
1113  * Move the supplied buffer to the indicated state.  The mutex
1114  * for the buffer must be held by the caller.
1115  */
1116 static void
1117 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1118 {
1119 	arc_state_t *old_state = ab->b_state;
1120 	int64_t refcnt = refcount_count(&ab->b_refcnt);
1121 	uint64_t from_delta, to_delta;
1122 
1123 	ASSERT(MUTEX_HELD(hash_lock));
1124 	ASSERT(new_state != old_state);
1125 	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1126 	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1127 	ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1128 
1129 	from_delta = to_delta = ab->b_datacnt * ab->b_size;
1130 
1131 	/*
1132 	 * If this buffer is evictable, transfer it from the
1133 	 * old state list to the new state list.
1134 	 */
1135 	if (refcnt == 0) {
1136 		if (old_state != arc_anon) {
1137 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
1138 			uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1139 
1140 			if (use_mutex)
1141 				mutex_enter(&old_state->arcs_mtx);
1142 
1143 			ASSERT(list_link_active(&ab->b_arc_node));
1144 			list_remove(&old_state->arcs_list[ab->b_type], ab);
1145 
1146 			/*
1147 			 * If prefetching out of the ghost cache,
1148 			 * we will have a non-zero datacnt.
1149 			 */
1150 			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
1151 				/* ghost elements have a ghost size */
1152 				ASSERT(ab->b_buf == NULL);
1153 				from_delta = ab->b_size;
1154 			}
1155 			ASSERT3U(*size, >=, from_delta);
1156 			atomic_add_64(size, -from_delta);
1157 
1158 			if (use_mutex)
1159 				mutex_exit(&old_state->arcs_mtx);
1160 		}
1161 		if (new_state != arc_anon) {
1162 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
1163 			uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1164 
1165 			if (use_mutex)
1166 				mutex_enter(&new_state->arcs_mtx);
1167 
1168 			list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1169 
1170 			/* ghost elements have a ghost size */
1171 			if (GHOST_STATE(new_state)) {
1172 				ASSERT(ab->b_datacnt == 0);
1173 				ASSERT(ab->b_buf == NULL);
1174 				to_delta = ab->b_size;
1175 			}
1176 			atomic_add_64(size, to_delta);
1177 
1178 			if (use_mutex)
1179 				mutex_exit(&new_state->arcs_mtx);
1180 		}
1181 	}
1182 
1183 	ASSERT(!BUF_EMPTY(ab));
1184 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
1185 		buf_hash_remove(ab);
1186 
1187 	/* adjust state sizes */
1188 	if (to_delta)
1189 		atomic_add_64(&new_state->arcs_size, to_delta);
1190 	if (from_delta) {
1191 		ASSERT3U(old_state->arcs_size, >=, from_delta);
1192 		atomic_add_64(&old_state->arcs_size, -from_delta);
1193 	}
1194 	ab->b_state = new_state;
1195 
1196 	/* adjust l2arc hdr stats */
1197 	if (new_state == arc_l2c_only)
1198 		l2arc_hdr_stat_add();
1199 	else if (old_state == arc_l2c_only)
1200 		l2arc_hdr_stat_remove();
1201 }
1202 
1203 void
1204 arc_space_consume(uint64_t space, arc_space_type_t type)
1205 {
1206 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1207 
1208 	switch (type) {
1209 	case ARC_SPACE_DATA:
1210 		ARCSTAT_INCR(arcstat_data_size, space);
1211 		break;
1212 	case ARC_SPACE_OTHER:
1213 		ARCSTAT_INCR(arcstat_other_size, space);
1214 		break;
1215 	case ARC_SPACE_HDRS:
1216 		ARCSTAT_INCR(arcstat_hdr_size, space);
1217 		break;
1218 	case ARC_SPACE_L2HDRS:
1219 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1220 		break;
1221 	}
1222 
1223 	atomic_add_64(&arc_meta_used, space);
1224 	atomic_add_64(&arc_size, space);
1225 }
1226 
1227 void
1228 arc_space_return(uint64_t space, arc_space_type_t type)
1229 {
1230 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1231 
1232 	switch (type) {
1233 	case ARC_SPACE_DATA:
1234 		ARCSTAT_INCR(arcstat_data_size, -space);
1235 		break;
1236 	case ARC_SPACE_OTHER:
1237 		ARCSTAT_INCR(arcstat_other_size, -space);
1238 		break;
1239 	case ARC_SPACE_HDRS:
1240 		ARCSTAT_INCR(arcstat_hdr_size, -space);
1241 		break;
1242 	case ARC_SPACE_L2HDRS:
1243 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1244 		break;
1245 	}
1246 
1247 	ASSERT(arc_meta_used >= space);
1248 	if (arc_meta_max < arc_meta_used)
1249 		arc_meta_max = arc_meta_used;
1250 	atomic_add_64(&arc_meta_used, -space);
1251 	ASSERT(arc_size >= space);
1252 	atomic_add_64(&arc_size, -space);
1253 }
1254 
1255 void *
1256 arc_data_buf_alloc(uint64_t size)
1257 {
1258 	if (arc_evict_needed(ARC_BUFC_DATA))
1259 		cv_signal(&arc_reclaim_thr_cv);
1260 	atomic_add_64(&arc_size, size);
1261 	return (zio_data_buf_alloc(size));
1262 }
1263 
1264 void
1265 arc_data_buf_free(void *buf, uint64_t size)
1266 {
1267 	zio_data_buf_free(buf, size);
1268 	ASSERT(arc_size >= size);
1269 	atomic_add_64(&arc_size, -size);
1270 }
1271 
1272 arc_buf_t *
1273 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1274 {
1275 	arc_buf_hdr_t *hdr;
1276 	arc_buf_t *buf;
1277 
1278 	ASSERT3U(size, >, 0);
1279 	hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1280 	ASSERT(BUF_EMPTY(hdr));
1281 	hdr->b_size = size;
1282 	hdr->b_type = type;
1283 	hdr->b_spa = spa_load_guid(spa);
1284 	hdr->b_state = arc_anon;
1285 	hdr->b_arc_access = 0;
1286 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1287 	buf->b_hdr = hdr;
1288 	buf->b_data = NULL;
1289 	buf->b_efunc = NULL;
1290 	buf->b_private = NULL;
1291 	buf->b_next = NULL;
1292 	hdr->b_buf = buf;
1293 	arc_get_data_buf(buf);
1294 	hdr->b_datacnt = 1;
1295 	hdr->b_flags = 0;
1296 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
1297 	(void) refcount_add(&hdr->b_refcnt, tag);
1298 
1299 	return (buf);
1300 }
1301 
1302 static char *arc_onloan_tag = "onloan";
1303 
1304 /*
1305  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1306  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1307  * buffers must be returned to the arc before they can be used by the DMU or
1308  * freed.
1309  */
1310 arc_buf_t *
1311 arc_loan_buf(spa_t *spa, int size)
1312 {
1313 	arc_buf_t *buf;
1314 
1315 	buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1316 
1317 	atomic_add_64(&arc_loaned_bytes, size);
1318 	return (buf);
1319 }
1320 
1321 /*
1322  * Return a loaned arc buffer to the arc.
1323  */
1324 void
1325 arc_return_buf(arc_buf_t *buf, void *tag)
1326 {
1327 	arc_buf_hdr_t *hdr = buf->b_hdr;
1328 
1329 	ASSERT(buf->b_data != NULL);
1330 	(void) refcount_add(&hdr->b_refcnt, tag);
1331 	(void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1332 
1333 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1334 }
1335 
1336 /* Detach an arc_buf from a dbuf (tag) */
1337 void
1338 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1339 {
1340 	arc_buf_hdr_t *hdr;
1341 
1342 	ASSERT(buf->b_data != NULL);
1343 	hdr = buf->b_hdr;
1344 	(void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1345 	(void) refcount_remove(&hdr->b_refcnt, tag);
1346 	buf->b_efunc = NULL;
1347 	buf->b_private = NULL;
1348 
1349 	atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1350 }
1351 
1352 static arc_buf_t *
1353 arc_buf_clone(arc_buf_t *from)
1354 {
1355 	arc_buf_t *buf;
1356 	arc_buf_hdr_t *hdr = from->b_hdr;
1357 	uint64_t size = hdr->b_size;
1358 
1359 	ASSERT(hdr->b_state != arc_anon);
1360 
1361 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1362 	buf->b_hdr = hdr;
1363 	buf->b_data = NULL;
1364 	buf->b_efunc = NULL;
1365 	buf->b_private = NULL;
1366 	buf->b_next = hdr->b_buf;
1367 	hdr->b_buf = buf;
1368 	arc_get_data_buf(buf);
1369 	bcopy(from->b_data, buf->b_data, size);
1370 
1371 	/*
1372 	 * This buffer already exists in the arc so create a duplicate
1373 	 * copy for the caller.  If the buffer is associated with user data
1374 	 * then track the size and number of duplicates.  These stats will be
1375 	 * updated as duplicate buffers are created and destroyed.
1376 	 */
1377 	if (hdr->b_type == ARC_BUFC_DATA) {
1378 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
1379 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1380 	}
1381 	hdr->b_datacnt += 1;
1382 	return (buf);
1383 }
1384 
1385 void
1386 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1387 {
1388 	arc_buf_hdr_t *hdr;
1389 	kmutex_t *hash_lock;
1390 
1391 	/*
1392 	 * Check to see if this buffer is evicted.  Callers
1393 	 * must verify b_data != NULL to know if the add_ref
1394 	 * was successful.
1395 	 */
1396 	mutex_enter(&buf->b_evict_lock);
1397 	if (buf->b_data == NULL) {
1398 		mutex_exit(&buf->b_evict_lock);
1399 		return;
1400 	}
1401 	hash_lock = HDR_LOCK(buf->b_hdr);
1402 	mutex_enter(hash_lock);
1403 	hdr = buf->b_hdr;
1404 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1405 	mutex_exit(&buf->b_evict_lock);
1406 
1407 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1408 	add_reference(hdr, hash_lock, tag);
1409 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1410 	arc_access(hdr, hash_lock);
1411 	mutex_exit(hash_lock);
1412 	ARCSTAT_BUMP(arcstat_hits);
1413 	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1414 	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1415 	    data, metadata, hits);
1416 }
1417 
1418 /*
1419  * Free the arc data buffer.  If it is an l2arc write in progress,
1420  * the buffer is placed on l2arc_free_on_write to be freed later.
1421  */
1422 static void
1423 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1424 {
1425 	arc_buf_hdr_t *hdr = buf->b_hdr;
1426 
1427 	if (HDR_L2_WRITING(hdr)) {
1428 		l2arc_data_free_t *df;
1429 		df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1430 		df->l2df_data = buf->b_data;
1431 		df->l2df_size = hdr->b_size;
1432 		df->l2df_func = free_func;
1433 		mutex_enter(&l2arc_free_on_write_mtx);
1434 		list_insert_head(l2arc_free_on_write, df);
1435 		mutex_exit(&l2arc_free_on_write_mtx);
1436 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
1437 	} else {
1438 		free_func(buf->b_data, hdr->b_size);
1439 	}
1440 }
1441 
1442 static void
1443 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1444 {
1445 	arc_buf_t **bufp;
1446 
1447 	/* free up data associated with the buf */
1448 	if (buf->b_data) {
1449 		arc_state_t *state = buf->b_hdr->b_state;
1450 		uint64_t size = buf->b_hdr->b_size;
1451 		arc_buf_contents_t type = buf->b_hdr->b_type;
1452 
1453 		arc_cksum_verify(buf);
1454 		arc_buf_unwatch(buf);
1455 
1456 		if (!recycle) {
1457 			if (type == ARC_BUFC_METADATA) {
1458 				arc_buf_data_free(buf, zio_buf_free);
1459 				arc_space_return(size, ARC_SPACE_DATA);
1460 			} else {
1461 				ASSERT(type == ARC_BUFC_DATA);
1462 				arc_buf_data_free(buf, zio_data_buf_free);
1463 				ARCSTAT_INCR(arcstat_data_size, -size);
1464 				atomic_add_64(&arc_size, -size);
1465 			}
1466 		}
1467 		if (list_link_active(&buf->b_hdr->b_arc_node)) {
1468 			uint64_t *cnt = &state->arcs_lsize[type];
1469 
1470 			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
1471 			ASSERT(state != arc_anon);
1472 
1473 			ASSERT3U(*cnt, >=, size);
1474 			atomic_add_64(cnt, -size);
1475 		}
1476 		ASSERT3U(state->arcs_size, >=, size);
1477 		atomic_add_64(&state->arcs_size, -size);
1478 		buf->b_data = NULL;
1479 
1480 		/*
1481 		 * If we're destroying a duplicate buffer make sure
1482 		 * that the appropriate statistics are updated.
1483 		 */
1484 		if (buf->b_hdr->b_datacnt > 1 &&
1485 		    buf->b_hdr->b_type == ARC_BUFC_DATA) {
1486 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
1487 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
1488 		}
1489 		ASSERT(buf->b_hdr->b_datacnt > 0);
1490 		buf->b_hdr->b_datacnt -= 1;
1491 	}
1492 
1493 	/* only remove the buf if requested */
1494 	if (!all)
1495 		return;
1496 
1497 	/* remove the buf from the hdr list */
1498 	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1499 		continue;
1500 	*bufp = buf->b_next;
1501 	buf->b_next = NULL;
1502 
1503 	ASSERT(buf->b_efunc == NULL);
1504 
1505 	/* clean up the buf */
1506 	buf->b_hdr = NULL;
1507 	kmem_cache_free(buf_cache, buf);
1508 }
1509 
1510 static void
1511 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1512 {
1513 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
1514 	ASSERT3P(hdr->b_state, ==, arc_anon);
1515 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1516 	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1517 
1518 	if (l2hdr != NULL) {
1519 		boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1520 		/*
1521 		 * To prevent arc_free() and l2arc_evict() from
1522 		 * attempting to free the same buffer at the same time,
1523 		 * a FREE_IN_PROGRESS flag is given to arc_free() to
1524 		 * give it priority.  l2arc_evict() can't destroy this
1525 		 * header while we are waiting on l2arc_buflist_mtx.
1526 		 *
1527 		 * The hdr may be removed from l2ad_buflist before we
1528 		 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1529 		 */
1530 		if (!buflist_held) {
1531 			mutex_enter(&l2arc_buflist_mtx);
1532 			l2hdr = hdr->b_l2hdr;
1533 		}
1534 
1535 		if (l2hdr != NULL) {
1536 			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1537 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1538 			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1539 			if (hdr->b_state == arc_l2c_only)
1540 				l2arc_hdr_stat_remove();
1541 			hdr->b_l2hdr = NULL;
1542 		}
1543 
1544 		if (!buflist_held)
1545 			mutex_exit(&l2arc_buflist_mtx);
1546 	}
1547 
1548 	if (!BUF_EMPTY(hdr)) {
1549 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
1550 		buf_discard_identity(hdr);
1551 	}
1552 	while (hdr->b_buf) {
1553 		arc_buf_t *buf = hdr->b_buf;
1554 
1555 		if (buf->b_efunc) {
1556 			mutex_enter(&arc_eviction_mtx);
1557 			mutex_enter(&buf->b_evict_lock);
1558 			ASSERT(buf->b_hdr != NULL);
1559 			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1560 			hdr->b_buf = buf->b_next;
1561 			buf->b_hdr = &arc_eviction_hdr;
1562 			buf->b_next = arc_eviction_list;
1563 			arc_eviction_list = buf;
1564 			mutex_exit(&buf->b_evict_lock);
1565 			mutex_exit(&arc_eviction_mtx);
1566 		} else {
1567 			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1568 		}
1569 	}
1570 	if (hdr->b_freeze_cksum != NULL) {
1571 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1572 		hdr->b_freeze_cksum = NULL;
1573 	}
1574 	if (hdr->b_thawed) {
1575 		kmem_free(hdr->b_thawed, 1);
1576 		hdr->b_thawed = NULL;
1577 	}
1578 
1579 	ASSERT(!list_link_active(&hdr->b_arc_node));
1580 	ASSERT3P(hdr->b_hash_next, ==, NULL);
1581 	ASSERT3P(hdr->b_acb, ==, NULL);
1582 	kmem_cache_free(hdr_cache, hdr);
1583 }
1584 
1585 void
1586 arc_buf_free(arc_buf_t *buf, void *tag)
1587 {
1588 	arc_buf_hdr_t *hdr = buf->b_hdr;
1589 	int hashed = hdr->b_state != arc_anon;
1590 
1591 	ASSERT(buf->b_efunc == NULL);
1592 	ASSERT(buf->b_data != NULL);
1593 
1594 	if (hashed) {
1595 		kmutex_t *hash_lock = HDR_LOCK(hdr);
1596 
1597 		mutex_enter(hash_lock);
1598 		hdr = buf->b_hdr;
1599 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1600 
1601 		(void) remove_reference(hdr, hash_lock, tag);
1602 		if (hdr->b_datacnt > 1) {
1603 			arc_buf_destroy(buf, FALSE, TRUE);
1604 		} else {
1605 			ASSERT(buf == hdr->b_buf);
1606 			ASSERT(buf->b_efunc == NULL);
1607 			hdr->b_flags |= ARC_BUF_AVAILABLE;
1608 		}
1609 		mutex_exit(hash_lock);
1610 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
1611 		int destroy_hdr;
1612 		/*
1613 		 * We are in the middle of an async write.  Don't destroy
1614 		 * this buffer unless the write completes before we finish
1615 		 * decrementing the reference count.
1616 		 */
1617 		mutex_enter(&arc_eviction_mtx);
1618 		(void) remove_reference(hdr, NULL, tag);
1619 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
1620 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1621 		mutex_exit(&arc_eviction_mtx);
1622 		if (destroy_hdr)
1623 			arc_hdr_destroy(hdr);
1624 	} else {
1625 		if (remove_reference(hdr, NULL, tag) > 0)
1626 			arc_buf_destroy(buf, FALSE, TRUE);
1627 		else
1628 			arc_hdr_destroy(hdr);
1629 	}
1630 }
1631 
1632 int
1633 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1634 {
1635 	arc_buf_hdr_t *hdr = buf->b_hdr;
1636 	kmutex_t *hash_lock = HDR_LOCK(hdr);
1637 	int no_callback = (buf->b_efunc == NULL);
1638 
1639 	if (hdr->b_state == arc_anon) {
1640 		ASSERT(hdr->b_datacnt == 1);
1641 		arc_buf_free(buf, tag);
1642 		return (no_callback);
1643 	}
1644 
1645 	mutex_enter(hash_lock);
1646 	hdr = buf->b_hdr;
1647 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1648 	ASSERT(hdr->b_state != arc_anon);
1649 	ASSERT(buf->b_data != NULL);
1650 
1651 	(void) remove_reference(hdr, hash_lock, tag);
1652 	if (hdr->b_datacnt > 1) {
1653 		if (no_callback)
1654 			arc_buf_destroy(buf, FALSE, TRUE);
1655 	} else if (no_callback) {
1656 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1657 		ASSERT(buf->b_efunc == NULL);
1658 		hdr->b_flags |= ARC_BUF_AVAILABLE;
1659 	}
1660 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
1661 	    refcount_is_zero(&hdr->b_refcnt));
1662 	mutex_exit(hash_lock);
1663 	return (no_callback);
1664 }
1665 
1666 int
1667 arc_buf_size(arc_buf_t *buf)
1668 {
1669 	return (buf->b_hdr->b_size);
1670 }
1671 
1672 /*
1673  * Called from the DMU to determine if the current buffer should be
1674  * evicted. In order to ensure proper locking, the eviction must be initiated
1675  * from the DMU. Return true if the buffer is associated with user data and
1676  * duplicate buffers still exist.
1677  */
1678 boolean_t
1679 arc_buf_eviction_needed(arc_buf_t *buf)
1680 {
1681 	arc_buf_hdr_t *hdr;
1682 	boolean_t evict_needed = B_FALSE;
1683 
1684 	if (zfs_disable_dup_eviction)
1685 		return (B_FALSE);
1686 
1687 	mutex_enter(&buf->b_evict_lock);
1688 	hdr = buf->b_hdr;
1689 	if (hdr == NULL) {
1690 		/*
1691 		 * We are in arc_do_user_evicts(); let that function
1692 		 * perform the eviction.
1693 		 */
1694 		ASSERT(buf->b_data == NULL);
1695 		mutex_exit(&buf->b_evict_lock);
1696 		return (B_FALSE);
1697 	} else if (buf->b_data == NULL) {
1698 		/*
1699 		 * We have already been added to the arc eviction list;
1700 		 * recommend eviction.
1701 		 */
1702 		ASSERT3P(hdr, ==, &arc_eviction_hdr);
1703 		mutex_exit(&buf->b_evict_lock);
1704 		return (B_TRUE);
1705 	}
1706 
1707 	if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
1708 		evict_needed = B_TRUE;
1709 
1710 	mutex_exit(&buf->b_evict_lock);
1711 	return (evict_needed);
1712 }
1713 
1714 /*
1715  * Evict buffers from list until we've removed the specified number of
1716  * bytes.  Move the removed buffers to the appropriate evict state.
1717  * If the recycle flag is set, then attempt to "recycle" a buffer:
1718  * - look for a buffer to evict that is `bytes' long.
1719  * - return the data block from this buffer rather than freeing it.
1720  * This flag is used by callers that are trying to make space for a
1721  * new buffer in a full arc cache.
1722  *
1723  * This function makes a "best effort".  It skips over any buffers
1724  * it can't get a hash_lock on, and so may not catch all candidates.
1725  * It may also return without evicting as much space as requested.
1726  */
1727 static void *
1728 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1729     arc_buf_contents_t type)
1730 {
1731 	arc_state_t *evicted_state;
1732 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1733 	arc_buf_hdr_t *ab, *ab_prev = NULL;
1734 	list_t *list = &state->arcs_list[type];
1735 	kmutex_t *hash_lock;
1736 	boolean_t have_lock;
1737 	void *stolen = NULL;
1738 
1739 	ASSERT(state == arc_mru || state == arc_mfu);
1740 
1741 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1742 
1743 	mutex_enter(&state->arcs_mtx);
1744 	mutex_enter(&evicted_state->arcs_mtx);
1745 
1746 	for (ab = list_tail(list); ab; ab = ab_prev) {
1747 		ab_prev = list_prev(list, ab);
1748 		/* prefetch buffers have a minimum lifespan */
1749 		if (HDR_IO_IN_PROGRESS(ab) ||
1750 		    (spa && ab->b_spa != spa) ||
1751 		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1752 		    ddi_get_lbolt() - ab->b_arc_access <
1753 		    arc_min_prefetch_lifespan)) {
1754 			skipped++;
1755 			continue;
1756 		}
1757 		/* "lookahead" for better eviction candidate */
1758 		if (recycle && ab->b_size != bytes &&
1759 		    ab_prev && ab_prev->b_size == bytes)
1760 			continue;
1761 		hash_lock = HDR_LOCK(ab);
1762 		have_lock = MUTEX_HELD(hash_lock);
1763 		if (have_lock || mutex_tryenter(hash_lock)) {
1764 			ASSERT0(refcount_count(&ab->b_refcnt));
1765 			ASSERT(ab->b_datacnt > 0);
1766 			while (ab->b_buf) {
1767 				arc_buf_t *buf = ab->b_buf;
1768 				if (!mutex_tryenter(&buf->b_evict_lock)) {
1769 					missed += 1;
1770 					break;
1771 				}
1772 				if (buf->b_data) {
1773 					bytes_evicted += ab->b_size;
1774 					if (recycle && ab->b_type == type &&
1775 					    ab->b_size == bytes &&
1776 					    !HDR_L2_WRITING(ab)) {
1777 						stolen = buf->b_data;
1778 						recycle = FALSE;
1779 					}
1780 				}
1781 				if (buf->b_efunc) {
1782 					mutex_enter(&arc_eviction_mtx);
1783 					arc_buf_destroy(buf,
1784 					    buf->b_data == stolen, FALSE);
1785 					ab->b_buf = buf->b_next;
1786 					buf->b_hdr = &arc_eviction_hdr;
1787 					buf->b_next = arc_eviction_list;
1788 					arc_eviction_list = buf;
1789 					mutex_exit(&arc_eviction_mtx);
1790 					mutex_exit(&buf->b_evict_lock);
1791 				} else {
1792 					mutex_exit(&buf->b_evict_lock);
1793 					arc_buf_destroy(buf,
1794 					    buf->b_data == stolen, TRUE);
1795 				}
1796 			}
1797 
1798 			if (ab->b_l2hdr) {
1799 				ARCSTAT_INCR(arcstat_evict_l2_cached,
1800 				    ab->b_size);
1801 			} else {
1802 				if (l2arc_write_eligible(ab->b_spa, ab)) {
1803 					ARCSTAT_INCR(arcstat_evict_l2_eligible,
1804 					    ab->b_size);
1805 				} else {
1806 					ARCSTAT_INCR(
1807 					    arcstat_evict_l2_ineligible,
1808 					    ab->b_size);
1809 				}
1810 			}
1811 
1812 			if (ab->b_datacnt == 0) {
1813 				arc_change_state(evicted_state, ab, hash_lock);
1814 				ASSERT(HDR_IN_HASH_TABLE(ab));
1815 				ab->b_flags |= ARC_IN_HASH_TABLE;
1816 				ab->b_flags &= ~ARC_BUF_AVAILABLE;
1817 				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
1818 			}
1819 			if (!have_lock)
1820 				mutex_exit(hash_lock);
1821 			if (bytes >= 0 && bytes_evicted >= bytes)
1822 				break;
1823 		} else {
1824 			missed += 1;
1825 		}
1826 	}
1827 
1828 	mutex_exit(&evicted_state->arcs_mtx);
1829 	mutex_exit(&state->arcs_mtx);
1830 
1831 	if (bytes_evicted < bytes)
1832 		dprintf("only evicted %lld bytes from %x",
1833 		    (longlong_t)bytes_evicted, state);
1834 
1835 	if (skipped)
1836 		ARCSTAT_INCR(arcstat_evict_skip, skipped);
1837 
1838 	if (missed)
1839 		ARCSTAT_INCR(arcstat_mutex_miss, missed);
1840 
1841 	/*
1842 	 * We have just evicted some date into the ghost state, make
1843 	 * sure we also adjust the ghost state size if necessary.
1844 	 */
1845 	if (arc_no_grow &&
1846 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1847 		int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1848 		    arc_mru_ghost->arcs_size - arc_c;
1849 
1850 		if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1851 			int64_t todelete =
1852 			    MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1853 			arc_evict_ghost(arc_mru_ghost, NULL, todelete);
1854 		} else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1855 			int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1856 			    arc_mru_ghost->arcs_size +
1857 			    arc_mfu_ghost->arcs_size - arc_c);
1858 			arc_evict_ghost(arc_mfu_ghost, NULL, todelete);
1859 		}
1860 	}
1861 
1862 	return (stolen);
1863 }
1864 
1865 /*
1866  * Remove buffers from list until we've removed the specified number of
1867  * bytes.  Destroy the buffers that are removed.
1868  */
1869 static void
1870 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1871 {
1872 	arc_buf_hdr_t *ab, *ab_prev;
1873 	arc_buf_hdr_t marker = { 0 };
1874 	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1875 	kmutex_t *hash_lock;
1876 	uint64_t bytes_deleted = 0;
1877 	uint64_t bufs_skipped = 0;
1878 
1879 	ASSERT(GHOST_STATE(state));
1880 top:
1881 	mutex_enter(&state->arcs_mtx);
1882 	for (ab = list_tail(list); ab; ab = ab_prev) {
1883 		ab_prev = list_prev(list, ab);
1884 		if (spa && ab->b_spa != spa)
1885 			continue;
1886 
1887 		/* ignore markers */
1888 		if (ab->b_spa == 0)
1889 			continue;
1890 
1891 		hash_lock = HDR_LOCK(ab);
1892 		/* caller may be trying to modify this buffer, skip it */
1893 		if (MUTEX_HELD(hash_lock))
1894 			continue;
1895 		if (mutex_tryenter(hash_lock)) {
1896 			ASSERT(!HDR_IO_IN_PROGRESS(ab));
1897 			ASSERT(ab->b_buf == NULL);
1898 			ARCSTAT_BUMP(arcstat_deleted);
1899 			bytes_deleted += ab->b_size;
1900 
1901 			if (ab->b_l2hdr != NULL) {
1902 				/*
1903 				 * This buffer is cached on the 2nd Level ARC;
1904 				 * don't destroy the header.
1905 				 */
1906 				arc_change_state(arc_l2c_only, ab, hash_lock);
1907 				mutex_exit(hash_lock);
1908 			} else {
1909 				arc_change_state(arc_anon, ab, hash_lock);
1910 				mutex_exit(hash_lock);
1911 				arc_hdr_destroy(ab);
1912 			}
1913 
1914 			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1915 			if (bytes >= 0 && bytes_deleted >= bytes)
1916 				break;
1917 		} else if (bytes < 0) {
1918 			/*
1919 			 * Insert a list marker and then wait for the
1920 			 * hash lock to become available. Once its
1921 			 * available, restart from where we left off.
1922 			 */
1923 			list_insert_after(list, ab, &marker);
1924 			mutex_exit(&state->arcs_mtx);
1925 			mutex_enter(hash_lock);
1926 			mutex_exit(hash_lock);
1927 			mutex_enter(&state->arcs_mtx);
1928 			ab_prev = list_prev(list, &marker);
1929 			list_remove(list, &marker);
1930 		} else
1931 			bufs_skipped += 1;
1932 	}
1933 	mutex_exit(&state->arcs_mtx);
1934 
1935 	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
1936 	    (bytes < 0 || bytes_deleted < bytes)) {
1937 		list = &state->arcs_list[ARC_BUFC_METADATA];
1938 		goto top;
1939 	}
1940 
1941 	if (bufs_skipped) {
1942 		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
1943 		ASSERT(bytes >= 0);
1944 	}
1945 
1946 	if (bytes_deleted < bytes)
1947 		dprintf("only deleted %lld bytes from %p",
1948 		    (longlong_t)bytes_deleted, state);
1949 }
1950 
1951 static void
1952 arc_adjust(void)
1953 {
1954 	int64_t adjustment, delta;
1955 
1956 	/*
1957 	 * Adjust MRU size
1958 	 */
1959 
1960 	adjustment = MIN((int64_t)(arc_size - arc_c),
1961 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
1962 	    arc_p));
1963 
1964 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
1965 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
1966 		(void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
1967 		adjustment -= delta;
1968 	}
1969 
1970 	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1971 		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
1972 		(void) arc_evict(arc_mru, NULL, delta, FALSE,
1973 		    ARC_BUFC_METADATA);
1974 	}
1975 
1976 	/*
1977 	 * Adjust MFU size
1978 	 */
1979 
1980 	adjustment = arc_size - arc_c;
1981 
1982 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
1983 		delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
1984 		(void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
1985 		adjustment -= delta;
1986 	}
1987 
1988 	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1989 		int64_t delta = MIN(adjustment,
1990 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
1991 		(void) arc_evict(arc_mfu, NULL, delta, FALSE,
1992 		    ARC_BUFC_METADATA);
1993 	}
1994 
1995 	/*
1996 	 * Adjust ghost lists
1997 	 */
1998 
1999 	adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2000 
2001 	if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
2002 		delta = MIN(arc_mru_ghost->arcs_size, adjustment);
2003 		arc_evict_ghost(arc_mru_ghost, NULL, delta);
2004 	}
2005 
2006 	adjustment =
2007 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2008 
2009 	if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2010 		delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2011 		arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2012 	}
2013 }
2014 
2015 static void
2016 arc_do_user_evicts(void)
2017 {
2018 	mutex_enter(&arc_eviction_mtx);
2019 	while (arc_eviction_list != NULL) {
2020 		arc_buf_t *buf = arc_eviction_list;
2021 		arc_eviction_list = buf->b_next;
2022 		mutex_enter(&buf->b_evict_lock);
2023 		buf->b_hdr = NULL;
2024 		mutex_exit(&buf->b_evict_lock);
2025 		mutex_exit(&arc_eviction_mtx);
2026 
2027 		if (buf->b_efunc != NULL)
2028 			VERIFY(buf->b_efunc(buf) == 0);
2029 
2030 		buf->b_efunc = NULL;
2031 		buf->b_private = NULL;
2032 		kmem_cache_free(buf_cache, buf);
2033 		mutex_enter(&arc_eviction_mtx);
2034 	}
2035 	mutex_exit(&arc_eviction_mtx);
2036 }
2037 
2038 /*
2039  * Flush all *evictable* data from the cache for the given spa.
2040  * NOTE: this will not touch "active" (i.e. referenced) data.
2041  */
2042 void
2043 arc_flush(spa_t *spa)
2044 {
2045 	uint64_t guid = 0;
2046 
2047 	if (spa)
2048 		guid = spa_load_guid(spa);
2049 
2050 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
2051 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2052 		if (spa)
2053 			break;
2054 	}
2055 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
2056 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2057 		if (spa)
2058 			break;
2059 	}
2060 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
2061 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2062 		if (spa)
2063 			break;
2064 	}
2065 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
2066 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2067 		if (spa)
2068 			break;
2069 	}
2070 
2071 	arc_evict_ghost(arc_mru_ghost, guid, -1);
2072 	arc_evict_ghost(arc_mfu_ghost, guid, -1);
2073 
2074 	mutex_enter(&arc_reclaim_thr_lock);
2075 	arc_do_user_evicts();
2076 	mutex_exit(&arc_reclaim_thr_lock);
2077 	ASSERT(spa || arc_eviction_list == NULL);
2078 }
2079 
2080 void
2081 arc_shrink(void)
2082 {
2083 	if (arc_c > arc_c_min) {
2084 		uint64_t to_free;
2085 
2086 #ifdef _KERNEL
2087 		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
2088 #else
2089 		to_free = arc_c >> arc_shrink_shift;
2090 #endif
2091 		if (arc_c > arc_c_min + to_free)
2092 			atomic_add_64(&arc_c, -to_free);
2093 		else
2094 			arc_c = arc_c_min;
2095 
2096 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
2097 		if (arc_c > arc_size)
2098 			arc_c = MAX(arc_size, arc_c_min);
2099 		if (arc_p > arc_c)
2100 			arc_p = (arc_c >> 1);
2101 		ASSERT(arc_c >= arc_c_min);
2102 		ASSERT((int64_t)arc_p >= 0);
2103 	}
2104 
2105 	if (arc_size > arc_c)
2106 		arc_adjust();
2107 }
2108 
2109 /*
2110  * Determine if the system is under memory pressure and is asking
2111  * to reclaim memory. A return value of 1 indicates that the system
2112  * is under memory pressure and that the arc should adjust accordingly.
2113  */
2114 static int
2115 arc_reclaim_needed(void)
2116 {
2117 	uint64_t extra;
2118 
2119 #ifdef _KERNEL
2120 
2121 	if (needfree)
2122 		return (1);
2123 
2124 	/*
2125 	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
2126 	 */
2127 	extra = desfree;
2128 
2129 	/*
2130 	 * check that we're out of range of the pageout scanner.  It starts to
2131 	 * schedule paging if freemem is less than lotsfree and needfree.
2132 	 * lotsfree is the high-water mark for pageout, and needfree is the
2133 	 * number of needed free pages.  We add extra pages here to make sure
2134 	 * the scanner doesn't start up while we're freeing memory.
2135 	 */
2136 	if (freemem < lotsfree + needfree + extra)
2137 		return (1);
2138 
2139 	/*
2140 	 * check to make sure that swapfs has enough space so that anon
2141 	 * reservations can still succeed. anon_resvmem() checks that the
2142 	 * availrmem is greater than swapfs_minfree, and the number of reserved
2143 	 * swap pages.  We also add a bit of extra here just to prevent
2144 	 * circumstances from getting really dire.
2145 	 */
2146 	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2147 		return (1);
2148 
2149 #if defined(__i386)
2150 	/*
2151 	 * If we're on an i386 platform, it's possible that we'll exhaust the
2152 	 * kernel heap space before we ever run out of available physical
2153 	 * memory.  Most checks of the size of the heap_area compare against
2154 	 * tune.t_minarmem, which is the minimum available real memory that we
2155 	 * can have in the system.  However, this is generally fixed at 25 pages
2156 	 * which is so low that it's useless.  In this comparison, we seek to
2157 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
2158 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
2159 	 * free)
2160 	 */
2161 	if (vmem_size(heap_arena, VMEM_FREE) <
2162 	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
2163 		return (1);
2164 #endif
2165 
2166 	/*
2167 	 * If zio data pages are being allocated out of a separate heap segment,
2168 	 * then enforce that the size of available vmem for this arena remains
2169 	 * above about 1/16th free.
2170 	 *
2171 	 * Note: The 1/16th arena free requirement was put in place
2172 	 * to aggressively evict memory from the arc in order to avoid
2173 	 * memory fragmentation issues.
2174 	 */
2175 	if (zio_arena != NULL &&
2176 	    vmem_size(zio_arena, VMEM_FREE) <
2177 	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
2178 		return (1);
2179 #else
2180 	if (spa_get_random(100) == 0)
2181 		return (1);
2182 #endif
2183 	return (0);
2184 }
2185 
2186 static void
2187 arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2188 {
2189 	size_t			i;
2190 	kmem_cache_t		*prev_cache = NULL;
2191 	kmem_cache_t		*prev_data_cache = NULL;
2192 	extern kmem_cache_t	*zio_buf_cache[];
2193 	extern kmem_cache_t	*zio_data_buf_cache[];
2194 
2195 #ifdef _KERNEL
2196 	if (arc_meta_used >= arc_meta_limit) {
2197 		/*
2198 		 * We are exceeding our meta-data cache limit.
2199 		 * Purge some DNLC entries to release holds on meta-data.
2200 		 */
2201 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
2202 	}
2203 #if defined(__i386)
2204 	/*
2205 	 * Reclaim unused memory from all kmem caches.
2206 	 */
2207 	kmem_reap();
2208 #endif
2209 #endif
2210 
2211 	/*
2212 	 * An aggressive reclamation will shrink the cache size as well as
2213 	 * reap free buffers from the arc kmem caches.
2214 	 */
2215 	if (strat == ARC_RECLAIM_AGGR)
2216 		arc_shrink();
2217 
2218 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2219 		if (zio_buf_cache[i] != prev_cache) {
2220 			prev_cache = zio_buf_cache[i];
2221 			kmem_cache_reap_now(zio_buf_cache[i]);
2222 		}
2223 		if (zio_data_buf_cache[i] != prev_data_cache) {
2224 			prev_data_cache = zio_data_buf_cache[i];
2225 			kmem_cache_reap_now(zio_data_buf_cache[i]);
2226 		}
2227 	}
2228 	kmem_cache_reap_now(buf_cache);
2229 	kmem_cache_reap_now(hdr_cache);
2230 
2231 	/*
2232 	 * Ask the vmem areana to reclaim unused memory from its
2233 	 * quantum caches.
2234 	 */
2235 	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
2236 		vmem_qcache_reap(zio_arena);
2237 }
2238 
2239 static void
2240 arc_reclaim_thread(void)
2241 {
2242 	clock_t			growtime = 0;
2243 	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
2244 	callb_cpr_t		cpr;
2245 
2246 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2247 
2248 	mutex_enter(&arc_reclaim_thr_lock);
2249 	while (arc_thread_exit == 0) {
2250 		if (arc_reclaim_needed()) {
2251 
2252 			if (arc_no_grow) {
2253 				if (last_reclaim == ARC_RECLAIM_CONS) {
2254 					last_reclaim = ARC_RECLAIM_AGGR;
2255 				} else {
2256 					last_reclaim = ARC_RECLAIM_CONS;
2257 				}
2258 			} else {
2259 				arc_no_grow = TRUE;
2260 				last_reclaim = ARC_RECLAIM_AGGR;
2261 				membar_producer();
2262 			}
2263 
2264 			/* reset the growth delay for every reclaim */
2265 			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2266 
2267 			arc_kmem_reap_now(last_reclaim);
2268 			arc_warm = B_TRUE;
2269 
2270 		} else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
2271 			arc_no_grow = FALSE;
2272 		}
2273 
2274 		arc_adjust();
2275 
2276 		if (arc_eviction_list != NULL)
2277 			arc_do_user_evicts();
2278 
2279 		/* block until needed, or one second, whichever is shorter */
2280 		CALLB_CPR_SAFE_BEGIN(&cpr);
2281 		(void) cv_timedwait(&arc_reclaim_thr_cv,
2282 		    &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2283 		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2284 	}
2285 
2286 	arc_thread_exit = 0;
2287 	cv_broadcast(&arc_reclaim_thr_cv);
2288 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
2289 	thread_exit();
2290 }
2291 
2292 /*
2293  * Adapt arc info given the number of bytes we are trying to add and
2294  * the state that we are comming from.  This function is only called
2295  * when we are adding new content to the cache.
2296  */
2297 static void
2298 arc_adapt(int bytes, arc_state_t *state)
2299 {
2300 	int mult;
2301 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2302 
2303 	if (state == arc_l2c_only)
2304 		return;
2305 
2306 	ASSERT(bytes > 0);
2307 	/*
2308 	 * Adapt the target size of the MRU list:
2309 	 *	- if we just hit in the MRU ghost list, then increase
2310 	 *	  the target size of the MRU list.
2311 	 *	- if we just hit in the MFU ghost list, then increase
2312 	 *	  the target size of the MFU list by decreasing the
2313 	 *	  target size of the MRU list.
2314 	 */
2315 	if (state == arc_mru_ghost) {
2316 		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
2317 		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
2318 		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
2319 
2320 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
2321 	} else if (state == arc_mfu_ghost) {
2322 		uint64_t delta;
2323 
2324 		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
2325 		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
2326 		mult = MIN(mult, 10);
2327 
2328 		delta = MIN(bytes * mult, arc_p);
2329 		arc_p = MAX(arc_p_min, arc_p - delta);
2330 	}
2331 	ASSERT((int64_t)arc_p >= 0);
2332 
2333 	if (arc_reclaim_needed()) {
2334 		cv_signal(&arc_reclaim_thr_cv);
2335 		return;
2336 	}
2337 
2338 	if (arc_no_grow)
2339 		return;
2340 
2341 	if (arc_c >= arc_c_max)
2342 		return;
2343 
2344 	/*
2345 	 * If we're within (2 * maxblocksize) bytes of the target
2346 	 * cache size, increment the target cache size
2347 	 */
2348 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
2349 		atomic_add_64(&arc_c, (int64_t)bytes);
2350 		if (arc_c > arc_c_max)
2351 			arc_c = arc_c_max;
2352 		else if (state == arc_anon)
2353 			atomic_add_64(&arc_p, (int64_t)bytes);
2354 		if (arc_p > arc_c)
2355 			arc_p = arc_c;
2356 	}
2357 	ASSERT((int64_t)arc_p >= 0);
2358 }
2359 
2360 /*
2361  * Check if the cache has reached its limits and eviction is required
2362  * prior to insert.
2363  */
2364 static int
2365 arc_evict_needed(arc_buf_contents_t type)
2366 {
2367 	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2368 		return (1);
2369 
2370 	if (arc_reclaim_needed())
2371 		return (1);
2372 
2373 	return (arc_size > arc_c);
2374 }
2375 
2376 /*
2377  * The buffer, supplied as the first argument, needs a data block.
2378  * So, if we are at cache max, determine which cache should be victimized.
2379  * We have the following cases:
2380  *
2381  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2382  * In this situation if we're out of space, but the resident size of the MFU is
2383  * under the limit, victimize the MFU cache to satisfy this insertion request.
2384  *
2385  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2386  * Here, we've used up all of the available space for the MRU, so we need to
2387  * evict from our own cache instead.  Evict from the set of resident MRU
2388  * entries.
2389  *
2390  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2391  * c minus p represents the MFU space in the cache, since p is the size of the
2392  * cache that is dedicated to the MRU.  In this situation there's still space on
2393  * the MFU side, so the MRU side needs to be victimized.
2394  *
2395  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2396  * MFU's resident set is consuming more space than it has been allotted.  In
2397  * this situation, we must victimize our own cache, the MFU, for this insertion.
2398  */
2399 static void
2400 arc_get_data_buf(arc_buf_t *buf)
2401 {
2402 	arc_state_t		*state = buf->b_hdr->b_state;
2403 	uint64_t		size = buf->b_hdr->b_size;
2404 	arc_buf_contents_t	type = buf->b_hdr->b_type;
2405 
2406 	arc_adapt(size, state);
2407 
2408 	/*
2409 	 * We have not yet reached cache maximum size,
2410 	 * just allocate a new buffer.
2411 	 */
2412 	if (!arc_evict_needed(type)) {
2413 		if (type == ARC_BUFC_METADATA) {
2414 			buf->b_data = zio_buf_alloc(size);
2415 			arc_space_consume(size, ARC_SPACE_DATA);
2416 		} else {
2417 			ASSERT(type == ARC_BUFC_DATA);
2418 			buf->b_data = zio_data_buf_alloc(size);
2419 			ARCSTAT_INCR(arcstat_data_size, size);
2420 			atomic_add_64(&arc_size, size);
2421 		}
2422 		goto out;
2423 	}
2424 
2425 	/*
2426 	 * If we are prefetching from the mfu ghost list, this buffer
2427 	 * will end up on the mru list; so steal space from there.
2428 	 */
2429 	if (state == arc_mfu_ghost)
2430 		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
2431 	else if (state == arc_mru_ghost)
2432 		state = arc_mru;
2433 
2434 	if (state == arc_mru || state == arc_anon) {
2435 		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
2436 		state = (arc_mfu->arcs_lsize[type] >= size &&
2437 		    arc_p > mru_used) ? arc_mfu : arc_mru;
2438 	} else {
2439 		/* MFU cases */
2440 		uint64_t mfu_space = arc_c - arc_p;
2441 		state =  (arc_mru->arcs_lsize[type] >= size &&
2442 		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2443 	}
2444 	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2445 		if (type == ARC_BUFC_METADATA) {
2446 			buf->b_data = zio_buf_alloc(size);
2447 			arc_space_consume(size, ARC_SPACE_DATA);
2448 		} else {
2449 			ASSERT(type == ARC_BUFC_DATA);
2450 			buf->b_data = zio_data_buf_alloc(size);
2451 			ARCSTAT_INCR(arcstat_data_size, size);
2452 			atomic_add_64(&arc_size, size);
2453 		}
2454 		ARCSTAT_BUMP(arcstat_recycle_miss);
2455 	}
2456 	ASSERT(buf->b_data != NULL);
2457 out:
2458 	/*
2459 	 * Update the state size.  Note that ghost states have a
2460 	 * "ghost size" and so don't need to be updated.
2461 	 */
2462 	if (!GHOST_STATE(buf->b_hdr->b_state)) {
2463 		arc_buf_hdr_t *hdr = buf->b_hdr;
2464 
2465 		atomic_add_64(&hdr->b_state->arcs_size, size);
2466 		if (list_link_active(&hdr->b_arc_node)) {
2467 			ASSERT(refcount_is_zero(&hdr->b_refcnt));
2468 			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2469 		}
2470 		/*
2471 		 * If we are growing the cache, and we are adding anonymous
2472 		 * data, and we have outgrown arc_p, update arc_p
2473 		 */
2474 		if (arc_size < arc_c && hdr->b_state == arc_anon &&
2475 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
2476 			arc_p = MIN(arc_c, arc_p + size);
2477 	}
2478 }
2479 
2480 /*
2481  * This routine is called whenever a buffer is accessed.
2482  * NOTE: the hash lock is dropped in this function.
2483  */
2484 static void
2485 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2486 {
2487 	clock_t now;
2488 
2489 	ASSERT(MUTEX_HELD(hash_lock));
2490 
2491 	if (buf->b_state == arc_anon) {
2492 		/*
2493 		 * This buffer is not in the cache, and does not
2494 		 * appear in our "ghost" list.  Add the new buffer
2495 		 * to the MRU state.
2496 		 */
2497 
2498 		ASSERT(buf->b_arc_access == 0);
2499 		buf->b_arc_access = ddi_get_lbolt();
2500 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2501 		arc_change_state(arc_mru, buf, hash_lock);
2502 
2503 	} else if (buf->b_state == arc_mru) {
2504 		now = ddi_get_lbolt();
2505 
2506 		/*
2507 		 * If this buffer is here because of a prefetch, then either:
2508 		 * - clear the flag if this is a "referencing" read
2509 		 *   (any subsequent access will bump this into the MFU state).
2510 		 * or
2511 		 * - move the buffer to the head of the list if this is
2512 		 *   another prefetch (to make it less likely to be evicted).
2513 		 */
2514 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
2515 			if (refcount_count(&buf->b_refcnt) == 0) {
2516 				ASSERT(list_link_active(&buf->b_arc_node));
2517 			} else {
2518 				buf->b_flags &= ~ARC_PREFETCH;
2519 				ARCSTAT_BUMP(arcstat_mru_hits);
2520 			}
2521 			buf->b_arc_access = now;
2522 			return;
2523 		}
2524 
2525 		/*
2526 		 * This buffer has been "accessed" only once so far,
2527 		 * but it is still in the cache. Move it to the MFU
2528 		 * state.
2529 		 */
2530 		if (now > buf->b_arc_access + ARC_MINTIME) {
2531 			/*
2532 			 * More than 125ms have passed since we
2533 			 * instantiated this buffer.  Move it to the
2534 			 * most frequently used state.
2535 			 */
2536 			buf->b_arc_access = now;
2537 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2538 			arc_change_state(arc_mfu, buf, hash_lock);
2539 		}
2540 		ARCSTAT_BUMP(arcstat_mru_hits);
2541 	} else if (buf->b_state == arc_mru_ghost) {
2542 		arc_state_t	*new_state;
2543 		/*
2544 		 * This buffer has been "accessed" recently, but
2545 		 * was evicted from the cache.  Move it to the
2546 		 * MFU state.
2547 		 */
2548 
2549 		if (buf->b_flags & ARC_PREFETCH) {
2550 			new_state = arc_mru;
2551 			if (refcount_count(&buf->b_refcnt) > 0)
2552 				buf->b_flags &= ~ARC_PREFETCH;
2553 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2554 		} else {
2555 			new_state = arc_mfu;
2556 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2557 		}
2558 
2559 		buf->b_arc_access = ddi_get_lbolt();
2560 		arc_change_state(new_state, buf, hash_lock);
2561 
2562 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
2563 	} else if (buf->b_state == arc_mfu) {
2564 		/*
2565 		 * This buffer has been accessed more than once and is
2566 		 * still in the cache.  Keep it in the MFU state.
2567 		 *
2568 		 * NOTE: an add_reference() that occurred when we did
2569 		 * the arc_read() will have kicked this off the list.
2570 		 * If it was a prefetch, we will explicitly move it to
2571 		 * the head of the list now.
2572 		 */
2573 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
2574 			ASSERT(refcount_count(&buf->b_refcnt) == 0);
2575 			ASSERT(list_link_active(&buf->b_arc_node));
2576 		}
2577 		ARCSTAT_BUMP(arcstat_mfu_hits);
2578 		buf->b_arc_access = ddi_get_lbolt();
2579 	} else if (buf->b_state == arc_mfu_ghost) {
2580 		arc_state_t	*new_state = arc_mfu;
2581 		/*
2582 		 * This buffer has been accessed more than once but has
2583 		 * been evicted from the cache.  Move it back to the
2584 		 * MFU state.
2585 		 */
2586 
2587 		if (buf->b_flags & ARC_PREFETCH) {
2588 			/*
2589 			 * This is a prefetch access...
2590 			 * move this block back to the MRU state.
2591 			 */
2592 			ASSERT0(refcount_count(&buf->b_refcnt));
2593 			new_state = arc_mru;
2594 		}
2595 
2596 		buf->b_arc_access = ddi_get_lbolt();
2597 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2598 		arc_change_state(new_state, buf, hash_lock);
2599 
2600 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2601 	} else if (buf->b_state == arc_l2c_only) {
2602 		/*
2603 		 * This buffer is on the 2nd Level ARC.
2604 		 */
2605 
2606 		buf->b_arc_access = ddi_get_lbolt();
2607 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2608 		arc_change_state(arc_mfu, buf, hash_lock);
2609 	} else {
2610 		ASSERT(!"invalid arc state");
2611 	}
2612 }
2613 
2614 /* a generic arc_done_func_t which you can use */
2615 /* ARGSUSED */
2616 void
2617 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2618 {
2619 	if (zio == NULL || zio->io_error == 0)
2620 		bcopy(buf->b_data, arg, buf->b_hdr->b_size);
2621 	VERIFY(arc_buf_remove_ref(buf, arg) == 1);
2622 }
2623 
2624 /* a generic arc_done_func_t */
2625 void
2626 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2627 {
2628 	arc_buf_t **bufp = arg;
2629 	if (zio && zio->io_error) {
2630 		VERIFY(arc_buf_remove_ref(buf, arg) == 1);
2631 		*bufp = NULL;
2632 	} else {
2633 		*bufp = buf;
2634 		ASSERT(buf->b_data);
2635 	}
2636 }
2637 
2638 static void
2639 arc_read_done(zio_t *zio)
2640 {
2641 	arc_buf_hdr_t	*hdr, *found;
2642 	arc_buf_t	*buf;
2643 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
2644 	kmutex_t	*hash_lock;
2645 	arc_callback_t	*callback_list, *acb;
2646 	int		freeable = FALSE;
2647 
2648 	buf = zio->io_private;
2649 	hdr = buf->b_hdr;
2650 
2651 	/*
2652 	 * The hdr was inserted into hash-table and removed from lists
2653 	 * prior to starting I/O.  We should find this header, since
2654 	 * it's in the hash table, and it should be legit since it's
2655 	 * not possible to evict it during the I/O.  The only possible
2656 	 * reason for it not to be found is if we were freed during the
2657 	 * read.
2658 	 */
2659 	found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2660 	    &hash_lock);
2661 
2662 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2663 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2664 	    (found == hdr && HDR_L2_READING(hdr)));
2665 
2666 	hdr->b_flags &= ~ARC_L2_EVICTED;
2667 	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2668 		hdr->b_flags &= ~ARC_L2CACHE;
2669 
2670 	/* byteswap if necessary */
2671 	callback_list = hdr->b_acb;
2672 	ASSERT(callback_list != NULL);
2673 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2674 		dmu_object_byteswap_t bswap =
2675 		    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2676 		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2677 		    byteswap_uint64_array :
2678 		    dmu_ot_byteswap[bswap].ob_func;
2679 		func(buf->b_data, hdr->b_size);
2680 	}
2681 
2682 	arc_cksum_compute(buf, B_FALSE);
2683 	arc_buf_watch(buf);
2684 
2685 	if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2686 		/*
2687 		 * Only call arc_access on anonymous buffers.  This is because
2688 		 * if we've issued an I/O for an evicted buffer, we've already
2689 		 * called arc_access (to prevent any simultaneous readers from
2690 		 * getting confused).
2691 		 */
2692 		arc_access(hdr, hash_lock);
2693 	}
2694 
2695 	/* create copies of the data buffer for the callers */
2696 	abuf = buf;
2697 	for (acb = callback_list; acb; acb = acb->acb_next) {
2698 		if (acb->acb_done) {
2699 			if (abuf == NULL) {
2700 				ARCSTAT_BUMP(arcstat_duplicate_reads);
2701 				abuf = arc_buf_clone(buf);
2702 			}
2703 			acb->acb_buf = abuf;
2704 			abuf = NULL;
2705 		}
2706 	}
2707 	hdr->b_acb = NULL;
2708 	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2709 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
2710 	if (abuf == buf) {
2711 		ASSERT(buf->b_efunc == NULL);
2712 		ASSERT(hdr->b_datacnt == 1);
2713 		hdr->b_flags |= ARC_BUF_AVAILABLE;
2714 	}
2715 
2716 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2717 
2718 	if (zio->io_error != 0) {
2719 		hdr->b_flags |= ARC_IO_ERROR;
2720 		if (hdr->b_state != arc_anon)
2721 			arc_change_state(arc_anon, hdr, hash_lock);
2722 		if (HDR_IN_HASH_TABLE(hdr))
2723 			buf_hash_remove(hdr);
2724 		freeable = refcount_is_zero(&hdr->b_refcnt);
2725 	}
2726 
2727 	/*
2728 	 * Broadcast before we drop the hash_lock to avoid the possibility
2729 	 * that the hdr (and hence the cv) might be freed before we get to
2730 	 * the cv_broadcast().
2731 	 */
2732 	cv_broadcast(&hdr->b_cv);
2733 
2734 	if (hash_lock) {
2735 		mutex_exit(hash_lock);
2736 	} else {
2737 		/*
2738 		 * This block was freed while we waited for the read to
2739 		 * complete.  It has been removed from the hash table and
2740 		 * moved to the anonymous state (so that it won't show up
2741 		 * in the cache).
2742 		 */
2743 		ASSERT3P(hdr->b_state, ==, arc_anon);
2744 		freeable = refcount_is_zero(&hdr->b_refcnt);
2745 	}
2746 
2747 	/* execute each callback and free its structure */
2748 	while ((acb = callback_list) != NULL) {
2749 		if (acb->acb_done)
2750 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2751 
2752 		if (acb->acb_zio_dummy != NULL) {
2753 			acb->acb_zio_dummy->io_error = zio->io_error;
2754 			zio_nowait(acb->acb_zio_dummy);
2755 		}
2756 
2757 		callback_list = acb->acb_next;
2758 		kmem_free(acb, sizeof (arc_callback_t));
2759 	}
2760 
2761 	if (freeable)
2762 		arc_hdr_destroy(hdr);
2763 }
2764 
2765 /*
2766  * "Read" the block at the specified DVA (in bp) via the
2767  * cache.  If the block is found in the cache, invoke the provided
2768  * callback immediately and return.  Note that the `zio' parameter
2769  * in the callback will be NULL in this case, since no IO was
2770  * required.  If the block is not in the cache pass the read request
2771  * on to the spa with a substitute callback function, so that the
2772  * requested block will be added to the cache.
2773  *
2774  * If a read request arrives for a block that has a read in-progress,
2775  * either wait for the in-progress read to complete (and return the
2776  * results); or, if this is a read with a "done" func, add a record
2777  * to the read to invoke the "done" func when the read completes,
2778  * and return; or just return.
2779  *
2780  * arc_read_done() will invoke all the requested "done" functions
2781  * for readers of this block.
2782  *
2783  * Normal callers should use arc_read and pass the arc buffer and offset
2784  * for the bp.  But if you know you don't need locking, you can use
2785  * arc_read_bp.
2786  */
2787 int
2788 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_buf_t *pbuf,
2789     arc_done_func_t *done, void *private, int priority, int zio_flags,
2790     uint32_t *arc_flags, const zbookmark_t *zb)
2791 {
2792 	int err;
2793 
2794 	if (pbuf == NULL) {
2795 		/*
2796 		 * XXX This happens from traverse callback funcs, for
2797 		 * the objset_phys_t block.
2798 		 */
2799 		return (arc_read_nolock(pio, spa, bp, done, private, priority,
2800 		    zio_flags, arc_flags, zb));
2801 	}
2802 
2803 	ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt));
2804 	ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size);
2805 	rw_enter(&pbuf->b_data_lock, RW_READER);
2806 
2807 	err = arc_read_nolock(pio, spa, bp, done, private, priority,
2808 	    zio_flags, arc_flags, zb);
2809 	rw_exit(&pbuf->b_data_lock);
2810 
2811 	return (err);
2812 }
2813 
2814 int
2815 arc_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bp,
2816     arc_done_func_t *done, void *private, int priority, int zio_flags,
2817     uint32_t *arc_flags, const zbookmark_t *zb)
2818 {
2819 	arc_buf_hdr_t *hdr;
2820 	arc_buf_t *buf;
2821 	kmutex_t *hash_lock;
2822 	zio_t *rzio;
2823 	uint64_t guid = spa_load_guid(spa);
2824 
2825 top:
2826 	hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2827 	    &hash_lock);
2828 	if (hdr && hdr->b_datacnt > 0) {
2829 
2830 		*arc_flags |= ARC_CACHED;
2831 
2832 		if (HDR_IO_IN_PROGRESS(hdr)) {
2833 
2834 			if (*arc_flags & ARC_WAIT) {
2835 				cv_wait(&hdr->b_cv, hash_lock);
2836 				mutex_exit(hash_lock);
2837 				goto top;
2838 			}
2839 			ASSERT(*arc_flags & ARC_NOWAIT);
2840 
2841 			if (done) {
2842 				arc_callback_t	*acb = NULL;
2843 
2844 				acb = kmem_zalloc(sizeof (arc_callback_t),
2845 				    KM_SLEEP);
2846 				acb->acb_done = done;
2847 				acb->acb_private = private;
2848 				if (pio != NULL)
2849 					acb->acb_zio_dummy = zio_null(pio,
2850 					    spa, NULL, NULL, NULL, zio_flags);
2851 
2852 				ASSERT(acb->acb_done != NULL);
2853 				acb->acb_next = hdr->b_acb;
2854 				hdr->b_acb = acb;
2855 				add_reference(hdr, hash_lock, private);
2856 				mutex_exit(hash_lock);
2857 				return (0);
2858 			}
2859 			mutex_exit(hash_lock);
2860 			return (0);
2861 		}
2862 
2863 		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2864 
2865 		if (done) {
2866 			add_reference(hdr, hash_lock, private);
2867 			/*
2868 			 * If this block is already in use, create a new
2869 			 * copy of the data so that we will be guaranteed
2870 			 * that arc_release() will always succeed.
2871 			 */
2872 			buf = hdr->b_buf;
2873 			ASSERT(buf);
2874 			ASSERT(buf->b_data);
2875 			if (HDR_BUF_AVAILABLE(hdr)) {
2876 				ASSERT(buf->b_efunc == NULL);
2877 				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2878 			} else {
2879 				buf = arc_buf_clone(buf);
2880 			}
2881 
2882 		} else if (*arc_flags & ARC_PREFETCH &&
2883 		    refcount_count(&hdr->b_refcnt) == 0) {
2884 			hdr->b_flags |= ARC_PREFETCH;
2885 		}
2886 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
2887 		arc_access(hdr, hash_lock);
2888 		if (*arc_flags & ARC_L2CACHE)
2889 			hdr->b_flags |= ARC_L2CACHE;
2890 		mutex_exit(hash_lock);
2891 		ARCSTAT_BUMP(arcstat_hits);
2892 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2893 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2894 		    data, metadata, hits);
2895 
2896 		if (done)
2897 			done(NULL, buf, private);
2898 	} else {
2899 		uint64_t size = BP_GET_LSIZE(bp);
2900 		arc_callback_t	*acb;
2901 		vdev_t *vd = NULL;
2902 		uint64_t addr;
2903 		boolean_t devw = B_FALSE;
2904 
2905 		if (hdr == NULL) {
2906 			/* this block is not in the cache */
2907 			arc_buf_hdr_t	*exists;
2908 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2909 			buf = arc_buf_alloc(spa, size, private, type);
2910 			hdr = buf->b_hdr;
2911 			hdr->b_dva = *BP_IDENTITY(bp);
2912 			hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2913 			hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2914 			exists = buf_hash_insert(hdr, &hash_lock);
2915 			if (exists) {
2916 				/* somebody beat us to the hash insert */
2917 				mutex_exit(hash_lock);
2918 				buf_discard_identity(hdr);
2919 				(void) arc_buf_remove_ref(buf, private);
2920 				goto top; /* restart the IO request */
2921 			}
2922 			/* if this is a prefetch, we don't have a reference */
2923 			if (*arc_flags & ARC_PREFETCH) {
2924 				(void) remove_reference(hdr, hash_lock,
2925 				    private);
2926 				hdr->b_flags |= ARC_PREFETCH;
2927 			}
2928 			if (*arc_flags & ARC_L2CACHE)
2929 				hdr->b_flags |= ARC_L2CACHE;
2930 			if (BP_GET_LEVEL(bp) > 0)
2931 				hdr->b_flags |= ARC_INDIRECT;
2932 		} else {
2933 			/* this block is in the ghost cache */
2934 			ASSERT(GHOST_STATE(hdr->b_state));
2935 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2936 			ASSERT0(refcount_count(&hdr->b_refcnt));
2937 			ASSERT(hdr->b_buf == NULL);
2938 
2939 			/* if this is a prefetch, we don't have a reference */
2940 			if (*arc_flags & ARC_PREFETCH)
2941 				hdr->b_flags |= ARC_PREFETCH;
2942 			else
2943 				add_reference(hdr, hash_lock, private);
2944 			if (*arc_flags & ARC_L2CACHE)
2945 				hdr->b_flags |= ARC_L2CACHE;
2946 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2947 			buf->b_hdr = hdr;
2948 			buf->b_data = NULL;
2949 			buf->b_efunc = NULL;
2950 			buf->b_private = NULL;
2951 			buf->b_next = NULL;
2952 			hdr->b_buf = buf;
2953 			ASSERT(hdr->b_datacnt == 0);
2954 			hdr->b_datacnt = 1;
2955 			arc_get_data_buf(buf);
2956 			arc_access(hdr, hash_lock);
2957 		}
2958 
2959 		ASSERT(!GHOST_STATE(hdr->b_state));
2960 
2961 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2962 		acb->acb_done = done;
2963 		acb->acb_private = private;
2964 
2965 		ASSERT(hdr->b_acb == NULL);
2966 		hdr->b_acb = acb;
2967 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
2968 
2969 		if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL &&
2970 		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
2971 			devw = hdr->b_l2hdr->b_dev->l2ad_writing;
2972 			addr = hdr->b_l2hdr->b_daddr;
2973 			/*
2974 			 * Lock out device removal.
2975 			 */
2976 			if (vdev_is_dead(vd) ||
2977 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
2978 				vd = NULL;
2979 		}
2980 
2981 		mutex_exit(hash_lock);
2982 
2983 		ASSERT3U(hdr->b_size, ==, size);
2984 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
2985 		    uint64_t, size, zbookmark_t *, zb);
2986 		ARCSTAT_BUMP(arcstat_misses);
2987 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2988 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2989 		    data, metadata, misses);
2990 
2991 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
2992 			/*
2993 			 * Read from the L2ARC if the following are true:
2994 			 * 1. The L2ARC vdev was previously cached.
2995 			 * 2. This buffer still has L2ARC metadata.
2996 			 * 3. This buffer isn't currently writing to the L2ARC.
2997 			 * 4. The L2ARC entry wasn't evicted, which may
2998 			 *    also have invalidated the vdev.
2999 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
3000 			 */
3001 			if (hdr->b_l2hdr != NULL &&
3002 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
3003 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
3004 				l2arc_read_callback_t *cb;
3005 
3006 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
3007 				ARCSTAT_BUMP(arcstat_l2_hits);
3008 
3009 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
3010 				    KM_SLEEP);
3011 				cb->l2rcb_buf = buf;
3012 				cb->l2rcb_spa = spa;
3013 				cb->l2rcb_bp = *bp;
3014 				cb->l2rcb_zb = *zb;
3015 				cb->l2rcb_flags = zio_flags;
3016 
3017 				/*
3018 				 * l2arc read.  The SCL_L2ARC lock will be
3019 				 * released by l2arc_read_done().
3020 				 */
3021 				rzio = zio_read_phys(pio, vd, addr, size,
3022 				    buf->b_data, ZIO_CHECKSUM_OFF,
3023 				    l2arc_read_done, cb, priority, zio_flags |
3024 				    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL |
3025 				    ZIO_FLAG_DONT_PROPAGATE |
3026 				    ZIO_FLAG_DONT_RETRY, B_FALSE);
3027 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
3028 				    zio_t *, rzio);
3029 				ARCSTAT_INCR(arcstat_l2_read_bytes, size);
3030 
3031 				if (*arc_flags & ARC_NOWAIT) {
3032 					zio_nowait(rzio);
3033 					return (0);
3034 				}
3035 
3036 				ASSERT(*arc_flags & ARC_WAIT);
3037 				if (zio_wait(rzio) == 0)
3038 					return (0);
3039 
3040 				/* l2arc read error; goto zio_read() */
3041 			} else {
3042 				DTRACE_PROBE1(l2arc__miss,
3043 				    arc_buf_hdr_t *, hdr);
3044 				ARCSTAT_BUMP(arcstat_l2_misses);
3045 				if (HDR_L2_WRITING(hdr))
3046 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
3047 				spa_config_exit(spa, SCL_L2ARC, vd);
3048 			}
3049 		} else {
3050 			if (vd != NULL)
3051 				spa_config_exit(spa, SCL_L2ARC, vd);
3052 			if (l2arc_ndev != 0) {
3053 				DTRACE_PROBE1(l2arc__miss,
3054 				    arc_buf_hdr_t *, hdr);
3055 				ARCSTAT_BUMP(arcstat_l2_misses);
3056 			}
3057 		}
3058 
3059 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
3060 		    arc_read_done, buf, priority, zio_flags, zb);
3061 
3062 		if (*arc_flags & ARC_WAIT)
3063 			return (zio_wait(rzio));
3064 
3065 		ASSERT(*arc_flags & ARC_NOWAIT);
3066 		zio_nowait(rzio);
3067 	}
3068 	return (0);
3069 }
3070 
3071 void
3072 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
3073 {
3074 	ASSERT(buf->b_hdr != NULL);
3075 	ASSERT(buf->b_hdr->b_state != arc_anon);
3076 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
3077 	ASSERT(buf->b_efunc == NULL);
3078 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
3079 
3080 	buf->b_efunc = func;
3081 	buf->b_private = private;
3082 }
3083 
3084 /*
3085  * This is used by the DMU to let the ARC know that a buffer is
3086  * being evicted, so the ARC should clean up.  If this arc buf
3087  * is not yet in the evicted state, it will be put there.
3088  */
3089 int
3090 arc_buf_evict(arc_buf_t *buf)
3091 {
3092 	arc_buf_hdr_t *hdr;
3093 	kmutex_t *hash_lock;
3094 	arc_buf_t **bufp;
3095 
3096 	mutex_enter(&buf->b_evict_lock);
3097 	hdr = buf->b_hdr;
3098 	if (hdr == NULL) {
3099 		/*
3100 		 * We are in arc_do_user_evicts().
3101 		 */
3102 		ASSERT(buf->b_data == NULL);
3103 		mutex_exit(&buf->b_evict_lock);
3104 		return (0);
3105 	} else if (buf->b_data == NULL) {
3106 		arc_buf_t copy = *buf; /* structure assignment */
3107 		/*
3108 		 * We are on the eviction list; process this buffer now
3109 		 * but let arc_do_user_evicts() do the reaping.
3110 		 */
3111 		buf->b_efunc = NULL;
3112 		mutex_exit(&buf->b_evict_lock);
3113 		VERIFY(copy.b_efunc(&copy) == 0);
3114 		return (1);
3115 	}
3116 	hash_lock = HDR_LOCK(hdr);
3117 	mutex_enter(hash_lock);
3118 	hdr = buf->b_hdr;
3119 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3120 
3121 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
3122 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3123 
3124 	/*
3125 	 * Pull this buffer off of the hdr
3126 	 */
3127 	bufp = &hdr->b_buf;
3128 	while (*bufp != buf)
3129 		bufp = &(*bufp)->b_next;
3130 	*bufp = buf->b_next;
3131 
3132 	ASSERT(buf->b_data != NULL);
3133 	arc_buf_destroy(buf, FALSE, FALSE);
3134 
3135 	if (hdr->b_datacnt == 0) {
3136 		arc_state_t *old_state = hdr->b_state;
3137 		arc_state_t *evicted_state;
3138 
3139 		ASSERT(hdr->b_buf == NULL);
3140 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
3141 
3142 		evicted_state =
3143 		    (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3144 
3145 		mutex_enter(&old_state->arcs_mtx);
3146 		mutex_enter(&evicted_state->arcs_mtx);
3147 
3148 		arc_change_state(evicted_state, hdr, hash_lock);
3149 		ASSERT(HDR_IN_HASH_TABLE(hdr));
3150 		hdr->b_flags |= ARC_IN_HASH_TABLE;
3151 		hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3152 
3153 		mutex_exit(&evicted_state->arcs_mtx);
3154 		mutex_exit(&old_state->arcs_mtx);
3155 	}
3156 	mutex_exit(hash_lock);
3157 	mutex_exit(&buf->b_evict_lock);
3158 
3159 	VERIFY(buf->b_efunc(buf) == 0);
3160 	buf->b_efunc = NULL;
3161 	buf->b_private = NULL;
3162 	buf->b_hdr = NULL;
3163 	buf->b_next = NULL;
3164 	kmem_cache_free(buf_cache, buf);
3165 	return (1);
3166 }
3167 
3168 /*
3169  * Release this buffer from the cache.  This must be done
3170  * after a read and prior to modifying the buffer contents.
3171  * If the buffer has more than one reference, we must make
3172  * a new hdr for the buffer.
3173  */
3174 void
3175 arc_release(arc_buf_t *buf, void *tag)
3176 {
3177 	arc_buf_hdr_t *hdr;
3178 	kmutex_t *hash_lock = NULL;
3179 	l2arc_buf_hdr_t *l2hdr;
3180 	uint64_t buf_size;
3181 
3182 	/*
3183 	 * It would be nice to assert that if it's DMU metadata (level >
3184 	 * 0 || it's the dnode file), then it must be syncing context.
3185 	 * But we don't know that information at this level.
3186 	 */
3187 
3188 	mutex_enter(&buf->b_evict_lock);
3189 	hdr = buf->b_hdr;
3190 
3191 	/* this buffer is not on any list */
3192 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3193 
3194 	if (hdr->b_state == arc_anon) {
3195 		/* this buffer is already released */
3196 		ASSERT(buf->b_efunc == NULL);
3197 	} else {
3198 		hash_lock = HDR_LOCK(hdr);
3199 		mutex_enter(hash_lock);
3200 		hdr = buf->b_hdr;
3201 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3202 	}
3203 
3204 	l2hdr = hdr->b_l2hdr;
3205 	if (l2hdr) {
3206 		mutex_enter(&l2arc_buflist_mtx);
3207 		hdr->b_l2hdr = NULL;
3208 		buf_size = hdr->b_size;
3209 	}
3210 
3211 	/*
3212 	 * Do we have more than one buf?
3213 	 */
3214 	if (hdr->b_datacnt > 1) {
3215 		arc_buf_hdr_t *nhdr;
3216 		arc_buf_t **bufp;
3217 		uint64_t blksz = hdr->b_size;
3218 		uint64_t spa = hdr->b_spa;
3219 		arc_buf_contents_t type = hdr->b_type;
3220 		uint32_t flags = hdr->b_flags;
3221 
3222 		ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3223 		/*
3224 		 * Pull the data off of this hdr and attach it to
3225 		 * a new anonymous hdr.
3226 		 */
3227 		(void) remove_reference(hdr, hash_lock, tag);
3228 		bufp = &hdr->b_buf;
3229 		while (*bufp != buf)
3230 			bufp = &(*bufp)->b_next;
3231 		*bufp = buf->b_next;
3232 		buf->b_next = NULL;
3233 
3234 		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3235 		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3236 		if (refcount_is_zero(&hdr->b_refcnt)) {
3237 			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3238 			ASSERT3U(*size, >=, hdr->b_size);
3239 			atomic_add_64(size, -hdr->b_size);
3240 		}
3241 
3242 		/*
3243 		 * We're releasing a duplicate user data buffer, update
3244 		 * our statistics accordingly.
3245 		 */
3246 		if (hdr->b_type == ARC_BUFC_DATA) {
3247 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
3248 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
3249 			    -hdr->b_size);
3250 		}
3251 		hdr->b_datacnt -= 1;
3252 		arc_cksum_verify(buf);
3253 		arc_buf_unwatch(buf);
3254 
3255 		mutex_exit(hash_lock);
3256 
3257 		nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3258 		nhdr->b_size = blksz;
3259 		nhdr->b_spa = spa;
3260 		nhdr->b_type = type;
3261 		nhdr->b_buf = buf;
3262 		nhdr->b_state = arc_anon;
3263 		nhdr->b_arc_access = 0;
3264 		nhdr->b_flags = flags & ARC_L2_WRITING;
3265 		nhdr->b_l2hdr = NULL;
3266 		nhdr->b_datacnt = 1;
3267 		nhdr->b_freeze_cksum = NULL;
3268 		(void) refcount_add(&nhdr->b_refcnt, tag);
3269 		buf->b_hdr = nhdr;
3270 		mutex_exit(&buf->b_evict_lock);
3271 		atomic_add_64(&arc_anon->arcs_size, blksz);
3272 	} else {
3273 		mutex_exit(&buf->b_evict_lock);
3274 		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3275 		ASSERT(!list_link_active(&hdr->b_arc_node));
3276 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3277 		if (hdr->b_state != arc_anon)
3278 			arc_change_state(arc_anon, hdr, hash_lock);
3279 		hdr->b_arc_access = 0;
3280 		if (hash_lock)
3281 			mutex_exit(hash_lock);
3282 
3283 		buf_discard_identity(hdr);
3284 		arc_buf_thaw(buf);
3285 	}
3286 	buf->b_efunc = NULL;
3287 	buf->b_private = NULL;
3288 
3289 	if (l2hdr) {
3290 		list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3291 		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3292 		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3293 		mutex_exit(&l2arc_buflist_mtx);
3294 	}
3295 }
3296 
3297 /*
3298  * Release this buffer.  If it does not match the provided BP, fill it
3299  * with that block's contents.
3300  */
3301 /* ARGSUSED */
3302 int
3303 arc_release_bp(arc_buf_t *buf, void *tag, blkptr_t *bp, spa_t *spa,
3304     zbookmark_t *zb)
3305 {
3306 	arc_release(buf, tag);
3307 	return (0);
3308 }
3309 
3310 int
3311 arc_released(arc_buf_t *buf)
3312 {
3313 	int released;
3314 
3315 	mutex_enter(&buf->b_evict_lock);
3316 	released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
3317 	mutex_exit(&buf->b_evict_lock);
3318 	return (released);
3319 }
3320 
3321 int
3322 arc_has_callback(arc_buf_t *buf)
3323 {
3324 	int callback;
3325 
3326 	mutex_enter(&buf->b_evict_lock);
3327 	callback = (buf->b_efunc != NULL);
3328 	mutex_exit(&buf->b_evict_lock);
3329 	return (callback);
3330 }
3331 
3332 #ifdef ZFS_DEBUG
3333 int
3334 arc_referenced(arc_buf_t *buf)
3335 {
3336 	int referenced;
3337 
3338 	mutex_enter(&buf->b_evict_lock);
3339 	referenced = (refcount_count(&buf->b_hdr->b_refcnt));
3340 	mutex_exit(&buf->b_evict_lock);
3341 	return (referenced);
3342 }
3343 #endif
3344 
3345 static void
3346 arc_write_ready(zio_t *zio)
3347 {
3348 	arc_write_callback_t *callback = zio->io_private;
3349 	arc_buf_t *buf = callback->awcb_buf;
3350 	arc_buf_hdr_t *hdr = buf->b_hdr;
3351 
3352 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3353 	callback->awcb_ready(zio, buf, callback->awcb_private);
3354 
3355 	/*
3356 	 * If the IO is already in progress, then this is a re-write
3357 	 * attempt, so we need to thaw and re-compute the cksum.
3358 	 * It is the responsibility of the callback to handle the
3359 	 * accounting for any re-write attempt.
3360 	 */
3361 	if (HDR_IO_IN_PROGRESS(hdr)) {
3362 		mutex_enter(&hdr->b_freeze_lock);
3363 		if (hdr->b_freeze_cksum != NULL) {
3364 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3365 			hdr->b_freeze_cksum = NULL;
3366 		}
3367 		mutex_exit(&hdr->b_freeze_lock);
3368 	}
3369 	arc_cksum_compute(buf, B_FALSE);
3370 	hdr->b_flags |= ARC_IO_IN_PROGRESS;
3371 }
3372 
3373 static void
3374 arc_write_done(zio_t *zio)
3375 {
3376 	arc_write_callback_t *callback = zio->io_private;
3377 	arc_buf_t *buf = callback->awcb_buf;
3378 	arc_buf_hdr_t *hdr = buf->b_hdr;
3379 
3380 	ASSERT(hdr->b_acb == NULL);
3381 
3382 	if (zio->io_error == 0) {
3383 		hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3384 		hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3385 		hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3386 	} else {
3387 		ASSERT(BUF_EMPTY(hdr));
3388 	}
3389 
3390 	/*
3391 	 * If the block to be written was all-zero, we may have
3392 	 * compressed it away.  In this case no write was performed
3393 	 * so there will be no dva/birth/checksum.  The buffer must
3394 	 * therefore remain anonymous (and uncached).
3395 	 */
3396 	if (!BUF_EMPTY(hdr)) {
3397 		arc_buf_hdr_t *exists;
3398 		kmutex_t *hash_lock;
3399 
3400 		ASSERT(zio->io_error == 0);
3401 
3402 		arc_cksum_verify(buf);
3403 
3404 		exists = buf_hash_insert(hdr, &hash_lock);
3405 		if (exists) {
3406 			/*
3407 			 * This can only happen if we overwrite for
3408 			 * sync-to-convergence, because we remove
3409 			 * buffers from the hash table when we arc_free().
3410 			 */
3411 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3412 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3413 					panic("bad overwrite, hdr=%p exists=%p",
3414 					    (void *)hdr, (void *)exists);
3415 				ASSERT(refcount_is_zero(&exists->b_refcnt));
3416 				arc_change_state(arc_anon, exists, hash_lock);
3417 				mutex_exit(hash_lock);
3418 				arc_hdr_destroy(exists);
3419 				exists = buf_hash_insert(hdr, &hash_lock);
3420 				ASSERT3P(exists, ==, NULL);
3421 			} else {
3422 				/* Dedup */
3423 				ASSERT(hdr->b_datacnt == 1);
3424 				ASSERT(hdr->b_state == arc_anon);
3425 				ASSERT(BP_GET_DEDUP(zio->io_bp));
3426 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3427 			}
3428 		}
3429 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3430 		/* if it's not anon, we are doing a scrub */
3431 		if (!exists && hdr->b_state == arc_anon)
3432 			arc_access(hdr, hash_lock);
3433 		mutex_exit(hash_lock);
3434 	} else {
3435 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3436 	}
3437 
3438 	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3439 	callback->awcb_done(zio, buf, callback->awcb_private);
3440 
3441 	kmem_free(callback, sizeof (arc_write_callback_t));
3442 }
3443 
3444 zio_t *
3445 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3446     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp,
3447     arc_done_func_t *ready, arc_done_func_t *done, void *private,
3448     int priority, int zio_flags, const zbookmark_t *zb)
3449 {
3450 	arc_buf_hdr_t *hdr = buf->b_hdr;
3451 	arc_write_callback_t *callback;
3452 	zio_t *zio;
3453 
3454 	ASSERT(ready != NULL);
3455 	ASSERT(done != NULL);
3456 	ASSERT(!HDR_IO_ERROR(hdr));
3457 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3458 	ASSERT(hdr->b_acb == NULL);
3459 	if (l2arc)
3460 		hdr->b_flags |= ARC_L2CACHE;
3461 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3462 	callback->awcb_ready = ready;
3463 	callback->awcb_done = done;
3464 	callback->awcb_private = private;
3465 	callback->awcb_buf = buf;
3466 
3467 	zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3468 	    arc_write_ready, arc_write_done, callback, priority, zio_flags, zb);
3469 
3470 	return (zio);
3471 }
3472 
3473 static int
3474 arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg)
3475 {
3476 #ifdef _KERNEL
3477 	uint64_t available_memory = ptob(freemem);
3478 	static uint64_t page_load = 0;
3479 	static uint64_t last_txg = 0;
3480 
3481 #if defined(__i386)
3482 	available_memory =
3483 	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3484 #endif
3485 	if (available_memory >= zfs_write_limit_max)
3486 		return (0);
3487 
3488 	if (txg > last_txg) {
3489 		last_txg = txg;
3490 		page_load = 0;
3491 	}
3492 	/*
3493 	 * If we are in pageout, we know that memory is already tight,
3494 	 * the arc is already going to be evicting, so we just want to
3495 	 * continue to let page writes occur as quickly as possible.
3496 	 */
3497 	if (curproc == proc_pageout) {
3498 		if (page_load > MAX(ptob(minfree), available_memory) / 4)
3499 			return (ERESTART);
3500 		/* Note: reserve is inflated, so we deflate */
3501 		page_load += reserve / 8;
3502 		return (0);
3503 	} else if (page_load > 0 && arc_reclaim_needed()) {
3504 		/* memory is low, delay before restarting */
3505 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3506 		return (EAGAIN);
3507 	}
3508 	page_load = 0;
3509 
3510 	if (arc_size > arc_c_min) {
3511 		uint64_t evictable_memory =
3512 		    arc_mru->arcs_lsize[ARC_BUFC_DATA] +
3513 		    arc_mru->arcs_lsize[ARC_BUFC_METADATA] +
3514 		    arc_mfu->arcs_lsize[ARC_BUFC_DATA] +
3515 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA];
3516 		available_memory += MIN(evictable_memory, arc_size - arc_c_min);
3517 	}
3518 
3519 	if (inflight_data > available_memory / 4) {
3520 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3521 		return (ERESTART);
3522 	}
3523 #endif
3524 	return (0);
3525 }
3526 
3527 void
3528 arc_tempreserve_clear(uint64_t reserve)
3529 {
3530 	atomic_add_64(&arc_tempreserve, -reserve);
3531 	ASSERT((int64_t)arc_tempreserve >= 0);
3532 }
3533 
3534 int
3535 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3536 {
3537 	int error;
3538 	uint64_t anon_size;
3539 
3540 #ifdef ZFS_DEBUG
3541 	/*
3542 	 * Once in a while, fail for no reason.  Everything should cope.
3543 	 */
3544 	if (spa_get_random(10000) == 0) {
3545 		dprintf("forcing random failure\n");
3546 		return (ERESTART);
3547 	}
3548 #endif
3549 	if (reserve > arc_c/4 && !arc_no_grow)
3550 		arc_c = MIN(arc_c_max, reserve * 4);
3551 	if (reserve > arc_c)
3552 		return (ENOMEM);
3553 
3554 	/*
3555 	 * Don't count loaned bufs as in flight dirty data to prevent long
3556 	 * network delays from blocking transactions that are ready to be
3557 	 * assigned to a txg.
3558 	 */
3559 	anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3560 
3561 	/*
3562 	 * Writes will, almost always, require additional memory allocations
3563 	 * in order to compress/encrypt/etc the data.  We therefor need to
3564 	 * make sure that there is sufficient available memory for this.
3565 	 */
3566 	if (error = arc_memory_throttle(reserve, anon_size, txg))
3567 		return (error);
3568 
3569 	/*
3570 	 * Throttle writes when the amount of dirty data in the cache
3571 	 * gets too large.  We try to keep the cache less than half full
3572 	 * of dirty blocks so that our sync times don't grow too large.
3573 	 * Note: if two requests come in concurrently, we might let them
3574 	 * both succeed, when one of them should fail.  Not a huge deal.
3575 	 */
3576 
3577 	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3578 	    anon_size > arc_c / 4) {
3579 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3580 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3581 		    arc_tempreserve>>10,
3582 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3583 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3584 		    reserve>>10, arc_c>>10);
3585 		return (ERESTART);
3586 	}
3587 	atomic_add_64(&arc_tempreserve, reserve);
3588 	return (0);
3589 }
3590 
3591 void
3592 arc_init(void)
3593 {
3594 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3595 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3596 
3597 	/* Convert seconds to clock ticks */
3598 	arc_min_prefetch_lifespan = 1 * hz;
3599 
3600 	/* Start out with 1/8 of all memory */
3601 	arc_c = physmem * PAGESIZE / 8;
3602 
3603 #ifdef _KERNEL
3604 	/*
3605 	 * On architectures where the physical memory can be larger
3606 	 * than the addressable space (intel in 32-bit mode), we may
3607 	 * need to limit the cache to 1/8 of VM size.
3608 	 */
3609 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3610 #endif
3611 
3612 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
3613 	arc_c_min = MAX(arc_c / 4, 64<<20);
3614 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
3615 	if (arc_c * 8 >= 1<<30)
3616 		arc_c_max = (arc_c * 8) - (1<<30);
3617 	else
3618 		arc_c_max = arc_c_min;
3619 	arc_c_max = MAX(arc_c * 6, arc_c_max);
3620 
3621 	/*
3622 	 * Allow the tunables to override our calculations if they are
3623 	 * reasonable (ie. over 64MB)
3624 	 */
3625 	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
3626 		arc_c_max = zfs_arc_max;
3627 	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
3628 		arc_c_min = zfs_arc_min;
3629 
3630 	arc_c = arc_c_max;
3631 	arc_p = (arc_c >> 1);
3632 
3633 	/* limit meta-data to 1/4 of the arc capacity */
3634 	arc_meta_limit = arc_c_max / 4;
3635 
3636 	/* Allow the tunable to override if it is reasonable */
3637 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
3638 		arc_meta_limit = zfs_arc_meta_limit;
3639 
3640 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
3641 		arc_c_min = arc_meta_limit / 2;
3642 
3643 	if (zfs_arc_grow_retry > 0)
3644 		arc_grow_retry = zfs_arc_grow_retry;
3645 
3646 	if (zfs_arc_shrink_shift > 0)
3647 		arc_shrink_shift = zfs_arc_shrink_shift;
3648 
3649 	if (zfs_arc_p_min_shift > 0)
3650 		arc_p_min_shift = zfs_arc_p_min_shift;
3651 
3652 	/* if kmem_flags are set, lets try to use less memory */
3653 	if (kmem_debugging())
3654 		arc_c = arc_c / 2;
3655 	if (arc_c < arc_c_min)
3656 		arc_c = arc_c_min;
3657 
3658 	arc_anon = &ARC_anon;
3659 	arc_mru = &ARC_mru;
3660 	arc_mru_ghost = &ARC_mru_ghost;
3661 	arc_mfu = &ARC_mfu;
3662 	arc_mfu_ghost = &ARC_mfu_ghost;
3663 	arc_l2c_only = &ARC_l2c_only;
3664 	arc_size = 0;
3665 
3666 	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3667 	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3668 	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3669 	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3670 	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3671 	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3672 
3673 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
3674 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3675 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
3676 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3677 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
3678 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3679 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
3680 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3681 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
3682 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3683 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
3684 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3685 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
3686 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3687 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
3688 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3689 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3690 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3691 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3692 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3693 
3694 	buf_init();
3695 
3696 	arc_thread_exit = 0;
3697 	arc_eviction_list = NULL;
3698 	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
3699 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3700 
3701 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
3702 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
3703 
3704 	if (arc_ksp != NULL) {
3705 		arc_ksp->ks_data = &arc_stats;
3706 		kstat_install(arc_ksp);
3707 	}
3708 
3709 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3710 	    TS_RUN, minclsyspri);
3711 
3712 	arc_dead = FALSE;
3713 	arc_warm = B_FALSE;
3714 
3715 	if (zfs_write_limit_max == 0)
3716 		zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
3717 	else
3718 		zfs_write_limit_shift = 0;
3719 	mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL);
3720 }
3721 
3722 void
3723 arc_fini(void)
3724 {
3725 	mutex_enter(&arc_reclaim_thr_lock);
3726 	arc_thread_exit = 1;
3727 	while (arc_thread_exit != 0)
3728 		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3729 	mutex_exit(&arc_reclaim_thr_lock);
3730 
3731 	arc_flush(NULL);
3732 
3733 	arc_dead = TRUE;
3734 
3735 	if (arc_ksp != NULL) {
3736 		kstat_delete(arc_ksp);
3737 		arc_ksp = NULL;
3738 	}
3739 
3740 	mutex_destroy(&arc_eviction_mtx);
3741 	mutex_destroy(&arc_reclaim_thr_lock);
3742 	cv_destroy(&arc_reclaim_thr_cv);
3743 
3744 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
3745 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
3746 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
3747 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
3748 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
3749 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
3750 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
3751 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3752 
3753 	mutex_destroy(&arc_anon->arcs_mtx);
3754 	mutex_destroy(&arc_mru->arcs_mtx);
3755 	mutex_destroy(&arc_mru_ghost->arcs_mtx);
3756 	mutex_destroy(&arc_mfu->arcs_mtx);
3757 	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3758 	mutex_destroy(&arc_l2c_only->arcs_mtx);
3759 
3760 	mutex_destroy(&zfs_write_limit_lock);
3761 
3762 	buf_fini();
3763 
3764 	ASSERT(arc_loaned_bytes == 0);
3765 }
3766 
3767 /*
3768  * Level 2 ARC
3769  *
3770  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3771  * It uses dedicated storage devices to hold cached data, which are populated
3772  * using large infrequent writes.  The main role of this cache is to boost
3773  * the performance of random read workloads.  The intended L2ARC devices
3774  * include short-stroked disks, solid state disks, and other media with
3775  * substantially faster read latency than disk.
3776  *
3777  *                 +-----------------------+
3778  *                 |         ARC           |
3779  *                 +-----------------------+
3780  *                    |         ^     ^
3781  *                    |         |     |
3782  *      l2arc_feed_thread()    arc_read()
3783  *                    |         |     |
3784  *                    |  l2arc read   |
3785  *                    V         |     |
3786  *               +---------------+    |
3787  *               |     L2ARC     |    |
3788  *               +---------------+    |
3789  *                   |    ^           |
3790  *          l2arc_write() |           |
3791  *                   |    |           |
3792  *                   V    |           |
3793  *                 +-------+      +-------+
3794  *                 | vdev  |      | vdev  |
3795  *                 | cache |      | cache |
3796  *                 +-------+      +-------+
3797  *                 +=========+     .-----.
3798  *                 :  L2ARC  :    |-_____-|
3799  *                 : devices :    | Disks |
3800  *                 +=========+    `-_____-'
3801  *
3802  * Read requests are satisfied from the following sources, in order:
3803  *
3804  *	1) ARC
3805  *	2) vdev cache of L2ARC devices
3806  *	3) L2ARC devices
3807  *	4) vdev cache of disks
3808  *	5) disks
3809  *
3810  * Some L2ARC device types exhibit extremely slow write performance.
3811  * To accommodate for this there are some significant differences between
3812  * the L2ARC and traditional cache design:
3813  *
3814  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3815  * the ARC behave as usual, freeing buffers and placing headers on ghost
3816  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3817  * this would add inflated write latencies for all ARC memory pressure.
3818  *
3819  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3820  * It does this by periodically scanning buffers from the eviction-end of
3821  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3822  * not already there.  It scans until a headroom of buffers is satisfied,
3823  * which itself is a buffer for ARC eviction.  The thread that does this is
3824  * l2arc_feed_thread(), illustrated below; example sizes are included to
3825  * provide a better sense of ratio than this diagram:
3826  *
3827  *	       head -->                        tail
3828  *	        +---------------------+----------+
3829  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3830  *	        +---------------------+----------+   |   o L2ARC eligible
3831  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3832  *	        +---------------------+----------+   |
3833  *	             15.9 Gbytes      ^ 32 Mbytes    |
3834  *	                           headroom          |
3835  *	                                      l2arc_feed_thread()
3836  *	                                             |
3837  *	                 l2arc write hand <--[oooo]--'
3838  *	                         |           8 Mbyte
3839  *	                         |          write max
3840  *	                         V
3841  *		  +==============================+
3842  *	L2ARC dev |####|#|###|###|    |####| ... |
3843  *	          +==============================+
3844  *	                     32 Gbytes
3845  *
3846  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3847  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3848  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3849  * safe to say that this is an uncommon case, since buffers at the end of
3850  * the ARC lists have moved there due to inactivity.
3851  *
3852  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3853  * then the L2ARC simply misses copying some buffers.  This serves as a
3854  * pressure valve to prevent heavy read workloads from both stalling the ARC
3855  * with waits and clogging the L2ARC with writes.  This also helps prevent
3856  * the potential for the L2ARC to churn if it attempts to cache content too
3857  * quickly, such as during backups of the entire pool.
3858  *
3859  * 5. After system boot and before the ARC has filled main memory, there are
3860  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
3861  * lists can remain mostly static.  Instead of searching from tail of these
3862  * lists as pictured, the l2arc_feed_thread() will search from the list heads
3863  * for eligible buffers, greatly increasing its chance of finding them.
3864  *
3865  * The L2ARC device write speed is also boosted during this time so that
3866  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
3867  * there are no L2ARC reads, and no fear of degrading read performance
3868  * through increased writes.
3869  *
3870  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
3871  * the vdev queue can aggregate them into larger and fewer writes.  Each
3872  * device is written to in a rotor fashion, sweeping writes through
3873  * available space then repeating.
3874  *
3875  * 7. The L2ARC does not store dirty content.  It never needs to flush
3876  * write buffers back to disk based storage.
3877  *
3878  * 8. If an ARC buffer is written (and dirtied) which also exists in the
3879  * L2ARC, the now stale L2ARC buffer is immediately dropped.
3880  *
3881  * The performance of the L2ARC can be tweaked by a number of tunables, which
3882  * may be necessary for different workloads:
3883  *
3884  *	l2arc_write_max		max write bytes per interval
3885  *	l2arc_write_boost	extra write bytes during device warmup
3886  *	l2arc_noprefetch	skip caching prefetched buffers
3887  *	l2arc_headroom		number of max device writes to precache
3888  *	l2arc_feed_secs		seconds between L2ARC writing
3889  *
3890  * Tunables may be removed or added as future performance improvements are
3891  * integrated, and also may become zpool properties.
3892  *
3893  * There are three key functions that control how the L2ARC warms up:
3894  *
3895  *	l2arc_write_eligible()	check if a buffer is eligible to cache
3896  *	l2arc_write_size()	calculate how much to write
3897  *	l2arc_write_interval()	calculate sleep delay between writes
3898  *
3899  * These three functions determine what to write, how much, and how quickly
3900  * to send writes.
3901  */
3902 
3903 static boolean_t
3904 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
3905 {
3906 	/*
3907 	 * A buffer is *not* eligible for the L2ARC if it:
3908 	 * 1. belongs to a different spa.
3909 	 * 2. is already cached on the L2ARC.
3910 	 * 3. has an I/O in progress (it may be an incomplete read).
3911 	 * 4. is flagged not eligible (zfs property).
3912 	 */
3913 	if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
3914 	    HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
3915 		return (B_FALSE);
3916 
3917 	return (B_TRUE);
3918 }
3919 
3920 static uint64_t
3921 l2arc_write_size(l2arc_dev_t *dev)
3922 {
3923 	uint64_t size;
3924 
3925 	size = dev->l2ad_write;
3926 
3927 	if (arc_warm == B_FALSE)
3928 		size += dev->l2ad_boost;
3929 
3930 	return (size);
3931 
3932 }
3933 
3934 static clock_t
3935 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
3936 {
3937 	clock_t interval, next, now;
3938 
3939 	/*
3940 	 * If the ARC lists are busy, increase our write rate; if the
3941 	 * lists are stale, idle back.  This is achieved by checking
3942 	 * how much we previously wrote - if it was more than half of
3943 	 * what we wanted, schedule the next write much sooner.
3944 	 */
3945 	if (l2arc_feed_again && wrote > (wanted / 2))
3946 		interval = (hz * l2arc_feed_min_ms) / 1000;
3947 	else
3948 		interval = hz * l2arc_feed_secs;
3949 
3950 	now = ddi_get_lbolt();
3951 	next = MAX(now, MIN(now + interval, began + interval));
3952 
3953 	return (next);
3954 }
3955 
3956 static void
3957 l2arc_hdr_stat_add(void)
3958 {
3959 	ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
3960 	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
3961 }
3962 
3963 static void
3964 l2arc_hdr_stat_remove(void)
3965 {
3966 	ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
3967 	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
3968 }
3969 
3970 /*
3971  * Cycle through L2ARC devices.  This is how L2ARC load balances.
3972  * If a device is returned, this also returns holding the spa config lock.
3973  */
3974 static l2arc_dev_t *
3975 l2arc_dev_get_next(void)
3976 {
3977 	l2arc_dev_t *first, *next = NULL;
3978 
3979 	/*
3980 	 * Lock out the removal of spas (spa_namespace_lock), then removal
3981 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
3982 	 * both locks will be dropped and a spa config lock held instead.
3983 	 */
3984 	mutex_enter(&spa_namespace_lock);
3985 	mutex_enter(&l2arc_dev_mtx);
3986 
3987 	/* if there are no vdevs, there is nothing to do */
3988 	if (l2arc_ndev == 0)
3989 		goto out;
3990 
3991 	first = NULL;
3992 	next = l2arc_dev_last;
3993 	do {
3994 		/* loop around the list looking for a non-faulted vdev */
3995 		if (next == NULL) {
3996 			next = list_head(l2arc_dev_list);
3997 		} else {
3998 			next = list_next(l2arc_dev_list, next);
3999 			if (next == NULL)
4000 				next = list_head(l2arc_dev_list);
4001 		}
4002 
4003 		/* if we have come back to the start, bail out */
4004 		if (first == NULL)
4005 			first = next;
4006 		else if (next == first)
4007 			break;
4008 
4009 	} while (vdev_is_dead(next->l2ad_vdev));
4010 
4011 	/* if we were unable to find any usable vdevs, return NULL */
4012 	if (vdev_is_dead(next->l2ad_vdev))
4013 		next = NULL;
4014 
4015 	l2arc_dev_last = next;
4016 
4017 out:
4018 	mutex_exit(&l2arc_dev_mtx);
4019 
4020 	/*
4021 	 * Grab the config lock to prevent the 'next' device from being
4022 	 * removed while we are writing to it.
4023 	 */
4024 	if (next != NULL)
4025 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
4026 	mutex_exit(&spa_namespace_lock);
4027 
4028 	return (next);
4029 }
4030 
4031 /*
4032  * Free buffers that were tagged for destruction.
4033  */
4034 static void
4035 l2arc_do_free_on_write()
4036 {
4037 	list_t *buflist;
4038 	l2arc_data_free_t *df, *df_prev;
4039 
4040 	mutex_enter(&l2arc_free_on_write_mtx);
4041 	buflist = l2arc_free_on_write;
4042 
4043 	for (df = list_tail(buflist); df; df = df_prev) {
4044 		df_prev = list_prev(buflist, df);
4045 		ASSERT(df->l2df_data != NULL);
4046 		ASSERT(df->l2df_func != NULL);
4047 		df->l2df_func(df->l2df_data, df->l2df_size);
4048 		list_remove(buflist, df);
4049 		kmem_free(df, sizeof (l2arc_data_free_t));
4050 	}
4051 
4052 	mutex_exit(&l2arc_free_on_write_mtx);
4053 }
4054 
4055 /*
4056  * A write to a cache device has completed.  Update all headers to allow
4057  * reads from these buffers to begin.
4058  */
4059 static void
4060 l2arc_write_done(zio_t *zio)
4061 {
4062 	l2arc_write_callback_t *cb;
4063 	l2arc_dev_t *dev;
4064 	list_t *buflist;
4065 	arc_buf_hdr_t *head, *ab, *ab_prev;
4066 	l2arc_buf_hdr_t *abl2;
4067 	kmutex_t *hash_lock;
4068 
4069 	cb = zio->io_private;
4070 	ASSERT(cb != NULL);
4071 	dev = cb->l2wcb_dev;
4072 	ASSERT(dev != NULL);
4073 	head = cb->l2wcb_head;
4074 	ASSERT(head != NULL);
4075 	buflist = dev->l2ad_buflist;
4076 	ASSERT(buflist != NULL);
4077 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
4078 	    l2arc_write_callback_t *, cb);
4079 
4080 	if (zio->io_error != 0)
4081 		ARCSTAT_BUMP(arcstat_l2_writes_error);
4082 
4083 	mutex_enter(&l2arc_buflist_mtx);
4084 
4085 	/*
4086 	 * All writes completed, or an error was hit.
4087 	 */
4088 	for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
4089 		ab_prev = list_prev(buflist, ab);
4090 
4091 		hash_lock = HDR_LOCK(ab);
4092 		if (!mutex_tryenter(hash_lock)) {
4093 			/*
4094 			 * This buffer misses out.  It may be in a stage
4095 			 * of eviction.  Its ARC_L2_WRITING flag will be
4096 			 * left set, denying reads to this buffer.
4097 			 */
4098 			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
4099 			continue;
4100 		}
4101 
4102 		if (zio->io_error != 0) {
4103 			/*
4104 			 * Error - drop L2ARC entry.
4105 			 */
4106 			list_remove(buflist, ab);
4107 			abl2 = ab->b_l2hdr;
4108 			ab->b_l2hdr = NULL;
4109 			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4110 			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4111 		}
4112 
4113 		/*
4114 		 * Allow ARC to begin reads to this L2ARC entry.
4115 		 */
4116 		ab->b_flags &= ~ARC_L2_WRITING;
4117 
4118 		mutex_exit(hash_lock);
4119 	}
4120 
4121 	atomic_inc_64(&l2arc_writes_done);
4122 	list_remove(buflist, head);
4123 	kmem_cache_free(hdr_cache, head);
4124 	mutex_exit(&l2arc_buflist_mtx);
4125 
4126 	l2arc_do_free_on_write();
4127 
4128 	kmem_free(cb, sizeof (l2arc_write_callback_t));
4129 }
4130 
4131 /*
4132  * A read to a cache device completed.  Validate buffer contents before
4133  * handing over to the regular ARC routines.
4134  */
4135 static void
4136 l2arc_read_done(zio_t *zio)
4137 {
4138 	l2arc_read_callback_t *cb;
4139 	arc_buf_hdr_t *hdr;
4140 	arc_buf_t *buf;
4141 	kmutex_t *hash_lock;
4142 	int equal;
4143 
4144 	ASSERT(zio->io_vd != NULL);
4145 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
4146 
4147 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
4148 
4149 	cb = zio->io_private;
4150 	ASSERT(cb != NULL);
4151 	buf = cb->l2rcb_buf;
4152 	ASSERT(buf != NULL);
4153 
4154 	hash_lock = HDR_LOCK(buf->b_hdr);
4155 	mutex_enter(hash_lock);
4156 	hdr = buf->b_hdr;
4157 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4158 
4159 	/*
4160 	 * Check this survived the L2ARC journey.
4161 	 */
4162 	equal = arc_cksum_equal(buf);
4163 	if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
4164 		mutex_exit(hash_lock);
4165 		zio->io_private = buf;
4166 		zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
4167 		zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
4168 		arc_read_done(zio);
4169 	} else {
4170 		mutex_exit(hash_lock);
4171 		/*
4172 		 * Buffer didn't survive caching.  Increment stats and
4173 		 * reissue to the original storage device.
4174 		 */
4175 		if (zio->io_error != 0) {
4176 			ARCSTAT_BUMP(arcstat_l2_io_error);
4177 		} else {
4178 			zio->io_error = EIO;
4179 		}
4180 		if (!equal)
4181 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4182 
4183 		/*
4184 		 * If there's no waiter, issue an async i/o to the primary
4185 		 * storage now.  If there *is* a waiter, the caller must
4186 		 * issue the i/o in a context where it's OK to block.
4187 		 */
4188 		if (zio->io_waiter == NULL) {
4189 			zio_t *pio = zio_unique_parent(zio);
4190 
4191 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4192 
4193 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4194 			    buf->b_data, zio->io_size, arc_read_done, buf,
4195 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4196 		}
4197 	}
4198 
4199 	kmem_free(cb, sizeof (l2arc_read_callback_t));
4200 }
4201 
4202 /*
4203  * This is the list priority from which the L2ARC will search for pages to
4204  * cache.  This is used within loops (0..3) to cycle through lists in the
4205  * desired order.  This order can have a significant effect on cache
4206  * performance.
4207  *
4208  * Currently the metadata lists are hit first, MFU then MRU, followed by
4209  * the data lists.  This function returns a locked list, and also returns
4210  * the lock pointer.
4211  */
4212 static list_t *
4213 l2arc_list_locked(int list_num, kmutex_t **lock)
4214 {
4215 	list_t *list;
4216 
4217 	ASSERT(list_num >= 0 && list_num <= 3);
4218 
4219 	switch (list_num) {
4220 	case 0:
4221 		list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4222 		*lock = &arc_mfu->arcs_mtx;
4223 		break;
4224 	case 1:
4225 		list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4226 		*lock = &arc_mru->arcs_mtx;
4227 		break;
4228 	case 2:
4229 		list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4230 		*lock = &arc_mfu->arcs_mtx;
4231 		break;
4232 	case 3:
4233 		list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4234 		*lock = &arc_mru->arcs_mtx;
4235 		break;
4236 	}
4237 
4238 	ASSERT(!(MUTEX_HELD(*lock)));
4239 	mutex_enter(*lock);
4240 	return (list);
4241 }
4242 
4243 /*
4244  * Evict buffers from the device write hand to the distance specified in
4245  * bytes.  This distance may span populated buffers, it may span nothing.
4246  * This is clearing a region on the L2ARC device ready for writing.
4247  * If the 'all' boolean is set, every buffer is evicted.
4248  */
4249 static void
4250 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4251 {
4252 	list_t *buflist;
4253 	l2arc_buf_hdr_t *abl2;
4254 	arc_buf_hdr_t *ab, *ab_prev;
4255 	kmutex_t *hash_lock;
4256 	uint64_t taddr;
4257 
4258 	buflist = dev->l2ad_buflist;
4259 
4260 	if (buflist == NULL)
4261 		return;
4262 
4263 	if (!all && dev->l2ad_first) {
4264 		/*
4265 		 * This is the first sweep through the device.  There is
4266 		 * nothing to evict.
4267 		 */
4268 		return;
4269 	}
4270 
4271 	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4272 		/*
4273 		 * When nearing the end of the device, evict to the end
4274 		 * before the device write hand jumps to the start.
4275 		 */
4276 		taddr = dev->l2ad_end;
4277 	} else {
4278 		taddr = dev->l2ad_hand + distance;
4279 	}
4280 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4281 	    uint64_t, taddr, boolean_t, all);
4282 
4283 top:
4284 	mutex_enter(&l2arc_buflist_mtx);
4285 	for (ab = list_tail(buflist); ab; ab = ab_prev) {
4286 		ab_prev = list_prev(buflist, ab);
4287 
4288 		hash_lock = HDR_LOCK(ab);
4289 		if (!mutex_tryenter(hash_lock)) {
4290 			/*
4291 			 * Missed the hash lock.  Retry.
4292 			 */
4293 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4294 			mutex_exit(&l2arc_buflist_mtx);
4295 			mutex_enter(hash_lock);
4296 			mutex_exit(hash_lock);
4297 			goto top;
4298 		}
4299 
4300 		if (HDR_L2_WRITE_HEAD(ab)) {
4301 			/*
4302 			 * We hit a write head node.  Leave it for
4303 			 * l2arc_write_done().
4304 			 */
4305 			list_remove(buflist, ab);
4306 			mutex_exit(hash_lock);
4307 			continue;
4308 		}
4309 
4310 		if (!all && ab->b_l2hdr != NULL &&
4311 		    (ab->b_l2hdr->b_daddr > taddr ||
4312 		    ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4313 			/*
4314 			 * We've evicted to the target address,
4315 			 * or the end of the device.
4316 			 */
4317 			mutex_exit(hash_lock);
4318 			break;
4319 		}
4320 
4321 		if (HDR_FREE_IN_PROGRESS(ab)) {
4322 			/*
4323 			 * Already on the path to destruction.
4324 			 */
4325 			mutex_exit(hash_lock);
4326 			continue;
4327 		}
4328 
4329 		if (ab->b_state == arc_l2c_only) {
4330 			ASSERT(!HDR_L2_READING(ab));
4331 			/*
4332 			 * This doesn't exist in the ARC.  Destroy.
4333 			 * arc_hdr_destroy() will call list_remove()
4334 			 * and decrement arcstat_l2_size.
4335 			 */
4336 			arc_change_state(arc_anon, ab, hash_lock);
4337 			arc_hdr_destroy(ab);
4338 		} else {
4339 			/*
4340 			 * Invalidate issued or about to be issued
4341 			 * reads, since we may be about to write
4342 			 * over this location.
4343 			 */
4344 			if (HDR_L2_READING(ab)) {
4345 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
4346 				ab->b_flags |= ARC_L2_EVICTED;
4347 			}
4348 
4349 			/*
4350 			 * Tell ARC this no longer exists in L2ARC.
4351 			 */
4352 			if (ab->b_l2hdr != NULL) {
4353 				abl2 = ab->b_l2hdr;
4354 				ab->b_l2hdr = NULL;
4355 				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4356 				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4357 			}
4358 			list_remove(buflist, ab);
4359 
4360 			/*
4361 			 * This may have been leftover after a
4362 			 * failed write.
4363 			 */
4364 			ab->b_flags &= ~ARC_L2_WRITING;
4365 		}
4366 		mutex_exit(hash_lock);
4367 	}
4368 	mutex_exit(&l2arc_buflist_mtx);
4369 
4370 	vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4371 	dev->l2ad_evict = taddr;
4372 }
4373 
4374 /*
4375  * Find and write ARC buffers to the L2ARC device.
4376  *
4377  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4378  * for reading until they have completed writing.
4379  */
4380 static uint64_t
4381 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
4382 {
4383 	arc_buf_hdr_t *ab, *ab_prev, *head;
4384 	l2arc_buf_hdr_t *hdrl2;
4385 	list_t *list;
4386 	uint64_t passed_sz, write_sz, buf_sz, headroom;
4387 	void *buf_data;
4388 	kmutex_t *hash_lock, *list_lock;
4389 	boolean_t have_lock, full;
4390 	l2arc_write_callback_t *cb;
4391 	zio_t *pio, *wzio;
4392 	uint64_t guid = spa_load_guid(spa);
4393 
4394 	ASSERT(dev->l2ad_vdev != NULL);
4395 
4396 	pio = NULL;
4397 	write_sz = 0;
4398 	full = B_FALSE;
4399 	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4400 	head->b_flags |= ARC_L2_WRITE_HEAD;
4401 
4402 	/*
4403 	 * Copy buffers for L2ARC writing.
4404 	 */
4405 	mutex_enter(&l2arc_buflist_mtx);
4406 	for (int try = 0; try <= 3; try++) {
4407 		list = l2arc_list_locked(try, &list_lock);
4408 		passed_sz = 0;
4409 
4410 		/*
4411 		 * L2ARC fast warmup.
4412 		 *
4413 		 * Until the ARC is warm and starts to evict, read from the
4414 		 * head of the ARC lists rather than the tail.
4415 		 */
4416 		headroom = target_sz * l2arc_headroom;
4417 		if (arc_warm == B_FALSE)
4418 			ab = list_head(list);
4419 		else
4420 			ab = list_tail(list);
4421 
4422 		for (; ab; ab = ab_prev) {
4423 			if (arc_warm == B_FALSE)
4424 				ab_prev = list_next(list, ab);
4425 			else
4426 				ab_prev = list_prev(list, ab);
4427 
4428 			hash_lock = HDR_LOCK(ab);
4429 			have_lock = MUTEX_HELD(hash_lock);
4430 			if (!have_lock && !mutex_tryenter(hash_lock)) {
4431 				/*
4432 				 * Skip this buffer rather than waiting.
4433 				 */
4434 				continue;
4435 			}
4436 
4437 			passed_sz += ab->b_size;
4438 			if (passed_sz > headroom) {
4439 				/*
4440 				 * Searched too far.
4441 				 */
4442 				mutex_exit(hash_lock);
4443 				break;
4444 			}
4445 
4446 			if (!l2arc_write_eligible(guid, ab)) {
4447 				mutex_exit(hash_lock);
4448 				continue;
4449 			}
4450 
4451 			if ((write_sz + ab->b_size) > target_sz) {
4452 				full = B_TRUE;
4453 				mutex_exit(hash_lock);
4454 				break;
4455 			}
4456 
4457 			if (pio == NULL) {
4458 				/*
4459 				 * Insert a dummy header on the buflist so
4460 				 * l2arc_write_done() can find where the
4461 				 * write buffers begin without searching.
4462 				 */
4463 				list_insert_head(dev->l2ad_buflist, head);
4464 
4465 				cb = kmem_alloc(
4466 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
4467 				cb->l2wcb_dev = dev;
4468 				cb->l2wcb_head = head;
4469 				pio = zio_root(spa, l2arc_write_done, cb,
4470 				    ZIO_FLAG_CANFAIL);
4471 			}
4472 
4473 			/*
4474 			 * Create and add a new L2ARC header.
4475 			 */
4476 			hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4477 			hdrl2->b_dev = dev;
4478 			hdrl2->b_daddr = dev->l2ad_hand;
4479 
4480 			ab->b_flags |= ARC_L2_WRITING;
4481 			ab->b_l2hdr = hdrl2;
4482 			list_insert_head(dev->l2ad_buflist, ab);
4483 			buf_data = ab->b_buf->b_data;
4484 			buf_sz = ab->b_size;
4485 
4486 			/*
4487 			 * Compute and store the buffer cksum before
4488 			 * writing.  On debug the cksum is verified first.
4489 			 */
4490 			arc_cksum_verify(ab->b_buf);
4491 			arc_cksum_compute(ab->b_buf, B_TRUE);
4492 
4493 			mutex_exit(hash_lock);
4494 
4495 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
4496 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4497 			    NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4498 			    ZIO_FLAG_CANFAIL, B_FALSE);
4499 
4500 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4501 			    zio_t *, wzio);
4502 			(void) zio_nowait(wzio);
4503 
4504 			/*
4505 			 * Keep the clock hand suitably device-aligned.
4506 			 */
4507 			buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4508 
4509 			write_sz += buf_sz;
4510 			dev->l2ad_hand += buf_sz;
4511 		}
4512 
4513 		mutex_exit(list_lock);
4514 
4515 		if (full == B_TRUE)
4516 			break;
4517 	}
4518 	mutex_exit(&l2arc_buflist_mtx);
4519 
4520 	if (pio == NULL) {
4521 		ASSERT0(write_sz);
4522 		kmem_cache_free(hdr_cache, head);
4523 		return (0);
4524 	}
4525 
4526 	ASSERT3U(write_sz, <=, target_sz);
4527 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
4528 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4529 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
4530 	vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0);
4531 
4532 	/*
4533 	 * Bump device hand to the device start if it is approaching the end.
4534 	 * l2arc_evict() will already have evicted ahead for this case.
4535 	 */
4536 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4537 		vdev_space_update(dev->l2ad_vdev,
4538 		    dev->l2ad_end - dev->l2ad_hand, 0, 0);
4539 		dev->l2ad_hand = dev->l2ad_start;
4540 		dev->l2ad_evict = dev->l2ad_start;
4541 		dev->l2ad_first = B_FALSE;
4542 	}
4543 
4544 	dev->l2ad_writing = B_TRUE;
4545 	(void) zio_wait(pio);
4546 	dev->l2ad_writing = B_FALSE;
4547 
4548 	return (write_sz);
4549 }
4550 
4551 /*
4552  * This thread feeds the L2ARC at regular intervals.  This is the beating
4553  * heart of the L2ARC.
4554  */
4555 static void
4556 l2arc_feed_thread(void)
4557 {
4558 	callb_cpr_t cpr;
4559 	l2arc_dev_t *dev;
4560 	spa_t *spa;
4561 	uint64_t size, wrote;
4562 	clock_t begin, next = ddi_get_lbolt();
4563 
4564 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4565 
4566 	mutex_enter(&l2arc_feed_thr_lock);
4567 
4568 	while (l2arc_thread_exit == 0) {
4569 		CALLB_CPR_SAFE_BEGIN(&cpr);
4570 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
4571 		    next);
4572 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4573 		next = ddi_get_lbolt() + hz;
4574 
4575 		/*
4576 		 * Quick check for L2ARC devices.
4577 		 */
4578 		mutex_enter(&l2arc_dev_mtx);
4579 		if (l2arc_ndev == 0) {
4580 			mutex_exit(&l2arc_dev_mtx);
4581 			continue;
4582 		}
4583 		mutex_exit(&l2arc_dev_mtx);
4584 		begin = ddi_get_lbolt();
4585 
4586 		/*
4587 		 * This selects the next l2arc device to write to, and in
4588 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
4589 		 * will return NULL if there are now no l2arc devices or if
4590 		 * they are all faulted.
4591 		 *
4592 		 * If a device is returned, its spa's config lock is also
4593 		 * held to prevent device removal.  l2arc_dev_get_next()
4594 		 * will grab and release l2arc_dev_mtx.
4595 		 */
4596 		if ((dev = l2arc_dev_get_next()) == NULL)
4597 			continue;
4598 
4599 		spa = dev->l2ad_spa;
4600 		ASSERT(spa != NULL);
4601 
4602 		/*
4603 		 * If the pool is read-only then force the feed thread to
4604 		 * sleep a little longer.
4605 		 */
4606 		if (!spa_writeable(spa)) {
4607 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
4608 			spa_config_exit(spa, SCL_L2ARC, dev);
4609 			continue;
4610 		}
4611 
4612 		/*
4613 		 * Avoid contributing to memory pressure.
4614 		 */
4615 		if (arc_reclaim_needed()) {
4616 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
4617 			spa_config_exit(spa, SCL_L2ARC, dev);
4618 			continue;
4619 		}
4620 
4621 		ARCSTAT_BUMP(arcstat_l2_feeds);
4622 
4623 		size = l2arc_write_size(dev);
4624 
4625 		/*
4626 		 * Evict L2ARC buffers that will be overwritten.
4627 		 */
4628 		l2arc_evict(dev, size, B_FALSE);
4629 
4630 		/*
4631 		 * Write ARC buffers.
4632 		 */
4633 		wrote = l2arc_write_buffers(spa, dev, size);
4634 
4635 		/*
4636 		 * Calculate interval between writes.
4637 		 */
4638 		next = l2arc_write_interval(begin, size, wrote);
4639 		spa_config_exit(spa, SCL_L2ARC, dev);
4640 	}
4641 
4642 	l2arc_thread_exit = 0;
4643 	cv_broadcast(&l2arc_feed_thr_cv);
4644 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
4645 	thread_exit();
4646 }
4647 
4648 boolean_t
4649 l2arc_vdev_present(vdev_t *vd)
4650 {
4651 	l2arc_dev_t *dev;
4652 
4653 	mutex_enter(&l2arc_dev_mtx);
4654 	for (dev = list_head(l2arc_dev_list); dev != NULL;
4655 	    dev = list_next(l2arc_dev_list, dev)) {
4656 		if (dev->l2ad_vdev == vd)
4657 			break;
4658 	}
4659 	mutex_exit(&l2arc_dev_mtx);
4660 
4661 	return (dev != NULL);
4662 }
4663 
4664 /*
4665  * Add a vdev for use by the L2ARC.  By this point the spa has already
4666  * validated the vdev and opened it.
4667  */
4668 void
4669 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
4670 {
4671 	l2arc_dev_t *adddev;
4672 
4673 	ASSERT(!l2arc_vdev_present(vd));
4674 
4675 	/*
4676 	 * Create a new l2arc device entry.
4677 	 */
4678 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
4679 	adddev->l2ad_spa = spa;
4680 	adddev->l2ad_vdev = vd;
4681 	adddev->l2ad_write = l2arc_write_max;
4682 	adddev->l2ad_boost = l2arc_write_boost;
4683 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
4684 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
4685 	adddev->l2ad_hand = adddev->l2ad_start;
4686 	adddev->l2ad_evict = adddev->l2ad_start;
4687 	adddev->l2ad_first = B_TRUE;
4688 	adddev->l2ad_writing = B_FALSE;
4689 	ASSERT3U(adddev->l2ad_write, >, 0);
4690 
4691 	/*
4692 	 * This is a list of all ARC buffers that are still valid on the
4693 	 * device.
4694 	 */
4695 	adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
4696 	list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
4697 	    offsetof(arc_buf_hdr_t, b_l2node));
4698 
4699 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
4700 
4701 	/*
4702 	 * Add device to global list
4703 	 */
4704 	mutex_enter(&l2arc_dev_mtx);
4705 	list_insert_head(l2arc_dev_list, adddev);
4706 	atomic_inc_64(&l2arc_ndev);
4707 	mutex_exit(&l2arc_dev_mtx);
4708 }
4709 
4710 /*
4711  * Remove a vdev from the L2ARC.
4712  */
4713 void
4714 l2arc_remove_vdev(vdev_t *vd)
4715 {
4716 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
4717 
4718 	/*
4719 	 * Find the device by vdev
4720 	 */
4721 	mutex_enter(&l2arc_dev_mtx);
4722 	for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
4723 		nextdev = list_next(l2arc_dev_list, dev);
4724 		if (vd == dev->l2ad_vdev) {
4725 			remdev = dev;
4726 			break;
4727 		}
4728 	}
4729 	ASSERT(remdev != NULL);
4730 
4731 	/*
4732 	 * Remove device from global list
4733 	 */
4734 	list_remove(l2arc_dev_list, remdev);
4735 	l2arc_dev_last = NULL;		/* may have been invalidated */
4736 	atomic_dec_64(&l2arc_ndev);
4737 	mutex_exit(&l2arc_dev_mtx);
4738 
4739 	/*
4740 	 * Clear all buflists and ARC references.  L2ARC device flush.
4741 	 */
4742 	l2arc_evict(remdev, 0, B_TRUE);
4743 	list_destroy(remdev->l2ad_buflist);
4744 	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
4745 	kmem_free(remdev, sizeof (l2arc_dev_t));
4746 }
4747 
4748 void
4749 l2arc_init(void)
4750 {
4751 	l2arc_thread_exit = 0;
4752 	l2arc_ndev = 0;
4753 	l2arc_writes_sent = 0;
4754 	l2arc_writes_done = 0;
4755 
4756 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
4757 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
4758 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
4759 	mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
4760 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
4761 
4762 	l2arc_dev_list = &L2ARC_dev_list;
4763 	l2arc_free_on_write = &L2ARC_free_on_write;
4764 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
4765 	    offsetof(l2arc_dev_t, l2ad_node));
4766 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
4767 	    offsetof(l2arc_data_free_t, l2df_list_node));
4768 }
4769 
4770 void
4771 l2arc_fini(void)
4772 {
4773 	/*
4774 	 * This is called from dmu_fini(), which is called from spa_fini();
4775 	 * Because of this, we can assume that all l2arc devices have
4776 	 * already been removed when the pools themselves were removed.
4777 	 */
4778 
4779 	l2arc_do_free_on_write();
4780 
4781 	mutex_destroy(&l2arc_feed_thr_lock);
4782 	cv_destroy(&l2arc_feed_thr_cv);
4783 	mutex_destroy(&l2arc_dev_mtx);
4784 	mutex_destroy(&l2arc_buflist_mtx);
4785 	mutex_destroy(&l2arc_free_on_write_mtx);
4786 
4787 	list_destroy(l2arc_dev_list);
4788 	list_destroy(l2arc_free_on_write);
4789 }
4790 
4791 void
4792 l2arc_start(void)
4793 {
4794 	if (!(spa_mode_global & FWRITE))
4795 		return;
4796 
4797 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
4798 	    TS_RUN, minclsyspri);
4799 }
4800 
4801 void
4802 l2arc_stop(void)
4803 {
4804 	if (!(spa_mode_global & FWRITE))
4805 		return;
4806 
4807 	mutex_enter(&l2arc_feed_thr_lock);
4808 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
4809 	l2arc_thread_exit = 1;
4810 	while (l2arc_thread_exit != 0)
4811 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
4812 	mutex_exit(&l2arc_feed_thr_lock);
4813 }
4814