1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2020, Delphix. All rights reserved.
25 * Copyright (c) 2014, Saso Kiselkov. All rights reserved.
26 * Copyright (c) 2017, Nexenta Systems, Inc. All rights reserved.
27 * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
28 * Copyright (c) 2020, George Amanakis. All rights reserved.
29 * Copyright (c) 2019, 2024, Klara Inc.
30 * Copyright (c) 2019, Allan Jude
31 * Copyright (c) 2020, The FreeBSD Foundation [1]
32 * Copyright (c) 2021, 2024 by George Melikov. All rights reserved.
33 *
34 * [1] Portions of this software were developed by Allan Jude
35 * under sponsorship from the FreeBSD Foundation.
36 */
37
38 /*
39 * DVA-based Adjustable Replacement Cache
40 *
41 * While much of the theory of operation used here is
42 * based on the self-tuning, low overhead replacement cache
43 * presented by Megiddo and Modha at FAST 2003, there are some
44 * significant differences:
45 *
46 * 1. The Megiddo and Modha model assumes any page is evictable.
47 * Pages in its cache cannot be "locked" into memory. This makes
48 * the eviction algorithm simple: evict the last page in the list.
49 * This also make the performance characteristics easy to reason
50 * about. Our cache is not so simple. At any given moment, some
51 * subset of the blocks in the cache are un-evictable because we
52 * have handed out a reference to them. Blocks are only evictable
53 * when there are no external references active. This makes
54 * eviction far more problematic: we choose to evict the evictable
55 * blocks that are the "lowest" in the list.
56 *
57 * There are times when it is not possible to evict the requested
58 * space. In these circumstances we are unable to adjust the cache
59 * size. To prevent the cache growing unbounded at these times we
60 * implement a "cache throttle" that slows the flow of new data
61 * into the cache until we can make space available.
62 *
63 * 2. The Megiddo and Modha model assumes a fixed cache size.
64 * Pages are evicted when the cache is full and there is a cache
65 * miss. Our model has a variable sized cache. It grows with
66 * high use, but also tries to react to memory pressure from the
67 * operating system: decreasing its size when system memory is
68 * tight.
69 *
70 * 3. The Megiddo and Modha model assumes a fixed page size. All
71 * elements of the cache are therefore exactly the same size. So
72 * when adjusting the cache size following a cache miss, its simply
73 * a matter of choosing a single page to evict. In our model, we
74 * have variable sized cache blocks (ranging from 512 bytes to
75 * 128K bytes). We therefore choose a set of blocks to evict to make
76 * space for a cache miss that approximates as closely as possible
77 * the space used by the new block.
78 *
79 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache"
80 * by N. Megiddo & D. Modha, FAST 2003
81 */
82
83 /*
84 * The locking model:
85 *
86 * A new reference to a cache buffer can be obtained in two
87 * ways: 1) via a hash table lookup using the DVA as a key,
88 * or 2) via one of the ARC lists. The arc_read() interface
89 * uses method 1, while the internal ARC algorithms for
90 * adjusting the cache use method 2. We therefore provide two
91 * types of locks: 1) the hash table lock array, and 2) the
92 * ARC list locks.
93 *
94 * Buffers do not have their own mutexes, rather they rely on the
95 * hash table mutexes for the bulk of their protection (i.e. most
96 * fields in the arc_buf_hdr_t are protected by these mutexes).
97 *
98 * buf_hash_find() returns the appropriate mutex (held) when it
99 * locates the requested buffer in the hash table. It returns
100 * NULL for the mutex if the buffer was not in the table.
101 *
102 * buf_hash_remove() expects the appropriate hash mutex to be
103 * already held before it is invoked.
104 *
105 * Each ARC state also has a mutex which is used to protect the
106 * buffer list associated with the state. When attempting to
107 * obtain a hash table lock while holding an ARC list lock you
108 * must use: mutex_tryenter() to avoid deadlock. Also note that
109 * the active state mutex must be held before the ghost state mutex.
110 *
111 * It as also possible to register a callback which is run when the
112 * metadata limit is reached and no buffers can be safely evicted. In
113 * this case the arc user should drop a reference on some arc buffers so
114 * they can be reclaimed. For example, when using the ZPL each dentry
115 * holds a references on a znode. These dentries must be pruned before
116 * the arc buffer holding the znode can be safely evicted.
117 *
118 * Note that the majority of the performance stats are manipulated
119 * with atomic operations.
120 *
121 * The L2ARC uses the l2ad_mtx on each vdev for the following:
122 *
123 * - L2ARC buflist creation
124 * - L2ARC buflist eviction
125 * - L2ARC write completion, which walks L2ARC buflists
126 * - ARC header destruction, as it removes from L2ARC buflists
127 * - ARC header release, as it removes from L2ARC buflists
128 */
129
130 /*
131 * ARC operation:
132 *
133 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
134 * This structure can point either to a block that is still in the cache or to
135 * one that is only accessible in an L2 ARC device, or it can provide
136 * information about a block that was recently evicted. If a block is
137 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
138 * information to retrieve it from the L2ARC device. This information is
139 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
140 * that is in this state cannot access the data directly.
141 *
142 * Blocks that are actively being referenced or have not been evicted
143 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
144 * the arc_buf_hdr_t that will point to the data block in memory. A block can
145 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
146 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
147 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
148 *
149 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
150 * ability to store the physical data (b_pabd) associated with the DVA of the
151 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
152 * it will match its on-disk compression characteristics. This behavior can be
153 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
154 * compressed ARC functionality is disabled, the b_pabd will point to an
155 * uncompressed version of the on-disk data.
156 *
157 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
158 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
159 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
160 * consumer. The ARC will provide references to this data and will keep it
161 * cached until it is no longer in use. The ARC caches only the L1ARC's physical
162 * data block and will evict any arc_buf_t that is no longer referenced. The
163 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
164 * "overhead_size" kstat.
165 *
166 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
167 * compressed form. The typical case is that consumers will want uncompressed
168 * data, and when that happens a new data buffer is allocated where the data is
169 * decompressed for them to use. Currently the only consumer who wants
170 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
171 * exists on disk. When this happens, the arc_buf_t's data buffer is shared
172 * with the arc_buf_hdr_t.
173 *
174 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
175 * first one is owned by a compressed send consumer (and therefore references
176 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
177 * used by any other consumer (and has its own uncompressed copy of the data
178 * buffer).
179 *
180 * arc_buf_hdr_t
181 * +-----------+
182 * | fields |
183 * | common to |
184 * | L1- and |
185 * | L2ARC |
186 * +-----------+
187 * | l2arc_buf_hdr_t
188 * | |
189 * +-----------+
190 * | l1arc_buf_hdr_t
191 * | | arc_buf_t
192 * | b_buf +------------>+-----------+ arc_buf_t
193 * | b_pabd +-+ |b_next +---->+-----------+
194 * +-----------+ | |-----------| |b_next +-->NULL
195 * | |b_comp = T | +-----------+
196 * | |b_data +-+ |b_comp = F |
197 * | +-----------+ | |b_data +-+
198 * +->+------+ | +-----------+ |
199 * compressed | | | |
200 * data | |<--------------+ | uncompressed
201 * +------+ compressed, | data
202 * shared +-->+------+
203 * data | |
204 * | |
205 * +------+
206 *
207 * When a consumer reads a block, the ARC must first look to see if the
208 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
209 * arc_buf_t and either copies uncompressed data into a new data buffer from an
210 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
211 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
212 * hdr is compressed and the desired compression characteristics of the
213 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
214 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
215 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
216 * be anywhere in the hdr's list.
217 *
218 * The diagram below shows an example of an uncompressed ARC hdr that is
219 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
220 * the last element in the buf list):
221 *
222 * arc_buf_hdr_t
223 * +-----------+
224 * | |
225 * | |
226 * | |
227 * +-----------+
228 * l2arc_buf_hdr_t| |
229 * | |
230 * +-----------+
231 * l1arc_buf_hdr_t| |
232 * | | arc_buf_t (shared)
233 * | b_buf +------------>+---------+ arc_buf_t
234 * | | |b_next +---->+---------+
235 * | b_pabd +-+ |---------| |b_next +-->NULL
236 * +-----------+ | | | +---------+
237 * | |b_data +-+ | |
238 * | +---------+ | |b_data +-+
239 * +->+------+ | +---------+ |
240 * | | | |
241 * uncompressed | | | |
242 * data +------+ | |
243 * ^ +->+------+ |
244 * | uncompressed | | |
245 * | data | | |
246 * | +------+ |
247 * +---------------------------------+
248 *
249 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
250 * since the physical block is about to be rewritten. The new data contents
251 * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
252 * it may compress the data before writing it to disk. The ARC will be called
253 * with the transformed data and will memcpy the transformed on-disk block into
254 * a newly allocated b_pabd. Writes are always done into buffers which have
255 * either been loaned (and hence are new and don't have other readers) or
256 * buffers which have been released (and hence have their own hdr, if there
257 * were originally other readers of the buf's original hdr). This ensures that
258 * the ARC only needs to update a single buf and its hdr after a write occurs.
259 *
260 * When the L2ARC is in use, it will also take advantage of the b_pabd. The
261 * L2ARC will always write the contents of b_pabd to the L2ARC. This means
262 * that when compressed ARC is enabled that the L2ARC blocks are identical
263 * to the on-disk block in the main data pool. This provides a significant
264 * advantage since the ARC can leverage the bp's checksum when reading from the
265 * L2ARC to determine if the contents are valid. However, if the compressed
266 * ARC is disabled, then the L2ARC's block must be transformed to look
267 * like the physical block in the main data pool before comparing the
268 * checksum and determining its validity.
269 *
270 * The L1ARC has a slightly different system for storing encrypted data.
271 * Raw (encrypted + possibly compressed) data has a few subtle differences from
272 * data that is just compressed. The biggest difference is that it is not
273 * possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
274 * The other difference is that encryption cannot be treated as a suggestion.
275 * If a caller would prefer compressed data, but they actually wind up with
276 * uncompressed data the worst thing that could happen is there might be a
277 * performance hit. If the caller requests encrypted data, however, we must be
278 * sure they actually get it or else secret information could be leaked. Raw
279 * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
280 * may have both an encrypted version and a decrypted version of its data at
281 * once. When a caller needs a raw arc_buf_t, it is allocated and the data is
282 * copied out of this header. To avoid complications with b_pabd, raw buffers
283 * cannot be shared.
284 */
285
286 #include <sys/spa.h>
287 #include <sys/zio.h>
288 #include <sys/spa_impl.h>
289 #include <sys/zio_compress.h>
290 #include <sys/zio_checksum.h>
291 #include <sys/zfs_context.h>
292 #include <sys/arc.h>
293 #include <sys/zfs_refcount.h>
294 #include <sys/vdev.h>
295 #include <sys/vdev_impl.h>
296 #include <sys/dsl_pool.h>
297 #include <sys/multilist.h>
298 #include <sys/abd.h>
299 #include <sys/zil.h>
300 #include <sys/fm/fs/zfs.h>
301 #include <sys/callb.h>
302 #include <sys/kstat.h>
303 #include <sys/zthr.h>
304 #include <zfs_fletcher.h>
305 #include <sys/arc_impl.h>
306 #include <sys/trace_zfs.h>
307 #include <sys/aggsum.h>
308 #include <sys/wmsum.h>
309 #include <cityhash.h>
310 #include <sys/vdev_trim.h>
311 #include <sys/zfs_racct.h>
312 #include <sys/zstd/zstd.h>
313
314 #ifndef _KERNEL
315 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
316 boolean_t arc_watch = B_FALSE;
317 #endif
318
319 /*
320 * This thread's job is to keep enough free memory in the system, by
321 * calling arc_kmem_reap_soon() plus arc_reduce_target_size(), which improves
322 * arc_available_memory().
323 */
324 static zthr_t *arc_reap_zthr;
325
326 /*
327 * This thread's job is to keep arc_size under arc_c, by calling
328 * arc_evict(), which improves arc_is_overflowing().
329 */
330 static zthr_t *arc_evict_zthr;
331 static arc_buf_hdr_t **arc_state_evict_markers;
332 static int arc_state_evict_marker_count;
333
334 static kmutex_t arc_evict_lock;
335 static boolean_t arc_evict_needed = B_FALSE;
336 static clock_t arc_last_uncached_flush;
337
338 /*
339 * Count of bytes evicted since boot.
340 */
341 static uint64_t arc_evict_count;
342
343 /*
344 * List of arc_evict_waiter_t's, representing threads waiting for the
345 * arc_evict_count to reach specific values.
346 */
347 static list_t arc_evict_waiters;
348
349 /*
350 * When arc_is_overflowing(), arc_get_data_impl() waits for this percent of
351 * the requested amount of data to be evicted. For example, by default for
352 * every 2KB that's evicted, 1KB of it may be "reused" by a new allocation.
353 * Since this is above 100%, it ensures that progress is made towards getting
354 * arc_size under arc_c. Since this is finite, it ensures that allocations
355 * can still happen, even during the potentially long time that arc_size is
356 * more than arc_c.
357 */
358 static uint_t zfs_arc_eviction_pct = 200;
359
360 /*
361 * The number of headers to evict in arc_evict_state_impl() before
362 * dropping the sublist lock and evicting from another sublist. A lower
363 * value means we're more likely to evict the "correct" header (i.e. the
364 * oldest header in the arc state), but comes with higher overhead
365 * (i.e. more invocations of arc_evict_state_impl()).
366 */
367 static uint_t zfs_arc_evict_batch_limit = 10;
368
369 /* number of seconds before growing cache again */
370 uint_t arc_grow_retry = 5;
371
372 /*
373 * Minimum time between calls to arc_kmem_reap_soon().
374 */
375 static const int arc_kmem_cache_reap_retry_ms = 1000;
376
377 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */
378 static int zfs_arc_overflow_shift = 8;
379
380 /* log2(fraction of arc to reclaim) */
381 uint_t arc_shrink_shift = 7;
382
383 /* percent of pagecache to reclaim arc to */
384 #ifdef _KERNEL
385 uint_t zfs_arc_pc_percent = 0;
386 #endif
387
388 /*
389 * log2(fraction of ARC which must be free to allow growing).
390 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
391 * when reading a new block into the ARC, we will evict an equal-sized block
392 * from the ARC.
393 *
394 * This must be less than arc_shrink_shift, so that when we shrink the ARC,
395 * we will still not allow it to grow.
396 */
397 uint_t arc_no_grow_shift = 5;
398
399
400 /*
401 * minimum lifespan of a prefetch block in clock ticks
402 * (initialized in arc_init())
403 */
404 static uint_t arc_min_prefetch_ms;
405 static uint_t arc_min_prescient_prefetch_ms;
406
407 /*
408 * If this percent of memory is free, don't throttle.
409 */
410 uint_t arc_lotsfree_percent = 10;
411
412 /*
413 * The arc has filled available memory and has now warmed up.
414 */
415 boolean_t arc_warm;
416
417 /*
418 * These tunables are for performance analysis.
419 */
420 uint64_t zfs_arc_max = 0;
421 uint64_t zfs_arc_min = 0;
422 static uint64_t zfs_arc_dnode_limit = 0;
423 static uint_t zfs_arc_dnode_reduce_percent = 10;
424 static uint_t zfs_arc_grow_retry = 0;
425 static uint_t zfs_arc_shrink_shift = 0;
426 uint_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
427
428 /*
429 * ARC dirty data constraints for arc_tempreserve_space() throttle:
430 * * total dirty data limit
431 * * anon block dirty limit
432 * * each pool's anon allowance
433 */
434 static const unsigned long zfs_arc_dirty_limit_percent = 50;
435 static const unsigned long zfs_arc_anon_limit_percent = 25;
436 static const unsigned long zfs_arc_pool_dirty_percent = 20;
437
438 /*
439 * Enable or disable compressed arc buffers.
440 */
441 int zfs_compressed_arc_enabled = B_TRUE;
442
443 /*
444 * Balance between metadata and data on ghost hits. Values above 100
445 * increase metadata caching by proportionally reducing effect of ghost
446 * data hits on target data/metadata rate.
447 */
448 static uint_t zfs_arc_meta_balance = 500;
449
450 /*
451 * Percentage that can be consumed by dnodes of ARC meta buffers.
452 */
453 static uint_t zfs_arc_dnode_limit_percent = 10;
454
455 /*
456 * These tunables are Linux-specific
457 */
458 static uint64_t zfs_arc_sys_free = 0;
459 static uint_t zfs_arc_min_prefetch_ms = 0;
460 static uint_t zfs_arc_min_prescient_prefetch_ms = 0;
461 static uint_t zfs_arc_lotsfree_percent = 10;
462
463 /*
464 * Number of arc_prune threads
465 */
466 static int zfs_arc_prune_task_threads = 1;
467
468 /* Used by spa_export/spa_destroy to flush the arc asynchronously */
469 static taskq_t *arc_flush_taskq;
470
471 /* The 7 states: */
472 arc_state_t ARC_anon;
473 arc_state_t ARC_mru;
474 arc_state_t ARC_mru_ghost;
475 arc_state_t ARC_mfu;
476 arc_state_t ARC_mfu_ghost;
477 arc_state_t ARC_l2c_only;
478 arc_state_t ARC_uncached;
479
480 arc_stats_t arc_stats = {
481 { "hits", KSTAT_DATA_UINT64 },
482 { "iohits", KSTAT_DATA_UINT64 },
483 { "misses", KSTAT_DATA_UINT64 },
484 { "demand_data_hits", KSTAT_DATA_UINT64 },
485 { "demand_data_iohits", KSTAT_DATA_UINT64 },
486 { "demand_data_misses", KSTAT_DATA_UINT64 },
487 { "demand_metadata_hits", KSTAT_DATA_UINT64 },
488 { "demand_metadata_iohits", KSTAT_DATA_UINT64 },
489 { "demand_metadata_misses", KSTAT_DATA_UINT64 },
490 { "prefetch_data_hits", KSTAT_DATA_UINT64 },
491 { "prefetch_data_iohits", KSTAT_DATA_UINT64 },
492 { "prefetch_data_misses", KSTAT_DATA_UINT64 },
493 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 },
494 { "prefetch_metadata_iohits", KSTAT_DATA_UINT64 },
495 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 },
496 { "mru_hits", KSTAT_DATA_UINT64 },
497 { "mru_ghost_hits", KSTAT_DATA_UINT64 },
498 { "mfu_hits", KSTAT_DATA_UINT64 },
499 { "mfu_ghost_hits", KSTAT_DATA_UINT64 },
500 { "uncached_hits", KSTAT_DATA_UINT64 },
501 { "deleted", KSTAT_DATA_UINT64 },
502 { "mutex_miss", KSTAT_DATA_UINT64 },
503 { "access_skip", KSTAT_DATA_UINT64 },
504 { "evict_skip", KSTAT_DATA_UINT64 },
505 { "evict_not_enough", KSTAT_DATA_UINT64 },
506 { "evict_l2_cached", KSTAT_DATA_UINT64 },
507 { "evict_l2_eligible", KSTAT_DATA_UINT64 },
508 { "evict_l2_eligible_mfu", KSTAT_DATA_UINT64 },
509 { "evict_l2_eligible_mru", KSTAT_DATA_UINT64 },
510 { "evict_l2_ineligible", KSTAT_DATA_UINT64 },
511 { "evict_l2_skip", KSTAT_DATA_UINT64 },
512 { "hash_elements", KSTAT_DATA_UINT64 },
513 { "hash_elements_max", KSTAT_DATA_UINT64 },
514 { "hash_collisions", KSTAT_DATA_UINT64 },
515 { "hash_chains", KSTAT_DATA_UINT64 },
516 { "hash_chain_max", KSTAT_DATA_UINT64 },
517 { "meta", KSTAT_DATA_UINT64 },
518 { "pd", KSTAT_DATA_UINT64 },
519 { "pm", KSTAT_DATA_UINT64 },
520 { "c", KSTAT_DATA_UINT64 },
521 { "c_min", KSTAT_DATA_UINT64 },
522 { "c_max", KSTAT_DATA_UINT64 },
523 { "size", KSTAT_DATA_UINT64 },
524 { "compressed_size", KSTAT_DATA_UINT64 },
525 { "uncompressed_size", KSTAT_DATA_UINT64 },
526 { "overhead_size", KSTAT_DATA_UINT64 },
527 { "hdr_size", KSTAT_DATA_UINT64 },
528 { "data_size", KSTAT_DATA_UINT64 },
529 { "metadata_size", KSTAT_DATA_UINT64 },
530 { "dbuf_size", KSTAT_DATA_UINT64 },
531 { "dnode_size", KSTAT_DATA_UINT64 },
532 { "bonus_size", KSTAT_DATA_UINT64 },
533 #if defined(COMPAT_FREEBSD11)
534 { "other_size", KSTAT_DATA_UINT64 },
535 #endif
536 { "anon_size", KSTAT_DATA_UINT64 },
537 { "anon_data", KSTAT_DATA_UINT64 },
538 { "anon_metadata", KSTAT_DATA_UINT64 },
539 { "anon_evictable_data", KSTAT_DATA_UINT64 },
540 { "anon_evictable_metadata", KSTAT_DATA_UINT64 },
541 { "mru_size", KSTAT_DATA_UINT64 },
542 { "mru_data", KSTAT_DATA_UINT64 },
543 { "mru_metadata", KSTAT_DATA_UINT64 },
544 { "mru_evictable_data", KSTAT_DATA_UINT64 },
545 { "mru_evictable_metadata", KSTAT_DATA_UINT64 },
546 { "mru_ghost_size", KSTAT_DATA_UINT64 },
547 { "mru_ghost_data", KSTAT_DATA_UINT64 },
548 { "mru_ghost_metadata", KSTAT_DATA_UINT64 },
549 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 },
550 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
551 { "mfu_size", KSTAT_DATA_UINT64 },
552 { "mfu_data", KSTAT_DATA_UINT64 },
553 { "mfu_metadata", KSTAT_DATA_UINT64 },
554 { "mfu_evictable_data", KSTAT_DATA_UINT64 },
555 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 },
556 { "mfu_ghost_size", KSTAT_DATA_UINT64 },
557 { "mfu_ghost_data", KSTAT_DATA_UINT64 },
558 { "mfu_ghost_metadata", KSTAT_DATA_UINT64 },
559 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 },
560 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
561 { "uncached_size", KSTAT_DATA_UINT64 },
562 { "uncached_data", KSTAT_DATA_UINT64 },
563 { "uncached_metadata", KSTAT_DATA_UINT64 },
564 { "uncached_evictable_data", KSTAT_DATA_UINT64 },
565 { "uncached_evictable_metadata", KSTAT_DATA_UINT64 },
566 { "l2_hits", KSTAT_DATA_UINT64 },
567 { "l2_misses", KSTAT_DATA_UINT64 },
568 { "l2_prefetch_asize", KSTAT_DATA_UINT64 },
569 { "l2_mru_asize", KSTAT_DATA_UINT64 },
570 { "l2_mfu_asize", KSTAT_DATA_UINT64 },
571 { "l2_bufc_data_asize", KSTAT_DATA_UINT64 },
572 { "l2_bufc_metadata_asize", KSTAT_DATA_UINT64 },
573 { "l2_feeds", KSTAT_DATA_UINT64 },
574 { "l2_rw_clash", KSTAT_DATA_UINT64 },
575 { "l2_read_bytes", KSTAT_DATA_UINT64 },
576 { "l2_write_bytes", KSTAT_DATA_UINT64 },
577 { "l2_writes_sent", KSTAT_DATA_UINT64 },
578 { "l2_writes_done", KSTAT_DATA_UINT64 },
579 { "l2_writes_error", KSTAT_DATA_UINT64 },
580 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 },
581 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 },
582 { "l2_evict_reading", KSTAT_DATA_UINT64 },
583 { "l2_evict_l1cached", KSTAT_DATA_UINT64 },
584 { "l2_free_on_write", KSTAT_DATA_UINT64 },
585 { "l2_abort_lowmem", KSTAT_DATA_UINT64 },
586 { "l2_cksum_bad", KSTAT_DATA_UINT64 },
587 { "l2_io_error", KSTAT_DATA_UINT64 },
588 { "l2_size", KSTAT_DATA_UINT64 },
589 { "l2_asize", KSTAT_DATA_UINT64 },
590 { "l2_hdr_size", KSTAT_DATA_UINT64 },
591 { "l2_log_blk_writes", KSTAT_DATA_UINT64 },
592 { "l2_log_blk_avg_asize", KSTAT_DATA_UINT64 },
593 { "l2_log_blk_asize", KSTAT_DATA_UINT64 },
594 { "l2_log_blk_count", KSTAT_DATA_UINT64 },
595 { "l2_data_to_meta_ratio", KSTAT_DATA_UINT64 },
596 { "l2_rebuild_success", KSTAT_DATA_UINT64 },
597 { "l2_rebuild_unsupported", KSTAT_DATA_UINT64 },
598 { "l2_rebuild_io_errors", KSTAT_DATA_UINT64 },
599 { "l2_rebuild_dh_errors", KSTAT_DATA_UINT64 },
600 { "l2_rebuild_cksum_lb_errors", KSTAT_DATA_UINT64 },
601 { "l2_rebuild_lowmem", KSTAT_DATA_UINT64 },
602 { "l2_rebuild_size", KSTAT_DATA_UINT64 },
603 { "l2_rebuild_asize", KSTAT_DATA_UINT64 },
604 { "l2_rebuild_bufs", KSTAT_DATA_UINT64 },
605 { "l2_rebuild_bufs_precached", KSTAT_DATA_UINT64 },
606 { "l2_rebuild_log_blks", KSTAT_DATA_UINT64 },
607 { "memory_throttle_count", KSTAT_DATA_UINT64 },
608 { "memory_direct_count", KSTAT_DATA_UINT64 },
609 { "memory_indirect_count", KSTAT_DATA_UINT64 },
610 { "memory_all_bytes", KSTAT_DATA_UINT64 },
611 { "memory_free_bytes", KSTAT_DATA_UINT64 },
612 { "memory_available_bytes", KSTAT_DATA_INT64 },
613 { "arc_no_grow", KSTAT_DATA_UINT64 },
614 { "arc_tempreserve", KSTAT_DATA_UINT64 },
615 { "arc_loaned_bytes", KSTAT_DATA_UINT64 },
616 { "arc_prune", KSTAT_DATA_UINT64 },
617 { "arc_meta_used", KSTAT_DATA_UINT64 },
618 { "arc_dnode_limit", KSTAT_DATA_UINT64 },
619 { "async_upgrade_sync", KSTAT_DATA_UINT64 },
620 { "predictive_prefetch", KSTAT_DATA_UINT64 },
621 { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
622 { "demand_iohit_predictive_prefetch", KSTAT_DATA_UINT64 },
623 { "prescient_prefetch", KSTAT_DATA_UINT64 },
624 { "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
625 { "demand_iohit_prescient_prefetch", KSTAT_DATA_UINT64 },
626 { "arc_need_free", KSTAT_DATA_UINT64 },
627 { "arc_sys_free", KSTAT_DATA_UINT64 },
628 { "arc_raw_size", KSTAT_DATA_UINT64 },
629 { "cached_only_in_progress", KSTAT_DATA_UINT64 },
630 { "abd_chunk_waste_size", KSTAT_DATA_UINT64 },
631 };
632
633 arc_sums_t arc_sums;
634
635 #define ARCSTAT_MAX(stat, val) { \
636 uint64_t m; \
637 while ((val) > (m = arc_stats.stat.value.ui64) && \
638 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \
639 continue; \
640 }
641
642 /*
643 * We define a macro to allow ARC hits/misses to be easily broken down by
644 * two separate conditions, giving a total of four different subtypes for
645 * each of hits and misses (so eight statistics total).
646 */
647 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
648 if (cond1) { \
649 if (cond2) { \
650 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
651 } else { \
652 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
653 } \
654 } else { \
655 if (cond2) { \
656 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
657 } else { \
658 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
659 } \
660 }
661
662 /*
663 * This macro allows us to use kstats as floating averages. Each time we
664 * update this kstat, we first factor it and the update value by
665 * ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall
666 * average. This macro assumes that integer loads and stores are atomic, but
667 * is not safe for multiple writers updating the kstat in parallel (only the
668 * last writer's update will remain).
669 */
670 #define ARCSTAT_F_AVG_FACTOR 3
671 #define ARCSTAT_F_AVG(stat, value) \
672 do { \
673 uint64_t x = ARCSTAT(stat); \
674 x = x - x / ARCSTAT_F_AVG_FACTOR + \
675 (value) / ARCSTAT_F_AVG_FACTOR; \
676 ARCSTAT(stat) = x; \
677 } while (0)
678
679 static kstat_t *arc_ksp;
680
681 /*
682 * There are several ARC variables that are critical to export as kstats --
683 * but we don't want to have to grovel around in the kstat whenever we wish to
684 * manipulate them. For these variables, we therefore define them to be in
685 * terms of the statistic variable. This assures that we are not introducing
686 * the possibility of inconsistency by having shadow copies of the variables,
687 * while still allowing the code to be readable.
688 */
689 #define arc_tempreserve ARCSTAT(arcstat_tempreserve)
690 #define arc_loaned_bytes ARCSTAT(arcstat_loaned_bytes)
691 #define arc_dnode_limit ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
692 #define arc_need_free ARCSTAT(arcstat_need_free) /* waiting to be evicted */
693
694 hrtime_t arc_growtime;
695 list_t arc_prune_list;
696 kmutex_t arc_prune_mtx;
697 taskq_t *arc_prune_taskq;
698
699 #define GHOST_STATE(state) \
700 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \
701 (state) == arc_l2c_only)
702
703 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
704 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
705 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
706 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
707 #define HDR_PRESCIENT_PREFETCH(hdr) \
708 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
709 #define HDR_COMPRESSION_ENABLED(hdr) \
710 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
711
712 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
713 #define HDR_UNCACHED(hdr) ((hdr)->b_flags & ARC_FLAG_UNCACHED)
714 #define HDR_L2_READING(hdr) \
715 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
716 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
717 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
718 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
719 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
720 #define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED)
721 #define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH)
722 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
723
724 #define HDR_ISTYPE_METADATA(hdr) \
725 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
726 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
727
728 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
729 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
730 #define HDR_HAS_RABD(hdr) \
731 (HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \
732 (hdr)->b_crypt_hdr.b_rabd != NULL)
733 #define HDR_ENCRYPTED(hdr) \
734 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
735 #define HDR_AUTHENTICATED(hdr) \
736 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
737
738 /* For storing compression mode in b_flags */
739 #define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1)
740
741 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
742 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
743 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
744 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
745
746 #define ARC_BUF_LAST(buf) ((buf)->b_next == NULL)
747 #define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED)
748 #define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
749 #define ARC_BUF_ENCRYPTED(buf) ((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)
750
751 /*
752 * Other sizes
753 */
754
755 #define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
756 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
757
758 /*
759 * Hash table routines
760 */
761
762 #define BUF_LOCKS 2048
763 typedef struct buf_hash_table {
764 uint64_t ht_mask;
765 arc_buf_hdr_t **ht_table;
766 kmutex_t ht_locks[BUF_LOCKS] ____cacheline_aligned;
767 } buf_hash_table_t;
768
769 static buf_hash_table_t buf_hash_table;
770
771 #define BUF_HASH_INDEX(spa, dva, birth) \
772 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
773 #define BUF_HASH_LOCK(idx) (&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
774 #define HDR_LOCK(hdr) \
775 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
776
777 uint64_t zfs_crc64_table[256];
778
779 /*
780 * Asynchronous ARC flush
781 *
782 * We track these in a list for arc_async_flush_guid_inuse().
783 * Used for both L1 and L2 async teardown.
784 */
785 static list_t arc_async_flush_list;
786 static kmutex_t arc_async_flush_lock;
787
788 typedef struct arc_async_flush {
789 uint64_t af_spa_guid;
790 taskq_ent_t af_tqent;
791 uint_t af_cache_level; /* 1 or 2 to differentiate node */
792 list_node_t af_node;
793 } arc_async_flush_t;
794
795
796 /*
797 * Level 2 ARC
798 */
799
800 #define L2ARC_WRITE_SIZE (32 * 1024 * 1024) /* initial write max */
801 #define L2ARC_HEADROOM 8 /* num of writes */
802
803 /*
804 * If we discover during ARC scan any buffers to be compressed, we boost
805 * our headroom for the next scanning cycle by this percentage multiple.
806 */
807 #define L2ARC_HEADROOM_BOOST 200
808 #define L2ARC_FEED_SECS 1 /* caching interval secs */
809 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */
810
811 /*
812 * We can feed L2ARC from two states of ARC buffers, mru and mfu,
813 * and each of the state has two types: data and metadata.
814 */
815 #define L2ARC_FEED_TYPES 4
816
817 /* L2ARC Performance Tunables */
818 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* def max write size */
819 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra warmup write */
820 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* # of dev writes */
821 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
822 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */
823 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval msecs */
824 int l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */
825 int l2arc_feed_again = B_TRUE; /* turbo warmup */
826 int l2arc_norw = B_FALSE; /* no reads during writes */
827 static uint_t l2arc_meta_percent = 33; /* limit on headers size */
828
829 /*
830 * L2ARC Internals
831 */
832 static list_t L2ARC_dev_list; /* device list */
833 static list_t *l2arc_dev_list; /* device list pointer */
834 static kmutex_t l2arc_dev_mtx; /* device list mutex */
835 static l2arc_dev_t *l2arc_dev_last; /* last device used */
836 static list_t L2ARC_free_on_write; /* free after write buf list */
837 static list_t *l2arc_free_on_write; /* free after write list ptr */
838 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */
839 static uint64_t l2arc_ndev; /* number of devices */
840
841 typedef struct l2arc_read_callback {
842 arc_buf_hdr_t *l2rcb_hdr; /* read header */
843 blkptr_t l2rcb_bp; /* original blkptr */
844 zbookmark_phys_t l2rcb_zb; /* original bookmark */
845 int l2rcb_flags; /* original flags */
846 abd_t *l2rcb_abd; /* temporary buffer */
847 } l2arc_read_callback_t;
848
849 typedef struct l2arc_data_free {
850 /* protected by l2arc_free_on_write_mtx */
851 abd_t *l2df_abd;
852 size_t l2df_size;
853 arc_buf_contents_t l2df_type;
854 list_node_t l2df_list_node;
855 } l2arc_data_free_t;
856
857 typedef enum arc_fill_flags {
858 ARC_FILL_LOCKED = 1 << 0, /* hdr lock is held */
859 ARC_FILL_COMPRESSED = 1 << 1, /* fill with compressed data */
860 ARC_FILL_ENCRYPTED = 1 << 2, /* fill with encrypted data */
861 ARC_FILL_NOAUTH = 1 << 3, /* don't attempt to authenticate */
862 ARC_FILL_IN_PLACE = 1 << 4 /* fill in place (special case) */
863 } arc_fill_flags_t;
864
865 typedef enum arc_ovf_level {
866 ARC_OVF_NONE, /* ARC within target size. */
867 ARC_OVF_SOME, /* ARC is slightly overflowed. */
868 ARC_OVF_SEVERE /* ARC is severely overflowed. */
869 } arc_ovf_level_t;
870
871 static kmutex_t l2arc_feed_thr_lock;
872 static kcondvar_t l2arc_feed_thr_cv;
873 static uint8_t l2arc_thread_exit;
874
875 static kmutex_t l2arc_rebuild_thr_lock;
876 static kcondvar_t l2arc_rebuild_thr_cv;
877
878 enum arc_hdr_alloc_flags {
879 ARC_HDR_ALLOC_RDATA = 0x1,
880 ARC_HDR_USE_RESERVE = 0x4,
881 ARC_HDR_ALLOC_LINEAR = 0x8,
882 };
883
884
885 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, const void *, int);
886 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, const void *);
887 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, const void *, int);
888 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, const void *);
889 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, const void *);
890 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
891 const void *tag);
892 static void arc_hdr_free_abd(arc_buf_hdr_t *, boolean_t);
893 static void arc_hdr_alloc_abd(arc_buf_hdr_t *, int);
894 static void arc_hdr_destroy(arc_buf_hdr_t *);
895 static void arc_access(arc_buf_hdr_t *, arc_flags_t, boolean_t);
896 static void arc_buf_watch(arc_buf_t *);
897 static void arc_change_state(arc_state_t *, arc_buf_hdr_t *);
898
899 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
900 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
901 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
902 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
903
904 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
905 static void l2arc_read_done(zio_t *);
906 static void l2arc_do_free_on_write(void);
907 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
908 boolean_t state_only);
909
910 static void arc_prune_async(uint64_t adjust);
911
912 #define l2arc_hdr_arcstats_increment(hdr) \
913 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
914 #define l2arc_hdr_arcstats_decrement(hdr) \
915 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
916 #define l2arc_hdr_arcstats_increment_state(hdr) \
917 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
918 #define l2arc_hdr_arcstats_decrement_state(hdr) \
919 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
920
921 /*
922 * l2arc_exclude_special : A zfs module parameter that controls whether buffers
923 * present on special vdevs are eligibile for caching in L2ARC. If
924 * set to 1, exclude dbufs on special vdevs from being cached to
925 * L2ARC.
926 */
927 int l2arc_exclude_special = 0;
928
929 /*
930 * l2arc_mfuonly : A ZFS module parameter that controls whether only MFU
931 * metadata and data are cached from ARC into L2ARC.
932 */
933 static int l2arc_mfuonly = 0;
934
935 /*
936 * L2ARC TRIM
937 * l2arc_trim_ahead : A ZFS module parameter that controls how much ahead of
938 * the current write size (l2arc_write_max) we should TRIM if we
939 * have filled the device. It is defined as a percentage of the
940 * write size. If set to 100 we trim twice the space required to
941 * accommodate upcoming writes. A minimum of 64MB will be trimmed.
942 * It also enables TRIM of the whole L2ARC device upon creation or
943 * addition to an existing pool or if the header of the device is
944 * invalid upon importing a pool or onlining a cache device. The
945 * default is 0, which disables TRIM on L2ARC altogether as it can
946 * put significant stress on the underlying storage devices. This
947 * will vary depending of how well the specific device handles
948 * these commands.
949 */
950 static uint64_t l2arc_trim_ahead = 0;
951
952 /*
953 * Performance tuning of L2ARC persistence:
954 *
955 * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
956 * an L2ARC device (either at pool import or later) will attempt
957 * to rebuild L2ARC buffer contents.
958 * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
959 * whether log blocks are written to the L2ARC device. If the L2ARC
960 * device is less than 1GB, the amount of data l2arc_evict()
961 * evicts is significant compared to the amount of restored L2ARC
962 * data. In this case do not write log blocks in L2ARC in order
963 * not to waste space.
964 */
965 static int l2arc_rebuild_enabled = B_TRUE;
966 static uint64_t l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
967
968 /* L2ARC persistence rebuild control routines. */
969 void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
970 static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
971 static int l2arc_rebuild(l2arc_dev_t *dev);
972
973 /* L2ARC persistence read I/O routines. */
974 static int l2arc_dev_hdr_read(l2arc_dev_t *dev);
975 static int l2arc_log_blk_read(l2arc_dev_t *dev,
976 const l2arc_log_blkptr_t *this_lp, const l2arc_log_blkptr_t *next_lp,
977 l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
978 zio_t *this_io, zio_t **next_io);
979 static zio_t *l2arc_log_blk_fetch(vdev_t *vd,
980 const l2arc_log_blkptr_t *lp, l2arc_log_blk_phys_t *lb);
981 static void l2arc_log_blk_fetch_abort(zio_t *zio);
982
983 /* L2ARC persistence block restoration routines. */
984 static void l2arc_log_blk_restore(l2arc_dev_t *dev,
985 const l2arc_log_blk_phys_t *lb, uint64_t lb_asize);
986 static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le,
987 l2arc_dev_t *dev);
988
989 /* L2ARC persistence write I/O routines. */
990 static uint64_t l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio,
991 l2arc_write_callback_t *cb);
992
993 /* L2ARC persistence auxiliary routines. */
994 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
995 const l2arc_log_blkptr_t *lbp);
996 static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev,
997 const arc_buf_hdr_t *ab);
998 boolean_t l2arc_range_check_overlap(uint64_t bottom,
999 uint64_t top, uint64_t check);
1000 static void l2arc_blk_fetch_done(zio_t *zio);
1001 static inline uint64_t
1002 l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev);
1003
1004 /*
1005 * We use Cityhash for this. It's fast, and has good hash properties without
1006 * requiring any large static buffers.
1007 */
1008 static uint64_t
buf_hash(uint64_t spa,const dva_t * dva,uint64_t birth)1009 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
1010 {
1011 return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
1012 }
1013
1014 #define HDR_EMPTY(hdr) \
1015 ((hdr)->b_dva.dva_word[0] == 0 && \
1016 (hdr)->b_dva.dva_word[1] == 0)
1017
1018 #define HDR_EMPTY_OR_LOCKED(hdr) \
1019 (HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
1020
1021 #define HDR_EQUAL(spa, dva, birth, hdr) \
1022 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1023 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1024 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1025
1026 static void
buf_discard_identity(arc_buf_hdr_t * hdr)1027 buf_discard_identity(arc_buf_hdr_t *hdr)
1028 {
1029 hdr->b_dva.dva_word[0] = 0;
1030 hdr->b_dva.dva_word[1] = 0;
1031 hdr->b_birth = 0;
1032 }
1033
1034 static arc_buf_hdr_t *
buf_hash_find(uint64_t spa,const blkptr_t * bp,kmutex_t ** lockp)1035 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
1036 {
1037 const dva_t *dva = BP_IDENTITY(bp);
1038 uint64_t birth = BP_GET_BIRTH(bp);
1039 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
1040 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1041 arc_buf_hdr_t *hdr;
1042
1043 mutex_enter(hash_lock);
1044 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1045 hdr = hdr->b_hash_next) {
1046 if (HDR_EQUAL(spa, dva, birth, hdr)) {
1047 *lockp = hash_lock;
1048 return (hdr);
1049 }
1050 }
1051 mutex_exit(hash_lock);
1052 *lockp = NULL;
1053 return (NULL);
1054 }
1055
1056 /*
1057 * Insert an entry into the hash table. If there is already an element
1058 * equal to elem in the hash table, then the already existing element
1059 * will be returned and the new element will not be inserted.
1060 * Otherwise returns NULL.
1061 * If lockp == NULL, the caller is assumed to already hold the hash lock.
1062 */
1063 static arc_buf_hdr_t *
buf_hash_insert(arc_buf_hdr_t * hdr,kmutex_t ** lockp)1064 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1065 {
1066 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1067 kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1068 arc_buf_hdr_t *fhdr;
1069 uint32_t i;
1070
1071 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1072 ASSERT(hdr->b_birth != 0);
1073 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1074
1075 if (lockp != NULL) {
1076 *lockp = hash_lock;
1077 mutex_enter(hash_lock);
1078 } else {
1079 ASSERT(MUTEX_HELD(hash_lock));
1080 }
1081
1082 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1083 fhdr = fhdr->b_hash_next, i++) {
1084 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1085 return (fhdr);
1086 }
1087
1088 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1089 buf_hash_table.ht_table[idx] = hdr;
1090 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1091
1092 /* collect some hash table performance data */
1093 if (i > 0) {
1094 ARCSTAT_BUMP(arcstat_hash_collisions);
1095 if (i == 1)
1096 ARCSTAT_BUMP(arcstat_hash_chains);
1097 ARCSTAT_MAX(arcstat_hash_chain_max, i);
1098 }
1099 ARCSTAT_BUMP(arcstat_hash_elements);
1100
1101 return (NULL);
1102 }
1103
1104 static void
buf_hash_remove(arc_buf_hdr_t * hdr)1105 buf_hash_remove(arc_buf_hdr_t *hdr)
1106 {
1107 arc_buf_hdr_t *fhdr, **hdrp;
1108 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1109
1110 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1111 ASSERT(HDR_IN_HASH_TABLE(hdr));
1112
1113 hdrp = &buf_hash_table.ht_table[idx];
1114 while ((fhdr = *hdrp) != hdr) {
1115 ASSERT3P(fhdr, !=, NULL);
1116 hdrp = &fhdr->b_hash_next;
1117 }
1118 *hdrp = hdr->b_hash_next;
1119 hdr->b_hash_next = NULL;
1120 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1121
1122 /* collect some hash table performance data */
1123 ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1124 if (buf_hash_table.ht_table[idx] &&
1125 buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1126 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1127 }
1128
1129 /*
1130 * Global data structures and functions for the buf kmem cache.
1131 */
1132
1133 static kmem_cache_t *hdr_full_cache;
1134 static kmem_cache_t *hdr_l2only_cache;
1135 static kmem_cache_t *buf_cache;
1136
1137 static void
buf_fini(void)1138 buf_fini(void)
1139 {
1140 #if defined(_KERNEL)
1141 /*
1142 * Large allocations which do not require contiguous pages
1143 * should be using vmem_free() in the linux kernel\
1144 */
1145 vmem_free(buf_hash_table.ht_table,
1146 (buf_hash_table.ht_mask + 1) * sizeof (void *));
1147 #else
1148 kmem_free(buf_hash_table.ht_table,
1149 (buf_hash_table.ht_mask + 1) * sizeof (void *));
1150 #endif
1151 for (int i = 0; i < BUF_LOCKS; i++)
1152 mutex_destroy(BUF_HASH_LOCK(i));
1153 kmem_cache_destroy(hdr_full_cache);
1154 kmem_cache_destroy(hdr_l2only_cache);
1155 kmem_cache_destroy(buf_cache);
1156 }
1157
1158 /*
1159 * Constructor callback - called when the cache is empty
1160 * and a new buf is requested.
1161 */
1162 static int
hdr_full_cons(void * vbuf,void * unused,int kmflag)1163 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1164 {
1165 (void) unused, (void) kmflag;
1166 arc_buf_hdr_t *hdr = vbuf;
1167
1168 memset(hdr, 0, HDR_FULL_SIZE);
1169 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
1170 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1171 #ifdef ZFS_DEBUG
1172 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1173 #endif
1174 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1175 list_link_init(&hdr->b_l2hdr.b_l2node);
1176 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1177
1178 return (0);
1179 }
1180
1181 static int
hdr_l2only_cons(void * vbuf,void * unused,int kmflag)1182 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1183 {
1184 (void) unused, (void) kmflag;
1185 arc_buf_hdr_t *hdr = vbuf;
1186
1187 memset(hdr, 0, HDR_L2ONLY_SIZE);
1188 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1189
1190 return (0);
1191 }
1192
1193 static int
buf_cons(void * vbuf,void * unused,int kmflag)1194 buf_cons(void *vbuf, void *unused, int kmflag)
1195 {
1196 (void) unused, (void) kmflag;
1197 arc_buf_t *buf = vbuf;
1198
1199 memset(buf, 0, sizeof (arc_buf_t));
1200 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1201
1202 return (0);
1203 }
1204
1205 /*
1206 * Destructor callback - called when a cached buf is
1207 * no longer required.
1208 */
1209 static void
hdr_full_dest(void * vbuf,void * unused)1210 hdr_full_dest(void *vbuf, void *unused)
1211 {
1212 (void) unused;
1213 arc_buf_hdr_t *hdr = vbuf;
1214
1215 ASSERT(HDR_EMPTY(hdr));
1216 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1217 #ifdef ZFS_DEBUG
1218 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1219 #endif
1220 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1221 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1222 }
1223
1224 static void
hdr_l2only_dest(void * vbuf,void * unused)1225 hdr_l2only_dest(void *vbuf, void *unused)
1226 {
1227 (void) unused;
1228 arc_buf_hdr_t *hdr = vbuf;
1229
1230 ASSERT(HDR_EMPTY(hdr));
1231 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1232 }
1233
1234 static void
buf_dest(void * vbuf,void * unused)1235 buf_dest(void *vbuf, void *unused)
1236 {
1237 (void) unused;
1238 (void) vbuf;
1239
1240 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1241 }
1242
1243 static void
buf_init(void)1244 buf_init(void)
1245 {
1246 uint64_t *ct = NULL;
1247 uint64_t hsize = 1ULL << 12;
1248 int i, j;
1249
1250 /*
1251 * The hash table is big enough to fill all of physical memory
1252 * with an average block size of zfs_arc_average_blocksize (default 8K).
1253 * By default, the table will take up
1254 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1255 */
1256 while (hsize * zfs_arc_average_blocksize < arc_all_memory())
1257 hsize <<= 1;
1258 retry:
1259 buf_hash_table.ht_mask = hsize - 1;
1260 #if defined(_KERNEL)
1261 /*
1262 * Large allocations which do not require contiguous pages
1263 * should be using vmem_alloc() in the linux kernel
1264 */
1265 buf_hash_table.ht_table =
1266 vmem_zalloc(hsize * sizeof (void*), KM_SLEEP);
1267 #else
1268 buf_hash_table.ht_table =
1269 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1270 #endif
1271 if (buf_hash_table.ht_table == NULL) {
1272 ASSERT(hsize > (1ULL << 8));
1273 hsize >>= 1;
1274 goto retry;
1275 }
1276
1277 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1278 0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
1279 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1280 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL,
1281 NULL, NULL, 0);
1282 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1283 0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1284
1285 for (i = 0; i < 256; i++)
1286 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1287 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1288
1289 for (i = 0; i < BUF_LOCKS; i++)
1290 mutex_init(BUF_HASH_LOCK(i), NULL, MUTEX_DEFAULT, NULL);
1291 }
1292
1293 #define ARC_MINTIME (hz>>4) /* 62 ms */
1294
1295 /*
1296 * This is the size that the buf occupies in memory. If the buf is compressed,
1297 * it will correspond to the compressed size. You should use this method of
1298 * getting the buf size unless you explicitly need the logical size.
1299 */
1300 uint64_t
arc_buf_size(arc_buf_t * buf)1301 arc_buf_size(arc_buf_t *buf)
1302 {
1303 return (ARC_BUF_COMPRESSED(buf) ?
1304 HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1305 }
1306
1307 uint64_t
arc_buf_lsize(arc_buf_t * buf)1308 arc_buf_lsize(arc_buf_t *buf)
1309 {
1310 return (HDR_GET_LSIZE(buf->b_hdr));
1311 }
1312
1313 /*
1314 * This function will return B_TRUE if the buffer is encrypted in memory.
1315 * This buffer can be decrypted by calling arc_untransform().
1316 */
1317 boolean_t
arc_is_encrypted(arc_buf_t * buf)1318 arc_is_encrypted(arc_buf_t *buf)
1319 {
1320 return (ARC_BUF_ENCRYPTED(buf) != 0);
1321 }
1322
1323 /*
1324 * Returns B_TRUE if the buffer represents data that has not had its MAC
1325 * verified yet.
1326 */
1327 boolean_t
arc_is_unauthenticated(arc_buf_t * buf)1328 arc_is_unauthenticated(arc_buf_t *buf)
1329 {
1330 return (HDR_NOAUTH(buf->b_hdr) != 0);
1331 }
1332
1333 void
arc_get_raw_params(arc_buf_t * buf,boolean_t * byteorder,uint8_t * salt,uint8_t * iv,uint8_t * mac)1334 arc_get_raw_params(arc_buf_t *buf, boolean_t *byteorder, uint8_t *salt,
1335 uint8_t *iv, uint8_t *mac)
1336 {
1337 arc_buf_hdr_t *hdr = buf->b_hdr;
1338
1339 ASSERT(HDR_PROTECTED(hdr));
1340
1341 memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
1342 memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
1343 memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
1344 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
1345 ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
1346 }
1347
1348 /*
1349 * Indicates how this buffer is compressed in memory. If it is not compressed
1350 * the value will be ZIO_COMPRESS_OFF. It can be made normally readable with
1351 * arc_untransform() as long as it is also unencrypted.
1352 */
1353 enum zio_compress
arc_get_compression(arc_buf_t * buf)1354 arc_get_compression(arc_buf_t *buf)
1355 {
1356 return (ARC_BUF_COMPRESSED(buf) ?
1357 HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1358 }
1359
1360 /*
1361 * Return the compression algorithm used to store this data in the ARC. If ARC
1362 * compression is enabled or this is an encrypted block, this will be the same
1363 * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
1364 */
1365 static inline enum zio_compress
arc_hdr_get_compress(arc_buf_hdr_t * hdr)1366 arc_hdr_get_compress(arc_buf_hdr_t *hdr)
1367 {
1368 return (HDR_COMPRESSION_ENABLED(hdr) ?
1369 HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
1370 }
1371
1372 uint8_t
arc_get_complevel(arc_buf_t * buf)1373 arc_get_complevel(arc_buf_t *buf)
1374 {
1375 return (buf->b_hdr->b_complevel);
1376 }
1377
1378 static inline boolean_t
arc_buf_is_shared(arc_buf_t * buf)1379 arc_buf_is_shared(arc_buf_t *buf)
1380 {
1381 boolean_t shared = (buf->b_data != NULL &&
1382 buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1383 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1384 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1385 IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1386 EQUIV(shared, ARC_BUF_SHARED(buf));
1387 IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
1388
1389 /*
1390 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1391 * already being shared" requirement prevents us from doing that.
1392 */
1393
1394 return (shared);
1395 }
1396
1397 /*
1398 * Free the checksum associated with this header. If there is no checksum, this
1399 * is a no-op.
1400 */
1401 static inline void
arc_cksum_free(arc_buf_hdr_t * hdr)1402 arc_cksum_free(arc_buf_hdr_t *hdr)
1403 {
1404 #ifdef ZFS_DEBUG
1405 ASSERT(HDR_HAS_L1HDR(hdr));
1406
1407 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1408 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1409 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1410 hdr->b_l1hdr.b_freeze_cksum = NULL;
1411 }
1412 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1413 #endif
1414 }
1415
1416 /*
1417 * Return true iff at least one of the bufs on hdr is not compressed.
1418 * Encrypted buffers count as compressed.
1419 */
1420 static boolean_t
arc_hdr_has_uncompressed_buf(arc_buf_hdr_t * hdr)1421 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1422 {
1423 ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
1424
1425 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1426 if (!ARC_BUF_COMPRESSED(b)) {
1427 return (B_TRUE);
1428 }
1429 }
1430 return (B_FALSE);
1431 }
1432
1433
1434 /*
1435 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
1436 * matches the checksum that is stored in the hdr. If there is no checksum,
1437 * or if the buf is compressed, this is a no-op.
1438 */
1439 static void
arc_cksum_verify(arc_buf_t * buf)1440 arc_cksum_verify(arc_buf_t *buf)
1441 {
1442 #ifdef ZFS_DEBUG
1443 arc_buf_hdr_t *hdr = buf->b_hdr;
1444 zio_cksum_t zc;
1445
1446 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1447 return;
1448
1449 if (ARC_BUF_COMPRESSED(buf))
1450 return;
1451
1452 ASSERT(HDR_HAS_L1HDR(hdr));
1453
1454 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1455
1456 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1457 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1458 return;
1459 }
1460
1461 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1462 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1463 panic("buffer modified while frozen!");
1464 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1465 #endif
1466 }
1467
1468 /*
1469 * This function makes the assumption that data stored in the L2ARC
1470 * will be transformed exactly as it is in the main pool. Because of
1471 * this we can verify the checksum against the reading process's bp.
1472 */
1473 static boolean_t
arc_cksum_is_equal(arc_buf_hdr_t * hdr,zio_t * zio)1474 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1475 {
1476 ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1477 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1478
1479 /*
1480 * Block pointers always store the checksum for the logical data.
1481 * If the block pointer has the gang bit set, then the checksum
1482 * it represents is for the reconstituted data and not for an
1483 * individual gang member. The zio pipeline, however, must be able to
1484 * determine the checksum of each of the gang constituents so it
1485 * treats the checksum comparison differently than what we need
1486 * for l2arc blocks. This prevents us from using the
1487 * zio_checksum_error() interface directly. Instead we must call the
1488 * zio_checksum_error_impl() so that we can ensure the checksum is
1489 * generated using the correct checksum algorithm and accounts for the
1490 * logical I/O size and not just a gang fragment.
1491 */
1492 return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1493 BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1494 zio->io_offset, NULL) == 0);
1495 }
1496
1497 /*
1498 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
1499 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1500 * isn't modified later on. If buf is compressed or there is already a checksum
1501 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1502 */
1503 static void
arc_cksum_compute(arc_buf_t * buf)1504 arc_cksum_compute(arc_buf_t *buf)
1505 {
1506 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1507 return;
1508
1509 #ifdef ZFS_DEBUG
1510 arc_buf_hdr_t *hdr = buf->b_hdr;
1511 ASSERT(HDR_HAS_L1HDR(hdr));
1512 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1513 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
1514 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1515 return;
1516 }
1517
1518 ASSERT(!ARC_BUF_ENCRYPTED(buf));
1519 ASSERT(!ARC_BUF_COMPRESSED(buf));
1520 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1521 KM_SLEEP);
1522 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1523 hdr->b_l1hdr.b_freeze_cksum);
1524 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1525 #endif
1526 arc_buf_watch(buf);
1527 }
1528
1529 #ifndef _KERNEL
1530 void
arc_buf_sigsegv(int sig,siginfo_t * si,void * unused)1531 arc_buf_sigsegv(int sig, siginfo_t *si, void *unused)
1532 {
1533 (void) sig, (void) unused;
1534 panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
1535 }
1536 #endif
1537
1538 static void
arc_buf_unwatch(arc_buf_t * buf)1539 arc_buf_unwatch(arc_buf_t *buf)
1540 {
1541 #ifndef _KERNEL
1542 if (arc_watch) {
1543 ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1544 PROT_READ | PROT_WRITE));
1545 }
1546 #else
1547 (void) buf;
1548 #endif
1549 }
1550
1551 static void
arc_buf_watch(arc_buf_t * buf)1552 arc_buf_watch(arc_buf_t *buf)
1553 {
1554 #ifndef _KERNEL
1555 if (arc_watch)
1556 ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1557 PROT_READ));
1558 #else
1559 (void) buf;
1560 #endif
1561 }
1562
1563 static arc_buf_contents_t
arc_buf_type(arc_buf_hdr_t * hdr)1564 arc_buf_type(arc_buf_hdr_t *hdr)
1565 {
1566 arc_buf_contents_t type;
1567 if (HDR_ISTYPE_METADATA(hdr)) {
1568 type = ARC_BUFC_METADATA;
1569 } else {
1570 type = ARC_BUFC_DATA;
1571 }
1572 VERIFY3U(hdr->b_type, ==, type);
1573 return (type);
1574 }
1575
1576 boolean_t
arc_is_metadata(arc_buf_t * buf)1577 arc_is_metadata(arc_buf_t *buf)
1578 {
1579 return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
1580 }
1581
1582 static uint32_t
arc_bufc_to_flags(arc_buf_contents_t type)1583 arc_bufc_to_flags(arc_buf_contents_t type)
1584 {
1585 switch (type) {
1586 case ARC_BUFC_DATA:
1587 /* metadata field is 0 if buffer contains normal data */
1588 return (0);
1589 case ARC_BUFC_METADATA:
1590 return (ARC_FLAG_BUFC_METADATA);
1591 default:
1592 break;
1593 }
1594 panic("undefined ARC buffer type!");
1595 return ((uint32_t)-1);
1596 }
1597
1598 void
arc_buf_thaw(arc_buf_t * buf)1599 arc_buf_thaw(arc_buf_t *buf)
1600 {
1601 arc_buf_hdr_t *hdr = buf->b_hdr;
1602
1603 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1604 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1605
1606 arc_cksum_verify(buf);
1607
1608 /*
1609 * Compressed buffers do not manipulate the b_freeze_cksum.
1610 */
1611 if (ARC_BUF_COMPRESSED(buf))
1612 return;
1613
1614 ASSERT(HDR_HAS_L1HDR(hdr));
1615 arc_cksum_free(hdr);
1616 arc_buf_unwatch(buf);
1617 }
1618
1619 void
arc_buf_freeze(arc_buf_t * buf)1620 arc_buf_freeze(arc_buf_t *buf)
1621 {
1622 if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1623 return;
1624
1625 if (ARC_BUF_COMPRESSED(buf))
1626 return;
1627
1628 ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
1629 arc_cksum_compute(buf);
1630 }
1631
1632 /*
1633 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
1634 * the following functions should be used to ensure that the flags are
1635 * updated in a thread-safe way. When manipulating the flags either
1636 * the hash_lock must be held or the hdr must be undiscoverable. This
1637 * ensures that we're not racing with any other threads when updating
1638 * the flags.
1639 */
1640 static inline void
arc_hdr_set_flags(arc_buf_hdr_t * hdr,arc_flags_t flags)1641 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1642 {
1643 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1644 hdr->b_flags |= flags;
1645 }
1646
1647 static inline void
arc_hdr_clear_flags(arc_buf_hdr_t * hdr,arc_flags_t flags)1648 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1649 {
1650 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1651 hdr->b_flags &= ~flags;
1652 }
1653
1654 /*
1655 * Setting the compression bits in the arc_buf_hdr_t's b_flags is
1656 * done in a special way since we have to clear and set bits
1657 * at the same time. Consumers that wish to set the compression bits
1658 * must use this function to ensure that the flags are updated in
1659 * thread-safe manner.
1660 */
1661 static void
arc_hdr_set_compress(arc_buf_hdr_t * hdr,enum zio_compress cmp)1662 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
1663 {
1664 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1665
1666 /*
1667 * Holes and embedded blocks will always have a psize = 0 so
1668 * we ignore the compression of the blkptr and set the
1669 * want to uncompress them. Mark them as uncompressed.
1670 */
1671 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
1672 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1673 ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
1674 } else {
1675 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1676 ASSERT(HDR_COMPRESSION_ENABLED(hdr));
1677 }
1678
1679 HDR_SET_COMPRESS(hdr, cmp);
1680 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
1681 }
1682
1683 /*
1684 * Looks for another buf on the same hdr which has the data decompressed, copies
1685 * from it, and returns true. If no such buf exists, returns false.
1686 */
1687 static boolean_t
arc_buf_try_copy_decompressed_data(arc_buf_t * buf)1688 arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
1689 {
1690 arc_buf_hdr_t *hdr = buf->b_hdr;
1691 boolean_t copied = B_FALSE;
1692
1693 ASSERT(HDR_HAS_L1HDR(hdr));
1694 ASSERT3P(buf->b_data, !=, NULL);
1695 ASSERT(!ARC_BUF_COMPRESSED(buf));
1696
1697 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1698 from = from->b_next) {
1699 /* can't use our own data buffer */
1700 if (from == buf) {
1701 continue;
1702 }
1703
1704 if (!ARC_BUF_COMPRESSED(from)) {
1705 memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
1706 copied = B_TRUE;
1707 break;
1708 }
1709 }
1710
1711 #ifdef ZFS_DEBUG
1712 /*
1713 * There were no decompressed bufs, so there should not be a
1714 * checksum on the hdr either.
1715 */
1716 if (zfs_flags & ZFS_DEBUG_MODIFY)
1717 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1718 #endif
1719
1720 return (copied);
1721 }
1722
1723 /*
1724 * Allocates an ARC buf header that's in an evicted & L2-cached state.
1725 * This is used during l2arc reconstruction to make empty ARC buffers
1726 * which circumvent the regular disk->arc->l2arc path and instead come
1727 * into being in the reverse order, i.e. l2arc->arc.
1728 */
1729 static arc_buf_hdr_t *
arc_buf_alloc_l2only(size_t size,arc_buf_contents_t type,l2arc_dev_t * dev,dva_t dva,uint64_t daddr,int32_t psize,uint64_t asize,uint64_t birth,enum zio_compress compress,uint8_t complevel,boolean_t protected,boolean_t prefetch,arc_state_type_t arcs_state)1730 arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
1731 dva_t dva, uint64_t daddr, int32_t psize, uint64_t asize, uint64_t birth,
1732 enum zio_compress compress, uint8_t complevel, boolean_t protected,
1733 boolean_t prefetch, arc_state_type_t arcs_state)
1734 {
1735 arc_buf_hdr_t *hdr;
1736
1737 ASSERT(size != 0);
1738 ASSERT(dev->l2ad_vdev != NULL);
1739
1740 hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
1741 hdr->b_birth = birth;
1742 hdr->b_type = type;
1743 hdr->b_flags = 0;
1744 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);
1745 HDR_SET_LSIZE(hdr, size);
1746 HDR_SET_PSIZE(hdr, psize);
1747 HDR_SET_L2SIZE(hdr, asize);
1748 arc_hdr_set_compress(hdr, compress);
1749 hdr->b_complevel = complevel;
1750 if (protected)
1751 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
1752 if (prefetch)
1753 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
1754 hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
1755
1756 hdr->b_dva = dva;
1757
1758 hdr->b_l2hdr.b_dev = dev;
1759 hdr->b_l2hdr.b_daddr = daddr;
1760 hdr->b_l2hdr.b_arcs_state = arcs_state;
1761
1762 return (hdr);
1763 }
1764
1765 /*
1766 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
1767 */
1768 static uint64_t
arc_hdr_size(arc_buf_hdr_t * hdr)1769 arc_hdr_size(arc_buf_hdr_t *hdr)
1770 {
1771 uint64_t size;
1772
1773 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
1774 HDR_GET_PSIZE(hdr) > 0) {
1775 size = HDR_GET_PSIZE(hdr);
1776 } else {
1777 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
1778 size = HDR_GET_LSIZE(hdr);
1779 }
1780 return (size);
1781 }
1782
1783 static int
arc_hdr_authenticate(arc_buf_hdr_t * hdr,spa_t * spa,uint64_t dsobj)1784 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
1785 {
1786 int ret;
1787 uint64_t csize;
1788 uint64_t lsize = HDR_GET_LSIZE(hdr);
1789 uint64_t psize = HDR_GET_PSIZE(hdr);
1790 abd_t *abd = hdr->b_l1hdr.b_pabd;
1791 boolean_t free_abd = B_FALSE;
1792
1793 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1794 ASSERT(HDR_AUTHENTICATED(hdr));
1795 ASSERT3P(abd, !=, NULL);
1796
1797 /*
1798 * The MAC is calculated on the compressed data that is stored on disk.
1799 * However, if compressed arc is disabled we will only have the
1800 * decompressed data available to us now. Compress it into a temporary
1801 * abd so we can verify the MAC. The performance overhead of this will
1802 * be relatively low, since most objects in an encrypted objset will
1803 * be encrypted (instead of authenticated) anyway.
1804 */
1805 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1806 !HDR_COMPRESSION_ENABLED(hdr)) {
1807 abd = NULL;
1808 csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
1809 hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize),
1810 hdr->b_complevel);
1811 if (csize >= lsize || csize > psize) {
1812 ret = SET_ERROR(EIO);
1813 return (ret);
1814 }
1815 ASSERT3P(abd, !=, NULL);
1816 abd_zero_off(abd, csize, psize - csize);
1817 free_abd = B_TRUE;
1818 }
1819
1820 /*
1821 * Authentication is best effort. We authenticate whenever the key is
1822 * available. If we succeed we clear ARC_FLAG_NOAUTH.
1823 */
1824 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
1825 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1826 ASSERT3U(lsize, ==, psize);
1827 ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd,
1828 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1829 } else {
1830 ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize,
1831 hdr->b_crypt_hdr.b_mac);
1832 }
1833
1834 if (ret == 0)
1835 arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
1836 else if (ret == ENOENT)
1837 ret = 0;
1838
1839 if (free_abd)
1840 abd_free(abd);
1841
1842 return (ret);
1843 }
1844
1845 /*
1846 * This function will take a header that only has raw encrypted data in
1847 * b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in
1848 * b_l1hdr.b_pabd. If designated in the header flags, this function will
1849 * also decompress the data.
1850 */
1851 static int
arc_hdr_decrypt(arc_buf_hdr_t * hdr,spa_t * spa,const zbookmark_phys_t * zb)1852 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
1853 {
1854 int ret;
1855 abd_t *cabd = NULL;
1856 boolean_t no_crypt = B_FALSE;
1857 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1858
1859 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1860 ASSERT(HDR_ENCRYPTED(hdr));
1861
1862 arc_hdr_alloc_abd(hdr, 0);
1863
1864 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
1865 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
1866 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
1867 hdr->b_crypt_hdr.b_rabd, &no_crypt);
1868 if (ret != 0)
1869 goto error;
1870
1871 if (no_crypt) {
1872 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
1873 HDR_GET_PSIZE(hdr));
1874 }
1875
1876 /*
1877 * If this header has disabled arc compression but the b_pabd is
1878 * compressed after decrypting it, we need to decompress the newly
1879 * decrypted data.
1880 */
1881 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1882 !HDR_COMPRESSION_ENABLED(hdr)) {
1883 /*
1884 * We want to make sure that we are correctly honoring the
1885 * zfs_abd_scatter_enabled setting, so we allocate an abd here
1886 * and then loan a buffer from it, rather than allocating a
1887 * linear buffer and wrapping it in an abd later.
1888 */
1889 cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
1890
1891 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
1892 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
1893 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
1894 if (ret != 0) {
1895 goto error;
1896 }
1897
1898 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
1899 arc_hdr_size(hdr), hdr);
1900 hdr->b_l1hdr.b_pabd = cabd;
1901 }
1902
1903 return (0);
1904
1905 error:
1906 arc_hdr_free_abd(hdr, B_FALSE);
1907 if (cabd != NULL)
1908 arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);
1909
1910 return (ret);
1911 }
1912
1913 /*
1914 * This function is called during arc_buf_fill() to prepare the header's
1915 * abd plaintext pointer for use. This involves authenticated protected
1916 * data and decrypting encrypted data into the plaintext abd.
1917 */
1918 static int
arc_fill_hdr_crypt(arc_buf_hdr_t * hdr,kmutex_t * hash_lock,spa_t * spa,const zbookmark_phys_t * zb,boolean_t noauth)1919 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
1920 const zbookmark_phys_t *zb, boolean_t noauth)
1921 {
1922 int ret;
1923
1924 ASSERT(HDR_PROTECTED(hdr));
1925
1926 if (hash_lock != NULL)
1927 mutex_enter(hash_lock);
1928
1929 if (HDR_NOAUTH(hdr) && !noauth) {
1930 /*
1931 * The caller requested authenticated data but our data has
1932 * not been authenticated yet. Verify the MAC now if we can.
1933 */
1934 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
1935 if (ret != 0)
1936 goto error;
1937 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
1938 /*
1939 * If we only have the encrypted version of the data, but the
1940 * unencrypted version was requested we take this opportunity
1941 * to store the decrypted version in the header for future use.
1942 */
1943 ret = arc_hdr_decrypt(hdr, spa, zb);
1944 if (ret != 0)
1945 goto error;
1946 }
1947
1948 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1949
1950 if (hash_lock != NULL)
1951 mutex_exit(hash_lock);
1952
1953 return (0);
1954
1955 error:
1956 if (hash_lock != NULL)
1957 mutex_exit(hash_lock);
1958
1959 return (ret);
1960 }
1961
1962 /*
1963 * This function is used by the dbuf code to decrypt bonus buffers in place.
1964 * The dbuf code itself doesn't have any locking for decrypting a shared dnode
1965 * block, so we use the hash lock here to protect against concurrent calls to
1966 * arc_buf_fill().
1967 */
1968 static void
arc_buf_untransform_in_place(arc_buf_t * buf)1969 arc_buf_untransform_in_place(arc_buf_t *buf)
1970 {
1971 arc_buf_hdr_t *hdr = buf->b_hdr;
1972
1973 ASSERT(HDR_ENCRYPTED(hdr));
1974 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
1975 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1976 ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);
1977
1978 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
1979 arc_buf_size(buf));
1980 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
1981 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
1982 }
1983
1984 /*
1985 * Given a buf that has a data buffer attached to it, this function will
1986 * efficiently fill the buf with data of the specified compression setting from
1987 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
1988 * are already sharing a data buf, no copy is performed.
1989 *
1990 * If the buf is marked as compressed but uncompressed data was requested, this
1991 * will allocate a new data buffer for the buf, remove that flag, and fill the
1992 * buf with uncompressed data. You can't request a compressed buf on a hdr with
1993 * uncompressed data, and (since we haven't added support for it yet) if you
1994 * want compressed data your buf must already be marked as compressed and have
1995 * the correct-sized data buffer.
1996 */
1997 static int
arc_buf_fill(arc_buf_t * buf,spa_t * spa,const zbookmark_phys_t * zb,arc_fill_flags_t flags)1998 arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
1999 arc_fill_flags_t flags)
2000 {
2001 int error = 0;
2002 arc_buf_hdr_t *hdr = buf->b_hdr;
2003 boolean_t hdr_compressed =
2004 (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
2005 boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0;
2006 boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0;
2007 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2008 kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);
2009
2010 ASSERT3P(buf->b_data, !=, NULL);
2011 IMPLY(compressed, hdr_compressed || ARC_BUF_ENCRYPTED(buf));
2012 IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
2013 IMPLY(encrypted, HDR_ENCRYPTED(hdr));
2014 IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf));
2015 IMPLY(encrypted, ARC_BUF_COMPRESSED(buf));
2016 IMPLY(encrypted, !arc_buf_is_shared(buf));
2017
2018 /*
2019 * If the caller wanted encrypted data we just need to copy it from
2020 * b_rabd and potentially byteswap it. We won't be able to do any
2021 * further transforms on it.
2022 */
2023 if (encrypted) {
2024 ASSERT(HDR_HAS_RABD(hdr));
2025 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
2026 HDR_GET_PSIZE(hdr));
2027 goto byteswap;
2028 }
2029
2030 /*
2031 * Adjust encrypted and authenticated headers to accommodate
2032 * the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are
2033 * allowed to fail decryption due to keys not being loaded
2034 * without being marked as an IO error.
2035 */
2036 if (HDR_PROTECTED(hdr)) {
2037 error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
2038 zb, !!(flags & ARC_FILL_NOAUTH));
2039 if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) {
2040 return (error);
2041 } else if (error != 0) {
2042 if (hash_lock != NULL)
2043 mutex_enter(hash_lock);
2044 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2045 if (hash_lock != NULL)
2046 mutex_exit(hash_lock);
2047 return (error);
2048 }
2049 }
2050
2051 /*
2052 * There is a special case here for dnode blocks which are
2053 * decrypting their bonus buffers. These blocks may request to
2054 * be decrypted in-place. This is necessary because there may
2055 * be many dnodes pointing into this buffer and there is
2056 * currently no method to synchronize replacing the backing
2057 * b_data buffer and updating all of the pointers. Here we use
2058 * the hash lock to ensure there are no races. If the need
2059 * arises for other types to be decrypted in-place, they must
2060 * add handling here as well.
2061 */
2062 if ((flags & ARC_FILL_IN_PLACE) != 0) {
2063 ASSERT(!hdr_compressed);
2064 ASSERT(!compressed);
2065 ASSERT(!encrypted);
2066
2067 if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
2068 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
2069
2070 if (hash_lock != NULL)
2071 mutex_enter(hash_lock);
2072 arc_buf_untransform_in_place(buf);
2073 if (hash_lock != NULL)
2074 mutex_exit(hash_lock);
2075
2076 /* Compute the hdr's checksum if necessary */
2077 arc_cksum_compute(buf);
2078 }
2079
2080 return (0);
2081 }
2082
2083 if (hdr_compressed == compressed) {
2084 if (ARC_BUF_SHARED(buf)) {
2085 ASSERT(arc_buf_is_shared(buf));
2086 } else {
2087 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2088 arc_buf_size(buf));
2089 }
2090 } else {
2091 ASSERT(hdr_compressed);
2092 ASSERT(!compressed);
2093
2094 /*
2095 * If the buf is sharing its data with the hdr, unlink it and
2096 * allocate a new data buffer for the buf.
2097 */
2098 if (ARC_BUF_SHARED(buf)) {
2099 ASSERTF(ARC_BUF_COMPRESSED(buf),
2100 "buf %p was uncompressed", buf);
2101
2102 /* We need to give the buf its own b_data */
2103 buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2104 buf->b_data =
2105 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2106 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2107
2108 /* Previously overhead was 0; just add new overhead */
2109 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2110 } else if (ARC_BUF_COMPRESSED(buf)) {
2111 ASSERT(!arc_buf_is_shared(buf));
2112
2113 /* We need to reallocate the buf's b_data */
2114 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2115 buf);
2116 buf->b_data =
2117 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2118
2119 /* We increased the size of b_data; update overhead */
2120 ARCSTAT_INCR(arcstat_overhead_size,
2121 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2122 }
2123
2124 /*
2125 * Regardless of the buf's previous compression settings, it
2126 * should not be compressed at the end of this function.
2127 */
2128 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2129
2130 /*
2131 * Try copying the data from another buf which already has a
2132 * decompressed version. If that's not possible, it's time to
2133 * bite the bullet and decompress the data from the hdr.
2134 */
2135 if (arc_buf_try_copy_decompressed_data(buf)) {
2136 /* Skip byteswapping and checksumming (already done) */
2137 return (0);
2138 } else {
2139 abd_t dabd;
2140 abd_get_from_buf_struct(&dabd, buf->b_data,
2141 HDR_GET_LSIZE(hdr));
2142 error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2143 hdr->b_l1hdr.b_pabd, &dabd,
2144 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
2145 &hdr->b_complevel);
2146 abd_free(&dabd);
2147
2148 /*
2149 * Absent hardware errors or software bugs, this should
2150 * be impossible, but log it anyway so we can debug it.
2151 */
2152 if (error != 0) {
2153 zfs_dbgmsg(
2154 "hdr %px, compress %d, psize %d, lsize %d",
2155 hdr, arc_hdr_get_compress(hdr),
2156 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2157 if (hash_lock != NULL)
2158 mutex_enter(hash_lock);
2159 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2160 if (hash_lock != NULL)
2161 mutex_exit(hash_lock);
2162 return (SET_ERROR(EIO));
2163 }
2164 }
2165 }
2166
2167 byteswap:
2168 /* Byteswap the buf's data if necessary */
2169 if (bswap != DMU_BSWAP_NUMFUNCS) {
2170 ASSERT(!HDR_SHARED_DATA(hdr));
2171 ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
2172 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2173 }
2174
2175 /* Compute the hdr's checksum if necessary */
2176 arc_cksum_compute(buf);
2177
2178 return (0);
2179 }
2180
2181 /*
2182 * If this function is being called to decrypt an encrypted buffer or verify an
2183 * authenticated one, the key must be loaded and a mapping must be made
2184 * available in the keystore via spa_keystore_create_mapping() or one of its
2185 * callers.
2186 */
2187 int
arc_untransform(arc_buf_t * buf,spa_t * spa,const zbookmark_phys_t * zb,boolean_t in_place)2188 arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
2189 boolean_t in_place)
2190 {
2191 int ret;
2192 arc_fill_flags_t flags = 0;
2193
2194 if (in_place)
2195 flags |= ARC_FILL_IN_PLACE;
2196
2197 ret = arc_buf_fill(buf, spa, zb, flags);
2198 if (ret == ECKSUM) {
2199 /*
2200 * Convert authentication and decryption errors to EIO
2201 * (and generate an ereport) before leaving the ARC.
2202 */
2203 ret = SET_ERROR(EIO);
2204 spa_log_error(spa, zb, buf->b_hdr->b_birth);
2205 (void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
2206 spa, NULL, zb, NULL, 0);
2207 }
2208
2209 return (ret);
2210 }
2211
2212 /*
2213 * Increment the amount of evictable space in the arc_state_t's refcount.
2214 * We account for the space used by the hdr and the arc buf individually
2215 * so that we can add and remove them from the refcount individually.
2216 */
2217 static void
arc_evictable_space_increment(arc_buf_hdr_t * hdr,arc_state_t * state)2218 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2219 {
2220 arc_buf_contents_t type = arc_buf_type(hdr);
2221
2222 ASSERT(HDR_HAS_L1HDR(hdr));
2223
2224 if (GHOST_STATE(state)) {
2225 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2226 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2227 ASSERT(!HDR_HAS_RABD(hdr));
2228 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2229 HDR_GET_LSIZE(hdr), hdr);
2230 return;
2231 }
2232
2233 if (hdr->b_l1hdr.b_pabd != NULL) {
2234 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2235 arc_hdr_size(hdr), hdr);
2236 }
2237 if (HDR_HAS_RABD(hdr)) {
2238 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2239 HDR_GET_PSIZE(hdr), hdr);
2240 }
2241
2242 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2243 buf = buf->b_next) {
2244 if (ARC_BUF_SHARED(buf))
2245 continue;
2246 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2247 arc_buf_size(buf), buf);
2248 }
2249 }
2250
2251 /*
2252 * Decrement the amount of evictable space in the arc_state_t's refcount.
2253 * We account for the space used by the hdr and the arc buf individually
2254 * so that we can add and remove them from the refcount individually.
2255 */
2256 static void
arc_evictable_space_decrement(arc_buf_hdr_t * hdr,arc_state_t * state)2257 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2258 {
2259 arc_buf_contents_t type = arc_buf_type(hdr);
2260
2261 ASSERT(HDR_HAS_L1HDR(hdr));
2262
2263 if (GHOST_STATE(state)) {
2264 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2265 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2266 ASSERT(!HDR_HAS_RABD(hdr));
2267 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2268 HDR_GET_LSIZE(hdr), hdr);
2269 return;
2270 }
2271
2272 if (hdr->b_l1hdr.b_pabd != NULL) {
2273 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2274 arc_hdr_size(hdr), hdr);
2275 }
2276 if (HDR_HAS_RABD(hdr)) {
2277 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2278 HDR_GET_PSIZE(hdr), hdr);
2279 }
2280
2281 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2282 buf = buf->b_next) {
2283 if (ARC_BUF_SHARED(buf))
2284 continue;
2285 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2286 arc_buf_size(buf), buf);
2287 }
2288 }
2289
2290 /*
2291 * Add a reference to this hdr indicating that someone is actively
2292 * referencing that memory. When the refcount transitions from 0 to 1,
2293 * we remove it from the respective arc_state_t list to indicate that
2294 * it is not evictable.
2295 */
2296 static void
add_reference(arc_buf_hdr_t * hdr,const void * tag)2297 add_reference(arc_buf_hdr_t *hdr, const void *tag)
2298 {
2299 arc_state_t *state = hdr->b_l1hdr.b_state;
2300
2301 ASSERT(HDR_HAS_L1HDR(hdr));
2302 if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
2303 ASSERT(state == arc_anon);
2304 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2305 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2306 }
2307
2308 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2309 state != arc_anon && state != arc_l2c_only) {
2310 /* We don't use the L2-only state list. */
2311 multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
2312 arc_evictable_space_decrement(hdr, state);
2313 }
2314 }
2315
2316 /*
2317 * Remove a reference from this hdr. When the reference transitions from
2318 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2319 * list making it eligible for eviction.
2320 */
2321 static int
remove_reference(arc_buf_hdr_t * hdr,const void * tag)2322 remove_reference(arc_buf_hdr_t *hdr, const void *tag)
2323 {
2324 int cnt;
2325 arc_state_t *state = hdr->b_l1hdr.b_state;
2326
2327 ASSERT(HDR_HAS_L1HDR(hdr));
2328 ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));
2329 ASSERT(!GHOST_STATE(state)); /* arc_l2c_only counts as a ghost. */
2330
2331 if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
2332 return (cnt);
2333
2334 if (state == arc_anon) {
2335 arc_hdr_destroy(hdr);
2336 return (0);
2337 }
2338 if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
2339 arc_change_state(arc_anon, hdr);
2340 arc_hdr_destroy(hdr);
2341 return (0);
2342 }
2343 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2344 arc_evictable_space_increment(hdr, state);
2345 return (0);
2346 }
2347
2348 /*
2349 * Returns detailed information about a specific arc buffer. When the
2350 * state_index argument is set the function will calculate the arc header
2351 * list position for its arc state. Since this requires a linear traversal
2352 * callers are strongly encourage not to do this. However, it can be helpful
2353 * for targeted analysis so the functionality is provided.
2354 */
2355 void
arc_buf_info(arc_buf_t * ab,arc_buf_info_t * abi,int state_index)2356 arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
2357 {
2358 (void) state_index;
2359 arc_buf_hdr_t *hdr = ab->b_hdr;
2360 l1arc_buf_hdr_t *l1hdr = NULL;
2361 l2arc_buf_hdr_t *l2hdr = NULL;
2362 arc_state_t *state = NULL;
2363
2364 memset(abi, 0, sizeof (arc_buf_info_t));
2365
2366 if (hdr == NULL)
2367 return;
2368
2369 abi->abi_flags = hdr->b_flags;
2370
2371 if (HDR_HAS_L1HDR(hdr)) {
2372 l1hdr = &hdr->b_l1hdr;
2373 state = l1hdr->b_state;
2374 }
2375 if (HDR_HAS_L2HDR(hdr))
2376 l2hdr = &hdr->b_l2hdr;
2377
2378 if (l1hdr) {
2379 abi->abi_bufcnt = 0;
2380 for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
2381 abi->abi_bufcnt++;
2382 abi->abi_access = l1hdr->b_arc_access;
2383 abi->abi_mru_hits = l1hdr->b_mru_hits;
2384 abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2385 abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2386 abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2387 abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
2388 }
2389
2390 if (l2hdr) {
2391 abi->abi_l2arc_dattr = l2hdr->b_daddr;
2392 abi->abi_l2arc_hits = l2hdr->b_hits;
2393 }
2394
2395 abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2396 abi->abi_state_contents = arc_buf_type(hdr);
2397 abi->abi_size = arc_hdr_size(hdr);
2398 }
2399
2400 /*
2401 * Move the supplied buffer to the indicated state. The hash lock
2402 * for the buffer must be held by the caller.
2403 */
2404 static void
arc_change_state(arc_state_t * new_state,arc_buf_hdr_t * hdr)2405 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
2406 {
2407 arc_state_t *old_state;
2408 int64_t refcnt;
2409 boolean_t update_old, update_new;
2410 arc_buf_contents_t type = arc_buf_type(hdr);
2411
2412 /*
2413 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2414 * in arc_read() when bringing a buffer out of the L2ARC. However, the
2415 * L1 hdr doesn't always exist when we change state to arc_anon before
2416 * destroying a header, in which case reallocating to add the L1 hdr is
2417 * pointless.
2418 */
2419 if (HDR_HAS_L1HDR(hdr)) {
2420 old_state = hdr->b_l1hdr.b_state;
2421 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2422 update_old = (hdr->b_l1hdr.b_buf != NULL ||
2423 hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
2424
2425 IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
2426 IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
2427 IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
2428 ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
2429 } else {
2430 old_state = arc_l2c_only;
2431 refcnt = 0;
2432 update_old = B_FALSE;
2433 }
2434 update_new = update_old;
2435 if (GHOST_STATE(old_state))
2436 update_old = B_TRUE;
2437 if (GHOST_STATE(new_state))
2438 update_new = B_TRUE;
2439
2440 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2441 ASSERT3P(new_state, !=, old_state);
2442
2443 /*
2444 * If this buffer is evictable, transfer it from the
2445 * old state list to the new state list.
2446 */
2447 if (refcnt == 0) {
2448 if (old_state != arc_anon && old_state != arc_l2c_only) {
2449 ASSERT(HDR_HAS_L1HDR(hdr));
2450 /* remove_reference() saves on insert. */
2451 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2452 multilist_remove(&old_state->arcs_list[type],
2453 hdr);
2454 arc_evictable_space_decrement(hdr, old_state);
2455 }
2456 }
2457 if (new_state != arc_anon && new_state != arc_l2c_only) {
2458 /*
2459 * An L1 header always exists here, since if we're
2460 * moving to some L1-cached state (i.e. not l2c_only or
2461 * anonymous), we realloc the header to add an L1hdr
2462 * beforehand.
2463 */
2464 ASSERT(HDR_HAS_L1HDR(hdr));
2465 multilist_insert(&new_state->arcs_list[type], hdr);
2466 arc_evictable_space_increment(hdr, new_state);
2467 }
2468 }
2469
2470 ASSERT(!HDR_EMPTY(hdr));
2471 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2472 buf_hash_remove(hdr);
2473
2474 /* adjust state sizes (ignore arc_l2c_only) */
2475
2476 if (update_new && new_state != arc_l2c_only) {
2477 ASSERT(HDR_HAS_L1HDR(hdr));
2478 if (GHOST_STATE(new_state)) {
2479
2480 /*
2481 * When moving a header to a ghost state, we first
2482 * remove all arc buffers. Thus, we'll have no arc
2483 * buffer to use for the reference. As a result, we
2484 * use the arc header pointer for the reference.
2485 */
2486 (void) zfs_refcount_add_many(
2487 &new_state->arcs_size[type],
2488 HDR_GET_LSIZE(hdr), hdr);
2489 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2490 ASSERT(!HDR_HAS_RABD(hdr));
2491 } else {
2492
2493 /*
2494 * Each individual buffer holds a unique reference,
2495 * thus we must remove each of these references one
2496 * at a time.
2497 */
2498 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2499 buf = buf->b_next) {
2500
2501 /*
2502 * When the arc_buf_t is sharing the data
2503 * block with the hdr, the owner of the
2504 * reference belongs to the hdr. Only
2505 * add to the refcount if the arc_buf_t is
2506 * not shared.
2507 */
2508 if (ARC_BUF_SHARED(buf))
2509 continue;
2510
2511 (void) zfs_refcount_add_many(
2512 &new_state->arcs_size[type],
2513 arc_buf_size(buf), buf);
2514 }
2515
2516 if (hdr->b_l1hdr.b_pabd != NULL) {
2517 (void) zfs_refcount_add_many(
2518 &new_state->arcs_size[type],
2519 arc_hdr_size(hdr), hdr);
2520 }
2521
2522 if (HDR_HAS_RABD(hdr)) {
2523 (void) zfs_refcount_add_many(
2524 &new_state->arcs_size[type],
2525 HDR_GET_PSIZE(hdr), hdr);
2526 }
2527 }
2528 }
2529
2530 if (update_old && old_state != arc_l2c_only) {
2531 ASSERT(HDR_HAS_L1HDR(hdr));
2532 if (GHOST_STATE(old_state)) {
2533 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2534 ASSERT(!HDR_HAS_RABD(hdr));
2535
2536 /*
2537 * When moving a header off of a ghost state,
2538 * the header will not contain any arc buffers.
2539 * We use the arc header pointer for the reference
2540 * which is exactly what we did when we put the
2541 * header on the ghost state.
2542 */
2543
2544 (void) zfs_refcount_remove_many(
2545 &old_state->arcs_size[type],
2546 HDR_GET_LSIZE(hdr), hdr);
2547 } else {
2548
2549 /*
2550 * Each individual buffer holds a unique reference,
2551 * thus we must remove each of these references one
2552 * at a time.
2553 */
2554 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2555 buf = buf->b_next) {
2556
2557 /*
2558 * When the arc_buf_t is sharing the data
2559 * block with the hdr, the owner of the
2560 * reference belongs to the hdr. Only
2561 * add to the refcount if the arc_buf_t is
2562 * not shared.
2563 */
2564 if (ARC_BUF_SHARED(buf))
2565 continue;
2566
2567 (void) zfs_refcount_remove_many(
2568 &old_state->arcs_size[type],
2569 arc_buf_size(buf), buf);
2570 }
2571 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
2572 HDR_HAS_RABD(hdr));
2573
2574 if (hdr->b_l1hdr.b_pabd != NULL) {
2575 (void) zfs_refcount_remove_many(
2576 &old_state->arcs_size[type],
2577 arc_hdr_size(hdr), hdr);
2578 }
2579
2580 if (HDR_HAS_RABD(hdr)) {
2581 (void) zfs_refcount_remove_many(
2582 &old_state->arcs_size[type],
2583 HDR_GET_PSIZE(hdr), hdr);
2584 }
2585 }
2586 }
2587
2588 if (HDR_HAS_L1HDR(hdr)) {
2589 hdr->b_l1hdr.b_state = new_state;
2590
2591 if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
2592 l2arc_hdr_arcstats_decrement_state(hdr);
2593 hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
2594 l2arc_hdr_arcstats_increment_state(hdr);
2595 }
2596 }
2597 }
2598
2599 void
arc_space_consume(uint64_t space,arc_space_type_t type)2600 arc_space_consume(uint64_t space, arc_space_type_t type)
2601 {
2602 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2603
2604 switch (type) {
2605 default:
2606 break;
2607 case ARC_SPACE_DATA:
2608 ARCSTAT_INCR(arcstat_data_size, space);
2609 break;
2610 case ARC_SPACE_META:
2611 ARCSTAT_INCR(arcstat_metadata_size, space);
2612 break;
2613 case ARC_SPACE_BONUS:
2614 ARCSTAT_INCR(arcstat_bonus_size, space);
2615 break;
2616 case ARC_SPACE_DNODE:
2617 ARCSTAT_INCR(arcstat_dnode_size, space);
2618 break;
2619 case ARC_SPACE_DBUF:
2620 ARCSTAT_INCR(arcstat_dbuf_size, space);
2621 break;
2622 case ARC_SPACE_HDRS:
2623 ARCSTAT_INCR(arcstat_hdr_size, space);
2624 break;
2625 case ARC_SPACE_L2HDRS:
2626 aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
2627 break;
2628 case ARC_SPACE_ABD_CHUNK_WASTE:
2629 /*
2630 * Note: this includes space wasted by all scatter ABD's, not
2631 * just those allocated by the ARC. But the vast majority of
2632 * scatter ABD's come from the ARC, because other users are
2633 * very short-lived.
2634 */
2635 ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
2636 break;
2637 }
2638
2639 if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2640 ARCSTAT_INCR(arcstat_meta_used, space);
2641
2642 aggsum_add(&arc_sums.arcstat_size, space);
2643 }
2644
2645 void
arc_space_return(uint64_t space,arc_space_type_t type)2646 arc_space_return(uint64_t space, arc_space_type_t type)
2647 {
2648 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
2649
2650 switch (type) {
2651 default:
2652 break;
2653 case ARC_SPACE_DATA:
2654 ARCSTAT_INCR(arcstat_data_size, -space);
2655 break;
2656 case ARC_SPACE_META:
2657 ARCSTAT_INCR(arcstat_metadata_size, -space);
2658 break;
2659 case ARC_SPACE_BONUS:
2660 ARCSTAT_INCR(arcstat_bonus_size, -space);
2661 break;
2662 case ARC_SPACE_DNODE:
2663 ARCSTAT_INCR(arcstat_dnode_size, -space);
2664 break;
2665 case ARC_SPACE_DBUF:
2666 ARCSTAT_INCR(arcstat_dbuf_size, -space);
2667 break;
2668 case ARC_SPACE_HDRS:
2669 ARCSTAT_INCR(arcstat_hdr_size, -space);
2670 break;
2671 case ARC_SPACE_L2HDRS:
2672 aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
2673 break;
2674 case ARC_SPACE_ABD_CHUNK_WASTE:
2675 ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
2676 break;
2677 }
2678
2679 if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
2680 ARCSTAT_INCR(arcstat_meta_used, -space);
2681
2682 ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
2683 aggsum_add(&arc_sums.arcstat_size, -space);
2684 }
2685
2686 /*
2687 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2688 * with the hdr's b_pabd.
2689 */
2690 static boolean_t
arc_can_share(arc_buf_hdr_t * hdr,arc_buf_t * buf)2691 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2692 {
2693 /*
2694 * The criteria for sharing a hdr's data are:
2695 * 1. the buffer is not encrypted
2696 * 2. the hdr's compression matches the buf's compression
2697 * 3. the hdr doesn't need to be byteswapped
2698 * 4. the hdr isn't already being shared
2699 * 5. the buf is either compressed or it is the last buf in the hdr list
2700 *
2701 * Criterion #5 maintains the invariant that shared uncompressed
2702 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2703 * might ask, "if a compressed buf is allocated first, won't that be the
2704 * last thing in the list?", but in that case it's impossible to create
2705 * a shared uncompressed buf anyway (because the hdr must be compressed
2706 * to have the compressed buf). You might also think that #3 is
2707 * sufficient to make this guarantee, however it's possible
2708 * (specifically in the rare L2ARC write race mentioned in
2709 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
2710 * is shareable, but wasn't at the time of its allocation. Rather than
2711 * allow a new shared uncompressed buf to be created and then shuffle
2712 * the list around to make it the last element, this simply disallows
2713 * sharing if the new buf isn't the first to be added.
2714 */
2715 ASSERT3P(buf->b_hdr, ==, hdr);
2716 boolean_t hdr_compressed =
2717 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
2718 boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
2719 return (!ARC_BUF_ENCRYPTED(buf) &&
2720 buf_compressed == hdr_compressed &&
2721 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2722 !HDR_SHARED_DATA(hdr) &&
2723 (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
2724 }
2725
2726 /*
2727 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2728 * or if you want a compressed buffer, pass those flags in. Returns 0 if the
2729 * copy was made successfully, or an error code otherwise.
2730 */
2731 static int
arc_buf_alloc_impl(arc_buf_hdr_t * hdr,spa_t * spa,const zbookmark_phys_t * zb,const void * tag,boolean_t encrypted,boolean_t compressed,boolean_t noauth,boolean_t fill,arc_buf_t ** ret)2732 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
2733 const void *tag, boolean_t encrypted, boolean_t compressed,
2734 boolean_t noauth, boolean_t fill, arc_buf_t **ret)
2735 {
2736 arc_buf_t *buf;
2737 arc_fill_flags_t flags = ARC_FILL_LOCKED;
2738
2739 ASSERT(HDR_HAS_L1HDR(hdr));
2740 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2741 VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2742 hdr->b_type == ARC_BUFC_METADATA);
2743 ASSERT3P(ret, !=, NULL);
2744 ASSERT3P(*ret, ==, NULL);
2745 IMPLY(encrypted, compressed);
2746
2747 buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2748 buf->b_hdr = hdr;
2749 buf->b_data = NULL;
2750 buf->b_next = hdr->b_l1hdr.b_buf;
2751 buf->b_flags = 0;
2752
2753 add_reference(hdr, tag);
2754
2755 /*
2756 * We're about to change the hdr's b_flags. We must either
2757 * hold the hash_lock or be undiscoverable.
2758 */
2759 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2760
2761 /*
2762 * Only honor requests for compressed bufs if the hdr is actually
2763 * compressed. This must be overridden if the buffer is encrypted since
2764 * encrypted buffers cannot be decompressed.
2765 */
2766 if (encrypted) {
2767 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2768 buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED;
2769 flags |= ARC_FILL_COMPRESSED | ARC_FILL_ENCRYPTED;
2770 } else if (compressed &&
2771 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
2772 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2773 flags |= ARC_FILL_COMPRESSED;
2774 }
2775
2776 if (noauth) {
2777 ASSERT0(encrypted);
2778 flags |= ARC_FILL_NOAUTH;
2779 }
2780
2781 /*
2782 * If the hdr's data can be shared then we share the data buffer and
2783 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2784 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
2785 * buffer to store the buf's data.
2786 *
2787 * There are two additional restrictions here because we're sharing
2788 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2789 * actively involved in an L2ARC write, because if this buf is used by
2790 * an arc_write() then the hdr's data buffer will be released when the
2791 * write completes, even though the L2ARC write might still be using it.
2792 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2793 * need to be ABD-aware. It must be allocated via
2794 * zio_[data_]buf_alloc(), not as a page, because we need to be able
2795 * to abd_release_ownership_of_buf(), which isn't allowed on "linear
2796 * page" buffers because the ABD code needs to handle freeing them
2797 * specially.
2798 */
2799 boolean_t can_share = arc_can_share(hdr, buf) &&
2800 !HDR_L2_WRITING(hdr) &&
2801 hdr->b_l1hdr.b_pabd != NULL &&
2802 abd_is_linear(hdr->b_l1hdr.b_pabd) &&
2803 !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
2804
2805 /* Set up b_data and sharing */
2806 if (can_share) {
2807 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2808 buf->b_flags |= ARC_BUF_FLAG_SHARED;
2809 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2810 } else {
2811 buf->b_data =
2812 arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2813 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
2814 }
2815 VERIFY3P(buf->b_data, !=, NULL);
2816
2817 hdr->b_l1hdr.b_buf = buf;
2818
2819 /*
2820 * If the user wants the data from the hdr, we need to either copy or
2821 * decompress the data.
2822 */
2823 if (fill) {
2824 ASSERT3P(zb, !=, NULL);
2825 return (arc_buf_fill(buf, spa, zb, flags));
2826 }
2827
2828 return (0);
2829 }
2830
2831 static const char *arc_onloan_tag = "onloan";
2832
2833 static inline void
arc_loaned_bytes_update(int64_t delta)2834 arc_loaned_bytes_update(int64_t delta)
2835 {
2836 atomic_add_64(&arc_loaned_bytes, delta);
2837
2838 /* assert that it did not wrap around */
2839 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
2840 }
2841
2842 /*
2843 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
2844 * flight data by arc_tempreserve_space() until they are "returned". Loaned
2845 * buffers must be returned to the arc before they can be used by the DMU or
2846 * freed.
2847 */
2848 arc_buf_t *
arc_loan_buf(spa_t * spa,boolean_t is_metadata,int size)2849 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
2850 {
2851 arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
2852 is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
2853
2854 arc_loaned_bytes_update(arc_buf_size(buf));
2855
2856 return (buf);
2857 }
2858
2859 arc_buf_t *
arc_loan_compressed_buf(spa_t * spa,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)2860 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
2861 enum zio_compress compression_type, uint8_t complevel)
2862 {
2863 arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
2864 psize, lsize, compression_type, complevel);
2865
2866 arc_loaned_bytes_update(arc_buf_size(buf));
2867
2868 return (buf);
2869 }
2870
2871 arc_buf_t *
arc_loan_raw_buf(spa_t * spa,uint64_t dsobj,boolean_t byteorder,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac,dmu_object_type_t ot,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)2872 arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
2873 const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
2874 dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
2875 enum zio_compress compression_type, uint8_t complevel)
2876 {
2877 arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
2878 byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
2879 complevel);
2880
2881 atomic_add_64(&arc_loaned_bytes, psize);
2882 return (buf);
2883 }
2884
2885
2886 /*
2887 * Return a loaned arc buffer to the arc.
2888 */
2889 void
arc_return_buf(arc_buf_t * buf,const void * tag)2890 arc_return_buf(arc_buf_t *buf, const void *tag)
2891 {
2892 arc_buf_hdr_t *hdr = buf->b_hdr;
2893
2894 ASSERT3P(buf->b_data, !=, NULL);
2895 ASSERT(HDR_HAS_L1HDR(hdr));
2896 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2897 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2898
2899 arc_loaned_bytes_update(-arc_buf_size(buf));
2900 }
2901
2902 /* Detach an arc_buf from a dbuf (tag) */
2903 void
arc_loan_inuse_buf(arc_buf_t * buf,const void * tag)2904 arc_loan_inuse_buf(arc_buf_t *buf, const void *tag)
2905 {
2906 arc_buf_hdr_t *hdr = buf->b_hdr;
2907
2908 ASSERT3P(buf->b_data, !=, NULL);
2909 ASSERT(HDR_HAS_L1HDR(hdr));
2910 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2911 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2912
2913 arc_loaned_bytes_update(arc_buf_size(buf));
2914 }
2915
2916 static void
l2arc_free_abd_on_write(abd_t * abd,size_t size,arc_buf_contents_t type)2917 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
2918 {
2919 l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
2920
2921 df->l2df_abd = abd;
2922 df->l2df_size = size;
2923 df->l2df_type = type;
2924 mutex_enter(&l2arc_free_on_write_mtx);
2925 list_insert_head(l2arc_free_on_write, df);
2926 mutex_exit(&l2arc_free_on_write_mtx);
2927 }
2928
2929 static void
arc_hdr_free_on_write(arc_buf_hdr_t * hdr,boolean_t free_rdata)2930 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
2931 {
2932 arc_state_t *state = hdr->b_l1hdr.b_state;
2933 arc_buf_contents_t type = arc_buf_type(hdr);
2934 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
2935
2936 /* protected by hash lock, if in the hash table */
2937 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2938 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2939 ASSERT(state != arc_anon && state != arc_l2c_only);
2940
2941 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2942 size, hdr);
2943 }
2944 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
2945 if (type == ARC_BUFC_METADATA) {
2946 arc_space_return(size, ARC_SPACE_META);
2947 } else {
2948 ASSERT(type == ARC_BUFC_DATA);
2949 arc_space_return(size, ARC_SPACE_DATA);
2950 }
2951
2952 if (free_rdata) {
2953 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
2954 } else {
2955 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2956 }
2957 }
2958
2959 /*
2960 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2961 * data buffer, we transfer the refcount ownership to the hdr and update
2962 * the appropriate kstats.
2963 */
2964 static void
arc_share_buf(arc_buf_hdr_t * hdr,arc_buf_t * buf)2965 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2966 {
2967 ASSERT(arc_can_share(hdr, buf));
2968 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2969 ASSERT(!ARC_BUF_ENCRYPTED(buf));
2970 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2971
2972 /*
2973 * Start sharing the data buffer. We transfer the
2974 * refcount ownership to the hdr since it always owns
2975 * the refcount whenever an arc_buf_t is shared.
2976 */
2977 zfs_refcount_transfer_ownership_many(
2978 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2979 arc_hdr_size(hdr), buf, hdr);
2980 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2981 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2982 HDR_ISTYPE_METADATA(hdr));
2983 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2984 buf->b_flags |= ARC_BUF_FLAG_SHARED;
2985
2986 /*
2987 * Since we've transferred ownership to the hdr we need
2988 * to increment its compressed and uncompressed kstats and
2989 * decrement the overhead size.
2990 */
2991 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2992 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2993 ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
2994 }
2995
2996 static void
arc_unshare_buf(arc_buf_hdr_t * hdr,arc_buf_t * buf)2997 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2998 {
2999 ASSERT(arc_buf_is_shared(buf));
3000 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3001 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3002
3003 /*
3004 * We are no longer sharing this buffer so we need
3005 * to transfer its ownership to the rightful owner.
3006 */
3007 zfs_refcount_transfer_ownership_many(
3008 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
3009 arc_hdr_size(hdr), hdr, buf);
3010 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3011 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3012 abd_free(hdr->b_l1hdr.b_pabd);
3013 hdr->b_l1hdr.b_pabd = NULL;
3014 buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
3015
3016 /*
3017 * Since the buffer is no longer shared between
3018 * the arc buf and the hdr, count it as overhead.
3019 */
3020 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3021 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3022 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
3023 }
3024
3025 /*
3026 * Remove an arc_buf_t from the hdr's buf list and return the last
3027 * arc_buf_t on the list. If no buffers remain on the list then return
3028 * NULL.
3029 */
3030 static arc_buf_t *
arc_buf_remove(arc_buf_hdr_t * hdr,arc_buf_t * buf)3031 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3032 {
3033 ASSERT(HDR_HAS_L1HDR(hdr));
3034 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3035
3036 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3037 arc_buf_t *lastbuf = NULL;
3038
3039 /*
3040 * Remove the buf from the hdr list and locate the last
3041 * remaining buffer on the list.
3042 */
3043 while (*bufp != NULL) {
3044 if (*bufp == buf)
3045 *bufp = buf->b_next;
3046
3047 /*
3048 * If we've removed a buffer in the middle of
3049 * the list then update the lastbuf and update
3050 * bufp.
3051 */
3052 if (*bufp != NULL) {
3053 lastbuf = *bufp;
3054 bufp = &(*bufp)->b_next;
3055 }
3056 }
3057 buf->b_next = NULL;
3058 ASSERT3P(lastbuf, !=, buf);
3059 IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
3060
3061 return (lastbuf);
3062 }
3063
3064 /*
3065 * Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
3066 * list and free it.
3067 */
3068 static void
arc_buf_destroy_impl(arc_buf_t * buf)3069 arc_buf_destroy_impl(arc_buf_t *buf)
3070 {
3071 arc_buf_hdr_t *hdr = buf->b_hdr;
3072
3073 /*
3074 * Free up the data associated with the buf but only if we're not
3075 * sharing this with the hdr. If we are sharing it with the hdr, the
3076 * hdr is responsible for doing the free.
3077 */
3078 if (buf->b_data != NULL) {
3079 /*
3080 * We're about to change the hdr's b_flags. We must either
3081 * hold the hash_lock or be undiscoverable.
3082 */
3083 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3084
3085 arc_cksum_verify(buf);
3086 arc_buf_unwatch(buf);
3087
3088 if (ARC_BUF_SHARED(buf)) {
3089 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3090 } else {
3091 ASSERT(!arc_buf_is_shared(buf));
3092 uint64_t size = arc_buf_size(buf);
3093 arc_free_data_buf(hdr, buf->b_data, size, buf);
3094 ARCSTAT_INCR(arcstat_overhead_size, -size);
3095 }
3096 buf->b_data = NULL;
3097
3098 /*
3099 * If we have no more encrypted buffers and we've already
3100 * gotten a copy of the decrypted data we can free b_rabd
3101 * to save some space.
3102 */
3103 if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
3104 hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
3105 arc_buf_t *b;
3106 for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
3107 if (b != buf && ARC_BUF_ENCRYPTED(b))
3108 break;
3109 }
3110 if (b == NULL)
3111 arc_hdr_free_abd(hdr, B_TRUE);
3112 }
3113 }
3114
3115 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3116
3117 if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
3118 /*
3119 * If the current arc_buf_t is sharing its data buffer with the
3120 * hdr, then reassign the hdr's b_pabd to share it with the new
3121 * buffer at the end of the list. The shared buffer is always
3122 * the last one on the hdr's buffer list.
3123 *
3124 * There is an equivalent case for compressed bufs, but since
3125 * they aren't guaranteed to be the last buf in the list and
3126 * that is an exceedingly rare case, we just allow that space be
3127 * wasted temporarily. We must also be careful not to share
3128 * encrypted buffers, since they cannot be shared.
3129 */
3130 if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) {
3131 /* Only one buf can be shared at once */
3132 ASSERT(!arc_buf_is_shared(lastbuf));
3133 /* hdr is uncompressed so can't have compressed buf */
3134 ASSERT(!ARC_BUF_COMPRESSED(lastbuf));
3135
3136 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3137 arc_hdr_free_abd(hdr, B_FALSE);
3138
3139 /*
3140 * We must setup a new shared block between the
3141 * last buffer and the hdr. The data would have
3142 * been allocated by the arc buf so we need to transfer
3143 * ownership to the hdr since it's now being shared.
3144 */
3145 arc_share_buf(hdr, lastbuf);
3146 }
3147 } else if (HDR_SHARED_DATA(hdr)) {
3148 /*
3149 * Uncompressed shared buffers are always at the end
3150 * of the list. Compressed buffers don't have the
3151 * same requirements. This makes it hard to
3152 * simply assert that the lastbuf is shared so
3153 * we rely on the hdr's compression flags to determine
3154 * if we have a compressed, shared buffer.
3155 */
3156 ASSERT3P(lastbuf, !=, NULL);
3157 ASSERT(arc_buf_is_shared(lastbuf) ||
3158 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
3159 }
3160
3161 /*
3162 * Free the checksum if we're removing the last uncompressed buf from
3163 * this hdr.
3164 */
3165 if (!arc_hdr_has_uncompressed_buf(hdr)) {
3166 arc_cksum_free(hdr);
3167 }
3168
3169 /* clean up the buf */
3170 buf->b_hdr = NULL;
3171 kmem_cache_free(buf_cache, buf);
3172 }
3173
3174 static void
arc_hdr_alloc_abd(arc_buf_hdr_t * hdr,int alloc_flags)3175 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
3176 {
3177 uint64_t size;
3178 boolean_t alloc_rdata = ((alloc_flags & ARC_HDR_ALLOC_RDATA) != 0);
3179
3180 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3181 ASSERT(HDR_HAS_L1HDR(hdr));
3182 ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);
3183 IMPLY(alloc_rdata, HDR_PROTECTED(hdr));
3184
3185 if (alloc_rdata) {
3186 size = HDR_GET_PSIZE(hdr);
3187 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
3188 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
3189 alloc_flags);
3190 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
3191 ARCSTAT_INCR(arcstat_raw_size, size);
3192 } else {
3193 size = arc_hdr_size(hdr);
3194 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3195 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
3196 alloc_flags);
3197 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3198 }
3199
3200 ARCSTAT_INCR(arcstat_compressed_size, size);
3201 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3202 }
3203
3204 static void
arc_hdr_free_abd(arc_buf_hdr_t * hdr,boolean_t free_rdata)3205 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
3206 {
3207 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
3208
3209 ASSERT(HDR_HAS_L1HDR(hdr));
3210 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
3211 IMPLY(free_rdata, HDR_HAS_RABD(hdr));
3212
3213 /*
3214 * If the hdr is currently being written to the l2arc then
3215 * we defer freeing the data by adding it to the l2arc_free_on_write
3216 * list. The l2arc will free the data once it's finished
3217 * writing it to the l2arc device.
3218 */
3219 if (HDR_L2_WRITING(hdr)) {
3220 arc_hdr_free_on_write(hdr, free_rdata);
3221 ARCSTAT_BUMP(arcstat_l2_free_on_write);
3222 } else if (free_rdata) {
3223 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
3224 } else {
3225 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
3226 }
3227
3228 if (free_rdata) {
3229 hdr->b_crypt_hdr.b_rabd = NULL;
3230 ARCSTAT_INCR(arcstat_raw_size, -size);
3231 } else {
3232 hdr->b_l1hdr.b_pabd = NULL;
3233 }
3234
3235 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
3236 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3237
3238 ARCSTAT_INCR(arcstat_compressed_size, -size);
3239 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3240 }
3241
3242 /*
3243 * Allocate empty anonymous ARC header. The header will get its identity
3244 * assigned and buffers attached later as part of read or write operations.
3245 *
3246 * In case of read arc_read() assigns header its identify (b_dva + b_birth),
3247 * inserts it into ARC hash to become globally visible and allocates physical
3248 * (b_pabd) or raw (b_rabd) ABD buffer to read into from disk. On disk read
3249 * completion arc_read_done() allocates ARC buffer(s) as needed, potentially
3250 * sharing one of them with the physical ABD buffer.
3251 *
3252 * In case of write arc_alloc_buf() allocates ARC buffer to be filled with
3253 * data. Then after compression and/or encryption arc_write_ready() allocates
3254 * and fills (or potentially shares) physical (b_pabd) or raw (b_rabd) ABD
3255 * buffer. On disk write completion arc_write_done() assigns the header its
3256 * new identity (b_dva + b_birth) and inserts into ARC hash.
3257 *
3258 * In case of partial overwrite the old data is read first as described. Then
3259 * arc_release() either allocates new anonymous ARC header and moves the ARC
3260 * buffer to it, or reuses the old ARC header by discarding its identity and
3261 * removing it from ARC hash. After buffer modification normal write process
3262 * follows as described.
3263 */
3264 static arc_buf_hdr_t *
arc_hdr_alloc(uint64_t spa,int32_t psize,int32_t lsize,boolean_t protected,enum zio_compress compression_type,uint8_t complevel,arc_buf_contents_t type)3265 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
3266 boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
3267 arc_buf_contents_t type)
3268 {
3269 arc_buf_hdr_t *hdr;
3270
3271 VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
3272 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3273
3274 ASSERT(HDR_EMPTY(hdr));
3275 #ifdef ZFS_DEBUG
3276 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3277 #endif
3278 HDR_SET_PSIZE(hdr, psize);
3279 HDR_SET_LSIZE(hdr, lsize);
3280 hdr->b_spa = spa;
3281 hdr->b_type = type;
3282 hdr->b_flags = 0;
3283 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3284 arc_hdr_set_compress(hdr, compression_type);
3285 hdr->b_complevel = complevel;
3286 if (protected)
3287 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3288
3289 hdr->b_l1hdr.b_state = arc_anon;
3290 hdr->b_l1hdr.b_arc_access = 0;
3291 hdr->b_l1hdr.b_mru_hits = 0;
3292 hdr->b_l1hdr.b_mru_ghost_hits = 0;
3293 hdr->b_l1hdr.b_mfu_hits = 0;
3294 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
3295 hdr->b_l1hdr.b_buf = NULL;
3296
3297 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3298
3299 return (hdr);
3300 }
3301
3302 /*
3303 * Transition between the two allocation states for the arc_buf_hdr struct.
3304 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
3305 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3306 * version is used when a cache buffer is only in the L2ARC in order to reduce
3307 * memory usage.
3308 */
3309 static arc_buf_hdr_t *
arc_hdr_realloc(arc_buf_hdr_t * hdr,kmem_cache_t * old,kmem_cache_t * new)3310 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3311 {
3312 ASSERT(HDR_HAS_L2HDR(hdr));
3313
3314 arc_buf_hdr_t *nhdr;
3315 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3316
3317 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
3318 (old == hdr_l2only_cache && new == hdr_full_cache));
3319
3320 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
3321
3322 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3323 buf_hash_remove(hdr);
3324
3325 memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);
3326
3327 if (new == hdr_full_cache) {
3328 arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3329 /*
3330 * arc_access and arc_change_state need to be aware that a
3331 * header has just come out of L2ARC, so we set its state to
3332 * l2c_only even though it's about to change.
3333 */
3334 nhdr->b_l1hdr.b_state = arc_l2c_only;
3335
3336 /* Verify previous threads set to NULL before freeing */
3337 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3338 ASSERT(!HDR_HAS_RABD(hdr));
3339 } else {
3340 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3341 #ifdef ZFS_DEBUG
3342 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3343 #endif
3344
3345 /*
3346 * If we've reached here, We must have been called from
3347 * arc_evict_hdr(), as such we should have already been
3348 * removed from any ghost list we were previously on
3349 * (which protects us from racing with arc_evict_state),
3350 * thus no locking is needed during this check.
3351 */
3352 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3353
3354 /*
3355 * A buffer must not be moved into the arc_l2c_only
3356 * state if it's not finished being written out to the
3357 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
3358 * might try to be accessed, even though it was removed.
3359 */
3360 VERIFY(!HDR_L2_WRITING(hdr));
3361 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3362 ASSERT(!HDR_HAS_RABD(hdr));
3363
3364 arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
3365 }
3366 /*
3367 * The header has been reallocated so we need to re-insert it into any
3368 * lists it was on.
3369 */
3370 (void) buf_hash_insert(nhdr, NULL);
3371
3372 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3373
3374 mutex_enter(&dev->l2ad_mtx);
3375
3376 /*
3377 * We must place the realloc'ed header back into the list at
3378 * the same spot. Otherwise, if it's placed earlier in the list,
3379 * l2arc_write_buffers() could find it during the function's
3380 * write phase, and try to write it out to the l2arc.
3381 */
3382 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3383 list_remove(&dev->l2ad_buflist, hdr);
3384
3385 mutex_exit(&dev->l2ad_mtx);
3386
3387 /*
3388 * Since we're using the pointer address as the tag when
3389 * incrementing and decrementing the l2ad_alloc refcount, we
3390 * must remove the old pointer (that we're about to destroy) and
3391 * add the new pointer to the refcount. Otherwise we'd remove
3392 * the wrong pointer address when calling arc_hdr_destroy() later.
3393 */
3394
3395 (void) zfs_refcount_remove_many(&dev->l2ad_alloc,
3396 arc_hdr_size(hdr), hdr);
3397 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
3398 arc_hdr_size(nhdr), nhdr);
3399
3400 buf_discard_identity(hdr);
3401 kmem_cache_free(old, hdr);
3402
3403 return (nhdr);
3404 }
3405
3406 /*
3407 * This function is used by the send / receive code to convert a newly
3408 * allocated arc_buf_t to one that is suitable for a raw encrypted write. It
3409 * is also used to allow the root objset block to be updated without altering
3410 * its embedded MACs. Both block types will always be uncompressed so we do not
3411 * have to worry about compression type or psize.
3412 */
3413 void
arc_convert_to_raw(arc_buf_t * buf,uint64_t dsobj,boolean_t byteorder,dmu_object_type_t ot,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac)3414 arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder,
3415 dmu_object_type_t ot, const uint8_t *salt, const uint8_t *iv,
3416 const uint8_t *mac)
3417 {
3418 arc_buf_hdr_t *hdr = buf->b_hdr;
3419
3420 ASSERT(ot == DMU_OT_DNODE || ot == DMU_OT_OBJSET);
3421 ASSERT(HDR_HAS_L1HDR(hdr));
3422 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3423
3424 buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED);
3425 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3426 hdr->b_crypt_hdr.b_dsobj = dsobj;
3427 hdr->b_crypt_hdr.b_ot = ot;
3428 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3429 DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3430 if (!arc_hdr_has_uncompressed_buf(hdr))
3431 arc_cksum_free(hdr);
3432
3433 if (salt != NULL)
3434 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3435 if (iv != NULL)
3436 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3437 if (mac != NULL)
3438 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3439 }
3440
3441 /*
3442 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
3443 * The buf is returned thawed since we expect the consumer to modify it.
3444 */
3445 arc_buf_t *
arc_alloc_buf(spa_t * spa,const void * tag,arc_buf_contents_t type,int32_t size)3446 arc_alloc_buf(spa_t *spa, const void *tag, arc_buf_contents_t type,
3447 int32_t size)
3448 {
3449 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3450 B_FALSE, ZIO_COMPRESS_OFF, 0, type);
3451
3452 arc_buf_t *buf = NULL;
3453 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
3454 B_FALSE, B_FALSE, &buf));
3455 arc_buf_thaw(buf);
3456
3457 return (buf);
3458 }
3459
3460 /*
3461 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
3462 * for bufs containing metadata.
3463 */
3464 arc_buf_t *
arc_alloc_compressed_buf(spa_t * spa,const void * tag,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)3465 arc_alloc_compressed_buf(spa_t *spa, const void *tag, uint64_t psize,
3466 uint64_t lsize, enum zio_compress compression_type, uint8_t complevel)
3467 {
3468 ASSERT3U(lsize, >, 0);
3469 ASSERT3U(lsize, >=, psize);
3470 ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF);
3471 ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3472
3473 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3474 B_FALSE, compression_type, complevel, ARC_BUFC_DATA);
3475
3476 arc_buf_t *buf = NULL;
3477 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
3478 B_TRUE, B_FALSE, B_FALSE, &buf));
3479 arc_buf_thaw(buf);
3480
3481 /*
3482 * To ensure that the hdr has the correct data in it if we call
3483 * arc_untransform() on this buf before it's been written to disk,
3484 * it's easiest if we just set up sharing between the buf and the hdr.
3485 */
3486 arc_share_buf(hdr, buf);
3487
3488 return (buf);
3489 }
3490
3491 arc_buf_t *
arc_alloc_raw_buf(spa_t * spa,const void * tag,uint64_t dsobj,boolean_t byteorder,const uint8_t * salt,const uint8_t * iv,const uint8_t * mac,dmu_object_type_t ot,uint64_t psize,uint64_t lsize,enum zio_compress compression_type,uint8_t complevel)3492 arc_alloc_raw_buf(spa_t *spa, const void *tag, uint64_t dsobj,
3493 boolean_t byteorder, const uint8_t *salt, const uint8_t *iv,
3494 const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
3495 enum zio_compress compression_type, uint8_t complevel)
3496 {
3497 arc_buf_hdr_t *hdr;
3498 arc_buf_t *buf;
3499 arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ?
3500 ARC_BUFC_METADATA : ARC_BUFC_DATA;
3501
3502 ASSERT3U(lsize, >, 0);
3503 ASSERT3U(lsize, >=, psize);
3504 ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF);
3505 ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
3506
3507 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
3508 compression_type, complevel, type);
3509
3510 hdr->b_crypt_hdr.b_dsobj = dsobj;
3511 hdr->b_crypt_hdr.b_ot = ot;
3512 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3513 DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
3514 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3515 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3516 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3517
3518 /*
3519 * This buffer will be considered encrypted even if the ot is not an
3520 * encrypted type. It will become authenticated instead in
3521 * arc_write_ready().
3522 */
3523 buf = NULL;
3524 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
3525 B_FALSE, B_FALSE, &buf));
3526 arc_buf_thaw(buf);
3527
3528 return (buf);
3529 }
3530
3531 static void
l2arc_hdr_arcstats_update(arc_buf_hdr_t * hdr,boolean_t incr,boolean_t state_only)3532 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
3533 boolean_t state_only)
3534 {
3535 uint64_t lsize = HDR_GET_LSIZE(hdr);
3536 uint64_t psize = HDR_GET_PSIZE(hdr);
3537 uint64_t asize = HDR_GET_L2SIZE(hdr);
3538 arc_buf_contents_t type = hdr->b_type;
3539 int64_t lsize_s;
3540 int64_t psize_s;
3541 int64_t asize_s;
3542
3543 /* For L2 we expect the header's b_l2size to be valid */
3544 ASSERT3U(asize, >=, psize);
3545
3546 if (incr) {
3547 lsize_s = lsize;
3548 psize_s = psize;
3549 asize_s = asize;
3550 } else {
3551 lsize_s = -lsize;
3552 psize_s = -psize;
3553 asize_s = -asize;
3554 }
3555
3556 /* If the buffer is a prefetch, count it as such. */
3557 if (HDR_PREFETCH(hdr)) {
3558 ARCSTAT_INCR(arcstat_l2_prefetch_asize, asize_s);
3559 } else {
3560 /*
3561 * We use the value stored in the L2 header upon initial
3562 * caching in L2ARC. This value will be updated in case
3563 * an MRU/MRU_ghost buffer transitions to MFU but the L2ARC
3564 * metadata (log entry) cannot currently be updated. Having
3565 * the ARC state in the L2 header solves the problem of a
3566 * possibly absent L1 header (apparent in buffers restored
3567 * from persistent L2ARC).
3568 */
3569 switch (hdr->b_l2hdr.b_arcs_state) {
3570 case ARC_STATE_MRU_GHOST:
3571 case ARC_STATE_MRU:
3572 ARCSTAT_INCR(arcstat_l2_mru_asize, asize_s);
3573 break;
3574 case ARC_STATE_MFU_GHOST:
3575 case ARC_STATE_MFU:
3576 ARCSTAT_INCR(arcstat_l2_mfu_asize, asize_s);
3577 break;
3578 default:
3579 break;
3580 }
3581 }
3582
3583 if (state_only)
3584 return;
3585
3586 ARCSTAT_INCR(arcstat_l2_psize, psize_s);
3587 ARCSTAT_INCR(arcstat_l2_lsize, lsize_s);
3588
3589 switch (type) {
3590 case ARC_BUFC_DATA:
3591 ARCSTAT_INCR(arcstat_l2_bufc_data_asize, asize_s);
3592 break;
3593 case ARC_BUFC_METADATA:
3594 ARCSTAT_INCR(arcstat_l2_bufc_metadata_asize, asize_s);
3595 break;
3596 default:
3597 break;
3598 }
3599 }
3600
3601
3602 static void
arc_hdr_l2hdr_destroy(arc_buf_hdr_t * hdr)3603 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3604 {
3605 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3606 l2arc_dev_t *dev = l2hdr->b_dev;
3607
3608 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3609 ASSERT(HDR_HAS_L2HDR(hdr));
3610
3611 list_remove(&dev->l2ad_buflist, hdr);
3612
3613 l2arc_hdr_arcstats_decrement(hdr);
3614 if (dev->l2ad_vdev != NULL) {
3615 uint64_t asize = HDR_GET_L2SIZE(hdr);
3616 vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3617 }
3618
3619 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3620 hdr);
3621 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3622 }
3623
3624 static void
arc_hdr_destroy(arc_buf_hdr_t * hdr)3625 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3626 {
3627 if (HDR_HAS_L1HDR(hdr)) {
3628 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3629 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3630 }
3631 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3632 ASSERT(!HDR_IN_HASH_TABLE(hdr));
3633
3634 if (HDR_HAS_L2HDR(hdr)) {
3635 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3636 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3637
3638 if (!buflist_held)
3639 mutex_enter(&dev->l2ad_mtx);
3640
3641 /*
3642 * Even though we checked this conditional above, we
3643 * need to check this again now that we have the
3644 * l2ad_mtx. This is because we could be racing with
3645 * another thread calling l2arc_evict() which might have
3646 * destroyed this header's L2 portion as we were waiting
3647 * to acquire the l2ad_mtx. If that happens, we don't
3648 * want to re-destroy the header's L2 portion.
3649 */
3650 if (HDR_HAS_L2HDR(hdr)) {
3651
3652 if (!HDR_EMPTY(hdr))
3653 buf_discard_identity(hdr);
3654
3655 arc_hdr_l2hdr_destroy(hdr);
3656 }
3657
3658 if (!buflist_held)
3659 mutex_exit(&dev->l2ad_mtx);
3660 }
3661
3662 /*
3663 * The header's identify can only be safely discarded once it is no
3664 * longer discoverable. This requires removing it from the hash table
3665 * and the l2arc header list. After this point the hash lock can not
3666 * be used to protect the header.
3667 */
3668 if (!HDR_EMPTY(hdr))
3669 buf_discard_identity(hdr);
3670
3671 if (HDR_HAS_L1HDR(hdr)) {
3672 arc_cksum_free(hdr);
3673
3674 while (hdr->b_l1hdr.b_buf != NULL)
3675 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3676
3677 if (hdr->b_l1hdr.b_pabd != NULL)
3678 arc_hdr_free_abd(hdr, B_FALSE);
3679
3680 if (HDR_HAS_RABD(hdr))
3681 arc_hdr_free_abd(hdr, B_TRUE);
3682 }
3683
3684 ASSERT3P(hdr->b_hash_next, ==, NULL);
3685 if (HDR_HAS_L1HDR(hdr)) {
3686 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3687 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3688 #ifdef ZFS_DEBUG
3689 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3690 #endif
3691 kmem_cache_free(hdr_full_cache, hdr);
3692 } else {
3693 kmem_cache_free(hdr_l2only_cache, hdr);
3694 }
3695 }
3696
3697 void
arc_buf_destroy(arc_buf_t * buf,const void * tag)3698 arc_buf_destroy(arc_buf_t *buf, const void *tag)
3699 {
3700 arc_buf_hdr_t *hdr = buf->b_hdr;
3701
3702 if (hdr->b_l1hdr.b_state == arc_anon) {
3703 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
3704 ASSERT(ARC_BUF_LAST(buf));
3705 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3706 VERIFY0(remove_reference(hdr, tag));
3707 return;
3708 }
3709
3710 kmutex_t *hash_lock = HDR_LOCK(hdr);
3711 mutex_enter(hash_lock);
3712
3713 ASSERT3P(hdr, ==, buf->b_hdr);
3714 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
3715 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3716 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3717 ASSERT3P(buf->b_data, !=, NULL);
3718
3719 arc_buf_destroy_impl(buf);
3720 (void) remove_reference(hdr, tag);
3721 mutex_exit(hash_lock);
3722 }
3723
3724 /*
3725 * Evict the arc_buf_hdr that is provided as a parameter. The resultant
3726 * state of the header is dependent on its state prior to entering this
3727 * function. The following transitions are possible:
3728 *
3729 * - arc_mru -> arc_mru_ghost
3730 * - arc_mfu -> arc_mfu_ghost
3731 * - arc_mru_ghost -> arc_l2c_only
3732 * - arc_mru_ghost -> deleted
3733 * - arc_mfu_ghost -> arc_l2c_only
3734 * - arc_mfu_ghost -> deleted
3735 * - arc_uncached -> deleted
3736 *
3737 * Return total size of evicted data buffers for eviction progress tracking.
3738 * When evicting from ghost states return logical buffer size to make eviction
3739 * progress at the same (or at least comparable) rate as from non-ghost states.
3740 *
3741 * Return *real_evicted for actual ARC size reduction to wake up threads
3742 * waiting for it. For non-ghost states it includes size of evicted data
3743 * buffers (the headers are not freed there). For ghost states it includes
3744 * only the evicted headers size.
3745 */
3746 static int64_t
arc_evict_hdr(arc_buf_hdr_t * hdr,uint64_t * real_evicted)3747 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
3748 {
3749 arc_state_t *evicted_state, *state;
3750 int64_t bytes_evicted = 0;
3751 uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3752 arc_min_prescient_prefetch_ms : arc_min_prefetch_ms;
3753
3754 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3755 ASSERT(HDR_HAS_L1HDR(hdr));
3756 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3757 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3758 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3759
3760 *real_evicted = 0;
3761 state = hdr->b_l1hdr.b_state;
3762 if (GHOST_STATE(state)) {
3763
3764 /*
3765 * l2arc_write_buffers() relies on a header's L1 portion
3766 * (i.e. its b_pabd field) during it's write phase.
3767 * Thus, we cannot push a header onto the arc_l2c_only
3768 * state (removing its L1 piece) until the header is
3769 * done being written to the l2arc.
3770 */
3771 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3772 ARCSTAT_BUMP(arcstat_evict_l2_skip);
3773 return (bytes_evicted);
3774 }
3775
3776 ARCSTAT_BUMP(arcstat_deleted);
3777 bytes_evicted += HDR_GET_LSIZE(hdr);
3778
3779 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3780
3781 if (HDR_HAS_L2HDR(hdr)) {
3782 ASSERT(hdr->b_l1hdr.b_pabd == NULL);
3783 ASSERT(!HDR_HAS_RABD(hdr));
3784 /*
3785 * This buffer is cached on the 2nd Level ARC;
3786 * don't destroy the header.
3787 */
3788 arc_change_state(arc_l2c_only, hdr);
3789 /*
3790 * dropping from L1+L2 cached to L2-only,
3791 * realloc to remove the L1 header.
3792 */
3793 (void) arc_hdr_realloc(hdr, hdr_full_cache,
3794 hdr_l2only_cache);
3795 *real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
3796 } else {
3797 arc_change_state(arc_anon, hdr);
3798 arc_hdr_destroy(hdr);
3799 *real_evicted += HDR_FULL_SIZE;
3800 }
3801 return (bytes_evicted);
3802 }
3803
3804 ASSERT(state == arc_mru || state == arc_mfu || state == arc_uncached);
3805 evicted_state = (state == arc_uncached) ? arc_anon :
3806 ((state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost);
3807
3808 /* prefetch buffers have a minimum lifespan */
3809 if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3810 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3811 MSEC_TO_TICK(min_lifetime)) {
3812 ARCSTAT_BUMP(arcstat_evict_skip);
3813 return (bytes_evicted);
3814 }
3815
3816 if (HDR_HAS_L2HDR(hdr)) {
3817 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3818 } else {
3819 if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3820 ARCSTAT_INCR(arcstat_evict_l2_eligible,
3821 HDR_GET_LSIZE(hdr));
3822
3823 switch (state->arcs_state) {
3824 case ARC_STATE_MRU:
3825 ARCSTAT_INCR(
3826 arcstat_evict_l2_eligible_mru,
3827 HDR_GET_LSIZE(hdr));
3828 break;
3829 case ARC_STATE_MFU:
3830 ARCSTAT_INCR(
3831 arcstat_evict_l2_eligible_mfu,
3832 HDR_GET_LSIZE(hdr));
3833 break;
3834 default:
3835 break;
3836 }
3837 } else {
3838 ARCSTAT_INCR(arcstat_evict_l2_ineligible,
3839 HDR_GET_LSIZE(hdr));
3840 }
3841 }
3842
3843 bytes_evicted += arc_hdr_size(hdr);
3844 *real_evicted += arc_hdr_size(hdr);
3845
3846 /*
3847 * If this hdr is being evicted and has a compressed buffer then we
3848 * discard it here before we change states. This ensures that the
3849 * accounting is updated correctly in arc_free_data_impl().
3850 */
3851 if (hdr->b_l1hdr.b_pabd != NULL)
3852 arc_hdr_free_abd(hdr, B_FALSE);
3853
3854 if (HDR_HAS_RABD(hdr))
3855 arc_hdr_free_abd(hdr, B_TRUE);
3856
3857 arc_change_state(evicted_state, hdr);
3858 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3859 if (evicted_state == arc_anon) {
3860 arc_hdr_destroy(hdr);
3861 *real_evicted += HDR_FULL_SIZE;
3862 } else {
3863 ASSERT(HDR_IN_HASH_TABLE(hdr));
3864 }
3865
3866 return (bytes_evicted);
3867 }
3868
3869 static void
arc_set_need_free(void)3870 arc_set_need_free(void)
3871 {
3872 ASSERT(MUTEX_HELD(&arc_evict_lock));
3873 int64_t remaining = arc_free_memory() - arc_sys_free / 2;
3874 arc_evict_waiter_t *aw = list_tail(&arc_evict_waiters);
3875 if (aw == NULL) {
3876 arc_need_free = MAX(-remaining, 0);
3877 } else {
3878 arc_need_free =
3879 MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
3880 }
3881 }
3882
3883 static uint64_t
arc_evict_state_impl(multilist_t * ml,int idx,arc_buf_hdr_t * marker,uint64_t spa,uint64_t bytes)3884 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
3885 uint64_t spa, uint64_t bytes)
3886 {
3887 multilist_sublist_t *mls;
3888 uint64_t bytes_evicted = 0, real_evicted = 0;
3889 arc_buf_hdr_t *hdr;
3890 kmutex_t *hash_lock;
3891 uint_t evict_count = zfs_arc_evict_batch_limit;
3892
3893 ASSERT3P(marker, !=, NULL);
3894
3895 mls = multilist_sublist_lock_idx(ml, idx);
3896
3897 for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
3898 hdr = multilist_sublist_prev(mls, marker)) {
3899 if ((evict_count == 0) || (bytes_evicted >= bytes))
3900 break;
3901
3902 /*
3903 * To keep our iteration location, move the marker
3904 * forward. Since we're not holding hdr's hash lock, we
3905 * must be very careful and not remove 'hdr' from the
3906 * sublist. Otherwise, other consumers might mistake the
3907 * 'hdr' as not being on a sublist when they call the
3908 * multilist_link_active() function (they all rely on
3909 * the hash lock protecting concurrent insertions and
3910 * removals). multilist_sublist_move_forward() was
3911 * specifically implemented to ensure this is the case
3912 * (only 'marker' will be removed and re-inserted).
3913 */
3914 multilist_sublist_move_forward(mls, marker);
3915
3916 /*
3917 * The only case where the b_spa field should ever be
3918 * zero, is the marker headers inserted by
3919 * arc_evict_state(). It's possible for multiple threads
3920 * to be calling arc_evict_state() concurrently (e.g.
3921 * dsl_pool_close() and zio_inject_fault()), so we must
3922 * skip any markers we see from these other threads.
3923 */
3924 if (hdr->b_spa == 0)
3925 continue;
3926
3927 /* we're only interested in evicting buffers of a certain spa */
3928 if (spa != 0 && hdr->b_spa != spa) {
3929 ARCSTAT_BUMP(arcstat_evict_skip);
3930 continue;
3931 }
3932
3933 hash_lock = HDR_LOCK(hdr);
3934
3935 /*
3936 * We aren't calling this function from any code path
3937 * that would already be holding a hash lock, so we're
3938 * asserting on this assumption to be defensive in case
3939 * this ever changes. Without this check, it would be
3940 * possible to incorrectly increment arcstat_mutex_miss
3941 * below (e.g. if the code changed such that we called
3942 * this function with a hash lock held).
3943 */
3944 ASSERT(!MUTEX_HELD(hash_lock));
3945
3946 if (mutex_tryenter(hash_lock)) {
3947 uint64_t revicted;
3948 uint64_t evicted = arc_evict_hdr(hdr, &revicted);
3949 mutex_exit(hash_lock);
3950
3951 bytes_evicted += evicted;
3952 real_evicted += revicted;
3953
3954 /*
3955 * If evicted is zero, arc_evict_hdr() must have
3956 * decided to skip this header, don't increment
3957 * evict_count in this case.
3958 */
3959 if (evicted != 0)
3960 evict_count--;
3961
3962 } else {
3963 ARCSTAT_BUMP(arcstat_mutex_miss);
3964 }
3965 }
3966
3967 multilist_sublist_unlock(mls);
3968
3969 /*
3970 * Increment the count of evicted bytes, and wake up any threads that
3971 * are waiting for the count to reach this value. Since the list is
3972 * ordered by ascending aew_count, we pop off the beginning of the
3973 * list until we reach the end, or a waiter that's past the current
3974 * "count". Doing this outside the loop reduces the number of times
3975 * we need to acquire the global arc_evict_lock.
3976 *
3977 * Only wake when there's sufficient free memory in the system
3978 * (specifically, arc_sys_free/2, which by default is a bit more than
3979 * 1/64th of RAM). See the comments in arc_wait_for_eviction().
3980 */
3981 mutex_enter(&arc_evict_lock);
3982 arc_evict_count += real_evicted;
3983
3984 if (arc_free_memory() > arc_sys_free / 2) {
3985 arc_evict_waiter_t *aw;
3986 while ((aw = list_head(&arc_evict_waiters)) != NULL &&
3987 aw->aew_count <= arc_evict_count) {
3988 list_remove(&arc_evict_waiters, aw);
3989 cv_broadcast(&aw->aew_cv);
3990 }
3991 }
3992 arc_set_need_free();
3993 mutex_exit(&arc_evict_lock);
3994
3995 /*
3996 * If the ARC size is reduced from arc_c_max to arc_c_min (especially
3997 * if the average cached block is small), eviction can be on-CPU for
3998 * many seconds. To ensure that other threads that may be bound to
3999 * this CPU are able to make progress, make a voluntary preemption
4000 * call here.
4001 */
4002 kpreempt(KPREEMPT_SYNC);
4003
4004 return (bytes_evicted);
4005 }
4006
4007 static arc_buf_hdr_t *
arc_state_alloc_marker(void)4008 arc_state_alloc_marker(void)
4009 {
4010 arc_buf_hdr_t *marker = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
4011
4012 /*
4013 * A b_spa of 0 is used to indicate that this header is
4014 * a marker. This fact is used in arc_evict_state_impl().
4015 */
4016 marker->b_spa = 0;
4017
4018 return (marker);
4019 }
4020
4021 static void
arc_state_free_marker(arc_buf_hdr_t * marker)4022 arc_state_free_marker(arc_buf_hdr_t *marker)
4023 {
4024 kmem_cache_free(hdr_full_cache, marker);
4025 }
4026
4027 /*
4028 * Allocate an array of buffer headers used as placeholders during arc state
4029 * eviction.
4030 */
4031 static arc_buf_hdr_t **
arc_state_alloc_markers(int count)4032 arc_state_alloc_markers(int count)
4033 {
4034 arc_buf_hdr_t **markers;
4035
4036 markers = kmem_zalloc(sizeof (*markers) * count, KM_SLEEP);
4037 for (int i = 0; i < count; i++)
4038 markers[i] = arc_state_alloc_marker();
4039 return (markers);
4040 }
4041
4042 static void
arc_state_free_markers(arc_buf_hdr_t ** markers,int count)4043 arc_state_free_markers(arc_buf_hdr_t **markers, int count)
4044 {
4045 for (int i = 0; i < count; i++)
4046 arc_state_free_marker(markers[i]);
4047 kmem_free(markers, sizeof (*markers) * count);
4048 }
4049
4050 /*
4051 * Evict buffers from the given arc state, until we've removed the
4052 * specified number of bytes. Move the removed buffers to the
4053 * appropriate evict state.
4054 *
4055 * This function makes a "best effort". It skips over any buffers
4056 * it can't get a hash_lock on, and so, may not catch all candidates.
4057 * It may also return without evicting as much space as requested.
4058 *
4059 * If bytes is specified using the special value ARC_EVICT_ALL, this
4060 * will evict all available (i.e. unlocked and evictable) buffers from
4061 * the given arc state; which is used by arc_flush().
4062 */
4063 static uint64_t
arc_evict_state(arc_state_t * state,arc_buf_contents_t type,uint64_t spa,uint64_t bytes)4064 arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
4065 uint64_t bytes)
4066 {
4067 uint64_t total_evicted = 0;
4068 multilist_t *ml = &state->arcs_list[type];
4069 int num_sublists;
4070 arc_buf_hdr_t **markers;
4071
4072 num_sublists = multilist_get_num_sublists(ml);
4073
4074 /*
4075 * If we've tried to evict from each sublist, made some
4076 * progress, but still have not hit the target number of bytes
4077 * to evict, we want to keep trying. The markers allow us to
4078 * pick up where we left off for each individual sublist, rather
4079 * than starting from the tail each time.
4080 */
4081 if (zthr_iscurthread(arc_evict_zthr)) {
4082 markers = arc_state_evict_markers;
4083 ASSERT3S(num_sublists, <=, arc_state_evict_marker_count);
4084 } else {
4085 markers = arc_state_alloc_markers(num_sublists);
4086 }
4087 for (int i = 0; i < num_sublists; i++) {
4088 multilist_sublist_t *mls;
4089
4090 mls = multilist_sublist_lock_idx(ml, i);
4091 multilist_sublist_insert_tail(mls, markers[i]);
4092 multilist_sublist_unlock(mls);
4093 }
4094
4095 /*
4096 * While we haven't hit our target number of bytes to evict, or
4097 * we're evicting all available buffers.
4098 */
4099 while (total_evicted < bytes) {
4100 int sublist_idx = multilist_get_random_index(ml);
4101 uint64_t scan_evicted = 0;
4102
4103 /*
4104 * Start eviction using a randomly selected sublist,
4105 * this is to try and evenly balance eviction across all
4106 * sublists. Always starting at the same sublist
4107 * (e.g. index 0) would cause evictions to favor certain
4108 * sublists over others.
4109 */
4110 for (int i = 0; i < num_sublists; i++) {
4111 uint64_t bytes_remaining;
4112 uint64_t bytes_evicted;
4113
4114 if (total_evicted < bytes)
4115 bytes_remaining = bytes - total_evicted;
4116 else
4117 break;
4118
4119 bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
4120 markers[sublist_idx], spa, bytes_remaining);
4121
4122 scan_evicted += bytes_evicted;
4123 total_evicted += bytes_evicted;
4124
4125 /* we've reached the end, wrap to the beginning */
4126 if (++sublist_idx >= num_sublists)
4127 sublist_idx = 0;
4128 }
4129
4130 /*
4131 * If we didn't evict anything during this scan, we have
4132 * no reason to believe we'll evict more during another
4133 * scan, so break the loop.
4134 */
4135 if (scan_evicted == 0) {
4136 /* This isn't possible, let's make that obvious */
4137 ASSERT3S(bytes, !=, 0);
4138
4139 /*
4140 * When bytes is ARC_EVICT_ALL, the only way to
4141 * break the loop is when scan_evicted is zero.
4142 * In that case, we actually have evicted enough,
4143 * so we don't want to increment the kstat.
4144 */
4145 if (bytes != ARC_EVICT_ALL) {
4146 ASSERT3S(total_evicted, <, bytes);
4147 ARCSTAT_BUMP(arcstat_evict_not_enough);
4148 }
4149
4150 break;
4151 }
4152 }
4153
4154 for (int i = 0; i < num_sublists; i++) {
4155 multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
4156 multilist_sublist_remove(mls, markers[i]);
4157 multilist_sublist_unlock(mls);
4158 }
4159 if (markers != arc_state_evict_markers)
4160 arc_state_free_markers(markers, num_sublists);
4161
4162 return (total_evicted);
4163 }
4164
4165 /*
4166 * Flush all "evictable" data of the given type from the arc state
4167 * specified. This will not evict any "active" buffers (i.e. referenced).
4168 *
4169 * When 'retry' is set to B_FALSE, the function will make a single pass
4170 * over the state and evict any buffers that it can. Since it doesn't
4171 * continually retry the eviction, it might end up leaving some buffers
4172 * in the ARC due to lock misses.
4173 *
4174 * When 'retry' is set to B_TRUE, the function will continually retry the
4175 * eviction until *all* evictable buffers have been removed from the
4176 * state. As a result, if concurrent insertions into the state are
4177 * allowed (e.g. if the ARC isn't shutting down), this function might
4178 * wind up in an infinite loop, continually trying to evict buffers.
4179 */
4180 static uint64_t
arc_flush_state(arc_state_t * state,uint64_t spa,arc_buf_contents_t type,boolean_t retry)4181 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
4182 boolean_t retry)
4183 {
4184 uint64_t evicted = 0;
4185
4186 while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
4187 evicted += arc_evict_state(state, type, spa, ARC_EVICT_ALL);
4188
4189 if (!retry)
4190 break;
4191 }
4192
4193 return (evicted);
4194 }
4195
4196 /*
4197 * Evict the specified number of bytes from the state specified. This
4198 * function prevents us from trying to evict more from a state's list
4199 * than is "evictable", and to skip evicting altogether when passed a
4200 * negative value for "bytes". In contrast, arc_evict_state() will
4201 * evict everything it can, when passed a negative value for "bytes".
4202 */
4203 static uint64_t
arc_evict_impl(arc_state_t * state,arc_buf_contents_t type,int64_t bytes)4204 arc_evict_impl(arc_state_t *state, arc_buf_contents_t type, int64_t bytes)
4205 {
4206 uint64_t delta;
4207
4208 if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
4209 delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
4210 bytes);
4211 return (arc_evict_state(state, type, 0, delta));
4212 }
4213
4214 return (0);
4215 }
4216
4217 /*
4218 * Adjust specified fraction, taking into account initial ghost state(s) size,
4219 * ghost hit bytes towards increasing the fraction, ghost hit bytes towards
4220 * decreasing it, plus a balance factor, controlling the decrease rate, used
4221 * to balance metadata vs data.
4222 */
4223 static uint64_t
arc_evict_adj(uint64_t frac,uint64_t total,uint64_t up,uint64_t down,uint_t balance)4224 arc_evict_adj(uint64_t frac, uint64_t total, uint64_t up, uint64_t down,
4225 uint_t balance)
4226 {
4227 if (total < 8 || up + down == 0)
4228 return (frac);
4229
4230 /*
4231 * We should not have more ghost hits than ghost size, but they
4232 * may get close. Restrict maximum adjustment in that case.
4233 */
4234 if (up + down >= total / 4) {
4235 uint64_t scale = (up + down) / (total / 8);
4236 up /= scale;
4237 down /= scale;
4238 }
4239
4240 /* Get maximal dynamic range by choosing optimal shifts. */
4241 int s = highbit64(total);
4242 s = MIN(64 - s, 32);
4243
4244 uint64_t ofrac = (1ULL << 32) - frac;
4245
4246 if (frac >= 4 * ofrac)
4247 up /= frac / (2 * ofrac + 1);
4248 up = (up << s) / (total >> (32 - s));
4249 if (ofrac >= 4 * frac)
4250 down /= ofrac / (2 * frac + 1);
4251 down = (down << s) / (total >> (32 - s));
4252 down = down * 100 / balance;
4253
4254 return (frac + up - down);
4255 }
4256
4257 /*
4258 * Calculate (x * multiplier / divisor) without unnecesary overflows.
4259 */
4260 static uint64_t
arc_mf(uint64_t x,uint64_t multiplier,uint64_t divisor)4261 arc_mf(uint64_t x, uint64_t multiplier, uint64_t divisor)
4262 {
4263 uint64_t q = (x / divisor);
4264 uint64_t r = (x % divisor);
4265
4266 return ((q * multiplier) + ((r * multiplier) / divisor));
4267 }
4268
4269 /*
4270 * Evict buffers from the cache, such that arcstat_size is capped by arc_c.
4271 */
4272 static uint64_t
arc_evict(void)4273 arc_evict(void)
4274 {
4275 uint64_t bytes, total_evicted = 0;
4276 int64_t e, mrud, mrum, mfud, mfum, w;
4277 static uint64_t ogrd, ogrm, ogfd, ogfm;
4278 static uint64_t gsrd, gsrm, gsfd, gsfm;
4279 uint64_t ngrd, ngrm, ngfd, ngfm;
4280
4281 /* Get current size of ARC states we can evict from. */
4282 mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
4283 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
4284 mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
4285 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
4286 mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
4287 mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
4288 uint64_t d = mrud + mfud;
4289 uint64_t m = mrum + mfum;
4290 uint64_t t = d + m;
4291
4292 /* Get ARC ghost hits since last eviction. */
4293 ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
4294 uint64_t grd = ngrd - ogrd;
4295 ogrd = ngrd;
4296 ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
4297 uint64_t grm = ngrm - ogrm;
4298 ogrm = ngrm;
4299 ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
4300 uint64_t gfd = ngfd - ogfd;
4301 ogfd = ngfd;
4302 ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
4303 uint64_t gfm = ngfm - ogfm;
4304 ogfm = ngfm;
4305
4306 /* Adjust ARC states balance based on ghost hits. */
4307 arc_meta = arc_evict_adj(arc_meta, gsrd + gsrm + gsfd + gsfm,
4308 grm + gfm, grd + gfd, zfs_arc_meta_balance);
4309 arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
4310 arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);
4311
4312 uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
4313 uint64_t ac = arc_c;
4314 int64_t wt = t - (asize - ac);
4315
4316 /*
4317 * Try to reduce pinned dnodes if more than 3/4 of wanted metadata
4318 * target is not evictable or if they go over arc_dnode_limit.
4319 */
4320 int64_t prune = 0;
4321 int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
4322 int64_t nem = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA])
4323 + zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA])
4324 - zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA])
4325 - zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
4326 w = wt * (int64_t)(arc_meta >> 16) >> 16;
4327 if (nem > w * 3 / 4) {
4328 prune = dn / sizeof (dnode_t) *
4329 zfs_arc_dnode_reduce_percent / 100;
4330 if (nem < w && w > 4)
4331 prune = arc_mf(prune, nem - w * 3 / 4, w / 4);
4332 }
4333 if (dn > arc_dnode_limit) {
4334 prune = MAX(prune, (dn - arc_dnode_limit) / sizeof (dnode_t) *
4335 zfs_arc_dnode_reduce_percent / 100);
4336 }
4337 if (prune > 0)
4338 arc_prune_async(prune);
4339
4340 /* Evict MRU metadata. */
4341 w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
4342 e = MIN((int64_t)(asize - ac), (int64_t)(mrum - w));
4343 bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
4344 total_evicted += bytes;
4345 mrum -= bytes;
4346 asize -= bytes;
4347
4348 /* Evict MFU metadata. */
4349 w = wt * (int64_t)(arc_meta >> 16) >> 16;
4350 e = MIN((int64_t)(asize - ac), (int64_t)(m - bytes - w));
4351 bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
4352 total_evicted += bytes;
4353 mfum -= bytes;
4354 asize -= bytes;
4355
4356 /* Evict MRU data. */
4357 wt -= m - total_evicted;
4358 w = wt * (int64_t)(arc_pd >> 16) >> 16;
4359 e = MIN((int64_t)(asize - ac), (int64_t)(mrud - w));
4360 bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
4361 total_evicted += bytes;
4362 mrud -= bytes;
4363 asize -= bytes;
4364
4365 /* Evict MFU data. */
4366 e = asize - ac;
4367 bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
4368 mfud -= bytes;
4369 total_evicted += bytes;
4370
4371 /*
4372 * Evict ghost lists
4373 *
4374 * Size of each state's ghost list represents how much that state
4375 * may grow by shrinking the other states. Would it need to shrink
4376 * other states to zero (that is unlikely), its ghost size would be
4377 * equal to sum of other three state sizes. But excessive ghost
4378 * size may result in false ghost hits (too far back), that may
4379 * never result in real cache hits if several states are competing.
4380 * So choose some arbitraty point of 1/2 of other state sizes.
4381 */
4382 gsrd = (mrum + mfud + mfum) / 2;
4383 e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
4384 gsrd;
4385 (void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_DATA, e);
4386
4387 gsrm = (mrud + mfud + mfum) / 2;
4388 e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
4389 gsrm;
4390 (void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_METADATA, e);
4391
4392 gsfd = (mrud + mrum + mfum) / 2;
4393 e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
4394 gsfd;
4395 (void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_DATA, e);
4396
4397 gsfm = (mrud + mrum + mfud) / 2;
4398 e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
4399 gsfm;
4400 (void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_METADATA, e);
4401
4402 return (total_evicted);
4403 }
4404
4405 static void
arc_flush_impl(uint64_t guid,boolean_t retry)4406 arc_flush_impl(uint64_t guid, boolean_t retry)
4407 {
4408 ASSERT(!retry || guid == 0);
4409
4410 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
4411 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
4412
4413 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
4414 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
4415
4416 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
4417 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
4418
4419 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
4420 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
4421
4422 (void) arc_flush_state(arc_uncached, guid, ARC_BUFC_DATA, retry);
4423 (void) arc_flush_state(arc_uncached, guid, ARC_BUFC_METADATA, retry);
4424 }
4425
4426 void
arc_flush(spa_t * spa,boolean_t retry)4427 arc_flush(spa_t *spa, boolean_t retry)
4428 {
4429 /*
4430 * If retry is B_TRUE, a spa must not be specified since we have
4431 * no good way to determine if all of a spa's buffers have been
4432 * evicted from an arc state.
4433 */
4434 ASSERT(!retry || spa == NULL);
4435
4436 arc_flush_impl(spa != NULL ? spa_load_guid(spa) : 0, retry);
4437 }
4438
4439 static arc_async_flush_t *
arc_async_flush_add(uint64_t spa_guid,uint_t level)4440 arc_async_flush_add(uint64_t spa_guid, uint_t level)
4441 {
4442 arc_async_flush_t *af = kmem_alloc(sizeof (*af), KM_SLEEP);
4443 af->af_spa_guid = spa_guid;
4444 af->af_cache_level = level;
4445 taskq_init_ent(&af->af_tqent);
4446 list_link_init(&af->af_node);
4447
4448 mutex_enter(&arc_async_flush_lock);
4449 list_insert_tail(&arc_async_flush_list, af);
4450 mutex_exit(&arc_async_flush_lock);
4451
4452 return (af);
4453 }
4454
4455 static void
arc_async_flush_remove(uint64_t spa_guid,uint_t level)4456 arc_async_flush_remove(uint64_t spa_guid, uint_t level)
4457 {
4458 mutex_enter(&arc_async_flush_lock);
4459 for (arc_async_flush_t *af = list_head(&arc_async_flush_list);
4460 af != NULL; af = list_next(&arc_async_flush_list, af)) {
4461 if (af->af_spa_guid == spa_guid &&
4462 af->af_cache_level == level) {
4463 list_remove(&arc_async_flush_list, af);
4464 kmem_free(af, sizeof (*af));
4465 break;
4466 }
4467 }
4468 mutex_exit(&arc_async_flush_lock);
4469 }
4470
4471 static void
arc_flush_task(void * arg)4472 arc_flush_task(void *arg)
4473 {
4474 arc_async_flush_t *af = arg;
4475 hrtime_t start_time = gethrtime();
4476 uint64_t spa_guid = af->af_spa_guid;
4477
4478 arc_flush_impl(spa_guid, B_FALSE);
4479 arc_async_flush_remove(spa_guid, af->af_cache_level);
4480
4481 uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
4482 if (elaspsed > 0) {
4483 zfs_dbgmsg("spa %llu arc flushed in %llu ms",
4484 (u_longlong_t)spa_guid, (u_longlong_t)elaspsed);
4485 }
4486 }
4487
4488 /*
4489 * ARC buffers use the spa's load guid and can continue to exist after
4490 * the spa_t is gone (exported). The blocks are orphaned since each
4491 * spa import has a different load guid.
4492 *
4493 * It's OK if the spa is re-imported while this asynchronous flush is
4494 * still in progress. The new spa_load_guid will be different.
4495 *
4496 * Also, arc_fini will wait for any arc_flush_task to finish.
4497 */
4498 void
arc_flush_async(spa_t * spa)4499 arc_flush_async(spa_t *spa)
4500 {
4501 uint64_t spa_guid = spa_load_guid(spa);
4502 arc_async_flush_t *af = arc_async_flush_add(spa_guid, 1);
4503
4504 taskq_dispatch_ent(arc_flush_taskq, arc_flush_task,
4505 af, TQ_SLEEP, &af->af_tqent);
4506 }
4507
4508 /*
4509 * Check if a guid is still in-use as part of an async teardown task
4510 */
4511 boolean_t
arc_async_flush_guid_inuse(uint64_t spa_guid)4512 arc_async_flush_guid_inuse(uint64_t spa_guid)
4513 {
4514 mutex_enter(&arc_async_flush_lock);
4515 for (arc_async_flush_t *af = list_head(&arc_async_flush_list);
4516 af != NULL; af = list_next(&arc_async_flush_list, af)) {
4517 if (af->af_spa_guid == spa_guid) {
4518 mutex_exit(&arc_async_flush_lock);
4519 return (B_TRUE);
4520 }
4521 }
4522 mutex_exit(&arc_async_flush_lock);
4523 return (B_FALSE);
4524 }
4525
4526 uint64_t
arc_reduce_target_size(uint64_t to_free)4527 arc_reduce_target_size(uint64_t to_free)
4528 {
4529 /*
4530 * Get the actual arc size. Even if we don't need it, this updates
4531 * the aggsum lower bound estimate for arc_is_overflowing().
4532 */
4533 uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
4534
4535 /*
4536 * All callers want the ARC to actually evict (at least) this much
4537 * memory. Therefore we reduce from the lower of the current size and
4538 * the target size. This way, even if arc_c is much higher than
4539 * arc_size (as can be the case after many calls to arc_freed(), we will
4540 * immediately have arc_c < arc_size and therefore the arc_evict_zthr
4541 * will evict.
4542 */
4543 uint64_t c = arc_c;
4544 if (c > arc_c_min) {
4545 c = MIN(c, MAX(asize, arc_c_min));
4546 to_free = MIN(to_free, c - arc_c_min);
4547 arc_c = c - to_free;
4548 } else {
4549 to_free = 0;
4550 }
4551
4552 /*
4553 * Whether or not we reduced the target size, request eviction if the
4554 * current size is over it now, since caller obviously wants some RAM.
4555 */
4556 if (asize > arc_c) {
4557 /* See comment in arc_evict_cb_check() on why lock+flag */
4558 mutex_enter(&arc_evict_lock);
4559 arc_evict_needed = B_TRUE;
4560 mutex_exit(&arc_evict_lock);
4561 zthr_wakeup(arc_evict_zthr);
4562 }
4563
4564 return (to_free);
4565 }
4566
4567 /*
4568 * Determine if the system is under memory pressure and is asking
4569 * to reclaim memory. A return value of B_TRUE indicates that the system
4570 * is under memory pressure and that the arc should adjust accordingly.
4571 */
4572 boolean_t
arc_reclaim_needed(void)4573 arc_reclaim_needed(void)
4574 {
4575 return (arc_available_memory() < 0);
4576 }
4577
4578 void
arc_kmem_reap_soon(void)4579 arc_kmem_reap_soon(void)
4580 {
4581 size_t i;
4582 kmem_cache_t *prev_cache = NULL;
4583 kmem_cache_t *prev_data_cache = NULL;
4584
4585 #ifdef _KERNEL
4586 #if defined(_ILP32)
4587 /*
4588 * Reclaim unused memory from all kmem caches.
4589 */
4590 kmem_reap();
4591 #endif
4592 #endif
4593
4594 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
4595 #if defined(_ILP32)
4596 /* reach upper limit of cache size on 32-bit */
4597 if (zio_buf_cache[i] == NULL)
4598 break;
4599 #endif
4600 if (zio_buf_cache[i] != prev_cache) {
4601 prev_cache = zio_buf_cache[i];
4602 kmem_cache_reap_now(zio_buf_cache[i]);
4603 }
4604 if (zio_data_buf_cache[i] != prev_data_cache) {
4605 prev_data_cache = zio_data_buf_cache[i];
4606 kmem_cache_reap_now(zio_data_buf_cache[i]);
4607 }
4608 }
4609 kmem_cache_reap_now(buf_cache);
4610 kmem_cache_reap_now(hdr_full_cache);
4611 kmem_cache_reap_now(hdr_l2only_cache);
4612 kmem_cache_reap_now(zfs_btree_leaf_cache);
4613 abd_cache_reap_now();
4614 }
4615
4616 static boolean_t
arc_evict_cb_check(void * arg,zthr_t * zthr)4617 arc_evict_cb_check(void *arg, zthr_t *zthr)
4618 {
4619 (void) arg, (void) zthr;
4620
4621 #ifdef ZFS_DEBUG
4622 /*
4623 * This is necessary in order to keep the kstat information
4624 * up to date for tools that display kstat data such as the
4625 * mdb ::arc dcmd and the Linux crash utility. These tools
4626 * typically do not call kstat's update function, but simply
4627 * dump out stats from the most recent update. Without
4628 * this call, these commands may show stale stats for the
4629 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
4630 * with this call, the data might be out of date if the
4631 * evict thread hasn't been woken recently; but that should
4632 * suffice. The arc_state_t structures can be queried
4633 * directly if more accurate information is needed.
4634 */
4635 if (arc_ksp != NULL)
4636 arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4637 #endif
4638
4639 /*
4640 * We have to rely on arc_wait_for_eviction() to tell us when to
4641 * evict, rather than checking if we are overflowing here, so that we
4642 * are sure to not leave arc_wait_for_eviction() waiting on aew_cv.
4643 * If we have become "not overflowing" since arc_wait_for_eviction()
4644 * checked, we need to wake it up. We could broadcast the CV here,
4645 * but arc_wait_for_eviction() may have not yet gone to sleep. We
4646 * would need to use a mutex to ensure that this function doesn't
4647 * broadcast until arc_wait_for_eviction() has gone to sleep (e.g.
4648 * the arc_evict_lock). However, the lock ordering of such a lock
4649 * would necessarily be incorrect with respect to the zthr_lock,
4650 * which is held before this function is called, and is held by
4651 * arc_wait_for_eviction() when it calls zthr_wakeup().
4652 */
4653 if (arc_evict_needed)
4654 return (B_TRUE);
4655
4656 /*
4657 * If we have buffers in uncached state, evict them periodically.
4658 */
4659 return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
4660 zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
4661 ddi_get_lbolt() - arc_last_uncached_flush >
4662 MSEC_TO_TICK(arc_min_prefetch_ms / 2)));
4663 }
4664
4665 /*
4666 * Keep arc_size under arc_c by running arc_evict which evicts data
4667 * from the ARC.
4668 */
4669 static void
arc_evict_cb(void * arg,zthr_t * zthr)4670 arc_evict_cb(void *arg, zthr_t *zthr)
4671 {
4672 (void) arg;
4673
4674 uint64_t evicted = 0;
4675 fstrans_cookie_t cookie = spl_fstrans_mark();
4676
4677 /* Always try to evict from uncached state. */
4678 arc_last_uncached_flush = ddi_get_lbolt();
4679 evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_DATA, B_FALSE);
4680 evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_METADATA, B_FALSE);
4681
4682 /* Evict from other states only if told to. */
4683 if (arc_evict_needed)
4684 evicted += arc_evict();
4685
4686 /*
4687 * If evicted is zero, we couldn't evict anything
4688 * via arc_evict(). This could be due to hash lock
4689 * collisions, but more likely due to the majority of
4690 * arc buffers being unevictable. Therefore, even if
4691 * arc_size is above arc_c, another pass is unlikely to
4692 * be helpful and could potentially cause us to enter an
4693 * infinite loop. Additionally, zthr_iscancelled() is
4694 * checked here so that if the arc is shutting down, the
4695 * broadcast will wake any remaining arc evict waiters.
4696 *
4697 * Note we cancel using zthr instead of arc_evict_zthr
4698 * because the latter may not yet be initializd when the
4699 * callback is first invoked.
4700 */
4701 mutex_enter(&arc_evict_lock);
4702 arc_evict_needed = !zthr_iscancelled(zthr) &&
4703 evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
4704 if (!arc_evict_needed) {
4705 /*
4706 * We're either no longer overflowing, or we
4707 * can't evict anything more, so we should wake
4708 * arc_get_data_impl() sooner.
4709 */
4710 arc_evict_waiter_t *aw;
4711 while ((aw = list_remove_head(&arc_evict_waiters)) != NULL) {
4712 cv_broadcast(&aw->aew_cv);
4713 }
4714 arc_set_need_free();
4715 }
4716 mutex_exit(&arc_evict_lock);
4717 spl_fstrans_unmark(cookie);
4718 }
4719
4720 static boolean_t
arc_reap_cb_check(void * arg,zthr_t * zthr)4721 arc_reap_cb_check(void *arg, zthr_t *zthr)
4722 {
4723 (void) arg, (void) zthr;
4724
4725 int64_t free_memory = arc_available_memory();
4726 static int reap_cb_check_counter = 0;
4727
4728 /*
4729 * If a kmem reap is already active, don't schedule more. We must
4730 * check for this because kmem_cache_reap_soon() won't actually
4731 * block on the cache being reaped (this is to prevent callers from
4732 * becoming implicitly blocked by a system-wide kmem reap -- which,
4733 * on a system with many, many full magazines, can take minutes).
4734 */
4735 if (!kmem_cache_reap_active() && free_memory < 0) {
4736
4737 arc_no_grow = B_TRUE;
4738 arc_warm = B_TRUE;
4739 /*
4740 * Wait at least zfs_grow_retry (default 5) seconds
4741 * before considering growing.
4742 */
4743 arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
4744 return (B_TRUE);
4745 } else if (free_memory < arc_c >> arc_no_grow_shift) {
4746 arc_no_grow = B_TRUE;
4747 } else if (gethrtime() >= arc_growtime) {
4748 arc_no_grow = B_FALSE;
4749 }
4750
4751 /*
4752 * Called unconditionally every 60 seconds to reclaim unused
4753 * zstd compression and decompression context. This is done
4754 * here to avoid the need for an independent thread.
4755 */
4756 if (!((reap_cb_check_counter++) % 60))
4757 zfs_zstd_cache_reap_now();
4758
4759 return (B_FALSE);
4760 }
4761
4762 /*
4763 * Keep enough free memory in the system by reaping the ARC's kmem
4764 * caches. To cause more slabs to be reapable, we may reduce the
4765 * target size of the cache (arc_c), causing the arc_evict_cb()
4766 * to free more buffers.
4767 */
4768 static void
arc_reap_cb(void * arg,zthr_t * zthr)4769 arc_reap_cb(void *arg, zthr_t *zthr)
4770 {
4771 int64_t can_free, free_memory, to_free;
4772
4773 (void) arg, (void) zthr;
4774 fstrans_cookie_t cookie = spl_fstrans_mark();
4775
4776 /*
4777 * Kick off asynchronous kmem_reap()'s of all our caches.
4778 */
4779 arc_kmem_reap_soon();
4780
4781 /*
4782 * Wait at least arc_kmem_cache_reap_retry_ms between
4783 * arc_kmem_reap_soon() calls. Without this check it is possible to
4784 * end up in a situation where we spend lots of time reaping
4785 * caches, while we're near arc_c_min. Waiting here also gives the
4786 * subsequent free memory check a chance of finding that the
4787 * asynchronous reap has already freed enough memory, and we don't
4788 * need to call arc_reduce_target_size().
4789 */
4790 delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);
4791
4792 /*
4793 * Reduce the target size as needed to maintain the amount of free
4794 * memory in the system at a fraction of the arc_size (1/128th by
4795 * default). If oversubscribed (free_memory < 0) then reduce the
4796 * target arc_size by the deficit amount plus the fractional
4797 * amount. If free memory is positive but less than the fractional
4798 * amount, reduce by what is needed to hit the fractional amount.
4799 */
4800 free_memory = arc_available_memory();
4801 can_free = arc_c - arc_c_min;
4802 to_free = (MAX(can_free, 0) >> arc_shrink_shift) - free_memory;
4803 if (to_free > 0)
4804 arc_reduce_target_size(to_free);
4805 spl_fstrans_unmark(cookie);
4806 }
4807
4808 #ifdef _KERNEL
4809 /*
4810 * Determine the amount of memory eligible for eviction contained in the
4811 * ARC. All clean data reported by the ghost lists can always be safely
4812 * evicted. Due to arc_c_min, the same does not hold for all clean data
4813 * contained by the regular mru and mfu lists.
4814 *
4815 * In the case of the regular mru and mfu lists, we need to report as
4816 * much clean data as possible, such that evicting that same reported
4817 * data will not bring arc_size below arc_c_min. Thus, in certain
4818 * circumstances, the total amount of clean data in the mru and mfu
4819 * lists might not actually be evictable.
4820 *
4821 * The following two distinct cases are accounted for:
4822 *
4823 * 1. The sum of the amount of dirty data contained by both the mru and
4824 * mfu lists, plus the ARC's other accounting (e.g. the anon list),
4825 * is greater than or equal to arc_c_min.
4826 * (i.e. amount of dirty data >= arc_c_min)
4827 *
4828 * This is the easy case; all clean data contained by the mru and mfu
4829 * lists is evictable. Evicting all clean data can only drop arc_size
4830 * to the amount of dirty data, which is greater than arc_c_min.
4831 *
4832 * 2. The sum of the amount of dirty data contained by both the mru and
4833 * mfu lists, plus the ARC's other accounting (e.g. the anon list),
4834 * is less than arc_c_min.
4835 * (i.e. arc_c_min > amount of dirty data)
4836 *
4837 * 2.1. arc_size is greater than or equal arc_c_min.
4838 * (i.e. arc_size >= arc_c_min > amount of dirty data)
4839 *
4840 * In this case, not all clean data from the regular mru and mfu
4841 * lists is actually evictable; we must leave enough clean data
4842 * to keep arc_size above arc_c_min. Thus, the maximum amount of
4843 * evictable data from the two lists combined, is exactly the
4844 * difference between arc_size and arc_c_min.
4845 *
4846 * 2.2. arc_size is less than arc_c_min
4847 * (i.e. arc_c_min > arc_size > amount of dirty data)
4848 *
4849 * In this case, none of the data contained in the mru and mfu
4850 * lists is evictable, even if it's clean. Since arc_size is
4851 * already below arc_c_min, evicting any more would only
4852 * increase this negative difference.
4853 */
4854
4855 #endif /* _KERNEL */
4856
4857 /*
4858 * Adapt arc info given the number of bytes we are trying to add and
4859 * the state that we are coming from. This function is only called
4860 * when we are adding new content to the cache.
4861 */
4862 static void
arc_adapt(uint64_t bytes)4863 arc_adapt(uint64_t bytes)
4864 {
4865 /*
4866 * Wake reap thread if we do not have any available memory
4867 */
4868 if (arc_reclaim_needed()) {
4869 zthr_wakeup(arc_reap_zthr);
4870 return;
4871 }
4872
4873 if (arc_no_grow)
4874 return;
4875
4876 if (arc_c >= arc_c_max)
4877 return;
4878
4879 /*
4880 * If we're within (2 * maxblocksize) bytes of the target
4881 * cache size, increment the target cache size
4882 */
4883 if (aggsum_upper_bound(&arc_sums.arcstat_size) +
4884 2 * SPA_MAXBLOCKSIZE >= arc_c) {
4885 uint64_t dc = MAX(bytes, SPA_OLD_MAXBLOCKSIZE);
4886 if (atomic_add_64_nv(&arc_c, dc) > arc_c_max)
4887 arc_c = arc_c_max;
4888 }
4889 }
4890
4891 /*
4892 * Check if ARC current size has grown past our upper thresholds.
4893 */
4894 static arc_ovf_level_t
arc_is_overflowing(boolean_t lax,boolean_t use_reserve)4895 arc_is_overflowing(boolean_t lax, boolean_t use_reserve)
4896 {
4897 /*
4898 * We just compare the lower bound here for performance reasons. Our
4899 * primary goals are to make sure that the arc never grows without
4900 * bound, and that it can reach its maximum size. This check
4901 * accomplishes both goals. The maximum amount we could run over by is
4902 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
4903 * in the ARC. In practice, that's in the tens of MB, which is low
4904 * enough to be safe.
4905 */
4906 int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) - arc_c -
4907 zfs_max_recordsize;
4908
4909 /* Always allow at least one block of overflow. */
4910 if (over < 0)
4911 return (ARC_OVF_NONE);
4912
4913 /* If we are under memory pressure, report severe overflow. */
4914 if (!lax)
4915 return (ARC_OVF_SEVERE);
4916
4917 /* We are not under pressure, so be more or less relaxed. */
4918 int64_t overflow = (arc_c >> zfs_arc_overflow_shift) / 2;
4919 if (use_reserve)
4920 overflow *= 3;
4921 return (over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
4922 }
4923
4924 static abd_t *
arc_get_data_abd(arc_buf_hdr_t * hdr,uint64_t size,const void * tag,int alloc_flags)4925 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
4926 int alloc_flags)
4927 {
4928 arc_buf_contents_t type = arc_buf_type(hdr);
4929
4930 arc_get_data_impl(hdr, size, tag, alloc_flags);
4931 if (alloc_flags & ARC_HDR_ALLOC_LINEAR)
4932 return (abd_alloc_linear(size, type == ARC_BUFC_METADATA));
4933 else
4934 return (abd_alloc(size, type == ARC_BUFC_METADATA));
4935 }
4936
4937 static void *
arc_get_data_buf(arc_buf_hdr_t * hdr,uint64_t size,const void * tag)4938 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
4939 {
4940 arc_buf_contents_t type = arc_buf_type(hdr);
4941
4942 arc_get_data_impl(hdr, size, tag, 0);
4943 if (type == ARC_BUFC_METADATA) {
4944 return (zio_buf_alloc(size));
4945 } else {
4946 ASSERT(type == ARC_BUFC_DATA);
4947 return (zio_data_buf_alloc(size));
4948 }
4949 }
4950
4951 /*
4952 * Wait for the specified amount of data (in bytes) to be evicted from the
4953 * ARC, and for there to be sufficient free memory in the system.
4954 * The lax argument specifies that caller does not have a specific reason
4955 * to wait, not aware of any memory pressure. Low memory handlers though
4956 * should set it to B_FALSE to wait for all required evictions to complete.
4957 * The use_reserve argument allows some callers to wait less than others
4958 * to not block critical code paths, possibly blocking other resources.
4959 */
4960 void
arc_wait_for_eviction(uint64_t amount,boolean_t lax,boolean_t use_reserve)4961 arc_wait_for_eviction(uint64_t amount, boolean_t lax, boolean_t use_reserve)
4962 {
4963 switch (arc_is_overflowing(lax, use_reserve)) {
4964 case ARC_OVF_NONE:
4965 return;
4966 case ARC_OVF_SOME:
4967 /*
4968 * This is a bit racy without taking arc_evict_lock, but the
4969 * worst that can happen is we either call zthr_wakeup() extra
4970 * time due to race with other thread here, or the set flag
4971 * get cleared by arc_evict_cb(), which is unlikely due to
4972 * big hysteresis, but also not important since at this level
4973 * of overflow the eviction is purely advisory. Same time
4974 * taking the global lock here every time without waiting for
4975 * the actual eviction creates a significant lock contention.
4976 */
4977 if (!arc_evict_needed) {
4978 arc_evict_needed = B_TRUE;
4979 zthr_wakeup(arc_evict_zthr);
4980 }
4981 return;
4982 case ARC_OVF_SEVERE:
4983 default:
4984 {
4985 arc_evict_waiter_t aw;
4986 list_link_init(&aw.aew_node);
4987 cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL);
4988
4989 uint64_t last_count = 0;
4990 mutex_enter(&arc_evict_lock);
4991 if (!list_is_empty(&arc_evict_waiters)) {
4992 arc_evict_waiter_t *last =
4993 list_tail(&arc_evict_waiters);
4994 last_count = last->aew_count;
4995 } else if (!arc_evict_needed) {
4996 arc_evict_needed = B_TRUE;
4997 zthr_wakeup(arc_evict_zthr);
4998 }
4999 /*
5000 * Note, the last waiter's count may be less than
5001 * arc_evict_count if we are low on memory in which
5002 * case arc_evict_state_impl() may have deferred
5003 * wakeups (but still incremented arc_evict_count).
5004 */
5005 aw.aew_count = MAX(last_count, arc_evict_count) + amount;
5006
5007 list_insert_tail(&arc_evict_waiters, &aw);
5008
5009 arc_set_need_free();
5010
5011 DTRACE_PROBE3(arc__wait__for__eviction,
5012 uint64_t, amount,
5013 uint64_t, arc_evict_count,
5014 uint64_t, aw.aew_count);
5015
5016 /*
5017 * We will be woken up either when arc_evict_count reaches
5018 * aew_count, or when the ARC is no longer overflowing and
5019 * eviction completes.
5020 * In case of "false" wakeup, we will still be on the list.
5021 */
5022 do {
5023 cv_wait(&aw.aew_cv, &arc_evict_lock);
5024 } while (list_link_active(&aw.aew_node));
5025 mutex_exit(&arc_evict_lock);
5026
5027 cv_destroy(&aw.aew_cv);
5028 }
5029 }
5030 }
5031
5032 /*
5033 * Allocate a block and return it to the caller. If we are hitting the
5034 * hard limit for the cache size, we must sleep, waiting for the eviction
5035 * thread to catch up. If we're past the target size but below the hard
5036 * limit, we'll only signal the reclaim thread and continue on.
5037 */
5038 static void
arc_get_data_impl(arc_buf_hdr_t * hdr,uint64_t size,const void * tag,int alloc_flags)5039 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
5040 int alloc_flags)
5041 {
5042 arc_adapt(size);
5043
5044 /*
5045 * If arc_size is currently overflowing, we must be adding data
5046 * faster than we are evicting. To ensure we don't compound the
5047 * problem by adding more data and forcing arc_size to grow even
5048 * further past it's target size, we wait for the eviction thread to
5049 * make some progress. We also wait for there to be sufficient free
5050 * memory in the system, as measured by arc_free_memory().
5051 *
5052 * Specifically, we wait for zfs_arc_eviction_pct percent of the
5053 * requested size to be evicted. This should be more than 100%, to
5054 * ensure that that progress is also made towards getting arc_size
5055 * under arc_c. See the comment above zfs_arc_eviction_pct.
5056 */
5057 arc_wait_for_eviction(size * zfs_arc_eviction_pct / 100,
5058 B_TRUE, alloc_flags & ARC_HDR_USE_RESERVE);
5059
5060 arc_buf_contents_t type = arc_buf_type(hdr);
5061 if (type == ARC_BUFC_METADATA) {
5062 arc_space_consume(size, ARC_SPACE_META);
5063 } else {
5064 arc_space_consume(size, ARC_SPACE_DATA);
5065 }
5066
5067 /*
5068 * Update the state size. Note that ghost states have a
5069 * "ghost size" and so don't need to be updated.
5070 */
5071 arc_state_t *state = hdr->b_l1hdr.b_state;
5072 if (!GHOST_STATE(state)) {
5073
5074 (void) zfs_refcount_add_many(&state->arcs_size[type], size,
5075 tag);
5076
5077 /*
5078 * If this is reached via arc_read, the link is
5079 * protected by the hash lock. If reached via
5080 * arc_buf_alloc, the header should not be accessed by
5081 * any other thread. And, if reached via arc_read_done,
5082 * the hash lock will protect it if it's found in the
5083 * hash table; otherwise no other thread should be
5084 * trying to [add|remove]_reference it.
5085 */
5086 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5087 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5088 (void) zfs_refcount_add_many(&state->arcs_esize[type],
5089 size, tag);
5090 }
5091 }
5092 }
5093
5094 static void
arc_free_data_abd(arc_buf_hdr_t * hdr,abd_t * abd,uint64_t size,const void * tag)5095 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,
5096 const void *tag)
5097 {
5098 arc_free_data_impl(hdr, size, tag);
5099 abd_free(abd);
5100 }
5101
5102 static void
arc_free_data_buf(arc_buf_hdr_t * hdr,void * buf,uint64_t size,const void * tag)5103 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)
5104 {
5105 arc_buf_contents_t type = arc_buf_type(hdr);
5106
5107 arc_free_data_impl(hdr, size, tag);
5108 if (type == ARC_BUFC_METADATA) {
5109 zio_buf_free(buf, size);
5110 } else {
5111 ASSERT(type == ARC_BUFC_DATA);
5112 zio_data_buf_free(buf, size);
5113 }
5114 }
5115
5116 /*
5117 * Free the arc data buffer.
5118 */
5119 static void
arc_free_data_impl(arc_buf_hdr_t * hdr,uint64_t size,const void * tag)5120 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
5121 {
5122 arc_state_t *state = hdr->b_l1hdr.b_state;
5123 arc_buf_contents_t type = arc_buf_type(hdr);
5124
5125 /* protected by hash lock, if in the hash table */
5126 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5127 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5128 ASSERT(state != arc_anon && state != arc_l2c_only);
5129
5130 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
5131 size, tag);
5132 }
5133 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);
5134
5135 VERIFY3U(hdr->b_type, ==, type);
5136 if (type == ARC_BUFC_METADATA) {
5137 arc_space_return(size, ARC_SPACE_META);
5138 } else {
5139 ASSERT(type == ARC_BUFC_DATA);
5140 arc_space_return(size, ARC_SPACE_DATA);
5141 }
5142 }
5143
5144 /*
5145 * This routine is called whenever a buffer is accessed.
5146 */
5147 static void
arc_access(arc_buf_hdr_t * hdr,arc_flags_t arc_flags,boolean_t hit)5148 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
5149 {
5150 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
5151 ASSERT(HDR_HAS_L1HDR(hdr));
5152
5153 /*
5154 * Update buffer prefetch status.
5155 */
5156 boolean_t was_prefetch = HDR_PREFETCH(hdr);
5157 boolean_t now_prefetch = arc_flags & ARC_FLAG_PREFETCH;
5158 if (was_prefetch != now_prefetch) {
5159 if (was_prefetch) {
5160 ARCSTAT_CONDSTAT(hit, demand_hit, demand_iohit,
5161 HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
5162 prefetch);
5163 }
5164 if (HDR_HAS_L2HDR(hdr))
5165 l2arc_hdr_arcstats_decrement_state(hdr);
5166 if (was_prefetch) {
5167 arc_hdr_clear_flags(hdr,
5168 ARC_FLAG_PREFETCH | ARC_FLAG_PRESCIENT_PREFETCH);
5169 } else {
5170 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5171 }
5172 if (HDR_HAS_L2HDR(hdr))
5173 l2arc_hdr_arcstats_increment_state(hdr);
5174 }
5175 if (now_prefetch) {
5176 if (arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
5177 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5178 ARCSTAT_BUMP(arcstat_prescient_prefetch);
5179 } else {
5180 ARCSTAT_BUMP(arcstat_predictive_prefetch);
5181 }
5182 }
5183 if (arc_flags & ARC_FLAG_L2CACHE)
5184 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5185
5186 clock_t now = ddi_get_lbolt();
5187 if (hdr->b_l1hdr.b_state == arc_anon) {
5188 arc_state_t *new_state;
5189 /*
5190 * This buffer is not in the cache, and does not appear in
5191 * our "ghost" lists. Add it to the MRU or uncached state.
5192 */
5193 ASSERT0(hdr->b_l1hdr.b_arc_access);
5194 hdr->b_l1hdr.b_arc_access = now;
5195 if (HDR_UNCACHED(hdr)) {
5196 new_state = arc_uncached;
5197 DTRACE_PROBE1(new_state__uncached, arc_buf_hdr_t *,
5198 hdr);
5199 } else {
5200 new_state = arc_mru;
5201 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5202 }
5203 arc_change_state(new_state, hdr);
5204 } else if (hdr->b_l1hdr.b_state == arc_mru) {
5205 /*
5206 * This buffer has been accessed once recently and either
5207 * its read is still in progress or it is in the cache.
5208 */
5209 if (HDR_IO_IN_PROGRESS(hdr)) {
5210 hdr->b_l1hdr.b_arc_access = now;
5211 return;
5212 }
5213 hdr->b_l1hdr.b_mru_hits++;
5214 ARCSTAT_BUMP(arcstat_mru_hits);
5215
5216 /*
5217 * If the previous access was a prefetch, then it already
5218 * handled possible promotion, so nothing more to do for now.
5219 */
5220 if (was_prefetch) {
5221 hdr->b_l1hdr.b_arc_access = now;
5222 return;
5223 }
5224
5225 /*
5226 * If more than ARC_MINTIME have passed from the previous
5227 * hit, promote the buffer to the MFU state.
5228 */
5229 if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
5230 ARC_MINTIME)) {
5231 hdr->b_l1hdr.b_arc_access = now;
5232 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5233 arc_change_state(arc_mfu, hdr);
5234 }
5235 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5236 arc_state_t *new_state;
5237 /*
5238 * This buffer has been accessed once recently, but was
5239 * evicted from the cache. Would we have bigger MRU, it
5240 * would be an MRU hit, so handle it the same way, except
5241 * we don't need to check the previous access time.
5242 */
5243 hdr->b_l1hdr.b_mru_ghost_hits++;
5244 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
5245 hdr->b_l1hdr.b_arc_access = now;
5246 wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
5247 arc_hdr_size(hdr));
5248 if (was_prefetch) {
5249 new_state = arc_mru;
5250 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5251 } else {
5252 new_state = arc_mfu;
5253 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5254 }
5255 arc_change_state(new_state, hdr);
5256 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
5257 /*
5258 * This buffer has been accessed more than once and either
5259 * still in the cache or being restored from one of ghosts.
5260 */
5261 if (!HDR_IO_IN_PROGRESS(hdr)) {
5262 hdr->b_l1hdr.b_mfu_hits++;
5263 ARCSTAT_BUMP(arcstat_mfu_hits);
5264 }
5265 hdr->b_l1hdr.b_arc_access = now;
5266 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5267 /*
5268 * This buffer has been accessed more than once recently, but
5269 * has been evicted from the cache. Would we have bigger MFU
5270 * it would stay in cache, so move it back to MFU state.
5271 */
5272 hdr->b_l1hdr.b_mfu_ghost_hits++;
5273 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
5274 hdr->b_l1hdr.b_arc_access = now;
5275 wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
5276 arc_hdr_size(hdr));
5277 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5278 arc_change_state(arc_mfu, hdr);
5279 } else if (hdr->b_l1hdr.b_state == arc_uncached) {
5280 /*
5281 * This buffer is uncacheable, but we got a hit. Probably
5282 * a demand read after prefetch. Nothing more to do here.
5283 */
5284 if (!HDR_IO_IN_PROGRESS(hdr))
5285 ARCSTAT_BUMP(arcstat_uncached_hits);
5286 hdr->b_l1hdr.b_arc_access = now;
5287 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5288 /*
5289 * This buffer is on the 2nd Level ARC and was not accessed
5290 * for a long time, so treat it as new and put into MRU.
5291 */
5292 hdr->b_l1hdr.b_arc_access = now;
5293 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5294 arc_change_state(arc_mru, hdr);
5295 } else {
5296 cmn_err(CE_PANIC, "invalid arc state 0x%p",
5297 hdr->b_l1hdr.b_state);
5298 }
5299 }
5300
5301 /*
5302 * This routine is called by dbuf_hold() to update the arc_access() state
5303 * which otherwise would be skipped for entries in the dbuf cache.
5304 */
5305 void
arc_buf_access(arc_buf_t * buf)5306 arc_buf_access(arc_buf_t *buf)
5307 {
5308 arc_buf_hdr_t *hdr = buf->b_hdr;
5309
5310 /*
5311 * Avoid taking the hash_lock when possible as an optimization.
5312 * The header must be checked again under the hash_lock in order
5313 * to handle the case where it is concurrently being released.
5314 */
5315 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
5316 return;
5317
5318 kmutex_t *hash_lock = HDR_LOCK(hdr);
5319 mutex_enter(hash_lock);
5320
5321 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5322 mutex_exit(hash_lock);
5323 ARCSTAT_BUMP(arcstat_access_skip);
5324 return;
5325 }
5326
5327 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5328 hdr->b_l1hdr.b_state == arc_mfu ||
5329 hdr->b_l1hdr.b_state == arc_uncached);
5330
5331 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5332 arc_access(hdr, 0, B_TRUE);
5333 mutex_exit(hash_lock);
5334
5335 ARCSTAT_BUMP(arcstat_hits);
5336 ARCSTAT_CONDSTAT(B_TRUE /* demand */, demand, prefetch,
5337 !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5338 }
5339
5340 /* a generic arc_read_done_func_t which you can use */
5341 void
arc_bcopy_func(zio_t * zio,const zbookmark_phys_t * zb,const blkptr_t * bp,arc_buf_t * buf,void * arg)5342 arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5343 arc_buf_t *buf, void *arg)
5344 {
5345 (void) zio, (void) zb, (void) bp;
5346
5347 if (buf == NULL)
5348 return;
5349
5350 memcpy(arg, buf->b_data, arc_buf_size(buf));
5351 arc_buf_destroy(buf, arg);
5352 }
5353
5354 /* a generic arc_read_done_func_t */
5355 void
arc_getbuf_func(zio_t * zio,const zbookmark_phys_t * zb,const blkptr_t * bp,arc_buf_t * buf,void * arg)5356 arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
5357 arc_buf_t *buf, void *arg)
5358 {
5359 (void) zb, (void) bp;
5360 arc_buf_t **bufp = arg;
5361
5362 if (buf == NULL) {
5363 ASSERT(zio == NULL || zio->io_error != 0);
5364 *bufp = NULL;
5365 } else {
5366 ASSERT(zio == NULL || zio->io_error == 0);
5367 *bufp = buf;
5368 ASSERT(buf->b_data != NULL);
5369 }
5370 }
5371
5372 static void
arc_hdr_verify(arc_buf_hdr_t * hdr,blkptr_t * bp)5373 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5374 {
5375 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
5376 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5377 ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
5378 } else {
5379 if (HDR_COMPRESSION_ENABLED(hdr)) {
5380 ASSERT3U(arc_hdr_get_compress(hdr), ==,
5381 BP_GET_COMPRESS(bp));
5382 }
5383 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5384 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5385 ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
5386 }
5387 }
5388
5389 static void
arc_read_done(zio_t * zio)5390 arc_read_done(zio_t *zio)
5391 {
5392 blkptr_t *bp = zio->io_bp;
5393 arc_buf_hdr_t *hdr = zio->io_private;
5394 kmutex_t *hash_lock = NULL;
5395 arc_callback_t *callback_list;
5396 arc_callback_t *acb;
5397
5398 /*
5399 * The hdr was inserted into hash-table and removed from lists
5400 * prior to starting I/O. We should find this header, since
5401 * it's in the hash table, and it should be legit since it's
5402 * not possible to evict it during the I/O. The only possible
5403 * reason for it not to be found is if we were freed during the
5404 * read.
5405 */
5406 if (HDR_IN_HASH_TABLE(hdr)) {
5407 arc_buf_hdr_t *found;
5408
5409 ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp));
5410 ASSERT3U(hdr->b_dva.dva_word[0], ==,
5411 BP_IDENTITY(zio->io_bp)->dva_word[0]);
5412 ASSERT3U(hdr->b_dva.dva_word[1], ==,
5413 BP_IDENTITY(zio->io_bp)->dva_word[1]);
5414
5415 found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
5416
5417 ASSERT((found == hdr &&
5418 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5419 (found == hdr && HDR_L2_READING(hdr)));
5420 ASSERT3P(hash_lock, !=, NULL);
5421 }
5422
5423 if (BP_IS_PROTECTED(bp)) {
5424 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
5425 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
5426 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
5427 hdr->b_crypt_hdr.b_iv);
5428
5429 if (zio->io_error == 0) {
5430 if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) {
5431 void *tmpbuf;
5432
5433 tmpbuf = abd_borrow_buf_copy(zio->io_abd,
5434 sizeof (zil_chain_t));
5435 zio_crypt_decode_mac_zil(tmpbuf,
5436 hdr->b_crypt_hdr.b_mac);
5437 abd_return_buf(zio->io_abd, tmpbuf,
5438 sizeof (zil_chain_t));
5439 } else {
5440 zio_crypt_decode_mac_bp(bp,
5441 hdr->b_crypt_hdr.b_mac);
5442 }
5443 }
5444 }
5445
5446 if (zio->io_error == 0) {
5447 /* byteswap if necessary */
5448 if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5449 if (BP_GET_LEVEL(zio->io_bp) > 0) {
5450 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5451 } else {
5452 hdr->b_l1hdr.b_byteswap =
5453 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5454 }
5455 } else {
5456 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5457 }
5458 if (!HDR_L2_READING(hdr)) {
5459 hdr->b_complevel = zio->io_prop.zp_complevel;
5460 }
5461 }
5462
5463 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5464 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5465 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5466
5467 callback_list = hdr->b_l1hdr.b_acb;
5468 ASSERT3P(callback_list, !=, NULL);
5469 hdr->b_l1hdr.b_acb = NULL;
5470
5471 /*
5472 * If a read request has a callback (i.e. acb_done is not NULL), then we
5473 * make a buf containing the data according to the parameters which were
5474 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
5475 * aren't needlessly decompressing the data multiple times.
5476 */
5477 int callback_cnt = 0;
5478 for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5479
5480 /* We need the last one to call below in original order. */
5481 callback_list = acb;
5482
5483 if (!acb->acb_done || acb->acb_nobuf)
5484 continue;
5485
5486 callback_cnt++;
5487
5488 if (zio->io_error != 0)
5489 continue;
5490
5491 int error = arc_buf_alloc_impl(hdr, zio->io_spa,
5492 &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
5493 acb->acb_compressed, acb->acb_noauth, B_TRUE,
5494 &acb->acb_buf);
5495
5496 /*
5497 * Assert non-speculative zios didn't fail because an
5498 * encryption key wasn't loaded
5499 */
5500 ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
5501 error != EACCES);
5502
5503 /*
5504 * If we failed to decrypt, report an error now (as the zio
5505 * layer would have done if it had done the transforms).
5506 */
5507 if (error == ECKSUM) {
5508 ASSERT(BP_IS_PROTECTED(bp));
5509 error = SET_ERROR(EIO);
5510 if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5511 spa_log_error(zio->io_spa, &acb->acb_zb,
5512 BP_GET_LOGICAL_BIRTH(zio->io_bp));
5513 (void) zfs_ereport_post(
5514 FM_EREPORT_ZFS_AUTHENTICATION,
5515 zio->io_spa, NULL, &acb->acb_zb, zio, 0);
5516 }
5517 }
5518
5519 if (error != 0) {
5520 /*
5521 * Decompression or decryption failed. Set
5522 * io_error so that when we call acb_done
5523 * (below), we will indicate that the read
5524 * failed. Note that in the unusual case
5525 * where one callback is compressed and another
5526 * uncompressed, we will mark all of them
5527 * as failed, even though the uncompressed
5528 * one can't actually fail. In this case,
5529 * the hdr will not be anonymous, because
5530 * if there are multiple callbacks, it's
5531 * because multiple threads found the same
5532 * arc buf in the hash table.
5533 */
5534 zio->io_error = error;
5535 }
5536 }
5537
5538 /*
5539 * If there are multiple callbacks, we must have the hash lock,
5540 * because the only way for multiple threads to find this hdr is
5541 * in the hash table. This ensures that if there are multiple
5542 * callbacks, the hdr is not anonymous. If it were anonymous,
5543 * we couldn't use arc_buf_destroy() in the error case below.
5544 */
5545 ASSERT(callback_cnt < 2 || hash_lock != NULL);
5546
5547 if (zio->io_error == 0) {
5548 arc_hdr_verify(hdr, zio->io_bp);
5549 } else {
5550 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5551 if (hdr->b_l1hdr.b_state != arc_anon)
5552 arc_change_state(arc_anon, hdr);
5553 if (HDR_IN_HASH_TABLE(hdr))
5554 buf_hash_remove(hdr);
5555 }
5556
5557 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5558 (void) remove_reference(hdr, hdr);
5559
5560 if (hash_lock != NULL)
5561 mutex_exit(hash_lock);
5562
5563 /* execute each callback and free its structure */
5564 while ((acb = callback_list) != NULL) {
5565 if (acb->acb_done != NULL) {
5566 if (zio->io_error != 0 && acb->acb_buf != NULL) {
5567 /*
5568 * If arc_buf_alloc_impl() fails during
5569 * decompression, the buf will still be
5570 * allocated, and needs to be freed here.
5571 */
5572 arc_buf_destroy(acb->acb_buf,
5573 acb->acb_private);
5574 acb->acb_buf = NULL;
5575 }
5576 acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5577 acb->acb_buf, acb->acb_private);
5578 }
5579
5580 if (acb->acb_zio_dummy != NULL) {
5581 acb->acb_zio_dummy->io_error = zio->io_error;
5582 zio_nowait(acb->acb_zio_dummy);
5583 }
5584
5585 callback_list = acb->acb_prev;
5586 if (acb->acb_wait) {
5587 mutex_enter(&acb->acb_wait_lock);
5588 acb->acb_wait_error = zio->io_error;
5589 acb->acb_wait = B_FALSE;
5590 cv_signal(&acb->acb_wait_cv);
5591 mutex_exit(&acb->acb_wait_lock);
5592 /* acb will be freed by the waiting thread. */
5593 } else {
5594 kmem_free(acb, sizeof (arc_callback_t));
5595 }
5596 }
5597 }
5598
5599 /*
5600 * Lookup the block at the specified DVA (in bp), and return the manner in
5601 * which the block is cached. A zero return indicates not cached.
5602 */
5603 int
arc_cached(spa_t * spa,const blkptr_t * bp)5604 arc_cached(spa_t *spa, const blkptr_t *bp)
5605 {
5606 arc_buf_hdr_t *hdr = NULL;
5607 kmutex_t *hash_lock = NULL;
5608 uint64_t guid = spa_load_guid(spa);
5609 int flags = 0;
5610
5611 if (BP_IS_EMBEDDED(bp))
5612 return (ARC_CACHED_EMBEDDED);
5613
5614 hdr = buf_hash_find(guid, bp, &hash_lock);
5615 if (hdr == NULL)
5616 return (0);
5617
5618 if (HDR_HAS_L1HDR(hdr)) {
5619 arc_state_t *state = hdr->b_l1hdr.b_state;
5620 /*
5621 * We switch to ensure that any future arc_state_type_t
5622 * changes are handled. This is just a shift to promote
5623 * more compile-time checking.
5624 */
5625 switch (state->arcs_state) {
5626 case ARC_STATE_ANON:
5627 break;
5628 case ARC_STATE_MRU:
5629 flags |= ARC_CACHED_IN_MRU | ARC_CACHED_IN_L1;
5630 break;
5631 case ARC_STATE_MFU:
5632 flags |= ARC_CACHED_IN_MFU | ARC_CACHED_IN_L1;
5633 break;
5634 case ARC_STATE_UNCACHED:
5635 /* The header is still in L1, probably not for long */
5636 flags |= ARC_CACHED_IN_L1;
5637 break;
5638 default:
5639 break;
5640 }
5641 }
5642 if (HDR_HAS_L2HDR(hdr))
5643 flags |= ARC_CACHED_IN_L2;
5644
5645 mutex_exit(hash_lock);
5646
5647 return (flags);
5648 }
5649
5650 /*
5651 * "Read" the block at the specified DVA (in bp) via the
5652 * cache. If the block is found in the cache, invoke the provided
5653 * callback immediately and return. Note that the `zio' parameter
5654 * in the callback will be NULL in this case, since no IO was
5655 * required. If the block is not in the cache pass the read request
5656 * on to the spa with a substitute callback function, so that the
5657 * requested block will be added to the cache.
5658 *
5659 * If a read request arrives for a block that has a read in-progress,
5660 * either wait for the in-progress read to complete (and return the
5661 * results); or, if this is a read with a "done" func, add a record
5662 * to the read to invoke the "done" func when the read completes,
5663 * and return; or just return.
5664 *
5665 * arc_read_done() will invoke all the requested "done" functions
5666 * for readers of this block.
5667 */
5668 int
arc_read(zio_t * pio,spa_t * spa,const blkptr_t * bp,arc_read_done_func_t * done,void * private,zio_priority_t priority,int zio_flags,arc_flags_t * arc_flags,const zbookmark_phys_t * zb)5669 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
5670 arc_read_done_func_t *done, void *private, zio_priority_t priority,
5671 int zio_flags, arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
5672 {
5673 arc_buf_hdr_t *hdr = NULL;
5674 kmutex_t *hash_lock = NULL;
5675 zio_t *rzio;
5676 uint64_t guid = spa_load_guid(spa);
5677 boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0;
5678 boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) &&
5679 (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5680 boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) &&
5681 (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
5682 boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
5683 boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
5684 arc_buf_t *buf = NULL;
5685 int rc = 0;
5686
5687 ASSERT(!embedded_bp ||
5688 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
5689 ASSERT(!BP_IS_HOLE(bp));
5690 ASSERT(!BP_IS_REDACTED(bp));
5691
5692 /*
5693 * Normally SPL_FSTRANS will already be set since kernel threads which
5694 * expect to call the DMU interfaces will set it when created. System
5695 * calls are similarly handled by setting/cleaning the bit in the
5696 * registered callback (module/os/.../zfs/zpl_*).
5697 *
5698 * External consumers such as Lustre which call the exported DMU
5699 * interfaces may not have set SPL_FSTRANS. To avoid a deadlock
5700 * on the hash_lock always set and clear the bit.
5701 */
5702 fstrans_cookie_t cookie = spl_fstrans_mark();
5703 top:
5704 if (!embedded_bp) {
5705 /*
5706 * Embedded BP's have no DVA and require no I/O to "read".
5707 * Create an anonymous arc buf to back it.
5708 */
5709 hdr = buf_hash_find(guid, bp, &hash_lock);
5710 }
5711
5712 /*
5713 * Determine if we have an L1 cache hit or a cache miss. For simplicity
5714 * we maintain encrypted data separately from compressed / uncompressed
5715 * data. If the user is requesting raw encrypted data and we don't have
5716 * that in the header we will read from disk to guarantee that we can
5717 * get it even if the encryption keys aren't loaded.
5718 */
5719 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
5720 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
5721 boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);
5722
5723 /*
5724 * Verify the block pointer contents are reasonable. This
5725 * should always be the case since the blkptr is protected by
5726 * a checksum.
5727 */
5728 if (!zfs_blkptr_verify(spa, bp, BLK_CONFIG_SKIP,
5729 BLK_VERIFY_LOG)) {
5730 mutex_exit(hash_lock);
5731 rc = SET_ERROR(ECKSUM);
5732 goto done;
5733 }
5734
5735 if (HDR_IO_IN_PROGRESS(hdr)) {
5736 if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
5737 mutex_exit(hash_lock);
5738 ARCSTAT_BUMP(arcstat_cached_only_in_progress);
5739 rc = SET_ERROR(ENOENT);
5740 goto done;
5741 }
5742
5743 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5744 ASSERT3P(head_zio, !=, NULL);
5745 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5746 priority == ZIO_PRIORITY_SYNC_READ) {
5747 /*
5748 * This is a sync read that needs to wait for
5749 * an in-flight async read. Request that the
5750 * zio have its priority upgraded.
5751 */
5752 zio_change_priority(head_zio, priority);
5753 DTRACE_PROBE1(arc__async__upgrade__sync,
5754 arc_buf_hdr_t *, hdr);
5755 ARCSTAT_BUMP(arcstat_async_upgrade_sync);
5756 }
5757
5758 DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
5759 arc_access(hdr, *arc_flags, B_FALSE);
5760
5761 /*
5762 * If there are multiple threads reading the same block
5763 * and that block is not yet in the ARC, then only one
5764 * thread will do the physical I/O and all other
5765 * threads will wait until that I/O completes.
5766 * Synchronous reads use the acb_wait_cv whereas nowait
5767 * reads register a callback. Both are signalled/called
5768 * in arc_read_done.
5769 *
5770 * Errors of the physical I/O may need to be propagated.
5771 * Synchronous read errors are returned here from
5772 * arc_read_done via acb_wait_error. Nowait reads
5773 * attach the acb_zio_dummy zio to pio and
5774 * arc_read_done propagates the physical I/O's io_error
5775 * to acb_zio_dummy, and thereby to pio.
5776 */
5777 arc_callback_t *acb = NULL;
5778 if (done || pio || *arc_flags & ARC_FLAG_WAIT) {
5779 acb = kmem_zalloc(sizeof (arc_callback_t),
5780 KM_SLEEP);
5781 acb->acb_done = done;
5782 acb->acb_private = private;
5783 acb->acb_compressed = compressed_read;
5784 acb->acb_encrypted = encrypted_read;
5785 acb->acb_noauth = noauth_read;
5786 acb->acb_nobuf = no_buf;
5787 if (*arc_flags & ARC_FLAG_WAIT) {
5788 acb->acb_wait = B_TRUE;
5789 mutex_init(&acb->acb_wait_lock, NULL,
5790 MUTEX_DEFAULT, NULL);
5791 cv_init(&acb->acb_wait_cv, NULL,
5792 CV_DEFAULT, NULL);
5793 }
5794 acb->acb_zb = *zb;
5795 if (pio != NULL) {
5796 acb->acb_zio_dummy = zio_null(pio,
5797 spa, NULL, NULL, NULL, zio_flags);
5798 }
5799 acb->acb_zio_head = head_zio;
5800 acb->acb_next = hdr->b_l1hdr.b_acb;
5801 hdr->b_l1hdr.b_acb->acb_prev = acb;
5802 hdr->b_l1hdr.b_acb = acb;
5803 }
5804 mutex_exit(hash_lock);
5805
5806 ARCSTAT_BUMP(arcstat_iohits);
5807 ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5808 demand, prefetch, is_data, data, metadata, iohits);
5809
5810 if (*arc_flags & ARC_FLAG_WAIT) {
5811 mutex_enter(&acb->acb_wait_lock);
5812 while (acb->acb_wait) {
5813 cv_wait(&acb->acb_wait_cv,
5814 &acb->acb_wait_lock);
5815 }
5816 rc = acb->acb_wait_error;
5817 mutex_exit(&acb->acb_wait_lock);
5818 mutex_destroy(&acb->acb_wait_lock);
5819 cv_destroy(&acb->acb_wait_cv);
5820 kmem_free(acb, sizeof (arc_callback_t));
5821 }
5822 goto out;
5823 }
5824
5825 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5826 hdr->b_l1hdr.b_state == arc_mfu ||
5827 hdr->b_l1hdr.b_state == arc_uncached);
5828
5829 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5830 arc_access(hdr, *arc_flags, B_TRUE);
5831
5832 if (done && !no_buf) {
5833 ASSERT(!embedded_bp || !BP_IS_HOLE(bp));
5834
5835 /* Get a buf with the desired data in it. */
5836 rc = arc_buf_alloc_impl(hdr, spa, zb, private,
5837 encrypted_read, compressed_read, noauth_read,
5838 B_TRUE, &buf);
5839 if (rc == ECKSUM) {
5840 /*
5841 * Convert authentication and decryption errors
5842 * to EIO (and generate an ereport if needed)
5843 * before leaving the ARC.
5844 */
5845 rc = SET_ERROR(EIO);
5846 if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5847 spa_log_error(spa, zb, hdr->b_birth);
5848 (void) zfs_ereport_post(
5849 FM_EREPORT_ZFS_AUTHENTICATION,
5850 spa, NULL, zb, NULL, 0);
5851 }
5852 }
5853 if (rc != 0) {
5854 arc_buf_destroy_impl(buf);
5855 buf = NULL;
5856 (void) remove_reference(hdr, private);
5857 }
5858
5859 /* assert any errors weren't due to unloaded keys */
5860 ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
5861 rc != EACCES);
5862 }
5863 mutex_exit(hash_lock);
5864 ARCSTAT_BUMP(arcstat_hits);
5865 ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
5866 demand, prefetch, is_data, data, metadata, hits);
5867 *arc_flags |= ARC_FLAG_CACHED;
5868 goto done;
5869 } else {
5870 uint64_t lsize = BP_GET_LSIZE(bp);
5871 uint64_t psize = BP_GET_PSIZE(bp);
5872 arc_callback_t *acb;
5873 vdev_t *vd = NULL;
5874 uint64_t addr = 0;
5875 boolean_t devw = B_FALSE;
5876 uint64_t size;
5877 abd_t *hdr_abd;
5878 int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;
5879 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
5880
5881 if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
5882 if (hash_lock != NULL)
5883 mutex_exit(hash_lock);
5884 rc = SET_ERROR(ENOENT);
5885 goto done;
5886 }
5887
5888 /*
5889 * Verify the block pointer contents are reasonable. This
5890 * should always be the case since the blkptr is protected by
5891 * a checksum.
5892 */
5893 if (!zfs_blkptr_verify(spa, bp,
5894 (zio_flags & ZIO_FLAG_CONFIG_WRITER) ?
5895 BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
5896 if (hash_lock != NULL)
5897 mutex_exit(hash_lock);
5898 rc = SET_ERROR(ECKSUM);
5899 goto done;
5900 }
5901
5902 if (hdr == NULL) {
5903 /*
5904 * This block is not in the cache or it has
5905 * embedded data.
5906 */
5907 arc_buf_hdr_t *exists = NULL;
5908 hdr = arc_hdr_alloc(guid, psize, lsize,
5909 BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type);
5910
5911 if (!embedded_bp) {
5912 hdr->b_dva = *BP_IDENTITY(bp);
5913 hdr->b_birth = BP_GET_BIRTH(bp);
5914 exists = buf_hash_insert(hdr, &hash_lock);
5915 }
5916 if (exists != NULL) {
5917 /* somebody beat us to the hash insert */
5918 mutex_exit(hash_lock);
5919 buf_discard_identity(hdr);
5920 arc_hdr_destroy(hdr);
5921 goto top; /* restart the IO request */
5922 }
5923 } else {
5924 /*
5925 * This block is in the ghost cache or encrypted data
5926 * was requested and we didn't have it. If it was
5927 * L2-only (and thus didn't have an L1 hdr),
5928 * we realloc the header to add an L1 hdr.
5929 */
5930 if (!HDR_HAS_L1HDR(hdr)) {
5931 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5932 hdr_full_cache);
5933 }
5934
5935 if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
5936 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5937 ASSERT(!HDR_HAS_RABD(hdr));
5938 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5939 ASSERT0(zfs_refcount_count(
5940 &hdr->b_l1hdr.b_refcnt));
5941 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5942 #ifdef ZFS_DEBUG
5943 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5944 #endif
5945 } else if (HDR_IO_IN_PROGRESS(hdr)) {
5946 /*
5947 * If this header already had an IO in progress
5948 * and we are performing another IO to fetch
5949 * encrypted data we must wait until the first
5950 * IO completes so as not to confuse
5951 * arc_read_done(). This should be very rare
5952 * and so the performance impact shouldn't
5953 * matter.
5954 */
5955 arc_callback_t *acb = kmem_zalloc(
5956 sizeof (arc_callback_t), KM_SLEEP);
5957 acb->acb_wait = B_TRUE;
5958 mutex_init(&acb->acb_wait_lock, NULL,
5959 MUTEX_DEFAULT, NULL);
5960 cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
5961 NULL);
5962 acb->acb_zio_head =
5963 hdr->b_l1hdr.b_acb->acb_zio_head;
5964 acb->acb_next = hdr->b_l1hdr.b_acb;
5965 hdr->b_l1hdr.b_acb->acb_prev = acb;
5966 hdr->b_l1hdr.b_acb = acb;
5967 mutex_exit(hash_lock);
5968 mutex_enter(&acb->acb_wait_lock);
5969 while (acb->acb_wait) {
5970 cv_wait(&acb->acb_wait_cv,
5971 &acb->acb_wait_lock);
5972 }
5973 mutex_exit(&acb->acb_wait_lock);
5974 mutex_destroy(&acb->acb_wait_lock);
5975 cv_destroy(&acb->acb_wait_cv);
5976 kmem_free(acb, sizeof (arc_callback_t));
5977 goto top;
5978 }
5979 }
5980 if (*arc_flags & ARC_FLAG_UNCACHED) {
5981 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
5982 if (!encrypted_read)
5983 alloc_flags |= ARC_HDR_ALLOC_LINEAR;
5984 }
5985
5986 /*
5987 * Take additional reference for IO_IN_PROGRESS. It stops
5988 * arc_access() from putting this header without any buffers
5989 * and so other references but obviously nonevictable onto
5990 * the evictable list of MRU or MFU state.
5991 */
5992 add_reference(hdr, hdr);
5993 if (!embedded_bp)
5994 arc_access(hdr, *arc_flags, B_FALSE);
5995 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5996 arc_hdr_alloc_abd(hdr, alloc_flags);
5997 if (encrypted_read) {
5998 ASSERT(HDR_HAS_RABD(hdr));
5999 size = HDR_GET_PSIZE(hdr);
6000 hdr_abd = hdr->b_crypt_hdr.b_rabd;
6001 zio_flags |= ZIO_FLAG_RAW;
6002 } else {
6003 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6004 size = arc_hdr_size(hdr);
6005 hdr_abd = hdr->b_l1hdr.b_pabd;
6006
6007 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
6008 zio_flags |= ZIO_FLAG_RAW_COMPRESS;
6009 }
6010
6011 /*
6012 * For authenticated bp's, we do not ask the ZIO layer
6013 * to authenticate them since this will cause the entire
6014 * IO to fail if the key isn't loaded. Instead, we
6015 * defer authentication until arc_buf_fill(), which will
6016 * verify the data when the key is available.
6017 */
6018 if (BP_IS_AUTHENTICATED(bp))
6019 zio_flags |= ZIO_FLAG_RAW_ENCRYPT;
6020 }
6021
6022 if (BP_IS_AUTHENTICATED(bp))
6023 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6024 if (BP_GET_LEVEL(bp) > 0)
6025 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
6026 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
6027
6028 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
6029 acb->acb_done = done;
6030 acb->acb_private = private;
6031 acb->acb_compressed = compressed_read;
6032 acb->acb_encrypted = encrypted_read;
6033 acb->acb_noauth = noauth_read;
6034 acb->acb_nobuf = no_buf;
6035 acb->acb_zb = *zb;
6036
6037 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6038 hdr->b_l1hdr.b_acb = acb;
6039
6040 if (HDR_HAS_L2HDR(hdr) &&
6041 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
6042 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
6043 addr = hdr->b_l2hdr.b_daddr;
6044 /*
6045 * Lock out L2ARC device removal.
6046 */
6047 if (vdev_is_dead(vd) ||
6048 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
6049 vd = NULL;
6050 }
6051
6052 /*
6053 * We count both async reads and scrub IOs as asynchronous so
6054 * that both can be upgraded in the event of a cache hit while
6055 * the read IO is still in-flight.
6056 */
6057 if (priority == ZIO_PRIORITY_ASYNC_READ ||
6058 priority == ZIO_PRIORITY_SCRUB)
6059 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6060 else
6061 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6062
6063 /*
6064 * At this point, we have a level 1 cache miss or a blkptr
6065 * with embedded data. Try again in L2ARC if possible.
6066 */
6067 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
6068
6069 /*
6070 * Skip ARC stat bump for block pointers with embedded
6071 * data. The data are read from the blkptr itself via
6072 * decode_embedded_bp_compressed().
6073 */
6074 if (!embedded_bp) {
6075 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
6076 blkptr_t *, bp, uint64_t, lsize,
6077 zbookmark_phys_t *, zb);
6078 ARCSTAT_BUMP(arcstat_misses);
6079 ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
6080 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
6081 metadata, misses);
6082 zfs_racct_read(spa, size, 1, 0);
6083 }
6084
6085 /* Check if the spa even has l2 configured */
6086 const boolean_t spa_has_l2 = l2arc_ndev != 0 &&
6087 spa->spa_l2cache.sav_count > 0;
6088
6089 if (vd != NULL && spa_has_l2 && !(l2arc_norw && devw)) {
6090 /*
6091 * Read from the L2ARC if the following are true:
6092 * 1. The L2ARC vdev was previously cached.
6093 * 2. This buffer still has L2ARC metadata.
6094 * 3. This buffer isn't currently writing to the L2ARC.
6095 * 4. The L2ARC entry wasn't evicted, which may
6096 * also have invalidated the vdev.
6097 */
6098 if (HDR_HAS_L2HDR(hdr) &&
6099 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
6100 l2arc_read_callback_t *cb;
6101 abd_t *abd;
6102 uint64_t asize;
6103
6104 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
6105 ARCSTAT_BUMP(arcstat_l2_hits);
6106 hdr->b_l2hdr.b_hits++;
6107
6108 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
6109 KM_SLEEP);
6110 cb->l2rcb_hdr = hdr;
6111 cb->l2rcb_bp = *bp;
6112 cb->l2rcb_zb = *zb;
6113 cb->l2rcb_flags = zio_flags;
6114
6115 /*
6116 * When Compressed ARC is disabled, but the
6117 * L2ARC block is compressed, arc_hdr_size()
6118 * will have returned LSIZE rather than PSIZE.
6119 */
6120 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
6121 !HDR_COMPRESSION_ENABLED(hdr) &&
6122 HDR_GET_PSIZE(hdr) != 0) {
6123 size = HDR_GET_PSIZE(hdr);
6124 }
6125
6126 asize = vdev_psize_to_asize(vd, size);
6127 if (asize != size) {
6128 abd = abd_alloc_for_io(asize,
6129 HDR_ISTYPE_METADATA(hdr));
6130 cb->l2rcb_abd = abd;
6131 } else {
6132 abd = hdr_abd;
6133 }
6134
6135 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
6136 addr + asize <= vd->vdev_psize -
6137 VDEV_LABEL_END_SIZE);
6138
6139 /*
6140 * l2arc read. The SCL_L2ARC lock will be
6141 * released by l2arc_read_done().
6142 * Issue a null zio if the underlying buffer
6143 * was squashed to zero size by compression.
6144 */
6145 ASSERT3U(arc_hdr_get_compress(hdr), !=,
6146 ZIO_COMPRESS_EMPTY);
6147 rzio = zio_read_phys(pio, vd, addr,
6148 asize, abd,
6149 ZIO_CHECKSUM_OFF,
6150 l2arc_read_done, cb, priority,
6151 zio_flags | ZIO_FLAG_CANFAIL |
6152 ZIO_FLAG_DONT_PROPAGATE |
6153 ZIO_FLAG_DONT_RETRY, B_FALSE);
6154 acb->acb_zio_head = rzio;
6155
6156 if (hash_lock != NULL)
6157 mutex_exit(hash_lock);
6158
6159 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
6160 zio_t *, rzio);
6161 ARCSTAT_INCR(arcstat_l2_read_bytes,
6162 HDR_GET_PSIZE(hdr));
6163
6164 if (*arc_flags & ARC_FLAG_NOWAIT) {
6165 zio_nowait(rzio);
6166 goto out;
6167 }
6168
6169 ASSERT(*arc_flags & ARC_FLAG_WAIT);
6170 if (zio_wait(rzio) == 0)
6171 goto out;
6172
6173 /* l2arc read error; goto zio_read() */
6174 if (hash_lock != NULL)
6175 mutex_enter(hash_lock);
6176 } else {
6177 DTRACE_PROBE1(l2arc__miss,
6178 arc_buf_hdr_t *, hdr);
6179 ARCSTAT_BUMP(arcstat_l2_misses);
6180 if (HDR_L2_WRITING(hdr))
6181 ARCSTAT_BUMP(arcstat_l2_rw_clash);
6182 spa_config_exit(spa, SCL_L2ARC, vd);
6183 }
6184 } else {
6185 if (vd != NULL)
6186 spa_config_exit(spa, SCL_L2ARC, vd);
6187
6188 /*
6189 * Only a spa with l2 should contribute to l2
6190 * miss stats. (Including the case of having a
6191 * faulted cache device - that's also a miss.)
6192 */
6193 if (spa_has_l2) {
6194 /*
6195 * Skip ARC stat bump for block pointers with
6196 * embedded data. The data are read from the
6197 * blkptr itself via
6198 * decode_embedded_bp_compressed().
6199 */
6200 if (!embedded_bp) {
6201 DTRACE_PROBE1(l2arc__miss,
6202 arc_buf_hdr_t *, hdr);
6203 ARCSTAT_BUMP(arcstat_l2_misses);
6204 }
6205 }
6206 }
6207
6208 rzio = zio_read(pio, spa, bp, hdr_abd, size,
6209 arc_read_done, hdr, priority, zio_flags, zb);
6210 acb->acb_zio_head = rzio;
6211
6212 if (hash_lock != NULL)
6213 mutex_exit(hash_lock);
6214
6215 if (*arc_flags & ARC_FLAG_WAIT) {
6216 rc = zio_wait(rzio);
6217 goto out;
6218 }
6219
6220 ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
6221 zio_nowait(rzio);
6222 }
6223
6224 out:
6225 /* embedded bps don't actually go to disk */
6226 if (!embedded_bp)
6227 spa_read_history_add(spa, zb, *arc_flags);
6228 spl_fstrans_unmark(cookie);
6229 return (rc);
6230
6231 done:
6232 if (done)
6233 done(NULL, zb, bp, buf, private);
6234 if (pio && rc != 0) {
6235 zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
6236 zio->io_error = rc;
6237 zio_nowait(zio);
6238 }
6239 goto out;
6240 }
6241
6242 arc_prune_t *
arc_add_prune_callback(arc_prune_func_t * func,void * private)6243 arc_add_prune_callback(arc_prune_func_t *func, void *private)
6244 {
6245 arc_prune_t *p;
6246
6247 p = kmem_alloc(sizeof (*p), KM_SLEEP);
6248 p->p_pfunc = func;
6249 p->p_private = private;
6250 list_link_init(&p->p_node);
6251 zfs_refcount_create(&p->p_refcnt);
6252
6253 mutex_enter(&arc_prune_mtx);
6254 zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
6255 list_insert_head(&arc_prune_list, p);
6256 mutex_exit(&arc_prune_mtx);
6257
6258 return (p);
6259 }
6260
6261 void
arc_remove_prune_callback(arc_prune_t * p)6262 arc_remove_prune_callback(arc_prune_t *p)
6263 {
6264 boolean_t wait = B_FALSE;
6265 mutex_enter(&arc_prune_mtx);
6266 list_remove(&arc_prune_list, p);
6267 if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6268 wait = B_TRUE;
6269 mutex_exit(&arc_prune_mtx);
6270
6271 /* wait for arc_prune_task to finish */
6272 if (wait)
6273 taskq_wait_outstanding(arc_prune_taskq, 0);
6274 ASSERT0(zfs_refcount_count(&p->p_refcnt));
6275 zfs_refcount_destroy(&p->p_refcnt);
6276 kmem_free(p, sizeof (*p));
6277 }
6278
6279 /*
6280 * Helper function for arc_prune_async() it is responsible for safely
6281 * handling the execution of a registered arc_prune_func_t.
6282 */
6283 static void
arc_prune_task(void * ptr)6284 arc_prune_task(void *ptr)
6285 {
6286 arc_prune_t *ap = (arc_prune_t *)ptr;
6287 arc_prune_func_t *func = ap->p_pfunc;
6288
6289 if (func != NULL)
6290 func(ap->p_adjust, ap->p_private);
6291
6292 (void) zfs_refcount_remove(&ap->p_refcnt, func);
6293 }
6294
6295 /*
6296 * Notify registered consumers they must drop holds on a portion of the ARC
6297 * buffers they reference. This provides a mechanism to ensure the ARC can
6298 * honor the metadata limit and reclaim otherwise pinned ARC buffers.
6299 *
6300 * This operation is performed asynchronously so it may be safely called
6301 * in the context of the arc_reclaim_thread(). A reference is taken here
6302 * for each registered arc_prune_t and the arc_prune_task() is responsible
6303 * for releasing it once the registered arc_prune_func_t has completed.
6304 */
6305 static void
arc_prune_async(uint64_t adjust)6306 arc_prune_async(uint64_t adjust)
6307 {
6308 arc_prune_t *ap;
6309
6310 mutex_enter(&arc_prune_mtx);
6311 for (ap = list_head(&arc_prune_list); ap != NULL;
6312 ap = list_next(&arc_prune_list, ap)) {
6313
6314 if (zfs_refcount_count(&ap->p_refcnt) >= 2)
6315 continue;
6316
6317 zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
6318 ap->p_adjust = adjust;
6319 if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
6320 ap, TQ_SLEEP) == TASKQID_INVALID) {
6321 (void) zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
6322 continue;
6323 }
6324 ARCSTAT_BUMP(arcstat_prune);
6325 }
6326 mutex_exit(&arc_prune_mtx);
6327 }
6328
6329 /*
6330 * Notify the arc that a block was freed, and thus will never be used again.
6331 */
6332 void
arc_freed(spa_t * spa,const blkptr_t * bp)6333 arc_freed(spa_t *spa, const blkptr_t *bp)
6334 {
6335 arc_buf_hdr_t *hdr;
6336 kmutex_t *hash_lock;
6337 uint64_t guid = spa_load_guid(spa);
6338
6339 ASSERT(!BP_IS_EMBEDDED(bp));
6340
6341 hdr = buf_hash_find(guid, bp, &hash_lock);
6342 if (hdr == NULL)
6343 return;
6344
6345 /*
6346 * We might be trying to free a block that is still doing I/O
6347 * (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
6348 * dmu_sync-ed block). A block may also have a reference if it is
6349 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6350 * have written the new block to its final resting place on disk but
6351 * without the dedup flag set. This would have left the hdr in the MRU
6352 * state and discoverable. When the txg finally syncs it detects that
6353 * the block was overridden in open context and issues an override I/O.
6354 * Since this is a dedup block, the override I/O will determine if the
6355 * block is already in the DDT. If so, then it will replace the io_bp
6356 * with the bp from the DDT and allow the I/O to finish. When the I/O
6357 * reaches the done callback, dbuf_write_override_done, it will
6358 * check to see if the io_bp and io_bp_override are identical.
6359 * If they are not, then it indicates that the bp was replaced with
6360 * the bp in the DDT and the override bp is freed. This allows
6361 * us to arrive here with a reference on a block that is being
6362 * freed. So if we have an I/O in progress, or a reference to
6363 * this hdr, then we don't destroy the hdr.
6364 */
6365 if (!HDR_HAS_L1HDR(hdr) ||
6366 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6367 arc_change_state(arc_anon, hdr);
6368 arc_hdr_destroy(hdr);
6369 mutex_exit(hash_lock);
6370 } else {
6371 mutex_exit(hash_lock);
6372 }
6373
6374 }
6375
6376 /*
6377 * Release this buffer from the cache, making it an anonymous buffer. This
6378 * must be done after a read and prior to modifying the buffer contents.
6379 * If the buffer has more than one reference, we must make
6380 * a new hdr for the buffer.
6381 */
6382 void
arc_release(arc_buf_t * buf,const void * tag)6383 arc_release(arc_buf_t *buf, const void *tag)
6384 {
6385 arc_buf_hdr_t *hdr = buf->b_hdr;
6386
6387 /*
6388 * It would be nice to assert that if its DMU metadata (level >
6389 * 0 || it's the dnode file), then it must be syncing context.
6390 * But we don't know that information at this level.
6391 */
6392
6393 ASSERT(HDR_HAS_L1HDR(hdr));
6394
6395 /*
6396 * We don't grab the hash lock prior to this check, because if
6397 * the buffer's header is in the arc_anon state, it won't be
6398 * linked into the hash table.
6399 */
6400 if (hdr->b_l1hdr.b_state == arc_anon) {
6401 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6402 ASSERT(!HDR_IN_HASH_TABLE(hdr));
6403 ASSERT(!HDR_HAS_L2HDR(hdr));
6404
6405 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6406 ASSERT(ARC_BUF_LAST(buf));
6407 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6408 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6409
6410 hdr->b_l1hdr.b_arc_access = 0;
6411
6412 /*
6413 * If the buf is being overridden then it may already
6414 * have a hdr that is not empty.
6415 */
6416 buf_discard_identity(hdr);
6417 arc_buf_thaw(buf);
6418
6419 return;
6420 }
6421
6422 kmutex_t *hash_lock = HDR_LOCK(hdr);
6423 mutex_enter(hash_lock);
6424
6425 /*
6426 * This assignment is only valid as long as the hash_lock is
6427 * held, we must be careful not to reference state or the
6428 * b_state field after dropping the lock.
6429 */
6430 arc_state_t *state = hdr->b_l1hdr.b_state;
6431 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6432 ASSERT3P(state, !=, arc_anon);
6433
6434 /* this buffer is not on any list */
6435 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6436
6437 if (HDR_HAS_L2HDR(hdr)) {
6438 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6439
6440 /*
6441 * We have to recheck this conditional again now that
6442 * we're holding the l2ad_mtx to prevent a race with
6443 * another thread which might be concurrently calling
6444 * l2arc_evict(). In that case, l2arc_evict() might have
6445 * destroyed the header's L2 portion as we were waiting
6446 * to acquire the l2ad_mtx.
6447 */
6448 if (HDR_HAS_L2HDR(hdr))
6449 arc_hdr_l2hdr_destroy(hdr);
6450
6451 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6452 }
6453
6454 /*
6455 * Do we have more than one buf?
6456 */
6457 if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
6458 arc_buf_hdr_t *nhdr;
6459 uint64_t spa = hdr->b_spa;
6460 uint64_t psize = HDR_GET_PSIZE(hdr);
6461 uint64_t lsize = HDR_GET_LSIZE(hdr);
6462 boolean_t protected = HDR_PROTECTED(hdr);
6463 enum zio_compress compress = arc_hdr_get_compress(hdr);
6464 arc_buf_contents_t type = arc_buf_type(hdr);
6465 VERIFY3U(hdr->b_type, ==, type);
6466
6467 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6468 VERIFY3S(remove_reference(hdr, tag), >, 0);
6469
6470 if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
6471 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6472 ASSERT(ARC_BUF_LAST(buf));
6473 }
6474
6475 /*
6476 * Pull the data off of this hdr and attach it to
6477 * a new anonymous hdr. Also find the last buffer
6478 * in the hdr's buffer list.
6479 */
6480 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6481 ASSERT3P(lastbuf, !=, NULL);
6482
6483 /*
6484 * If the current arc_buf_t and the hdr are sharing their data
6485 * buffer, then we must stop sharing that block.
6486 */
6487 if (ARC_BUF_SHARED(buf)) {
6488 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6489 ASSERT(!arc_buf_is_shared(lastbuf));
6490
6491 /*
6492 * First, sever the block sharing relationship between
6493 * buf and the arc_buf_hdr_t.
6494 */
6495 arc_unshare_buf(hdr, buf);
6496
6497 /*
6498 * Now we need to recreate the hdr's b_pabd. Since we
6499 * have lastbuf handy, we try to share with it, but if
6500 * we can't then we allocate a new b_pabd and copy the
6501 * data from buf into it.
6502 */
6503 if (arc_can_share(hdr, lastbuf)) {
6504 arc_share_buf(hdr, lastbuf);
6505 } else {
6506 arc_hdr_alloc_abd(hdr, 0);
6507 abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6508 buf->b_data, psize);
6509 }
6510 VERIFY3P(lastbuf->b_data, !=, NULL);
6511 } else if (HDR_SHARED_DATA(hdr)) {
6512 /*
6513 * Uncompressed shared buffers are always at the end
6514 * of the list. Compressed buffers don't have the
6515 * same requirements. This makes it hard to
6516 * simply assert that the lastbuf is shared so
6517 * we rely on the hdr's compression flags to determine
6518 * if we have a compressed, shared buffer.
6519 */
6520 ASSERT(arc_buf_is_shared(lastbuf) ||
6521 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
6522 ASSERT(!arc_buf_is_shared(buf));
6523 }
6524
6525 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
6526 ASSERT3P(state, !=, arc_l2c_only);
6527
6528 (void) zfs_refcount_remove_many(&state->arcs_size[type],
6529 arc_buf_size(buf), buf);
6530
6531 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6532 ASSERT3P(state, !=, arc_l2c_only);
6533 (void) zfs_refcount_remove_many(
6534 &state->arcs_esize[type],
6535 arc_buf_size(buf), buf);
6536 }
6537
6538 arc_cksum_verify(buf);
6539 arc_buf_unwatch(buf);
6540
6541 /* if this is the last uncompressed buf free the checksum */
6542 if (!arc_hdr_has_uncompressed_buf(hdr))
6543 arc_cksum_free(hdr);
6544
6545 mutex_exit(hash_lock);
6546
6547 nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
6548 compress, hdr->b_complevel, type);
6549 ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6550 ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
6551 VERIFY3U(nhdr->b_type, ==, type);
6552 ASSERT(!HDR_SHARED_DATA(nhdr));
6553
6554 nhdr->b_l1hdr.b_buf = buf;
6555 (void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6556 buf->b_hdr = nhdr;
6557
6558 (void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
6559 arc_buf_size(buf), buf);
6560 } else {
6561 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6562 /* protected by hash lock, or hdr is on arc_anon */
6563 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6564 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6565 hdr->b_l1hdr.b_mru_hits = 0;
6566 hdr->b_l1hdr.b_mru_ghost_hits = 0;
6567 hdr->b_l1hdr.b_mfu_hits = 0;
6568 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
6569 arc_change_state(arc_anon, hdr);
6570 hdr->b_l1hdr.b_arc_access = 0;
6571
6572 mutex_exit(hash_lock);
6573 buf_discard_identity(hdr);
6574 arc_buf_thaw(buf);
6575 }
6576 }
6577
6578 int
arc_released(arc_buf_t * buf)6579 arc_released(arc_buf_t *buf)
6580 {
6581 return (buf->b_data != NULL &&
6582 buf->b_hdr->b_l1hdr.b_state == arc_anon);
6583 }
6584
6585 #ifdef ZFS_DEBUG
6586 int
arc_referenced(arc_buf_t * buf)6587 arc_referenced(arc_buf_t *buf)
6588 {
6589 return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6590 }
6591 #endif
6592
6593 static void
arc_write_ready(zio_t * zio)6594 arc_write_ready(zio_t *zio)
6595 {
6596 arc_write_callback_t *callback = zio->io_private;
6597 arc_buf_t *buf = callback->awcb_buf;
6598 arc_buf_hdr_t *hdr = buf->b_hdr;
6599 blkptr_t *bp = zio->io_bp;
6600 uint64_t psize = BP_IS_HOLE(bp) ? 0 : BP_GET_PSIZE(bp);
6601 fstrans_cookie_t cookie = spl_fstrans_mark();
6602
6603 ASSERT(HDR_HAS_L1HDR(hdr));
6604 ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6605 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6606
6607 /*
6608 * If we're reexecuting this zio because the pool suspended, then
6609 * cleanup any state that was previously set the first time the
6610 * callback was invoked.
6611 */
6612 if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6613 arc_cksum_free(hdr);
6614 arc_buf_unwatch(buf);
6615 if (hdr->b_l1hdr.b_pabd != NULL) {
6616 if (ARC_BUF_SHARED(buf)) {
6617 arc_unshare_buf(hdr, buf);
6618 } else {
6619 ASSERT(!arc_buf_is_shared(buf));
6620 arc_hdr_free_abd(hdr, B_FALSE);
6621 }
6622 }
6623
6624 if (HDR_HAS_RABD(hdr))
6625 arc_hdr_free_abd(hdr, B_TRUE);
6626 }
6627 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6628 ASSERT(!HDR_HAS_RABD(hdr));
6629 ASSERT(!HDR_SHARED_DATA(hdr));
6630 ASSERT(!arc_buf_is_shared(buf));
6631
6632 callback->awcb_ready(zio, buf, callback->awcb_private);
6633
6634 if (HDR_IO_IN_PROGRESS(hdr)) {
6635 ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6636 } else {
6637 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6638 add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
6639 }
6640
6641 if (BP_IS_PROTECTED(bp)) {
6642 /* ZIL blocks are written through zio_rewrite */
6643 ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
6644
6645 if (BP_SHOULD_BYTESWAP(bp)) {
6646 if (BP_GET_LEVEL(bp) > 0) {
6647 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
6648 } else {
6649 hdr->b_l1hdr.b_byteswap =
6650 DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
6651 }
6652 } else {
6653 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
6654 }
6655
6656 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
6657 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
6658 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
6659 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
6660 hdr->b_crypt_hdr.b_iv);
6661 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
6662 } else {
6663 arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
6664 }
6665
6666 /*
6667 * If this block was written for raw encryption but the zio layer
6668 * ended up only authenticating it, adjust the buffer flags now.
6669 */
6670 if (BP_IS_AUTHENTICATED(bp) && ARC_BUF_ENCRYPTED(buf)) {
6671 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6672 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6673 if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF)
6674 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6675 } else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) {
6676 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6677 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6678 }
6679
6680 /* this must be done after the buffer flags are adjusted */
6681 arc_cksum_compute(buf);
6682
6683 enum zio_compress compress;
6684 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
6685 compress = ZIO_COMPRESS_OFF;
6686 } else {
6687 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
6688 compress = BP_GET_COMPRESS(bp);
6689 }
6690 HDR_SET_PSIZE(hdr, psize);
6691 arc_hdr_set_compress(hdr, compress);
6692 hdr->b_complevel = zio->io_prop.zp_complevel;
6693
6694 if (zio->io_error != 0 || psize == 0)
6695 goto out;
6696
6697 /*
6698 * Fill the hdr with data. If the buffer is encrypted we have no choice
6699 * but to copy the data into b_radb. If the hdr is compressed, the data
6700 * we want is available from the zio, otherwise we can take it from
6701 * the buf.
6702 *
6703 * We might be able to share the buf's data with the hdr here. However,
6704 * doing so would cause the ARC to be full of linear ABDs if we write a
6705 * lot of shareable data. As a compromise, we check whether scattered
6706 * ABDs are allowed, and assume that if they are then the user wants
6707 * the ARC to be primarily filled with them regardless of the data being
6708 * written. Therefore, if they're allowed then we allocate one and copy
6709 * the data into it; otherwise, we share the data directly if we can.
6710 */
6711 if (ARC_BUF_ENCRYPTED(buf)) {
6712 ASSERT3U(psize, >, 0);
6713 ASSERT(ARC_BUF_COMPRESSED(buf));
6714 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6715 ARC_HDR_USE_RESERVE);
6716 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6717 } else if (!(HDR_UNCACHED(hdr) ||
6718 abd_size_alloc_linear(arc_buf_size(buf))) ||
6719 !arc_can_share(hdr, buf)) {
6720 /*
6721 * Ideally, we would always copy the io_abd into b_pabd, but the
6722 * user may have disabled compressed ARC, thus we must check the
6723 * hdr's compression setting rather than the io_bp's.
6724 */
6725 if (BP_IS_ENCRYPTED(bp)) {
6726 ASSERT3U(psize, >, 0);
6727 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6728 ARC_HDR_USE_RESERVE);
6729 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6730 } else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
6731 !ARC_BUF_COMPRESSED(buf)) {
6732 ASSERT3U(psize, >, 0);
6733 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6734 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6735 } else {
6736 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6737 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6738 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6739 arc_buf_size(buf));
6740 }
6741 } else {
6742 ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
6743 ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
6744 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6745 ASSERT(ARC_BUF_LAST(buf));
6746
6747 arc_share_buf(hdr, buf);
6748 }
6749
6750 out:
6751 arc_hdr_verify(hdr, bp);
6752 spl_fstrans_unmark(cookie);
6753 }
6754
6755 static void
arc_write_children_ready(zio_t * zio)6756 arc_write_children_ready(zio_t *zio)
6757 {
6758 arc_write_callback_t *callback = zio->io_private;
6759 arc_buf_t *buf = callback->awcb_buf;
6760
6761 callback->awcb_children_ready(zio, buf, callback->awcb_private);
6762 }
6763
6764 static void
arc_write_done(zio_t * zio)6765 arc_write_done(zio_t *zio)
6766 {
6767 arc_write_callback_t *callback = zio->io_private;
6768 arc_buf_t *buf = callback->awcb_buf;
6769 arc_buf_hdr_t *hdr = buf->b_hdr;
6770
6771 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6772
6773 if (zio->io_error == 0) {
6774 arc_hdr_verify(hdr, zio->io_bp);
6775
6776 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6777 buf_discard_identity(hdr);
6778 } else {
6779 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
6780 hdr->b_birth = BP_GET_BIRTH(zio->io_bp);
6781 }
6782 } else {
6783 ASSERT(HDR_EMPTY(hdr));
6784 }
6785
6786 /*
6787 * If the block to be written was all-zero or compressed enough to be
6788 * embedded in the BP, no write was performed so there will be no
6789 * dva/birth/checksum. The buffer must therefore remain anonymous
6790 * (and uncached).
6791 */
6792 if (!HDR_EMPTY(hdr)) {
6793 arc_buf_hdr_t *exists;
6794 kmutex_t *hash_lock;
6795
6796 ASSERT3U(zio->io_error, ==, 0);
6797
6798 arc_cksum_verify(buf);
6799
6800 exists = buf_hash_insert(hdr, &hash_lock);
6801 if (exists != NULL) {
6802 /*
6803 * This can only happen if we overwrite for
6804 * sync-to-convergence, because we remove
6805 * buffers from the hash table when we arc_free().
6806 */
6807 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
6808 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6809 panic("bad overwrite, hdr=%p exists=%p",
6810 (void *)hdr, (void *)exists);
6811 ASSERT(zfs_refcount_is_zero(
6812 &exists->b_l1hdr.b_refcnt));
6813 arc_change_state(arc_anon, exists);
6814 arc_hdr_destroy(exists);
6815 mutex_exit(hash_lock);
6816 exists = buf_hash_insert(hdr, &hash_lock);
6817 ASSERT3P(exists, ==, NULL);
6818 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
6819 /* nopwrite */
6820 ASSERT(zio->io_prop.zp_nopwrite);
6821 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6822 panic("bad nopwrite, hdr=%p exists=%p",
6823 (void *)hdr, (void *)exists);
6824 } else {
6825 /* Dedup */
6826 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6827 ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
6828 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6829 ASSERT(BP_GET_DEDUP(zio->io_bp));
6830 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
6831 }
6832 }
6833 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6834 VERIFY3S(remove_reference(hdr, hdr), >, 0);
6835 /* if it's not anon, we are doing a scrub */
6836 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6837 arc_access(hdr, 0, B_FALSE);
6838 mutex_exit(hash_lock);
6839 } else {
6840 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6841 VERIFY3S(remove_reference(hdr, hdr), >, 0);
6842 }
6843
6844 callback->awcb_done(zio, buf, callback->awcb_private);
6845
6846 abd_free(zio->io_abd);
6847 kmem_free(callback, sizeof (arc_write_callback_t));
6848 }
6849
6850 zio_t *
arc_write(zio_t * pio,spa_t * spa,uint64_t txg,blkptr_t * bp,arc_buf_t * buf,boolean_t uncached,boolean_t l2arc,const zio_prop_t * zp,arc_write_done_func_t * ready,arc_write_done_func_t * children_ready,arc_write_done_func_t * done,void * private,zio_priority_t priority,int zio_flags,const zbookmark_phys_t * zb)6851 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
6852 blkptr_t *bp, arc_buf_t *buf, boolean_t uncached, boolean_t l2arc,
6853 const zio_prop_t *zp, arc_write_done_func_t *ready,
6854 arc_write_done_func_t *children_ready, arc_write_done_func_t *done,
6855 void *private, zio_priority_t priority, int zio_flags,
6856 const zbookmark_phys_t *zb)
6857 {
6858 arc_buf_hdr_t *hdr = buf->b_hdr;
6859 arc_write_callback_t *callback;
6860 zio_t *zio;
6861 zio_prop_t localprop = *zp;
6862
6863 ASSERT3P(ready, !=, NULL);
6864 ASSERT3P(done, !=, NULL);
6865 ASSERT(!HDR_IO_ERROR(hdr));
6866 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6867 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6868 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6869 if (uncached)
6870 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
6871 else if (l2arc)
6872 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
6873
6874 if (ARC_BUF_ENCRYPTED(buf)) {
6875 ASSERT(ARC_BUF_COMPRESSED(buf));
6876 localprop.zp_encrypt = B_TRUE;
6877 localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6878 localprop.zp_complevel = hdr->b_complevel;
6879 localprop.zp_byteorder =
6880 (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
6881 ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
6882 memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
6883 ZIO_DATA_SALT_LEN);
6884 memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
6885 ZIO_DATA_IV_LEN);
6886 memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
6887 ZIO_DATA_MAC_LEN);
6888 if (DMU_OT_IS_ENCRYPTED(localprop.zp_type)) {
6889 localprop.zp_nopwrite = B_FALSE;
6890 localprop.zp_copies =
6891 MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
6892 }
6893 zio_flags |= ZIO_FLAG_RAW;
6894 } else if (ARC_BUF_COMPRESSED(buf)) {
6895 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
6896 localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6897 localprop.zp_complevel = hdr->b_complevel;
6898 zio_flags |= ZIO_FLAG_RAW_COMPRESS;
6899 }
6900 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
6901 callback->awcb_ready = ready;
6902 callback->awcb_children_ready = children_ready;
6903 callback->awcb_done = done;
6904 callback->awcb_private = private;
6905 callback->awcb_buf = buf;
6906
6907 /*
6908 * The hdr's b_pabd is now stale, free it now. A new data block
6909 * will be allocated when the zio pipeline calls arc_write_ready().
6910 */
6911 if (hdr->b_l1hdr.b_pabd != NULL) {
6912 /*
6913 * If the buf is currently sharing the data block with
6914 * the hdr then we need to break that relationship here.
6915 * The hdr will remain with a NULL data pointer and the
6916 * buf will take sole ownership of the block.
6917 */
6918 if (ARC_BUF_SHARED(buf)) {
6919 arc_unshare_buf(hdr, buf);
6920 } else {
6921 ASSERT(!arc_buf_is_shared(buf));
6922 arc_hdr_free_abd(hdr, B_FALSE);
6923 }
6924 VERIFY3P(buf->b_data, !=, NULL);
6925 }
6926
6927 if (HDR_HAS_RABD(hdr))
6928 arc_hdr_free_abd(hdr, B_TRUE);
6929
6930 if (!(zio_flags & ZIO_FLAG_RAW))
6931 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
6932
6933 ASSERT(!arc_buf_is_shared(buf));
6934 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6935
6936 zio = zio_write(pio, spa, txg, bp,
6937 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6938 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
6939 (children_ready != NULL) ? arc_write_children_ready : NULL,
6940 arc_write_done, callback, priority, zio_flags, zb);
6941
6942 return (zio);
6943 }
6944
6945 void
arc_tempreserve_clear(uint64_t reserve)6946 arc_tempreserve_clear(uint64_t reserve)
6947 {
6948 atomic_add_64(&arc_tempreserve, -reserve);
6949 ASSERT((int64_t)arc_tempreserve >= 0);
6950 }
6951
6952 int
arc_tempreserve_space(spa_t * spa,uint64_t reserve,uint64_t txg)6953 arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg)
6954 {
6955 int error;
6956 uint64_t anon_size;
6957
6958 if (!arc_no_grow &&
6959 reserve > arc_c/4 &&
6960 reserve * 4 > (2ULL << SPA_MAXBLOCKSHIFT))
6961 arc_c = MIN(arc_c_max, reserve * 4);
6962
6963 /*
6964 * Throttle when the calculated memory footprint for the TXG
6965 * exceeds the target ARC size.
6966 */
6967 if (reserve > arc_c) {
6968 DMU_TX_STAT_BUMP(dmu_tx_memory_reserve);
6969 return (SET_ERROR(ERESTART));
6970 }
6971
6972 /*
6973 * Don't count loaned bufs as in flight dirty data to prevent long
6974 * network delays from blocking transactions that are ready to be
6975 * assigned to a txg.
6976 */
6977
6978 /* assert that it has not wrapped around */
6979 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
6980
6981 anon_size = MAX((int64_t)
6982 (zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
6983 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
6984 arc_loaned_bytes), 0);
6985
6986 /*
6987 * Writes will, almost always, require additional memory allocations
6988 * in order to compress/encrypt/etc the data. We therefore need to
6989 * make sure that there is sufficient available memory for this.
6990 */
6991 error = arc_memory_throttle(spa, reserve, txg);
6992 if (error != 0)
6993 return (error);
6994
6995 /*
6996 * Throttle writes when the amount of dirty data in the cache
6997 * gets too large. We try to keep the cache less than half full
6998 * of dirty blocks so that our sync times don't grow too large.
6999 *
7000 * In the case of one pool being built on another pool, we want
7001 * to make sure we don't end up throttling the lower (backing)
7002 * pool when the upper pool is the majority contributor to dirty
7003 * data. To insure we make forward progress during throttling, we
7004 * also check the current pool's net dirty data and only throttle
7005 * if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty
7006 * data in the cache.
7007 *
7008 * Note: if two requests come in concurrently, we might let them
7009 * both succeed, when one of them should fail. Not a huge deal.
7010 */
7011 uint64_t total_dirty = reserve + arc_tempreserve + anon_size;
7012 uint64_t spa_dirty_anon = spa_dirty_data(spa);
7013 uint64_t rarc_c = arc_warm ? arc_c : arc_c_max;
7014 if (total_dirty > rarc_c * zfs_arc_dirty_limit_percent / 100 &&
7015 anon_size > rarc_c * zfs_arc_anon_limit_percent / 100 &&
7016 spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) {
7017 #ifdef ZFS_DEBUG
7018 uint64_t meta_esize = zfs_refcount_count(
7019 &arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7020 uint64_t data_esize =
7021 zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7022 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
7023 "anon_data=%lluK tempreserve=%lluK rarc_c=%lluK\n",
7024 (u_longlong_t)arc_tempreserve >> 10,
7025 (u_longlong_t)meta_esize >> 10,
7026 (u_longlong_t)data_esize >> 10,
7027 (u_longlong_t)reserve >> 10,
7028 (u_longlong_t)rarc_c >> 10);
7029 #endif
7030 DMU_TX_STAT_BUMP(dmu_tx_dirty_throttle);
7031 return (SET_ERROR(ERESTART));
7032 }
7033 atomic_add_64(&arc_tempreserve, reserve);
7034 return (0);
7035 }
7036
7037 static void
arc_kstat_update_state(arc_state_t * state,kstat_named_t * size,kstat_named_t * data,kstat_named_t * metadata,kstat_named_t * evict_data,kstat_named_t * evict_metadata)7038 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
7039 kstat_named_t *data, kstat_named_t *metadata,
7040 kstat_named_t *evict_data, kstat_named_t *evict_metadata)
7041 {
7042 data->value.ui64 =
7043 zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
7044 metadata->value.ui64 =
7045 zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
7046 size->value.ui64 = data->value.ui64 + metadata->value.ui64;
7047 evict_data->value.ui64 =
7048 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
7049 evict_metadata->value.ui64 =
7050 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
7051 }
7052
7053 static int
arc_kstat_update(kstat_t * ksp,int rw)7054 arc_kstat_update(kstat_t *ksp, int rw)
7055 {
7056 arc_stats_t *as = ksp->ks_data;
7057
7058 if (rw == KSTAT_WRITE)
7059 return (SET_ERROR(EACCES));
7060
7061 as->arcstat_hits.value.ui64 =
7062 wmsum_value(&arc_sums.arcstat_hits);
7063 as->arcstat_iohits.value.ui64 =
7064 wmsum_value(&arc_sums.arcstat_iohits);
7065 as->arcstat_misses.value.ui64 =
7066 wmsum_value(&arc_sums.arcstat_misses);
7067 as->arcstat_demand_data_hits.value.ui64 =
7068 wmsum_value(&arc_sums.arcstat_demand_data_hits);
7069 as->arcstat_demand_data_iohits.value.ui64 =
7070 wmsum_value(&arc_sums.arcstat_demand_data_iohits);
7071 as->arcstat_demand_data_misses.value.ui64 =
7072 wmsum_value(&arc_sums.arcstat_demand_data_misses);
7073 as->arcstat_demand_metadata_hits.value.ui64 =
7074 wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
7075 as->arcstat_demand_metadata_iohits.value.ui64 =
7076 wmsum_value(&arc_sums.arcstat_demand_metadata_iohits);
7077 as->arcstat_demand_metadata_misses.value.ui64 =
7078 wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
7079 as->arcstat_prefetch_data_hits.value.ui64 =
7080 wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
7081 as->arcstat_prefetch_data_iohits.value.ui64 =
7082 wmsum_value(&arc_sums.arcstat_prefetch_data_iohits);
7083 as->arcstat_prefetch_data_misses.value.ui64 =
7084 wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
7085 as->arcstat_prefetch_metadata_hits.value.ui64 =
7086 wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
7087 as->arcstat_prefetch_metadata_iohits.value.ui64 =
7088 wmsum_value(&arc_sums.arcstat_prefetch_metadata_iohits);
7089 as->arcstat_prefetch_metadata_misses.value.ui64 =
7090 wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
7091 as->arcstat_mru_hits.value.ui64 =
7092 wmsum_value(&arc_sums.arcstat_mru_hits);
7093 as->arcstat_mru_ghost_hits.value.ui64 =
7094 wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
7095 as->arcstat_mfu_hits.value.ui64 =
7096 wmsum_value(&arc_sums.arcstat_mfu_hits);
7097 as->arcstat_mfu_ghost_hits.value.ui64 =
7098 wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
7099 as->arcstat_uncached_hits.value.ui64 =
7100 wmsum_value(&arc_sums.arcstat_uncached_hits);
7101 as->arcstat_deleted.value.ui64 =
7102 wmsum_value(&arc_sums.arcstat_deleted);
7103 as->arcstat_mutex_miss.value.ui64 =
7104 wmsum_value(&arc_sums.arcstat_mutex_miss);
7105 as->arcstat_access_skip.value.ui64 =
7106 wmsum_value(&arc_sums.arcstat_access_skip);
7107 as->arcstat_evict_skip.value.ui64 =
7108 wmsum_value(&arc_sums.arcstat_evict_skip);
7109 as->arcstat_evict_not_enough.value.ui64 =
7110 wmsum_value(&arc_sums.arcstat_evict_not_enough);
7111 as->arcstat_evict_l2_cached.value.ui64 =
7112 wmsum_value(&arc_sums.arcstat_evict_l2_cached);
7113 as->arcstat_evict_l2_eligible.value.ui64 =
7114 wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
7115 as->arcstat_evict_l2_eligible_mfu.value.ui64 =
7116 wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
7117 as->arcstat_evict_l2_eligible_mru.value.ui64 =
7118 wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
7119 as->arcstat_evict_l2_ineligible.value.ui64 =
7120 wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
7121 as->arcstat_evict_l2_skip.value.ui64 =
7122 wmsum_value(&arc_sums.arcstat_evict_l2_skip);
7123 as->arcstat_hash_elements.value.ui64 =
7124 as->arcstat_hash_elements_max.value.ui64 =
7125 wmsum_value(&arc_sums.arcstat_hash_elements);
7126 as->arcstat_hash_collisions.value.ui64 =
7127 wmsum_value(&arc_sums.arcstat_hash_collisions);
7128 as->arcstat_hash_chains.value.ui64 =
7129 wmsum_value(&arc_sums.arcstat_hash_chains);
7130 as->arcstat_size.value.ui64 =
7131 aggsum_value(&arc_sums.arcstat_size);
7132 as->arcstat_compressed_size.value.ui64 =
7133 wmsum_value(&arc_sums.arcstat_compressed_size);
7134 as->arcstat_uncompressed_size.value.ui64 =
7135 wmsum_value(&arc_sums.arcstat_uncompressed_size);
7136 as->arcstat_overhead_size.value.ui64 =
7137 wmsum_value(&arc_sums.arcstat_overhead_size);
7138 as->arcstat_hdr_size.value.ui64 =
7139 wmsum_value(&arc_sums.arcstat_hdr_size);
7140 as->arcstat_data_size.value.ui64 =
7141 wmsum_value(&arc_sums.arcstat_data_size);
7142 as->arcstat_metadata_size.value.ui64 =
7143 wmsum_value(&arc_sums.arcstat_metadata_size);
7144 as->arcstat_dbuf_size.value.ui64 =
7145 wmsum_value(&arc_sums.arcstat_dbuf_size);
7146 #if defined(COMPAT_FREEBSD11)
7147 as->arcstat_other_size.value.ui64 =
7148 wmsum_value(&arc_sums.arcstat_bonus_size) +
7149 wmsum_value(&arc_sums.arcstat_dnode_size) +
7150 wmsum_value(&arc_sums.arcstat_dbuf_size);
7151 #endif
7152
7153 arc_kstat_update_state(arc_anon,
7154 &as->arcstat_anon_size,
7155 &as->arcstat_anon_data,
7156 &as->arcstat_anon_metadata,
7157 &as->arcstat_anon_evictable_data,
7158 &as->arcstat_anon_evictable_metadata);
7159 arc_kstat_update_state(arc_mru,
7160 &as->arcstat_mru_size,
7161 &as->arcstat_mru_data,
7162 &as->arcstat_mru_metadata,
7163 &as->arcstat_mru_evictable_data,
7164 &as->arcstat_mru_evictable_metadata);
7165 arc_kstat_update_state(arc_mru_ghost,
7166 &as->arcstat_mru_ghost_size,
7167 &as->arcstat_mru_ghost_data,
7168 &as->arcstat_mru_ghost_metadata,
7169 &as->arcstat_mru_ghost_evictable_data,
7170 &as->arcstat_mru_ghost_evictable_metadata);
7171 arc_kstat_update_state(arc_mfu,
7172 &as->arcstat_mfu_size,
7173 &as->arcstat_mfu_data,
7174 &as->arcstat_mfu_metadata,
7175 &as->arcstat_mfu_evictable_data,
7176 &as->arcstat_mfu_evictable_metadata);
7177 arc_kstat_update_state(arc_mfu_ghost,
7178 &as->arcstat_mfu_ghost_size,
7179 &as->arcstat_mfu_ghost_data,
7180 &as->arcstat_mfu_ghost_metadata,
7181 &as->arcstat_mfu_ghost_evictable_data,
7182 &as->arcstat_mfu_ghost_evictable_metadata);
7183 arc_kstat_update_state(arc_uncached,
7184 &as->arcstat_uncached_size,
7185 &as->arcstat_uncached_data,
7186 &as->arcstat_uncached_metadata,
7187 &as->arcstat_uncached_evictable_data,
7188 &as->arcstat_uncached_evictable_metadata);
7189
7190 as->arcstat_dnode_size.value.ui64 =
7191 wmsum_value(&arc_sums.arcstat_dnode_size);
7192 as->arcstat_bonus_size.value.ui64 =
7193 wmsum_value(&arc_sums.arcstat_bonus_size);
7194 as->arcstat_l2_hits.value.ui64 =
7195 wmsum_value(&arc_sums.arcstat_l2_hits);
7196 as->arcstat_l2_misses.value.ui64 =
7197 wmsum_value(&arc_sums.arcstat_l2_misses);
7198 as->arcstat_l2_prefetch_asize.value.ui64 =
7199 wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
7200 as->arcstat_l2_mru_asize.value.ui64 =
7201 wmsum_value(&arc_sums.arcstat_l2_mru_asize);
7202 as->arcstat_l2_mfu_asize.value.ui64 =
7203 wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
7204 as->arcstat_l2_bufc_data_asize.value.ui64 =
7205 wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
7206 as->arcstat_l2_bufc_metadata_asize.value.ui64 =
7207 wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
7208 as->arcstat_l2_feeds.value.ui64 =
7209 wmsum_value(&arc_sums.arcstat_l2_feeds);
7210 as->arcstat_l2_rw_clash.value.ui64 =
7211 wmsum_value(&arc_sums.arcstat_l2_rw_clash);
7212 as->arcstat_l2_read_bytes.value.ui64 =
7213 wmsum_value(&arc_sums.arcstat_l2_read_bytes);
7214 as->arcstat_l2_write_bytes.value.ui64 =
7215 wmsum_value(&arc_sums.arcstat_l2_write_bytes);
7216 as->arcstat_l2_writes_sent.value.ui64 =
7217 wmsum_value(&arc_sums.arcstat_l2_writes_sent);
7218 as->arcstat_l2_writes_done.value.ui64 =
7219 wmsum_value(&arc_sums.arcstat_l2_writes_done);
7220 as->arcstat_l2_writes_error.value.ui64 =
7221 wmsum_value(&arc_sums.arcstat_l2_writes_error);
7222 as->arcstat_l2_writes_lock_retry.value.ui64 =
7223 wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
7224 as->arcstat_l2_evict_lock_retry.value.ui64 =
7225 wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
7226 as->arcstat_l2_evict_reading.value.ui64 =
7227 wmsum_value(&arc_sums.arcstat_l2_evict_reading);
7228 as->arcstat_l2_evict_l1cached.value.ui64 =
7229 wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
7230 as->arcstat_l2_free_on_write.value.ui64 =
7231 wmsum_value(&arc_sums.arcstat_l2_free_on_write);
7232 as->arcstat_l2_abort_lowmem.value.ui64 =
7233 wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
7234 as->arcstat_l2_cksum_bad.value.ui64 =
7235 wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
7236 as->arcstat_l2_io_error.value.ui64 =
7237 wmsum_value(&arc_sums.arcstat_l2_io_error);
7238 as->arcstat_l2_lsize.value.ui64 =
7239 wmsum_value(&arc_sums.arcstat_l2_lsize);
7240 as->arcstat_l2_psize.value.ui64 =
7241 wmsum_value(&arc_sums.arcstat_l2_psize);
7242 as->arcstat_l2_hdr_size.value.ui64 =
7243 aggsum_value(&arc_sums.arcstat_l2_hdr_size);
7244 as->arcstat_l2_log_blk_writes.value.ui64 =
7245 wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
7246 as->arcstat_l2_log_blk_asize.value.ui64 =
7247 wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
7248 as->arcstat_l2_log_blk_count.value.ui64 =
7249 wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
7250 as->arcstat_l2_rebuild_success.value.ui64 =
7251 wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
7252 as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
7253 wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
7254 as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
7255 wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
7256 as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
7257 wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
7258 as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
7259 wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
7260 as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
7261 wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
7262 as->arcstat_l2_rebuild_size.value.ui64 =
7263 wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
7264 as->arcstat_l2_rebuild_asize.value.ui64 =
7265 wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
7266 as->arcstat_l2_rebuild_bufs.value.ui64 =
7267 wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
7268 as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
7269 wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
7270 as->arcstat_l2_rebuild_log_blks.value.ui64 =
7271 wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
7272 as->arcstat_memory_throttle_count.value.ui64 =
7273 wmsum_value(&arc_sums.arcstat_memory_throttle_count);
7274 as->arcstat_memory_direct_count.value.ui64 =
7275 wmsum_value(&arc_sums.arcstat_memory_direct_count);
7276 as->arcstat_memory_indirect_count.value.ui64 =
7277 wmsum_value(&arc_sums.arcstat_memory_indirect_count);
7278
7279 as->arcstat_memory_all_bytes.value.ui64 =
7280 arc_all_memory();
7281 as->arcstat_memory_free_bytes.value.ui64 =
7282 arc_free_memory();
7283 as->arcstat_memory_available_bytes.value.i64 =
7284 arc_available_memory();
7285
7286 as->arcstat_prune.value.ui64 =
7287 wmsum_value(&arc_sums.arcstat_prune);
7288 as->arcstat_meta_used.value.ui64 =
7289 wmsum_value(&arc_sums.arcstat_meta_used);
7290 as->arcstat_async_upgrade_sync.value.ui64 =
7291 wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
7292 as->arcstat_predictive_prefetch.value.ui64 =
7293 wmsum_value(&arc_sums.arcstat_predictive_prefetch);
7294 as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
7295 wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7296 as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
7297 wmsum_value(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7298 as->arcstat_prescient_prefetch.value.ui64 =
7299 wmsum_value(&arc_sums.arcstat_prescient_prefetch);
7300 as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
7301 wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7302 as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
7303 wmsum_value(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7304 as->arcstat_raw_size.value.ui64 =
7305 wmsum_value(&arc_sums.arcstat_raw_size);
7306 as->arcstat_cached_only_in_progress.value.ui64 =
7307 wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
7308 as->arcstat_abd_chunk_waste_size.value.ui64 =
7309 wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);
7310
7311 return (0);
7312 }
7313
7314 /*
7315 * This function *must* return indices evenly distributed between all
7316 * sublists of the multilist. This is needed due to how the ARC eviction
7317 * code is laid out; arc_evict_state() assumes ARC buffers are evenly
7318 * distributed between all sublists and uses this assumption when
7319 * deciding which sublist to evict from and how much to evict from it.
7320 */
7321 static unsigned int
arc_state_multilist_index_func(multilist_t * ml,void * obj)7322 arc_state_multilist_index_func(multilist_t *ml, void *obj)
7323 {
7324 arc_buf_hdr_t *hdr = obj;
7325
7326 /*
7327 * We rely on b_dva to generate evenly distributed index
7328 * numbers using buf_hash below. So, as an added precaution,
7329 * let's make sure we never add empty buffers to the arc lists.
7330 */
7331 ASSERT(!HDR_EMPTY(hdr));
7332
7333 /*
7334 * The assumption here, is the hash value for a given
7335 * arc_buf_hdr_t will remain constant throughout its lifetime
7336 * (i.e. its b_spa, b_dva, and b_birth fields don't change).
7337 * Thus, we don't need to store the header's sublist index
7338 * on insertion, as this index can be recalculated on removal.
7339 *
7340 * Also, the low order bits of the hash value are thought to be
7341 * distributed evenly. Otherwise, in the case that the multilist
7342 * has a power of two number of sublists, each sublists' usage
7343 * would not be evenly distributed. In this context full 64bit
7344 * division would be a waste of time, so limit it to 32 bits.
7345 */
7346 return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
7347 multilist_get_num_sublists(ml));
7348 }
7349
7350 static unsigned int
arc_state_l2c_multilist_index_func(multilist_t * ml,void * obj)7351 arc_state_l2c_multilist_index_func(multilist_t *ml, void *obj)
7352 {
7353 panic("Header %p insert into arc_l2c_only %p", obj, ml);
7354 }
7355
7356 #define WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do { \
7357 if ((do_warn) && (tuning) && ((tuning) != (value))) { \
7358 cmn_err(CE_WARN, \
7359 "ignoring tunable %s (using %llu instead)", \
7360 (#tuning), (u_longlong_t)(value)); \
7361 } \
7362 } while (0)
7363
7364 /*
7365 * Called during module initialization and periodically thereafter to
7366 * apply reasonable changes to the exposed performance tunings. Can also be
7367 * called explicitly by param_set_arc_*() functions when ARC tunables are
7368 * updated manually. Non-zero zfs_* values which differ from the currently set
7369 * values will be applied.
7370 */
7371 void
arc_tuning_update(boolean_t verbose)7372 arc_tuning_update(boolean_t verbose)
7373 {
7374 uint64_t allmem = arc_all_memory();
7375
7376 /* Valid range: 32M - <arc_c_max> */
7377 if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) &&
7378 (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) &&
7379 (zfs_arc_min <= arc_c_max)) {
7380 arc_c_min = zfs_arc_min;
7381 arc_c = MAX(arc_c, arc_c_min);
7382 }
7383 WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose);
7384
7385 /* Valid range: 64M - <all physical memory> */
7386 if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
7387 (zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
7388 (zfs_arc_max > arc_c_min)) {
7389 arc_c_max = zfs_arc_max;
7390 arc_c = MIN(arc_c, arc_c_max);
7391 if (arc_dnode_limit > arc_c_max)
7392 arc_dnode_limit = arc_c_max;
7393 }
7394 WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose);
7395
7396 /* Valid range: 0 - <all physical memory> */
7397 arc_dnode_limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit :
7398 MIN(zfs_arc_dnode_limit_percent, 100) * arc_c_max / 100;
7399 WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_limit, verbose);
7400
7401 /* Valid range: 1 - N */
7402 if (zfs_arc_grow_retry)
7403 arc_grow_retry = zfs_arc_grow_retry;
7404
7405 /* Valid range: 1 - N */
7406 if (zfs_arc_shrink_shift) {
7407 arc_shrink_shift = zfs_arc_shrink_shift;
7408 arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
7409 }
7410
7411 /* Valid range: 1 - N ms */
7412 if (zfs_arc_min_prefetch_ms)
7413 arc_min_prefetch_ms = zfs_arc_min_prefetch_ms;
7414
7415 /* Valid range: 1 - N ms */
7416 if (zfs_arc_min_prescient_prefetch_ms) {
7417 arc_min_prescient_prefetch_ms =
7418 zfs_arc_min_prescient_prefetch_ms;
7419 }
7420
7421 /* Valid range: 0 - 100 */
7422 if (zfs_arc_lotsfree_percent <= 100)
7423 arc_lotsfree_percent = zfs_arc_lotsfree_percent;
7424 WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent,
7425 verbose);
7426
7427 /* Valid range: 0 - <all physical memory> */
7428 if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free))
7429 arc_sys_free = MIN(zfs_arc_sys_free, allmem);
7430 WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose);
7431 }
7432
7433 static void
arc_state_multilist_init(multilist_t * ml,multilist_sublist_index_func_t * index_func,int * maxcountp)7434 arc_state_multilist_init(multilist_t *ml,
7435 multilist_sublist_index_func_t *index_func, int *maxcountp)
7436 {
7437 multilist_create(ml, sizeof (arc_buf_hdr_t),
7438 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func);
7439 *maxcountp = MAX(*maxcountp, multilist_get_num_sublists(ml));
7440 }
7441
7442 static void
arc_state_init(void)7443 arc_state_init(void)
7444 {
7445 int num_sublists = 0;
7446
7447 arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
7448 arc_state_multilist_index_func, &num_sublists);
7449 arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
7450 arc_state_multilist_index_func, &num_sublists);
7451 arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
7452 arc_state_multilist_index_func, &num_sublists);
7453 arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
7454 arc_state_multilist_index_func, &num_sublists);
7455 arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
7456 arc_state_multilist_index_func, &num_sublists);
7457 arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
7458 arc_state_multilist_index_func, &num_sublists);
7459 arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
7460 arc_state_multilist_index_func, &num_sublists);
7461 arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
7462 arc_state_multilist_index_func, &num_sublists);
7463 arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
7464 arc_state_multilist_index_func, &num_sublists);
7465 arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
7466 arc_state_multilist_index_func, &num_sublists);
7467
7468 /*
7469 * L2 headers should never be on the L2 state list since they don't
7470 * have L1 headers allocated. Special index function asserts that.
7471 */
7472 arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
7473 arc_state_l2c_multilist_index_func, &num_sublists);
7474 arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
7475 arc_state_l2c_multilist_index_func, &num_sublists);
7476
7477 /*
7478 * Keep track of the number of markers needed to reclaim buffers from
7479 * any ARC state. The markers will be pre-allocated so as to minimize
7480 * the number of memory allocations performed by the eviction thread.
7481 */
7482 arc_state_evict_marker_count = num_sublists;
7483
7484 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7485 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7486 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7487 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7488 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7489 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7490 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7491 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7492 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7493 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7494 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7495 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7496 zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7497 zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7498
7499 zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7500 zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7501 zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7502 zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7503 zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7504 zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7505 zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7506 zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7507 zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7508 zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7509 zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7510 zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7511 zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7512 zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7513
7514 wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7515 wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7516 wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7517 wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7518
7519 wmsum_init(&arc_sums.arcstat_hits, 0);
7520 wmsum_init(&arc_sums.arcstat_iohits, 0);
7521 wmsum_init(&arc_sums.arcstat_misses, 0);
7522 wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
7523 wmsum_init(&arc_sums.arcstat_demand_data_iohits, 0);
7524 wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
7525 wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
7526 wmsum_init(&arc_sums.arcstat_demand_metadata_iohits, 0);
7527 wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
7528 wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
7529 wmsum_init(&arc_sums.arcstat_prefetch_data_iohits, 0);
7530 wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
7531 wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
7532 wmsum_init(&arc_sums.arcstat_prefetch_metadata_iohits, 0);
7533 wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
7534 wmsum_init(&arc_sums.arcstat_mru_hits, 0);
7535 wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
7536 wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
7537 wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
7538 wmsum_init(&arc_sums.arcstat_uncached_hits, 0);
7539 wmsum_init(&arc_sums.arcstat_deleted, 0);
7540 wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
7541 wmsum_init(&arc_sums.arcstat_access_skip, 0);
7542 wmsum_init(&arc_sums.arcstat_evict_skip, 0);
7543 wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
7544 wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
7545 wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
7546 wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
7547 wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
7548 wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
7549 wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
7550 wmsum_init(&arc_sums.arcstat_hash_elements, 0);
7551 wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
7552 wmsum_init(&arc_sums.arcstat_hash_chains, 0);
7553 aggsum_init(&arc_sums.arcstat_size, 0);
7554 wmsum_init(&arc_sums.arcstat_compressed_size, 0);
7555 wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
7556 wmsum_init(&arc_sums.arcstat_overhead_size, 0);
7557 wmsum_init(&arc_sums.arcstat_hdr_size, 0);
7558 wmsum_init(&arc_sums.arcstat_data_size, 0);
7559 wmsum_init(&arc_sums.arcstat_metadata_size, 0);
7560 wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
7561 wmsum_init(&arc_sums.arcstat_dnode_size, 0);
7562 wmsum_init(&arc_sums.arcstat_bonus_size, 0);
7563 wmsum_init(&arc_sums.arcstat_l2_hits, 0);
7564 wmsum_init(&arc_sums.arcstat_l2_misses, 0);
7565 wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
7566 wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
7567 wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
7568 wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
7569 wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
7570 wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
7571 wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
7572 wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
7573 wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
7574 wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
7575 wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
7576 wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
7577 wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
7578 wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
7579 wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
7580 wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
7581 wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
7582 wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
7583 wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
7584 wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
7585 wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
7586 wmsum_init(&arc_sums.arcstat_l2_psize, 0);
7587 aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
7588 wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
7589 wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
7590 wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
7591 wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
7592 wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
7593 wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
7594 wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
7595 wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
7596 wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
7597 wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
7598 wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
7599 wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
7600 wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
7601 wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
7602 wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
7603 wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
7604 wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
7605 wmsum_init(&arc_sums.arcstat_prune, 0);
7606 wmsum_init(&arc_sums.arcstat_meta_used, 0);
7607 wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
7608 wmsum_init(&arc_sums.arcstat_predictive_prefetch, 0);
7609 wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
7610 wmsum_init(&arc_sums.arcstat_demand_iohit_predictive_prefetch, 0);
7611 wmsum_init(&arc_sums.arcstat_prescient_prefetch, 0);
7612 wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
7613 wmsum_init(&arc_sums.arcstat_demand_iohit_prescient_prefetch, 0);
7614 wmsum_init(&arc_sums.arcstat_raw_size, 0);
7615 wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
7616 wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);
7617
7618 arc_anon->arcs_state = ARC_STATE_ANON;
7619 arc_mru->arcs_state = ARC_STATE_MRU;
7620 arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
7621 arc_mfu->arcs_state = ARC_STATE_MFU;
7622 arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
7623 arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
7624 arc_uncached->arcs_state = ARC_STATE_UNCACHED;
7625 }
7626
7627 static void
arc_state_fini(void)7628 arc_state_fini(void)
7629 {
7630 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7631 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7632 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7633 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7634 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7635 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7636 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7637 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7638 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7639 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7640 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7641 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7642 zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7643 zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7644
7645 zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7646 zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7647 zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7648 zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7649 zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7650 zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7651 zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7652 zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7653 zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7654 zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7655 zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7656 zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7657 zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7658 zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7659
7660 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
7661 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7662 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7663 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7664 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
7665 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7666 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
7667 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7668 multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7669 multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
7670 multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
7671 multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);
7672
7673 wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
7674 wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
7675 wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
7676 wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
7677
7678 wmsum_fini(&arc_sums.arcstat_hits);
7679 wmsum_fini(&arc_sums.arcstat_iohits);
7680 wmsum_fini(&arc_sums.arcstat_misses);
7681 wmsum_fini(&arc_sums.arcstat_demand_data_hits);
7682 wmsum_fini(&arc_sums.arcstat_demand_data_iohits);
7683 wmsum_fini(&arc_sums.arcstat_demand_data_misses);
7684 wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
7685 wmsum_fini(&arc_sums.arcstat_demand_metadata_iohits);
7686 wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
7687 wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
7688 wmsum_fini(&arc_sums.arcstat_prefetch_data_iohits);
7689 wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
7690 wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
7691 wmsum_fini(&arc_sums.arcstat_prefetch_metadata_iohits);
7692 wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
7693 wmsum_fini(&arc_sums.arcstat_mru_hits);
7694 wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
7695 wmsum_fini(&arc_sums.arcstat_mfu_hits);
7696 wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
7697 wmsum_fini(&arc_sums.arcstat_uncached_hits);
7698 wmsum_fini(&arc_sums.arcstat_deleted);
7699 wmsum_fini(&arc_sums.arcstat_mutex_miss);
7700 wmsum_fini(&arc_sums.arcstat_access_skip);
7701 wmsum_fini(&arc_sums.arcstat_evict_skip);
7702 wmsum_fini(&arc_sums.arcstat_evict_not_enough);
7703 wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
7704 wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
7705 wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
7706 wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
7707 wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
7708 wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
7709 wmsum_fini(&arc_sums.arcstat_hash_elements);
7710 wmsum_fini(&arc_sums.arcstat_hash_collisions);
7711 wmsum_fini(&arc_sums.arcstat_hash_chains);
7712 aggsum_fini(&arc_sums.arcstat_size);
7713 wmsum_fini(&arc_sums.arcstat_compressed_size);
7714 wmsum_fini(&arc_sums.arcstat_uncompressed_size);
7715 wmsum_fini(&arc_sums.arcstat_overhead_size);
7716 wmsum_fini(&arc_sums.arcstat_hdr_size);
7717 wmsum_fini(&arc_sums.arcstat_data_size);
7718 wmsum_fini(&arc_sums.arcstat_metadata_size);
7719 wmsum_fini(&arc_sums.arcstat_dbuf_size);
7720 wmsum_fini(&arc_sums.arcstat_dnode_size);
7721 wmsum_fini(&arc_sums.arcstat_bonus_size);
7722 wmsum_fini(&arc_sums.arcstat_l2_hits);
7723 wmsum_fini(&arc_sums.arcstat_l2_misses);
7724 wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
7725 wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
7726 wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
7727 wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
7728 wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
7729 wmsum_fini(&arc_sums.arcstat_l2_feeds);
7730 wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
7731 wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
7732 wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
7733 wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
7734 wmsum_fini(&arc_sums.arcstat_l2_writes_done);
7735 wmsum_fini(&arc_sums.arcstat_l2_writes_error);
7736 wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
7737 wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
7738 wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
7739 wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
7740 wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
7741 wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
7742 wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
7743 wmsum_fini(&arc_sums.arcstat_l2_io_error);
7744 wmsum_fini(&arc_sums.arcstat_l2_lsize);
7745 wmsum_fini(&arc_sums.arcstat_l2_psize);
7746 aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
7747 wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
7748 wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
7749 wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
7750 wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
7751 wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
7752 wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
7753 wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
7754 wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
7755 wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
7756 wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
7757 wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
7758 wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
7759 wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
7760 wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
7761 wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
7762 wmsum_fini(&arc_sums.arcstat_memory_direct_count);
7763 wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
7764 wmsum_fini(&arc_sums.arcstat_prune);
7765 wmsum_fini(&arc_sums.arcstat_meta_used);
7766 wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
7767 wmsum_fini(&arc_sums.arcstat_predictive_prefetch);
7768 wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
7769 wmsum_fini(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
7770 wmsum_fini(&arc_sums.arcstat_prescient_prefetch);
7771 wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
7772 wmsum_fini(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
7773 wmsum_fini(&arc_sums.arcstat_raw_size);
7774 wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
7775 wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
7776 }
7777
7778 uint64_t
arc_target_bytes(void)7779 arc_target_bytes(void)
7780 {
7781 return (arc_c);
7782 }
7783
7784 void
arc_set_limits(uint64_t allmem)7785 arc_set_limits(uint64_t allmem)
7786 {
7787 /* Set min cache to 1/32 of all memory, or 32MB, whichever is more. */
7788 arc_c_min = MAX(allmem / 32, 2ULL << SPA_MAXBLOCKSHIFT);
7789
7790 /* How to set default max varies by platform. */
7791 arc_c_max = arc_default_max(arc_c_min, allmem);
7792 }
7793 void
arc_init(void)7794 arc_init(void)
7795 {
7796 uint64_t percent, allmem = arc_all_memory();
7797 mutex_init(&arc_evict_lock, NULL, MUTEX_DEFAULT, NULL);
7798 list_create(&arc_evict_waiters, sizeof (arc_evict_waiter_t),
7799 offsetof(arc_evict_waiter_t, aew_node));
7800
7801 arc_min_prefetch_ms = 1000;
7802 arc_min_prescient_prefetch_ms = 6000;
7803
7804 #if defined(_KERNEL)
7805 arc_lowmem_init();
7806 #endif
7807
7808 arc_set_limits(allmem);
7809
7810 #ifdef _KERNEL
7811 /*
7812 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
7813 * environment before the module was loaded, don't block setting the
7814 * maximum because it is less than arc_c_min, instead, reset arc_c_min
7815 * to a lower value.
7816 * zfs_arc_min will be handled by arc_tuning_update().
7817 */
7818 if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
7819 zfs_arc_max < allmem) {
7820 arc_c_max = zfs_arc_max;
7821 if (arc_c_min >= arc_c_max) {
7822 arc_c_min = MAX(zfs_arc_max / 2,
7823 2ULL << SPA_MAXBLOCKSHIFT);
7824 }
7825 }
7826 #else
7827 /*
7828 * In userland, there's only the memory pressure that we artificially
7829 * create (see arc_available_memory()). Don't let arc_c get too
7830 * small, because it can cause transactions to be larger than
7831 * arc_c, causing arc_tempreserve_space() to fail.
7832 */
7833 arc_c_min = MAX(arc_c_max / 2, 2ULL << SPA_MAXBLOCKSHIFT);
7834 #endif
7835
7836 arc_c = arc_c_min;
7837 /*
7838 * 32-bit fixed point fractions of metadata from total ARC size,
7839 * MRU data from all data and MRU metadata from all metadata.
7840 */
7841 arc_meta = (1ULL << 32) / 4; /* Metadata is 25% of arc_c. */
7842 arc_pd = (1ULL << 32) / 2; /* Data MRU is 50% of data. */
7843 arc_pm = (1ULL << 32) / 2; /* Metadata MRU is 50% of metadata. */
7844
7845 percent = MIN(zfs_arc_dnode_limit_percent, 100);
7846 arc_dnode_limit = arc_c_max * percent / 100;
7847
7848 /* Apply user specified tunings */
7849 arc_tuning_update(B_TRUE);
7850
7851 /* if kmem_flags are set, lets try to use less memory */
7852 if (kmem_debugging())
7853 arc_c = arc_c / 2;
7854 if (arc_c < arc_c_min)
7855 arc_c = arc_c_min;
7856
7857 arc_register_hotplug();
7858
7859 arc_state_init();
7860
7861 buf_init();
7862
7863 list_create(&arc_prune_list, sizeof (arc_prune_t),
7864 offsetof(arc_prune_t, p_node));
7865 mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL);
7866
7867 arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
7868 defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
7869
7870 list_create(&arc_async_flush_list, sizeof (arc_async_flush_t),
7871 offsetof(arc_async_flush_t, af_node));
7872 mutex_init(&arc_async_flush_lock, NULL, MUTEX_DEFAULT, NULL);
7873 arc_flush_taskq = taskq_create("arc_flush", MIN(boot_ncpus, 4),
7874 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC);
7875
7876 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
7877 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
7878
7879 if (arc_ksp != NULL) {
7880 arc_ksp->ks_data = &arc_stats;
7881 arc_ksp->ks_update = arc_kstat_update;
7882 kstat_install(arc_ksp);
7883 }
7884
7885 arc_state_evict_markers =
7886 arc_state_alloc_markers(arc_state_evict_marker_count);
7887 arc_evict_zthr = zthr_create_timer("arc_evict",
7888 arc_evict_cb_check, arc_evict_cb, NULL, SEC2NSEC(1), defclsyspri);
7889 arc_reap_zthr = zthr_create_timer("arc_reap",
7890 arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);
7891
7892 arc_warm = B_FALSE;
7893
7894 /*
7895 * Calculate maximum amount of dirty data per pool.
7896 *
7897 * If it has been set by a module parameter, take that.
7898 * Otherwise, use a percentage of physical memory defined by
7899 * zfs_dirty_data_max_percent (default 10%) with a cap at
7900 * zfs_dirty_data_max_max (default 4G or 25% of physical memory).
7901 */
7902 #ifdef __LP64__
7903 if (zfs_dirty_data_max_max == 0)
7904 zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
7905 allmem * zfs_dirty_data_max_max_percent / 100);
7906 #else
7907 if (zfs_dirty_data_max_max == 0)
7908 zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
7909 allmem * zfs_dirty_data_max_max_percent / 100);
7910 #endif
7911
7912 if (zfs_dirty_data_max == 0) {
7913 zfs_dirty_data_max = allmem *
7914 zfs_dirty_data_max_percent / 100;
7915 zfs_dirty_data_max = MIN(zfs_dirty_data_max,
7916 zfs_dirty_data_max_max);
7917 }
7918
7919 if (zfs_wrlog_data_max == 0) {
7920
7921 /*
7922 * dp_wrlog_total is reduced for each txg at the end of
7923 * spa_sync(). However, dp_dirty_total is reduced every time
7924 * a block is written out. Thus under normal operation,
7925 * dp_wrlog_total could grow 2 times as big as
7926 * zfs_dirty_data_max.
7927 */
7928 zfs_wrlog_data_max = zfs_dirty_data_max * 2;
7929 }
7930 }
7931
7932 void
arc_fini(void)7933 arc_fini(void)
7934 {
7935 arc_prune_t *p;
7936
7937 #ifdef _KERNEL
7938 arc_lowmem_fini();
7939 #endif /* _KERNEL */
7940
7941 /* Wait for any background flushes */
7942 taskq_wait(arc_flush_taskq);
7943 taskq_destroy(arc_flush_taskq);
7944
7945 /* Use B_TRUE to ensure *all* buffers are evicted */
7946 arc_flush(NULL, B_TRUE);
7947
7948 if (arc_ksp != NULL) {
7949 kstat_delete(arc_ksp);
7950 arc_ksp = NULL;
7951 }
7952
7953 taskq_wait(arc_prune_taskq);
7954 taskq_destroy(arc_prune_taskq);
7955
7956 list_destroy(&arc_async_flush_list);
7957 mutex_destroy(&arc_async_flush_lock);
7958
7959 mutex_enter(&arc_prune_mtx);
7960 while ((p = list_remove_head(&arc_prune_list)) != NULL) {
7961 (void) zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
7962 zfs_refcount_destroy(&p->p_refcnt);
7963 kmem_free(p, sizeof (*p));
7964 }
7965 mutex_exit(&arc_prune_mtx);
7966
7967 list_destroy(&arc_prune_list);
7968 mutex_destroy(&arc_prune_mtx);
7969
7970 (void) zthr_cancel(arc_evict_zthr);
7971 (void) zthr_cancel(arc_reap_zthr);
7972 arc_state_free_markers(arc_state_evict_markers,
7973 arc_state_evict_marker_count);
7974
7975 mutex_destroy(&arc_evict_lock);
7976 list_destroy(&arc_evict_waiters);
7977
7978 /*
7979 * Free any buffers that were tagged for destruction. This needs
7980 * to occur before arc_state_fini() runs and destroys the aggsum
7981 * values which are updated when freeing scatter ABDs.
7982 */
7983 l2arc_do_free_on_write();
7984
7985 /*
7986 * buf_fini() must proceed arc_state_fini() because buf_fin() may
7987 * trigger the release of kmem magazines, which can callback to
7988 * arc_space_return() which accesses aggsums freed in act_state_fini().
7989 */
7990 buf_fini();
7991 arc_state_fini();
7992
7993 arc_unregister_hotplug();
7994
7995 /*
7996 * We destroy the zthrs after all the ARC state has been
7997 * torn down to avoid the case of them receiving any
7998 * wakeup() signals after they are destroyed.
7999 */
8000 zthr_destroy(arc_evict_zthr);
8001 zthr_destroy(arc_reap_zthr);
8002
8003 ASSERT0(arc_loaned_bytes);
8004 }
8005
8006 /*
8007 * Level 2 ARC
8008 *
8009 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
8010 * It uses dedicated storage devices to hold cached data, which are populated
8011 * using large infrequent writes. The main role of this cache is to boost
8012 * the performance of random read workloads. The intended L2ARC devices
8013 * include short-stroked disks, solid state disks, and other media with
8014 * substantially faster read latency than disk.
8015 *
8016 * +-----------------------+
8017 * | ARC |
8018 * +-----------------------+
8019 * | ^ ^
8020 * | | |
8021 * l2arc_feed_thread() arc_read()
8022 * | | |
8023 * | l2arc read |
8024 * V | |
8025 * +---------------+ |
8026 * | L2ARC | |
8027 * +---------------+ |
8028 * | ^ |
8029 * l2arc_write() | |
8030 * | | |
8031 * V | |
8032 * +-------+ +-------+
8033 * | vdev | | vdev |
8034 * | cache | | cache |
8035 * +-------+ +-------+
8036 * +=========+ .-----.
8037 * : L2ARC : |-_____-|
8038 * : devices : | Disks |
8039 * +=========+ `-_____-'
8040 *
8041 * Read requests are satisfied from the following sources, in order:
8042 *
8043 * 1) ARC
8044 * 2) vdev cache of L2ARC devices
8045 * 3) L2ARC devices
8046 * 4) vdev cache of disks
8047 * 5) disks
8048 *
8049 * Some L2ARC device types exhibit extremely slow write performance.
8050 * To accommodate for this there are some significant differences between
8051 * the L2ARC and traditional cache design:
8052 *
8053 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from
8054 * the ARC behave as usual, freeing buffers and placing headers on ghost
8055 * lists. The ARC does not send buffers to the L2ARC during eviction as
8056 * this would add inflated write latencies for all ARC memory pressure.
8057 *
8058 * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
8059 * It does this by periodically scanning buffers from the eviction-end of
8060 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
8061 * not already there. It scans until a headroom of buffers is satisfied,
8062 * which itself is a buffer for ARC eviction. If a compressible buffer is
8063 * found during scanning and selected for writing to an L2ARC device, we
8064 * temporarily boost scanning headroom during the next scan cycle to make
8065 * sure we adapt to compression effects (which might significantly reduce
8066 * the data volume we write to L2ARC). The thread that does this is
8067 * l2arc_feed_thread(), illustrated below; example sizes are included to
8068 * provide a better sense of ratio than this diagram:
8069 *
8070 * head --> tail
8071 * +---------------------+----------+
8072 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC
8073 * +---------------------+----------+ | o L2ARC eligible
8074 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer
8075 * +---------------------+----------+ |
8076 * 15.9 Gbytes ^ 32 Mbytes |
8077 * headroom |
8078 * l2arc_feed_thread()
8079 * |
8080 * l2arc write hand <--[oooo]--'
8081 * | 8 Mbyte
8082 * | write max
8083 * V
8084 * +==============================+
8085 * L2ARC dev |####|#|###|###| |####| ... |
8086 * +==============================+
8087 * 32 Gbytes
8088 *
8089 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
8090 * evicted, then the L2ARC has cached a buffer much sooner than it probably
8091 * needed to, potentially wasting L2ARC device bandwidth and storage. It is
8092 * safe to say that this is an uncommon case, since buffers at the end of
8093 * the ARC lists have moved there due to inactivity.
8094 *
8095 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
8096 * then the L2ARC simply misses copying some buffers. This serves as a
8097 * pressure valve to prevent heavy read workloads from both stalling the ARC
8098 * with waits and clogging the L2ARC with writes. This also helps prevent
8099 * the potential for the L2ARC to churn if it attempts to cache content too
8100 * quickly, such as during backups of the entire pool.
8101 *
8102 * 5. After system boot and before the ARC has filled main memory, there are
8103 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
8104 * lists can remain mostly static. Instead of searching from tail of these
8105 * lists as pictured, the l2arc_feed_thread() will search from the list heads
8106 * for eligible buffers, greatly increasing its chance of finding them.
8107 *
8108 * The L2ARC device write speed is also boosted during this time so that
8109 * the L2ARC warms up faster. Since there have been no ARC evictions yet,
8110 * there are no L2ARC reads, and no fear of degrading read performance
8111 * through increased writes.
8112 *
8113 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
8114 * the vdev queue can aggregate them into larger and fewer writes. Each
8115 * device is written to in a rotor fashion, sweeping writes through
8116 * available space then repeating.
8117 *
8118 * 7. The L2ARC does not store dirty content. It never needs to flush
8119 * write buffers back to disk based storage.
8120 *
8121 * 8. If an ARC buffer is written (and dirtied) which also exists in the
8122 * L2ARC, the now stale L2ARC buffer is immediately dropped.
8123 *
8124 * The performance of the L2ARC can be tweaked by a number of tunables, which
8125 * may be necessary for different workloads:
8126 *
8127 * l2arc_write_max max write bytes per interval
8128 * l2arc_write_boost extra write bytes during device warmup
8129 * l2arc_noprefetch skip caching prefetched buffers
8130 * l2arc_headroom number of max device writes to precache
8131 * l2arc_headroom_boost when we find compressed buffers during ARC
8132 * scanning, we multiply headroom by this
8133 * percentage factor for the next scan cycle,
8134 * since more compressed buffers are likely to
8135 * be present
8136 * l2arc_feed_secs seconds between L2ARC writing
8137 *
8138 * Tunables may be removed or added as future performance improvements are
8139 * integrated, and also may become zpool properties.
8140 *
8141 * There are three key functions that control how the L2ARC warms up:
8142 *
8143 * l2arc_write_eligible() check if a buffer is eligible to cache
8144 * l2arc_write_size() calculate how much to write
8145 * l2arc_write_interval() calculate sleep delay between writes
8146 *
8147 * These three functions determine what to write, how much, and how quickly
8148 * to send writes.
8149 *
8150 * L2ARC persistence:
8151 *
8152 * When writing buffers to L2ARC, we periodically add some metadata to
8153 * make sure we can pick them up after reboot, thus dramatically reducing
8154 * the impact that any downtime has on the performance of storage systems
8155 * with large caches.
8156 *
8157 * The implementation works fairly simply by integrating the following two
8158 * modifications:
8159 *
8160 * *) When writing to the L2ARC, we occasionally write a "l2arc log block",
8161 * which is an additional piece of metadata which describes what's been
8162 * written. This allows us to rebuild the arc_buf_hdr_t structures of the
8163 * main ARC buffers. There are 2 linked-lists of log blocks headed by
8164 * dh_start_lbps[2]. We alternate which chain we append to, so they are
8165 * time-wise and offset-wise interleaved, but that is an optimization rather
8166 * than for correctness. The log block also includes a pointer to the
8167 * previous block in its chain.
8168 *
8169 * *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device
8170 * for our header bookkeeping purposes. This contains a device header,
8171 * which contains our top-level reference structures. We update it each
8172 * time we write a new log block, so that we're able to locate it in the
8173 * L2ARC device. If this write results in an inconsistent device header
8174 * (e.g. due to power failure), we detect this by verifying the header's
8175 * checksum and simply fail to reconstruct the L2ARC after reboot.
8176 *
8177 * Implementation diagram:
8178 *
8179 * +=== L2ARC device (not to scale) ======================================+
8180 * | ___two newest log block pointers__.__________ |
8181 * | / \dh_start_lbps[1] |
8182 * | / \ \dh_start_lbps[0]|
8183 * |.___/__. V V |
8184 * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---|
8185 * || hdr| ^ /^ /^ / / |
8186 * |+------+ ...--\-------/ \-----/--\------/ / |
8187 * | \--------------/ \--------------/ |
8188 * +======================================================================+
8189 *
8190 * As can be seen on the diagram, rather than using a simple linked list,
8191 * we use a pair of linked lists with alternating elements. This is a
8192 * performance enhancement due to the fact that we only find out the
8193 * address of the next log block access once the current block has been
8194 * completely read in. Obviously, this hurts performance, because we'd be
8195 * keeping the device's I/O queue at only a 1 operation deep, thus
8196 * incurring a large amount of I/O round-trip latency. Having two lists
8197 * allows us to fetch two log blocks ahead of where we are currently
8198 * rebuilding L2ARC buffers.
8199 *
8200 * On-device data structures:
8201 *
8202 * L2ARC device header: l2arc_dev_hdr_phys_t
8203 * L2ARC log block: l2arc_log_blk_phys_t
8204 *
8205 * L2ARC reconstruction:
8206 *
8207 * When writing data, we simply write in the standard rotary fashion,
8208 * evicting buffers as we go and simply writing new data over them (writing
8209 * a new log block every now and then). This obviously means that once we
8210 * loop around the end of the device, we will start cutting into an already
8211 * committed log block (and its referenced data buffers), like so:
8212 *
8213 * current write head__ __old tail
8214 * \ /
8215 * V V
8216 * <--|bufs |lb |bufs |lb | |bufs |lb |bufs |lb |-->
8217 * ^ ^^^^^^^^^___________________________________
8218 * | \
8219 * <<nextwrite>> may overwrite this blk and/or its bufs --'
8220 *
8221 * When importing the pool, we detect this situation and use it to stop
8222 * our scanning process (see l2arc_rebuild).
8223 *
8224 * There is one significant caveat to consider when rebuilding ARC contents
8225 * from an L2ARC device: what about invalidated buffers? Given the above
8226 * construction, we cannot update blocks which we've already written to amend
8227 * them to remove buffers which were invalidated. Thus, during reconstruction,
8228 * we might be populating the cache with buffers for data that's not on the
8229 * main pool anymore, or may have been overwritten!
8230 *
8231 * As it turns out, this isn't a problem. Every arc_read request includes
8232 * both the DVA and, crucially, the birth TXG of the BP the caller is
8233 * looking for. So even if the cache were populated by completely rotten
8234 * blocks for data that had been long deleted and/or overwritten, we'll
8235 * never actually return bad data from the cache, since the DVA with the
8236 * birth TXG uniquely identify a block in space and time - once created,
8237 * a block is immutable on disk. The worst thing we have done is wasted
8238 * some time and memory at l2arc rebuild to reconstruct outdated ARC
8239 * entries that will get dropped from the l2arc as it is being updated
8240 * with new blocks.
8241 *
8242 * L2ARC buffers that have been evicted by l2arc_evict() ahead of the write
8243 * hand are not restored. This is done by saving the offset (in bytes)
8244 * l2arc_evict() has evicted to in the L2ARC device header and taking it
8245 * into account when restoring buffers.
8246 */
8247
8248 static boolean_t
l2arc_write_eligible(uint64_t spa_guid,arc_buf_hdr_t * hdr)8249 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
8250 {
8251 /*
8252 * A buffer is *not* eligible for the L2ARC if it:
8253 * 1. belongs to a different spa.
8254 * 2. is already cached on the L2ARC.
8255 * 3. has an I/O in progress (it may be an incomplete read).
8256 * 4. is flagged not eligible (zfs property).
8257 */
8258 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
8259 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
8260 return (B_FALSE);
8261
8262 return (B_TRUE);
8263 }
8264
8265 static uint64_t
l2arc_write_size(l2arc_dev_t * dev)8266 l2arc_write_size(l2arc_dev_t *dev)
8267 {
8268 uint64_t size;
8269
8270 /*
8271 * Make sure our globals have meaningful values in case the user
8272 * altered them.
8273 */
8274 size = l2arc_write_max;
8275 if (size == 0) {
8276 cmn_err(CE_NOTE, "l2arc_write_max must be greater than zero, "
8277 "resetting it to the default (%d)", L2ARC_WRITE_SIZE);
8278 size = l2arc_write_max = L2ARC_WRITE_SIZE;
8279 }
8280
8281 if (arc_warm == B_FALSE)
8282 size += l2arc_write_boost;
8283
8284 /* We need to add in the worst case scenario of log block overhead. */
8285 size += l2arc_log_blk_overhead(size, dev);
8286 if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
8287 /*
8288 * Trim ahead of the write size 64MB or (l2arc_trim_ahead/100)
8289 * times the writesize, whichever is greater.
8290 */
8291 size += MAX(64 * 1024 * 1024,
8292 (size * l2arc_trim_ahead) / 100);
8293 }
8294
8295 /*
8296 * Make sure the write size does not exceed the size of the cache
8297 * device. This is important in l2arc_evict(), otherwise infinite
8298 * iteration can occur.
8299 */
8300 size = MIN(size, (dev->l2ad_end - dev->l2ad_start) / 4);
8301
8302 size = P2ROUNDUP(size, 1ULL << dev->l2ad_vdev->vdev_ashift);
8303
8304 return (size);
8305
8306 }
8307
8308 static clock_t
l2arc_write_interval(clock_t began,uint64_t wanted,uint64_t wrote)8309 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
8310 {
8311 clock_t interval, next, now;
8312
8313 /*
8314 * If the ARC lists are busy, increase our write rate; if the
8315 * lists are stale, idle back. This is achieved by checking
8316 * how much we previously wrote - if it was more than half of
8317 * what we wanted, schedule the next write much sooner.
8318 */
8319 if (l2arc_feed_again && wrote > (wanted / 2))
8320 interval = (hz * l2arc_feed_min_ms) / 1000;
8321 else
8322 interval = hz * l2arc_feed_secs;
8323
8324 now = ddi_get_lbolt();
8325 next = MAX(now, MIN(now + interval, began + interval));
8326
8327 return (next);
8328 }
8329
8330 static boolean_t
l2arc_dev_invalid(const l2arc_dev_t * dev)8331 l2arc_dev_invalid(const l2arc_dev_t *dev)
8332 {
8333 /*
8334 * We want to skip devices that are being rebuilt, trimmed,
8335 * removed, or belong to a spa that is being exported.
8336 */
8337 return (dev->l2ad_vdev == NULL || vdev_is_dead(dev->l2ad_vdev) ||
8338 dev->l2ad_rebuild || dev->l2ad_trim_all ||
8339 dev->l2ad_spa == NULL || dev->l2ad_spa->spa_is_exporting);
8340 }
8341
8342 /*
8343 * Cycle through L2ARC devices. This is how L2ARC load balances.
8344 * If a device is returned, this also returns holding the spa config lock.
8345 */
8346 static l2arc_dev_t *
l2arc_dev_get_next(void)8347 l2arc_dev_get_next(void)
8348 {
8349 l2arc_dev_t *first, *next = NULL;
8350
8351 /*
8352 * Lock out the removal of spas (spa_namespace_lock), then removal
8353 * of cache devices (l2arc_dev_mtx). Once a device has been selected,
8354 * both locks will be dropped and a spa config lock held instead.
8355 */
8356 mutex_enter(&spa_namespace_lock);
8357 mutex_enter(&l2arc_dev_mtx);
8358
8359 /* if there are no vdevs, there is nothing to do */
8360 if (l2arc_ndev == 0)
8361 goto out;
8362
8363 first = NULL;
8364 next = l2arc_dev_last;
8365 do {
8366 /* loop around the list looking for a non-faulted vdev */
8367 if (next == NULL) {
8368 next = list_head(l2arc_dev_list);
8369 } else {
8370 next = list_next(l2arc_dev_list, next);
8371 if (next == NULL)
8372 next = list_head(l2arc_dev_list);
8373 }
8374
8375 /* if we have come back to the start, bail out */
8376 if (first == NULL)
8377 first = next;
8378 else if (next == first)
8379 break;
8380
8381 ASSERT3P(next, !=, NULL);
8382 } while (l2arc_dev_invalid(next));
8383
8384 /* if we were unable to find any usable vdevs, return NULL */
8385 if (l2arc_dev_invalid(next))
8386 next = NULL;
8387
8388 l2arc_dev_last = next;
8389
8390 out:
8391 mutex_exit(&l2arc_dev_mtx);
8392
8393 /*
8394 * Grab the config lock to prevent the 'next' device from being
8395 * removed while we are writing to it.
8396 */
8397 if (next != NULL)
8398 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
8399 mutex_exit(&spa_namespace_lock);
8400
8401 return (next);
8402 }
8403
8404 /*
8405 * Free buffers that were tagged for destruction.
8406 */
8407 static void
l2arc_do_free_on_write(void)8408 l2arc_do_free_on_write(void)
8409 {
8410 l2arc_data_free_t *df;
8411
8412 mutex_enter(&l2arc_free_on_write_mtx);
8413 while ((df = list_remove_head(l2arc_free_on_write)) != NULL) {
8414 ASSERT3P(df->l2df_abd, !=, NULL);
8415 abd_free(df->l2df_abd);
8416 kmem_free(df, sizeof (l2arc_data_free_t));
8417 }
8418 mutex_exit(&l2arc_free_on_write_mtx);
8419 }
8420
8421 /*
8422 * A write to a cache device has completed. Update all headers to allow
8423 * reads from these buffers to begin.
8424 */
8425 static void
l2arc_write_done(zio_t * zio)8426 l2arc_write_done(zio_t *zio)
8427 {
8428 l2arc_write_callback_t *cb;
8429 l2arc_lb_abd_buf_t *abd_buf;
8430 l2arc_lb_ptr_buf_t *lb_ptr_buf;
8431 l2arc_dev_t *dev;
8432 l2arc_dev_hdr_phys_t *l2dhdr;
8433 list_t *buflist;
8434 arc_buf_hdr_t *head, *hdr, *hdr_prev;
8435 kmutex_t *hash_lock;
8436 int64_t bytes_dropped = 0;
8437
8438 cb = zio->io_private;
8439 ASSERT3P(cb, !=, NULL);
8440 dev = cb->l2wcb_dev;
8441 l2dhdr = dev->l2ad_dev_hdr;
8442 ASSERT3P(dev, !=, NULL);
8443 head = cb->l2wcb_head;
8444 ASSERT3P(head, !=, NULL);
8445 buflist = &dev->l2ad_buflist;
8446 ASSERT3P(buflist, !=, NULL);
8447 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
8448 l2arc_write_callback_t *, cb);
8449
8450 /*
8451 * All writes completed, or an error was hit.
8452 */
8453 top:
8454 mutex_enter(&dev->l2ad_mtx);
8455 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
8456 hdr_prev = list_prev(buflist, hdr);
8457
8458 hash_lock = HDR_LOCK(hdr);
8459
8460 /*
8461 * We cannot use mutex_enter or else we can deadlock
8462 * with l2arc_write_buffers (due to swapping the order
8463 * the hash lock and l2ad_mtx are taken).
8464 */
8465 if (!mutex_tryenter(hash_lock)) {
8466 /*
8467 * Missed the hash lock. We must retry so we
8468 * don't leave the ARC_FLAG_L2_WRITING bit set.
8469 */
8470 ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
8471
8472 /*
8473 * We don't want to rescan the headers we've
8474 * already marked as having been written out, so
8475 * we reinsert the head node so we can pick up
8476 * where we left off.
8477 */
8478 list_remove(buflist, head);
8479 list_insert_after(buflist, hdr, head);
8480
8481 mutex_exit(&dev->l2ad_mtx);
8482
8483 /*
8484 * We wait for the hash lock to become available
8485 * to try and prevent busy waiting, and increase
8486 * the chance we'll be able to acquire the lock
8487 * the next time around.
8488 */
8489 mutex_enter(hash_lock);
8490 mutex_exit(hash_lock);
8491 goto top;
8492 }
8493
8494 /*
8495 * We could not have been moved into the arc_l2c_only
8496 * state while in-flight due to our ARC_FLAG_L2_WRITING
8497 * bit being set. Let's just ensure that's being enforced.
8498 */
8499 ASSERT(HDR_HAS_L1HDR(hdr));
8500
8501 /*
8502 * Skipped - drop L2ARC entry and mark the header as no
8503 * longer L2 eligibile.
8504 */
8505 if (zio->io_error != 0) {
8506 /*
8507 * Error - drop L2ARC entry.
8508 */
8509 list_remove(buflist, hdr);
8510 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
8511
8512 uint64_t psize = HDR_GET_PSIZE(hdr);
8513 l2arc_hdr_arcstats_decrement(hdr);
8514
8515 ASSERT(dev->l2ad_vdev != NULL);
8516
8517 bytes_dropped +=
8518 vdev_psize_to_asize(dev->l2ad_vdev, psize);
8519 (void) zfs_refcount_remove_many(&dev->l2ad_alloc,
8520 arc_hdr_size(hdr), hdr);
8521 }
8522
8523 /*
8524 * Allow ARC to begin reads and ghost list evictions to
8525 * this L2ARC entry.
8526 */
8527 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
8528
8529 mutex_exit(hash_lock);
8530 }
8531
8532 /*
8533 * Free the allocated abd buffers for writing the log blocks.
8534 * If the zio failed reclaim the allocated space and remove the
8535 * pointers to these log blocks from the log block pointer list
8536 * of the L2ARC device.
8537 */
8538 while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
8539 abd_free(abd_buf->abd);
8540 zio_buf_free(abd_buf, sizeof (*abd_buf));
8541 if (zio->io_error != 0) {
8542 lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
8543 /*
8544 * L2BLK_GET_PSIZE returns aligned size for log
8545 * blocks.
8546 */
8547 uint64_t asize =
8548 L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
8549 bytes_dropped += asize;
8550 ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
8551 ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
8552 zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
8553 lb_ptr_buf);
8554 (void) zfs_refcount_remove(&dev->l2ad_lb_count,
8555 lb_ptr_buf);
8556 kmem_free(lb_ptr_buf->lb_ptr,
8557 sizeof (l2arc_log_blkptr_t));
8558 kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
8559 }
8560 }
8561 list_destroy(&cb->l2wcb_abd_list);
8562
8563 if (zio->io_error != 0) {
8564 ARCSTAT_BUMP(arcstat_l2_writes_error);
8565
8566 /*
8567 * Restore the lbps array in the header to its previous state.
8568 * If the list of log block pointers is empty, zero out the
8569 * log block pointers in the device header.
8570 */
8571 lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
8572 for (int i = 0; i < 2; i++) {
8573 if (lb_ptr_buf == NULL) {
8574 /*
8575 * If the list is empty zero out the device
8576 * header. Otherwise zero out the second log
8577 * block pointer in the header.
8578 */
8579 if (i == 0) {
8580 memset(l2dhdr, 0,
8581 dev->l2ad_dev_hdr_asize);
8582 } else {
8583 memset(&l2dhdr->dh_start_lbps[i], 0,
8584 sizeof (l2arc_log_blkptr_t));
8585 }
8586 break;
8587 }
8588 memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
8589 sizeof (l2arc_log_blkptr_t));
8590 lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
8591 lb_ptr_buf);
8592 }
8593 }
8594
8595 ARCSTAT_BUMP(arcstat_l2_writes_done);
8596 list_remove(buflist, head);
8597 ASSERT(!HDR_HAS_L1HDR(head));
8598 kmem_cache_free(hdr_l2only_cache, head);
8599 mutex_exit(&dev->l2ad_mtx);
8600
8601 ASSERT(dev->l2ad_vdev != NULL);
8602 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
8603
8604 l2arc_do_free_on_write();
8605
8606 kmem_free(cb, sizeof (l2arc_write_callback_t));
8607 }
8608
8609 static int
l2arc_untransform(zio_t * zio,l2arc_read_callback_t * cb)8610 l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
8611 {
8612 int ret;
8613 spa_t *spa = zio->io_spa;
8614 arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
8615 blkptr_t *bp = zio->io_bp;
8616 uint8_t salt[ZIO_DATA_SALT_LEN];
8617 uint8_t iv[ZIO_DATA_IV_LEN];
8618 uint8_t mac[ZIO_DATA_MAC_LEN];
8619 boolean_t no_crypt = B_FALSE;
8620
8621 /*
8622 * ZIL data is never be written to the L2ARC, so we don't need
8623 * special handling for its unique MAC storage.
8624 */
8625 ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
8626 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
8627 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8628
8629 /*
8630 * If the data was encrypted, decrypt it now. Note that
8631 * we must check the bp here and not the hdr, since the
8632 * hdr does not have its encryption parameters updated
8633 * until arc_read_done().
8634 */
8635 if (BP_IS_ENCRYPTED(bp)) {
8636 abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8637 ARC_HDR_USE_RESERVE);
8638
8639 zio_crypt_decode_params_bp(bp, salt, iv);
8640 zio_crypt_decode_mac_bp(bp, mac);
8641
8642 ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
8643 BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
8644 salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
8645 hdr->b_l1hdr.b_pabd, &no_crypt);
8646 if (ret != 0) {
8647 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8648 goto error;
8649 }
8650
8651 /*
8652 * If we actually performed decryption, replace b_pabd
8653 * with the decrypted data. Otherwise we can just throw
8654 * our decryption buffer away.
8655 */
8656 if (!no_crypt) {
8657 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8658 arc_hdr_size(hdr), hdr);
8659 hdr->b_l1hdr.b_pabd = eabd;
8660 zio->io_abd = eabd;
8661 } else {
8662 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8663 }
8664 }
8665
8666 /*
8667 * If the L2ARC block was compressed, but ARC compression
8668 * is disabled we decompress the data into a new buffer and
8669 * replace the existing data.
8670 */
8671 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
8672 !HDR_COMPRESSION_ENABLED(hdr)) {
8673 abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8674 ARC_HDR_USE_RESERVE);
8675
8676 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
8677 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
8678 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
8679 if (ret != 0) {
8680 arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
8681 goto error;
8682 }
8683
8684 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8685 arc_hdr_size(hdr), hdr);
8686 hdr->b_l1hdr.b_pabd = cabd;
8687 zio->io_abd = cabd;
8688 zio->io_size = HDR_GET_LSIZE(hdr);
8689 }
8690
8691 return (0);
8692
8693 error:
8694 return (ret);
8695 }
8696
8697
8698 /*
8699 * A read to a cache device completed. Validate buffer contents before
8700 * handing over to the regular ARC routines.
8701 */
8702 static void
l2arc_read_done(zio_t * zio)8703 l2arc_read_done(zio_t *zio)
8704 {
8705 int tfm_error = 0;
8706 l2arc_read_callback_t *cb = zio->io_private;
8707 arc_buf_hdr_t *hdr;
8708 kmutex_t *hash_lock;
8709 boolean_t valid_cksum;
8710 boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
8711 (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));
8712
8713 ASSERT3P(zio->io_vd, !=, NULL);
8714 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
8715
8716 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
8717
8718 ASSERT3P(cb, !=, NULL);
8719 hdr = cb->l2rcb_hdr;
8720 ASSERT3P(hdr, !=, NULL);
8721
8722 hash_lock = HDR_LOCK(hdr);
8723 mutex_enter(hash_lock);
8724 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
8725
8726 /*
8727 * If the data was read into a temporary buffer,
8728 * move it and free the buffer.
8729 */
8730 if (cb->l2rcb_abd != NULL) {
8731 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
8732 if (zio->io_error == 0) {
8733 if (using_rdata) {
8734 abd_copy(hdr->b_crypt_hdr.b_rabd,
8735 cb->l2rcb_abd, arc_hdr_size(hdr));
8736 } else {
8737 abd_copy(hdr->b_l1hdr.b_pabd,
8738 cb->l2rcb_abd, arc_hdr_size(hdr));
8739 }
8740 }
8741
8742 /*
8743 * The following must be done regardless of whether
8744 * there was an error:
8745 * - free the temporary buffer
8746 * - point zio to the real ARC buffer
8747 * - set zio size accordingly
8748 * These are required because zio is either re-used for
8749 * an I/O of the block in the case of the error
8750 * or the zio is passed to arc_read_done() and it
8751 * needs real data.
8752 */
8753 abd_free(cb->l2rcb_abd);
8754 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
8755
8756 if (using_rdata) {
8757 ASSERT(HDR_HAS_RABD(hdr));
8758 zio->io_abd = zio->io_orig_abd =
8759 hdr->b_crypt_hdr.b_rabd;
8760 } else {
8761 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8762 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
8763 }
8764 }
8765
8766 ASSERT3P(zio->io_abd, !=, NULL);
8767
8768 /*
8769 * Check this survived the L2ARC journey.
8770 */
8771 ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
8772 (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
8773 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
8774 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
8775 zio->io_prop.zp_complevel = hdr->b_complevel;
8776
8777 valid_cksum = arc_cksum_is_equal(hdr, zio);
8778
8779 /*
8780 * b_rabd will always match the data as it exists on disk if it is
8781 * being used. Therefore if we are reading into b_rabd we do not
8782 * attempt to untransform the data.
8783 */
8784 if (valid_cksum && !using_rdata)
8785 tfm_error = l2arc_untransform(zio, cb);
8786
8787 if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
8788 !HDR_L2_EVICTED(hdr)) {
8789 mutex_exit(hash_lock);
8790 zio->io_private = hdr;
8791 arc_read_done(zio);
8792 } else {
8793 /*
8794 * Buffer didn't survive caching. Increment stats and
8795 * reissue to the original storage device.
8796 */
8797 if (zio->io_error != 0) {
8798 ARCSTAT_BUMP(arcstat_l2_io_error);
8799 } else {
8800 zio->io_error = SET_ERROR(EIO);
8801 }
8802 if (!valid_cksum || tfm_error != 0)
8803 ARCSTAT_BUMP(arcstat_l2_cksum_bad);
8804
8805 /*
8806 * If there's no waiter, issue an async i/o to the primary
8807 * storage now. If there *is* a waiter, the caller must
8808 * issue the i/o in a context where it's OK to block.
8809 */
8810 if (zio->io_waiter == NULL) {
8811 zio_t *pio = zio_unique_parent(zio);
8812 void *abd = (using_rdata) ?
8813 hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
8814
8815 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
8816
8817 zio = zio_read(pio, zio->io_spa, zio->io_bp,
8818 abd, zio->io_size, arc_read_done,
8819 hdr, zio->io_priority, cb->l2rcb_flags,
8820 &cb->l2rcb_zb);
8821
8822 /*
8823 * Original ZIO will be freed, so we need to update
8824 * ARC header with the new ZIO pointer to be used
8825 * by zio_change_priority() in arc_read().
8826 */
8827 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
8828 acb != NULL; acb = acb->acb_next)
8829 acb->acb_zio_head = zio;
8830
8831 mutex_exit(hash_lock);
8832 zio_nowait(zio);
8833 } else {
8834 mutex_exit(hash_lock);
8835 }
8836 }
8837
8838 kmem_free(cb, sizeof (l2arc_read_callback_t));
8839 }
8840
8841 /*
8842 * This is the list priority from which the L2ARC will search for pages to
8843 * cache. This is used within loops (0..3) to cycle through lists in the
8844 * desired order. This order can have a significant effect on cache
8845 * performance.
8846 *
8847 * Currently the metadata lists are hit first, MFU then MRU, followed by
8848 * the data lists. This function returns a locked list, and also returns
8849 * the lock pointer.
8850 */
8851 static multilist_sublist_t *
l2arc_sublist_lock(int list_num)8852 l2arc_sublist_lock(int list_num)
8853 {
8854 multilist_t *ml = NULL;
8855 unsigned int idx;
8856
8857 ASSERT(list_num >= 0 && list_num < L2ARC_FEED_TYPES);
8858
8859 switch (list_num) {
8860 case 0:
8861 ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
8862 break;
8863 case 1:
8864 ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
8865 break;
8866 case 2:
8867 ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
8868 break;
8869 case 3:
8870 ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
8871 break;
8872 default:
8873 return (NULL);
8874 }
8875
8876 /*
8877 * Return a randomly-selected sublist. This is acceptable
8878 * because the caller feeds only a little bit of data for each
8879 * call (8MB). Subsequent calls will result in different
8880 * sublists being selected.
8881 */
8882 idx = multilist_get_random_index(ml);
8883 return (multilist_sublist_lock_idx(ml, idx));
8884 }
8885
8886 /*
8887 * Calculates the maximum overhead of L2ARC metadata log blocks for a given
8888 * L2ARC write size. l2arc_evict and l2arc_write_size need to include this
8889 * overhead in processing to make sure there is enough headroom available
8890 * when writing buffers.
8891 */
8892 static inline uint64_t
l2arc_log_blk_overhead(uint64_t write_sz,l2arc_dev_t * dev)8893 l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev)
8894 {
8895 if (dev->l2ad_log_entries == 0) {
8896 return (0);
8897 } else {
8898 ASSERT(dev->l2ad_vdev != NULL);
8899
8900 uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT;
8901
8902 uint64_t log_blocks = (log_entries +
8903 dev->l2ad_log_entries - 1) /
8904 dev->l2ad_log_entries;
8905
8906 return (vdev_psize_to_asize(dev->l2ad_vdev,
8907 sizeof (l2arc_log_blk_phys_t)) * log_blocks);
8908 }
8909 }
8910
8911 /*
8912 * Evict buffers from the device write hand to the distance specified in
8913 * bytes. This distance may span populated buffers, it may span nothing.
8914 * This is clearing a region on the L2ARC device ready for writing.
8915 * If the 'all' boolean is set, every buffer is evicted.
8916 */
8917 static void
l2arc_evict(l2arc_dev_t * dev,uint64_t distance,boolean_t all)8918 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
8919 {
8920 list_t *buflist;
8921 arc_buf_hdr_t *hdr, *hdr_prev;
8922 kmutex_t *hash_lock;
8923 uint64_t taddr;
8924 l2arc_lb_ptr_buf_t *lb_ptr_buf, *lb_ptr_buf_prev;
8925 vdev_t *vd = dev->l2ad_vdev;
8926 boolean_t rerun;
8927
8928 ASSERT(vd != NULL || all);
8929 ASSERT(dev->l2ad_spa != NULL || all);
8930
8931 buflist = &dev->l2ad_buflist;
8932
8933 top:
8934 rerun = B_FALSE;
8935 if (dev->l2ad_hand + distance > dev->l2ad_end) {
8936 /*
8937 * When there is no space to accommodate upcoming writes,
8938 * evict to the end. Then bump the write and evict hands
8939 * to the start and iterate. This iteration does not
8940 * happen indefinitely as we make sure in
8941 * l2arc_write_size() that when the write hand is reset,
8942 * the write size does not exceed the end of the device.
8943 */
8944 rerun = B_TRUE;
8945 taddr = dev->l2ad_end;
8946 } else {
8947 taddr = dev->l2ad_hand + distance;
8948 }
8949 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
8950 uint64_t, taddr, boolean_t, all);
8951
8952 if (!all) {
8953 /*
8954 * This check has to be placed after deciding whether to
8955 * iterate (rerun).
8956 */
8957 if (dev->l2ad_first) {
8958 /*
8959 * This is the first sweep through the device. There is
8960 * nothing to evict. We have already trimmmed the
8961 * whole device.
8962 */
8963 goto out;
8964 } else {
8965 /*
8966 * Trim the space to be evicted.
8967 */
8968 if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
8969 l2arc_trim_ahead > 0) {
8970 /*
8971 * We have to drop the spa_config lock because
8972 * vdev_trim_range() will acquire it.
8973 * l2ad_evict already accounts for the label
8974 * size. To prevent vdev_trim_ranges() from
8975 * adding it again, we subtract it from
8976 * l2ad_evict.
8977 */
8978 spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
8979 vdev_trim_simple(vd,
8980 dev->l2ad_evict - VDEV_LABEL_START_SIZE,
8981 taddr - dev->l2ad_evict);
8982 spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
8983 RW_READER);
8984 }
8985
8986 /*
8987 * When rebuilding L2ARC we retrieve the evict hand
8988 * from the header of the device. Of note, l2arc_evict()
8989 * does not actually delete buffers from the cache
8990 * device, but trimming may do so depending on the
8991 * hardware implementation. Thus keeping track of the
8992 * evict hand is useful.
8993 */
8994 dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
8995 }
8996 }
8997
8998 retry:
8999 mutex_enter(&dev->l2ad_mtx);
9000 /*
9001 * We have to account for evicted log blocks. Run vdev_space_update()
9002 * on log blocks whose offset (in bytes) is before the evicted offset
9003 * (in bytes) by searching in the list of pointers to log blocks
9004 * present in the L2ARC device.
9005 */
9006 for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
9007 lb_ptr_buf = lb_ptr_buf_prev) {
9008
9009 lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);
9010
9011 /* L2BLK_GET_PSIZE returns aligned size for log blocks */
9012 uint64_t asize = L2BLK_GET_PSIZE(
9013 (lb_ptr_buf->lb_ptr)->lbp_prop);
9014
9015 /*
9016 * We don't worry about log blocks left behind (ie
9017 * lbp_payload_start < l2ad_hand) because l2arc_write_buffers()
9018 * will never write more than l2arc_evict() evicts.
9019 */
9020 if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
9021 break;
9022 } else {
9023 if (vd != NULL)
9024 vdev_space_update(vd, -asize, 0, 0);
9025 ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
9026 ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
9027 zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
9028 lb_ptr_buf);
9029 (void) zfs_refcount_remove(&dev->l2ad_lb_count,
9030 lb_ptr_buf);
9031 list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
9032 kmem_free(lb_ptr_buf->lb_ptr,
9033 sizeof (l2arc_log_blkptr_t));
9034 kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
9035 }
9036 }
9037
9038 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
9039 hdr_prev = list_prev(buflist, hdr);
9040
9041 ASSERT(!HDR_EMPTY(hdr));
9042 hash_lock = HDR_LOCK(hdr);
9043
9044 /*
9045 * We cannot use mutex_enter or else we can deadlock
9046 * with l2arc_write_buffers (due to swapping the order
9047 * the hash lock and l2ad_mtx are taken).
9048 */
9049 if (!mutex_tryenter(hash_lock)) {
9050 /*
9051 * Missed the hash lock. Retry.
9052 */
9053 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
9054 mutex_exit(&dev->l2ad_mtx);
9055 mutex_enter(hash_lock);
9056 mutex_exit(hash_lock);
9057 goto retry;
9058 }
9059
9060 /*
9061 * A header can't be on this list if it doesn't have L2 header.
9062 */
9063 ASSERT(HDR_HAS_L2HDR(hdr));
9064
9065 /* Ensure this header has finished being written. */
9066 ASSERT(!HDR_L2_WRITING(hdr));
9067 ASSERT(!HDR_L2_WRITE_HEAD(hdr));
9068
9069 if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
9070 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
9071 /*
9072 * We've evicted to the target address,
9073 * or the end of the device.
9074 */
9075 mutex_exit(hash_lock);
9076 break;
9077 }
9078
9079 if (!HDR_HAS_L1HDR(hdr)) {
9080 ASSERT(!HDR_L2_READING(hdr));
9081 /*
9082 * This doesn't exist in the ARC. Destroy.
9083 * arc_hdr_destroy() will call list_remove()
9084 * and decrement arcstat_l2_lsize.
9085 */
9086 arc_change_state(arc_anon, hdr);
9087 arc_hdr_destroy(hdr);
9088 } else {
9089 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
9090 ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
9091 /*
9092 * Invalidate issued or about to be issued
9093 * reads, since we may be about to write
9094 * over this location.
9095 */
9096 if (HDR_L2_READING(hdr)) {
9097 ARCSTAT_BUMP(arcstat_l2_evict_reading);
9098 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
9099 }
9100
9101 arc_hdr_l2hdr_destroy(hdr);
9102 }
9103 mutex_exit(hash_lock);
9104 }
9105 mutex_exit(&dev->l2ad_mtx);
9106
9107 out:
9108 /*
9109 * We need to check if we evict all buffers, otherwise we may iterate
9110 * unnecessarily.
9111 */
9112 if (!all && rerun) {
9113 /*
9114 * Bump device hand to the device start if it is approaching the
9115 * end. l2arc_evict() has already evicted ahead for this case.
9116 */
9117 dev->l2ad_hand = dev->l2ad_start;
9118 dev->l2ad_evict = dev->l2ad_start;
9119 dev->l2ad_first = B_FALSE;
9120 goto top;
9121 }
9122
9123 if (!all) {
9124 /*
9125 * In case of cache device removal (all) the following
9126 * assertions may be violated without functional consequences
9127 * as the device is about to be removed.
9128 */
9129 ASSERT3U(dev->l2ad_hand + distance, <=, dev->l2ad_end);
9130 if (!dev->l2ad_first)
9131 ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9132 }
9133 }
9134
9135 /*
9136 * Handle any abd transforms that might be required for writing to the L2ARC.
9137 * If successful, this function will always return an abd with the data
9138 * transformed as it is on disk in a new abd of asize bytes.
9139 */
9140 static int
l2arc_apply_transforms(spa_t * spa,arc_buf_hdr_t * hdr,uint64_t asize,abd_t ** abd_out)9141 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
9142 abd_t **abd_out)
9143 {
9144 int ret;
9145 abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
9146 enum zio_compress compress = HDR_GET_COMPRESS(hdr);
9147 uint64_t psize = HDR_GET_PSIZE(hdr);
9148 uint64_t size = arc_hdr_size(hdr);
9149 boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
9150 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
9151 dsl_crypto_key_t *dck = NULL;
9152 uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
9153 boolean_t no_crypt = B_FALSE;
9154
9155 ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
9156 !HDR_COMPRESSION_ENABLED(hdr)) ||
9157 HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);
9158 ASSERT3U(psize, <=, asize);
9159
9160 /*
9161 * If this data simply needs its own buffer, we simply allocate it
9162 * and copy the data. This may be done to eliminate a dependency on a
9163 * shared buffer or to reallocate the buffer to match asize.
9164 */
9165 if (HDR_HAS_RABD(hdr)) {
9166 ASSERT3U(asize, >, psize);
9167 to_write = abd_alloc_for_io(asize, ismd);
9168 abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
9169 abd_zero_off(to_write, psize, asize - psize);
9170 goto out;
9171 }
9172
9173 if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
9174 !HDR_ENCRYPTED(hdr)) {
9175 ASSERT3U(size, ==, psize);
9176 to_write = abd_alloc_for_io(asize, ismd);
9177 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
9178 if (asize > size)
9179 abd_zero_off(to_write, size, asize - size);
9180 goto out;
9181 }
9182
9183 if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
9184 cabd = abd_alloc_for_io(MAX(size, asize), ismd);
9185 uint64_t csize = zio_compress_data(compress, to_write, &cabd,
9186 size, MIN(size, psize), hdr->b_complevel);
9187 if (csize >= size || csize > psize) {
9188 /*
9189 * We can't re-compress the block into the original
9190 * psize. Even if it fits into asize, it does not
9191 * matter, since checksum will never match on read.
9192 */
9193 abd_free(cabd);
9194 return (SET_ERROR(EIO));
9195 }
9196 if (asize > csize)
9197 abd_zero_off(cabd, csize, asize - csize);
9198 to_write = cabd;
9199 }
9200
9201 if (HDR_ENCRYPTED(hdr)) {
9202 eabd = abd_alloc_for_io(asize, ismd);
9203
9204 /*
9205 * If the dataset was disowned before the buffer
9206 * made it to this point, the key to re-encrypt
9207 * it won't be available. In this case we simply
9208 * won't write the buffer to the L2ARC.
9209 */
9210 ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
9211 FTAG, &dck);
9212 if (ret != 0)
9213 goto error;
9214
9215 ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
9216 hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
9217 hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
9218 &no_crypt);
9219 if (ret != 0)
9220 goto error;
9221
9222 if (no_crypt)
9223 abd_copy(eabd, to_write, psize);
9224
9225 if (psize != asize)
9226 abd_zero_off(eabd, psize, asize - psize);
9227
9228 /* assert that the MAC we got here matches the one we saved */
9229 ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
9230 spa_keystore_dsl_key_rele(spa, dck, FTAG);
9231
9232 if (to_write == cabd)
9233 abd_free(cabd);
9234
9235 to_write = eabd;
9236 }
9237
9238 out:
9239 ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
9240 *abd_out = to_write;
9241 return (0);
9242
9243 error:
9244 if (dck != NULL)
9245 spa_keystore_dsl_key_rele(spa, dck, FTAG);
9246 if (cabd != NULL)
9247 abd_free(cabd);
9248 if (eabd != NULL)
9249 abd_free(eabd);
9250
9251 *abd_out = NULL;
9252 return (ret);
9253 }
9254
9255 static void
l2arc_blk_fetch_done(zio_t * zio)9256 l2arc_blk_fetch_done(zio_t *zio)
9257 {
9258 l2arc_read_callback_t *cb;
9259
9260 cb = zio->io_private;
9261 if (cb->l2rcb_abd != NULL)
9262 abd_free(cb->l2rcb_abd);
9263 kmem_free(cb, sizeof (l2arc_read_callback_t));
9264 }
9265
9266 /*
9267 * Find and write ARC buffers to the L2ARC device.
9268 *
9269 * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
9270 * for reading until they have completed writing.
9271 * The headroom_boost is an in-out parameter used to maintain headroom boost
9272 * state between calls to this function.
9273 *
9274 * Returns the number of bytes actually written (which may be smaller than
9275 * the delta by which the device hand has changed due to alignment and the
9276 * writing of log blocks).
9277 */
9278 static uint64_t
l2arc_write_buffers(spa_t * spa,l2arc_dev_t * dev,uint64_t target_sz)9279 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
9280 {
9281 arc_buf_hdr_t *hdr, *head, *marker;
9282 uint64_t write_asize, write_psize, headroom;
9283 boolean_t full, from_head = !arc_warm;
9284 l2arc_write_callback_t *cb = NULL;
9285 zio_t *pio, *wzio;
9286 uint64_t guid = spa_load_guid(spa);
9287 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9288
9289 ASSERT3P(dev->l2ad_vdev, !=, NULL);
9290
9291 pio = NULL;
9292 write_asize = write_psize = 0;
9293 full = B_FALSE;
9294 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
9295 arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
9296 marker = arc_state_alloc_marker();
9297
9298 /*
9299 * Copy buffers for L2ARC writing.
9300 */
9301 for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) {
9302 /*
9303 * pass == 0: MFU meta
9304 * pass == 1: MRU meta
9305 * pass == 2: MFU data
9306 * pass == 3: MRU data
9307 */
9308 if (l2arc_mfuonly == 1) {
9309 if (pass == 1 || pass == 3)
9310 continue;
9311 } else if (l2arc_mfuonly > 1) {
9312 if (pass == 3)
9313 continue;
9314 }
9315
9316 uint64_t passed_sz = 0;
9317 headroom = target_sz * l2arc_headroom;
9318 if (zfs_compressed_arc_enabled)
9319 headroom = (headroom * l2arc_headroom_boost) / 100;
9320
9321 /*
9322 * Until the ARC is warm and starts to evict, read from the
9323 * head of the ARC lists rather than the tail.
9324 */
9325 multilist_sublist_t *mls = l2arc_sublist_lock(pass);
9326 ASSERT3P(mls, !=, NULL);
9327 if (from_head)
9328 hdr = multilist_sublist_head(mls);
9329 else
9330 hdr = multilist_sublist_tail(mls);
9331
9332 while (hdr != NULL) {
9333 kmutex_t *hash_lock;
9334 abd_t *to_write = NULL;
9335
9336 hash_lock = HDR_LOCK(hdr);
9337 if (!mutex_tryenter(hash_lock)) {
9338 skip:
9339 /* Skip this buffer rather than waiting. */
9340 if (from_head)
9341 hdr = multilist_sublist_next(mls, hdr);
9342 else
9343 hdr = multilist_sublist_prev(mls, hdr);
9344 continue;
9345 }
9346
9347 passed_sz += HDR_GET_LSIZE(hdr);
9348 if (l2arc_headroom != 0 && passed_sz > headroom) {
9349 /*
9350 * Searched too far.
9351 */
9352 mutex_exit(hash_lock);
9353 break;
9354 }
9355
9356 if (!l2arc_write_eligible(guid, hdr)) {
9357 mutex_exit(hash_lock);
9358 goto skip;
9359 }
9360
9361 ASSERT(HDR_HAS_L1HDR(hdr));
9362 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
9363 ASSERT3U(arc_hdr_size(hdr), >, 0);
9364 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
9365 HDR_HAS_RABD(hdr));
9366 uint64_t psize = HDR_GET_PSIZE(hdr);
9367 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
9368 psize);
9369
9370 /*
9371 * If the allocated size of this buffer plus the max
9372 * size for the pending log block exceeds the evicted
9373 * target size, terminate writing buffers for this run.
9374 */
9375 if (write_asize + asize +
9376 sizeof (l2arc_log_blk_phys_t) > target_sz) {
9377 full = B_TRUE;
9378 mutex_exit(hash_lock);
9379 break;
9380 }
9381
9382 /*
9383 * We should not sleep with sublist lock held or it
9384 * may block ARC eviction. Insert a marker to save
9385 * the position and drop the lock.
9386 */
9387 if (from_head) {
9388 multilist_sublist_insert_after(mls, hdr,
9389 marker);
9390 } else {
9391 multilist_sublist_insert_before(mls, hdr,
9392 marker);
9393 }
9394 multilist_sublist_unlock(mls);
9395
9396 /*
9397 * If this header has b_rabd, we can use this since it
9398 * must always match the data exactly as it exists on
9399 * disk. Otherwise, the L2ARC can normally use the
9400 * hdr's data, but if we're sharing data between the
9401 * hdr and one of its bufs, L2ARC needs its own copy of
9402 * the data so that the ZIO below can't race with the
9403 * buf consumer. To ensure that this copy will be
9404 * available for the lifetime of the ZIO and be cleaned
9405 * up afterwards, we add it to the l2arc_free_on_write
9406 * queue. If we need to apply any transforms to the
9407 * data (compression, encryption) we will also need the
9408 * extra buffer.
9409 */
9410 if (HDR_HAS_RABD(hdr) && psize == asize) {
9411 to_write = hdr->b_crypt_hdr.b_rabd;
9412 } else if ((HDR_COMPRESSION_ENABLED(hdr) ||
9413 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
9414 !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
9415 psize == asize) {
9416 to_write = hdr->b_l1hdr.b_pabd;
9417 } else {
9418 int ret;
9419 arc_buf_contents_t type = arc_buf_type(hdr);
9420
9421 ret = l2arc_apply_transforms(spa, hdr, asize,
9422 &to_write);
9423 if (ret != 0) {
9424 arc_hdr_clear_flags(hdr,
9425 ARC_FLAG_L2CACHE);
9426 mutex_exit(hash_lock);
9427 goto next;
9428 }
9429
9430 l2arc_free_abd_on_write(to_write, asize, type);
9431 }
9432
9433 hdr->b_l2hdr.b_dev = dev;
9434 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
9435 hdr->b_l2hdr.b_hits = 0;
9436 hdr->b_l2hdr.b_arcs_state =
9437 hdr->b_l1hdr.b_state->arcs_state;
9438 /* l2arc_hdr_arcstats_update() expects a valid asize */
9439 HDR_SET_L2SIZE(hdr, asize);
9440 arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR |
9441 ARC_FLAG_L2_WRITING);
9442
9443 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
9444 arc_hdr_size(hdr), hdr);
9445 l2arc_hdr_arcstats_increment(hdr);
9446 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
9447
9448 mutex_enter(&dev->l2ad_mtx);
9449 if (pio == NULL) {
9450 /*
9451 * Insert a dummy header on the buflist so
9452 * l2arc_write_done() can find where the
9453 * write buffers begin without searching.
9454 */
9455 list_insert_head(&dev->l2ad_buflist, head);
9456 }
9457 list_insert_head(&dev->l2ad_buflist, hdr);
9458 mutex_exit(&dev->l2ad_mtx);
9459
9460 boolean_t commit = l2arc_log_blk_insert(dev, hdr);
9461 mutex_exit(hash_lock);
9462
9463 if (pio == NULL) {
9464 cb = kmem_alloc(
9465 sizeof (l2arc_write_callback_t), KM_SLEEP);
9466 cb->l2wcb_dev = dev;
9467 cb->l2wcb_head = head;
9468 list_create(&cb->l2wcb_abd_list,
9469 sizeof (l2arc_lb_abd_buf_t),
9470 offsetof(l2arc_lb_abd_buf_t, node));
9471 pio = zio_root(spa, l2arc_write_done, cb,
9472 ZIO_FLAG_CANFAIL);
9473 }
9474
9475 wzio = zio_write_phys(pio, dev->l2ad_vdev,
9476 dev->l2ad_hand, asize, to_write,
9477 ZIO_CHECKSUM_OFF, NULL, hdr,
9478 ZIO_PRIORITY_ASYNC_WRITE,
9479 ZIO_FLAG_CANFAIL, B_FALSE);
9480
9481 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
9482 zio_t *, wzio);
9483 zio_nowait(wzio);
9484
9485 write_psize += psize;
9486 write_asize += asize;
9487 dev->l2ad_hand += asize;
9488
9489 if (commit) {
9490 /* l2ad_hand will be adjusted inside. */
9491 write_asize +=
9492 l2arc_log_blk_commit(dev, pio, cb);
9493 }
9494
9495 next:
9496 multilist_sublist_lock(mls);
9497 if (from_head)
9498 hdr = multilist_sublist_next(mls, marker);
9499 else
9500 hdr = multilist_sublist_prev(mls, marker);
9501 multilist_sublist_remove(mls, marker);
9502 }
9503
9504 multilist_sublist_unlock(mls);
9505
9506 if (full == B_TRUE)
9507 break;
9508 }
9509
9510 arc_state_free_marker(marker);
9511
9512 /* No buffers selected for writing? */
9513 if (pio == NULL) {
9514 ASSERT0(write_psize);
9515 ASSERT(!HDR_HAS_L1HDR(head));
9516 kmem_cache_free(hdr_l2only_cache, head);
9517
9518 /*
9519 * Although we did not write any buffers l2ad_evict may
9520 * have advanced.
9521 */
9522 if (dev->l2ad_evict != l2dhdr->dh_evict)
9523 l2arc_dev_hdr_update(dev);
9524
9525 return (0);
9526 }
9527
9528 if (!dev->l2ad_first)
9529 ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9530
9531 ASSERT3U(write_asize, <=, target_sz);
9532 ARCSTAT_BUMP(arcstat_l2_writes_sent);
9533 ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);
9534
9535 dev->l2ad_writing = B_TRUE;
9536 (void) zio_wait(pio);
9537 dev->l2ad_writing = B_FALSE;
9538
9539 /*
9540 * Update the device header after the zio completes as
9541 * l2arc_write_done() may have updated the memory holding the log block
9542 * pointers in the device header.
9543 */
9544 l2arc_dev_hdr_update(dev);
9545
9546 return (write_asize);
9547 }
9548
9549 static boolean_t
l2arc_hdr_limit_reached(void)9550 l2arc_hdr_limit_reached(void)
9551 {
9552 int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);
9553
9554 return (arc_reclaim_needed() ||
9555 (s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
9556 }
9557
9558 /*
9559 * This thread feeds the L2ARC at regular intervals. This is the beating
9560 * heart of the L2ARC.
9561 */
9562 static __attribute__((noreturn)) void
l2arc_feed_thread(void * unused)9563 l2arc_feed_thread(void *unused)
9564 {
9565 (void) unused;
9566 callb_cpr_t cpr;
9567 l2arc_dev_t *dev;
9568 spa_t *spa;
9569 uint64_t size, wrote;
9570 clock_t begin, next = ddi_get_lbolt();
9571 fstrans_cookie_t cookie;
9572
9573 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
9574
9575 mutex_enter(&l2arc_feed_thr_lock);
9576
9577 cookie = spl_fstrans_mark();
9578 while (l2arc_thread_exit == 0) {
9579 CALLB_CPR_SAFE_BEGIN(&cpr);
9580 (void) cv_timedwait_idle(&l2arc_feed_thr_cv,
9581 &l2arc_feed_thr_lock, next);
9582 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
9583 next = ddi_get_lbolt() + hz;
9584
9585 /*
9586 * Quick check for L2ARC devices.
9587 */
9588 mutex_enter(&l2arc_dev_mtx);
9589 if (l2arc_ndev == 0) {
9590 mutex_exit(&l2arc_dev_mtx);
9591 continue;
9592 }
9593 mutex_exit(&l2arc_dev_mtx);
9594 begin = ddi_get_lbolt();
9595
9596 /*
9597 * This selects the next l2arc device to write to, and in
9598 * doing so the next spa to feed from: dev->l2ad_spa. This
9599 * will return NULL if there are now no l2arc devices or if
9600 * they are all faulted.
9601 *
9602 * If a device is returned, its spa's config lock is also
9603 * held to prevent device removal. l2arc_dev_get_next()
9604 * will grab and release l2arc_dev_mtx.
9605 */
9606 if ((dev = l2arc_dev_get_next()) == NULL)
9607 continue;
9608
9609 spa = dev->l2ad_spa;
9610 ASSERT3P(spa, !=, NULL);
9611
9612 /*
9613 * If the pool is read-only then force the feed thread to
9614 * sleep a little longer.
9615 */
9616 if (!spa_writeable(spa)) {
9617 next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
9618 spa_config_exit(spa, SCL_L2ARC, dev);
9619 continue;
9620 }
9621
9622 /*
9623 * Avoid contributing to memory pressure.
9624 */
9625 if (l2arc_hdr_limit_reached()) {
9626 ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
9627 spa_config_exit(spa, SCL_L2ARC, dev);
9628 continue;
9629 }
9630
9631 ARCSTAT_BUMP(arcstat_l2_feeds);
9632
9633 size = l2arc_write_size(dev);
9634
9635 /*
9636 * Evict L2ARC buffers that will be overwritten.
9637 */
9638 l2arc_evict(dev, size, B_FALSE);
9639
9640 /*
9641 * Write ARC buffers.
9642 */
9643 wrote = l2arc_write_buffers(spa, dev, size);
9644
9645 /*
9646 * Calculate interval between writes.
9647 */
9648 next = l2arc_write_interval(begin, size, wrote);
9649 spa_config_exit(spa, SCL_L2ARC, dev);
9650 }
9651 spl_fstrans_unmark(cookie);
9652
9653 l2arc_thread_exit = 0;
9654 cv_broadcast(&l2arc_feed_thr_cv);
9655 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */
9656 thread_exit();
9657 }
9658
9659 boolean_t
l2arc_vdev_present(vdev_t * vd)9660 l2arc_vdev_present(vdev_t *vd)
9661 {
9662 return (l2arc_vdev_get(vd) != NULL);
9663 }
9664
9665 /*
9666 * Returns the l2arc_dev_t associated with a particular vdev_t or NULL if
9667 * the vdev_t isn't an L2ARC device.
9668 */
9669 l2arc_dev_t *
l2arc_vdev_get(vdev_t * vd)9670 l2arc_vdev_get(vdev_t *vd)
9671 {
9672 l2arc_dev_t *dev;
9673
9674 mutex_enter(&l2arc_dev_mtx);
9675 for (dev = list_head(l2arc_dev_list); dev != NULL;
9676 dev = list_next(l2arc_dev_list, dev)) {
9677 if (dev->l2ad_vdev == vd)
9678 break;
9679 }
9680 mutex_exit(&l2arc_dev_mtx);
9681
9682 return (dev);
9683 }
9684
9685 static void
l2arc_rebuild_dev(l2arc_dev_t * dev,boolean_t reopen)9686 l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
9687 {
9688 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9689 uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
9690 spa_t *spa = dev->l2ad_spa;
9691
9692 /*
9693 * After a l2arc_remove_vdev(), the spa_t will no longer be valid
9694 */
9695 if (spa == NULL)
9696 return;
9697
9698 /*
9699 * The L2ARC has to hold at least the payload of one log block for
9700 * them to be restored (persistent L2ARC). The payload of a log block
9701 * depends on the amount of its log entries. We always write log blocks
9702 * with 1022 entries. How many of them are committed or restored depends
9703 * on the size of the L2ARC device. Thus the maximum payload of
9704 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
9705 * is less than that, we reduce the amount of committed and restored
9706 * log entries per block so as to enable persistence.
9707 */
9708 if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
9709 dev->l2ad_log_entries = 0;
9710 } else {
9711 dev->l2ad_log_entries = MIN((dev->l2ad_end -
9712 dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
9713 L2ARC_LOG_BLK_MAX_ENTRIES);
9714 }
9715
9716 /*
9717 * Read the device header, if an error is returned do not rebuild L2ARC.
9718 */
9719 if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
9720 /*
9721 * If we are onlining a cache device (vdev_reopen) that was
9722 * still present (l2arc_vdev_present()) and rebuild is enabled,
9723 * we should evict all ARC buffers and pointers to log blocks
9724 * and reclaim their space before restoring its contents to
9725 * L2ARC.
9726 */
9727 if (reopen) {
9728 if (!l2arc_rebuild_enabled) {
9729 return;
9730 } else {
9731 l2arc_evict(dev, 0, B_TRUE);
9732 /* start a new log block */
9733 dev->l2ad_log_ent_idx = 0;
9734 dev->l2ad_log_blk_payload_asize = 0;
9735 dev->l2ad_log_blk_payload_start = 0;
9736 }
9737 }
9738 /*
9739 * Just mark the device as pending for a rebuild. We won't
9740 * be starting a rebuild in line here as it would block pool
9741 * import. Instead spa_load_impl will hand that off to an
9742 * async task which will call l2arc_spa_rebuild_start.
9743 */
9744 dev->l2ad_rebuild = B_TRUE;
9745 } else if (spa_writeable(spa)) {
9746 /*
9747 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
9748 * otherwise create a new header. We zero out the memory holding
9749 * the header to reset dh_start_lbps. If we TRIM the whole
9750 * device the new header will be written by
9751 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
9752 * trim_state in the header too. When reading the header, if
9753 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
9754 * we opt to TRIM the whole device again.
9755 */
9756 if (l2arc_trim_ahead > 0) {
9757 dev->l2ad_trim_all = B_TRUE;
9758 } else {
9759 memset(l2dhdr, 0, l2dhdr_asize);
9760 l2arc_dev_hdr_update(dev);
9761 }
9762 }
9763 }
9764
9765 /*
9766 * Add a vdev for use by the L2ARC. By this point the spa has already
9767 * validated the vdev and opened it.
9768 */
9769 void
l2arc_add_vdev(spa_t * spa,vdev_t * vd)9770 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
9771 {
9772 l2arc_dev_t *adddev;
9773 uint64_t l2dhdr_asize;
9774
9775 ASSERT(!l2arc_vdev_present(vd));
9776
9777 /*
9778 * Create a new l2arc device entry.
9779 */
9780 adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
9781 adddev->l2ad_spa = spa;
9782 adddev->l2ad_vdev = vd;
9783 /* leave extra size for an l2arc device header */
9784 l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
9785 MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
9786 adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
9787 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
9788 ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
9789 adddev->l2ad_hand = adddev->l2ad_start;
9790 adddev->l2ad_evict = adddev->l2ad_start;
9791 adddev->l2ad_first = B_TRUE;
9792 adddev->l2ad_writing = B_FALSE;
9793 adddev->l2ad_trim_all = B_FALSE;
9794 list_link_init(&adddev->l2ad_node);
9795 adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);
9796
9797 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
9798 /*
9799 * This is a list of all ARC buffers that are still valid on the
9800 * device.
9801 */
9802 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
9803 offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
9804
9805 /*
9806 * This is a list of pointers to log blocks that are still present
9807 * on the device.
9808 */
9809 list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
9810 offsetof(l2arc_lb_ptr_buf_t, node));
9811
9812 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
9813 zfs_refcount_create(&adddev->l2ad_alloc);
9814 zfs_refcount_create(&adddev->l2ad_lb_asize);
9815 zfs_refcount_create(&adddev->l2ad_lb_count);
9816
9817 /*
9818 * Decide if dev is eligible for L2ARC rebuild or whole device
9819 * trimming. This has to happen before the device is added in the
9820 * cache device list and l2arc_dev_mtx is released. Otherwise
9821 * l2arc_feed_thread() might already start writing on the
9822 * device.
9823 */
9824 l2arc_rebuild_dev(adddev, B_FALSE);
9825
9826 /*
9827 * Add device to global list
9828 */
9829 mutex_enter(&l2arc_dev_mtx);
9830 list_insert_head(l2arc_dev_list, adddev);
9831 atomic_inc_64(&l2arc_ndev);
9832 mutex_exit(&l2arc_dev_mtx);
9833 }
9834
9835 /*
9836 * Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
9837 * in case of onlining a cache device.
9838 */
9839 void
l2arc_rebuild_vdev(vdev_t * vd,boolean_t reopen)9840 l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
9841 {
9842 l2arc_dev_t *dev = NULL;
9843
9844 dev = l2arc_vdev_get(vd);
9845 ASSERT3P(dev, !=, NULL);
9846
9847 /*
9848 * In contrast to l2arc_add_vdev() we do not have to worry about
9849 * l2arc_feed_thread() invalidating previous content when onlining a
9850 * cache device. The device parameters (l2ad*) are not cleared when
9851 * offlining the device and writing new buffers will not invalidate
9852 * all previous content. In worst case only buffers that have not had
9853 * their log block written to the device will be lost.
9854 * When onlining the cache device (ie offline->online without exporting
9855 * the pool in between) this happens:
9856 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
9857 * | |
9858 * vdev_is_dead() = B_FALSE l2ad_rebuild = B_TRUE
9859 * During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
9860 * is set to B_TRUE we might write additional buffers to the device.
9861 */
9862 l2arc_rebuild_dev(dev, reopen);
9863 }
9864
9865 typedef struct {
9866 l2arc_dev_t *rva_l2arc_dev;
9867 uint64_t rva_spa_gid;
9868 uint64_t rva_vdev_gid;
9869 boolean_t rva_async;
9870
9871 } remove_vdev_args_t;
9872
9873 static void
l2arc_device_teardown(void * arg)9874 l2arc_device_teardown(void *arg)
9875 {
9876 remove_vdev_args_t *rva = arg;
9877 l2arc_dev_t *remdev = rva->rva_l2arc_dev;
9878 hrtime_t start_time = gethrtime();
9879
9880 /*
9881 * Clear all buflists and ARC references. L2ARC device flush.
9882 */
9883 l2arc_evict(remdev, 0, B_TRUE);
9884 list_destroy(&remdev->l2ad_buflist);
9885 ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
9886 list_destroy(&remdev->l2ad_lbptr_list);
9887 mutex_destroy(&remdev->l2ad_mtx);
9888 zfs_refcount_destroy(&remdev->l2ad_alloc);
9889 zfs_refcount_destroy(&remdev->l2ad_lb_asize);
9890 zfs_refcount_destroy(&remdev->l2ad_lb_count);
9891 kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
9892 vmem_free(remdev, sizeof (l2arc_dev_t));
9893
9894 uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
9895 if (elaspsed > 0) {
9896 zfs_dbgmsg("spa %llu, vdev %llu removed in %llu ms",
9897 (u_longlong_t)rva->rva_spa_gid,
9898 (u_longlong_t)rva->rva_vdev_gid,
9899 (u_longlong_t)elaspsed);
9900 }
9901
9902 if (rva->rva_async)
9903 arc_async_flush_remove(rva->rva_spa_gid, 2);
9904 kmem_free(rva, sizeof (remove_vdev_args_t));
9905 }
9906
9907 /*
9908 * Remove a vdev from the L2ARC.
9909 */
9910 void
l2arc_remove_vdev(vdev_t * vd)9911 l2arc_remove_vdev(vdev_t *vd)
9912 {
9913 spa_t *spa = vd->vdev_spa;
9914 boolean_t asynchronous = spa->spa_state == POOL_STATE_EXPORTED ||
9915 spa->spa_state == POOL_STATE_DESTROYED;
9916
9917 /*
9918 * Find the device by vdev
9919 */
9920 l2arc_dev_t *remdev = l2arc_vdev_get(vd);
9921 ASSERT3P(remdev, !=, NULL);
9922
9923 /*
9924 * Save info for final teardown
9925 */
9926 remove_vdev_args_t *rva = kmem_alloc(sizeof (remove_vdev_args_t),
9927 KM_SLEEP);
9928 rva->rva_l2arc_dev = remdev;
9929 rva->rva_spa_gid = spa_load_guid(spa);
9930 rva->rva_vdev_gid = remdev->l2ad_vdev->vdev_guid;
9931
9932 /*
9933 * Cancel any ongoing or scheduled rebuild.
9934 */
9935 mutex_enter(&l2arc_rebuild_thr_lock);
9936 remdev->l2ad_rebuild_cancel = B_TRUE;
9937 if (remdev->l2ad_rebuild_began == B_TRUE) {
9938 while (remdev->l2ad_rebuild == B_TRUE)
9939 cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
9940 }
9941 mutex_exit(&l2arc_rebuild_thr_lock);
9942 rva->rva_async = asynchronous;
9943
9944 /*
9945 * Remove device from global list
9946 */
9947 ASSERT(spa_config_held(spa, SCL_L2ARC, RW_WRITER) & SCL_L2ARC);
9948 mutex_enter(&l2arc_dev_mtx);
9949 list_remove(l2arc_dev_list, remdev);
9950 l2arc_dev_last = NULL; /* may have been invalidated */
9951 atomic_dec_64(&l2arc_ndev);
9952
9953 /* During a pool export spa & vdev will no longer be valid */
9954 if (asynchronous) {
9955 remdev->l2ad_spa = NULL;
9956 remdev->l2ad_vdev = NULL;
9957 }
9958 mutex_exit(&l2arc_dev_mtx);
9959
9960 if (!asynchronous) {
9961 l2arc_device_teardown(rva);
9962 return;
9963 }
9964
9965 arc_async_flush_t *af = arc_async_flush_add(rva->rva_spa_gid, 2);
9966
9967 taskq_dispatch_ent(arc_flush_taskq, l2arc_device_teardown, rva,
9968 TQ_SLEEP, &af->af_tqent);
9969 }
9970
9971 void
l2arc_init(void)9972 l2arc_init(void)
9973 {
9974 l2arc_thread_exit = 0;
9975 l2arc_ndev = 0;
9976
9977 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
9978 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
9979 mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL);
9980 cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL);
9981 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
9982 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
9983
9984 l2arc_dev_list = &L2ARC_dev_list;
9985 l2arc_free_on_write = &L2ARC_free_on_write;
9986 list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
9987 offsetof(l2arc_dev_t, l2ad_node));
9988 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
9989 offsetof(l2arc_data_free_t, l2df_list_node));
9990 }
9991
9992 void
l2arc_fini(void)9993 l2arc_fini(void)
9994 {
9995 mutex_destroy(&l2arc_feed_thr_lock);
9996 cv_destroy(&l2arc_feed_thr_cv);
9997 mutex_destroy(&l2arc_rebuild_thr_lock);
9998 cv_destroy(&l2arc_rebuild_thr_cv);
9999 mutex_destroy(&l2arc_dev_mtx);
10000 mutex_destroy(&l2arc_free_on_write_mtx);
10001
10002 list_destroy(l2arc_dev_list);
10003 list_destroy(l2arc_free_on_write);
10004 }
10005
10006 void
l2arc_start(void)10007 l2arc_start(void)
10008 {
10009 if (!(spa_mode_global & SPA_MODE_WRITE))
10010 return;
10011
10012 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
10013 TS_RUN, defclsyspri);
10014 }
10015
10016 void
l2arc_stop(void)10017 l2arc_stop(void)
10018 {
10019 if (!(spa_mode_global & SPA_MODE_WRITE))
10020 return;
10021
10022 mutex_enter(&l2arc_feed_thr_lock);
10023 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */
10024 l2arc_thread_exit = 1;
10025 while (l2arc_thread_exit != 0)
10026 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
10027 mutex_exit(&l2arc_feed_thr_lock);
10028 }
10029
10030 /*
10031 * Punches out rebuild threads for the L2ARC devices in a spa. This should
10032 * be called after pool import from the spa async thread, since starting
10033 * these threads directly from spa_import() will make them part of the
10034 * "zpool import" context and delay process exit (and thus pool import).
10035 */
10036 void
l2arc_spa_rebuild_start(spa_t * spa)10037 l2arc_spa_rebuild_start(spa_t *spa)
10038 {
10039 ASSERT(MUTEX_HELD(&spa_namespace_lock));
10040
10041 /*
10042 * Locate the spa's l2arc devices and kick off rebuild threads.
10043 */
10044 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10045 l2arc_dev_t *dev =
10046 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10047 if (dev == NULL) {
10048 /* Don't attempt a rebuild if the vdev is UNAVAIL */
10049 continue;
10050 }
10051 mutex_enter(&l2arc_rebuild_thr_lock);
10052 if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
10053 dev->l2ad_rebuild_began = B_TRUE;
10054 (void) thread_create(NULL, 0, l2arc_dev_rebuild_thread,
10055 dev, 0, &p0, TS_RUN, minclsyspri);
10056 }
10057 mutex_exit(&l2arc_rebuild_thr_lock);
10058 }
10059 }
10060
10061 void
l2arc_spa_rebuild_stop(spa_t * spa)10062 l2arc_spa_rebuild_stop(spa_t *spa)
10063 {
10064 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
10065 spa->spa_export_thread == curthread);
10066
10067 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10068 l2arc_dev_t *dev =
10069 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10070 if (dev == NULL)
10071 continue;
10072 mutex_enter(&l2arc_rebuild_thr_lock);
10073 dev->l2ad_rebuild_cancel = B_TRUE;
10074 mutex_exit(&l2arc_rebuild_thr_lock);
10075 }
10076 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10077 l2arc_dev_t *dev =
10078 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10079 if (dev == NULL)
10080 continue;
10081 mutex_enter(&l2arc_rebuild_thr_lock);
10082 if (dev->l2ad_rebuild_began == B_TRUE) {
10083 while (dev->l2ad_rebuild == B_TRUE) {
10084 cv_wait(&l2arc_rebuild_thr_cv,
10085 &l2arc_rebuild_thr_lock);
10086 }
10087 }
10088 mutex_exit(&l2arc_rebuild_thr_lock);
10089 }
10090 }
10091
10092 /*
10093 * Main entry point for L2ARC rebuilding.
10094 */
10095 static __attribute__((noreturn)) void
l2arc_dev_rebuild_thread(void * arg)10096 l2arc_dev_rebuild_thread(void *arg)
10097 {
10098 l2arc_dev_t *dev = arg;
10099
10100 VERIFY(dev->l2ad_rebuild);
10101 (void) l2arc_rebuild(dev);
10102 mutex_enter(&l2arc_rebuild_thr_lock);
10103 dev->l2ad_rebuild_began = B_FALSE;
10104 dev->l2ad_rebuild = B_FALSE;
10105 cv_signal(&l2arc_rebuild_thr_cv);
10106 mutex_exit(&l2arc_rebuild_thr_lock);
10107
10108 thread_exit();
10109 }
10110
10111 /*
10112 * This function implements the actual L2ARC metadata rebuild. It:
10113 * starts reading the log block chain and restores each block's contents
10114 * to memory (reconstructing arc_buf_hdr_t's).
10115 *
10116 * Operation stops under any of the following conditions:
10117 *
10118 * 1) We reach the end of the log block chain.
10119 * 2) We encounter *any* error condition (cksum errors, io errors)
10120 */
10121 static int
l2arc_rebuild(l2arc_dev_t * dev)10122 l2arc_rebuild(l2arc_dev_t *dev)
10123 {
10124 vdev_t *vd = dev->l2ad_vdev;
10125 spa_t *spa = vd->vdev_spa;
10126 int err = 0;
10127 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10128 l2arc_log_blk_phys_t *this_lb, *next_lb;
10129 zio_t *this_io = NULL, *next_io = NULL;
10130 l2arc_log_blkptr_t lbps[2];
10131 l2arc_lb_ptr_buf_t *lb_ptr_buf;
10132 boolean_t lock_held;
10133
10134 this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP);
10135 next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP);
10136
10137 /*
10138 * We prevent device removal while issuing reads to the device,
10139 * then during the rebuilding phases we drop this lock again so
10140 * that a spa_unload or device remove can be initiated - this is
10141 * safe, because the spa will signal us to stop before removing
10142 * our device and wait for us to stop.
10143 */
10144 spa_config_enter(spa, SCL_L2ARC, vd, RW_READER);
10145 lock_held = B_TRUE;
10146
10147 /*
10148 * Retrieve the persistent L2ARC device state.
10149 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10150 */
10151 dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
10152 dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
10153 L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
10154 dev->l2ad_start);
10155 dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);
10156
10157 vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
10158 vd->vdev_trim_state = l2dhdr->dh_trim_state;
10159
10160 /*
10161 * In case the zfs module parameter l2arc_rebuild_enabled is false
10162 * we do not start the rebuild process.
10163 */
10164 if (!l2arc_rebuild_enabled)
10165 goto out;
10166
10167 /* Prepare the rebuild process */
10168 memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));
10169
10170 /* Start the rebuild process */
10171 for (;;) {
10172 if (!l2arc_log_blkptr_valid(dev, &lbps[0]))
10173 break;
10174
10175 if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1],
10176 this_lb, next_lb, this_io, &next_io)) != 0)
10177 goto out;
10178
10179 /*
10180 * Our memory pressure valve. If the system is running low
10181 * on memory, rather than swamping memory with new ARC buf
10182 * hdrs, we opt not to rebuild the L2ARC. At this point,
10183 * however, we have already set up our L2ARC dev to chain in
10184 * new metadata log blocks, so the user may choose to offline/
10185 * online the L2ARC dev at a later time (or re-import the pool)
10186 * to reconstruct it (when there's less memory pressure).
10187 */
10188 if (l2arc_hdr_limit_reached()) {
10189 ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem);
10190 cmn_err(CE_NOTE, "System running low on memory, "
10191 "aborting L2ARC rebuild.");
10192 err = SET_ERROR(ENOMEM);
10193 goto out;
10194 }
10195
10196 spa_config_exit(spa, SCL_L2ARC, vd);
10197 lock_held = B_FALSE;
10198
10199 /*
10200 * Now that we know that the next_lb checks out alright, we
10201 * can start reconstruction from this log block.
10202 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10203 */
10204 uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
10205 l2arc_log_blk_restore(dev, this_lb, asize);
10206
10207 /*
10208 * log block restored, include its pointer in the list of
10209 * pointers to log blocks present in the L2ARC device.
10210 */
10211 lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
10212 lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
10213 KM_SLEEP);
10214 memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
10215 sizeof (l2arc_log_blkptr_t));
10216 mutex_enter(&dev->l2ad_mtx);
10217 list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
10218 ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
10219 ARCSTAT_BUMP(arcstat_l2_log_blk_count);
10220 zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10221 zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10222 mutex_exit(&dev->l2ad_mtx);
10223 vdev_space_update(vd, asize, 0, 0);
10224
10225 /*
10226 * Protection against loops of log blocks:
10227 *
10228 * l2ad_hand l2ad_evict
10229 * V V
10230 * l2ad_start |=======================================| l2ad_end
10231 * -----|||----|||---|||----|||
10232 * (3) (2) (1) (0)
10233 * ---|||---|||----|||---|||
10234 * (7) (6) (5) (4)
10235 *
10236 * In this situation the pointer of log block (4) passes
10237 * l2arc_log_blkptr_valid() but the log block should not be
10238 * restored as it is overwritten by the payload of log block
10239 * (0). Only log blocks (0)-(3) should be restored. We check
10240 * whether l2ad_evict lies in between the payload starting
10241 * offset of the next log block (lbps[1].lbp_payload_start)
10242 * and the payload starting offset of the present log block
10243 * (lbps[0].lbp_payload_start). If true and this isn't the
10244 * first pass, we are looping from the beginning and we should
10245 * stop.
10246 */
10247 if (l2arc_range_check_overlap(lbps[1].lbp_payload_start,
10248 lbps[0].lbp_payload_start, dev->l2ad_evict) &&
10249 !dev->l2ad_first)
10250 goto out;
10251
10252 kpreempt(KPREEMPT_SYNC);
10253 for (;;) {
10254 mutex_enter(&l2arc_rebuild_thr_lock);
10255 if (dev->l2ad_rebuild_cancel) {
10256 mutex_exit(&l2arc_rebuild_thr_lock);
10257 err = SET_ERROR(ECANCELED);
10258 goto out;
10259 }
10260 mutex_exit(&l2arc_rebuild_thr_lock);
10261 if (spa_config_tryenter(spa, SCL_L2ARC, vd,
10262 RW_READER)) {
10263 lock_held = B_TRUE;
10264 break;
10265 }
10266 /*
10267 * L2ARC config lock held by somebody in writer,
10268 * possibly due to them trying to remove us. They'll
10269 * likely to want us to shut down, so after a little
10270 * delay, we check l2ad_rebuild_cancel and retry
10271 * the lock again.
10272 */
10273 delay(1);
10274 }
10275
10276 /*
10277 * Continue with the next log block.
10278 */
10279 lbps[0] = lbps[1];
10280 lbps[1] = this_lb->lb_prev_lbp;
10281 PTR_SWAP(this_lb, next_lb);
10282 this_io = next_io;
10283 next_io = NULL;
10284 }
10285
10286 if (this_io != NULL)
10287 l2arc_log_blk_fetch_abort(this_io);
10288 out:
10289 if (next_io != NULL)
10290 l2arc_log_blk_fetch_abort(next_io);
10291 vmem_free(this_lb, sizeof (*this_lb));
10292 vmem_free(next_lb, sizeof (*next_lb));
10293
10294 if (err == ECANCELED) {
10295 /*
10296 * In case the rebuild was canceled do not log to spa history
10297 * log as the pool may be in the process of being removed.
10298 */
10299 zfs_dbgmsg("L2ARC rebuild aborted, restored %llu blocks",
10300 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10301 return (err);
10302 } else if (!l2arc_rebuild_enabled) {
10303 spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10304 "disabled");
10305 } else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
10306 ARCSTAT_BUMP(arcstat_l2_rebuild_success);
10307 spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10308 "successful, restored %llu blocks",
10309 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10310 } else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
10311 /*
10312 * No error but also nothing restored, meaning the lbps array
10313 * in the device header points to invalid/non-present log
10314 * blocks. Reset the header.
10315 */
10316 spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10317 "no valid log blocks");
10318 memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
10319 l2arc_dev_hdr_update(dev);
10320 } else if (err != 0) {
10321 spa_history_log_internal(spa, "L2ARC rebuild", NULL,
10322 "aborted, restored %llu blocks",
10323 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10324 }
10325
10326 if (lock_held)
10327 spa_config_exit(spa, SCL_L2ARC, vd);
10328
10329 return (err);
10330 }
10331
10332 /*
10333 * Attempts to read the device header on the provided L2ARC device and writes
10334 * it to `hdr'. On success, this function returns 0, otherwise the appropriate
10335 * error code is returned.
10336 */
10337 static int
l2arc_dev_hdr_read(l2arc_dev_t * dev)10338 l2arc_dev_hdr_read(l2arc_dev_t *dev)
10339 {
10340 int err;
10341 uint64_t guid;
10342 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10343 const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10344 abd_t *abd;
10345
10346 guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10347
10348 abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
10349
10350 err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
10351 VDEV_LABEL_START_SIZE, l2dhdr_asize, abd,
10352 ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
10353 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY |
10354 ZIO_FLAG_SPECULATIVE, B_FALSE));
10355
10356 abd_free(abd);
10357
10358 if (err != 0) {
10359 ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors);
10360 zfs_dbgmsg("L2ARC IO error (%d) while reading device header, "
10361 "vdev guid: %llu", err,
10362 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10363 return (err);
10364 }
10365
10366 if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
10367 byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr));
10368
10369 if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC ||
10370 l2dhdr->dh_spa_guid != guid ||
10371 l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid ||
10372 l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION ||
10373 l2dhdr->dh_log_entries != dev->l2ad_log_entries ||
10374 l2dhdr->dh_end != dev->l2ad_end ||
10375 !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
10376 l2dhdr->dh_evict) ||
10377 (l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
10378 l2arc_trim_ahead > 0)) {
10379 /*
10380 * Attempt to rebuild a device containing no actual dev hdr
10381 * or containing a header from some other pool or from another
10382 * version of persistent L2ARC.
10383 */
10384 ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported);
10385 return (SET_ERROR(ENOTSUP));
10386 }
10387
10388 return (0);
10389 }
10390
10391 /*
10392 * Reads L2ARC log blocks from storage and validates their contents.
10393 *
10394 * This function implements a simple fetcher to make sure that while
10395 * we're processing one buffer the L2ARC is already fetching the next
10396 * one in the chain.
10397 *
10398 * The arguments this_lp and next_lp point to the current and next log block
10399 * address in the block chain. Similarly, this_lb and next_lb hold the
10400 * l2arc_log_blk_phys_t's of the current and next L2ARC blk.
10401 *
10402 * The `this_io' and `next_io' arguments are used for block fetching.
10403 * When issuing the first blk IO during rebuild, you should pass NULL for
10404 * `this_io'. This function will then issue a sync IO to read the block and
10405 * also issue an async IO to fetch the next block in the block chain. The
10406 * fetched IO is returned in `next_io'. On subsequent calls to this
10407 * function, pass the value returned in `next_io' from the previous call
10408 * as `this_io' and a fresh `next_io' pointer to hold the next fetch IO.
10409 * Prior to the call, you should initialize your `next_io' pointer to be
10410 * NULL. If no fetch IO was issued, the pointer is left set at NULL.
10411 *
10412 * On success, this function returns 0, otherwise it returns an appropriate
10413 * error code. On error the fetching IO is aborted and cleared before
10414 * returning from this function. Therefore, if we return `success', the
10415 * caller can assume that we have taken care of cleanup of fetch IOs.
10416 */
10417 static int
l2arc_log_blk_read(l2arc_dev_t * dev,const l2arc_log_blkptr_t * this_lbp,const l2arc_log_blkptr_t * next_lbp,l2arc_log_blk_phys_t * this_lb,l2arc_log_blk_phys_t * next_lb,zio_t * this_io,zio_t ** next_io)10418 l2arc_log_blk_read(l2arc_dev_t *dev,
10419 const l2arc_log_blkptr_t *this_lbp, const l2arc_log_blkptr_t *next_lbp,
10420 l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
10421 zio_t *this_io, zio_t **next_io)
10422 {
10423 int err = 0;
10424 zio_cksum_t cksum;
10425 uint64_t asize;
10426
10427 ASSERT(this_lbp != NULL && next_lbp != NULL);
10428 ASSERT(this_lb != NULL && next_lb != NULL);
10429 ASSERT(next_io != NULL && *next_io == NULL);
10430 ASSERT(l2arc_log_blkptr_valid(dev, this_lbp));
10431
10432 /*
10433 * Check to see if we have issued the IO for this log block in a
10434 * previous run. If not, this is the first call, so issue it now.
10435 */
10436 if (this_io == NULL) {
10437 this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
10438 this_lb);
10439 }
10440
10441 /*
10442 * Peek to see if we can start issuing the next IO immediately.
10443 */
10444 if (l2arc_log_blkptr_valid(dev, next_lbp)) {
10445 /*
10446 * Start issuing IO for the next log block early - this
10447 * should help keep the L2ARC device busy while we
10448 * decompress and restore this log block.
10449 */
10450 *next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
10451 next_lb);
10452 }
10453
10454 /* Wait for the IO to read this log block to complete */
10455 if ((err = zio_wait(this_io)) != 0) {
10456 ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors);
10457 zfs_dbgmsg("L2ARC IO error (%d) while reading log block, "
10458 "offset: %llu, vdev guid: %llu", err,
10459 (u_longlong_t)this_lbp->lbp_daddr,
10460 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10461 goto cleanup;
10462 }
10463
10464 /*
10465 * Make sure the buffer checks out.
10466 * L2BLK_GET_PSIZE returns aligned size for log blocks.
10467 */
10468 asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
10469 fletcher_4_native(this_lb, asize, NULL, &cksum);
10470 if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
10471 ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors);
10472 zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, "
10473 "vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu",
10474 (u_longlong_t)this_lbp->lbp_daddr,
10475 (u_longlong_t)dev->l2ad_vdev->vdev_guid,
10476 (u_longlong_t)dev->l2ad_hand,
10477 (u_longlong_t)dev->l2ad_evict);
10478 err = SET_ERROR(ECKSUM);
10479 goto cleanup;
10480 }
10481
10482 /* Now we can take our time decoding this buffer */
10483 switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
10484 case ZIO_COMPRESS_OFF:
10485 break;
10486 case ZIO_COMPRESS_LZ4: {
10487 abd_t *abd = abd_alloc_linear(asize, B_TRUE);
10488 abd_copy_from_buf_off(abd, this_lb, 0, asize);
10489 abd_t dabd;
10490 abd_get_from_buf_struct(&dabd, this_lb, sizeof (*this_lb));
10491 err = zio_decompress_data(
10492 L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
10493 abd, &dabd, asize, sizeof (*this_lb), NULL);
10494 abd_free(&dabd);
10495 abd_free(abd);
10496 if (err != 0) {
10497 err = SET_ERROR(EINVAL);
10498 goto cleanup;
10499 }
10500 break;
10501 }
10502 default:
10503 err = SET_ERROR(EINVAL);
10504 goto cleanup;
10505 }
10506 if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
10507 byteswap_uint64_array(this_lb, sizeof (*this_lb));
10508 if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
10509 err = SET_ERROR(EINVAL);
10510 goto cleanup;
10511 }
10512 cleanup:
10513 /* Abort an in-flight fetch I/O in case of error */
10514 if (err != 0 && *next_io != NULL) {
10515 l2arc_log_blk_fetch_abort(*next_io);
10516 *next_io = NULL;
10517 }
10518 return (err);
10519 }
10520
10521 /*
10522 * Restores the payload of a log block to ARC. This creates empty ARC hdr
10523 * entries which only contain an l2arc hdr, essentially restoring the
10524 * buffers to their L2ARC evicted state. This function also updates space
10525 * usage on the L2ARC vdev to make sure it tracks restored buffers.
10526 */
10527 static void
l2arc_log_blk_restore(l2arc_dev_t * dev,const l2arc_log_blk_phys_t * lb,uint64_t lb_asize)10528 l2arc_log_blk_restore(l2arc_dev_t *dev, const l2arc_log_blk_phys_t *lb,
10529 uint64_t lb_asize)
10530 {
10531 uint64_t size = 0, asize = 0;
10532 uint64_t log_entries = dev->l2ad_log_entries;
10533
10534 /*
10535 * Usually arc_adapt() is called only for data, not headers, but
10536 * since we may allocate significant amount of memory here, let ARC
10537 * grow its arc_c.
10538 */
10539 arc_adapt(log_entries * HDR_L2ONLY_SIZE);
10540
10541 for (int i = log_entries - 1; i >= 0; i--) {
10542 /*
10543 * Restore goes in the reverse temporal direction to preserve
10544 * correct temporal ordering of buffers in the l2ad_buflist.
10545 * l2arc_hdr_restore also does a list_insert_tail instead of
10546 * list_insert_head on the l2ad_buflist:
10547 *
10548 * LIST l2ad_buflist LIST
10549 * HEAD <------ (time) ------ TAIL
10550 * direction +-----+-----+-----+-----+-----+ direction
10551 * of l2arc <== | buf | buf | buf | buf | buf | ===> of rebuild
10552 * fill +-----+-----+-----+-----+-----+
10553 * ^ ^
10554 * | |
10555 * | |
10556 * l2arc_feed_thread l2arc_rebuild
10557 * will place new bufs here restores bufs here
10558 *
10559 * During l2arc_rebuild() the device is not used by
10560 * l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
10561 */
10562 size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
10563 asize += vdev_psize_to_asize(dev->l2ad_vdev,
10564 L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
10565 l2arc_hdr_restore(&lb->lb_entries[i], dev);
10566 }
10567
10568 /*
10569 * Record rebuild stats:
10570 * size Logical size of restored buffers in the L2ARC
10571 * asize Aligned size of restored buffers in the L2ARC
10572 */
10573 ARCSTAT_INCR(arcstat_l2_rebuild_size, size);
10574 ARCSTAT_INCR(arcstat_l2_rebuild_asize, asize);
10575 ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries);
10576 ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, lb_asize);
10577 ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, asize / lb_asize);
10578 ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks);
10579 }
10580
10581 /*
10582 * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
10583 * into a state indicating that it has been evicted to L2ARC.
10584 */
10585 static void
l2arc_hdr_restore(const l2arc_log_ent_phys_t * le,l2arc_dev_t * dev)10586 l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, l2arc_dev_t *dev)
10587 {
10588 arc_buf_hdr_t *hdr, *exists;
10589 kmutex_t *hash_lock;
10590 arc_buf_contents_t type = L2BLK_GET_TYPE((le)->le_prop);
10591 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
10592 L2BLK_GET_PSIZE((le)->le_prop));
10593
10594 /*
10595 * Do all the allocation before grabbing any locks, this lets us
10596 * sleep if memory is full and we don't have to deal with failed
10597 * allocations.
10598 */
10599 hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
10600 dev, le->le_dva, le->le_daddr,
10601 L2BLK_GET_PSIZE((le)->le_prop), asize, le->le_birth,
10602 L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
10603 L2BLK_GET_PROTECTED((le)->le_prop),
10604 L2BLK_GET_PREFETCH((le)->le_prop),
10605 L2BLK_GET_STATE((le)->le_prop));
10606
10607 /*
10608 * vdev_space_update() has to be called before arc_hdr_destroy() to
10609 * avoid underflow since the latter also calls vdev_space_update().
10610 */
10611 l2arc_hdr_arcstats_increment(hdr);
10612 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10613
10614 mutex_enter(&dev->l2ad_mtx);
10615 list_insert_tail(&dev->l2ad_buflist, hdr);
10616 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
10617 mutex_exit(&dev->l2ad_mtx);
10618
10619 exists = buf_hash_insert(hdr, &hash_lock);
10620 if (exists) {
10621 /* Buffer was already cached, no need to restore it. */
10622 arc_hdr_destroy(hdr);
10623 /*
10624 * If the buffer is already cached, check whether it has
10625 * L2ARC metadata. If not, enter them and update the flag.
10626 * This is important is case of onlining a cache device, since
10627 * we previously evicted all L2ARC metadata from ARC.
10628 */
10629 if (!HDR_HAS_L2HDR(exists)) {
10630 arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR);
10631 exists->b_l2hdr.b_dev = dev;
10632 exists->b_l2hdr.b_daddr = le->le_daddr;
10633 exists->b_l2hdr.b_arcs_state =
10634 L2BLK_GET_STATE((le)->le_prop);
10635 /* l2arc_hdr_arcstats_update() expects a valid asize */
10636 HDR_SET_L2SIZE(exists, asize);
10637 mutex_enter(&dev->l2ad_mtx);
10638 list_insert_tail(&dev->l2ad_buflist, exists);
10639 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
10640 arc_hdr_size(exists), exists);
10641 mutex_exit(&dev->l2ad_mtx);
10642 l2arc_hdr_arcstats_increment(exists);
10643 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10644 }
10645 ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached);
10646 }
10647
10648 mutex_exit(hash_lock);
10649 }
10650
10651 /*
10652 * Starts an asynchronous read IO to read a log block. This is used in log
10653 * block reconstruction to start reading the next block before we are done
10654 * decoding and reconstructing the current block, to keep the l2arc device
10655 * nice and hot with read IO to process.
10656 * The returned zio will contain a newly allocated memory buffers for the IO
10657 * data which should then be freed by the caller once the zio is no longer
10658 * needed (i.e. due to it having completed). If you wish to abort this
10659 * zio, you should do so using l2arc_log_blk_fetch_abort, which takes
10660 * care of disposing of the allocated buffers correctly.
10661 */
10662 static zio_t *
l2arc_log_blk_fetch(vdev_t * vd,const l2arc_log_blkptr_t * lbp,l2arc_log_blk_phys_t * lb)10663 l2arc_log_blk_fetch(vdev_t *vd, const l2arc_log_blkptr_t *lbp,
10664 l2arc_log_blk_phys_t *lb)
10665 {
10666 uint32_t asize;
10667 zio_t *pio;
10668 l2arc_read_callback_t *cb;
10669
10670 /* L2BLK_GET_PSIZE returns aligned size for log blocks */
10671 asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10672 ASSERT(asize <= sizeof (l2arc_log_blk_phys_t));
10673
10674 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP);
10675 cb->l2rcb_abd = abd_get_from_buf(lb, asize);
10676 pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
10677 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY);
10678 (void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
10679 cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10680 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL |
10681 ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE));
10682
10683 return (pio);
10684 }
10685
10686 /*
10687 * Aborts a zio returned from l2arc_log_blk_fetch and frees the data
10688 * buffers allocated for it.
10689 */
10690 static void
l2arc_log_blk_fetch_abort(zio_t * zio)10691 l2arc_log_blk_fetch_abort(zio_t *zio)
10692 {
10693 (void) zio_wait(zio);
10694 }
10695
10696 /*
10697 * Creates a zio to update the device header on an l2arc device.
10698 */
10699 void
l2arc_dev_hdr_update(l2arc_dev_t * dev)10700 l2arc_dev_hdr_update(l2arc_dev_t *dev)
10701 {
10702 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10703 const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10704 abd_t *abd;
10705 int err;
10706
10707 VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));
10708
10709 l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
10710 l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
10711 l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10712 l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
10713 l2dhdr->dh_log_entries = dev->l2ad_log_entries;
10714 l2dhdr->dh_evict = dev->l2ad_evict;
10715 l2dhdr->dh_start = dev->l2ad_start;
10716 l2dhdr->dh_end = dev->l2ad_end;
10717 l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
10718 l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
10719 l2dhdr->dh_flags = 0;
10720 l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
10721 l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
10722 if (dev->l2ad_first)
10723 l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST;
10724
10725 abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
10726
10727 err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
10728 VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL,
10729 NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE));
10730
10731 abd_free(abd);
10732
10733 if (err != 0) {
10734 zfs_dbgmsg("L2ARC IO error (%d) while writing device header, "
10735 "vdev guid: %llu", err,
10736 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10737 }
10738 }
10739
10740 /*
10741 * Commits a log block to the L2ARC device. This routine is invoked from
10742 * l2arc_write_buffers when the log block fills up.
10743 * This function allocates some memory to temporarily hold the serialized
10744 * buffer to be written. This is then released in l2arc_write_done.
10745 */
10746 static uint64_t
l2arc_log_blk_commit(l2arc_dev_t * dev,zio_t * pio,l2arc_write_callback_t * cb)10747 l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
10748 {
10749 l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
10750 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10751 uint64_t psize, asize;
10752 zio_t *wzio;
10753 l2arc_lb_abd_buf_t *abd_buf;
10754 abd_t *abd = NULL;
10755 l2arc_lb_ptr_buf_t *lb_ptr_buf;
10756
10757 VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
10758
10759 abd_buf = zio_buf_alloc(sizeof (*abd_buf));
10760 abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
10761 lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
10762 lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);
10763
10764 /* link the buffer into the block chain */
10765 lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
10766 lb->lb_magic = L2ARC_LOG_BLK_MAGIC;
10767
10768 /*
10769 * l2arc_log_blk_commit() may be called multiple times during a single
10770 * l2arc_write_buffers() call. Save the allocated abd buffers in a list
10771 * so we can free them in l2arc_write_done() later on.
10772 */
10773 list_insert_tail(&cb->l2wcb_abd_list, abd_buf);
10774
10775 /* try to compress the buffer, at least one sector to save */
10776 psize = zio_compress_data(ZIO_COMPRESS_LZ4,
10777 abd_buf->abd, &abd, sizeof (*lb),
10778 zio_get_compression_max_size(ZIO_COMPRESS_LZ4,
10779 dev->l2ad_vdev->vdev_ashift,
10780 dev->l2ad_vdev->vdev_ashift, sizeof (*lb)), 0);
10781
10782 /* a log block is never entirely zero */
10783 ASSERT(psize != 0);
10784 asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
10785 ASSERT(asize <= sizeof (*lb));
10786
10787 /*
10788 * Update the start log block pointer in the device header to point
10789 * to the log block we're about to write.
10790 */
10791 l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
10792 l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
10793 l2dhdr->dh_start_lbps[0].lbp_payload_asize =
10794 dev->l2ad_log_blk_payload_asize;
10795 l2dhdr->dh_start_lbps[0].lbp_payload_start =
10796 dev->l2ad_log_blk_payload_start;
10797 L2BLK_SET_LSIZE(
10798 (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
10799 L2BLK_SET_PSIZE(
10800 (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
10801 L2BLK_SET_CHECKSUM(
10802 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10803 ZIO_CHECKSUM_FLETCHER_4);
10804 if (asize < sizeof (*lb)) {
10805 /* compression succeeded */
10806 abd_zero_off(abd, psize, asize - psize);
10807 L2BLK_SET_COMPRESS(
10808 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10809 ZIO_COMPRESS_LZ4);
10810 } else {
10811 /* compression failed */
10812 abd_copy_from_buf_off(abd, lb, 0, sizeof (*lb));
10813 L2BLK_SET_COMPRESS(
10814 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10815 ZIO_COMPRESS_OFF);
10816 }
10817
10818 /* checksum what we're about to write */
10819 abd_fletcher_4_native(abd, asize, NULL,
10820 &l2dhdr->dh_start_lbps[0].lbp_cksum);
10821
10822 abd_free(abd_buf->abd);
10823
10824 /* perform the write itself */
10825 abd_buf->abd = abd;
10826 wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
10827 asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10828 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
10829 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio);
10830 (void) zio_nowait(wzio);
10831
10832 dev->l2ad_hand += asize;
10833 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10834
10835 /*
10836 * Include the committed log block's pointer in the list of pointers
10837 * to log blocks present in the L2ARC device.
10838 */
10839 memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
10840 sizeof (l2arc_log_blkptr_t));
10841 mutex_enter(&dev->l2ad_mtx);
10842 list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
10843 ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
10844 ARCSTAT_BUMP(arcstat_l2_log_blk_count);
10845 zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10846 zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10847 mutex_exit(&dev->l2ad_mtx);
10848
10849 /* bump the kstats */
10850 ARCSTAT_INCR(arcstat_l2_write_bytes, asize);
10851 ARCSTAT_BUMP(arcstat_l2_log_blk_writes);
10852 ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, asize);
10853 ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio,
10854 dev->l2ad_log_blk_payload_asize / asize);
10855
10856 /* start a new log block */
10857 dev->l2ad_log_ent_idx = 0;
10858 dev->l2ad_log_blk_payload_asize = 0;
10859 dev->l2ad_log_blk_payload_start = 0;
10860
10861 return (asize);
10862 }
10863
10864 /*
10865 * Validates an L2ARC log block address to make sure that it can be read
10866 * from the provided L2ARC device.
10867 */
10868 boolean_t
l2arc_log_blkptr_valid(l2arc_dev_t * dev,const l2arc_log_blkptr_t * lbp)10869 l2arc_log_blkptr_valid(l2arc_dev_t *dev, const l2arc_log_blkptr_t *lbp)
10870 {
10871 /* L2BLK_GET_PSIZE returns aligned size for log blocks */
10872 uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10873 uint64_t end = lbp->lbp_daddr + asize - 1;
10874 uint64_t start = lbp->lbp_payload_start;
10875 boolean_t evicted = B_FALSE;
10876
10877 /*
10878 * A log block is valid if all of the following conditions are true:
10879 * - it fits entirely (including its payload) between l2ad_start and
10880 * l2ad_end
10881 * - it has a valid size
10882 * - neither the log block itself nor part of its payload was evicted
10883 * by l2arc_evict():
10884 *
10885 * l2ad_hand l2ad_evict
10886 * | | lbp_daddr
10887 * | start | | end
10888 * | | | | |
10889 * V V V V V
10890 * l2ad_start ============================================ l2ad_end
10891 * --------------------------||||
10892 * ^ ^
10893 * | log block
10894 * payload
10895 */
10896
10897 evicted =
10898 l2arc_range_check_overlap(start, end, dev->l2ad_hand) ||
10899 l2arc_range_check_overlap(start, end, dev->l2ad_evict) ||
10900 l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) ||
10901 l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);
10902
10903 return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
10904 asize > 0 && asize <= sizeof (l2arc_log_blk_phys_t) &&
10905 (!evicted || dev->l2ad_first));
10906 }
10907
10908 /*
10909 * Inserts ARC buffer header `hdr' into the current L2ARC log block on
10910 * the device. The buffer being inserted must be present in L2ARC.
10911 * Returns B_TRUE if the L2ARC log block is full and needs to be committed
10912 * to L2ARC, or B_FALSE if it still has room for more ARC buffers.
10913 */
10914 static boolean_t
l2arc_log_blk_insert(l2arc_dev_t * dev,const arc_buf_hdr_t * hdr)10915 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)
10916 {
10917 l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
10918 l2arc_log_ent_phys_t *le;
10919
10920 if (dev->l2ad_log_entries == 0)
10921 return (B_FALSE);
10922
10923 int index = dev->l2ad_log_ent_idx++;
10924
10925 ASSERT3S(index, <, dev->l2ad_log_entries);
10926 ASSERT(HDR_HAS_L2HDR(hdr));
10927
10928 le = &lb->lb_entries[index];
10929 memset(le, 0, sizeof (*le));
10930 le->le_dva = hdr->b_dva;
10931 le->le_birth = hdr->b_birth;
10932 le->le_daddr = hdr->b_l2hdr.b_daddr;
10933 if (index == 0)
10934 dev->l2ad_log_blk_payload_start = le->le_daddr;
10935 L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
10936 L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
10937 L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
10938 le->le_complevel = hdr->b_complevel;
10939 L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
10940 L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
10941 L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
10942 L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);
10943
10944 dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
10945 HDR_GET_PSIZE(hdr));
10946
10947 return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
10948 }
10949
10950 /*
10951 * Checks whether a given L2ARC device address sits in a time-sequential
10952 * range. The trick here is that the L2ARC is a rotary buffer, so we can't
10953 * just do a range comparison, we need to handle the situation in which the
10954 * range wraps around the end of the L2ARC device. Arguments:
10955 * bottom -- Lower end of the range to check (written to earlier).
10956 * top -- Upper end of the range to check (written to later).
10957 * check -- The address for which we want to determine if it sits in
10958 * between the top and bottom.
10959 *
10960 * The 3-way conditional below represents the following cases:
10961 *
10962 * bottom < top : Sequentially ordered case:
10963 * <check>--------+-------------------+
10964 * | (overlap here?) |
10965 * L2ARC dev V V
10966 * |---------------<bottom>============<top>--------------|
10967 *
10968 * bottom > top: Looped-around case:
10969 * <check>--------+------------------+
10970 * | (overlap here?) |
10971 * L2ARC dev V V
10972 * |===============<top>---------------<bottom>===========|
10973 * ^ ^
10974 * | (or here?) |
10975 * +---------------+---------<check>
10976 *
10977 * top == bottom : Just a single address comparison.
10978 */
10979 boolean_t
l2arc_range_check_overlap(uint64_t bottom,uint64_t top,uint64_t check)10980 l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check)
10981 {
10982 if (bottom < top)
10983 return (bottom <= check && check <= top);
10984 else if (bottom > top)
10985 return (check <= top || bottom <= check);
10986 else
10987 return (check == top);
10988 }
10989
10990 EXPORT_SYMBOL(arc_buf_size);
10991 EXPORT_SYMBOL(arc_write);
10992 EXPORT_SYMBOL(arc_read);
10993 EXPORT_SYMBOL(arc_buf_info);
10994 EXPORT_SYMBOL(arc_getbuf_func);
10995 EXPORT_SYMBOL(arc_add_prune_callback);
10996 EXPORT_SYMBOL(arc_remove_prune_callback);
10997
10998 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
10999 spl_param_get_u64, ZMOD_RW, "Minimum ARC size in bytes");
11000
11001 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
11002 spl_param_get_u64, ZMOD_RW, "Maximum ARC size in bytes");
11003
11004 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, meta_balance, UINT, ZMOD_RW,
11005 "Balance between metadata and data on ghost hits.");
11006
11007 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, grow_retry, param_set_arc_int,
11008 param_get_uint, ZMOD_RW, "Seconds before growing ARC size");
11009
11010 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, shrink_shift, param_set_arc_int,
11011 param_get_uint, ZMOD_RW, "log2(fraction of ARC to reclaim)");
11012
11013 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, pc_percent, UINT, ZMOD_RW,
11014 "Percent of pagecache to reclaim ARC to");
11015
11016 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, average_blocksize, UINT, ZMOD_RD,
11017 "Target average block size");
11018
11019 ZFS_MODULE_PARAM(zfs, zfs_, compressed_arc_enabled, INT, ZMOD_RW,
11020 "Disable compressed ARC buffers");
11021
11022 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prefetch_ms, param_set_arc_int,
11023 param_get_uint, ZMOD_RW, "Min life of prefetch block in ms");
11024
11025 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prescient_prefetch_ms,
11026 param_set_arc_int, param_get_uint, ZMOD_RW,
11027 "Min life of prescient prefetched block in ms");
11028
11029 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_max, U64, ZMOD_RW,
11030 "Max write bytes per interval");
11031
11032 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_boost, U64, ZMOD_RW,
11033 "Extra write bytes during device warmup");
11034
11035 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom, U64, ZMOD_RW,
11036 "Number of max device writes to precache");
11037
11038 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom_boost, U64, ZMOD_RW,
11039 "Compressed l2arc_headroom multiplier");
11040
11041 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, trim_ahead, U64, ZMOD_RW,
11042 "TRIM ahead L2ARC write size multiplier");
11043
11044 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_secs, U64, ZMOD_RW,
11045 "Seconds between L2ARC writing");
11046
11047 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_min_ms, U64, ZMOD_RW,
11048 "Min feed interval in milliseconds");
11049
11050 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, noprefetch, INT, ZMOD_RW,
11051 "Skip caching prefetched buffers");
11052
11053 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW,
11054 "Turbo L2ARC warmup");
11055
11056 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW,
11057 "No reads during writes");
11058
11059 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, UINT, ZMOD_RW,
11060 "Percent of ARC size allowed for L2ARC-only headers");
11061
11062 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
11063 "Rebuild the L2ARC when importing a pool");
11064
11065 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, U64, ZMOD_RW,
11066 "Min size in bytes to write rebuild log blocks in L2ARC");
11067
11068 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, mfuonly, INT, ZMOD_RW,
11069 "Cache only MFU data from ARC into L2ARC");
11070
11071 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, exclude_special, INT, ZMOD_RW,
11072 "Exclude dbufs on special vdevs from being cached to L2ARC if set.");
11073
11074 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int,
11075 param_get_uint, ZMOD_RW, "System free memory I/O throttle in bytes");
11076
11077 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, sys_free, param_set_arc_u64,
11078 spl_param_get_u64, ZMOD_RW, "System free memory target size in bytes");
11079
11080 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit, param_set_arc_u64,
11081 spl_param_get_u64, ZMOD_RW, "Minimum bytes of dnodes in ARC");
11082
11083 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit_percent,
11084 param_set_arc_int, param_get_uint, ZMOD_RW,
11085 "Percent of ARC meta buffers for dnodes");
11086
11087 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, dnode_reduce_percent, UINT, ZMOD_RW,
11088 "Percentage of excess dnodes to try to unpin");
11089
11090 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, UINT, ZMOD_RW,
11091 "When full, ARC allocation waits for eviction of this % of alloc size");
11092
11093 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, UINT, ZMOD_RW,
11094 "The number of headers to evict per sublist before moving to the next");
11095
11096 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
11097 "Number of arc_prune threads");
11098