xref: /illumos-gate/usr/src/uts/common/fs/zfs/zio.c (revision bbf215553c7233fbab8a0afdf1fac74c44781867)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
24  * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
25  * Copyright (c) 2014 Integros [integros.com]
26  * Copyright (c) 2017, Intel Corporation.
27  * Copyright 2020 Joyent, Inc.
28  */
29 
30 #include <sys/sysmacros.h>
31 #include <sys/zfs_context.h>
32 #include <sys/fm/fs/zfs.h>
33 #include <sys/spa.h>
34 #include <sys/txg.h>
35 #include <sys/spa_impl.h>
36 #include <sys/vdev_impl.h>
37 #include <sys/vdev_trim.h>
38 #include <sys/zio_impl.h>
39 #include <sys/zio_compress.h>
40 #include <sys/zio_checksum.h>
41 #include <sys/dmu_objset.h>
42 #include <sys/arc.h>
43 #include <sys/ddt.h>
44 #include <sys/blkptr.h>
45 #include <sys/zfeature.h>
46 #include <sys/time.h>
47 #include <sys/dsl_scan.h>
48 #include <sys/metaslab_impl.h>
49 #include <sys/abd.h>
50 #include <sys/cityhash.h>
51 #include <sys/dsl_crypt.h>
52 #include <sys/stdbool.h>
53 
54 /*
55  * ==========================================================================
56  * I/O type descriptions
57  * ==========================================================================
58  */
59 const char *zio_type_name[ZIO_TYPES] = {
60 	"zio_null", "zio_read", "zio_write", "zio_free", "zio_claim",
61 	"zio_ioctl", "z_trim"
62 };
63 
64 boolean_t zio_dva_throttle_enabled = B_TRUE;
65 
66 /*
67  * ==========================================================================
68  * I/O kmem caches
69  * ==========================================================================
70  */
71 kmem_cache_t *zio_cache;
72 kmem_cache_t *zio_link_cache;
73 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
74 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
75 
76 #ifdef _KERNEL
77 extern vmem_t *zio_alloc_arena;
78 #endif
79 
80 #define	ZIO_PIPELINE_CONTINUE		0x100
81 #define	ZIO_PIPELINE_STOP		0x101
82 
83 /* Mark IOs as "slow" if they take longer than 30 seconds */
84 int zio_slow_io_ms = (30 * MILLISEC);
85 
86 #define	BP_SPANB(indblkshift, level) \
87 	(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
88 #define	COMPARE_META_LEVEL	0x80000000ul
89 /*
90  * The following actions directly effect the spa's sync-to-convergence logic.
91  * The values below define the sync pass when we start performing the action.
92  * Care should be taken when changing these values as they directly impact
93  * spa_sync() performance. Tuning these values may introduce subtle performance
94  * pathologies and should only be done in the context of performance analysis.
95  * These tunables will eventually be removed and replaced with #defines once
96  * enough analysis has been done to determine optimal values.
97  *
98  * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that
99  * regular blocks are not deferred.
100  */
101 int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */
102 int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */
103 int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */
104 
105 /*
106  * An allocating zio is one that either currently has the DVA allocate
107  * stage set or will have it later in its lifetime.
108  */
109 #define	IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE)
110 
111 boolean_t	zio_requeue_io_start_cut_in_line = B_TRUE;
112 
113 #ifdef ZFS_DEBUG
114 int zio_buf_debug_limit = 16384;
115 #else
116 int zio_buf_debug_limit = 0;
117 #endif
118 
119 static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t);
120 
121 void
zio_init(void)122 zio_init(void)
123 {
124 	size_t c;
125 	vmem_t *data_alloc_arena = NULL;
126 
127 #ifdef _KERNEL
128 	data_alloc_arena = zio_alloc_arena;
129 #endif
130 	zio_cache = kmem_cache_create("zio_cache",
131 	    sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
132 	zio_link_cache = kmem_cache_create("zio_link_cache",
133 	    sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
134 
135 	/*
136 	 * For small buffers, we want a cache for each multiple of
137 	 * SPA_MINBLOCKSIZE.  For larger buffers, we want a cache
138 	 * for each quarter-power of 2.
139 	 */
140 	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
141 		size_t size = (c + 1) << SPA_MINBLOCKSHIFT;
142 		size_t p2 = size;
143 		size_t align = 0;
144 		size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0;
145 
146 		while (!ISP2(p2))
147 			p2 &= p2 - 1;
148 
149 #ifndef _KERNEL
150 		/*
151 		 * If we are using watchpoints, put each buffer on its own page,
152 		 * to eliminate the performance overhead of trapping to the
153 		 * kernel when modifying a non-watched buffer that shares the
154 		 * page with a watched buffer.
155 		 */
156 		if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE))
157 			continue;
158 #endif
159 		if (size <= 4 * SPA_MINBLOCKSIZE) {
160 			align = SPA_MINBLOCKSIZE;
161 		} else if (IS_P2ALIGNED(size, p2 >> 2)) {
162 			align = MIN(p2 >> 2, PAGESIZE);
163 		}
164 
165 		if (align != 0) {
166 			char name[36];
167 			(void) sprintf(name, "zio_buf_%lu", (ulong_t)size);
168 			zio_buf_cache[c] = kmem_cache_create(name, size,
169 			    align, NULL, NULL, NULL, NULL, NULL, cflags);
170 
171 			/*
172 			 * Since zio_data bufs do not appear in crash dumps, we
173 			 * pass KMC_NOTOUCH so that no allocator metadata is
174 			 * stored with the buffers.
175 			 */
176 			(void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size);
177 			zio_data_buf_cache[c] = kmem_cache_create(name, size,
178 			    align, NULL, NULL, NULL, NULL, data_alloc_arena,
179 			    cflags | KMC_NOTOUCH);
180 		}
181 	}
182 
183 	while (--c != 0) {
184 		ASSERT(zio_buf_cache[c] != NULL);
185 		if (zio_buf_cache[c - 1] == NULL)
186 			zio_buf_cache[c - 1] = zio_buf_cache[c];
187 
188 		ASSERT(zio_data_buf_cache[c] != NULL);
189 		if (zio_data_buf_cache[c - 1] == NULL)
190 			zio_data_buf_cache[c - 1] = zio_data_buf_cache[c];
191 	}
192 
193 	zio_inject_init();
194 }
195 
196 void
zio_fini(void)197 zio_fini(void)
198 {
199 	size_t c;
200 	kmem_cache_t *last_cache = NULL;
201 	kmem_cache_t *last_data_cache = NULL;
202 
203 	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
204 		if (zio_buf_cache[c] != last_cache) {
205 			last_cache = zio_buf_cache[c];
206 			kmem_cache_destroy(zio_buf_cache[c]);
207 		}
208 		zio_buf_cache[c] = NULL;
209 
210 		if (zio_data_buf_cache[c] != last_data_cache) {
211 			last_data_cache = zio_data_buf_cache[c];
212 			kmem_cache_destroy(zio_data_buf_cache[c]);
213 		}
214 		zio_data_buf_cache[c] = NULL;
215 	}
216 
217 	kmem_cache_destroy(zio_link_cache);
218 	kmem_cache_destroy(zio_cache);
219 
220 	zio_inject_fini();
221 }
222 
223 /*
224  * ==========================================================================
225  * Allocate and free I/O buffers
226  * ==========================================================================
227  */
228 
229 /*
230  * Use zio_buf_alloc to allocate ZFS metadata.  This data will appear in a
231  * crashdump if the kernel panics, so use it judiciously.  Obviously, it's
232  * useful to inspect ZFS metadata, but if possible, we should avoid keeping
233  * excess / transient data in-core during a crashdump.
234  */
235 void *
zio_buf_alloc(size_t size)236 zio_buf_alloc(size_t size)
237 {
238 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
239 
240 	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
241 
242 	return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));
243 }
244 
245 /*
246  * Use zio_data_buf_alloc to allocate data.  The data will not appear in a
247  * crashdump if the kernel panics.  This exists so that we will limit the amount
248  * of ZFS data that shows up in a kernel crashdump.  (Thus reducing the amount
249  * of kernel heap dumped to disk when the kernel panics)
250  */
251 void *
zio_data_buf_alloc(size_t size)252 zio_data_buf_alloc(size_t size)
253 {
254 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
255 
256 	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
257 
258 	return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE));
259 }
260 
261 void
zio_buf_free(void * buf,size_t size)262 zio_buf_free(void *buf, size_t size)
263 {
264 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
265 
266 	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
267 
268 	kmem_cache_free(zio_buf_cache[c], buf);
269 }
270 
271 void
zio_data_buf_free(void * buf,size_t size)272 zio_data_buf_free(void *buf, size_t size)
273 {
274 	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;
275 
276 	VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
277 
278 	kmem_cache_free(zio_data_buf_cache[c], buf);
279 }
280 
281 /* ARGSUSED */
282 static void
zio_abd_free(void * abd,size_t size)283 zio_abd_free(void *abd, size_t size)
284 {
285 	abd_free((abd_t *)abd);
286 }
287 
288 /*
289  * ==========================================================================
290  * Push and pop I/O transform buffers
291  * ==========================================================================
292  */
293 void
zio_push_transform(zio_t * zio,abd_t * data,uint64_t size,uint64_t bufsize,zio_transform_func_t * transform)294 zio_push_transform(zio_t *zio, abd_t *data, uint64_t size, uint64_t bufsize,
295     zio_transform_func_t *transform)
296 {
297 	zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);
298 
299 	/*
300 	 * Ensure that anyone expecting this zio to contain a linear ABD isn't
301 	 * going to get a nasty surprise when they try to access the data.
302 	 */
303 	IMPLY(abd_is_linear(zio->io_abd), abd_is_linear(data));
304 
305 	zt->zt_orig_abd = zio->io_abd;
306 	zt->zt_orig_size = zio->io_size;
307 	zt->zt_bufsize = bufsize;
308 	zt->zt_transform = transform;
309 
310 	zt->zt_next = zio->io_transform_stack;
311 	zio->io_transform_stack = zt;
312 
313 	zio->io_abd = data;
314 	zio->io_size = size;
315 }
316 
317 void
zio_pop_transforms(zio_t * zio)318 zio_pop_transforms(zio_t *zio)
319 {
320 	zio_transform_t *zt;
321 
322 	while ((zt = zio->io_transform_stack) != NULL) {
323 		if (zt->zt_transform != NULL)
324 			zt->zt_transform(zio,
325 			    zt->zt_orig_abd, zt->zt_orig_size);
326 
327 		if (zt->zt_bufsize != 0)
328 			abd_free(zio->io_abd);
329 
330 		zio->io_abd = zt->zt_orig_abd;
331 		zio->io_size = zt->zt_orig_size;
332 		zio->io_transform_stack = zt->zt_next;
333 
334 		kmem_free(zt, sizeof (zio_transform_t));
335 	}
336 }
337 
338 /*
339  * ==========================================================================
340  * I/O transform callbacks for subblocks, decompression, and decryption
341  * ==========================================================================
342  */
343 static void
zio_subblock(zio_t * zio,abd_t * data,uint64_t size)344 zio_subblock(zio_t *zio, abd_t *data, uint64_t size)
345 {
346 	ASSERT(zio->io_size > size);
347 
348 	if (zio->io_type == ZIO_TYPE_READ)
349 		abd_copy(data, zio->io_abd, size);
350 }
351 
352 static void
zio_decompress(zio_t * zio,abd_t * data,uint64_t size)353 zio_decompress(zio_t *zio, abd_t *data, uint64_t size)
354 {
355 	if (zio->io_error == 0) {
356 		void *tmp = abd_borrow_buf(data, size);
357 		int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
358 		    zio->io_abd, tmp, zio->io_size, size);
359 		abd_return_buf_copy(data, tmp, size);
360 
361 		if (ret != 0)
362 			zio->io_error = SET_ERROR(EIO);
363 	}
364 }
365 
366 static void
zio_decrypt(zio_t * zio,abd_t * data,uint64_t size)367 zio_decrypt(zio_t *zio, abd_t *data, uint64_t size)
368 {
369 	int ret;
370 	void *tmp;
371 	blkptr_t *bp = zio->io_bp;
372 	spa_t *spa = zio->io_spa;
373 	uint64_t dsobj = zio->io_bookmark.zb_objset;
374 	uint64_t lsize = BP_GET_LSIZE(bp);
375 	dmu_object_type_t ot = BP_GET_TYPE(bp);
376 	uint8_t salt[ZIO_DATA_SALT_LEN];
377 	uint8_t iv[ZIO_DATA_IV_LEN];
378 	uint8_t mac[ZIO_DATA_MAC_LEN];
379 	boolean_t no_crypt = B_FALSE;
380 
381 	ASSERT(BP_USES_CRYPT(bp));
382 	ASSERT3U(size, !=, 0);
383 
384 	if (zio->io_error != 0)
385 		return;
386 
387 	/*
388 	 * Verify the cksum of MACs stored in an indirect bp. It will always
389 	 * be possible to verify this since it does not require an encryption
390 	 * key.
391 	 */
392 	if (BP_HAS_INDIRECT_MAC_CKSUM(bp)) {
393 		zio_crypt_decode_mac_bp(bp, mac);
394 
395 		if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
396 			/*
397 			 * We haven't decompressed the data yet, but
398 			 * zio_crypt_do_indirect_mac_checksum() requires
399 			 * decompressed data to be able to parse out the MACs
400 			 * from the indirect block. We decompress it now and
401 			 * throw away the result after we are finished.
402 			 */
403 			tmp = zio_buf_alloc(lsize);
404 			ret = zio_decompress_data(BP_GET_COMPRESS(bp),
405 			    zio->io_abd, tmp, zio->io_size, lsize);
406 			if (ret != 0) {
407 				ret = SET_ERROR(EIO);
408 				goto error;
409 			}
410 			ret = zio_crypt_do_indirect_mac_checksum(B_FALSE,
411 			    tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac);
412 			zio_buf_free(tmp, lsize);
413 		} else {
414 			ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE,
415 			    zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac);
416 		}
417 		abd_copy(data, zio->io_abd, size);
418 
419 		if (ret != 0)
420 			goto error;
421 
422 		return;
423 	}
424 
425 	/*
426 	 * If this is an authenticated block, just check the MAC. It would be
427 	 * nice to separate this out into its own flag, but for the moment
428 	 * enum zio_flag is out of bits.
429 	 */
430 	if (BP_IS_AUTHENTICATED(bp)) {
431 		if (ot == DMU_OT_OBJSET) {
432 			ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa,
433 			    dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp));
434 		} else {
435 			zio_crypt_decode_mac_bp(bp, mac);
436 			ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj,
437 			    zio->io_abd, size, mac);
438 		}
439 		abd_copy(data, zio->io_abd, size);
440 
441 		if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) {
442 			ret = zio_handle_decrypt_injection(spa,
443 			    &zio->io_bookmark, ot, ECKSUM);
444 		}
445 		if (ret != 0)
446 			goto error;
447 
448 		return;
449 	}
450 
451 	zio_crypt_decode_params_bp(bp, salt, iv);
452 
453 	if (ot == DMU_OT_INTENT_LOG) {
454 		tmp = abd_borrow_buf_copy(zio->io_abd, sizeof (zil_chain_t));
455 		zio_crypt_decode_mac_zil(tmp, mac);
456 		abd_return_buf(zio->io_abd, tmp, sizeof (zil_chain_t));
457 	} else {
458 		zio_crypt_decode_mac_bp(bp, mac);
459 	}
460 
461 	ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp),
462 	    BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data,
463 	    zio->io_abd, &no_crypt);
464 	if (no_crypt)
465 		abd_copy(data, zio->io_abd, size);
466 
467 	if (ret != 0)
468 		goto error;
469 
470 	return;
471 
472 error:
473 	/* assert that the key was found unless this was speculative */
474 	ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE));
475 
476 	/*
477 	 * If there was a decryption / authentication error return EIO as
478 	 * the io_error. If this was not a speculative zio, create an ereport.
479 	 */
480 	if (ret == ECKSUM) {
481 		zio->io_error = SET_ERROR(EIO);
482 		if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
483 			spa_log_error(spa, &zio->io_bookmark);
484 			(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
485 			    spa, NULL, &zio->io_bookmark, zio, 0, 0);
486 		}
487 	} else {
488 		zio->io_error = ret;
489 	}
490 }
491 
492 /*
493  * ==========================================================================
494  * I/O parent/child relationships and pipeline interlocks
495  * ==========================================================================
496  */
497 zio_t *
zio_walk_parents(zio_t * cio,zio_link_t ** zl)498 zio_walk_parents(zio_t *cio, zio_link_t **zl)
499 {
500 	list_t *pl = &cio->io_parent_list;
501 
502 	*zl = (*zl == NULL) ? list_head(pl) : list_next(pl, *zl);
503 	if (*zl == NULL)
504 		return (NULL);
505 
506 	ASSERT((*zl)->zl_child == cio);
507 	return ((*zl)->zl_parent);
508 }
509 
510 zio_t *
zio_walk_children(zio_t * pio,zio_link_t ** zl)511 zio_walk_children(zio_t *pio, zio_link_t **zl)
512 {
513 	list_t *cl = &pio->io_child_list;
514 
515 	ASSERT(MUTEX_HELD(&pio->io_lock));
516 
517 	*zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl);
518 	if (*zl == NULL)
519 		return (NULL);
520 
521 	ASSERT((*zl)->zl_parent == pio);
522 	return ((*zl)->zl_child);
523 }
524 
525 zio_t *
zio_unique_parent(zio_t * cio)526 zio_unique_parent(zio_t *cio)
527 {
528 	zio_link_t *zl = NULL;
529 	zio_t *pio = zio_walk_parents(cio, &zl);
530 
531 	VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL);
532 	return (pio);
533 }
534 
535 void
zio_add_child(zio_t * pio,zio_t * cio)536 zio_add_child(zio_t *pio, zio_t *cio)
537 {
538 	zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
539 
540 	/*
541 	 * Logical I/Os can have logical, gang, or vdev children.
542 	 * Gang I/Os can have gang or vdev children.
543 	 * Vdev I/Os can only have vdev children.
544 	 * The following ASSERT captures all of these constraints.
545 	 */
546 	ASSERT3S(cio->io_child_type, <=, pio->io_child_type);
547 
548 	zl->zl_parent = pio;
549 	zl->zl_child = cio;
550 
551 	mutex_enter(&pio->io_lock);
552 	mutex_enter(&cio->io_lock);
553 
554 	ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
555 
556 	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
557 		pio->io_children[cio->io_child_type][w] += !cio->io_state[w];
558 
559 	list_insert_head(&pio->io_child_list, zl);
560 	list_insert_head(&cio->io_parent_list, zl);
561 
562 	pio->io_child_count++;
563 	cio->io_parent_count++;
564 
565 	mutex_exit(&cio->io_lock);
566 	mutex_exit(&pio->io_lock);
567 }
568 
569 static void
zio_remove_child(zio_t * pio,zio_t * cio,zio_link_t * zl)570 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl)
571 {
572 	ASSERT(zl->zl_parent == pio);
573 	ASSERT(zl->zl_child == cio);
574 
575 	mutex_enter(&pio->io_lock);
576 	mutex_enter(&cio->io_lock);
577 
578 	list_remove(&pio->io_child_list, zl);
579 	list_remove(&cio->io_parent_list, zl);
580 
581 	pio->io_child_count--;
582 	cio->io_parent_count--;
583 
584 	mutex_exit(&cio->io_lock);
585 	mutex_exit(&pio->io_lock);
586 
587 	kmem_cache_free(zio_link_cache, zl);
588 }
589 
590 static boolean_t
zio_wait_for_children(zio_t * zio,uint8_t childbits,enum zio_wait_type wait)591 zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
592 {
593 	boolean_t waiting = B_FALSE;
594 
595 	mutex_enter(&zio->io_lock);
596 	ASSERT(zio->io_stall == NULL);
597 	for (int c = 0; c < ZIO_CHILD_TYPES; c++) {
598 		if (!(ZIO_CHILD_BIT_IS_SET(childbits, c)))
599 			continue;
600 
601 		uint64_t *countp = &zio->io_children[c][wait];
602 		if (*countp != 0) {
603 			zio->io_stage >>= 1;
604 			ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN);
605 			zio->io_stall = countp;
606 			waiting = B_TRUE;
607 			break;
608 		}
609 	}
610 	mutex_exit(&zio->io_lock);
611 	return (waiting);
612 }
613 
614 static void
zio_notify_parent(zio_t * pio,zio_t * zio,enum zio_wait_type wait)615 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
616 {
617 	uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
618 	int *errorp = &pio->io_child_error[zio->io_child_type];
619 
620 	mutex_enter(&pio->io_lock);
621 	if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
622 		*errorp = zio_worst_error(*errorp, zio->io_error);
623 	pio->io_reexecute |= zio->io_reexecute;
624 	ASSERT3U(*countp, >, 0);
625 
626 	(*countp)--;
627 
628 	if (*countp == 0 && pio->io_stall == countp) {
629 		zio_taskq_type_t type =
630 		    pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE :
631 		    ZIO_TASKQ_INTERRUPT;
632 		pio->io_stall = NULL;
633 		mutex_exit(&pio->io_lock);
634 		/*
635 		 * Dispatch the parent zio in its own taskq so that
636 		 * the child can continue to make progress. This also
637 		 * prevents overflowing the stack when we have deeply nested
638 		 * parent-child relationships.
639 		 */
640 		zio_taskq_dispatch(pio, type, B_FALSE);
641 	} else {
642 		mutex_exit(&pio->io_lock);
643 	}
644 }
645 
646 static void
zio_inherit_child_errors(zio_t * zio,enum zio_child c)647 zio_inherit_child_errors(zio_t *zio, enum zio_child c)
648 {
649 	if (zio->io_child_error[c] != 0 && zio->io_error == 0)
650 		zio->io_error = zio->io_child_error[c];
651 }
652 
653 int
zio_bookmark_compare(const void * x1,const void * x2)654 zio_bookmark_compare(const void *x1, const void *x2)
655 {
656 	const zio_t *z1 = x1;
657 	const zio_t *z2 = x2;
658 
659 	if (z1->io_bookmark.zb_objset < z2->io_bookmark.zb_objset)
660 		return (-1);
661 	if (z1->io_bookmark.zb_objset > z2->io_bookmark.zb_objset)
662 		return (1);
663 
664 	if (z1->io_bookmark.zb_object < z2->io_bookmark.zb_object)
665 		return (-1);
666 	if (z1->io_bookmark.zb_object > z2->io_bookmark.zb_object)
667 		return (1);
668 
669 	if (z1->io_bookmark.zb_level < z2->io_bookmark.zb_level)
670 		return (-1);
671 	if (z1->io_bookmark.zb_level > z2->io_bookmark.zb_level)
672 		return (1);
673 
674 	if (z1->io_bookmark.zb_blkid < z2->io_bookmark.zb_blkid)
675 		return (-1);
676 	if (z1->io_bookmark.zb_blkid > z2->io_bookmark.zb_blkid)
677 		return (1);
678 
679 	if (z1 < z2)
680 		return (-1);
681 	if (z1 > z2)
682 		return (1);
683 
684 	return (0);
685 }
686 
687 /*
688  * ==========================================================================
689  * Create the various types of I/O (read, write, free, etc)
690  * ==========================================================================
691  */
692 static zio_t *
zio_create(zio_t * pio,spa_t * spa,uint64_t txg,const blkptr_t * bp,abd_t * data,uint64_t lsize,uint64_t psize,zio_done_func_t * done,void * private,zio_type_t type,zio_priority_t priority,enum zio_flag flags,vdev_t * vd,uint64_t offset,const zbookmark_phys_t * zb,enum zio_stage stage,enum zio_stage pipeline)693 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
694     abd_t *data, uint64_t lsize, uint64_t psize, zio_done_func_t *done,
695     void *private, zio_type_t type, zio_priority_t priority,
696     enum zio_flag flags, vdev_t *vd, uint64_t offset,
697     const zbookmark_phys_t *zb, enum zio_stage stage, enum zio_stage pipeline)
698 {
699 	zio_t *zio;
700 
701 	IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE);
702 	ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0);
703 	ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);
704 
705 	ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER));
706 	ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER));
707 	ASSERT(vd || stage == ZIO_STAGE_OPEN);
708 
709 	IMPLY(lsize != psize, (flags & ZIO_FLAG_RAW_COMPRESS) != 0);
710 
711 	zio = kmem_cache_alloc(zio_cache, KM_SLEEP);
712 	bzero(zio, sizeof (zio_t));
713 
714 	mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL);
715 	cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);
716 
717 	list_create(&zio->io_parent_list, sizeof (zio_link_t),
718 	    offsetof(zio_link_t, zl_parent_node));
719 	list_create(&zio->io_child_list, sizeof (zio_link_t),
720 	    offsetof(zio_link_t, zl_child_node));
721 	metaslab_trace_init(&zio->io_alloc_list);
722 
723 	if (vd != NULL)
724 		zio->io_child_type = ZIO_CHILD_VDEV;
725 	else if (flags & ZIO_FLAG_GANG_CHILD)
726 		zio->io_child_type = ZIO_CHILD_GANG;
727 	else if (flags & ZIO_FLAG_DDT_CHILD)
728 		zio->io_child_type = ZIO_CHILD_DDT;
729 	else
730 		zio->io_child_type = ZIO_CHILD_LOGICAL;
731 
732 	if (bp != NULL) {
733 		zio->io_bp = (blkptr_t *)bp;
734 		zio->io_bp_copy = *bp;
735 		zio->io_bp_orig = *bp;
736 		if (type != ZIO_TYPE_WRITE ||
737 		    zio->io_child_type == ZIO_CHILD_DDT)
738 			zio->io_bp = &zio->io_bp_copy;	/* so caller can free */
739 		if (zio->io_child_type == ZIO_CHILD_LOGICAL)
740 			zio->io_logical = zio;
741 		if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
742 			pipeline |= ZIO_GANG_STAGES;
743 	}
744 
745 	zio->io_spa = spa;
746 	zio->io_txg = txg;
747 	zio->io_done = done;
748 	zio->io_private = private;
749 	zio->io_type = type;
750 	zio->io_priority = priority;
751 	zio->io_vd = vd;
752 	zio->io_offset = offset;
753 	zio->io_orig_abd = zio->io_abd = data;
754 	zio->io_orig_size = zio->io_size = psize;
755 	zio->io_lsize = lsize;
756 	zio->io_orig_flags = zio->io_flags = flags;
757 	zio->io_orig_stage = zio->io_stage = stage;
758 	zio->io_orig_pipeline = zio->io_pipeline = pipeline;
759 	zio->io_pipeline_trace = ZIO_STAGE_OPEN;
760 
761 	zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY);
762 	zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE);
763 
764 	if (zb != NULL)
765 		zio->io_bookmark = *zb;
766 
767 	if (pio != NULL) {
768 		if (zio->io_metaslab_class == NULL)
769 			zio->io_metaslab_class = pio->io_metaslab_class;
770 		if (zio->io_logical == NULL)
771 			zio->io_logical = pio->io_logical;
772 		if (zio->io_child_type == ZIO_CHILD_GANG)
773 			zio->io_gang_leader = pio->io_gang_leader;
774 		zio_add_child(pio, zio);
775 	}
776 
777 	return (zio);
778 }
779 
780 static void
zio_destroy(zio_t * zio)781 zio_destroy(zio_t *zio)
782 {
783 	metaslab_trace_fini(&zio->io_alloc_list);
784 	list_destroy(&zio->io_parent_list);
785 	list_destroy(&zio->io_child_list);
786 	mutex_destroy(&zio->io_lock);
787 	cv_destroy(&zio->io_cv);
788 	kmem_cache_free(zio_cache, zio);
789 }
790 
791 zio_t *
zio_null(zio_t * pio,spa_t * spa,vdev_t * vd,zio_done_func_t * done,void * private,enum zio_flag flags)792 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done,
793     void *private, enum zio_flag flags)
794 {
795 	zio_t *zio;
796 
797 	zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
798 	    ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
799 	    ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE);
800 
801 	return (zio);
802 }
803 
804 zio_t *
zio_root(spa_t * spa,zio_done_func_t * done,void * private,enum zio_flag flags)805 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags)
806 {
807 	return (zio_null(NULL, spa, NULL, done, private, flags));
808 }
809 
810 void
zfs_blkptr_verify(spa_t * spa,const blkptr_t * bp)811 zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp)
812 {
813 	if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) {
814 		zfs_panic_recover("blkptr at %p has invalid TYPE %llu",
815 		    bp, (longlong_t)BP_GET_TYPE(bp));
816 	}
817 	if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS ||
818 	    BP_GET_CHECKSUM(bp) <= ZIO_CHECKSUM_ON) {
819 		zfs_panic_recover("blkptr at %p has invalid CHECKSUM %llu",
820 		    bp, (longlong_t)BP_GET_CHECKSUM(bp));
821 	}
822 	if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS ||
823 	    BP_GET_COMPRESS(bp) <= ZIO_COMPRESS_ON) {
824 		zfs_panic_recover("blkptr at %p has invalid COMPRESS %llu",
825 		    bp, (longlong_t)BP_GET_COMPRESS(bp));
826 	}
827 	if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) {
828 		zfs_panic_recover("blkptr at %p has invalid LSIZE %llu",
829 		    bp, (longlong_t)BP_GET_LSIZE(bp));
830 	}
831 	if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) {
832 		zfs_panic_recover("blkptr at %p has invalid PSIZE %llu",
833 		    bp, (longlong_t)BP_GET_PSIZE(bp));
834 	}
835 
836 	if (BP_IS_EMBEDDED(bp)) {
837 		if (BPE_GET_ETYPE(bp) > NUM_BP_EMBEDDED_TYPES) {
838 			zfs_panic_recover("blkptr at %p has invalid ETYPE %llu",
839 			    bp, (longlong_t)BPE_GET_ETYPE(bp));
840 		}
841 	}
842 
843 	/*
844 	 * Do not verify individual DVAs if the config is not trusted. This
845 	 * will be done once the zio is executed in vdev_mirror_map_alloc.
846 	 */
847 	if (!spa->spa_trust_config)
848 		return;
849 
850 	/*
851 	 * Pool-specific checks.
852 	 *
853 	 * Note: it would be nice to verify that the blk_birth and
854 	 * BP_PHYSICAL_BIRTH() are not too large.  However, spa_freeze()
855 	 * allows the birth time of log blocks (and dmu_sync()-ed blocks
856 	 * that are in the log) to be arbitrarily large.
857 	 */
858 	for (int i = 0; i < BP_GET_NDVAS(bp); i++) {
859 		uint64_t vdevid = DVA_GET_VDEV(&bp->blk_dva[i]);
860 		if (vdevid >= spa->spa_root_vdev->vdev_children) {
861 			zfs_panic_recover("blkptr at %p DVA %u has invalid "
862 			    "VDEV %llu",
863 			    bp, i, (longlong_t)vdevid);
864 			continue;
865 		}
866 		vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
867 		if (vd == NULL) {
868 			zfs_panic_recover("blkptr at %p DVA %u has invalid "
869 			    "VDEV %llu",
870 			    bp, i, (longlong_t)vdevid);
871 			continue;
872 		}
873 		if (vd->vdev_ops == &vdev_hole_ops) {
874 			zfs_panic_recover("blkptr at %p DVA %u has hole "
875 			    "VDEV %llu",
876 			    bp, i, (longlong_t)vdevid);
877 			continue;
878 		}
879 		if (vd->vdev_ops == &vdev_missing_ops) {
880 			/*
881 			 * "missing" vdevs are valid during import, but we
882 			 * don't have their detailed info (e.g. asize), so
883 			 * we can't perform any more checks on them.
884 			 */
885 			continue;
886 		}
887 		uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]);
888 		uint64_t asize = DVA_GET_ASIZE(&bp->blk_dva[i]);
889 		if (BP_IS_GANG(bp))
890 			asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE);
891 		if (offset + asize > vd->vdev_asize) {
892 			zfs_panic_recover("blkptr at %p DVA %u has invalid "
893 			    "OFFSET %llu",
894 			    bp, i, (longlong_t)offset);
895 		}
896 	}
897 }
898 
899 boolean_t
zfs_dva_valid(spa_t * spa,const dva_t * dva,const blkptr_t * bp)900 zfs_dva_valid(spa_t *spa, const dva_t *dva, const blkptr_t *bp)
901 {
902 	uint64_t vdevid = DVA_GET_VDEV(dva);
903 
904 	if (vdevid >= spa->spa_root_vdev->vdev_children)
905 		return (B_FALSE);
906 
907 	vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
908 	if (vd == NULL)
909 		return (B_FALSE);
910 
911 	if (vd->vdev_ops == &vdev_hole_ops)
912 		return (B_FALSE);
913 
914 	if (vd->vdev_ops == &vdev_missing_ops) {
915 		return (B_FALSE);
916 	}
917 
918 	uint64_t offset = DVA_GET_OFFSET(dva);
919 	uint64_t asize = DVA_GET_ASIZE(dva);
920 
921 	if (BP_IS_GANG(bp))
922 		asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE);
923 	if (offset + asize > vd->vdev_asize)
924 		return (B_FALSE);
925 
926 	return (B_TRUE);
927 }
928 
929 zio_t *
zio_read(zio_t * pio,spa_t * spa,const blkptr_t * bp,abd_t * data,uint64_t size,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,const zbookmark_phys_t * zb)930 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
931     abd_t *data, uint64_t size, zio_done_func_t *done, void *private,
932     zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb)
933 {
934 	zio_t *zio;
935 
936 	zfs_blkptr_verify(spa, bp);
937 
938 	zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp,
939 	    data, size, size, done, private,
940 	    ZIO_TYPE_READ, priority, flags, NULL, 0, zb,
941 	    ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
942 	    ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE);
943 
944 	return (zio);
945 }
946 
947 zio_t *
zio_write(zio_t * pio,spa_t * spa,uint64_t txg,blkptr_t * bp,abd_t * data,uint64_t lsize,uint64_t psize,const zio_prop_t * zp,zio_done_func_t * ready,zio_done_func_t * children_ready,zio_done_func_t * physdone,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,const zbookmark_phys_t * zb)948 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
949     abd_t *data, uint64_t lsize, uint64_t psize, const zio_prop_t *zp,
950     zio_done_func_t *ready, zio_done_func_t *children_ready,
951     zio_done_func_t *physdone, zio_done_func_t *done,
952     void *private, zio_priority_t priority, enum zio_flag flags,
953     const zbookmark_phys_t *zb)
954 {
955 	zio_t *zio;
956 
957 	ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF &&
958 	    zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS &&
959 	    zp->zp_compress >= ZIO_COMPRESS_OFF &&
960 	    zp->zp_compress < ZIO_COMPRESS_FUNCTIONS &&
961 	    DMU_OT_IS_VALID(zp->zp_type) &&
962 	    zp->zp_level < 32 &&
963 	    zp->zp_copies > 0 &&
964 	    zp->zp_copies <= spa_max_replication(spa));
965 
966 	zio = zio_create(pio, spa, txg, bp, data, lsize, psize, done, private,
967 	    ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb,
968 	    ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ?
969 	    ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
970 
971 	zio->io_ready = ready;
972 	zio->io_children_ready = children_ready;
973 	zio->io_physdone = physdone;
974 	zio->io_prop = *zp;
975 
976 	/*
977 	 * Data can be NULL if we are going to call zio_write_override() to
978 	 * provide the already-allocated BP.  But we may need the data to
979 	 * verify a dedup hit (if requested).  In this case, don't try to
980 	 * dedup (just take the already-allocated BP verbatim). Encrypted
981 	 * dedup blocks need data as well so we also disable dedup in this
982 	 * case.
983 	 */
984 	if (data == NULL &&
985 	    (zio->io_prop.zp_dedup_verify || zio->io_prop.zp_encrypt)) {
986 		zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE;
987 	}
988 
989 	return (zio);
990 }
991 
992 zio_t *
zio_rewrite(zio_t * pio,spa_t * spa,uint64_t txg,blkptr_t * bp,abd_t * data,uint64_t size,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,zbookmark_phys_t * zb)993 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, abd_t *data,
994     uint64_t size, zio_done_func_t *done, void *private,
995     zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb)
996 {
997 	zio_t *zio;
998 
999 	zio = zio_create(pio, spa, txg, bp, data, size, size, done, private,
1000 	    ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_IO_REWRITE, NULL, 0, zb,
1001 	    ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE);
1002 
1003 	return (zio);
1004 }
1005 
1006 void
zio_write_override(zio_t * zio,blkptr_t * bp,int copies,boolean_t nopwrite)1007 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite)
1008 {
1009 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
1010 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1011 	ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
1012 	ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
1013 
1014 	/*
1015 	 * We must reset the io_prop to match the values that existed
1016 	 * when the bp was first written by dmu_sync() keeping in mind
1017 	 * that nopwrite and dedup are mutually exclusive.
1018 	 */
1019 	zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;
1020 	zio->io_prop.zp_nopwrite = nopwrite;
1021 	zio->io_prop.zp_copies = copies;
1022 	zio->io_bp_override = bp;
1023 }
1024 
1025 void
zio_free(spa_t * spa,uint64_t txg,const blkptr_t * bp)1026 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
1027 {
1028 
1029 	zfs_blkptr_verify(spa, bp);
1030 
1031 	/*
1032 	 * The check for EMBEDDED is a performance optimization.  We
1033 	 * process the free here (by ignoring it) rather than
1034 	 * putting it on the list and then processing it in zio_free_sync().
1035 	 */
1036 	if (BP_IS_EMBEDDED(bp))
1037 		return;
1038 	metaslab_check_free(spa, bp);
1039 
1040 	/*
1041 	 * Frees that are for the currently-syncing txg, are not going to be
1042 	 * deferred, and which will not need to do a read (i.e. not GANG or
1043 	 * DEDUP), can be processed immediately.  Otherwise, put them on the
1044 	 * in-memory list for later processing.
1045 	 *
1046 	 * Note that we only defer frees after zfs_sync_pass_deferred_free
1047 	 * when the log space map feature is disabled. [see relevant comment
1048 	 * in spa_sync_iterate_to_convergence()]
1049 	 */
1050 	if (BP_IS_GANG(bp) ||
1051 	    BP_GET_DEDUP(bp) ||
1052 	    txg != spa->spa_syncing_txg ||
1053 	    (spa_sync_pass(spa) >= zfs_sync_pass_deferred_free &&
1054 	    !spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))) {
1055 		bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
1056 	} else {
1057 		VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 0)));
1058 	}
1059 }
1060 
1061 zio_t *
zio_free_sync(zio_t * pio,spa_t * spa,uint64_t txg,const blkptr_t * bp,enum zio_flag flags)1062 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
1063     enum zio_flag flags)
1064 {
1065 	zio_t *zio;
1066 	enum zio_stage stage = ZIO_FREE_PIPELINE;
1067 
1068 	ASSERT(!BP_IS_HOLE(bp));
1069 	ASSERT(spa_syncing_txg(spa) == txg);
1070 
1071 	if (BP_IS_EMBEDDED(bp))
1072 		return (zio_null(pio, spa, NULL, NULL, NULL, 0));
1073 
1074 	metaslab_check_free(spa, bp);
1075 	arc_freed(spa, bp);
1076 	dsl_scan_freed(spa, bp);
1077 
1078 	/*
1079 	 * GANG and DEDUP blocks can induce a read (for the gang block header,
1080 	 * or the DDT), so issue them asynchronously so that this thread is
1081 	 * not tied up.
1082 	 */
1083 	if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp))
1084 		stage |= ZIO_STAGE_ISSUE_ASYNC;
1085 
1086 	zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
1087 	    BP_GET_PSIZE(bp), NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW,
1088 	    flags, NULL, 0, NULL, ZIO_STAGE_OPEN, stage);
1089 
1090 	return (zio);
1091 }
1092 
1093 zio_t *
zio_claim(zio_t * pio,spa_t * spa,uint64_t txg,const blkptr_t * bp,zio_done_func_t * done,void * private,enum zio_flag flags)1094 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
1095     zio_done_func_t *done, void *private, enum zio_flag flags)
1096 {
1097 	zio_t *zio;
1098 
1099 	zfs_blkptr_verify(spa, bp);
1100 
1101 	if (BP_IS_EMBEDDED(bp))
1102 		return (zio_null(pio, spa, NULL, NULL, NULL, 0));
1103 
1104 	/*
1105 	 * A claim is an allocation of a specific block.  Claims are needed
1106 	 * to support immediate writes in the intent log.  The issue is that
1107 	 * immediate writes contain committed data, but in a txg that was
1108 	 * *not* committed.  Upon opening the pool after an unclean shutdown,
1109 	 * the intent log claims all blocks that contain immediate write data
1110 	 * so that the SPA knows they're in use.
1111 	 *
1112 	 * All claims *must* be resolved in the first txg -- before the SPA
1113 	 * starts allocating blocks -- so that nothing is allocated twice.
1114 	 * If txg == 0 we just verify that the block is claimable.
1115 	 */
1116 	ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <,
1117 	    spa_min_claim_txg(spa));
1118 	ASSERT(txg == spa_min_claim_txg(spa) || txg == 0);
1119 	ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa));	/* zdb(8) */
1120 
1121 	zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
1122 	    BP_GET_PSIZE(bp), done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW,
1123 	    flags, NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE);
1124 	ASSERT0(zio->io_queued_timestamp);
1125 
1126 	return (zio);
1127 }
1128 
1129 zio_t *
zio_ioctl(zio_t * pio,spa_t * spa,vdev_t * vd,int cmd,zio_done_func_t * done,void * private,enum zio_flag flags)1130 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd,
1131     zio_done_func_t *done, void *private, enum zio_flag flags)
1132 {
1133 	zio_t *zio;
1134 	int c;
1135 
1136 	if (vd->vdev_children == 0) {
1137 		zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
1138 		    ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
1139 		    ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE);
1140 
1141 		zio->io_cmd = cmd;
1142 	} else {
1143 		zio = zio_null(pio, spa, NULL, NULL, NULL, flags);
1144 
1145 		for (c = 0; c < vd->vdev_children; c++)
1146 			zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd,
1147 			    done, private, flags));
1148 	}
1149 
1150 	return (zio);
1151 }
1152 
1153 zio_t *
zio_trim(zio_t * pio,vdev_t * vd,uint64_t offset,uint64_t size,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,enum trim_flag trim_flags)1154 zio_trim(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
1155     zio_done_func_t *done, void *private, zio_priority_t priority,
1156     enum zio_flag flags, enum trim_flag trim_flags)
1157 {
1158 	zio_t *zio;
1159 
1160 	ASSERT0(vd->vdev_children);
1161 	ASSERT0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
1162 	ASSERT0(P2PHASE(size, 1ULL << vd->vdev_ashift));
1163 	ASSERT3U(size, !=, 0);
1164 
1165 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, NULL, size, size, done,
1166 	    private, ZIO_TYPE_TRIM, priority, flags | ZIO_FLAG_PHYSICAL,
1167 	    vd, offset, NULL, ZIO_STAGE_OPEN, ZIO_TRIM_PIPELINE);
1168 	zio->io_trim_flags = trim_flags;
1169 
1170 	return (zio);
1171 }
1172 
1173 zio_t *
zio_read_phys(zio_t * pio,vdev_t * vd,uint64_t offset,uint64_t size,abd_t * data,int checksum,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,boolean_t labels)1174 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
1175     abd_t *data, int checksum, zio_done_func_t *done, void *private,
1176     zio_priority_t priority, enum zio_flag flags, boolean_t labels)
1177 {
1178 	zio_t *zio;
1179 
1180 	ASSERT(vd->vdev_children == 0);
1181 	ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
1182 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
1183 	ASSERT3U(offset + size, <=, vd->vdev_psize);
1184 
1185 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
1186 	    private, ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd,
1187 	    offset, NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);
1188 
1189 	zio->io_prop.zp_checksum = checksum;
1190 
1191 	return (zio);
1192 }
1193 
1194 zio_t *
zio_write_phys(zio_t * pio,vdev_t * vd,uint64_t offset,uint64_t size,abd_t * data,int checksum,zio_done_func_t * done,void * private,zio_priority_t priority,enum zio_flag flags,boolean_t labels)1195 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
1196     abd_t *data, int checksum, zio_done_func_t *done, void *private,
1197     zio_priority_t priority, enum zio_flag flags, boolean_t labels)
1198 {
1199 	zio_t *zio;
1200 
1201 	ASSERT(vd->vdev_children == 0);
1202 	ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
1203 	    offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
1204 	ASSERT3U(offset + size, <=, vd->vdev_psize);
1205 
1206 	zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
1207 	    private, ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd,
1208 	    offset, NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);
1209 
1210 	zio->io_prop.zp_checksum = checksum;
1211 
1212 	if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
1213 		/*
1214 		 * zec checksums are necessarily destructive -- they modify
1215 		 * the end of the write buffer to hold the verifier/checksum.
1216 		 * Therefore, we must make a local copy in case the data is
1217 		 * being written to multiple places in parallel.
1218 		 */
1219 		abd_t *wbuf = abd_alloc_sametype(data, size);
1220 		abd_copy(wbuf, data, size);
1221 
1222 		zio_push_transform(zio, wbuf, size, size, NULL);
1223 	}
1224 
1225 	return (zio);
1226 }
1227 
1228 /*
1229  * Create a child I/O to do some work for us.
1230  */
1231 zio_t *
zio_vdev_child_io(zio_t * pio,blkptr_t * bp,vdev_t * vd,uint64_t offset,abd_t * data,uint64_t size,int type,zio_priority_t priority,enum zio_flag flags,zio_done_func_t * done,void * private)1232 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset,
1233     abd_t *data, uint64_t size, int type, zio_priority_t priority,
1234     enum zio_flag flags, zio_done_func_t *done, void *private)
1235 {
1236 	enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE;
1237 	zio_t *zio;
1238 
1239 	/*
1240 	 * vdev child I/Os do not propagate their error to the parent.
1241 	 * Therefore, for correct operation the caller *must* check for
1242 	 * and handle the error in the child i/o's done callback.
1243 	 * The only exceptions are i/os that we don't care about
1244 	 * (OPTIONAL or REPAIR).
1245 	 */
1246 	ASSERT((flags & ZIO_FLAG_OPTIONAL) || (flags & ZIO_FLAG_IO_REPAIR) ||
1247 	    done != NULL);
1248 
1249 	if (type == ZIO_TYPE_READ && bp != NULL) {
1250 		/*
1251 		 * If we have the bp, then the child should perform the
1252 		 * checksum and the parent need not.  This pushes error
1253 		 * detection as close to the leaves as possible and
1254 		 * eliminates redundant checksums in the interior nodes.
1255 		 */
1256 		pipeline |= ZIO_STAGE_CHECKSUM_VERIFY;
1257 		pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
1258 	}
1259 
1260 	if (vd->vdev_ops->vdev_op_leaf) {
1261 		ASSERT0(vd->vdev_children);
1262 		offset += VDEV_LABEL_START_SIZE;
1263 	}
1264 
1265 	flags |= ZIO_VDEV_CHILD_FLAGS(pio);
1266 
1267 	/*
1268 	 * If we've decided to do a repair, the write is not speculative --
1269 	 * even if the original read was.
1270 	 */
1271 	if (flags & ZIO_FLAG_IO_REPAIR)
1272 		flags &= ~ZIO_FLAG_SPECULATIVE;
1273 
1274 	/*
1275 	 * If we're creating a child I/O that is not associated with a
1276 	 * top-level vdev, then the child zio is not an allocating I/O.
1277 	 * If this is a retried I/O then we ignore it since we will
1278 	 * have already processed the original allocating I/O.
1279 	 */
1280 	if (flags & ZIO_FLAG_IO_ALLOCATING &&
1281 	    (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) {
1282 		ASSERT(pio->io_metaslab_class != NULL);
1283 		ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled);
1284 		ASSERT(type == ZIO_TYPE_WRITE);
1285 		ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE);
1286 		ASSERT(!(flags & ZIO_FLAG_IO_REPAIR));
1287 		ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) ||
1288 		    pio->io_child_type == ZIO_CHILD_GANG);
1289 
1290 		flags &= ~ZIO_FLAG_IO_ALLOCATING;
1291 	}
1292 
1293 	zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size,
1294 	    done, private, type, priority, flags, vd, offset, &pio->io_bookmark,
1295 	    ZIO_STAGE_VDEV_IO_START >> 1, pipeline);
1296 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
1297 
1298 	zio->io_physdone = pio->io_physdone;
1299 	if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL)
1300 		zio->io_logical->io_phys_children++;
1301 
1302 	return (zio);
1303 }
1304 
1305 zio_t *
zio_vdev_delegated_io(vdev_t * vd,uint64_t offset,abd_t * data,uint64_t size,zio_type_t type,zio_priority_t priority,enum zio_flag flags,zio_done_func_t * done,void * private)1306 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, abd_t *data, uint64_t size,
1307     zio_type_t type, zio_priority_t priority, enum zio_flag flags,
1308     zio_done_func_t *done, void *private)
1309 {
1310 	zio_t *zio;
1311 
1312 	ASSERT(vd->vdev_ops->vdev_op_leaf);
1313 
1314 	zio = zio_create(NULL, vd->vdev_spa, 0, NULL,
1315 	    data, size, size, done, private, type, priority,
1316 	    flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED,
1317 	    vd, offset, NULL,
1318 	    ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE);
1319 
1320 	return (zio);
1321 }
1322 
1323 void
zio_flush(zio_t * zio,vdev_t * vd)1324 zio_flush(zio_t *zio, vdev_t *vd)
1325 {
1326 	zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE,
1327 	    NULL, NULL,
1328 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY));
1329 }
1330 
1331 void
zio_shrink(zio_t * zio,uint64_t size)1332 zio_shrink(zio_t *zio, uint64_t size)
1333 {
1334 	ASSERT3P(zio->io_executor, ==, NULL);
1335 	ASSERT3P(zio->io_orig_size, ==, zio->io_size);
1336 	ASSERT3U(size, <=, zio->io_size);
1337 
1338 	/*
1339 	 * We don't shrink for raidz because of problems with the
1340 	 * reconstruction when reading back less than the block size.
1341 	 * Note, BP_IS_RAIDZ() assumes no compression.
1342 	 */
1343 	ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF);
1344 	if (!BP_IS_RAIDZ(zio->io_bp)) {
1345 		/* we are not doing a raw write */
1346 		ASSERT3U(zio->io_size, ==, zio->io_lsize);
1347 		zio->io_orig_size = zio->io_size = zio->io_lsize = size;
1348 	}
1349 }
1350 
1351 /*
1352  * ==========================================================================
1353  * Prepare to read and write logical blocks
1354  * ==========================================================================
1355  */
1356 
1357 static int
zio_read_bp_init(zio_t * zio)1358 zio_read_bp_init(zio_t *zio)
1359 {
1360 	blkptr_t *bp = zio->io_bp;
1361 	uint64_t psize =
1362 	    BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp);
1363 
1364 	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
1365 
1366 	if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
1367 	    zio->io_child_type == ZIO_CHILD_LOGICAL &&
1368 	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
1369 		zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),
1370 		    psize, psize, zio_decompress);
1371 	}
1372 
1373 	if (((BP_IS_PROTECTED(bp) && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) ||
1374 	    BP_HAS_INDIRECT_MAC_CKSUM(bp)) &&
1375 	    zio->io_child_type == ZIO_CHILD_LOGICAL) {
1376 		zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize),
1377 		    psize, psize, zio_decrypt);
1378 	}
1379 
1380 	if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) {
1381 		int psize = BPE_GET_PSIZE(bp);
1382 		void *data = abd_borrow_buf(zio->io_abd, psize);
1383 
1384 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1385 		decode_embedded_bp_compressed(bp, data);
1386 		abd_return_buf_copy(zio->io_abd, data, psize);
1387 	} else {
1388 		ASSERT(!BP_IS_EMBEDDED(bp));
1389 		ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
1390 	}
1391 
1392 	if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0)
1393 		zio->io_flags |= ZIO_FLAG_DONT_CACHE;
1394 
1395 	if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP)
1396 		zio->io_flags |= ZIO_FLAG_DONT_CACHE;
1397 
1398 	if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
1399 		zio->io_pipeline = ZIO_DDT_READ_PIPELINE;
1400 
1401 	return (ZIO_PIPELINE_CONTINUE);
1402 }
1403 
1404 static int
zio_write_bp_init(zio_t * zio)1405 zio_write_bp_init(zio_t *zio)
1406 {
1407 	if (!IO_IS_ALLOCATING(zio))
1408 		return (ZIO_PIPELINE_CONTINUE);
1409 
1410 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
1411 
1412 	if (zio->io_bp_override) {
1413 		blkptr_t *bp = zio->io_bp;
1414 		zio_prop_t *zp = &zio->io_prop;
1415 
1416 		ASSERT(bp->blk_birth != zio->io_txg);
1417 		ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
1418 
1419 		*bp = *zio->io_bp_override;
1420 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1421 
1422 		if (BP_IS_EMBEDDED(bp))
1423 			return (ZIO_PIPELINE_CONTINUE);
1424 
1425 		/*
1426 		 * If we've been overridden and nopwrite is set then
1427 		 * set the flag accordingly to indicate that a nopwrite
1428 		 * has already occurred.
1429 		 */
1430 		if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) {
1431 			ASSERT(!zp->zp_dedup);
1432 			ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);
1433 			zio->io_flags |= ZIO_FLAG_NOPWRITE;
1434 			return (ZIO_PIPELINE_CONTINUE);
1435 		}
1436 
1437 		ASSERT(!zp->zp_nopwrite);
1438 
1439 		if (BP_IS_HOLE(bp) || !zp->zp_dedup)
1440 			return (ZIO_PIPELINE_CONTINUE);
1441 
1442 		ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
1443 		    ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);
1444 
1445 		if (BP_GET_CHECKSUM(bp) == zp->zp_checksum &&
1446 		    !zp->zp_encrypt) {
1447 			BP_SET_DEDUP(bp, 1);
1448 			zio->io_pipeline |= ZIO_STAGE_DDT_WRITE;
1449 			return (ZIO_PIPELINE_CONTINUE);
1450 		}
1451 
1452 		/*
1453 		 * We were unable to handle this as an override bp, treat
1454 		 * it as a regular write I/O.
1455 		 */
1456 		zio->io_bp_override = NULL;
1457 		*bp = zio->io_bp_orig;
1458 		zio->io_pipeline = zio->io_orig_pipeline;
1459 	}
1460 
1461 	return (ZIO_PIPELINE_CONTINUE);
1462 }
1463 
1464 static int
zio_write_compress(zio_t * zio)1465 zio_write_compress(zio_t *zio)
1466 {
1467 	spa_t *spa = zio->io_spa;
1468 	zio_prop_t *zp = &zio->io_prop;
1469 	enum zio_compress compress = zp->zp_compress;
1470 	blkptr_t *bp = zio->io_bp;
1471 	uint64_t lsize = zio->io_lsize;
1472 	uint64_t psize = zio->io_size;
1473 	int pass = 1;
1474 
1475 	/*
1476 	 * If our children haven't all reached the ready stage,
1477 	 * wait for them and then repeat this pipeline stage.
1478 	 */
1479 	if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT |
1480 	    ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) {
1481 		return (ZIO_PIPELINE_STOP);
1482 	}
1483 
1484 	if (!IO_IS_ALLOCATING(zio))
1485 		return (ZIO_PIPELINE_CONTINUE);
1486 
1487 	if (zio->io_children_ready != NULL) {
1488 		/*
1489 		 * Now that all our children are ready, run the callback
1490 		 * associated with this zio in case it wants to modify the
1491 		 * data to be written.
1492 		 */
1493 		ASSERT3U(zp->zp_level, >, 0);
1494 		zio->io_children_ready(zio);
1495 	}
1496 
1497 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
1498 	ASSERT(zio->io_bp_override == NULL);
1499 
1500 	if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) {
1501 		/*
1502 		 * We're rewriting an existing block, which means we're
1503 		 * working on behalf of spa_sync().  For spa_sync() to
1504 		 * converge, it must eventually be the case that we don't
1505 		 * have to allocate new blocks.  But compression changes
1506 		 * the blocksize, which forces a reallocate, and makes
1507 		 * convergence take longer.  Therefore, after the first
1508 		 * few passes, stop compressing to ensure convergence.
1509 		 */
1510 		pass = spa_sync_pass(spa);
1511 
1512 		ASSERT(zio->io_txg == spa_syncing_txg(spa));
1513 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1514 		ASSERT(!BP_GET_DEDUP(bp));
1515 
1516 		if (pass >= zfs_sync_pass_dont_compress)
1517 			compress = ZIO_COMPRESS_OFF;
1518 
1519 		/* Make sure someone doesn't change their mind on overwrites */
1520 		ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp),
1521 		    spa_max_replication(spa)) == BP_GET_NDVAS(bp));
1522 	}
1523 
1524 	/* If it's a compressed write that is not raw, compress the buffer. */
1525 	if (compress != ZIO_COMPRESS_OFF &&
1526 	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
1527 		void *cbuf = zio_buf_alloc(lsize);
1528 		psize = zio_compress_data(compress, zio->io_abd, cbuf, lsize);
1529 		if (psize == 0 || psize == lsize) {
1530 			compress = ZIO_COMPRESS_OFF;
1531 			zio_buf_free(cbuf, lsize);
1532 		} else if (!zp->zp_dedup && !zp->zp_encrypt &&
1533 		    psize <= BPE_PAYLOAD_SIZE &&
1534 		    zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&
1535 		    spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) {
1536 			encode_embedded_bp_compressed(bp,
1537 			    cbuf, compress, lsize, psize);
1538 			BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA);
1539 			BP_SET_TYPE(bp, zio->io_prop.zp_type);
1540 			BP_SET_LEVEL(bp, zio->io_prop.zp_level);
1541 			zio_buf_free(cbuf, lsize);
1542 			bp->blk_birth = zio->io_txg;
1543 			zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1544 			ASSERT(spa_feature_is_active(spa,
1545 			    SPA_FEATURE_EMBEDDED_DATA));
1546 			return (ZIO_PIPELINE_CONTINUE);
1547 		} else {
1548 			/*
1549 			 * Round up compressed size up to the ashift
1550 			 * of the smallest-ashift device, and zero the tail.
1551 			 * This ensures that the compressed size of the BP
1552 			 * (and thus compressratio property) are correct,
1553 			 * in that we charge for the padding used to fill out
1554 			 * the last sector.
1555 			 */
1556 			ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
1557 			size_t rounded = (size_t)P2ROUNDUP(psize,
1558 			    1ULL << spa->spa_min_ashift);
1559 			if (rounded >= lsize) {
1560 				compress = ZIO_COMPRESS_OFF;
1561 				zio_buf_free(cbuf, lsize);
1562 				psize = lsize;
1563 			} else {
1564 				abd_t *cdata = abd_get_from_buf(cbuf, lsize);
1565 				abd_take_ownership_of_buf(cdata, B_TRUE);
1566 				abd_zero_off(cdata, psize, rounded - psize);
1567 				psize = rounded;
1568 				zio_push_transform(zio, cdata,
1569 				    psize, lsize, NULL);
1570 			}
1571 		}
1572 
1573 		/*
1574 		 * We were unable to handle this as an override bp, treat
1575 		 * it as a regular write I/O.
1576 		 */
1577 		zio->io_bp_override = NULL;
1578 		*bp = zio->io_bp_orig;
1579 		zio->io_pipeline = zio->io_orig_pipeline;
1580 
1581 	} else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 &&
1582 	    zp->zp_type == DMU_OT_DNODE) {
1583 		/*
1584 		 * The DMU actually relies on the zio layer's compression
1585 		 * to free metadnode blocks that have had all contained
1586 		 * dnodes freed. As a result, even when doing a raw
1587 		 * receive, we must check whether the block can be compressed
1588 		 * to a hole.
1589 		 */
1590 		psize = zio_compress_data(ZIO_COMPRESS_EMPTY,
1591 		    zio->io_abd, NULL, lsize);
1592 		if (psize == 0)
1593 			compress = ZIO_COMPRESS_OFF;
1594 	} else {
1595 		ASSERT3U(psize, !=, 0);
1596 	}
1597 
1598 	/*
1599 	 * The final pass of spa_sync() must be all rewrites, but the first
1600 	 * few passes offer a trade-off: allocating blocks defers convergence,
1601 	 * but newly allocated blocks are sequential, so they can be written
1602 	 * to disk faster.  Therefore, we allow the first few passes of
1603 	 * spa_sync() to allocate new blocks, but force rewrites after that.
1604 	 * There should only be a handful of blocks after pass 1 in any case.
1605 	 */
1606 	if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg &&
1607 	    BP_GET_PSIZE(bp) == psize &&
1608 	    pass >= zfs_sync_pass_rewrite) {
1609 		VERIFY3U(psize, !=, 0);
1610 		enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;
1611 		zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages;
1612 		zio->io_flags |= ZIO_FLAG_IO_REWRITE;
1613 	} else {
1614 		BP_ZERO(bp);
1615 		zio->io_pipeline = ZIO_WRITE_PIPELINE;
1616 	}
1617 
1618 	if (psize == 0) {
1619 		if (zio->io_bp_orig.blk_birth != 0 &&
1620 		    spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
1621 			BP_SET_LSIZE(bp, lsize);
1622 			BP_SET_TYPE(bp, zp->zp_type);
1623 			BP_SET_LEVEL(bp, zp->zp_level);
1624 			BP_SET_BIRTH(bp, zio->io_txg, 0);
1625 		}
1626 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
1627 	} else {
1628 		ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER);
1629 		BP_SET_LSIZE(bp, lsize);
1630 		BP_SET_TYPE(bp, zp->zp_type);
1631 		BP_SET_LEVEL(bp, zp->zp_level);
1632 		BP_SET_PSIZE(bp, psize);
1633 		BP_SET_COMPRESS(bp, compress);
1634 		BP_SET_CHECKSUM(bp, zp->zp_checksum);
1635 		BP_SET_DEDUP(bp, zp->zp_dedup);
1636 		BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
1637 		if (zp->zp_dedup) {
1638 			ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1639 			ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1640 			ASSERT(!zp->zp_encrypt ||
1641 			    DMU_OT_IS_ENCRYPTED(zp->zp_type));
1642 			zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE;
1643 		}
1644 		if (zp->zp_nopwrite) {
1645 			ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
1646 			ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
1647 			zio->io_pipeline |= ZIO_STAGE_NOP_WRITE;
1648 		}
1649 	}
1650 	return (ZIO_PIPELINE_CONTINUE);
1651 }
1652 
1653 static int
zio_free_bp_init(zio_t * zio)1654 zio_free_bp_init(zio_t *zio)
1655 {
1656 	blkptr_t *bp = zio->io_bp;
1657 
1658 	if (zio->io_child_type == ZIO_CHILD_LOGICAL) {
1659 		if (BP_GET_DEDUP(bp))
1660 			zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;
1661 	}
1662 
1663 	ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
1664 
1665 	return (ZIO_PIPELINE_CONTINUE);
1666 }
1667 
1668 /*
1669  * ==========================================================================
1670  * Execute the I/O pipeline
1671  * ==========================================================================
1672  */
1673 
1674 static void
zio_taskq_dispatch(zio_t * zio,zio_taskq_type_t q,boolean_t cutinline)1675 zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
1676 {
1677 	spa_t *spa = zio->io_spa;
1678 	zio_type_t t = zio->io_type;
1679 	int flags = (cutinline ? TQ_FRONT : 0);
1680 
1681 	/*
1682 	 * If we're a config writer or a probe, the normal issue and
1683 	 * interrupt threads may all be blocked waiting for the config lock.
1684 	 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL.
1685 	 */
1686 	if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE))
1687 		t = ZIO_TYPE_NULL;
1688 
1689 	/*
1690 	 * A similar issue exists for the L2ARC write thread until L2ARC 2.0.
1691 	 */
1692 	if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux)
1693 		t = ZIO_TYPE_NULL;
1694 
1695 	/*
1696 	 * If this is a high priority I/O, then use the high priority taskq if
1697 	 * available.
1698 	 */
1699 	if ((zio->io_priority == ZIO_PRIORITY_NOW ||
1700 	    zio->io_priority == ZIO_PRIORITY_SYNC_WRITE) &&
1701 	    spa->spa_zio_taskq[t][q + 1].stqs_count != 0)
1702 		q++;
1703 
1704 	ASSERT3U(q, <, ZIO_TASKQ_TYPES);
1705 
1706 	/*
1707 	 * NB: We are assuming that the zio can only be dispatched
1708 	 * to a single taskq at a time.  It would be a grievous error
1709 	 * to dispatch the zio to another taskq at the same time.
1710 	 */
1711 	ASSERT(zio->io_tqent.tqent_next == NULL);
1712 	spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio,
1713 	    flags, &zio->io_tqent);
1714 }
1715 
1716 static boolean_t
zio_taskq_member(zio_t * zio,zio_taskq_type_t q)1717 zio_taskq_member(zio_t *zio, zio_taskq_type_t q)
1718 {
1719 	kthread_t *executor = zio->io_executor;
1720 	spa_t *spa = zio->io_spa;
1721 
1722 	for (zio_type_t t = 0; t < ZIO_TYPES; t++) {
1723 		spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
1724 		uint_t i;
1725 		for (i = 0; i < tqs->stqs_count; i++) {
1726 			if (taskq_member(tqs->stqs_taskq[i], executor))
1727 				return (B_TRUE);
1728 		}
1729 	}
1730 
1731 	return (B_FALSE);
1732 }
1733 
1734 static int
zio_issue_async(zio_t * zio)1735 zio_issue_async(zio_t *zio)
1736 {
1737 	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
1738 
1739 	return (ZIO_PIPELINE_STOP);
1740 }
1741 
1742 void
zio_interrupt(zio_t * zio)1743 zio_interrupt(zio_t *zio)
1744 {
1745 	zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE);
1746 }
1747 
1748 void
zio_delay_interrupt(zio_t * zio)1749 zio_delay_interrupt(zio_t *zio)
1750 {
1751 	/*
1752 	 * The timeout_generic() function isn't defined in userspace, so
1753 	 * rather than trying to implement the function, the zio delay
1754 	 * functionality has been disabled for userspace builds.
1755 	 */
1756 
1757 #ifdef _KERNEL
1758 	/*
1759 	 * If io_target_timestamp is zero, then no delay has been registered
1760 	 * for this IO, thus jump to the end of this function and "skip" the
1761 	 * delay; issuing it directly to the zio layer.
1762 	 */
1763 	if (zio->io_target_timestamp != 0) {
1764 		hrtime_t now = gethrtime();
1765 
1766 		if (now >= zio->io_target_timestamp) {
1767 			/*
1768 			 * This IO has already taken longer than the target
1769 			 * delay to complete, so we don't want to delay it
1770 			 * any longer; we "miss" the delay and issue it
1771 			 * directly to the zio layer. This is likely due to
1772 			 * the target latency being set to a value less than
1773 			 * the underlying hardware can satisfy (e.g. delay
1774 			 * set to 1ms, but the disks take 10ms to complete an
1775 			 * IO request).
1776 			 */
1777 
1778 			DTRACE_PROBE2(zio__delay__miss, zio_t *, zio,
1779 			    hrtime_t, now);
1780 
1781 			zio_interrupt(zio);
1782 		} else {
1783 			hrtime_t diff = zio->io_target_timestamp - now;
1784 
1785 			DTRACE_PROBE3(zio__delay__hit, zio_t *, zio,
1786 			    hrtime_t, now, hrtime_t, diff);
1787 
1788 			(void) timeout_generic(CALLOUT_NORMAL,
1789 			    (void (*)(void *))zio_interrupt, zio, diff, 1, 0);
1790 		}
1791 
1792 		return;
1793 	}
1794 #endif
1795 
1796 	DTRACE_PROBE1(zio__delay__skip, zio_t *, zio);
1797 	zio_interrupt(zio);
1798 }
1799 
1800 /*
1801  * Execute the I/O pipeline until one of the following occurs:
1802  *
1803  *	(1) the I/O completes
1804  *	(2) the pipeline stalls waiting for dependent child I/Os
1805  *	(3) the I/O issues, so we're waiting for an I/O completion interrupt
1806  *	(4) the I/O is delegated by vdev-level caching or aggregation
1807  *	(5) the I/O is deferred due to vdev-level queueing
1808  *	(6) the I/O is handed off to another thread.
1809  *
1810  * In all cases, the pipeline stops whenever there's no CPU work; it never
1811  * burns a thread in cv_wait().
1812  *
1813  * There's no locking on io_stage because there's no legitimate way
1814  * for multiple threads to be attempting to process the same I/O.
1815  */
1816 static zio_pipe_stage_t *zio_pipeline[];
1817 
1818 void
zio_execute(zio_t * zio)1819 zio_execute(zio_t *zio)
1820 {
1821 	zio->io_executor = curthread;
1822 
1823 	ASSERT3U(zio->io_queued_timestamp, >, 0);
1824 
1825 	while (zio->io_stage < ZIO_STAGE_DONE) {
1826 		enum zio_stage pipeline = zio->io_pipeline;
1827 		enum zio_stage stage = zio->io_stage;
1828 		int rv;
1829 
1830 		ASSERT(!MUTEX_HELD(&zio->io_lock));
1831 		ASSERT(ISP2(stage));
1832 		ASSERT(zio->io_stall == NULL);
1833 
1834 		do {
1835 			stage <<= 1;
1836 		} while ((stage & pipeline) == 0);
1837 
1838 		ASSERT(stage <= ZIO_STAGE_DONE);
1839 
1840 		/*
1841 		 * If we are in interrupt context and this pipeline stage
1842 		 * will grab a config lock that is held across I/O,
1843 		 * or may wait for an I/O that needs an interrupt thread
1844 		 * to complete, issue async to avoid deadlock.
1845 		 *
1846 		 * For VDEV_IO_START, we cut in line so that the io will
1847 		 * be sent to disk promptly.
1848 		 */
1849 		if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
1850 		    zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) {
1851 			boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
1852 			    zio_requeue_io_start_cut_in_line : B_FALSE;
1853 			zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
1854 			return;
1855 		}
1856 
1857 #ifdef _KERNEL
1858 		/*
1859 		 * The I/O pipeline is a part of the machinery responsible for
1860 		 * evacuation of memory pages to disk when we are under
1861 		 * sufficient memory pressure for pageout to run.  By setting
1862 		 * this flag, allocations may dip into pages in the pageout
1863 		 * reserved pool in order to try to make forward progress.
1864 		 */
1865 		bool set_pushpage = false;
1866 		if (!(curthread->t_flag & T_PUSHPAGE)) {
1867 			/*
1868 			 * We can be called recursively, so we need to remember
1869 			 * if this frame was the one that first set the flag or
1870 			 * not.
1871 			 */
1872 			set_pushpage = true;
1873 			curthread->t_flag |= T_PUSHPAGE;
1874 		}
1875 #endif
1876 
1877 		zio->io_stage = stage;
1878 		zio->io_pipeline_trace |= zio->io_stage;
1879 		rv = zio_pipeline[highbit64(stage) - 1](zio);
1880 
1881 #ifdef _KERNEL
1882 		if (set_pushpage) {
1883 			curthread->t_flag &= ~T_PUSHPAGE;
1884 		}
1885 #endif
1886 
1887 		if (rv == ZIO_PIPELINE_STOP)
1888 			return;
1889 
1890 		ASSERT(rv == ZIO_PIPELINE_CONTINUE);
1891 	}
1892 }
1893 
1894 /*
1895  * ==========================================================================
1896  * Initiate I/O, either sync or async
1897  * ==========================================================================
1898  */
1899 int
zio_wait(zio_t * zio)1900 zio_wait(zio_t *zio)
1901 {
1902 	int error;
1903 
1904 	ASSERT3P(zio->io_stage, ==, ZIO_STAGE_OPEN);
1905 	ASSERT3P(zio->io_executor, ==, NULL);
1906 
1907 	zio->io_waiter = curthread;
1908 	ASSERT0(zio->io_queued_timestamp);
1909 	zio->io_queued_timestamp = gethrtime();
1910 
1911 	zio_execute(zio);
1912 
1913 	mutex_enter(&zio->io_lock);
1914 	while (zio->io_executor != NULL)
1915 		cv_wait(&zio->io_cv, &zio->io_lock);
1916 	mutex_exit(&zio->io_lock);
1917 
1918 	error = zio->io_error;
1919 	zio_destroy(zio);
1920 
1921 	return (error);
1922 }
1923 
1924 void
zio_nowait(zio_t * zio)1925 zio_nowait(zio_t *zio)
1926 {
1927 	ASSERT3P(zio->io_executor, ==, NULL);
1928 
1929 	if (zio->io_child_type == ZIO_CHILD_LOGICAL &&
1930 	    zio_unique_parent(zio) == NULL) {
1931 		/*
1932 		 * This is a logical async I/O with no parent to wait for it.
1933 		 * We add it to the spa_async_root_zio "Godfather" I/O which
1934 		 * will ensure they complete prior to unloading the pool.
1935 		 */
1936 		spa_t *spa = zio->io_spa;
1937 
1938 		zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio);
1939 	}
1940 
1941 	ASSERT0(zio->io_queued_timestamp);
1942 	zio->io_queued_timestamp = gethrtime();
1943 	zio_execute(zio);
1944 }
1945 
1946 /*
1947  * ==========================================================================
1948  * Reexecute, cancel, or suspend/resume failed I/O
1949  * ==========================================================================
1950  */
1951 
1952 static void
zio_reexecute(zio_t * pio)1953 zio_reexecute(zio_t *pio)
1954 {
1955 	zio_t *cio, *cio_next;
1956 
1957 	ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);
1958 	ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
1959 	ASSERT(pio->io_gang_leader == NULL);
1960 	ASSERT(pio->io_gang_tree == NULL);
1961 
1962 	pio->io_flags = pio->io_orig_flags;
1963 	pio->io_stage = pio->io_orig_stage;
1964 	pio->io_pipeline = pio->io_orig_pipeline;
1965 	pio->io_reexecute = 0;
1966 	pio->io_flags |= ZIO_FLAG_REEXECUTED;
1967 	pio->io_pipeline_trace = 0;
1968 	pio->io_error = 0;
1969 	for (int w = 0; w < ZIO_WAIT_TYPES; w++)
1970 		pio->io_state[w] = 0;
1971 	for (int c = 0; c < ZIO_CHILD_TYPES; c++)
1972 		pio->io_child_error[c] = 0;
1973 
1974 	if (IO_IS_ALLOCATING(pio))
1975 		BP_ZERO(pio->io_bp);
1976 
1977 	/*
1978 	 * As we reexecute pio's children, new children could be created.
1979 	 * New children go to the head of pio's io_child_list, however,
1980 	 * so we will (correctly) not reexecute them.  The key is that
1981 	 * the remainder of pio's io_child_list, from 'cio_next' onward,
1982 	 * cannot be affected by any side effects of reexecuting 'cio'.
1983 	 */
1984 	zio_link_t *zl = NULL;
1985 	mutex_enter(&pio->io_lock);
1986 	for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
1987 		cio_next = zio_walk_children(pio, &zl);
1988 		for (int w = 0; w < ZIO_WAIT_TYPES; w++)
1989 			pio->io_children[cio->io_child_type][w]++;
1990 		mutex_exit(&pio->io_lock);
1991 		zio_reexecute(cio);
1992 		mutex_enter(&pio->io_lock);
1993 	}
1994 	mutex_exit(&pio->io_lock);
1995 
1996 	/*
1997 	 * Now that all children have been reexecuted, execute the parent.
1998 	 * We don't reexecute "The Godfather" I/O here as it's the
1999 	 * responsibility of the caller to wait on it.
2000 	 */
2001 	if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) {
2002 		pio->io_queued_timestamp = gethrtime();
2003 		zio_execute(pio);
2004 	}
2005 }
2006 
2007 void
zio_suspend(spa_t * spa,zio_t * zio,zio_suspend_reason_t reason)2008 zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason)
2009 {
2010 	if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC)
2011 		fm_panic("Pool '%s' has encountered an uncorrectable I/O "
2012 		    "failure and the failure mode property for this pool "
2013 		    "is set to panic.", spa_name(spa));
2014 
2015 	cmn_err(CE_WARN, "Pool '%s' has encountered an uncorrectable I/O "
2016 	    "failure and has been suspended; `zpool clear` will be required "
2017 	    "before the pool can be written to.", spa_name(spa));
2018 
2019 	(void) zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL,
2020 	    NULL, NULL, 0, 0);
2021 
2022 	mutex_enter(&spa->spa_suspend_lock);
2023 
2024 	if (spa->spa_suspend_zio_root == NULL)
2025 		spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL,
2026 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
2027 		    ZIO_FLAG_GODFATHER);
2028 
2029 	spa->spa_suspended = reason;
2030 
2031 	if (zio != NULL) {
2032 		ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
2033 		ASSERT(zio != spa->spa_suspend_zio_root);
2034 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
2035 		ASSERT(zio_unique_parent(zio) == NULL);
2036 		ASSERT(zio->io_stage == ZIO_STAGE_DONE);
2037 		zio_add_child(spa->spa_suspend_zio_root, zio);
2038 	}
2039 
2040 	mutex_exit(&spa->spa_suspend_lock);
2041 }
2042 
2043 int
zio_resume(spa_t * spa)2044 zio_resume(spa_t *spa)
2045 {
2046 	zio_t *pio;
2047 
2048 	/*
2049 	 * Reexecute all previously suspended i/o.
2050 	 */
2051 	mutex_enter(&spa->spa_suspend_lock);
2052 	spa->spa_suspended = ZIO_SUSPEND_NONE;
2053 	cv_broadcast(&spa->spa_suspend_cv);
2054 	pio = spa->spa_suspend_zio_root;
2055 	spa->spa_suspend_zio_root = NULL;
2056 	mutex_exit(&spa->spa_suspend_lock);
2057 
2058 	if (pio == NULL)
2059 		return (0);
2060 
2061 	zio_reexecute(pio);
2062 	return (zio_wait(pio));
2063 }
2064 
2065 void
zio_resume_wait(spa_t * spa)2066 zio_resume_wait(spa_t *spa)
2067 {
2068 	mutex_enter(&spa->spa_suspend_lock);
2069 	while (spa_suspended(spa))
2070 		cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock);
2071 	mutex_exit(&spa->spa_suspend_lock);
2072 }
2073 
2074 /*
2075  * ==========================================================================
2076  * Gang blocks.
2077  *
2078  * A gang block is a collection of small blocks that looks to the DMU
2079  * like one large block.  When zio_dva_allocate() cannot find a block
2080  * of the requested size, due to either severe fragmentation or the pool
2081  * being nearly full, it calls zio_write_gang_block() to construct the
2082  * block from smaller fragments.
2083  *
2084  * A gang block consists of a gang header (zio_gbh_phys_t) and up to
2085  * three (SPA_GBH_NBLKPTRS) gang members.  The gang header is just like
2086  * an indirect block: it's an array of block pointers.  It consumes
2087  * only one sector and hence is allocatable regardless of fragmentation.
2088  * The gang header's bps point to its gang members, which hold the data.
2089  *
2090  * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg>
2091  * as the verifier to ensure uniqueness of the SHA256 checksum.
2092  * Critically, the gang block bp's blk_cksum is the checksum of the data,
2093  * not the gang header.  This ensures that data block signatures (needed for
2094  * deduplication) are independent of how the block is physically stored.
2095  *
2096  * Gang blocks can be nested: a gang member may itself be a gang block.
2097  * Thus every gang block is a tree in which root and all interior nodes are
2098  * gang headers, and the leaves are normal blocks that contain user data.
2099  * The root of the gang tree is called the gang leader.
2100  *
2101  * To perform any operation (read, rewrite, free, claim) on a gang block,
2102  * zio_gang_assemble() first assembles the gang tree (minus data leaves)
2103  * in the io_gang_tree field of the original logical i/o by recursively
2104  * reading the gang leader and all gang headers below it.  This yields
2105  * an in-core tree containing the contents of every gang header and the
2106  * bps for every constituent of the gang block.
2107  *
2108  * With the gang tree now assembled, zio_gang_issue() just walks the gang tree
2109  * and invokes a callback on each bp.  To free a gang block, zio_gang_issue()
2110  * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp.
2111  * zio_claim_gang() provides a similarly trivial wrapper for zio_claim().
2112  * zio_read_gang() is a wrapper around zio_read() that omits reading gang
2113  * headers, since we already have those in io_gang_tree.  zio_rewrite_gang()
2114  * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite()
2115  * of the gang header plus zio_checksum_compute() of the data to update the
2116  * gang header's blk_cksum as described above.
2117  *
2118  * The two-phase assemble/issue model solves the problem of partial failure --
2119  * what if you'd freed part of a gang block but then couldn't read the
2120  * gang header for another part?  Assembling the entire gang tree first
2121  * ensures that all the necessary gang header I/O has succeeded before
2122  * starting the actual work of free, claim, or write.  Once the gang tree
2123  * is assembled, free and claim are in-memory operations that cannot fail.
2124  *
2125  * In the event that a gang write fails, zio_dva_unallocate() walks the
2126  * gang tree to immediately free (i.e. insert back into the space map)
2127  * everything we've allocated.  This ensures that we don't get ENOSPC
2128  * errors during repeated suspend/resume cycles due to a flaky device.
2129  *
2130  * Gang rewrites only happen during sync-to-convergence.  If we can't assemble
2131  * the gang tree, we won't modify the block, so we can safely defer the free
2132  * (knowing that the block is still intact).  If we *can* assemble the gang
2133  * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free
2134  * each constituent bp and we can allocate a new block on the next sync pass.
2135  *
2136  * In all cases, the gang tree allows complete recovery from partial failure.
2137  * ==========================================================================
2138  */
2139 
2140 static void
zio_gang_issue_func_done(zio_t * zio)2141 zio_gang_issue_func_done(zio_t *zio)
2142 {
2143 	abd_put(zio->io_abd);
2144 }
2145 
2146 static zio_t *
zio_read_gang(zio_t * pio,blkptr_t * bp,zio_gang_node_t * gn,abd_t * data,uint64_t offset)2147 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data,
2148     uint64_t offset)
2149 {
2150 	if (gn != NULL)
2151 		return (pio);
2152 
2153 	return (zio_read(pio, pio->io_spa, bp, abd_get_offset(data, offset),
2154 	    BP_GET_PSIZE(bp), zio_gang_issue_func_done,
2155 	    NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
2156 	    &pio->io_bookmark));
2157 }
2158 
2159 static zio_t *
zio_rewrite_gang(zio_t * pio,blkptr_t * bp,zio_gang_node_t * gn,abd_t * data,uint64_t offset)2160 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data,
2161     uint64_t offset)
2162 {
2163 	zio_t *zio;
2164 
2165 	if (gn != NULL) {
2166 		abd_t *gbh_abd =
2167 		    abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE);
2168 		zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
2169 		    gbh_abd, SPA_GANGBLOCKSIZE, zio_gang_issue_func_done, NULL,
2170 		    pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
2171 		    &pio->io_bookmark);
2172 		/*
2173 		 * As we rewrite each gang header, the pipeline will compute
2174 		 * a new gang block header checksum for it; but no one will
2175 		 * compute a new data checksum, so we do that here.  The one
2176 		 * exception is the gang leader: the pipeline already computed
2177 		 * its data checksum because that stage precedes gang assembly.
2178 		 * (Presently, nothing actually uses interior data checksums;
2179 		 * this is just good hygiene.)
2180 		 */
2181 		if (gn != pio->io_gang_leader->io_gang_tree) {
2182 			abd_t *buf = abd_get_offset(data, offset);
2183 
2184 			zio_checksum_compute(zio, BP_GET_CHECKSUM(bp),
2185 			    buf, BP_GET_PSIZE(bp));
2186 
2187 			abd_put(buf);
2188 		}
2189 		/*
2190 		 * If we are here to damage data for testing purposes,
2191 		 * leave the GBH alone so that we can detect the damage.
2192 		 */
2193 		if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE)
2194 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
2195 	} else {
2196 		zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
2197 		    abd_get_offset(data, offset), BP_GET_PSIZE(bp),
2198 		    zio_gang_issue_func_done, NULL, pio->io_priority,
2199 		    ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
2200 	}
2201 
2202 	return (zio);
2203 }
2204 
2205 /* ARGSUSED */
2206 static zio_t *
zio_free_gang(zio_t * pio,blkptr_t * bp,zio_gang_node_t * gn,abd_t * data,uint64_t offset)2207 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data,
2208     uint64_t offset)
2209 {
2210 	return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp,
2211 	    ZIO_GANG_CHILD_FLAGS(pio)));
2212 }
2213 
2214 /* ARGSUSED */
2215 static zio_t *
zio_claim_gang(zio_t * pio,blkptr_t * bp,zio_gang_node_t * gn,abd_t * data,uint64_t offset)2216 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data,
2217     uint64_t offset)
2218 {
2219 	return (zio_claim(pio, pio->io_spa, pio->io_txg, bp,
2220 	    NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio)));
2221 }
2222 
2223 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = {
2224 	NULL,
2225 	zio_read_gang,
2226 	zio_rewrite_gang,
2227 	zio_free_gang,
2228 	zio_claim_gang,
2229 	NULL
2230 };
2231 
2232 static void zio_gang_tree_assemble_done(zio_t *zio);
2233 
2234 static zio_gang_node_t *
zio_gang_node_alloc(zio_gang_node_t ** gnpp)2235 zio_gang_node_alloc(zio_gang_node_t **gnpp)
2236 {
2237 	zio_gang_node_t *gn;
2238 
2239 	ASSERT(*gnpp == NULL);
2240 
2241 	gn = kmem_zalloc(sizeof (*gn), KM_SLEEP);
2242 	gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE);
2243 	*gnpp = gn;
2244 
2245 	return (gn);
2246 }
2247 
2248 static void
zio_gang_node_free(zio_gang_node_t ** gnpp)2249 zio_gang_node_free(zio_gang_node_t **gnpp)
2250 {
2251 	zio_gang_node_t *gn = *gnpp;
2252 
2253 	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
2254 		ASSERT(gn->gn_child[g] == NULL);
2255 
2256 	zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE);
2257 	kmem_free(gn, sizeof (*gn));
2258 	*gnpp = NULL;
2259 }
2260 
2261 static void
zio_gang_tree_free(zio_gang_node_t ** gnpp)2262 zio_gang_tree_free(zio_gang_node_t **gnpp)
2263 {
2264 	zio_gang_node_t *gn = *gnpp;
2265 
2266 	if (gn == NULL)
2267 		return;
2268 
2269 	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
2270 		zio_gang_tree_free(&gn->gn_child[g]);
2271 
2272 	zio_gang_node_free(gnpp);
2273 }
2274 
2275 static void
zio_gang_tree_assemble(zio_t * gio,blkptr_t * bp,zio_gang_node_t ** gnpp)2276 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp)
2277 {
2278 	zio_gang_node_t *gn = zio_gang_node_alloc(gnpp);
2279 	abd_t *gbh_abd = abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE);
2280 
2281 	ASSERT(gio->io_gang_leader == gio);
2282 	ASSERT(BP_IS_GANG(bp));
2283 
2284 	zio_nowait(zio_read(gio, gio->io_spa, bp, gbh_abd, SPA_GANGBLOCKSIZE,
2285 	    zio_gang_tree_assemble_done, gn, gio->io_priority,
2286 	    ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark));
2287 }
2288 
2289 static void
zio_gang_tree_assemble_done(zio_t * zio)2290 zio_gang_tree_assemble_done(zio_t *zio)
2291 {
2292 	zio_t *gio = zio->io_gang_leader;
2293 	zio_gang_node_t *gn = zio->io_private;
2294 	blkptr_t *bp = zio->io_bp;
2295 
2296 	ASSERT(gio == zio_unique_parent(zio));
2297 	ASSERT(zio->io_child_count == 0);
2298 
2299 	if (zio->io_error)
2300 		return;
2301 
2302 	/* this ABD was created from a linear buf in zio_gang_tree_assemble */
2303 	if (BP_SHOULD_BYTESWAP(bp))
2304 		byteswap_uint64_array(abd_to_buf(zio->io_abd), zio->io_size);
2305 
2306 	ASSERT3P(abd_to_buf(zio->io_abd), ==, gn->gn_gbh);
2307 	ASSERT(zio->io_size == SPA_GANGBLOCKSIZE);
2308 	ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
2309 
2310 	abd_put(zio->io_abd);
2311 
2312 	for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
2313 		blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
2314 		if (!BP_IS_GANG(gbp))
2315 			continue;
2316 		zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]);
2317 	}
2318 }
2319 
2320 static void
zio_gang_tree_issue(zio_t * pio,zio_gang_node_t * gn,blkptr_t * bp,abd_t * data,uint64_t offset)2321 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, abd_t *data,
2322     uint64_t offset)
2323 {
2324 	zio_t *gio = pio->io_gang_leader;
2325 	zio_t *zio;
2326 
2327 	ASSERT(BP_IS_GANG(bp) == !!gn);
2328 	ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp));
2329 	ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree);
2330 
2331 	/*
2332 	 * If you're a gang header, your data is in gn->gn_gbh.
2333 	 * If you're a gang member, your data is in 'data' and gn == NULL.
2334 	 */
2335 	zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data, offset);
2336 
2337 	if (gn != NULL) {
2338 		ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
2339 
2340 		for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
2341 			blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g];
2342 			if (BP_IS_HOLE(gbp))
2343 				continue;
2344 			zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data,
2345 			    offset);
2346 			offset += BP_GET_PSIZE(gbp);
2347 		}
2348 	}
2349 
2350 	if (gn == gio->io_gang_tree)
2351 		ASSERT3U(gio->io_size, ==, offset);
2352 
2353 	if (zio != pio)
2354 		zio_nowait(zio);
2355 }
2356 
2357 static int
zio_gang_assemble(zio_t * zio)2358 zio_gang_assemble(zio_t *zio)
2359 {
2360 	blkptr_t *bp = zio->io_bp;
2361 
2362 	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL);
2363 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2364 
2365 	zio->io_gang_leader = zio;
2366 
2367 	zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);
2368 
2369 	return (ZIO_PIPELINE_CONTINUE);
2370 }
2371 
2372 static int
zio_gang_issue(zio_t * zio)2373 zio_gang_issue(zio_t *zio)
2374 {
2375 	blkptr_t *bp = zio->io_bp;
2376 
2377 	if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) {
2378 		return (ZIO_PIPELINE_STOP);
2379 	}
2380 
2381 	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
2382 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
2383 
2384 	if (zio->io_child_error[ZIO_CHILD_GANG] == 0)
2385 		zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_abd,
2386 		    0);
2387 	else
2388 		zio_gang_tree_free(&zio->io_gang_tree);
2389 
2390 	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
2391 
2392 	return (ZIO_PIPELINE_CONTINUE);
2393 }
2394 
2395 static void
zio_write_gang_member_ready(zio_t * zio)2396 zio_write_gang_member_ready(zio_t *zio)
2397 {
2398 	zio_t *pio = zio_unique_parent(zio);
2399 	zio_t *gio = zio->io_gang_leader;
2400 	dva_t *cdva = zio->io_bp->blk_dva;
2401 	dva_t *pdva = pio->io_bp->blk_dva;
2402 	uint64_t asize;
2403 
2404 	if (BP_IS_HOLE(zio->io_bp))
2405 		return;
2406 
2407 	ASSERT(BP_IS_HOLE(&zio->io_bp_orig));
2408 
2409 	ASSERT(zio->io_child_type == ZIO_CHILD_GANG);
2410 	ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies);
2411 	ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp));
2412 	ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp));
2413 	ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp));
2414 
2415 	mutex_enter(&pio->io_lock);
2416 	for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) {
2417 		ASSERT(DVA_GET_GANG(&pdva[d]));
2418 		asize = DVA_GET_ASIZE(&pdva[d]);
2419 		asize += DVA_GET_ASIZE(&cdva[d]);
2420 		DVA_SET_ASIZE(&pdva[d], asize);
2421 	}
2422 	mutex_exit(&pio->io_lock);
2423 }
2424 
2425 static void
zio_write_gang_done(zio_t * zio)2426 zio_write_gang_done(zio_t *zio)
2427 {
2428 	/*
2429 	 * The io_abd field will be NULL for a zio with no data.  The io_flags
2430 	 * will initially have the ZIO_FLAG_NODATA bit flag set, but we can't
2431 	 * check for it here as it is cleared in zio_ready.
2432 	 */
2433 	if (zio->io_abd != NULL)
2434 		abd_put(zio->io_abd);
2435 }
2436 
2437 static int
zio_write_gang_block(zio_t * pio)2438 zio_write_gang_block(zio_t *pio)
2439 {
2440 	spa_t *spa = pio->io_spa;
2441 	metaslab_class_t *mc = spa_normal_class(spa);
2442 	blkptr_t *bp = pio->io_bp;
2443 	zio_t *gio = pio->io_gang_leader;
2444 	zio_t *zio;
2445 	zio_gang_node_t *gn, **gnpp;
2446 	zio_gbh_phys_t *gbh;
2447 	abd_t *gbh_abd;
2448 	uint64_t txg = pio->io_txg;
2449 	uint64_t resid = pio->io_size;
2450 	uint64_t lsize;
2451 	int copies = gio->io_prop.zp_copies;
2452 	int gbh_copies = MIN(copies + 1, spa_max_replication(spa));
2453 	zio_prop_t zp;
2454 	int error;
2455 	boolean_t has_data = !(pio->io_flags & ZIO_FLAG_NODATA);
2456 
2457 	/*
2458 	 * encrypted blocks need DVA[2] free so encrypted gang headers can't
2459 	 * have a third copy.
2460 	 */
2461 	if (gio->io_prop.zp_encrypt && gbh_copies >= SPA_DVAS_PER_BP)
2462 		gbh_copies = SPA_DVAS_PER_BP - 1;
2463 
2464 	int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER;
2465 	if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2466 		ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2467 		ASSERT(has_data);
2468 
2469 		flags |= METASLAB_ASYNC_ALLOC;
2470 		VERIFY(zfs_refcount_held(&mc->mc_alloc_slots[pio->io_allocator],
2471 		    pio));
2472 
2473 		/*
2474 		 * The logical zio has already placed a reservation for
2475 		 * 'copies' allocation slots but gang blocks may require
2476 		 * additional copies. These additional copies
2477 		 * (i.e. gbh_copies - copies) are guaranteed to succeed
2478 		 * since metaslab_class_throttle_reserve() always allows
2479 		 * additional reservations for gang blocks.
2480 		 */
2481 		VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
2482 		    pio->io_allocator, pio, flags));
2483 	}
2484 
2485 	error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
2486 	    bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
2487 	    &pio->io_alloc_list, pio, pio->io_allocator);
2488 	if (error) {
2489 		if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2490 			ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2491 			ASSERT(has_data);
2492 
2493 			/*
2494 			 * If we failed to allocate the gang block header then
2495 			 * we remove any additional allocation reservations that
2496 			 * we placed here. The original reservation will
2497 			 * be removed when the logical I/O goes to the ready
2498 			 * stage.
2499 			 */
2500 			metaslab_class_throttle_unreserve(mc,
2501 			    gbh_copies - copies, pio->io_allocator, pio);
2502 		}
2503 		pio->io_error = error;
2504 		return (ZIO_PIPELINE_CONTINUE);
2505 	}
2506 
2507 	if (pio == gio) {
2508 		gnpp = &gio->io_gang_tree;
2509 	} else {
2510 		gnpp = pio->io_private;
2511 		ASSERT(pio->io_ready == zio_write_gang_member_ready);
2512 	}
2513 
2514 	gn = zio_gang_node_alloc(gnpp);
2515 	gbh = gn->gn_gbh;
2516 	bzero(gbh, SPA_GANGBLOCKSIZE);
2517 	gbh_abd = abd_get_from_buf(gbh, SPA_GANGBLOCKSIZE);
2518 
2519 	/*
2520 	 * Create the gang header.
2521 	 */
2522 	zio = zio_rewrite(pio, spa, txg, bp, gbh_abd, SPA_GANGBLOCKSIZE,
2523 	    zio_write_gang_done, NULL, pio->io_priority,
2524 	    ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
2525 
2526 	/*
2527 	 * Create and nowait the gang children.
2528 	 */
2529 	for (int g = 0; resid != 0; resid -= lsize, g++) {
2530 		lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g),
2531 		    SPA_MINBLOCKSIZE);
2532 		ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);
2533 
2534 		zp.zp_checksum = gio->io_prop.zp_checksum;
2535 		zp.zp_compress = ZIO_COMPRESS_OFF;
2536 		zp.zp_type = DMU_OT_NONE;
2537 		zp.zp_level = 0;
2538 		zp.zp_copies = gio->io_prop.zp_copies;
2539 		zp.zp_dedup = B_FALSE;
2540 		zp.zp_dedup_verify = B_FALSE;
2541 		zp.zp_nopwrite = B_FALSE;
2542 		zp.zp_encrypt = gio->io_prop.zp_encrypt;
2543 		zp.zp_byteorder = gio->io_prop.zp_byteorder;
2544 		bzero(zp.zp_salt, ZIO_DATA_SALT_LEN);
2545 		bzero(zp.zp_iv, ZIO_DATA_IV_LEN);
2546 		bzero(zp.zp_mac, ZIO_DATA_MAC_LEN);
2547 
2548 		zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
2549 		    has_data ? abd_get_offset(pio->io_abd, pio->io_size -
2550 		    resid) : NULL, lsize, lsize, &zp,
2551 		    zio_write_gang_member_ready, NULL, NULL,
2552 		    zio_write_gang_done, &gn->gn_child[g], pio->io_priority,
2553 		    ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
2554 
2555 		if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
2556 			ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
2557 			ASSERT(has_data);
2558 
2559 			/*
2560 			 * Gang children won't throttle but we should
2561 			 * account for their work, so reserve an allocation
2562 			 * slot for them here.
2563 			 */
2564 			VERIFY(metaslab_class_throttle_reserve(mc,
2565 			    zp.zp_copies, cio->io_allocator, cio, flags));
2566 		}
2567 		zio_nowait(cio);
2568 	}
2569 
2570 	/*
2571 	 * Set pio's pipeline to just wait for zio to finish.
2572 	 */
2573 	pio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
2574 
2575 	zio_nowait(zio);
2576 
2577 	return (ZIO_PIPELINE_CONTINUE);
2578 }
2579 
2580 /*
2581  * The zio_nop_write stage in the pipeline determines if allocating a
2582  * new bp is necessary.  The nopwrite feature can handle writes in
2583  * either syncing or open context (i.e. zil writes) and as a result is
2584  * mutually exclusive with dedup.
2585  *
2586  * By leveraging a cryptographically secure checksum, such as SHA256, we
2587  * can compare the checksums of the new data and the old to determine if
2588  * allocating a new block is required.  Note that our requirements for
2589  * cryptographic strength are fairly weak: there can't be any accidental
2590  * hash collisions, but we don't need to be secure against intentional
2591  * (malicious) collisions.  To trigger a nopwrite, you have to be able
2592  * to write the file to begin with, and triggering an incorrect (hash
2593  * collision) nopwrite is no worse than simply writing to the file.
2594  * That said, there are no known attacks against the checksum algorithms
2595  * used for nopwrite, assuming that the salt and the checksums
2596  * themselves remain secret.
2597  */
2598 static int
zio_nop_write(zio_t * zio)2599 zio_nop_write(zio_t *zio)
2600 {
2601 	blkptr_t *bp = zio->io_bp;
2602 	blkptr_t *bp_orig = &zio->io_bp_orig;
2603 	zio_prop_t *zp = &zio->io_prop;
2604 
2605 	ASSERT(BP_GET_LEVEL(bp) == 0);
2606 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
2607 	ASSERT(zp->zp_nopwrite);
2608 	ASSERT(!zp->zp_dedup);
2609 	ASSERT(zio->io_bp_override == NULL);
2610 	ASSERT(IO_IS_ALLOCATING(zio));
2611 
2612 	/*
2613 	 * Check to see if the original bp and the new bp have matching
2614 	 * characteristics (i.e. same checksum, compression algorithms, etc).
2615 	 * If they don't then just continue with the pipeline which will
2616 	 * allocate a new bp.
2617 	 */
2618 	if (BP_IS_HOLE(bp_orig) ||
2619 	    !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags &
2620 	    ZCHECKSUM_FLAG_NOPWRITE) ||
2621 	    BP_IS_ENCRYPTED(bp) || BP_IS_ENCRYPTED(bp_orig) ||
2622 	    BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) ||
2623 	    BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
2624 	    BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) ||
2625 	    zp->zp_copies != BP_GET_NDVAS(bp_orig))
2626 		return (ZIO_PIPELINE_CONTINUE);
2627 
2628 	/*
2629 	 * If the checksums match then reset the pipeline so that we
2630 	 * avoid allocating a new bp and issuing any I/O.
2631 	 */
2632 	if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {
2633 		ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags &
2634 		    ZCHECKSUM_FLAG_NOPWRITE);
2635 		ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig));
2636 		ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig));
2637 		ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF);
2638 		ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop,
2639 		    sizeof (uint64_t)) == 0);
2640 
2641 		*bp = *bp_orig;
2642 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
2643 		zio->io_flags |= ZIO_FLAG_NOPWRITE;
2644 	}
2645 
2646 	return (ZIO_PIPELINE_CONTINUE);
2647 }
2648 
2649 /*
2650  * ==========================================================================
2651  * Dedup
2652  * ==========================================================================
2653  */
2654 static void
zio_ddt_child_read_done(zio_t * zio)2655 zio_ddt_child_read_done(zio_t *zio)
2656 {
2657 	blkptr_t *bp = zio->io_bp;
2658 	ddt_entry_t *dde = zio->io_private;
2659 	ddt_phys_t *ddp;
2660 	zio_t *pio = zio_unique_parent(zio);
2661 
2662 	mutex_enter(&pio->io_lock);
2663 	ddp = ddt_phys_select(dde, bp);
2664 	if (zio->io_error == 0)
2665 		ddt_phys_clear(ddp);	/* this ddp doesn't need repair */
2666 
2667 	if (zio->io_error == 0 && dde->dde_repair_abd == NULL)
2668 		dde->dde_repair_abd = zio->io_abd;
2669 	else
2670 		abd_free(zio->io_abd);
2671 	mutex_exit(&pio->io_lock);
2672 }
2673 
2674 static int
zio_ddt_read_start(zio_t * zio)2675 zio_ddt_read_start(zio_t *zio)
2676 {
2677 	blkptr_t *bp = zio->io_bp;
2678 
2679 	ASSERT(BP_GET_DEDUP(bp));
2680 	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
2681 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
2682 
2683 	if (zio->io_child_error[ZIO_CHILD_DDT]) {
2684 		ddt_t *ddt = ddt_select(zio->io_spa, bp);
2685 		ddt_entry_t *dde = ddt_repair_start(ddt, bp);
2686 		ddt_phys_t *ddp = dde->dde_phys;
2687 		ddt_phys_t *ddp_self = ddt_phys_select(dde, bp);
2688 		blkptr_t blk;
2689 
2690 		ASSERT(zio->io_vsd == NULL);
2691 		zio->io_vsd = dde;
2692 
2693 		if (ddp_self == NULL)
2694 			return (ZIO_PIPELINE_CONTINUE);
2695 
2696 		for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
2697 			if (ddp->ddp_phys_birth == 0 || ddp == ddp_self)
2698 				continue;
2699 			ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp,
2700 			    &blk);
2701 			zio_nowait(zio_read(zio, zio->io_spa, &blk,
2702 			    abd_alloc_for_io(zio->io_size, B_TRUE),
2703 			    zio->io_size, zio_ddt_child_read_done, dde,
2704 			    zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) |
2705 			    ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark));
2706 		}
2707 		return (ZIO_PIPELINE_CONTINUE);
2708 	}
2709 
2710 	zio_nowait(zio_read(zio, zio->io_spa, bp,
2711 	    zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority,
2712 	    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));
2713 
2714 	return (ZIO_PIPELINE_CONTINUE);
2715 }
2716 
2717 static int
zio_ddt_read_done(zio_t * zio)2718 zio_ddt_read_done(zio_t *zio)
2719 {
2720 	blkptr_t *bp = zio->io_bp;
2721 
2722 	if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) {
2723 		return (ZIO_PIPELINE_STOP);
2724 	}
2725 
2726 	ASSERT(BP_GET_DEDUP(bp));
2727 	ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
2728 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
2729 
2730 	if (zio->io_child_error[ZIO_CHILD_DDT]) {
2731 		ddt_t *ddt = ddt_select(zio->io_spa, bp);
2732 		ddt_entry_t *dde = zio->io_vsd;
2733 		if (ddt == NULL) {
2734 			ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
2735 			return (ZIO_PIPELINE_CONTINUE);
2736 		}
2737 		if (dde == NULL) {
2738 			zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;
2739 			zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
2740 			return (ZIO_PIPELINE_STOP);
2741 		}
2742 		if (dde->dde_repair_abd != NULL) {
2743 			abd_copy(zio->io_abd, dde->dde_repair_abd,
2744 			    zio->io_size);
2745 			zio->io_child_error[ZIO_CHILD_DDT] = 0;
2746 		}
2747 		ddt_repair_done(ddt, dde);
2748 		zio->io_vsd = NULL;
2749 	}
2750 
2751 	ASSERT(zio->io_vsd == NULL);
2752 
2753 	return (ZIO_PIPELINE_CONTINUE);
2754 }
2755 
2756 static boolean_t
zio_ddt_collision(zio_t * zio,ddt_t * ddt,ddt_entry_t * dde)2757 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde)
2758 {
2759 	spa_t *spa = zio->io_spa;
2760 	boolean_t do_raw = !!(zio->io_flags & ZIO_FLAG_RAW);
2761 
2762 	/* We should never get a raw, override zio */
2763 	ASSERT(!(zio->io_bp_override && do_raw));
2764 
2765 	/*
2766 	 * Note: we compare the original data, not the transformed data,
2767 	 * because when zio->io_bp is an override bp, we will not have
2768 	 * pushed the I/O transforms.  That's an important optimization
2769 	 * because otherwise we'd compress/encrypt all dmu_sync() data twice.
2770 	 * However, we should never get a raw, override zio so in these
2771 	 * cases we can compare the io_data directly. This is useful because
2772 	 * it allows us to do dedup verification even if we don't have access
2773 	 * to the original data (for instance, if the encryption keys aren't
2774 	 * loaded).
2775 	 */
2776 
2777 	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
2778 		zio_t *lio = dde->dde_lead_zio[p];
2779 
2780 		if (lio != NULL && do_raw) {
2781 			return (lio->io_size != zio->io_size ||
2782 			    abd_cmp(zio->io_abd, lio->io_abd,
2783 			    zio->io_size) != 0);
2784 		} else if (lio != NULL) {
2785 			return (lio->io_orig_size != zio->io_orig_size ||
2786 			    abd_cmp(zio->io_orig_abd, lio->io_orig_abd,
2787 			    zio->io_orig_size) != 0);
2788 		}
2789 	}
2790 
2791 	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
2792 		ddt_phys_t *ddp = &dde->dde_phys[p];
2793 
2794 		if (ddp->ddp_phys_birth != 0 && do_raw) {
2795 			blkptr_t blk = *zio->io_bp;
2796 			uint64_t psize;
2797 			abd_t *tmpabd;
2798 			int error;
2799 
2800 			ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth);
2801 			psize = BP_GET_PSIZE(&blk);
2802 
2803 			if (psize != zio->io_size)
2804 				return (B_TRUE);
2805 
2806 			ddt_exit(ddt);
2807 
2808 			tmpabd = abd_alloc_for_io(psize, B_TRUE);
2809 
2810 			error = zio_wait(zio_read(NULL, spa, &blk, tmpabd,
2811 			    psize, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
2812 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
2813 			    ZIO_FLAG_RAW, &zio->io_bookmark));
2814 
2815 			if (error == 0) {
2816 				if (abd_cmp(tmpabd, zio->io_abd, psize) != 0)
2817 					error = SET_ERROR(ENOENT);
2818 			}
2819 
2820 			abd_free(tmpabd);
2821 			ddt_enter(ddt);
2822 			return (error != 0);
2823 		} else if (ddp->ddp_phys_birth != 0) {
2824 			arc_buf_t *abuf = NULL;
2825 			arc_flags_t aflags = ARC_FLAG_WAIT;
2826 			int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE;
2827 			blkptr_t blk = *zio->io_bp;
2828 			int error;
2829 
2830 			ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth);
2831 
2832 			if (BP_GET_LSIZE(&blk) != zio->io_orig_size)
2833 				return (B_TRUE);
2834 
2835 			ddt_exit(ddt);
2836 
2837 			/*
2838 			 * Intuitively, it would make more sense to compare
2839 			 * io_abd than io_orig_abd in the raw case since you
2840 			 * don't want to look at any transformations that have
2841 			 * happened to the data. However, for raw I/Os the
2842 			 * data will actually be the same in io_abd and
2843 			 * io_orig_abd, so all we have to do is issue this as
2844 			 * a raw ARC read.
2845 			 */
2846 			if (do_raw) {
2847 				zio_flags |= ZIO_FLAG_RAW;
2848 				ASSERT3U(zio->io_size, ==, zio->io_orig_size);
2849 				ASSERT0(abd_cmp(zio->io_abd, zio->io_orig_abd,
2850 				    zio->io_size));
2851 				ASSERT3P(zio->io_transform_stack, ==, NULL);
2852 			}
2853 
2854 			error = arc_read(NULL, spa, &blk,
2855 			    arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ,
2856 			    zio_flags, &aflags, &zio->io_bookmark);
2857 
2858 			if (error == 0) {
2859 				if (abd_cmp_buf(zio->io_orig_abd, abuf->b_data,
2860 				    zio->io_orig_size) != 0)
2861 					error = SET_ERROR(ENOENT);
2862 				arc_buf_destroy(abuf, &abuf);
2863 			}
2864 
2865 			ddt_enter(ddt);
2866 			return (error != 0);
2867 		}
2868 	}
2869 
2870 	return (B_FALSE);
2871 }
2872 
2873 static void
zio_ddt_child_write_ready(zio_t * zio)2874 zio_ddt_child_write_ready(zio_t *zio)
2875 {
2876 	int p = zio->io_prop.zp_copies;
2877 	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
2878 	ddt_entry_t *dde = zio->io_private;
2879 	ddt_phys_t *ddp = &dde->dde_phys[p];
2880 	zio_t *pio;
2881 
2882 	if (zio->io_error)
2883 		return;
2884 
2885 	ddt_enter(ddt);
2886 
2887 	ASSERT(dde->dde_lead_zio[p] == zio);
2888 
2889 	ddt_phys_fill(ddp, zio->io_bp);
2890 
2891 	zio_link_t *zl = NULL;
2892 	while ((pio = zio_walk_parents(zio, &zl)) != NULL)
2893 		ddt_bp_fill(ddp, pio->io_bp, zio->io_txg);
2894 
2895 	ddt_exit(ddt);
2896 }
2897 
2898 static void
zio_ddt_child_write_done(zio_t * zio)2899 zio_ddt_child_write_done(zio_t *zio)
2900 {
2901 	int p = zio->io_prop.zp_copies;
2902 	ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
2903 	ddt_entry_t *dde = zio->io_private;
2904 	ddt_phys_t *ddp = &dde->dde_phys[p];
2905 
2906 	ddt_enter(ddt);
2907 
2908 	ASSERT(ddp->ddp_refcnt == 0);
2909 	ASSERT(dde->dde_lead_zio[p] == zio);
2910 	dde->dde_lead_zio[p] = NULL;
2911 
2912 	if (zio->io_error == 0) {
2913 		zio_link_t *zl = NULL;
2914 		while (zio_walk_parents(zio, &zl) != NULL)
2915 			ddt_phys_addref(ddp);
2916 	} else {
2917 		ddt_phys_clear(ddp);
2918 	}
2919 
2920 	ddt_exit(ddt);
2921 }
2922 
2923 static void
zio_ddt_ditto_write_done(zio_t * zio)2924 zio_ddt_ditto_write_done(zio_t *zio)
2925 {
2926 	int p = DDT_PHYS_DITTO;
2927 	zio_prop_t *zp = &zio->io_prop;
2928 	blkptr_t *bp = zio->io_bp;
2929 	ddt_t *ddt = ddt_select(zio->io_spa, bp);
2930 	ddt_entry_t *dde = zio->io_private;
2931 	ddt_phys_t *ddp = &dde->dde_phys[p];
2932 	ddt_key_t *ddk = &dde->dde_key;
2933 
2934 	ddt_enter(ddt);
2935 
2936 	ASSERT(ddp->ddp_refcnt == 0);
2937 	ASSERT(dde->dde_lead_zio[p] == zio);
2938 	dde->dde_lead_zio[p] = NULL;
2939 
2940 	if (zio->io_error == 0) {
2941 		ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum));
2942 		ASSERT(zp->zp_copies < SPA_DVAS_PER_BP);
2943 		ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp));
2944 		if (ddp->ddp_phys_birth != 0)
2945 			ddt_phys_free(ddt, ddk, ddp, zio->io_txg);
2946 		ddt_phys_fill(ddp, bp);
2947 	}
2948 
2949 	ddt_exit(ddt);
2950 }
2951 
2952 static int
zio_ddt_write(zio_t * zio)2953 zio_ddt_write(zio_t *zio)
2954 {
2955 	spa_t *spa = zio->io_spa;
2956 	blkptr_t *bp = zio->io_bp;
2957 	uint64_t txg = zio->io_txg;
2958 	zio_prop_t *zp = &zio->io_prop;
2959 	int p = zp->zp_copies;
2960 	int ditto_copies;
2961 	zio_t *cio = NULL;
2962 	zio_t *dio = NULL;
2963 	ddt_t *ddt = ddt_select(spa, bp);
2964 	ddt_entry_t *dde;
2965 	ddt_phys_t *ddp;
2966 
2967 	ASSERT(BP_GET_DEDUP(bp));
2968 	ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum);
2969 	ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override);
2970 	ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW)));
2971 
2972 	ddt_enter(ddt);
2973 	dde = ddt_lookup(ddt, bp, B_TRUE);
2974 	ddp = &dde->dde_phys[p];
2975 
2976 	if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) {
2977 		/*
2978 		 * If we're using a weak checksum, upgrade to a strong checksum
2979 		 * and try again.  If we're already using a strong checksum,
2980 		 * we can't resolve it, so just convert to an ordinary write.
2981 		 * (And automatically e-mail a paper to Nature?)
2982 		 */
2983 		if (!(zio_checksum_table[zp->zp_checksum].ci_flags &
2984 		    ZCHECKSUM_FLAG_DEDUP)) {
2985 			zp->zp_checksum = spa_dedup_checksum(spa);
2986 			zio_pop_transforms(zio);
2987 			zio->io_stage = ZIO_STAGE_OPEN;
2988 			BP_ZERO(bp);
2989 		} else {
2990 			zp->zp_dedup = B_FALSE;
2991 			BP_SET_DEDUP(bp, B_FALSE);
2992 		}
2993 		ASSERT(!BP_GET_DEDUP(bp));
2994 		zio->io_pipeline = ZIO_WRITE_PIPELINE;
2995 		ddt_exit(ddt);
2996 		return (ZIO_PIPELINE_CONTINUE);
2997 	}
2998 
2999 	ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
3000 	ASSERT(ditto_copies < SPA_DVAS_PER_BP);
3001 
3002 	if (ditto_copies > ddt_ditto_copies_present(dde) &&
3003 	    dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) {
3004 		zio_prop_t czp = *zp;
3005 
3006 		czp.zp_copies = ditto_copies;
3007 
3008 		/*
3009 		 * If we arrived here with an override bp, we won't have run
3010 		 * the transform stack, so we won't have the data we need to
3011 		 * generate a child i/o.  So, toss the override bp and restart.
3012 		 * This is safe, because using the override bp is just an
3013 		 * optimization; and it's rare, so the cost doesn't matter.
3014 		 */
3015 		if (zio->io_bp_override) {
3016 			zio_pop_transforms(zio);
3017 			zio->io_stage = ZIO_STAGE_OPEN;
3018 			zio->io_pipeline = ZIO_WRITE_PIPELINE;
3019 			zio->io_bp_override = NULL;
3020 			BP_ZERO(bp);
3021 			ddt_exit(ddt);
3022 			return (ZIO_PIPELINE_CONTINUE);
3023 		}
3024 
3025 		dio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,
3026 		    zio->io_orig_size, zio->io_orig_size, &czp, NULL, NULL,
3027 		    NULL, zio_ddt_ditto_write_done, dde, zio->io_priority,
3028 		    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);
3029 
3030 		zio_push_transform(dio, zio->io_abd, zio->io_size, 0, NULL);
3031 		dde->dde_lead_zio[DDT_PHYS_DITTO] = dio;
3032 	}
3033 
3034 	if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) {
3035 		if (ddp->ddp_phys_birth != 0)
3036 			ddt_bp_fill(ddp, bp, txg);
3037 		if (dde->dde_lead_zio[p] != NULL)
3038 			zio_add_child(zio, dde->dde_lead_zio[p]);
3039 		else
3040 			ddt_phys_addref(ddp);
3041 	} else if (zio->io_bp_override) {
3042 		ASSERT(bp->blk_birth == txg);
3043 		ASSERT(BP_EQUAL(bp, zio->io_bp_override));
3044 		ddt_phys_fill(ddp, bp);
3045 		ddt_phys_addref(ddp);
3046 	} else {
3047 		cio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,
3048 		    zio->io_orig_size, zio->io_orig_size, zp,
3049 		    zio_ddt_child_write_ready, NULL, NULL,
3050 		    zio_ddt_child_write_done, dde, zio->io_priority,
3051 		    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark);
3052 
3053 		zio_push_transform(cio, zio->io_abd, zio->io_size, 0, NULL);
3054 		dde->dde_lead_zio[p] = cio;
3055 	}
3056 
3057 	ddt_exit(ddt);
3058 
3059 	if (cio)
3060 		zio_nowait(cio);
3061 	if (dio)
3062 		zio_nowait(dio);
3063 
3064 	return (ZIO_PIPELINE_CONTINUE);
3065 }
3066 
3067 ddt_entry_t *freedde; /* for debugging */
3068 
3069 static int
zio_ddt_free(zio_t * zio)3070 zio_ddt_free(zio_t *zio)
3071 {
3072 	spa_t *spa = zio->io_spa;
3073 	blkptr_t *bp = zio->io_bp;
3074 	ddt_t *ddt = ddt_select(spa, bp);
3075 	ddt_entry_t *dde;
3076 	ddt_phys_t *ddp;
3077 
3078 	ASSERT(BP_GET_DEDUP(bp));
3079 	ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
3080 
3081 	ddt_enter(ddt);
3082 	freedde = dde = ddt_lookup(ddt, bp, B_TRUE);
3083 	ddp = ddt_phys_select(dde, bp);
3084 	ddt_phys_decref(ddp);
3085 	ddt_exit(ddt);
3086 
3087 	return (ZIO_PIPELINE_CONTINUE);
3088 }
3089 
3090 /*
3091  * ==========================================================================
3092  * Allocate and free blocks
3093  * ==========================================================================
3094  */
3095 
3096 static zio_t *
zio_io_to_allocate(spa_t * spa,int allocator)3097 zio_io_to_allocate(spa_t *spa, int allocator)
3098 {
3099 	zio_t *zio;
3100 
3101 	ASSERT(MUTEX_HELD(&spa->spa_alloc_locks[allocator]));
3102 
3103 	zio = avl_first(&spa->spa_alloc_trees[allocator]);
3104 	if (zio == NULL)
3105 		return (NULL);
3106 
3107 	ASSERT(IO_IS_ALLOCATING(zio));
3108 
3109 	/*
3110 	 * Try to place a reservation for this zio. If we're unable to
3111 	 * reserve then we throttle.
3112 	 */
3113 	ASSERT3U(zio->io_allocator, ==, allocator);
3114 	if (!metaslab_class_throttle_reserve(zio->io_metaslab_class,
3115 	    zio->io_prop.zp_copies, zio->io_allocator, zio, 0)) {
3116 		return (NULL);
3117 	}
3118 
3119 	avl_remove(&spa->spa_alloc_trees[allocator], zio);
3120 	ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);
3121 
3122 	return (zio);
3123 }
3124 
3125 static int
zio_dva_throttle(zio_t * zio)3126 zio_dva_throttle(zio_t *zio)
3127 {
3128 	spa_t *spa = zio->io_spa;
3129 	zio_t *nio;
3130 	metaslab_class_t *mc;
3131 
3132 	/* locate an appropriate allocation class */
3133 	mc = spa_preferred_class(spa, zio->io_size, zio->io_prop.zp_type,
3134 	    zio->io_prop.zp_level, zio->io_prop.zp_zpl_smallblk);
3135 
3136 	if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE ||
3137 	    !mc->mc_alloc_throttle_enabled ||
3138 	    zio->io_child_type == ZIO_CHILD_GANG ||
3139 	    zio->io_flags & ZIO_FLAG_NODATA) {
3140 		return (ZIO_PIPELINE_CONTINUE);
3141 	}
3142 
3143 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
3144 
3145 	ASSERT3U(zio->io_queued_timestamp, >, 0);
3146 	ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);
3147 
3148 	zbookmark_phys_t *bm = &zio->io_bookmark;
3149 	/*
3150 	 * We want to try to use as many allocators as possible to help improve
3151 	 * performance, but we also want logically adjacent IOs to be physically
3152 	 * adjacent to improve sequential read performance. We chunk each object
3153 	 * into 2^20 block regions, and then hash based on the objset, object,
3154 	 * level, and region to accomplish both of these goals.
3155 	 */
3156 	zio->io_allocator = cityhash4(bm->zb_objset, bm->zb_object,
3157 	    bm->zb_level, bm->zb_blkid >> 20) % spa->spa_alloc_count;
3158 	mutex_enter(&spa->spa_alloc_locks[zio->io_allocator]);
3159 	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
3160 	zio->io_metaslab_class = mc;
3161 	avl_add(&spa->spa_alloc_trees[zio->io_allocator], zio);
3162 	nio = zio_io_to_allocate(spa, zio->io_allocator);
3163 	mutex_exit(&spa->spa_alloc_locks[zio->io_allocator]);
3164 
3165 	if (nio == zio)
3166 		return (ZIO_PIPELINE_CONTINUE);
3167 
3168 	if (nio != NULL) {
3169 		ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
3170 		/*
3171 		 * We are passing control to a new zio so make sure that
3172 		 * it is processed by a different thread. We do this to
3173 		 * avoid stack overflows that can occur when parents are
3174 		 * throttled and children are making progress. We allow
3175 		 * it to go to the head of the taskq since it's already
3176 		 * been waiting.
3177 		 */
3178 		zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
3179 	}
3180 	return (ZIO_PIPELINE_STOP);
3181 }
3182 
3183 static void
zio_allocate_dispatch(spa_t * spa,int allocator)3184 zio_allocate_dispatch(spa_t *spa, int allocator)
3185 {
3186 	zio_t *zio;
3187 
3188 	mutex_enter(&spa->spa_alloc_locks[allocator]);
3189 	zio = zio_io_to_allocate(spa, allocator);
3190 	mutex_exit(&spa->spa_alloc_locks[allocator]);
3191 	if (zio == NULL)
3192 		return;
3193 
3194 	ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);
3195 	ASSERT0(zio->io_error);
3196 	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
3197 }
3198 
3199 static int
zio_dva_allocate(zio_t * zio)3200 zio_dva_allocate(zio_t *zio)
3201 {
3202 	spa_t *spa = zio->io_spa;
3203 	metaslab_class_t *mc;
3204 	blkptr_t *bp = zio->io_bp;
3205 	int error;
3206 	int flags = 0;
3207 
3208 	if (zio->io_gang_leader == NULL) {
3209 		ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
3210 		zio->io_gang_leader = zio;
3211 	}
3212 
3213 	ASSERT(BP_IS_HOLE(bp));
3214 	ASSERT0(BP_GET_NDVAS(bp));
3215 	ASSERT3U(zio->io_prop.zp_copies, >, 0);
3216 	ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa));
3217 	ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp));
3218 
3219 	if (zio->io_flags & ZIO_FLAG_NODATA)
3220 		flags |= METASLAB_DONT_THROTTLE;
3221 	if (zio->io_flags & ZIO_FLAG_GANG_CHILD)
3222 		flags |= METASLAB_GANG_CHILD;
3223 	if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE)
3224 		flags |= METASLAB_ASYNC_ALLOC;
3225 
3226 	/*
3227 	 * if not already chosen, locate an appropriate allocation class
3228 	 */
3229 	mc = zio->io_metaslab_class;
3230 	if (mc == NULL) {
3231 		mc = spa_preferred_class(spa, zio->io_size,
3232 		    zio->io_prop.zp_type, zio->io_prop.zp_level,
3233 		    zio->io_prop.zp_zpl_smallblk);
3234 		zio->io_metaslab_class = mc;
3235 	}
3236 
3237 	error = metaslab_alloc(spa, mc, zio->io_size, bp,
3238 	    zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
3239 	    &zio->io_alloc_list, zio, zio->io_allocator);
3240 
3241 	/*
3242 	 * Fallback to normal class when an alloc class is full
3243 	 */
3244 	if (error == ENOSPC && mc != spa_normal_class(spa)) {
3245 		/*
3246 		 * If throttling, transfer reservation over to normal class.
3247 		 * The io_allocator slot can remain the same even though we
3248 		 * are switching classes.
3249 		 */
3250 		if (mc->mc_alloc_throttle_enabled &&
3251 		    (zio->io_flags & ZIO_FLAG_IO_ALLOCATING)) {
3252 			metaslab_class_throttle_unreserve(mc,
3253 			    zio->io_prop.zp_copies, zio->io_allocator, zio);
3254 			zio->io_flags &= ~ZIO_FLAG_IO_ALLOCATING;
3255 
3256 			mc = spa_normal_class(spa);
3257 			VERIFY(metaslab_class_throttle_reserve(mc,
3258 			    zio->io_prop.zp_copies, zio->io_allocator, zio,
3259 			    flags | METASLAB_MUST_RESERVE));
3260 		} else {
3261 			mc = spa_normal_class(spa);
3262 		}
3263 		zio->io_metaslab_class = mc;
3264 
3265 		error = metaslab_alloc(spa, mc, zio->io_size, bp,
3266 		    zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
3267 		    &zio->io_alloc_list, zio, zio->io_allocator);
3268 	}
3269 
3270 	if (error != 0) {
3271 		zfs_dbgmsg("%s: metaslab allocation failure: zio %p, "
3272 		    "size %llu, error %d", spa_name(spa), zio, zio->io_size,
3273 		    error);
3274 		if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
3275 			return (zio_write_gang_block(zio));
3276 		zio->io_error = error;
3277 	}
3278 
3279 	return (ZIO_PIPELINE_CONTINUE);
3280 }
3281 
3282 static int
zio_dva_free(zio_t * zio)3283 zio_dva_free(zio_t *zio)
3284 {
3285 	metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);
3286 
3287 	return (ZIO_PIPELINE_CONTINUE);
3288 }
3289 
3290 static int
zio_dva_claim(zio_t * zio)3291 zio_dva_claim(zio_t *zio)
3292 {
3293 	int error;
3294 
3295 	error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg);
3296 	if (error)
3297 		zio->io_error = error;
3298 
3299 	return (ZIO_PIPELINE_CONTINUE);
3300 }
3301 
3302 /*
3303  * Undo an allocation.  This is used by zio_done() when an I/O fails
3304  * and we want to give back the block we just allocated.
3305  * This handles both normal blocks and gang blocks.
3306  */
3307 static void
zio_dva_unallocate(zio_t * zio,zio_gang_node_t * gn,blkptr_t * bp)3308 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
3309 {
3310 	ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp));
3311 	ASSERT(zio->io_bp_override == NULL);
3312 
3313 	if (!BP_IS_HOLE(bp))
3314 		metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE);
3315 
3316 	if (gn != NULL) {
3317 		for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
3318 			zio_dva_unallocate(zio, gn->gn_child[g],
3319 			    &gn->gn_gbh->zg_blkptr[g]);
3320 		}
3321 	}
3322 }
3323 
3324 /*
3325  * Try to allocate an intent log block.  Return 0 on success, errno on failure.
3326  */
3327 int
zio_alloc_zil(spa_t * spa,objset_t * os,uint64_t txg,blkptr_t * new_bp,blkptr_t * old_bp,uint64_t size,boolean_t * slog)3328 zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
3329     blkptr_t *old_bp, uint64_t size, boolean_t *slog)
3330 {
3331 	int error = 1;
3332 	zio_alloc_list_t io_alloc_list;
3333 
3334 	ASSERT(txg > spa_syncing_txg(spa));
3335 
3336 	metaslab_trace_init(&io_alloc_list);
3337 
3338 	/*
3339 	 * Block pointer fields are useful to metaslabs for stats and debugging.
3340 	 * Fill in the obvious ones before calling into metaslab_alloc().
3341 	 */
3342 	BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
3343 	BP_SET_PSIZE(new_bp, size);
3344 	BP_SET_LEVEL(new_bp, 0);
3345 
3346 	/*
3347 	 * When allocating a zil block, we don't have information about
3348 	 * the final destination of the block except the objset it's part
3349 	 * of, so we just hash the objset ID to pick the allocator to get
3350 	 * some parallelism.
3351 	 */
3352 	error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
3353 	    txg, old_bp, METASLAB_HINTBP_AVOID, &io_alloc_list, NULL,
3354 	    cityhash4(0, 0, 0,
3355 	    os->os_dsl_dataset->ds_object) % spa->spa_alloc_count);
3356 	if (error == 0) {
3357 		*slog = TRUE;
3358 	} else {
3359 		error = metaslab_alloc(spa, spa_normal_class(spa), size,
3360 		    new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID,
3361 		    &io_alloc_list, NULL, cityhash4(0, 0, 0,
3362 		    os->os_dsl_dataset->ds_object) % spa->spa_alloc_count);
3363 		if (error == 0)
3364 			*slog = FALSE;
3365 	}
3366 	metaslab_trace_fini(&io_alloc_list);
3367 
3368 	if (error == 0) {
3369 		BP_SET_LSIZE(new_bp, size);
3370 		BP_SET_PSIZE(new_bp, size);
3371 		BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF);
3372 		BP_SET_CHECKSUM(new_bp,
3373 		    spa_version(spa) >= SPA_VERSION_SLIM_ZIL
3374 		    ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG);
3375 		BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
3376 		BP_SET_LEVEL(new_bp, 0);
3377 		BP_SET_DEDUP(new_bp, 0);
3378 		BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER);
3379 
3380 		/*
3381 		 * encrypted blocks will require an IV and salt. We generate
3382 		 * these now since we will not be rewriting the bp at
3383 		 * rewrite time.
3384 		 */
3385 		if (os->os_encrypted) {
3386 			uint8_t iv[ZIO_DATA_IV_LEN];
3387 			uint8_t salt[ZIO_DATA_SALT_LEN];
3388 
3389 			BP_SET_CRYPT(new_bp, B_TRUE);
3390 			VERIFY0(spa_crypt_get_salt(spa,
3391 			    dmu_objset_id(os), salt));
3392 			VERIFY0(zio_crypt_generate_iv(iv));
3393 
3394 			zio_crypt_encode_params_bp(new_bp, salt, iv);
3395 		}
3396 	} else {
3397 		zfs_dbgmsg("%s: zil block allocation failure: "
3398 		    "size %llu, error %d", spa_name(spa), size, error);
3399 	}
3400 
3401 	return (error);
3402 }
3403 
3404 /*
3405  * ==========================================================================
3406  * Read and write to physical devices
3407  * ==========================================================================
3408  */
3409 
3410 /*
3411  * Issue an I/O to the underlying vdev. Typically the issue pipeline
3412  * stops after this stage and will resume upon I/O completion.
3413  * However, there are instances where the vdev layer may need to
3414  * continue the pipeline when an I/O was not issued. Since the I/O
3415  * that was sent to the vdev layer might be different than the one
3416  * currently active in the pipeline (see vdev_queue_io()), we explicitly
3417  * force the underlying vdev layers to call either zio_execute() or
3418  * zio_interrupt() to ensure that the pipeline continues with the correct I/O.
3419  */
3420 static int
zio_vdev_io_start(zio_t * zio)3421 zio_vdev_io_start(zio_t *zio)
3422 {
3423 	vdev_t *vd = zio->io_vd;
3424 	uint64_t align;
3425 	spa_t *spa = zio->io_spa;
3426 
3427 	zio->io_delay = 0;
3428 
3429 	ASSERT(zio->io_error == 0);
3430 	ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);
3431 
3432 	if (vd == NULL) {
3433 		if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
3434 			spa_config_enter(spa, SCL_ZIO, zio, RW_READER);
3435 
3436 		/*
3437 		 * The mirror_ops handle multiple DVAs in a single BP.
3438 		 */
3439 		vdev_mirror_ops.vdev_op_io_start(zio);
3440 		return (ZIO_PIPELINE_STOP);
3441 	}
3442 
3443 	ASSERT3P(zio->io_logical, !=, zio);
3444 	if (zio->io_type == ZIO_TYPE_WRITE) {
3445 		ASSERT(spa->spa_trust_config);
3446 
3447 		/*
3448 		 * Note: the code can handle other kinds of writes,
3449 		 * but we don't expect them.
3450 		 */
3451 		if (zio->io_vd->vdev_removing) {
3452 			ASSERT(zio->io_flags &
3453 			    (ZIO_FLAG_PHYSICAL | ZIO_FLAG_SELF_HEAL |
3454 			    ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE));
3455 		}
3456 	}
3457 
3458 	align = 1ULL << vd->vdev_top->vdev_ashift;
3459 
3460 	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) &&
3461 	    P2PHASE(zio->io_size, align) != 0) {
3462 		/* Transform logical writes to be a full physical block size. */
3463 		uint64_t asize = P2ROUNDUP(zio->io_size, align);
3464 		abd_t *abuf = abd_alloc_sametype(zio->io_abd, asize);
3465 		ASSERT(vd == vd->vdev_top);
3466 		if (zio->io_type == ZIO_TYPE_WRITE) {
3467 			abd_copy(abuf, zio->io_abd, zio->io_size);
3468 			abd_zero_off(abuf, zio->io_size, asize - zio->io_size);
3469 		}
3470 		zio_push_transform(zio, abuf, asize, asize, zio_subblock);
3471 	}
3472 
3473 	/*
3474 	 * If this is not a physical io, make sure that it is properly aligned
3475 	 * before proceeding.
3476 	 */
3477 	if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) {
3478 		ASSERT0(P2PHASE(zio->io_offset, align));
3479 		ASSERT0(P2PHASE(zio->io_size, align));
3480 	} else {
3481 		/*
3482 		 * For physical writes, we allow 512b aligned writes and assume
3483 		 * the device will perform a read-modify-write as necessary.
3484 		 */
3485 		ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE));
3486 		ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE));
3487 	}
3488 
3489 	VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa));
3490 
3491 	/*
3492 	 * If this is a repair I/O, and there's no self-healing involved --
3493 	 * that is, we're just resilvering what we expect to resilver --
3494 	 * then don't do the I/O unless zio's txg is actually in vd's DTL.
3495 	 * This prevents spurious resilvering.
3496 	 *
3497 	 * There are a few ways that we can end up creating these spurious
3498 	 * resilver i/os:
3499 	 *
3500 	 * 1. A resilver i/o will be issued if any DVA in the BP has a
3501 	 * dirty DTL.  The mirror code will issue resilver writes to
3502 	 * each DVA, including the one(s) that are not on vdevs with dirty
3503 	 * DTLs.
3504 	 *
3505 	 * 2. With nested replication, which happens when we have a
3506 	 * "replacing" or "spare" vdev that's a child of a mirror or raidz.
3507 	 * For example, given mirror(replacing(A+B), C), it's likely that
3508 	 * only A is out of date (it's the new device). In this case, we'll
3509 	 * read from C, then use the data to resilver A+B -- but we don't
3510 	 * actually want to resilver B, just A. The top-level mirror has no
3511 	 * way to know this, so instead we just discard unnecessary repairs
3512 	 * as we work our way down the vdev tree.
3513 	 *
3514 	 * 3. ZTEST also creates mirrors of mirrors, mirrors of raidz, etc.
3515 	 * The same logic applies to any form of nested replication: ditto
3516 	 * + mirror, RAID-Z + replacing, etc.
3517 	 *
3518 	 * However, indirect vdevs point off to other vdevs which may have
3519 	 * DTL's, so we never bypass them.  The child i/os on concrete vdevs
3520 	 * will be properly bypassed instead.
3521 	 */
3522 	if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
3523 	    !(zio->io_flags & ZIO_FLAG_SELF_HEAL) &&
3524 	    zio->io_txg != 0 &&	/* not a delegated i/o */
3525 	    vd->vdev_ops != &vdev_indirect_ops &&
3526 	    !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
3527 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);
3528 		zio_vdev_io_bypass(zio);
3529 		return (ZIO_PIPELINE_CONTINUE);
3530 	}
3531 
3532 	if (vd->vdev_ops->vdev_op_leaf && (zio->io_type == ZIO_TYPE_READ ||
3533 	    zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM)) {
3534 
3535 		if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio))
3536 			return (ZIO_PIPELINE_CONTINUE);
3537 
3538 		if ((zio = vdev_queue_io(zio)) == NULL)
3539 			return (ZIO_PIPELINE_STOP);
3540 
3541 		if (!vdev_accessible(vd, zio)) {
3542 			zio->io_error = SET_ERROR(ENXIO);
3543 			zio_interrupt(zio);
3544 			return (ZIO_PIPELINE_STOP);
3545 		}
3546 		zio->io_delay = gethrtime();
3547 	}
3548 
3549 	vd->vdev_ops->vdev_op_io_start(zio);
3550 	return (ZIO_PIPELINE_STOP);
3551 }
3552 
3553 static int
zio_vdev_io_done(zio_t * zio)3554 zio_vdev_io_done(zio_t *zio)
3555 {
3556 	vdev_t *vd = zio->io_vd;
3557 	vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;
3558 	boolean_t unexpected_error = B_FALSE;
3559 
3560 	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
3561 		return (ZIO_PIPELINE_STOP);
3562 	}
3563 
3564 	ASSERT(zio->io_type == ZIO_TYPE_READ ||
3565 	    zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM);
3566 
3567 	if (zio->io_delay)
3568 		zio->io_delay = gethrtime() - zio->io_delay;
3569 
3570 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf) {
3571 
3572 		vdev_queue_io_done(zio);
3573 
3574 		if (zio->io_type == ZIO_TYPE_WRITE)
3575 			vdev_cache_write(zio);
3576 
3577 		if (zio_injection_enabled && zio->io_error == 0)
3578 			zio->io_error = zio_handle_device_injection(vd,
3579 			    zio, EIO);
3580 
3581 		if (zio_injection_enabled && zio->io_error == 0)
3582 			zio->io_error = zio_handle_label_injection(zio, EIO);
3583 
3584 		if (zio->io_error && zio->io_type != ZIO_TYPE_TRIM) {
3585 			if (!vdev_accessible(vd, zio)) {
3586 				zio->io_error = SET_ERROR(ENXIO);
3587 			} else {
3588 				unexpected_error = B_TRUE;
3589 			}
3590 		}
3591 	}
3592 
3593 	ops->vdev_op_io_done(zio);
3594 
3595 	if (unexpected_error)
3596 		VERIFY(vdev_probe(vd, zio) == NULL);
3597 
3598 	return (ZIO_PIPELINE_CONTINUE);
3599 }
3600 
3601 /*
3602  * This function is used to change the priority of an existing zio that is
3603  * currently in-flight. This is used by the arc to upgrade priority in the
3604  * event that a demand read is made for a block that is currently queued
3605  * as a scrub or async read IO. Otherwise, the high priority read request
3606  * would end up having to wait for the lower priority IO.
3607  */
3608 void
zio_change_priority(zio_t * pio,zio_priority_t priority)3609 zio_change_priority(zio_t *pio, zio_priority_t priority)
3610 {
3611 	zio_t *cio, *cio_next;
3612 	zio_link_t *zl = NULL;
3613 
3614 	ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
3615 
3616 	if (pio->io_vd != NULL && pio->io_vd->vdev_ops->vdev_op_leaf) {
3617 		vdev_queue_change_io_priority(pio, priority);
3618 	} else {
3619 		pio->io_priority = priority;
3620 	}
3621 
3622 	mutex_enter(&pio->io_lock);
3623 	for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
3624 		cio_next = zio_walk_children(pio, &zl);
3625 		zio_change_priority(cio, priority);
3626 	}
3627 	mutex_exit(&pio->io_lock);
3628 }
3629 
3630 /*
3631  * For non-raidz ZIOs, we can just copy aside the bad data read from the
3632  * disk, and use that to finish the checksum ereport later.
3633  */
3634 static void
zio_vsd_default_cksum_finish(zio_cksum_report_t * zcr,const abd_t * good_buf)3635 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr,
3636     const abd_t *good_buf)
3637 {
3638 	/* no processing needed */
3639 	zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE);
3640 }
3641 
3642 /*ARGSUSED*/
3643 void
zio_vsd_default_cksum_report(zio_t * zio,zio_cksum_report_t * zcr,void * ignored)3644 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored)
3645 {
3646 	void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size);
3647 
3648 	abd_copy(abd, zio->io_abd, zio->io_size);
3649 
3650 	zcr->zcr_cbinfo = zio->io_size;
3651 	zcr->zcr_cbdata = abd;
3652 	zcr->zcr_finish = zio_vsd_default_cksum_finish;
3653 	zcr->zcr_free = zio_abd_free;
3654 }
3655 
3656 static int
zio_vdev_io_assess(zio_t * zio)3657 zio_vdev_io_assess(zio_t *zio)
3658 {
3659 	vdev_t *vd = zio->io_vd;
3660 
3661 	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
3662 		return (ZIO_PIPELINE_STOP);
3663 	}
3664 
3665 	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
3666 		spa_config_exit(zio->io_spa, SCL_ZIO, zio);
3667 
3668 	if (zio->io_vsd != NULL) {
3669 		zio->io_vsd_ops->vsd_free(zio);
3670 		zio->io_vsd = NULL;
3671 	}
3672 
3673 	if (zio_injection_enabled && zio->io_error == 0)
3674 		zio->io_error = zio_handle_fault_injection(zio, EIO);
3675 
3676 	/*
3677 	 * If the I/O failed, determine whether we should attempt to retry it.
3678 	 *
3679 	 * On retry, we cut in line in the issue queue, since we don't want
3680 	 * compression/checksumming/etc. work to prevent our (cheap) IO reissue.
3681 	 */
3682 	if (zio->io_error && vd == NULL &&
3683 	    !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) {
3684 		ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE));	/* not a leaf */
3685 		ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS));	/* not a leaf */
3686 		zio->io_error = 0;
3687 		zio->io_flags |= ZIO_FLAG_IO_RETRY |
3688 		    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE;
3689 		zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;
3690 		zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE,
3691 		    zio_requeue_io_start_cut_in_line);
3692 		return (ZIO_PIPELINE_STOP);
3693 	}
3694 
3695 	/*
3696 	 * If we got an error on a leaf device, convert it to ENXIO
3697 	 * if the device is not accessible at all.
3698 	 */
3699 	if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf &&
3700 	    !vdev_accessible(vd, zio))
3701 		zio->io_error = SET_ERROR(ENXIO);
3702 
3703 	/*
3704 	 * If we can't write to an interior vdev (mirror or RAID-Z),
3705 	 * set vdev_cant_write so that we stop trying to allocate from it.
3706 	 */
3707 	if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE &&
3708 	    vd != NULL && !vd->vdev_ops->vdev_op_leaf) {
3709 		vd->vdev_cant_write = B_TRUE;
3710 	}
3711 
3712 	/*
3713 	 * If a cache flush returns ENOTSUP or ENOTTY, we know that no future
3714 	 * attempts will ever succeed. In this case we set a persistent
3715 	 * boolean flag so that we don't bother with it in the future.
3716 	 */
3717 	if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) &&
3718 	    zio->io_type == ZIO_TYPE_IOCTL &&
3719 	    zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL)
3720 		vd->vdev_nowritecache = B_TRUE;
3721 
3722 	if (zio->io_error)
3723 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
3724 
3725 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&
3726 	    zio->io_physdone != NULL) {
3727 		ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED));
3728 		ASSERT(zio->io_child_type == ZIO_CHILD_VDEV);
3729 		zio->io_physdone(zio->io_logical);
3730 	}
3731 
3732 	return (ZIO_PIPELINE_CONTINUE);
3733 }
3734 
3735 void
zio_vdev_io_reissue(zio_t * zio)3736 zio_vdev_io_reissue(zio_t *zio)
3737 {
3738 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
3739 	ASSERT(zio->io_error == 0);
3740 
3741 	zio->io_stage >>= 1;
3742 }
3743 
3744 void
zio_vdev_io_redone(zio_t * zio)3745 zio_vdev_io_redone(zio_t *zio)
3746 {
3747 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE);
3748 
3749 	zio->io_stage >>= 1;
3750 }
3751 
3752 void
zio_vdev_io_bypass(zio_t * zio)3753 zio_vdev_io_bypass(zio_t *zio)
3754 {
3755 	ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
3756 	ASSERT(zio->io_error == 0);
3757 
3758 	zio->io_flags |= ZIO_FLAG_IO_BYPASS;
3759 	zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;
3760 }
3761 
3762 /*
3763  * ==========================================================================
3764  * Encrypt and store encryption parameters
3765  * ==========================================================================
3766  */
3767 
3768 
3769 /*
3770  * This function is used for ZIO_STAGE_ENCRYPT. It is responsible for
3771  * managing the storage of encryption parameters and passing them to the
3772  * lower-level encryption functions.
3773  */
3774 static int
zio_encrypt(zio_t * zio)3775 zio_encrypt(zio_t *zio)
3776 {
3777 	zio_prop_t *zp = &zio->io_prop;
3778 	spa_t *spa = zio->io_spa;
3779 	blkptr_t *bp = zio->io_bp;
3780 	uint64_t psize = BP_GET_PSIZE(bp);
3781 	uint64_t dsobj = zio->io_bookmark.zb_objset;
3782 	dmu_object_type_t ot = BP_GET_TYPE(bp);
3783 	void *enc_buf = NULL;
3784 	abd_t *eabd = NULL;
3785 	uint8_t salt[ZIO_DATA_SALT_LEN];
3786 	uint8_t iv[ZIO_DATA_IV_LEN];
3787 	uint8_t mac[ZIO_DATA_MAC_LEN];
3788 	boolean_t no_crypt = B_FALSE;
3789 
3790 	/* the root zio already encrypted the data */
3791 	if (zio->io_child_type == ZIO_CHILD_GANG)
3792 		return (ZIO_PIPELINE_CONTINUE);
3793 
3794 	/* only ZIL blocks are re-encrypted on rewrite */
3795 	if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG)
3796 		return (ZIO_PIPELINE_CONTINUE);
3797 
3798 	if (!(zp->zp_encrypt || BP_IS_ENCRYPTED(bp))) {
3799 		BP_SET_CRYPT(bp, B_FALSE);
3800 		return (ZIO_PIPELINE_CONTINUE);
3801 	}
3802 
3803 	/* if we are doing raw encryption set the provided encryption params */
3804 	if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) {
3805 		ASSERT0(BP_GET_LEVEL(bp));
3806 		BP_SET_CRYPT(bp, B_TRUE);
3807 		BP_SET_BYTEORDER(bp, zp->zp_byteorder);
3808 		if (ot != DMU_OT_OBJSET)
3809 			zio_crypt_encode_mac_bp(bp, zp->zp_mac);
3810 
3811 		/* dnode blocks must be written out in the provided byteorder */
3812 		if (zp->zp_byteorder != ZFS_HOST_BYTEORDER &&
3813 		    ot == DMU_OT_DNODE) {
3814 			void *bswap_buf = zio_buf_alloc(psize);
3815 			abd_t *babd = abd_get_from_buf(bswap_buf, psize);
3816 
3817 			ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
3818 			abd_copy_to_buf(bswap_buf, zio->io_abd, psize);
3819 			dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf,
3820 			    psize);
3821 
3822 			abd_take_ownership_of_buf(babd, B_TRUE);
3823 			zio_push_transform(zio, babd, psize, psize, NULL);
3824 		}
3825 
3826 		if (DMU_OT_IS_ENCRYPTED(ot))
3827 			zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv);
3828 		return (ZIO_PIPELINE_CONTINUE);
3829 	}
3830 
3831 	/* indirect blocks only maintain a cksum of the lower level MACs */
3832 	if (BP_GET_LEVEL(bp) > 0) {
3833 		BP_SET_CRYPT(bp, B_TRUE);
3834 		VERIFY0(zio_crypt_do_indirect_mac_checksum_abd(B_TRUE,
3835 		    zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp),
3836 		    mac));
3837 		zio_crypt_encode_mac_bp(bp, mac);
3838 		return (ZIO_PIPELINE_CONTINUE);
3839 	}
3840 
3841 	/*
3842 	 * Objset blocks are a special case since they have 2 256-bit MACs
3843 	 * embedded within them.
3844 	 */
3845 	if (ot == DMU_OT_OBJSET) {
3846 		ASSERT0(DMU_OT_IS_ENCRYPTED(ot));
3847 		ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
3848 		BP_SET_CRYPT(bp, B_TRUE);
3849 		VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj,
3850 		    zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp)));
3851 		return (ZIO_PIPELINE_CONTINUE);
3852 	}
3853 
3854 	/* unencrypted object types are only authenticated with a MAC */
3855 	if (!DMU_OT_IS_ENCRYPTED(ot)) {
3856 		BP_SET_CRYPT(bp, B_TRUE);
3857 		VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj,
3858 		    zio->io_abd, psize, mac));
3859 		zio_crypt_encode_mac_bp(bp, mac);
3860 		return (ZIO_PIPELINE_CONTINUE);
3861 	}
3862 
3863 	/*
3864 	 * Later passes of sync-to-convergence may decide to rewrite data
3865 	 * in place to avoid more disk reallocations. This presents a problem
3866 	 * for encryption because this consitutes rewriting the new data with
3867 	 * the same encryption key and IV. However, this only applies to blocks
3868 	 * in the MOS (particularly the spacemaps) and we do not encrypt the
3869 	 * MOS. We assert that the zio is allocating or an intent log write
3870 	 * to enforce this.
3871 	 */
3872 	ASSERT(IO_IS_ALLOCATING(zio) || ot == DMU_OT_INTENT_LOG);
3873 	ASSERT(BP_GET_LEVEL(bp) == 0 || ot == DMU_OT_INTENT_LOG);
3874 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION));
3875 	ASSERT3U(psize, !=, 0);
3876 
3877 	enc_buf = zio_buf_alloc(psize);
3878 	eabd = abd_get_from_buf(enc_buf, psize);
3879 	abd_take_ownership_of_buf(eabd, B_TRUE);
3880 
3881 	/*
3882 	 * For an explanation of what encryption parameters are stored
3883 	 * where, see the block comment in zio_crypt.c.
3884 	 */
3885 	if (ot == DMU_OT_INTENT_LOG) {
3886 		zio_crypt_decode_params_bp(bp, salt, iv);
3887 	} else {
3888 		BP_SET_CRYPT(bp, B_TRUE);
3889 	}
3890 
3891 	/* Perform the encryption. This should not fail */
3892 	VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark,
3893 	    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
3894 	    salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt));
3895 
3896 	/* encode encryption metadata into the bp */
3897 	if (ot == DMU_OT_INTENT_LOG) {
3898 		/*
3899 		 * ZIL blocks store the MAC in the embedded checksum, so the
3900 		 * transform must always be applied.
3901 		 */
3902 		zio_crypt_encode_mac_zil(enc_buf, mac);
3903 		zio_push_transform(zio, eabd, psize, psize, NULL);
3904 	} else {
3905 		BP_SET_CRYPT(bp, B_TRUE);
3906 		zio_crypt_encode_params_bp(bp, salt, iv);
3907 		zio_crypt_encode_mac_bp(bp, mac);
3908 
3909 		if (no_crypt) {
3910 			ASSERT3U(ot, ==, DMU_OT_DNODE);
3911 			abd_free(eabd);
3912 		} else {
3913 			zio_push_transform(zio, eabd, psize, psize, NULL);
3914 		}
3915 	}
3916 
3917 	return (ZIO_PIPELINE_CONTINUE);
3918 }
3919 
3920 /*
3921  * ==========================================================================
3922  * Generate and verify checksums
3923  * ==========================================================================
3924  */
3925 static int
zio_checksum_generate(zio_t * zio)3926 zio_checksum_generate(zio_t *zio)
3927 {
3928 	blkptr_t *bp = zio->io_bp;
3929 	enum zio_checksum checksum;
3930 
3931 	if (bp == NULL) {
3932 		/*
3933 		 * This is zio_write_phys().
3934 		 * We're either generating a label checksum, or none at all.
3935 		 */
3936 		checksum = zio->io_prop.zp_checksum;
3937 
3938 		if (checksum == ZIO_CHECKSUM_OFF)
3939 			return (ZIO_PIPELINE_CONTINUE);
3940 
3941 		ASSERT(checksum == ZIO_CHECKSUM_LABEL);
3942 	} else {
3943 		if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) {
3944 			ASSERT(!IO_IS_ALLOCATING(zio));
3945 			checksum = ZIO_CHECKSUM_GANG_HEADER;
3946 		} else {
3947 			checksum = BP_GET_CHECKSUM(bp);
3948 		}
3949 	}
3950 
3951 	zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size);
3952 
3953 	return (ZIO_PIPELINE_CONTINUE);
3954 }
3955 
3956 static int
zio_checksum_verify(zio_t * zio)3957 zio_checksum_verify(zio_t *zio)
3958 {
3959 	zio_bad_cksum_t info;
3960 	blkptr_t *bp = zio->io_bp;
3961 	int error;
3962 
3963 	ASSERT(zio->io_vd != NULL);
3964 
3965 	if (bp == NULL) {
3966 		/*
3967 		 * This is zio_read_phys().
3968 		 * We're either verifying a label checksum, or nothing at all.
3969 		 */
3970 		if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
3971 			return (ZIO_PIPELINE_CONTINUE);
3972 
3973 		ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL);
3974 	}
3975 
3976 	if ((error = zio_checksum_error(zio, &info)) != 0) {
3977 		zio->io_error = error;
3978 		if (error == ECKSUM &&
3979 		    !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) {
3980 			zfs_ereport_start_checksum(zio->io_spa,
3981 			    zio->io_vd, &zio->io_bookmark, zio,
3982 			    zio->io_offset, zio->io_size, NULL, &info);
3983 		}
3984 	}
3985 
3986 	return (ZIO_PIPELINE_CONTINUE);
3987 }
3988 
3989 /*
3990  * Called by RAID-Z to ensure we don't compute the checksum twice.
3991  */
3992 void
zio_checksum_verified(zio_t * zio)3993 zio_checksum_verified(zio_t *zio)
3994 {
3995 	zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
3996 }
3997 
3998 /*
3999  * ==========================================================================
4000  * Error rank.  Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other.
4001  * An error of 0 indicates success.  ENXIO indicates whole-device failure,
4002  * which may be transient (e.g. unplugged) or permament.  ECKSUM and EIO
4003  * indicate errors that are specific to one I/O, and most likely permanent.
4004  * Any other error is presumed to be worse because we weren't expecting it.
4005  * ==========================================================================
4006  */
4007 int
zio_worst_error(int e1,int e2)4008 zio_worst_error(int e1, int e2)
4009 {
4010 	static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO };
4011 	int r1, r2;
4012 
4013 	for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++)
4014 		if (e1 == zio_error_rank[r1])
4015 			break;
4016 
4017 	for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++)
4018 		if (e2 == zio_error_rank[r2])
4019 			break;
4020 
4021 	return (r1 > r2 ? e1 : e2);
4022 }
4023 
4024 /*
4025  * ==========================================================================
4026  * I/O completion
4027  * ==========================================================================
4028  */
4029 static int
zio_ready(zio_t * zio)4030 zio_ready(zio_t *zio)
4031 {
4032 	blkptr_t *bp = zio->io_bp;
4033 	zio_t *pio, *pio_next;
4034 	zio_link_t *zl = NULL;
4035 
4036 	if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT,
4037 	    ZIO_WAIT_READY)) {
4038 		return (ZIO_PIPELINE_STOP);
4039 	}
4040 
4041 	if (zio->io_ready) {
4042 		ASSERT(IO_IS_ALLOCATING(zio));
4043 		ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) ||
4044 		    (zio->io_flags & ZIO_FLAG_NOPWRITE));
4045 		ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
4046 
4047 		zio->io_ready(zio);
4048 	}
4049 
4050 	if (bp != NULL && bp != &zio->io_bp_copy)
4051 		zio->io_bp_copy = *bp;
4052 
4053 	if (zio->io_error != 0) {
4054 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
4055 
4056 		if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
4057 			ASSERT(IO_IS_ALLOCATING(zio));
4058 			ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
4059 			ASSERT(zio->io_metaslab_class != NULL);
4060 
4061 			/*
4062 			 * We were unable to allocate anything, unreserve and
4063 			 * issue the next I/O to allocate.
4064 			 */
4065 			metaslab_class_throttle_unreserve(
4066 			    zio->io_metaslab_class, zio->io_prop.zp_copies,
4067 			    zio->io_allocator, zio);
4068 			zio_allocate_dispatch(zio->io_spa, zio->io_allocator);
4069 		}
4070 	}
4071 
4072 	mutex_enter(&zio->io_lock);
4073 	zio->io_state[ZIO_WAIT_READY] = 1;
4074 	pio = zio_walk_parents(zio, &zl);
4075 	mutex_exit(&zio->io_lock);
4076 
4077 	/*
4078 	 * As we notify zio's parents, new parents could be added.
4079 	 * New parents go to the head of zio's io_parent_list, however,
4080 	 * so we will (correctly) not notify them.  The remainder of zio's
4081 	 * io_parent_list, from 'pio_next' onward, cannot change because
4082 	 * all parents must wait for us to be done before they can be done.
4083 	 */
4084 	for (; pio != NULL; pio = pio_next) {
4085 		pio_next = zio_walk_parents(zio, &zl);
4086 		zio_notify_parent(pio, zio, ZIO_WAIT_READY);
4087 	}
4088 
4089 	if (zio->io_flags & ZIO_FLAG_NODATA) {
4090 		if (BP_IS_GANG(bp)) {
4091 			zio->io_flags &= ~ZIO_FLAG_NODATA;
4092 		} else {
4093 			ASSERT((uintptr_t)zio->io_abd < SPA_MAXBLOCKSIZE);
4094 			zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
4095 		}
4096 	}
4097 
4098 	if (zio_injection_enabled &&
4099 	    zio->io_spa->spa_syncing_txg == zio->io_txg)
4100 		zio_handle_ignored_writes(zio);
4101 
4102 	return (ZIO_PIPELINE_CONTINUE);
4103 }
4104 
4105 /*
4106  * Update the allocation throttle accounting.
4107  */
4108 static void
zio_dva_throttle_done(zio_t * zio)4109 zio_dva_throttle_done(zio_t *zio)
4110 {
4111 	zio_t *lio = zio->io_logical;
4112 	zio_t *pio = zio_unique_parent(zio);
4113 	vdev_t *vd = zio->io_vd;
4114 	int flags = METASLAB_ASYNC_ALLOC;
4115 
4116 	ASSERT3P(zio->io_bp, !=, NULL);
4117 	ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE);
4118 	ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE);
4119 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
4120 	ASSERT(vd != NULL);
4121 	ASSERT3P(vd, ==, vd->vdev_top);
4122 	ASSERT(!(zio->io_flags & (ZIO_FLAG_IO_REPAIR | ZIO_FLAG_IO_RETRY)));
4123 	ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING);
4124 	ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE));
4125 	ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA));
4126 
4127 	/*
4128 	 * Parents of gang children can have two flavors -- ones that
4129 	 * allocated the gang header (will have ZIO_FLAG_IO_REWRITE set)
4130 	 * and ones that allocated the constituent blocks. The allocation
4131 	 * throttle needs to know the allocating parent zio so we must find
4132 	 * it here.
4133 	 */
4134 	if (pio->io_child_type == ZIO_CHILD_GANG) {
4135 		/*
4136 		 * If our parent is a rewrite gang child then our grandparent
4137 		 * would have been the one that performed the allocation.
4138 		 */
4139 		if (pio->io_flags & ZIO_FLAG_IO_REWRITE)
4140 			pio = zio_unique_parent(pio);
4141 		flags |= METASLAB_GANG_CHILD;
4142 	}
4143 
4144 	ASSERT(IO_IS_ALLOCATING(pio));
4145 	ASSERT3P(zio, !=, zio->io_logical);
4146 	ASSERT(zio->io_logical != NULL);
4147 	ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
4148 	ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);
4149 	ASSERT(zio->io_metaslab_class != NULL);
4150 
4151 	mutex_enter(&pio->io_lock);
4152 	metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags,
4153 	    pio->io_allocator, B_TRUE);
4154 	mutex_exit(&pio->io_lock);
4155 
4156 	metaslab_class_throttle_unreserve(zio->io_metaslab_class, 1,
4157 	    pio->io_allocator, pio);
4158 
4159 	/*
4160 	 * Call into the pipeline to see if there is more work that
4161 	 * needs to be done. If there is work to be done it will be
4162 	 * dispatched to another taskq thread.
4163 	 */
4164 	zio_allocate_dispatch(zio->io_spa, pio->io_allocator);
4165 }
4166 
4167 static int
zio_done(zio_t * zio)4168 zio_done(zio_t *zio)
4169 {
4170 	spa_t *spa = zio->io_spa;
4171 	zio_t *lio = zio->io_logical;
4172 	blkptr_t *bp = zio->io_bp;
4173 	vdev_t *vd = zio->io_vd;
4174 	uint64_t psize = zio->io_size;
4175 	zio_t *pio, *pio_next;
4176 	zio_link_t *zl = NULL;
4177 
4178 	/*
4179 	 * If our children haven't all completed,
4180 	 * wait for them and then repeat this pipeline stage.
4181 	 */
4182 	if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) {
4183 		return (ZIO_PIPELINE_STOP);
4184 	}
4185 
4186 	/*
4187 	 * If the allocation throttle is enabled, then update the accounting.
4188 	 * We only track child I/Os that are part of an allocating async
4189 	 * write. We must do this since the allocation is performed
4190 	 * by the logical I/O but the actual write is done by child I/Os.
4191 	 */
4192 	if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING &&
4193 	    zio->io_child_type == ZIO_CHILD_VDEV) {
4194 		ASSERT(zio->io_metaslab_class != NULL);
4195 		ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled);
4196 		zio_dva_throttle_done(zio);
4197 	}
4198 
4199 	/*
4200 	 * If the allocation throttle is enabled, verify that
4201 	 * we have decremented the refcounts for every I/O that was throttled.
4202 	 */
4203 	if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
4204 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);
4205 		ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
4206 		ASSERT(bp != NULL);
4207 
4208 		metaslab_group_alloc_verify(spa, zio->io_bp, zio,
4209 		    zio->io_allocator);
4210 		VERIFY(zfs_refcount_not_held(
4211 		    &zio->io_metaslab_class->mc_alloc_slots[zio->io_allocator],
4212 		    zio));
4213 	}
4214 
4215 	for (int c = 0; c < ZIO_CHILD_TYPES; c++)
4216 		for (int w = 0; w < ZIO_WAIT_TYPES; w++)
4217 			ASSERT(zio->io_children[c][w] == 0);
4218 
4219 	if (bp != NULL && !BP_IS_EMBEDDED(bp)) {
4220 		ASSERT(bp->blk_pad[0] == 0);
4221 		ASSERT(bp->blk_pad[1] == 0);
4222 		ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 ||
4223 		    (bp == zio_unique_parent(zio)->io_bp));
4224 		if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
4225 		    zio->io_bp_override == NULL &&
4226 		    !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
4227 			ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp));
4228 			ASSERT(BP_COUNT_GANG(bp) == 0 ||
4229 			    (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp)));
4230 		}
4231 		if (zio->io_flags & ZIO_FLAG_NOPWRITE)
4232 			VERIFY(BP_EQUAL(bp, &zio->io_bp_orig));
4233 	}
4234 
4235 	/*
4236 	 * If there were child vdev/gang/ddt errors, they apply to us now.
4237 	 */
4238 	zio_inherit_child_errors(zio, ZIO_CHILD_VDEV);
4239 	zio_inherit_child_errors(zio, ZIO_CHILD_GANG);
4240 	zio_inherit_child_errors(zio, ZIO_CHILD_DDT);
4241 
4242 	/*
4243 	 * If the I/O on the transformed data was successful, generate any
4244 	 * checksum reports now while we still have the transformed data.
4245 	 */
4246 	if (zio->io_error == 0) {
4247 		while (zio->io_cksum_report != NULL) {
4248 			zio_cksum_report_t *zcr = zio->io_cksum_report;
4249 			uint64_t align = zcr->zcr_align;
4250 			uint64_t asize = P2ROUNDUP(psize, align);
4251 			abd_t *adata = zio->io_abd;
4252 
4253 			if (asize != psize) {
4254 				adata = abd_alloc(asize, B_TRUE);
4255 				abd_copy(adata, zio->io_abd, psize);
4256 				abd_zero_off(adata, psize, asize - psize);
4257 			}
4258 
4259 			zio->io_cksum_report = zcr->zcr_next;
4260 			zcr->zcr_next = NULL;
4261 			zcr->zcr_finish(zcr, adata);
4262 			zfs_ereport_free_checksum(zcr);
4263 
4264 			if (asize != psize)
4265 				abd_free(adata);
4266 		}
4267 	}
4268 
4269 	zio_pop_transforms(zio);	/* note: may set zio->io_error */
4270 
4271 	vdev_stat_update(zio, psize);
4272 
4273 	if (zio->io_delay >= MSEC2NSEC(zio_slow_io_ms)) {
4274 		if (zio->io_vd != NULL && !vdev_is_dead(zio->io_vd)) {
4275 			/*
4276 			 * We want to only increment our slow IO counters if
4277 			 * the IO is valid (i.e. not if the drive is removed).
4278 			 *
4279 			 * zfs_ereport_post() will also do these checks, but
4280 			 * it can also have other failures, so we need to
4281 			 * increment the slow_io counters independent of it.
4282 			 */
4283 			if (zfs_ereport_is_valid(FM_EREPORT_ZFS_DELAY,
4284 			    zio->io_spa, zio->io_vd, zio)) {
4285 				mutex_enter(&zio->io_vd->vdev_stat_lock);
4286 				zio->io_vd->vdev_stat.vs_slow_ios++;
4287 				mutex_exit(&zio->io_vd->vdev_stat_lock);
4288 
4289 				(void) zfs_ereport_post(FM_EREPORT_ZFS_DELAY,
4290 				    zio->io_spa, zio->io_vd, &zio->io_bookmark,
4291 				    zio, 0, 0);
4292 			}
4293 		}
4294 	}
4295 
4296 	if (zio->io_error) {
4297 		/*
4298 		 * If this I/O is attached to a particular vdev,
4299 		 * generate an error message describing the I/O failure
4300 		 * at the block level.  We ignore these errors if the
4301 		 * device is currently unavailable.
4302 		 */
4303 		if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd))
4304 			(void) zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd,
4305 			    &zio->io_bookmark, zio, 0, 0);
4306 
4307 		if ((zio->io_error == EIO || !(zio->io_flags &
4308 		    (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) &&
4309 		    zio == lio) {
4310 			/*
4311 			 * For logical I/O requests, tell the SPA to log the
4312 			 * error and generate a logical data ereport.
4313 			 */
4314 			spa_log_error(spa, &zio->io_bookmark);
4315 			(void) zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL,
4316 			    &zio->io_bookmark, zio, 0, 0);
4317 		}
4318 	}
4319 
4320 	if (zio->io_error && zio == lio) {
4321 		/*
4322 		 * Determine whether zio should be reexecuted.  This will
4323 		 * propagate all the way to the root via zio_notify_parent().
4324 		 */
4325 		ASSERT(vd == NULL && bp != NULL);
4326 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
4327 
4328 		if (IO_IS_ALLOCATING(zio) &&
4329 		    !(zio->io_flags & ZIO_FLAG_CANFAIL)) {
4330 			if (zio->io_error != ENOSPC)
4331 				zio->io_reexecute |= ZIO_REEXECUTE_NOW;
4332 			else
4333 				zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND;
4334 		}
4335 
4336 		if ((zio->io_type == ZIO_TYPE_READ ||
4337 		    zio->io_type == ZIO_TYPE_FREE) &&
4338 		    !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) &&
4339 		    zio->io_error == ENXIO &&
4340 		    spa_load_state(spa) == SPA_LOAD_NONE &&
4341 		    spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE)
4342 			zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND;
4343 
4344 		if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute)
4345 			zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND;
4346 
4347 		/*
4348 		 * Here is a possibly good place to attempt to do
4349 		 * either combinatorial reconstruction or error correction
4350 		 * based on checksums.  It also might be a good place
4351 		 * to send out preliminary ereports before we suspend
4352 		 * processing.
4353 		 */
4354 	}
4355 
4356 	/*
4357 	 * If there were logical child errors, they apply to us now.
4358 	 * We defer this until now to avoid conflating logical child
4359 	 * errors with errors that happened to the zio itself when
4360 	 * updating vdev stats and reporting FMA events above.
4361 	 */
4362 	zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);
4363 
4364 	if ((zio->io_error || zio->io_reexecute) &&
4365 	    IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
4366 	    !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)))
4367 		zio_dva_unallocate(zio, zio->io_gang_tree, bp);
4368 
4369 	zio_gang_tree_free(&zio->io_gang_tree);
4370 
4371 	/*
4372 	 * Godfather I/Os should never suspend.
4373 	 */
4374 	if ((zio->io_flags & ZIO_FLAG_GODFATHER) &&
4375 	    (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND))
4376 		zio->io_reexecute = 0;
4377 
4378 	if (zio->io_reexecute) {
4379 		/*
4380 		 * This is a logical I/O that wants to reexecute.
4381 		 *
4382 		 * Reexecute is top-down.  When an i/o fails, if it's not
4383 		 * the root, it simply notifies its parent and sticks around.
4384 		 * The parent, seeing that it still has children in zio_done(),
4385 		 * does the same.  This percolates all the way up to the root.
4386 		 * The root i/o will reexecute or suspend the entire tree.
4387 		 *
4388 		 * This approach ensures that zio_reexecute() honors
4389 		 * all the original i/o dependency relationships, e.g.
4390 		 * parents not executing until children are ready.
4391 		 */
4392 		ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
4393 
4394 		zio->io_gang_leader = NULL;
4395 
4396 		mutex_enter(&zio->io_lock);
4397 		zio->io_state[ZIO_WAIT_DONE] = 1;
4398 		mutex_exit(&zio->io_lock);
4399 
4400 		/*
4401 		 * "The Godfather" I/O monitors its children but is
4402 		 * not a true parent to them. It will track them through
4403 		 * the pipeline but severs its ties whenever they get into
4404 		 * trouble (e.g. suspended). This allows "The Godfather"
4405 		 * I/O to return status without blocking.
4406 		 */
4407 		zl = NULL;
4408 		for (pio = zio_walk_parents(zio, &zl); pio != NULL;
4409 		    pio = pio_next) {
4410 			zio_link_t *remove_zl = zl;
4411 			pio_next = zio_walk_parents(zio, &zl);
4412 
4413 			if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&
4414 			    (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
4415 				zio_remove_child(pio, zio, remove_zl);
4416 				zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
4417 			}
4418 		}
4419 
4420 		if ((pio = zio_unique_parent(zio)) != NULL) {
4421 			/*
4422 			 * We're not a root i/o, so there's nothing to do
4423 			 * but notify our parent.  Don't propagate errors
4424 			 * upward since we haven't permanently failed yet.
4425 			 */
4426 			ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
4427 			zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE;
4428 			zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
4429 		} else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
4430 			/*
4431 			 * We'd fail again if we reexecuted now, so suspend
4432 			 * until conditions improve (e.g. device comes online).
4433 			 */
4434 			zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR);
4435 		} else {
4436 			/*
4437 			 * Reexecution is potentially a huge amount of work.
4438 			 * Hand it off to the otherwise-unused claim taskq.
4439 			 */
4440 			ASSERT(zio->io_tqent.tqent_next == NULL);
4441 			spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM,
4442 			    ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio,
4443 			    0, &zio->io_tqent);
4444 		}
4445 		return (ZIO_PIPELINE_STOP);
4446 	}
4447 
4448 	ASSERT(zio->io_child_count == 0);
4449 	ASSERT(zio->io_reexecute == 0);
4450 	ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL));
4451 
4452 	/*
4453 	 * Report any checksum errors, since the I/O is complete.
4454 	 */
4455 	while (zio->io_cksum_report != NULL) {
4456 		zio_cksum_report_t *zcr = zio->io_cksum_report;
4457 		zio->io_cksum_report = zcr->zcr_next;
4458 		zcr->zcr_next = NULL;
4459 		zcr->zcr_finish(zcr, NULL);
4460 		zfs_ereport_free_checksum(zcr);
4461 	}
4462 
4463 	/*
4464 	 * It is the responsibility of the done callback to ensure that this
4465 	 * particular zio is no longer discoverable for adoption, and as
4466 	 * such, cannot acquire any new parents.
4467 	 */
4468 	if (zio->io_done)
4469 		zio->io_done(zio);
4470 
4471 	mutex_enter(&zio->io_lock);
4472 	zio->io_state[ZIO_WAIT_DONE] = 1;
4473 	mutex_exit(&zio->io_lock);
4474 
4475 	zl = NULL;
4476 	for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
4477 		zio_link_t *remove_zl = zl;
4478 		pio_next = zio_walk_parents(zio, &zl);
4479 		zio_remove_child(pio, zio, remove_zl);
4480 		zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
4481 	}
4482 
4483 	if (zio->io_waiter != NULL) {
4484 		mutex_enter(&zio->io_lock);
4485 		zio->io_executor = NULL;
4486 		cv_broadcast(&zio->io_cv);
4487 		mutex_exit(&zio->io_lock);
4488 	} else {
4489 		zio_destroy(zio);
4490 	}
4491 
4492 	return (ZIO_PIPELINE_STOP);
4493 }
4494 
4495 /*
4496  * ==========================================================================
4497  * I/O pipeline definition
4498  * ==========================================================================
4499  */
4500 static zio_pipe_stage_t *zio_pipeline[] = {
4501 	NULL,
4502 	zio_read_bp_init,
4503 	zio_write_bp_init,
4504 	zio_free_bp_init,
4505 	zio_issue_async,
4506 	zio_write_compress,
4507 	zio_encrypt,
4508 	zio_checksum_generate,
4509 	zio_nop_write,
4510 	zio_ddt_read_start,
4511 	zio_ddt_read_done,
4512 	zio_ddt_write,
4513 	zio_ddt_free,
4514 	zio_gang_assemble,
4515 	zio_gang_issue,
4516 	zio_dva_throttle,
4517 	zio_dva_allocate,
4518 	zio_dva_free,
4519 	zio_dva_claim,
4520 	zio_ready,
4521 	zio_vdev_io_start,
4522 	zio_vdev_io_done,
4523 	zio_vdev_io_assess,
4524 	zio_checksum_verify,
4525 	zio_done
4526 };
4527 
4528 
4529 
4530 
4531 /*
4532  * Compare two zbookmark_phys_t's to see which we would reach first in a
4533  * pre-order traversal of the object tree.
4534  *
4535  * This is simple in every case aside from the meta-dnode object. For all other
4536  * objects, we traverse them in order (object 1 before object 2, and so on).
4537  * However, all of these objects are traversed while traversing object 0, since
4538  * the data it points to is the list of objects.  Thus, we need to convert to a
4539  * canonical representation so we can compare meta-dnode bookmarks to
4540  * non-meta-dnode bookmarks.
4541  *
4542  * We do this by calculating "equivalents" for each field of the zbookmark.
4543  * zbookmarks outside of the meta-dnode use their own object and level, and
4544  * calculate the level 0 equivalent (the first L0 blkid that is contained in the
4545  * blocks this bookmark refers to) by multiplying their blkid by their span
4546  * (the number of L0 blocks contained within one block at their level).
4547  * zbookmarks inside the meta-dnode calculate their object equivalent
4548  * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use
4549  * level + 1<<31 (any value larger than a level could ever be) for their level.
4550  * This causes them to always compare before a bookmark in their object
4551  * equivalent, compare appropriately to bookmarks in other objects, and to
4552  * compare appropriately to other bookmarks in the meta-dnode.
4553  */
4554 int
zbookmark_compare(uint16_t dbss1,uint8_t ibs1,uint16_t dbss2,uint8_t ibs2,const zbookmark_phys_t * zb1,const zbookmark_phys_t * zb2)4555 zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2,
4556     const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2)
4557 {
4558 	/*
4559 	 * These variables represent the "equivalent" values for the zbookmark,
4560 	 * after converting zbookmarks inside the meta dnode to their
4561 	 * normal-object equivalents.
4562 	 */
4563 	uint64_t zb1obj, zb2obj;
4564 	uint64_t zb1L0, zb2L0;
4565 	uint64_t zb1level, zb2level;
4566 
4567 	if (zb1->zb_object == zb2->zb_object &&
4568 	    zb1->zb_level == zb2->zb_level &&
4569 	    zb1->zb_blkid == zb2->zb_blkid)
4570 		return (0);
4571 
4572 	/*
4573 	 * BP_SPANB calculates the span in blocks.
4574 	 */
4575 	zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level);
4576 	zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level);
4577 
4578 	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
4579 		zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
4580 		zb1L0 = 0;
4581 		zb1level = zb1->zb_level + COMPARE_META_LEVEL;
4582 	} else {
4583 		zb1obj = zb1->zb_object;
4584 		zb1level = zb1->zb_level;
4585 	}
4586 
4587 	if (zb2->zb_object == DMU_META_DNODE_OBJECT) {
4588 		zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
4589 		zb2L0 = 0;
4590 		zb2level = zb2->zb_level + COMPARE_META_LEVEL;
4591 	} else {
4592 		zb2obj = zb2->zb_object;
4593 		zb2level = zb2->zb_level;
4594 	}
4595 
4596 	/* Now that we have a canonical representation, do the comparison. */
4597 	if (zb1obj != zb2obj)
4598 		return (zb1obj < zb2obj ? -1 : 1);
4599 	else if (zb1L0 != zb2L0)
4600 		return (zb1L0 < zb2L0 ? -1 : 1);
4601 	else if (zb1level != zb2level)
4602 		return (zb1level > zb2level ? -1 : 1);
4603 	/*
4604 	 * This can (theoretically) happen if the bookmarks have the same object
4605 	 * and level, but different blkids, if the block sizes are not the same.
4606 	 * There is presently no way to change the indirect block sizes
4607 	 */
4608 	return (0);
4609 }
4610 
4611 /*
4612  *  This function checks the following: given that last_block is the place that
4613  *  our traversal stopped last time, does that guarantee that we've visited
4614  *  every node under subtree_root?  Therefore, we can't just use the raw output
4615  *  of zbookmark_compare.  We have to pass in a modified version of
4616  *  subtree_root; by incrementing the block id, and then checking whether
4617  *  last_block is before or equal to that, we can tell whether or not having
4618  *  visited last_block implies that all of subtree_root's children have been
4619  *  visited.
4620  */
4621 boolean_t
zbookmark_subtree_completed(const dnode_phys_t * dnp,const zbookmark_phys_t * subtree_root,const zbookmark_phys_t * last_block)4622 zbookmark_subtree_completed(const dnode_phys_t *dnp,
4623     const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block)
4624 {
4625 	zbookmark_phys_t mod_zb = *subtree_root;
4626 	mod_zb.zb_blkid++;
4627 	ASSERT(last_block->zb_level == 0);
4628 
4629 	/* The objset_phys_t isn't before anything. */
4630 	if (dnp == NULL)
4631 		return (B_FALSE);
4632 
4633 	/*
4634 	 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the
4635 	 * data block size in sectors, because that variable is only used if
4636 	 * the bookmark refers to a block in the meta-dnode.  Since we don't
4637 	 * know without examining it what object it refers to, and there's no
4638 	 * harm in passing in this value in other cases, we always pass it in.
4639 	 *
4640 	 * We pass in 0 for the indirect block size shift because zb2 must be
4641 	 * level 0.  The indirect block size is only used to calculate the span
4642 	 * of the bookmark, but since the bookmark must be level 0, the span is
4643 	 * always 1, so the math works out.
4644 	 *
4645 	 * If you make changes to how the zbookmark_compare code works, be sure
4646 	 * to make sure that this code still works afterwards.
4647 	 */
4648 	return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,
4649 	    1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb,
4650 	    last_block) <= 0);
4651 }
4652