xref: /illumos-gate/usr/src/boot/libsa/zfs/zfsimpl.c (revision 076ad4c710ebdb269f6341db447a83b5781f0b05)
1 /*
2  * Copyright (c) 2007 Doug Rabson
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 
29 /*
30  *	Stand-alone ZFS file reader.
31  */
32 
33 #include <stdbool.h>
34 #include <sys/endian.h>
35 #include <sys/stat.h>
36 #include <sys/stdint.h>
37 #include <sys/list.h>
38 #include <sys/zfs_bootenv.h>
39 #include <inttypes.h>
40 
41 #include "zfsimpl.h"
42 #include "zfssubr.c"
43 
44 
45 struct zfsmount {
46 	const spa_t	*spa;
47 	objset_phys_t	objset;
48 	uint64_t	rootobj;
49 };
50 
51 /*
52  * The indirect_child_t represents the vdev that we will read from, when we
53  * need to read all copies of the data (e.g. for scrub or reconstruction).
54  * For plain (non-mirror) top-level vdevs (i.e. is_vdev is not a mirror),
55  * ic_vdev is the same as is_vdev.  However, for mirror top-level vdevs,
56  * ic_vdev is a child of the mirror.
57  */
58 typedef struct indirect_child {
59 	void *ic_data;
60 	vdev_t *ic_vdev;
61 } indirect_child_t;
62 
63 /*
64  * The indirect_split_t represents one mapped segment of an i/o to the
65  * indirect vdev. For non-split (contiguously-mapped) blocks, there will be
66  * only one indirect_split_t, with is_split_offset==0 and is_size==io_size.
67  * For split blocks, there will be several of these.
68  */
69 typedef struct indirect_split {
70 	list_node_t is_node; /* link on iv_splits */
71 
72 	/*
73 	 * is_split_offset is the offset into the i/o.
74 	 * This is the sum of the previous splits' is_size's.
75 	 */
76 	uint64_t is_split_offset;
77 
78 	vdev_t *is_vdev; /* top-level vdev */
79 	uint64_t is_target_offset; /* offset on is_vdev */
80 	uint64_t is_size;
81 	int is_children; /* number of entries in is_child[] */
82 
83 	/*
84 	 * is_good_child is the child that we are currently using to
85 	 * attempt reconstruction.
86 	 */
87 	int is_good_child;
88 
89 	indirect_child_t is_child[1]; /* variable-length */
90 } indirect_split_t;
91 
92 /*
93  * The indirect_vsd_t is associated with each i/o to the indirect vdev.
94  * It is the "Vdev-Specific Data" in the zio_t's io_vsd.
95  */
96 typedef struct indirect_vsd {
97 	boolean_t iv_split_block;
98 	boolean_t iv_reconstruct;
99 
100 	list_t iv_splits; /* list of indirect_split_t's */
101 } indirect_vsd_t;
102 
103 /*
104  * List of all vdevs, chained through v_alllink.
105  */
106 static vdev_list_t zfs_vdevs;
107 
108 /*
109  * List of ZFS features supported for read
110  */
111 static const char *features_for_read[] = {
112 	"org.illumos:lz4_compress",
113 	"com.delphix:hole_birth",
114 	"com.delphix:extensible_dataset",
115 	"com.delphix:embedded_data",
116 	"org.open-zfs:large_blocks",
117 	"org.illumos:sha512",
118 	"org.illumos:skein",
119 	"org.illumos:edonr",
120 	"org.zfsonlinux:large_dnode",
121 	"com.joyent:multi_vdev_crash_dump",
122 	"com.delphix:spacemap_histogram",
123 	"com.delphix:zpool_checkpoint",
124 	"com.delphix:spacemap_v2",
125 	"com.datto:encryption",
126 	"com.datto:bookmark_v2",
127 	"org.zfsonlinux:allocation_classes",
128 	"com.datto:resilver_defer",
129 	"com.delphix:device_removal",
130 	"com.delphix:obsolete_counts",
131 	NULL
132 };
133 
134 /*
135  * List of all pools, chained through spa_link.
136  */
137 static spa_list_t zfs_pools;
138 
139 static const dnode_phys_t *dnode_cache_obj;
140 static uint64_t dnode_cache_bn;
141 static char *dnode_cache_buf;
142 
143 static int zio_read(const spa_t *spa, const blkptr_t *bp, void *buf);
144 static int zfs_get_root(const spa_t *spa, uint64_t *objid);
145 static int zfs_rlookup(const spa_t *spa, uint64_t objnum, char *result);
146 static int zap_lookup(const spa_t *spa, const dnode_phys_t *dnode,
147     const char *name, uint64_t integer_size, uint64_t num_integers,
148     void *value);
149 static int objset_get_dnode(const spa_t *, const objset_phys_t *, uint64_t,
150     dnode_phys_t *);
151 static int dnode_read(const spa_t *, const dnode_phys_t *, off_t, void *,
152     size_t);
153 static int vdev_indirect_read(vdev_t *, const blkptr_t *, void *, off_t,
154     size_t);
155 static int vdev_mirror_read(vdev_t *, const blkptr_t *, void *, off_t,
156     size_t);
157 
158 static void
159 zfs_init(void)
160 {
161 	STAILQ_INIT(&zfs_vdevs);
162 	STAILQ_INIT(&zfs_pools);
163 
164 	dnode_cache_buf = malloc(SPA_MAXBLOCKSIZE);
165 
166 	zfs_init_crc();
167 }
168 
169 static int
170 nvlist_check_features_for_read(nvlist_t *nvl)
171 {
172 	nvlist_t *features = NULL;
173 	nvs_data_t *data;
174 	nvp_header_t *nvp;
175 	nv_string_t *nvp_name;
176 	int rc;
177 
178 	/*
179 	 * We may have all features disabled.
180 	 */
181 	rc = nvlist_find(nvl, ZPOOL_CONFIG_FEATURES_FOR_READ,
182 	    DATA_TYPE_NVLIST, NULL, &features, NULL);
183 	switch (rc) {
184 	case 0:
185 		break;		/* Continue with checks */
186 
187 	case ENOENT:
188 		return (0);	/* All features are disabled */
189 
190 	default:
191 		return (rc);	/* Error while reading nvlist */
192 	}
193 
194 	data = (nvs_data_t *)features->nv_data;
195 	nvp = &data->nvl_pair;	/* first pair in nvlist */
196 
197 	while (nvp->encoded_size != 0 && nvp->decoded_size != 0) {
198 		int i, found;
199 
200 		nvp_name = (nv_string_t *)((uintptr_t)nvp + sizeof (*nvp));
201 		found = 0;
202 
203 		for (i = 0; features_for_read[i] != NULL; i++) {
204 			if (memcmp(nvp_name->nv_data, features_for_read[i],
205 			    nvp_name->nv_size) == 0) {
206 				found = 1;
207 				break;
208 			}
209 		}
210 
211 		if (!found) {
212 			printf("ZFS: unsupported feature: %.*s\n",
213 			    nvp_name->nv_size, nvp_name->nv_data);
214 			rc = EIO;
215 		}
216 		nvp = (nvp_header_t *)((uint8_t *)nvp + nvp->encoded_size);
217 	}
218 	nvlist_destroy(features);
219 
220 	return (rc);
221 }
222 
223 static int
224 vdev_read_phys(vdev_t *vdev, const blkptr_t *bp, void *buf,
225     off_t offset, size_t size)
226 {
227 	size_t psize;
228 	int rc;
229 
230 	if (vdev->v_phys_read == NULL)
231 		return (ENOTSUP);
232 
233 	if (bp) {
234 		psize = BP_GET_PSIZE(bp);
235 	} else {
236 		psize = size;
237 	}
238 
239 	rc = vdev->v_phys_read(vdev, vdev->v_priv, offset, buf, psize);
240 	if (rc == 0) {
241 		if (bp != NULL)
242 			rc = zio_checksum_verify(vdev->v_spa, bp, buf);
243 	}
244 
245 	return (rc);
246 }
247 
248 static int
249 vdev_write_phys(vdev_t *vdev, void *buf, off_t offset, size_t size)
250 {
251 	if (vdev->v_phys_write == NULL)
252 		return (ENOTSUP);
253 
254 	return (vdev->v_phys_write(vdev, offset, buf, size));
255 }
256 
257 typedef struct remap_segment {
258 	vdev_t *rs_vd;
259 	uint64_t rs_offset;
260 	uint64_t rs_asize;
261 	uint64_t rs_split_offset;
262 	list_node_t rs_node;
263 } remap_segment_t;
264 
265 static remap_segment_t *
266 rs_alloc(vdev_t *vd, uint64_t offset, uint64_t asize, uint64_t split_offset)
267 {
268 	remap_segment_t *rs = malloc(sizeof (remap_segment_t));
269 
270 	if (rs != NULL) {
271 		rs->rs_vd = vd;
272 		rs->rs_offset = offset;
273 		rs->rs_asize = asize;
274 		rs->rs_split_offset = split_offset;
275 	}
276 
277 	return (rs);
278 }
279 
280 vdev_indirect_mapping_t *
281 vdev_indirect_mapping_open(spa_t *spa, objset_phys_t *os,
282     uint64_t mapping_object)
283 {
284 	vdev_indirect_mapping_t *vim;
285 	vdev_indirect_mapping_phys_t *vim_phys;
286 	int rc;
287 
288 	vim = calloc(1, sizeof (*vim));
289 	if (vim == NULL)
290 		return (NULL);
291 
292 	vim->vim_dn = calloc(1, sizeof (*vim->vim_dn));
293 	if (vim->vim_dn == NULL) {
294 		free(vim);
295 		return (NULL);
296 	}
297 
298 	rc = objset_get_dnode(spa, os, mapping_object, vim->vim_dn);
299 	if (rc != 0) {
300 		free(vim->vim_dn);
301 		free(vim);
302 		return (NULL);
303 	}
304 
305 	vim->vim_spa = spa;
306 	vim->vim_phys = malloc(sizeof (*vim->vim_phys));
307 	if (vim->vim_phys == NULL) {
308 		free(vim->vim_dn);
309 		free(vim);
310 		return (NULL);
311 	}
312 
313 	vim_phys = (vdev_indirect_mapping_phys_t *)DN_BONUS(vim->vim_dn);
314 	*vim->vim_phys = *vim_phys;
315 
316 	vim->vim_objset = os;
317 	vim->vim_object = mapping_object;
318 	vim->vim_entries = NULL;
319 
320 	vim->vim_havecounts =
321 	    (vim->vim_dn->dn_bonuslen > VDEV_INDIRECT_MAPPING_SIZE_V0);
322 
323 	return (vim);
324 }
325 
326 /*
327  * Compare an offset with an indirect mapping entry; there are three
328  * possible scenarios:
329  *
330  *     1. The offset is "less than" the mapping entry; meaning the
331  *        offset is less than the source offset of the mapping entry. In
332  *        this case, there is no overlap between the offset and the
333  *        mapping entry and -1 will be returned.
334  *
335  *     2. The offset is "greater than" the mapping entry; meaning the
336  *        offset is greater than the mapping entry's source offset plus
337  *        the entry's size. In this case, there is no overlap between
338  *        the offset and the mapping entry and 1 will be returned.
339  *
340  *        NOTE: If the offset is actually equal to the entry's offset
341  *        plus size, this is considered to be "greater" than the entry,
342  *        and this case applies (i.e. 1 will be returned). Thus, the
343  *        entry's "range" can be considered to be inclusive at its
344  *        start, but exclusive at its end: e.g. [src, src + size).
345  *
346  *     3. The last case to consider is if the offset actually falls
347  *        within the mapping entry's range. If this is the case, the
348  *        offset is considered to be "equal to" the mapping entry and
349  *        0 will be returned.
350  *
351  *        NOTE: If the offset is equal to the entry's source offset,
352  *        this case applies and 0 will be returned. If the offset is
353  *        equal to the entry's source plus its size, this case does
354  *        *not* apply (see "NOTE" above for scenario 2), and 1 will be
355  *        returned.
356  */
357 static int
358 dva_mapping_overlap_compare(const void *v_key, const void *v_array_elem)
359 {
360 	const uint64_t *key = v_key;
361 	const vdev_indirect_mapping_entry_phys_t *array_elem =
362 	    v_array_elem;
363 	uint64_t src_offset = DVA_MAPPING_GET_SRC_OFFSET(array_elem);
364 
365 	if (*key < src_offset) {
366 		return (-1);
367 	} else if (*key < src_offset + DVA_GET_ASIZE(&array_elem->vimep_dst)) {
368 		return (0);
369 	} else {
370 		return (1);
371 	}
372 }
373 
374 /*
375  * Return array entry.
376  */
377 static vdev_indirect_mapping_entry_phys_t *
378 vdev_indirect_mapping_entry(vdev_indirect_mapping_t *vim, uint64_t index)
379 {
380 	uint64_t size;
381 	off_t offset = 0;
382 	int rc;
383 
384 	if (vim->vim_phys->vimp_num_entries == 0)
385 		return (NULL);
386 
387 	if (vim->vim_entries == NULL) {
388 		uint64_t bsize;
389 
390 		bsize = vim->vim_dn->dn_datablkszsec << SPA_MINBLOCKSHIFT;
391 		size = vim->vim_phys->vimp_num_entries *
392 		    sizeof (*vim->vim_entries);
393 		if (size > bsize) {
394 			size = bsize / sizeof (*vim->vim_entries);
395 			size *= sizeof (*vim->vim_entries);
396 		}
397 		vim->vim_entries = malloc(size);
398 		if (vim->vim_entries == NULL)
399 			return (NULL);
400 		vim->vim_num_entries = size / sizeof (*vim->vim_entries);
401 		offset = index * sizeof (*vim->vim_entries);
402 	}
403 
404 	/* We have data in vim_entries */
405 	if (offset == 0) {
406 		if (index >= vim->vim_entry_offset &&
407 		    index <= vim->vim_entry_offset + vim->vim_num_entries) {
408 			index -= vim->vim_entry_offset;
409 			return (&vim->vim_entries[index]);
410 		}
411 		offset = index * sizeof (*vim->vim_entries);
412 	}
413 
414 	vim->vim_entry_offset = index;
415 	size = vim->vim_num_entries * sizeof (*vim->vim_entries);
416 	rc = dnode_read(vim->vim_spa, vim->vim_dn, offset, vim->vim_entries,
417 	    size);
418 	if (rc != 0) {
419 		/* Read error, invalidate vim_entries. */
420 		free(vim->vim_entries);
421 		vim->vim_entries = NULL;
422 		return (NULL);
423 	}
424 	index -= vim->vim_entry_offset;
425 	return (&vim->vim_entries[index]);
426 }
427 
428 /*
429  * Returns the mapping entry for the given offset.
430  *
431  * It's possible that the given offset will not be in the mapping table
432  * (i.e. no mapping entries contain this offset), in which case, the
433  * return value value depends on the "next_if_missing" parameter.
434  *
435  * If the offset is not found in the table and "next_if_missing" is
436  * B_FALSE, then NULL will always be returned. The behavior is intended
437  * to allow consumers to get the entry corresponding to the offset
438  * parameter, iff the offset overlaps with an entry in the table.
439  *
440  * If the offset is not found in the table and "next_if_missing" is
441  * B_TRUE, then the entry nearest to the given offset will be returned,
442  * such that the entry's source offset is greater than the offset
443  * passed in (i.e. the "next" mapping entry in the table is returned, if
444  * the offset is missing from the table). If there are no entries whose
445  * source offset is greater than the passed in offset, NULL is returned.
446  */
447 static vdev_indirect_mapping_entry_phys_t *
448 vdev_indirect_mapping_entry_for_offset(vdev_indirect_mapping_t *vim,
449     uint64_t offset)
450 {
451 	ASSERT(vim->vim_phys->vimp_num_entries > 0);
452 
453 	vdev_indirect_mapping_entry_phys_t *entry;
454 
455 	uint64_t last = vim->vim_phys->vimp_num_entries - 1;
456 	uint64_t base = 0;
457 
458 	/*
459 	 * We don't define these inside of the while loop because we use
460 	 * their value in the case that offset isn't in the mapping.
461 	 */
462 	uint64_t mid;
463 	int result;
464 
465 	while (last >= base) {
466 		mid = base + ((last - base) >> 1);
467 
468 		entry = vdev_indirect_mapping_entry(vim, mid);
469 		if (entry == NULL)
470 			break;
471 		result = dva_mapping_overlap_compare(&offset, entry);
472 
473 		if (result == 0) {
474 			break;
475 		} else if (result < 0) {
476 			last = mid - 1;
477 		} else {
478 			base = mid + 1;
479 		}
480 	}
481 	return (entry);
482 }
483 
484 /*
485  * Given an indirect vdev and an extent on that vdev, it duplicates the
486  * physical entries of the indirect mapping that correspond to the extent
487  * to a new array and returns a pointer to it. In addition, copied_entries
488  * is populated with the number of mapping entries that were duplicated.
489  *
490  * Finally, since we are doing an allocation, it is up to the caller to
491  * free the array allocated in this function.
492  */
493 vdev_indirect_mapping_entry_phys_t *
494 vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t *vd, uint64_t offset,
495     uint64_t asize, uint64_t *copied_entries)
496 {
497 	vdev_indirect_mapping_entry_phys_t *duplicate_mappings = NULL;
498 	vdev_indirect_mapping_t *vim = vd->v_mapping;
499 	uint64_t entries = 0;
500 
501 	vdev_indirect_mapping_entry_phys_t *first_mapping =
502 	    vdev_indirect_mapping_entry_for_offset(vim, offset);
503 	ASSERT3P(first_mapping, !=, NULL);
504 
505 	vdev_indirect_mapping_entry_phys_t *m = first_mapping;
506 	while (asize > 0) {
507 		uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
508 		uint64_t inner_offset = offset - DVA_MAPPING_GET_SRC_OFFSET(m);
509 		uint64_t inner_size = MIN(asize, size - inner_offset);
510 
511 		offset += inner_size;
512 		asize -= inner_size;
513 		entries++;
514 		m++;
515 	}
516 
517 	size_t copy_length = entries * sizeof (*first_mapping);
518 	duplicate_mappings = malloc(copy_length);
519 	if (duplicate_mappings != NULL)
520 		bcopy(first_mapping, duplicate_mappings, copy_length);
521 	else
522 		entries = 0;
523 
524 	*copied_entries = entries;
525 
526 	return (duplicate_mappings);
527 }
528 
529 static vdev_t *
530 vdev_lookup_top(spa_t *spa, uint64_t vdev)
531 {
532 	vdev_t *rvd;
533 	vdev_list_t *vlist;
534 
535 	vlist = &spa->spa_root_vdev->v_children;
536 	STAILQ_FOREACH(rvd, vlist, v_childlink)
537 		if (rvd->v_id == vdev)
538 			break;
539 
540 	return (rvd);
541 }
542 
543 /*
544  * This is a callback for vdev_indirect_remap() which allocates an
545  * indirect_split_t for each split segment and adds it to iv_splits.
546  */
547 static void
548 vdev_indirect_gather_splits(uint64_t split_offset, vdev_t *vd, uint64_t offset,
549     uint64_t size, void *arg)
550 {
551 	int n = 1;
552 	zio_t *zio = arg;
553 	indirect_vsd_t *iv = zio->io_vsd;
554 
555 	if (vd->v_read == vdev_indirect_read)
556 		return;
557 
558 	if (vd->v_read == vdev_mirror_read)
559 		n = vd->v_nchildren;
560 
561 	indirect_split_t *is =
562 	    malloc(offsetof(indirect_split_t, is_child[n]));
563 	if (is == NULL) {
564 		zio->io_error = ENOMEM;
565 		return;
566 	}
567 	bzero(is, offsetof(indirect_split_t, is_child[n]));
568 
569 	is->is_children = n;
570 	is->is_size = size;
571 	is->is_split_offset = split_offset;
572 	is->is_target_offset = offset;
573 	is->is_vdev = vd;
574 
575 	/*
576 	 * Note that we only consider multiple copies of the data for
577 	 * *mirror* vdevs.  We don't for "replacing" or "spare" vdevs, even
578 	 * though they use the same ops as mirror, because there's only one
579 	 * "good" copy under the replacing/spare.
580 	 */
581 	if (vd->v_read == vdev_mirror_read) {
582 		int i = 0;
583 		vdev_t *kid;
584 
585 		STAILQ_FOREACH(kid, &vd->v_children, v_childlink) {
586 			is->is_child[i++].ic_vdev = kid;
587 		}
588 	} else {
589 		is->is_child[0].ic_vdev = vd;
590 	}
591 
592 	list_insert_tail(&iv->iv_splits, is);
593 }
594 
595 static void
596 vdev_indirect_remap(vdev_t *vd, uint64_t offset, uint64_t asize, void *arg)
597 {
598 	list_t stack;
599 	spa_t *spa = vd->v_spa;
600 	zio_t *zio = arg;
601 	remap_segment_t *rs;
602 
603 	list_create(&stack, sizeof (remap_segment_t),
604 	    offsetof(remap_segment_t, rs_node));
605 
606 	rs = rs_alloc(vd, offset, asize, 0);
607 	if (rs == NULL) {
608 		printf("vdev_indirect_remap: out of memory.\n");
609 		zio->io_error = ENOMEM;
610 	}
611 	for (; rs != NULL; rs = list_remove_head(&stack)) {
612 		vdev_t *v = rs->rs_vd;
613 		uint64_t num_entries = 0;
614 		/* vdev_indirect_mapping_t *vim = v->v_mapping; */
615 		vdev_indirect_mapping_entry_phys_t *mapping =
616 		    vdev_indirect_mapping_duplicate_adjacent_entries(v,
617 		    rs->rs_offset, rs->rs_asize, &num_entries);
618 
619 		if (num_entries == 0)
620 			zio->io_error = ENOMEM;
621 
622 		for (uint64_t i = 0; i < num_entries; i++) {
623 			vdev_indirect_mapping_entry_phys_t *m = &mapping[i];
624 			uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
625 			uint64_t dst_offset = DVA_GET_OFFSET(&m->vimep_dst);
626 			uint64_t dst_vdev = DVA_GET_VDEV(&m->vimep_dst);
627 			uint64_t inner_offset = rs->rs_offset -
628 			    DVA_MAPPING_GET_SRC_OFFSET(m);
629 			uint64_t inner_size =
630 			    MIN(rs->rs_asize, size - inner_offset);
631 			vdev_t *dst_v = vdev_lookup_top(spa, dst_vdev);
632 
633 			if (dst_v->v_read == vdev_indirect_read) {
634 				remap_segment_t *o;
635 
636 				o = rs_alloc(dst_v, dst_offset + inner_offset,
637 				    inner_size, rs->rs_split_offset);
638 				if (o == NULL) {
639 					printf("vdev_indirect_remap: "
640 					    "out of memory.\n");
641 					zio->io_error = ENOMEM;
642 					break;
643 				}
644 
645 				list_insert_head(&stack, o);
646 			}
647 			vdev_indirect_gather_splits(rs->rs_split_offset, dst_v,
648 			    dst_offset + inner_offset,
649 			    inner_size, arg);
650 
651 			/*
652 			 * vdev_indirect_gather_splits can have memory
653 			 * allocation error, we can not recover from it.
654 			 */
655 			if (zio->io_error != 0)
656 				break;
657 			rs->rs_offset += inner_size;
658 			rs->rs_asize -= inner_size;
659 			rs->rs_split_offset += inner_size;
660 		}
661 
662 		free(mapping);
663 		free(rs);
664 		if (zio->io_error != 0)
665 			break;
666 	}
667 
668 	list_destroy(&stack);
669 }
670 
671 static void
672 vdev_indirect_map_free(zio_t *zio)
673 {
674 	indirect_vsd_t *iv = zio->io_vsd;
675 	indirect_split_t *is;
676 
677 	while ((is = list_head(&iv->iv_splits)) != NULL) {
678 		for (int c = 0; c < is->is_children; c++) {
679 			indirect_child_t *ic = &is->is_child[c];
680 			free(ic->ic_data);
681 		}
682 		list_remove(&iv->iv_splits, is);
683 		free(is);
684 	}
685 	free(iv);
686 }
687 
688 static int
689 vdev_indirect_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
690     off_t offset, size_t bytes)
691 {
692 	zio_t zio;
693 	spa_t *spa = vdev->v_spa;
694 	indirect_vsd_t *iv;
695 	indirect_split_t *first;
696 	int rc = EIO;
697 
698 	iv = calloc(1, sizeof (*iv));
699 	if (iv == NULL)
700 		return (ENOMEM);
701 
702 	list_create(&iv->iv_splits,
703 	    sizeof (indirect_split_t), offsetof(indirect_split_t, is_node));
704 
705 	bzero(&zio, sizeof (zio));
706 	zio.io_spa = spa;
707 	zio.io_bp = (blkptr_t *)bp;
708 	zio.io_data = buf;
709 	zio.io_size = bytes;
710 	zio.io_offset = offset;
711 	zio.io_vd = vdev;
712 	zio.io_vsd = iv;
713 
714 	if (vdev->v_mapping == NULL) {
715 		vdev_indirect_config_t *vic;
716 
717 		vic = &vdev->vdev_indirect_config;
718 		vdev->v_mapping = vdev_indirect_mapping_open(spa,
719 		    &spa->spa_mos, vic->vic_mapping_object);
720 	}
721 
722 	vdev_indirect_remap(vdev, offset, bytes, &zio);
723 	if (zio.io_error != 0)
724 		return (zio.io_error);
725 
726 	first = list_head(&iv->iv_splits);
727 	if (first->is_size == zio.io_size) {
728 		/*
729 		 * This is not a split block; we are pointing to the entire
730 		 * data, which will checksum the same as the original data.
731 		 * Pass the BP down so that the child i/o can verify the
732 		 * checksum, and try a different location if available
733 		 * (e.g. on a mirror).
734 		 *
735 		 * While this special case could be handled the same as the
736 		 * general (split block) case, doing it this way ensures
737 		 * that the vast majority of blocks on indirect vdevs
738 		 * (which are not split) are handled identically to blocks
739 		 * on non-indirect vdevs.  This allows us to be less strict
740 		 * about performance in the general (but rare) case.
741 		 */
742 		rc = first->is_vdev->v_read(first->is_vdev, zio.io_bp,
743 		    zio.io_data, first->is_target_offset, bytes);
744 	} else {
745 		iv->iv_split_block = B_TRUE;
746 		/*
747 		 * Read one copy of each split segment, from the
748 		 * top-level vdev.  Since we don't know the
749 		 * checksum of each split individually, the child
750 		 * zio can't ensure that we get the right data.
751 		 * E.g. if it's a mirror, it will just read from a
752 		 * random (healthy) leaf vdev.  We have to verify
753 		 * the checksum in vdev_indirect_io_done().
754 		 */
755 		for (indirect_split_t *is = list_head(&iv->iv_splits);
756 		    is != NULL; is = list_next(&iv->iv_splits, is)) {
757 			char *ptr = zio.io_data;
758 
759 			rc = is->is_vdev->v_read(is->is_vdev, zio.io_bp,
760 			    ptr + is->is_split_offset, is->is_target_offset,
761 			    is->is_size);
762 		}
763 		if (zio_checksum_verify(spa, zio.io_bp, zio.io_data))
764 			rc = ECKSUM;
765 		else
766 			rc = 0;
767 	}
768 
769 	vdev_indirect_map_free(&zio);
770 	if (rc == 0)
771 		rc = zio.io_error;
772 
773 	return (rc);
774 }
775 
776 static int
777 vdev_disk_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
778     off_t offset, size_t bytes)
779 {
780 
781 	return (vdev_read_phys(vdev, bp, buf,
782 	    offset + VDEV_LABEL_START_SIZE, bytes));
783 }
784 
785 static int
786 vdev_missing_read(vdev_t *vdev __unused, const blkptr_t *bp __unused,
787     void *buf __unused, off_t offset __unused, size_t bytes __unused)
788 {
789 
790 	return (ENOTSUP);
791 }
792 
793 static int
794 vdev_mirror_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
795     off_t offset, size_t bytes)
796 {
797 	vdev_t *kid;
798 	int rc;
799 
800 	rc = EIO;
801 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
802 		if (kid->v_state != VDEV_STATE_HEALTHY)
803 			continue;
804 		rc = kid->v_read(kid, bp, buf, offset, bytes);
805 		if (!rc)
806 			return (0);
807 	}
808 
809 	return (rc);
810 }
811 
812 static int
813 vdev_replacing_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
814     off_t offset, size_t bytes)
815 {
816 	vdev_t *kid;
817 
818 	/*
819 	 * Here we should have two kids:
820 	 * First one which is the one we are replacing and we can trust
821 	 * only this one to have valid data, but it might not be present.
822 	 * Second one is that one we are replacing with. It is most likely
823 	 * healthy, but we can't trust it has needed data, so we won't use it.
824 	 */
825 	kid = STAILQ_FIRST(&vdev->v_children);
826 	if (kid == NULL)
827 		return (EIO);
828 	if (kid->v_state != VDEV_STATE_HEALTHY)
829 		return (EIO);
830 	return (kid->v_read(kid, bp, buf, offset, bytes));
831 }
832 
833 static vdev_t *
834 vdev_find(uint64_t guid)
835 {
836 	vdev_t *vdev;
837 
838 	STAILQ_FOREACH(vdev, &zfs_vdevs, v_alllink)
839 		if (vdev->v_guid == guid)
840 			return (vdev);
841 
842 	return (0);
843 }
844 
845 static vdev_t *
846 vdev_create(uint64_t guid, vdev_read_t *vdev_read)
847 {
848 	vdev_t *vdev;
849 	vdev_indirect_config_t *vic;
850 
851 	vdev = calloc(1, sizeof (vdev_t));
852 	if (vdev != NULL) {
853 		STAILQ_INIT(&vdev->v_children);
854 		vdev->v_guid = guid;
855 		vdev->v_read = vdev_read;
856 
857 		/*
858 		 * root vdev has no read function, we use this fact to
859 		 * skip setting up data we do not need for root vdev.
860 		 * We only point root vdev from spa.
861 		 */
862 		if (vdev_read != NULL) {
863 			vic = &vdev->vdev_indirect_config;
864 			vic->vic_prev_indirect_vdev = UINT64_MAX;
865 			STAILQ_INSERT_TAIL(&zfs_vdevs, vdev, v_alllink);
866 		}
867 	}
868 
869 	return (vdev);
870 }
871 
872 static void
873 vdev_set_initial_state(vdev_t *vdev, const nvlist_t *nvlist)
874 {
875 	uint64_t is_offline, is_faulted, is_degraded, is_removed, isnt_present;
876 	uint64_t is_log;
877 
878 	is_offline = is_removed = is_faulted = is_degraded = isnt_present = 0;
879 	is_log = 0;
880 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_OFFLINE, DATA_TYPE_UINT64, NULL,
881 	    &is_offline, NULL);
882 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_REMOVED, DATA_TYPE_UINT64, NULL,
883 	    &is_removed, NULL);
884 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_FAULTED, DATA_TYPE_UINT64, NULL,
885 	    &is_faulted, NULL);
886 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_DEGRADED, DATA_TYPE_UINT64,
887 	    NULL, &is_degraded, NULL);
888 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_NOT_PRESENT, DATA_TYPE_UINT64,
889 	    NULL, &isnt_present, NULL);
890 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_IS_LOG, DATA_TYPE_UINT64, NULL,
891 	    &is_log, NULL);
892 
893 	if (is_offline != 0)
894 		vdev->v_state = VDEV_STATE_OFFLINE;
895 	else if (is_removed != 0)
896 		vdev->v_state = VDEV_STATE_REMOVED;
897 	else if (is_faulted != 0)
898 		vdev->v_state = VDEV_STATE_FAULTED;
899 	else if (is_degraded != 0)
900 		vdev->v_state = VDEV_STATE_DEGRADED;
901 	else if (isnt_present != 0)
902 		vdev->v_state = VDEV_STATE_CANT_OPEN;
903 
904 	vdev->v_islog = is_log != 0;
905 }
906 
907 static int
908 vdev_init(uint64_t guid, const nvlist_t *nvlist, vdev_t **vdevp)
909 {
910 	uint64_t id, ashift, asize, nparity;
911 	const char *path;
912 	const char *type;
913 	int len, pathlen;
914 	char *name;
915 	vdev_t *vdev;
916 
917 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ID, DATA_TYPE_UINT64, NULL, &id,
918 	    NULL) ||
919 	    nvlist_find(nvlist, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING,
920 	    NULL, &type, &len)) {
921 		return (ENOENT);
922 	}
923 
924 	if (memcmp(type, VDEV_TYPE_MIRROR, len) != 0 &&
925 	    memcmp(type, VDEV_TYPE_DISK, len) != 0 &&
926 #ifdef ZFS_TEST
927 	    memcmp(type, VDEV_TYPE_FILE, len) != 0 &&
928 #endif
929 	    memcmp(type, VDEV_TYPE_RAIDZ, len) != 0 &&
930 	    memcmp(type, VDEV_TYPE_INDIRECT, len) != 0 &&
931 	    memcmp(type, VDEV_TYPE_REPLACING, len) != 0 &&
932 	    memcmp(type, VDEV_TYPE_HOLE, len) != 0) {
933 		printf("ZFS: can only boot from disk, mirror, raidz1, "
934 		    "raidz2 and raidz3 vdevs, got: %.*s\n", len, type);
935 		return (EIO);
936 	}
937 
938 	if (memcmp(type, VDEV_TYPE_MIRROR, len) == 0)
939 		vdev = vdev_create(guid, vdev_mirror_read);
940 	else if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0)
941 		vdev = vdev_create(guid, vdev_raidz_read);
942 	else if (memcmp(type, VDEV_TYPE_REPLACING, len) == 0)
943 		vdev = vdev_create(guid, vdev_replacing_read);
944 	else if (memcmp(type, VDEV_TYPE_INDIRECT, len) == 0) {
945 		vdev_indirect_config_t *vic;
946 
947 		vdev = vdev_create(guid, vdev_indirect_read);
948 		if (vdev != NULL) {
949 			vdev->v_state = VDEV_STATE_HEALTHY;
950 			vic = &vdev->vdev_indirect_config;
951 
952 			nvlist_find(nvlist,
953 			    ZPOOL_CONFIG_INDIRECT_OBJECT,
954 			    DATA_TYPE_UINT64,
955 			    NULL, &vic->vic_mapping_object, NULL);
956 			nvlist_find(nvlist,
957 			    ZPOOL_CONFIG_INDIRECT_BIRTHS,
958 			    DATA_TYPE_UINT64,
959 			    NULL, &vic->vic_births_object, NULL);
960 			nvlist_find(nvlist,
961 			    ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
962 			    DATA_TYPE_UINT64,
963 			    NULL, &vic->vic_prev_indirect_vdev, NULL);
964 		}
965 	} else if (memcmp(type, VDEV_TYPE_HOLE, len) == 0) {
966 		vdev = vdev_create(guid, vdev_missing_read);
967 	} else {
968 		vdev = vdev_create(guid, vdev_disk_read);
969 	}
970 
971 	if (vdev == NULL)
972 		return (ENOMEM);
973 
974 	vdev_set_initial_state(vdev, nvlist);
975 	vdev->v_id = id;
976 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASHIFT,
977 	    DATA_TYPE_UINT64, NULL, &ashift, NULL) == 0)
978 		vdev->v_ashift = ashift;
979 
980 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASIZE,
981 	    DATA_TYPE_UINT64, NULL, &asize, NULL) == 0) {
982 		vdev->v_psize = asize +
983 		    VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
984 	}
985 
986 	if (nvlist_find(nvlist, ZPOOL_CONFIG_NPARITY,
987 	    DATA_TYPE_UINT64, NULL, &nparity, NULL) == 0)
988 		vdev->v_nparity = nparity;
989 
990 	if (nvlist_find(nvlist, ZPOOL_CONFIG_PATH,
991 	    DATA_TYPE_STRING, NULL, &path, &pathlen) == 0) {
992 		char prefix[] = "/dev/dsk/";
993 
994 		len = strlen(prefix);
995 		if (len < pathlen && memcmp(path, prefix, len) == 0) {
996 			path += len;
997 			pathlen -= len;
998 		}
999 		name = malloc(pathlen + 1);
1000 		if (name != NULL) {
1001 			bcopy(path, name, pathlen);
1002 			name[pathlen] = '\0';
1003 		}
1004 		vdev->v_name = name;
1005 		vdev->v_phys_path = NULL;
1006 		vdev->v_devid = NULL;
1007 		if (nvlist_find(nvlist, ZPOOL_CONFIG_PHYS_PATH,
1008 		    DATA_TYPE_STRING, NULL, &path, &pathlen) == 0) {
1009 			name = malloc(pathlen + 1);
1010 			if (name != NULL) {
1011 				bcopy(path, name, pathlen);
1012 				name[pathlen] = '\0';
1013 				vdev->v_phys_path = name;
1014 			}
1015 		}
1016 		if (nvlist_find(nvlist, ZPOOL_CONFIG_DEVID,
1017 		    DATA_TYPE_STRING, NULL, &path, &pathlen) == 0) {
1018 			name = malloc(pathlen + 1);
1019 			if (name != NULL) {
1020 				bcopy(path, name, pathlen);
1021 				name[pathlen] = '\0';
1022 				vdev->v_devid = name;
1023 			}
1024 		}
1025 	} else {
1026 		name = NULL;
1027 		if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0) {
1028 			if (vdev->v_nparity < 1 ||
1029 			    vdev->v_nparity > 3) {
1030 				printf("ZFS: invalid raidz parity: %d\n",
1031 				    vdev->v_nparity);
1032 				return (EIO);
1033 			}
1034 			(void) asprintf(&name, "%.*s%d-%" PRIu64, len, type,
1035 			    vdev->v_nparity, id);
1036 		} else {
1037 			(void) asprintf(&name, "%.*s-%" PRIu64, len, type, id);
1038 		}
1039 		vdev->v_name = name;
1040 	}
1041 	*vdevp = vdev;
1042 	return (0);
1043 }
1044 
1045 /*
1046  * Find slot for vdev. We return either NULL to signal to use
1047  * STAILQ_INSERT_HEAD, or we return link element to be used with
1048  * STAILQ_INSERT_AFTER.
1049  */
1050 static vdev_t *
1051 vdev_find_previous(vdev_t *top_vdev, vdev_t *vdev)
1052 {
1053 	vdev_t *v, *previous;
1054 
1055 	if (STAILQ_EMPTY(&top_vdev->v_children))
1056 		return (NULL);
1057 
1058 	previous = NULL;
1059 	STAILQ_FOREACH(v, &top_vdev->v_children, v_childlink) {
1060 		if (v->v_id > vdev->v_id)
1061 			return (previous);
1062 
1063 		if (v->v_id == vdev->v_id)
1064 			return (v);
1065 
1066 		if (v->v_id < vdev->v_id)
1067 			previous = v;
1068 	}
1069 	return (previous);
1070 }
1071 
1072 static size_t
1073 vdev_child_count(vdev_t *vdev)
1074 {
1075 	vdev_t *v;
1076 	size_t count;
1077 
1078 	count = 0;
1079 	STAILQ_FOREACH(v, &vdev->v_children, v_childlink) {
1080 		count++;
1081 	}
1082 	return (count);
1083 }
1084 
1085 /*
1086  * Insert vdev into top_vdev children list. List is ordered by v_id.
1087  */
1088 static void
1089 vdev_insert(vdev_t *top_vdev, vdev_t *vdev)
1090 {
1091 	vdev_t *previous;
1092 	size_t count;
1093 
1094 	/*
1095 	 * The top level vdev can appear in random order, depending how
1096 	 * the firmware is presenting the disk devices.
1097 	 * However, we will insert vdev to create list ordered by v_id,
1098 	 * so we can use either STAILQ_INSERT_HEAD or STAILQ_INSERT_AFTER
1099 	 * as STAILQ does not have insert before.
1100 	 */
1101 	previous = vdev_find_previous(top_vdev, vdev);
1102 
1103 	if (previous == NULL) {
1104 		STAILQ_INSERT_HEAD(&top_vdev->v_children, vdev, v_childlink);
1105 	} else if (previous->v_id == vdev->v_id) {
1106 		/*
1107 		 * This vdev was configured from label config,
1108 		 * do not insert duplicate.
1109 		 */
1110 		return;
1111 	} else {
1112 		STAILQ_INSERT_AFTER(&top_vdev->v_children, previous, vdev,
1113 		    v_childlink);
1114 	}
1115 
1116 	count = vdev_child_count(top_vdev);
1117 	if (top_vdev->v_nchildren < count)
1118 		top_vdev->v_nchildren = count;
1119 }
1120 
1121 static int
1122 vdev_from_nvlist(spa_t *spa, uint64_t top_guid, const nvlist_t *nvlist)
1123 {
1124 	vdev_t *top_vdev, *vdev;
1125 	nvlist_t **kids = NULL;
1126 	int rc, nkids;
1127 
1128 	/* Get top vdev. */
1129 	top_vdev = vdev_find(top_guid);
1130 	if (top_vdev == NULL) {
1131 		rc = vdev_init(top_guid, nvlist, &top_vdev);
1132 		if (rc != 0)
1133 			return (rc);
1134 		top_vdev->v_spa = spa;
1135 		top_vdev->v_top = top_vdev;
1136 		vdev_insert(spa->spa_root_vdev, top_vdev);
1137 	}
1138 
1139 	/* Add children if there are any. */
1140 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1141 	    &nkids, &kids, NULL);
1142 	if (rc == 0) {
1143 		for (int i = 0; i < nkids; i++) {
1144 			uint64_t guid;
1145 
1146 			rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID,
1147 			    DATA_TYPE_UINT64, NULL, &guid, NULL);
1148 			if (rc != 0)
1149 				goto done;
1150 
1151 			rc = vdev_init(guid, kids[i], &vdev);
1152 			if (rc != 0)
1153 				goto done;
1154 
1155 			vdev->v_spa = spa;
1156 			vdev->v_top = top_vdev;
1157 			vdev_insert(top_vdev, vdev);
1158 		}
1159 	} else {
1160 		/*
1161 		 * When there are no children, nvlist_find() does return
1162 		 * error, reset it because leaf devices have no children.
1163 		 */
1164 		rc = 0;
1165 	}
1166 done:
1167 	if (kids != NULL) {
1168 		for (int i = 0; i < nkids; i++)
1169 			nvlist_destroy(kids[i]);
1170 		free(kids);
1171 	}
1172 
1173 	return (rc);
1174 }
1175 
1176 static int
1177 vdev_init_from_label(spa_t *spa, const nvlist_t *nvlist)
1178 {
1179 	uint64_t pool_guid, top_guid;
1180 	nvlist_t *vdevs;
1181 	int rc;
1182 
1183 	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
1184 	    NULL, &pool_guid, NULL) ||
1185 	    nvlist_find(nvlist, ZPOOL_CONFIG_TOP_GUID, DATA_TYPE_UINT64,
1186 	    NULL, &top_guid, NULL) ||
1187 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1188 	    NULL, &vdevs, NULL)) {
1189 		printf("ZFS: can't find vdev details\n");
1190 		return (ENOENT);
1191 	}
1192 
1193 	rc = vdev_from_nvlist(spa, top_guid, vdevs);
1194 	nvlist_destroy(vdevs);
1195 	return (rc);
1196 }
1197 
1198 static void
1199 vdev_set_state(vdev_t *vdev)
1200 {
1201 	vdev_t *kid;
1202 	int good_kids;
1203 	int bad_kids;
1204 
1205 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1206 		vdev_set_state(kid);
1207 	}
1208 
1209 	/*
1210 	 * A mirror or raidz is healthy if all its kids are healthy. A
1211 	 * mirror is degraded if any of its kids is healthy; a raidz
1212 	 * is degraded if at most nparity kids are offline.
1213 	 */
1214 	if (STAILQ_FIRST(&vdev->v_children)) {
1215 		good_kids = 0;
1216 		bad_kids = 0;
1217 		STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1218 			if (kid->v_state == VDEV_STATE_HEALTHY)
1219 				good_kids++;
1220 			else
1221 				bad_kids++;
1222 		}
1223 		if (bad_kids == 0) {
1224 			vdev->v_state = VDEV_STATE_HEALTHY;
1225 		} else {
1226 			if (vdev->v_read == vdev_mirror_read) {
1227 				if (good_kids) {
1228 					vdev->v_state = VDEV_STATE_DEGRADED;
1229 				} else {
1230 					vdev->v_state = VDEV_STATE_OFFLINE;
1231 				}
1232 			} else if (vdev->v_read == vdev_raidz_read) {
1233 				if (bad_kids > vdev->v_nparity) {
1234 					vdev->v_state = VDEV_STATE_OFFLINE;
1235 				} else {
1236 					vdev->v_state = VDEV_STATE_DEGRADED;
1237 				}
1238 			}
1239 		}
1240 	}
1241 }
1242 
1243 static int
1244 vdev_update_from_nvlist(uint64_t top_guid, const nvlist_t *nvlist)
1245 {
1246 	vdev_t *vdev;
1247 	nvlist_t **kids = NULL;
1248 	int rc, nkids;
1249 
1250 	/* Update top vdev. */
1251 	vdev = vdev_find(top_guid);
1252 	if (vdev != NULL)
1253 		vdev_set_initial_state(vdev, nvlist);
1254 
1255 	/* Update children if there are any. */
1256 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1257 	    &nkids, &kids, NULL);
1258 	if (rc == 0) {
1259 		for (int i = 0; i < nkids; i++) {
1260 			uint64_t guid;
1261 
1262 			rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID,
1263 			    DATA_TYPE_UINT64, NULL, &guid, NULL);
1264 			if (rc != 0)
1265 				break;
1266 
1267 			vdev = vdev_find(guid);
1268 			if (vdev != NULL)
1269 				vdev_set_initial_state(vdev, kids[i]);
1270 		}
1271 	} else {
1272 		rc = 0;
1273 	}
1274 	if (kids != NULL) {
1275 		for (int i = 0; i < nkids; i++)
1276 			nvlist_destroy(kids[i]);
1277 		free(kids);
1278 	}
1279 
1280 	return (rc);
1281 }
1282 
1283 static int
1284 vdev_init_from_nvlist(spa_t *spa, const nvlist_t *nvlist)
1285 {
1286 	uint64_t pool_guid, vdev_children;
1287 	nvlist_t *vdevs = NULL, **kids = NULL;
1288 	int rc, nkids;
1289 
1290 	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
1291 	    NULL, &pool_guid, NULL) ||
1292 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_CHILDREN, DATA_TYPE_UINT64,
1293 	    NULL, &vdev_children, NULL) ||
1294 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1295 	    NULL, &vdevs, NULL)) {
1296 		printf("ZFS: can't find vdev details\n");
1297 		return (ENOENT);
1298 	}
1299 
1300 	/* Wrong guid?! */
1301 	if (spa->spa_guid != pool_guid) {
1302 		nvlist_destroy(vdevs);
1303 		return (EINVAL);
1304 	}
1305 
1306 	spa->spa_root_vdev->v_nchildren = vdev_children;
1307 
1308 	rc = nvlist_find(vdevs, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1309 	    &nkids, &kids, NULL);
1310 	nvlist_destroy(vdevs);
1311 
1312 	/*
1313 	 * MOS config has at least one child for root vdev.
1314 	 */
1315 	if (rc != 0)
1316 		return (rc);
1317 
1318 	for (int i = 0; i < nkids; i++) {
1319 		uint64_t guid;
1320 		vdev_t *vdev;
1321 
1322 		rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
1323 		    NULL, &guid, NULL);
1324 		if (rc != 0)
1325 			break;
1326 		vdev = vdev_find(guid);
1327 		/*
1328 		 * Top level vdev is missing, create it.
1329 		 */
1330 		if (vdev == NULL)
1331 			rc = vdev_from_nvlist(spa, guid, kids[i]);
1332 		else
1333 			rc = vdev_update_from_nvlist(guid, kids[i]);
1334 		if (rc != 0)
1335 			break;
1336 	}
1337 	if (kids != NULL) {
1338 		for (int i = 0; i < nkids; i++)
1339 			nvlist_destroy(kids[i]);
1340 		free(kids);
1341 	}
1342 
1343 	/*
1344 	 * Re-evaluate top-level vdev state.
1345 	 */
1346 	vdev_set_state(spa->spa_root_vdev);
1347 
1348 	return (rc);
1349 }
1350 
1351 static spa_t *
1352 spa_find_by_guid(uint64_t guid)
1353 {
1354 	spa_t *spa;
1355 
1356 	STAILQ_FOREACH(spa, &zfs_pools, spa_link)
1357 		if (spa->spa_guid == guid)
1358 			return (spa);
1359 
1360 	return (NULL);
1361 }
1362 
1363 static spa_t *
1364 spa_find_by_name(const char *name)
1365 {
1366 	spa_t *spa;
1367 
1368 	STAILQ_FOREACH(spa, &zfs_pools, spa_link)
1369 		if (strcmp(spa->spa_name, name) == 0)
1370 			return (spa);
1371 
1372 	return (NULL);
1373 }
1374 
1375 static spa_t *
1376 spa_find_by_dev(struct zfs_devdesc *dev)
1377 {
1378 
1379 	if (dev->dd.d_dev->dv_type != DEVT_ZFS)
1380 		return (NULL);
1381 
1382 	if (dev->pool_guid == 0)
1383 		return (STAILQ_FIRST(&zfs_pools));
1384 
1385 	return (spa_find_by_guid(dev->pool_guid));
1386 }
1387 
1388 static spa_t *
1389 spa_create(uint64_t guid, const char *name)
1390 {
1391 	spa_t *spa;
1392 
1393 	if ((spa = calloc(1, sizeof (spa_t))) == NULL)
1394 		return (NULL);
1395 	if ((spa->spa_name = strdup(name)) == NULL) {
1396 		free(spa);
1397 		return (NULL);
1398 	}
1399 	spa->spa_guid = guid;
1400 	spa->spa_root_vdev = vdev_create(guid, NULL);
1401 	if (spa->spa_root_vdev == NULL) {
1402 		free(spa->spa_name);
1403 		free(spa);
1404 		return (NULL);
1405 	}
1406 	spa->spa_root_vdev->v_name = strdup("root");
1407 	STAILQ_INSERT_TAIL(&zfs_pools, spa, spa_link);
1408 
1409 	return (spa);
1410 }
1411 
1412 static const char *
1413 state_name(vdev_state_t state)
1414 {
1415 	static const char *names[] = {
1416 		"UNKNOWN",
1417 		"CLOSED",
1418 		"OFFLINE",
1419 		"REMOVED",
1420 		"CANT_OPEN",
1421 		"FAULTED",
1422 		"DEGRADED",
1423 		"ONLINE"
1424 	};
1425 	return (names[state]);
1426 }
1427 
1428 static int
1429 pager_printf(const char *fmt, ...)
1430 {
1431 	char line[80];
1432 	va_list args;
1433 
1434 	va_start(args, fmt);
1435 	vsnprintf(line, sizeof (line), fmt, args);
1436 	va_end(args);
1437 	return (pager_output(line));
1438 }
1439 
1440 #define	STATUS_FORMAT	"        %s %s\n"
1441 
1442 static int
1443 print_state(int indent, const char *name, vdev_state_t state)
1444 {
1445 	int i;
1446 	char buf[512];
1447 
1448 	buf[0] = 0;
1449 	for (i = 0; i < indent; i++)
1450 		strcat(buf, "  ");
1451 	strcat(buf, name);
1452 	return (pager_printf(STATUS_FORMAT, buf, state_name(state)));
1453 }
1454 
1455 static int
1456 vdev_status(vdev_t *vdev, int indent)
1457 {
1458 	vdev_t *kid;
1459 	int ret;
1460 
1461 	if (vdev->v_islog) {
1462 		(void) pager_output("        logs\n");
1463 		indent++;
1464 	}
1465 
1466 	ret = print_state(indent, vdev->v_name, vdev->v_state);
1467 	if (ret != 0)
1468 		return (ret);
1469 
1470 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1471 		ret = vdev_status(kid, indent + 1);
1472 		if (ret != 0)
1473 			return (ret);
1474 	}
1475 	return (ret);
1476 }
1477 
1478 static int
1479 spa_status(spa_t *spa)
1480 {
1481 	static char bootfs[ZFS_MAXNAMELEN];
1482 	uint64_t rootid;
1483 	vdev_list_t *vlist;
1484 	vdev_t *vdev;
1485 	int good_kids, bad_kids, degraded_kids, ret;
1486 	vdev_state_t state;
1487 
1488 	ret = pager_printf("  pool: %s\n", spa->spa_name);
1489 	if (ret != 0)
1490 		return (ret);
1491 
1492 	if (zfs_get_root(spa, &rootid) == 0 &&
1493 	    zfs_rlookup(spa, rootid, bootfs) == 0) {
1494 		if (bootfs[0] == '\0')
1495 			ret = pager_printf("bootfs: %s\n", spa->spa_name);
1496 		else
1497 			ret = pager_printf("bootfs: %s/%s\n", spa->spa_name,
1498 			    bootfs);
1499 		if (ret != 0)
1500 			return (ret);
1501 	}
1502 	ret = pager_printf("config:\n\n");
1503 	if (ret != 0)
1504 		return (ret);
1505 	ret = pager_printf(STATUS_FORMAT, "NAME", "STATE");
1506 	if (ret != 0)
1507 		return (ret);
1508 
1509 	good_kids = 0;
1510 	degraded_kids = 0;
1511 	bad_kids = 0;
1512 	vlist = &spa->spa_root_vdev->v_children;
1513 	STAILQ_FOREACH(vdev, vlist, v_childlink) {
1514 		if (vdev->v_state == VDEV_STATE_HEALTHY)
1515 			good_kids++;
1516 		else if (vdev->v_state == VDEV_STATE_DEGRADED)
1517 			degraded_kids++;
1518 		else
1519 			bad_kids++;
1520 	}
1521 
1522 	state = VDEV_STATE_CLOSED;
1523 	if (good_kids > 0 && (degraded_kids + bad_kids) == 0)
1524 		state = VDEV_STATE_HEALTHY;
1525 	else if ((good_kids + degraded_kids) > 0)
1526 		state = VDEV_STATE_DEGRADED;
1527 
1528 	ret = print_state(0, spa->spa_name, state);
1529 	if (ret != 0)
1530 		return (ret);
1531 
1532 	STAILQ_FOREACH(vdev, vlist, v_childlink) {
1533 		ret = vdev_status(vdev, 1);
1534 		if (ret != 0)
1535 			return (ret);
1536 	}
1537 	return (ret);
1538 }
1539 
1540 int
1541 spa_all_status(void)
1542 {
1543 	spa_t *spa;
1544 	int first = 1, ret = 0;
1545 
1546 	STAILQ_FOREACH(spa, &zfs_pools, spa_link) {
1547 		if (!first) {
1548 			ret = pager_printf("\n");
1549 			if (ret != 0)
1550 				return (ret);
1551 		}
1552 		first = 0;
1553 		ret = spa_status(spa);
1554 		if (ret != 0)
1555 			return (ret);
1556 	}
1557 	return (ret);
1558 }
1559 
1560 uint64_t
1561 vdev_label_offset(uint64_t psize, int l, uint64_t offset)
1562 {
1563 	uint64_t label_offset;
1564 
1565 	if (l < VDEV_LABELS / 2)
1566 		label_offset = 0;
1567 	else
1568 		label_offset = psize - VDEV_LABELS * sizeof (vdev_label_t);
1569 
1570 	return (offset + l * sizeof (vdev_label_t) + label_offset);
1571 }
1572 
1573 static int
1574 vdev_uberblock_compare(const uberblock_t *ub1, const uberblock_t *ub2)
1575 {
1576 	unsigned int seq1 = 0;
1577 	unsigned int seq2 = 0;
1578 	int cmp = AVL_CMP(ub1->ub_txg, ub2->ub_txg);
1579 
1580 	if (cmp != 0)
1581 		return (cmp);
1582 
1583 	cmp = AVL_CMP(ub1->ub_timestamp, ub2->ub_timestamp);
1584 	if (cmp != 0)
1585 		return (cmp);
1586 
1587 	if (MMP_VALID(ub1) && MMP_SEQ_VALID(ub1))
1588 		seq1 = MMP_SEQ(ub1);
1589 
1590 	if (MMP_VALID(ub2) && MMP_SEQ_VALID(ub2))
1591 		seq2 = MMP_SEQ(ub2);
1592 
1593 	return (AVL_CMP(seq1, seq2));
1594 }
1595 
1596 static int
1597 uberblock_verify(uberblock_t *ub)
1598 {
1599 	if (ub->ub_magic == BSWAP_64((uint64_t)UBERBLOCK_MAGIC)) {
1600 		byteswap_uint64_array(ub, sizeof (uberblock_t));
1601 	}
1602 
1603 	if (ub->ub_magic != UBERBLOCK_MAGIC ||
1604 	    !SPA_VERSION_IS_SUPPORTED(ub->ub_version))
1605 		return (EINVAL);
1606 
1607 	return (0);
1608 }
1609 
1610 static int
1611 vdev_label_read(vdev_t *vd, int l, void *buf, uint64_t offset,
1612     size_t size)
1613 {
1614 	blkptr_t bp;
1615 	off_t off;
1616 
1617 	off = vdev_label_offset(vd->v_psize, l, offset);
1618 
1619 	BP_ZERO(&bp);
1620 	BP_SET_LSIZE(&bp, size);
1621 	BP_SET_PSIZE(&bp, size);
1622 	BP_SET_CHECKSUM(&bp, ZIO_CHECKSUM_LABEL);
1623 	BP_SET_COMPRESS(&bp, ZIO_COMPRESS_OFF);
1624 	DVA_SET_OFFSET(BP_IDENTITY(&bp), off);
1625 	ZIO_SET_CHECKSUM(&bp.blk_cksum, off, 0, 0, 0);
1626 
1627 	return (vdev_read_phys(vd, &bp, buf, off, size));
1628 }
1629 
1630 /*
1631  * We do need to be sure we write to correct location.
1632  * Our vdev label does consist of 4 fields:
1633  * pad1 (8k), reserved.
1634  * bootenv (8k), checksummed, previously reserved, may contain garbage.
1635  * vdev_phys (112k), checksummed
1636  * uberblock ring (128k), checksummed.
1637  *
1638  * Since bootenv area may contain garbage, we can not reliably read it, as
1639  * we can get checksum errors.
1640  * Next best thing is vdev_phys - it is just after bootenv. It still may
1641  * be corrupted, but in such case we will miss this one write.
1642  */
1643 static int
1644 vdev_label_write_validate(vdev_t *vd, int l, uint64_t offset)
1645 {
1646 	uint64_t off, o_phys;
1647 	void *buf;
1648 	size_t size = VDEV_PHYS_SIZE;
1649 	int rc;
1650 
1651 	o_phys = offsetof(vdev_label_t, vl_vdev_phys);
1652 	off = vdev_label_offset(vd->v_psize, l, o_phys);
1653 
1654 	/* off should be 8K from bootenv */
1655 	if (vdev_label_offset(vd->v_psize, l, offset) + VDEV_PAD_SIZE != off)
1656 		return (EINVAL);
1657 
1658 	buf = malloc(size);
1659 	if (buf == NULL)
1660 		return (ENOMEM);
1661 
1662 	/* Read vdev_phys */
1663 	rc = vdev_label_read(vd, l, buf, o_phys, size);
1664 	free(buf);
1665 	return (rc);
1666 }
1667 
1668 static int
1669 vdev_label_write(vdev_t *vd, int l, vdev_boot_envblock_t *be, uint64_t offset)
1670 {
1671 	zio_checksum_info_t *ci;
1672 	zio_cksum_t cksum;
1673 	off_t off;
1674 	size_t size = VDEV_PAD_SIZE;
1675 	int rc;
1676 
1677 	if (vd->v_phys_write == NULL)
1678 		return (ENOTSUP);
1679 
1680 	off = vdev_label_offset(vd->v_psize, l, offset);
1681 
1682 	rc = vdev_label_write_validate(vd, l, offset);
1683 	if (rc != 0) {
1684 		return (rc);
1685 	}
1686 
1687 	ci = &zio_checksum_table[ZIO_CHECKSUM_LABEL];
1688 	be->vbe_zbt.zec_magic = ZEC_MAGIC;
1689 	zio_checksum_label_verifier(&be->vbe_zbt.zec_cksum, off);
1690 	ci->ci_func[0](be, size, NULL, &cksum);
1691 	be->vbe_zbt.zec_cksum = cksum;
1692 
1693 	return (vdev_write_phys(vd, be, off, size));
1694 }
1695 
1696 static int
1697 vdev_write_bootenv_impl(vdev_t *vdev, vdev_boot_envblock_t *be)
1698 {
1699 	vdev_t *kid;
1700 	int rv = 0, rc;
1701 
1702 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1703 		if (kid->v_state != VDEV_STATE_HEALTHY)
1704 			continue;
1705 		rc = vdev_write_bootenv_impl(kid, be);
1706 		if (rv == 0)
1707 			rv = rc;
1708 	}
1709 
1710 	/*
1711 	 * Non-leaf vdevs do not have v_phys_write.
1712 	 */
1713 	if (vdev->v_phys_write == NULL)
1714 		return (rv);
1715 
1716 	for (int l = 0; l < VDEV_LABELS; l++) {
1717 		rc = vdev_label_write(vdev, l, be,
1718 		    offsetof(vdev_label_t, vl_be));
1719 		if (rc != 0) {
1720 			printf("failed to write bootenv to %s label %d: %d\n",
1721 			    vdev->v_name ? vdev->v_name : "unknown", l, rc);
1722 			rv = rc;
1723 		}
1724 	}
1725 	return (rv);
1726 }
1727 
1728 int
1729 vdev_write_bootenv(vdev_t *vdev, nvlist_t *nvl)
1730 {
1731 	vdev_boot_envblock_t *be;
1732 	nvlist_t nv, *nvp;
1733 	uint64_t version;
1734 	int rv;
1735 
1736 	if (nvl->nv_size > sizeof (be->vbe_bootenv))
1737 		return (E2BIG);
1738 
1739 	version = VB_RAW;
1740 	nvp = vdev_read_bootenv(vdev);
1741 	if (nvp != NULL) {
1742 		nvlist_find(nvp, BOOTENV_VERSION, DATA_TYPE_UINT64, NULL,
1743 		    &version, NULL);
1744 		nvlist_destroy(nvp);
1745 	}
1746 
1747 	be = calloc(1, sizeof (*be));
1748 	if (be == NULL)
1749 		return (ENOMEM);
1750 
1751 	be->vbe_version = version;
1752 	switch (version) {
1753 	case VB_RAW:
1754 		/*
1755 		 * If there is no envmap, we will just wipe bootenv.
1756 		 */
1757 		nvlist_find(nvl, GRUB_ENVMAP, DATA_TYPE_STRING, NULL,
1758 		    be->vbe_bootenv, NULL);
1759 		rv = 0;
1760 		break;
1761 
1762 	case VB_NVLIST:
1763 		nv.nv_header = nvl->nv_header;
1764 		nv.nv_asize = nvl->nv_asize;
1765 		nv.nv_size = nvl->nv_size;
1766 
1767 		bcopy(&nv.nv_header, be->vbe_bootenv, sizeof (nv.nv_header));
1768 		nv.nv_data = (uint8_t *)be->vbe_bootenv + sizeof (nvs_header_t);
1769 		bcopy(nvl->nv_data, nv.nv_data, nv.nv_size);
1770 		rv = nvlist_export(&nv);
1771 		break;
1772 
1773 	default:
1774 		rv = EINVAL;
1775 		break;
1776 	}
1777 
1778 	if (rv == 0) {
1779 		be->vbe_version = htobe64(be->vbe_version);
1780 		rv = vdev_write_bootenv_impl(vdev, be);
1781 	}
1782 	free(be);
1783 	return (rv);
1784 }
1785 
1786 /*
1787  * Read the bootenv area from pool label, return the nvlist from it.
1788  * We return from first successful read.
1789  */
1790 nvlist_t *
1791 vdev_read_bootenv(vdev_t *vdev)
1792 {
1793 	vdev_t *kid;
1794 	nvlist_t *benv;
1795 	vdev_boot_envblock_t *be;
1796 	char *command;
1797 	bool ok;
1798 	int rv;
1799 
1800 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1801 		if (kid->v_state != VDEV_STATE_HEALTHY)
1802 			continue;
1803 
1804 		benv = vdev_read_bootenv(kid);
1805 		if (benv != NULL)
1806 			return (benv);
1807 	}
1808 
1809 	be = malloc(sizeof (*be));
1810 	if (be == NULL)
1811 		return (NULL);
1812 
1813 	rv = 0;
1814 	for (int l = 0; l < VDEV_LABELS; l++) {
1815 		rv = vdev_label_read(vdev, l, be,
1816 		    offsetof(vdev_label_t, vl_be),
1817 		    sizeof (*be));
1818 		if (rv == 0)
1819 			break;
1820 	}
1821 	if (rv != 0) {
1822 		free(be);
1823 		return (NULL);
1824 	}
1825 
1826 	be->vbe_version = be64toh(be->vbe_version);
1827 	switch (be->vbe_version) {
1828 	case VB_RAW:
1829 		/*
1830 		 * if we have textual data in vbe_bootenv, create nvlist
1831 		 * with key "envmap".
1832 		 */
1833 		benv = nvlist_create(NV_UNIQUE_NAME);
1834 		if (benv != NULL) {
1835 			if (*be->vbe_bootenv == '\0') {
1836 				nvlist_add_uint64(benv, BOOTENV_VERSION,
1837 				    VB_NVLIST);
1838 				break;
1839 			}
1840 			nvlist_add_uint64(benv, BOOTENV_VERSION, VB_RAW);
1841 			be->vbe_bootenv[sizeof (be->vbe_bootenv) - 1] = '\0';
1842 			nvlist_add_string(benv, GRUB_ENVMAP, be->vbe_bootenv);
1843 		}
1844 		break;
1845 
1846 	case VB_NVLIST:
1847 		benv = nvlist_import(be->vbe_bootenv, sizeof (be->vbe_bootenv));
1848 		break;
1849 
1850 	default:
1851 		command = (char *)be;
1852 		ok = false;
1853 
1854 		/* Check for legacy zfsbootcfg command string */
1855 		for (int i = 0; command[i] != '\0'; i++) {
1856 			if (iscntrl(command[i])) {
1857 				ok = false;
1858 				break;
1859 			} else {
1860 				ok = true;
1861 			}
1862 		}
1863 		benv = nvlist_create(NV_UNIQUE_NAME);
1864 		if (benv != NULL) {
1865 			if (ok)
1866 				nvlist_add_string(benv, FREEBSD_BOOTONCE,
1867 				    command);
1868 			else
1869 				nvlist_add_uint64(benv, BOOTENV_VERSION,
1870 				    VB_NVLIST);
1871 		}
1872 		break;
1873 	}
1874 	free(be);
1875 	return (benv);
1876 }
1877 
1878 static uint64_t
1879 vdev_get_label_asize(nvlist_t *nvl)
1880 {
1881 	nvlist_t *vdevs;
1882 	uint64_t asize;
1883 	const char *type;
1884 	int len;
1885 
1886 	asize = 0;
1887 	/* Get vdev tree */
1888 	if (nvlist_find(nvl, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1889 	    NULL, &vdevs, NULL) != 0)
1890 		return (asize);
1891 
1892 	/*
1893 	 * Get vdev type. We will calculate asize for raidz, mirror and disk.
1894 	 * For raidz, the asize is raw size of all children.
1895 	 */
1896 	if (nvlist_find(vdevs, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING,
1897 	    NULL, &type, &len) != 0)
1898 		goto done;
1899 
1900 	if (memcmp(type, VDEV_TYPE_MIRROR, len) != 0 &&
1901 	    memcmp(type, VDEV_TYPE_DISK, len) != 0 &&
1902 	    memcmp(type, VDEV_TYPE_RAIDZ, len) != 0)
1903 		goto done;
1904 
1905 	if (nvlist_find(vdevs, ZPOOL_CONFIG_ASIZE, DATA_TYPE_UINT64,
1906 	    NULL, &asize, NULL) != 0)
1907 		goto done;
1908 
1909 	if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0) {
1910 		nvlist_t **kids;
1911 		int nkids;
1912 
1913 		if (nvlist_find(vdevs, ZPOOL_CONFIG_CHILDREN,
1914 		    DATA_TYPE_NVLIST_ARRAY, &nkids, &kids, NULL) != 0) {
1915 			asize = 0;
1916 			goto done;
1917 		}
1918 
1919 		asize /= nkids;
1920 		for (int i = 0; i < nkids; i++)
1921 			nvlist_destroy(kids[i]);
1922 		free(kids);
1923 	}
1924 
1925 	asize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
1926 done:
1927 	return (asize);
1928 }
1929 
1930 static nvlist_t *
1931 vdev_label_read_config(vdev_t *vd, uint64_t txg)
1932 {
1933 	vdev_phys_t *label;
1934 	uint64_t best_txg = 0;
1935 	uint64_t label_txg = 0;
1936 	uint64_t asize;
1937 	nvlist_t *nvl = NULL, *tmp;
1938 	int error;
1939 
1940 	label = malloc(sizeof (vdev_phys_t));
1941 	if (label == NULL)
1942 		return (NULL);
1943 
1944 	for (int l = 0; l < VDEV_LABELS; l++) {
1945 		if (vdev_label_read(vd, l, label,
1946 		    offsetof(vdev_label_t, vl_vdev_phys),
1947 		    sizeof (vdev_phys_t)))
1948 			continue;
1949 
1950 		tmp = nvlist_import(label->vp_nvlist,
1951 		    sizeof (label->vp_nvlist));
1952 		if (tmp == NULL)
1953 			continue;
1954 
1955 		error = nvlist_find(tmp, ZPOOL_CONFIG_POOL_TXG,
1956 		    DATA_TYPE_UINT64, NULL, &label_txg, NULL);
1957 		if (error != 0 || label_txg == 0) {
1958 			nvlist_destroy(nvl);
1959 			nvl = tmp;
1960 			goto done;
1961 		}
1962 
1963 		if (label_txg <= txg && label_txg > best_txg) {
1964 			best_txg = label_txg;
1965 			nvlist_destroy(nvl);
1966 			nvl = tmp;
1967 			tmp = NULL;
1968 
1969 			/*
1970 			 * Use asize from pool config. We need this
1971 			 * because we can get bad value from BIOS.
1972 			 */
1973 			asize = vdev_get_label_asize(nvl);
1974 			if (asize != 0) {
1975 				vd->v_psize = asize;
1976 			}
1977 		}
1978 		nvlist_destroy(tmp);
1979 	}
1980 
1981 	if (best_txg == 0) {
1982 		nvlist_destroy(nvl);
1983 		nvl = NULL;
1984 	}
1985 done:
1986 	free(label);
1987 	return (nvl);
1988 }
1989 
1990 static void
1991 vdev_uberblock_load(vdev_t *vd, uberblock_t *ub)
1992 {
1993 	uberblock_t *buf;
1994 
1995 	buf = malloc(VDEV_UBERBLOCK_SIZE(vd));
1996 	if (buf == NULL)
1997 		return;
1998 
1999 	for (int l = 0; l < VDEV_LABELS; l++) {
2000 		for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
2001 			if (vdev_label_read(vd, l, buf,
2002 			    VDEV_UBERBLOCK_OFFSET(vd, n),
2003 			    VDEV_UBERBLOCK_SIZE(vd)))
2004 				continue;
2005 			if (uberblock_verify(buf) != 0)
2006 				continue;
2007 
2008 			if (vdev_uberblock_compare(buf, ub) > 0)
2009 				*ub = *buf;
2010 		}
2011 	}
2012 	free(buf);
2013 }
2014 
2015 static int
2016 vdev_probe(vdev_phys_read_t *_read, vdev_phys_write_t *_write, void *priv,
2017     spa_t **spap)
2018 {
2019 	vdev_t vtmp;
2020 	spa_t *spa;
2021 	vdev_t *vdev;
2022 	nvlist_t *nvl;
2023 	uint64_t val;
2024 	uint64_t guid, vdev_children;
2025 	uint64_t pool_txg, pool_guid;
2026 	const char *pool_name;
2027 	int rc, namelen;
2028 
2029 	/*
2030 	 * Load the vdev label and figure out which
2031 	 * uberblock is most current.
2032 	 */
2033 	memset(&vtmp, 0, sizeof (vtmp));
2034 	vtmp.v_phys_read = _read;
2035 	vtmp.v_phys_write = _write;
2036 	vtmp.v_priv = priv;
2037 	vtmp.v_psize = P2ALIGN(ldi_get_size(priv),
2038 	    (uint64_t)sizeof (vdev_label_t));
2039 
2040 	/* Test for minimum device size. */
2041 	if (vtmp.v_psize < SPA_MINDEVSIZE)
2042 		return (EIO);
2043 
2044 	nvl = vdev_label_read_config(&vtmp, UINT64_MAX);
2045 	if (nvl == NULL)
2046 		return (EIO);
2047 
2048 	if (nvlist_find(nvl, ZPOOL_CONFIG_VERSION, DATA_TYPE_UINT64,
2049 	    NULL, &val, NULL) != 0) {
2050 		nvlist_destroy(nvl);
2051 		return (EIO);
2052 	}
2053 
2054 	if (!SPA_VERSION_IS_SUPPORTED(val)) {
2055 		printf("ZFS: unsupported ZFS version %u (should be %u)\n",
2056 		    (unsigned)val, (unsigned)SPA_VERSION);
2057 		nvlist_destroy(nvl);
2058 		return (EIO);
2059 	}
2060 
2061 	/* Check ZFS features for read */
2062 	rc = nvlist_check_features_for_read(nvl);
2063 	if (rc != 0) {
2064 		nvlist_destroy(nvl);
2065 		return (EIO);
2066 	}
2067 
2068 	if (nvlist_find(nvl, ZPOOL_CONFIG_POOL_STATE, DATA_TYPE_UINT64,
2069 	    NULL, &val, NULL) != 0) {
2070 		nvlist_destroy(nvl);
2071 		return (EIO);
2072 	}
2073 
2074 	if (val == POOL_STATE_DESTROYED) {
2075 		/* We don't boot only from destroyed pools. */
2076 		nvlist_destroy(nvl);
2077 		return (EIO);
2078 	}
2079 
2080 	if (nvlist_find(nvl, ZPOOL_CONFIG_POOL_TXG, DATA_TYPE_UINT64,
2081 	    NULL, &pool_txg, NULL) != 0 ||
2082 	    nvlist_find(nvl, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
2083 	    NULL, &pool_guid, NULL) != 0 ||
2084 	    nvlist_find(nvl, ZPOOL_CONFIG_POOL_NAME, DATA_TYPE_STRING,
2085 	    NULL, &pool_name, &namelen) != 0) {
2086 		/*
2087 		 * Cache and spare devices end up here - just ignore
2088 		 * them.
2089 		 */
2090 		nvlist_destroy(nvl);
2091 		return (EIO);
2092 	}
2093 
2094 	/*
2095 	 * Create the pool if this is the first time we've seen it.
2096 	 */
2097 	spa = spa_find_by_guid(pool_guid);
2098 	if (spa == NULL) {
2099 		char *name;
2100 
2101 		nvlist_find(nvl, ZPOOL_CONFIG_VDEV_CHILDREN,
2102 		    DATA_TYPE_UINT64, NULL, &vdev_children, NULL);
2103 		name = malloc(namelen + 1);
2104 		if (name == NULL) {
2105 			nvlist_destroy(nvl);
2106 			return (ENOMEM);
2107 		}
2108 		bcopy(pool_name, name, namelen);
2109 		name[namelen] = '\0';
2110 		spa = spa_create(pool_guid, name);
2111 		free(name);
2112 		if (spa == NULL) {
2113 			nvlist_destroy(nvl);
2114 			return (ENOMEM);
2115 		}
2116 		spa->spa_root_vdev->v_nchildren = vdev_children;
2117 	}
2118 	if (pool_txg > spa->spa_txg)
2119 		spa->spa_txg = pool_txg;
2120 
2121 	/*
2122 	 * Get the vdev tree and create our in-core copy of it.
2123 	 * If we already have a vdev with this guid, this must
2124 	 * be some kind of alias (overlapping slices, dangerously dedicated
2125 	 * disks etc).
2126 	 */
2127 	if (nvlist_find(nvl, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
2128 	    NULL, &guid, NULL) != 0) {
2129 		nvlist_destroy(nvl);
2130 		return (EIO);
2131 	}
2132 	vdev = vdev_find(guid);
2133 	/* Has this vdev already been inited? */
2134 	if (vdev && vdev->v_phys_read) {
2135 		nvlist_destroy(nvl);
2136 		return (EIO);
2137 	}
2138 
2139 	rc = vdev_init_from_label(spa, nvl);
2140 	nvlist_destroy(nvl);
2141 	if (rc != 0)
2142 		return (rc);
2143 
2144 	/*
2145 	 * We should already have created an incomplete vdev for this
2146 	 * vdev. Find it and initialise it with our read proc.
2147 	 */
2148 	vdev = vdev_find(guid);
2149 	if (vdev != NULL) {
2150 		vdev->v_phys_read = _read;
2151 		vdev->v_phys_write = _write;
2152 		vdev->v_priv = priv;
2153 		vdev->v_psize = vtmp.v_psize;
2154 		/*
2155 		 * If no other state is set, mark vdev healthy.
2156 		 */
2157 		if (vdev->v_state == VDEV_STATE_UNKNOWN)
2158 			vdev->v_state = VDEV_STATE_HEALTHY;
2159 	} else {
2160 		printf("ZFS: inconsistent nvlist contents\n");
2161 		return (EIO);
2162 	}
2163 
2164 	if (vdev->v_islog)
2165 		spa->spa_with_log = vdev->v_islog;
2166 
2167 	/* Record boot vdev for spa. */
2168 	if (spa->spa_boot_vdev == NULL)
2169 		spa->spa_boot_vdev = vdev;
2170 
2171 	/*
2172 	 * Re-evaluate top-level vdev state.
2173 	 */
2174 	vdev_set_state(vdev->v_top);
2175 
2176 	/*
2177 	 * Ok, we are happy with the pool so far. Lets find
2178 	 * the best uberblock and then we can actually access
2179 	 * the contents of the pool.
2180 	 */
2181 	vdev_uberblock_load(vdev, &spa->spa_uberblock);
2182 
2183 	if (spap != NULL)
2184 		*spap = spa;
2185 	return (0);
2186 }
2187 
2188 static int
2189 ilog2(int n)
2190 {
2191 	int v;
2192 
2193 	for (v = 0; v < 32; v++)
2194 		if (n == (1 << v))
2195 			return (v);
2196 	return (-1);
2197 }
2198 
2199 static int
2200 zio_read_gang(const spa_t *spa, const blkptr_t *bp, void *buf)
2201 {
2202 	blkptr_t gbh_bp;
2203 	zio_gbh_phys_t zio_gb;
2204 	char *pbuf;
2205 	int i;
2206 
2207 	/* Artificial BP for gang block header. */
2208 	gbh_bp = *bp;
2209 	BP_SET_PSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
2210 	BP_SET_LSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
2211 	BP_SET_CHECKSUM(&gbh_bp, ZIO_CHECKSUM_GANG_HEADER);
2212 	BP_SET_COMPRESS(&gbh_bp, ZIO_COMPRESS_OFF);
2213 	for (i = 0; i < SPA_DVAS_PER_BP; i++)
2214 		DVA_SET_GANG(&gbh_bp.blk_dva[i], 0);
2215 
2216 	/* Read gang header block using the artificial BP. */
2217 	if (zio_read(spa, &gbh_bp, &zio_gb))
2218 		return (EIO);
2219 
2220 	pbuf = buf;
2221 	for (i = 0; i < SPA_GBH_NBLKPTRS; i++) {
2222 		blkptr_t *gbp = &zio_gb.zg_blkptr[i];
2223 
2224 		if (BP_IS_HOLE(gbp))
2225 			continue;
2226 		if (zio_read(spa, gbp, pbuf))
2227 			return (EIO);
2228 		pbuf += BP_GET_PSIZE(gbp);
2229 	}
2230 
2231 	if (zio_checksum_verify(spa, bp, buf))
2232 		return (EIO);
2233 	return (0);
2234 }
2235 
2236 static int
2237 zio_read(const spa_t *spa, const blkptr_t *bp, void *buf)
2238 {
2239 	int cpfunc = BP_GET_COMPRESS(bp);
2240 	uint64_t align, size;
2241 	void *pbuf;
2242 	int i, error;
2243 
2244 	/*
2245 	 * Process data embedded in block pointer
2246 	 */
2247 	if (BP_IS_EMBEDDED(bp)) {
2248 		ASSERT(BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
2249 
2250 		size = BPE_GET_PSIZE(bp);
2251 		ASSERT(size <= BPE_PAYLOAD_SIZE);
2252 
2253 		if (cpfunc != ZIO_COMPRESS_OFF)
2254 			pbuf = malloc(size);
2255 		else
2256 			pbuf = buf;
2257 
2258 		if (pbuf == NULL)
2259 			return (ENOMEM);
2260 
2261 		decode_embedded_bp_compressed(bp, pbuf);
2262 		error = 0;
2263 
2264 		if (cpfunc != ZIO_COMPRESS_OFF) {
2265 			error = zio_decompress_data(cpfunc, pbuf,
2266 			    size, buf, BP_GET_LSIZE(bp));
2267 			free(pbuf);
2268 		}
2269 		if (error != 0)
2270 			printf("ZFS: i/o error - unable to decompress "
2271 			    "block pointer data, error %d\n", error);
2272 		return (error);
2273 	}
2274 
2275 	error = EIO;
2276 
2277 	for (i = 0; i < SPA_DVAS_PER_BP; i++) {
2278 		const dva_t *dva = &bp->blk_dva[i];
2279 		vdev_t *vdev;
2280 		vdev_list_t *vlist;
2281 		uint64_t vdevid;
2282 		off_t offset;
2283 
2284 		if (!dva->dva_word[0] && !dva->dva_word[1])
2285 			continue;
2286 
2287 		vdevid = DVA_GET_VDEV(dva);
2288 		offset = DVA_GET_OFFSET(dva);
2289 		vlist = &spa->spa_root_vdev->v_children;
2290 		STAILQ_FOREACH(vdev, vlist, v_childlink) {
2291 			if (vdev->v_id == vdevid)
2292 				break;
2293 		}
2294 		if (!vdev || !vdev->v_read)
2295 			continue;
2296 
2297 		size = BP_GET_PSIZE(bp);
2298 		if (vdev->v_read == vdev_raidz_read) {
2299 			align = 1ULL << vdev->v_ashift;
2300 			if (P2PHASE(size, align) != 0)
2301 				size = P2ROUNDUP(size, align);
2302 		}
2303 		if (size != BP_GET_PSIZE(bp) || cpfunc != ZIO_COMPRESS_OFF)
2304 			pbuf = malloc(size);
2305 		else
2306 			pbuf = buf;
2307 
2308 		if (pbuf == NULL) {
2309 			error = ENOMEM;
2310 			break;
2311 		}
2312 
2313 		if (DVA_GET_GANG(dva))
2314 			error = zio_read_gang(spa, bp, pbuf);
2315 		else
2316 			error = vdev->v_read(vdev, bp, pbuf, offset, size);
2317 		if (error == 0) {
2318 			if (cpfunc != ZIO_COMPRESS_OFF)
2319 				error = zio_decompress_data(cpfunc, pbuf,
2320 				    BP_GET_PSIZE(bp), buf, BP_GET_LSIZE(bp));
2321 			else if (size != BP_GET_PSIZE(bp))
2322 				bcopy(pbuf, buf, BP_GET_PSIZE(bp));
2323 		}
2324 		if (buf != pbuf)
2325 			free(pbuf);
2326 		if (error == 0)
2327 			break;
2328 	}
2329 	if (error != 0)
2330 		printf("ZFS: i/o error - all block copies unavailable\n");
2331 
2332 	return (error);
2333 }
2334 
2335 static int
2336 dnode_read(const spa_t *spa, const dnode_phys_t *dnode, off_t offset,
2337     void *buf, size_t buflen)
2338 {
2339 	int ibshift = dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
2340 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2341 	int nlevels = dnode->dn_nlevels;
2342 	int i, rc;
2343 
2344 	if (bsize > SPA_MAXBLOCKSIZE) {
2345 		printf("ZFS: I/O error - blocks larger than %llu are not "
2346 		    "supported\n", SPA_MAXBLOCKSIZE);
2347 		return (EIO);
2348 	}
2349 
2350 	/*
2351 	 * Handle odd block sizes, mirrors dmu_read_impl().  Data can't exist
2352 	 * past the first block, so we'll clip the read to the portion of the
2353 	 * buffer within bsize and zero out the remainder.
2354 	 */
2355 	if (dnode->dn_maxblkid == 0) {
2356 		size_t newbuflen;
2357 
2358 		newbuflen = offset > bsize ? 0 : MIN(buflen, bsize - offset);
2359 		bzero((char *)buf + newbuflen, buflen - newbuflen);
2360 		buflen = newbuflen;
2361 	}
2362 
2363 	/*
2364 	 * Note: bsize may not be a power of two here so we need to do an
2365 	 * actual divide rather than a bitshift.
2366 	 */
2367 	while (buflen > 0) {
2368 		uint64_t bn = offset / bsize;
2369 		int boff = offset % bsize;
2370 		int ibn;
2371 		const blkptr_t *indbp;
2372 		blkptr_t bp;
2373 
2374 		if (bn > dnode->dn_maxblkid) {
2375 			printf("warning: zfs bug: bn %llx > dn_maxblkid %llx\n",
2376 			    (unsigned long long)bn,
2377 			    (unsigned long long)dnode->dn_maxblkid);
2378 			/*
2379 			 * zfs bug, will not return error
2380 			 * return (EIO);
2381 			 */
2382 		}
2383 
2384 		if (dnode == dnode_cache_obj && bn == dnode_cache_bn)
2385 			goto cached;
2386 
2387 		indbp = dnode->dn_blkptr;
2388 		for (i = 0; i < nlevels; i++) {
2389 			/*
2390 			 * Copy the bp from the indirect array so that
2391 			 * we can re-use the scratch buffer for multi-level
2392 			 * objects.
2393 			 */
2394 			ibn = bn >> ((nlevels - i - 1) * ibshift);
2395 			ibn &= ((1 << ibshift) - 1);
2396 			bp = indbp[ibn];
2397 			if (BP_IS_HOLE(&bp)) {
2398 				memset(dnode_cache_buf, 0, bsize);
2399 				break;
2400 			}
2401 			rc = zio_read(spa, &bp, dnode_cache_buf);
2402 			if (rc)
2403 				return (rc);
2404 			indbp = (const blkptr_t *) dnode_cache_buf;
2405 		}
2406 		dnode_cache_obj = dnode;
2407 		dnode_cache_bn = bn;
2408 	cached:
2409 
2410 		/*
2411 		 * The buffer contains our data block. Copy what we
2412 		 * need from it and loop.
2413 		 */
2414 		i = bsize - boff;
2415 		if (i > buflen) i = buflen;
2416 		memcpy(buf, &dnode_cache_buf[boff], i);
2417 		buf = ((char *)buf) + i;
2418 		offset += i;
2419 		buflen -= i;
2420 	}
2421 
2422 	return (0);
2423 }
2424 
2425 /*
2426  * Lookup a value in a microzap directory.
2427  */
2428 static int
2429 mzap_lookup(const mzap_phys_t *mz, size_t size, const char *name,
2430     uint64_t *value)
2431 {
2432 	const mzap_ent_phys_t *mze;
2433 	int chunks, i;
2434 
2435 	/*
2436 	 * Microzap objects use exactly one block. Read the whole
2437 	 * thing.
2438 	 */
2439 	chunks = size / MZAP_ENT_LEN - 1;
2440 	for (i = 0; i < chunks; i++) {
2441 		mze = &mz->mz_chunk[i];
2442 		if (strcmp(mze->mze_name, name) == 0) {
2443 			*value = mze->mze_value;
2444 			return (0);
2445 		}
2446 	}
2447 
2448 	return (ENOENT);
2449 }
2450 
2451 /*
2452  * Compare a name with a zap leaf entry. Return non-zero if the name
2453  * matches.
2454  */
2455 static int
2456 fzap_name_equal(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc,
2457     const char *name)
2458 {
2459 	size_t namelen;
2460 	const zap_leaf_chunk_t *nc;
2461 	const char *p;
2462 
2463 	namelen = zc->l_entry.le_name_numints;
2464 
2465 	nc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_name_chunk);
2466 	p = name;
2467 	while (namelen > 0) {
2468 		size_t len;
2469 
2470 		len = namelen;
2471 		if (len > ZAP_LEAF_ARRAY_BYTES)
2472 			len = ZAP_LEAF_ARRAY_BYTES;
2473 		if (memcmp(p, nc->l_array.la_array, len))
2474 			return (0);
2475 		p += len;
2476 		namelen -= len;
2477 		nc = &ZAP_LEAF_CHUNK(zl, nc->l_array.la_next);
2478 	}
2479 
2480 	return (1);
2481 }
2482 
2483 /*
2484  * Extract a uint64_t value from a zap leaf entry.
2485  */
2486 static uint64_t
2487 fzap_leaf_value(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc)
2488 {
2489 	const zap_leaf_chunk_t *vc;
2490 	int i;
2491 	uint64_t value;
2492 	const uint8_t *p;
2493 
2494 	vc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_value_chunk);
2495 	for (i = 0, value = 0, p = vc->l_array.la_array; i < 8; i++) {
2496 		value = (value << 8) | p[i];
2497 	}
2498 
2499 	return (value);
2500 }
2501 
2502 static void
2503 stv(int len, void *addr, uint64_t value)
2504 {
2505 	switch (len) {
2506 	case 1:
2507 		*(uint8_t *)addr = value;
2508 		return;
2509 	case 2:
2510 		*(uint16_t *)addr = value;
2511 		return;
2512 	case 4:
2513 		*(uint32_t *)addr = value;
2514 		return;
2515 	case 8:
2516 		*(uint64_t *)addr = value;
2517 		return;
2518 	}
2519 }
2520 
2521 /*
2522  * Extract a array from a zap leaf entry.
2523  */
2524 static void
2525 fzap_leaf_array(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc,
2526     uint64_t integer_size, uint64_t num_integers, void *buf)
2527 {
2528 	uint64_t array_int_len = zc->l_entry.le_value_intlen;
2529 	uint64_t value = 0;
2530 	uint64_t *u64 = buf;
2531 	char *p = buf;
2532 	int len = MIN(zc->l_entry.le_value_numints, num_integers);
2533 	int chunk = zc->l_entry.le_value_chunk;
2534 	int byten = 0;
2535 
2536 	if (integer_size == 8 && len == 1) {
2537 		*u64 = fzap_leaf_value(zl, zc);
2538 		return;
2539 	}
2540 
2541 	while (len > 0) {
2542 		struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(zl, chunk).l_array;
2543 		int i;
2544 
2545 		ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(zl));
2546 		for (i = 0; i < ZAP_LEAF_ARRAY_BYTES && len > 0; i++) {
2547 			value = (value << 8) | la->la_array[i];
2548 			byten++;
2549 			if (byten == array_int_len) {
2550 				stv(integer_size, p, value);
2551 				byten = 0;
2552 				len--;
2553 				if (len == 0)
2554 					return;
2555 				p += integer_size;
2556 			}
2557 		}
2558 		chunk = la->la_next;
2559 	}
2560 }
2561 
2562 static int
2563 fzap_check_size(uint64_t integer_size, uint64_t num_integers)
2564 {
2565 
2566 	switch (integer_size) {
2567 	case 1:
2568 	case 2:
2569 	case 4:
2570 	case 8:
2571 		break;
2572 	default:
2573 		return (EINVAL);
2574 	}
2575 
2576 	if (integer_size * num_integers > ZAP_MAXVALUELEN)
2577 		return (E2BIG);
2578 
2579 	return (0);
2580 }
2581 
2582 static void
2583 zap_leaf_free(zap_leaf_t *leaf)
2584 {
2585 	free(leaf->l_phys);
2586 	free(leaf);
2587 }
2588 
2589 static int
2590 zap_get_leaf_byblk(fat_zap_t *zap, uint64_t blk, zap_leaf_t **lp)
2591 {
2592 	int bs = FZAP_BLOCK_SHIFT(zap);
2593 	int err;
2594 
2595 	*lp = malloc(sizeof (**lp));
2596 	if (*lp == NULL)
2597 		return (ENOMEM);
2598 
2599 	(*lp)->l_bs = bs;
2600 	(*lp)->l_phys = malloc(1 << bs);
2601 
2602 	if ((*lp)->l_phys == NULL) {
2603 		free(*lp);
2604 		return (ENOMEM);
2605 	}
2606 	err = dnode_read(zap->zap_spa, zap->zap_dnode, blk << bs, (*lp)->l_phys,
2607 	    1 << bs);
2608 	if (err != 0) {
2609 		zap_leaf_free(*lp);
2610 	}
2611 	return (err);
2612 }
2613 
2614 static int
2615 zap_table_load(fat_zap_t *zap, zap_table_phys_t *tbl, uint64_t idx,
2616     uint64_t *valp)
2617 {
2618 	int bs = FZAP_BLOCK_SHIFT(zap);
2619 	uint64_t blk = idx >> (bs - 3);
2620 	uint64_t off = idx & ((1 << (bs - 3)) - 1);
2621 	uint64_t *buf;
2622 	int rc;
2623 
2624 	buf = malloc(1 << zap->zap_block_shift);
2625 	if (buf == NULL)
2626 		return (ENOMEM);
2627 	rc = dnode_read(zap->zap_spa, zap->zap_dnode, (tbl->zt_blk + blk) << bs,
2628 	    buf, 1 << zap->zap_block_shift);
2629 	if (rc == 0)
2630 		*valp = buf[off];
2631 	free(buf);
2632 	return (rc);
2633 }
2634 
2635 static int
2636 zap_idx_to_blk(fat_zap_t *zap, uint64_t idx, uint64_t *valp)
2637 {
2638 	if (zap->zap_phys->zap_ptrtbl.zt_numblks == 0) {
2639 		*valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx);
2640 		return (0);
2641 	} else {
2642 		return (zap_table_load(zap, &zap->zap_phys->zap_ptrtbl,
2643 		    idx, valp));
2644 	}
2645 }
2646 
2647 #define	ZAP_HASH_IDX(hash, n)	(((n) == 0) ? 0 : ((hash) >> (64 - (n))))
2648 static int
2649 zap_deref_leaf(fat_zap_t *zap, uint64_t h, zap_leaf_t **lp)
2650 {
2651 	uint64_t idx, blk;
2652 	int err;
2653 
2654 	idx = ZAP_HASH_IDX(h, zap->zap_phys->zap_ptrtbl.zt_shift);
2655 	err = zap_idx_to_blk(zap, idx, &blk);
2656 	if (err != 0)
2657 		return (err);
2658 	return (zap_get_leaf_byblk(zap, blk, lp));
2659 }
2660 
2661 #define	CHAIN_END	0xffff	/* end of the chunk chain */
2662 #define	LEAF_HASH(l, h) \
2663 	((ZAP_LEAF_HASH_NUMENTRIES(l)-1) & \
2664 	((h) >> \
2665 	(64 - ZAP_LEAF_HASH_SHIFT(l) - (l)->l_phys->l_hdr.lh_prefix_len)))
2666 #define	LEAF_HASH_ENTPTR(l, h)	(&(l)->l_phys->l_hash[LEAF_HASH(l, h)])
2667 
2668 static int
2669 zap_leaf_lookup(zap_leaf_t *zl, uint64_t hash, const char *name,
2670     uint64_t integer_size, uint64_t num_integers, void *value)
2671 {
2672 	int rc;
2673 	uint16_t *chunkp;
2674 	struct zap_leaf_entry *le;
2675 
2676 	/*
2677 	 * Make sure this chunk matches our hash.
2678 	 */
2679 	if (zl->l_phys->l_hdr.lh_prefix_len > 0 &&
2680 	    zl->l_phys->l_hdr.lh_prefix !=
2681 	    hash >> (64 - zl->l_phys->l_hdr.lh_prefix_len))
2682 		return (EIO);
2683 
2684 	rc = ENOENT;
2685 	for (chunkp = LEAF_HASH_ENTPTR(zl, hash);
2686 	    *chunkp != CHAIN_END; chunkp = &le->le_next) {
2687 		zap_leaf_chunk_t *zc;
2688 		uint16_t chunk = *chunkp;
2689 
2690 		le = ZAP_LEAF_ENTRY(zl, chunk);
2691 		if (le->le_hash != hash)
2692 			continue;
2693 		zc = &ZAP_LEAF_CHUNK(zl, chunk);
2694 		if (fzap_name_equal(zl, zc, name)) {
2695 			if (zc->l_entry.le_value_intlen > integer_size) {
2696 				rc = EINVAL;
2697 			} else {
2698 				fzap_leaf_array(zl, zc, integer_size,
2699 				    num_integers, value);
2700 				rc = 0;
2701 			}
2702 			break;
2703 		}
2704 	}
2705 	return (rc);
2706 }
2707 
2708 /*
2709  * Lookup a value in a fatzap directory.
2710  */
2711 static int
2712 fzap_lookup(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2713     const char *name, uint64_t integer_size, uint64_t num_integers,
2714     void *value)
2715 {
2716 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2717 	fat_zap_t z;
2718 	zap_leaf_t *zl;
2719 	uint64_t hash;
2720 	int rc;
2721 
2722 	if (zh->zap_magic != ZAP_MAGIC)
2723 		return (EIO);
2724 
2725 	if ((rc = fzap_check_size(integer_size, num_integers)) != 0)
2726 		return (rc);
2727 
2728 	z.zap_block_shift = ilog2(bsize);
2729 	z.zap_phys = zh;
2730 	z.zap_spa = spa;
2731 	z.zap_dnode = dnode;
2732 
2733 	hash = zap_hash(zh->zap_salt, name);
2734 	rc = zap_deref_leaf(&z, hash, &zl);
2735 	if (rc != 0)
2736 		return (rc);
2737 
2738 	rc = zap_leaf_lookup(zl, hash, name, integer_size, num_integers, value);
2739 
2740 	zap_leaf_free(zl);
2741 	return (rc);
2742 }
2743 
2744 /*
2745  * Lookup a name in a zap object and return its value as a uint64_t.
2746  */
2747 static int
2748 zap_lookup(const spa_t *spa, const dnode_phys_t *dnode, const char *name,
2749     uint64_t integer_size, uint64_t num_integers, void *value)
2750 {
2751 	int rc;
2752 	zap_phys_t *zap;
2753 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2754 
2755 	zap = malloc(size);
2756 	if (zap == NULL)
2757 		return (ENOMEM);
2758 
2759 	rc = dnode_read(spa, dnode, 0, zap, size);
2760 	if (rc)
2761 		goto done;
2762 
2763 	switch (zap->zap_block_type) {
2764 	case ZBT_MICRO:
2765 		rc = mzap_lookup((const mzap_phys_t *)zap, size, name, value);
2766 		break;
2767 	case ZBT_HEADER:
2768 		rc = fzap_lookup(spa, dnode, zap, name, integer_size,
2769 		    num_integers, value);
2770 		break;
2771 	default:
2772 		printf("ZFS: invalid zap_type=%" PRIx64 "\n",
2773 		    zap->zap_block_type);
2774 		rc = EIO;
2775 	}
2776 done:
2777 	free(zap);
2778 	return (rc);
2779 }
2780 
2781 /*
2782  * List a microzap directory.
2783  */
2784 static int
2785 mzap_list(const mzap_phys_t *mz, size_t size,
2786     int (*callback)(const char *, uint64_t))
2787 {
2788 	const mzap_ent_phys_t *mze;
2789 	int chunks, i, rc;
2790 
2791 	/*
2792 	 * Microzap objects use exactly one block. Read the whole
2793 	 * thing.
2794 	 */
2795 	rc = 0;
2796 	chunks = size / MZAP_ENT_LEN - 1;
2797 	for (i = 0; i < chunks; i++) {
2798 		mze = &mz->mz_chunk[i];
2799 		if (mze->mze_name[0]) {
2800 			rc = callback(mze->mze_name, mze->mze_value);
2801 			if (rc != 0)
2802 				break;
2803 		}
2804 	}
2805 
2806 	return (rc);
2807 }
2808 
2809 /*
2810  * List a fatzap directory.
2811  */
2812 static int
2813 fzap_list(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2814     int (*callback)(const char *, uint64_t))
2815 {
2816 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2817 	fat_zap_t z;
2818 	int i, j, rc;
2819 
2820 	if (zh->zap_magic != ZAP_MAGIC)
2821 		return (EIO);
2822 
2823 	z.zap_block_shift = ilog2(bsize);
2824 	z.zap_phys = zh;
2825 
2826 	/*
2827 	 * This assumes that the leaf blocks start at block 1. The
2828 	 * documentation isn't exactly clear on this.
2829 	 */
2830 	zap_leaf_t zl;
2831 	zl.l_bs = z.zap_block_shift;
2832 	zl.l_phys = malloc(bsize);
2833 	if (zl.l_phys == NULL)
2834 		return (ENOMEM);
2835 
2836 	for (i = 0; i < zh->zap_num_leafs; i++) {
2837 		off_t off = ((off_t)(i + 1)) << zl.l_bs;
2838 		char name[256], *p;
2839 		uint64_t value;
2840 
2841 		if (dnode_read(spa, dnode, off, zl.l_phys, bsize)) {
2842 			free(zl.l_phys);
2843 			return (EIO);
2844 		}
2845 
2846 		for (j = 0; j < ZAP_LEAF_NUMCHUNKS(&zl); j++) {
2847 			zap_leaf_chunk_t *zc, *nc;
2848 			int namelen;
2849 
2850 			zc = &ZAP_LEAF_CHUNK(&zl, j);
2851 			if (zc->l_entry.le_type != ZAP_CHUNK_ENTRY)
2852 				continue;
2853 			namelen = zc->l_entry.le_name_numints;
2854 			if (namelen > sizeof (name))
2855 				namelen = sizeof (name);
2856 
2857 			/*
2858 			 * Paste the name back together.
2859 			 */
2860 			nc = &ZAP_LEAF_CHUNK(&zl, zc->l_entry.le_name_chunk);
2861 			p = name;
2862 			while (namelen > 0) {
2863 				int len;
2864 				len = namelen;
2865 				if (len > ZAP_LEAF_ARRAY_BYTES)
2866 					len = ZAP_LEAF_ARRAY_BYTES;
2867 				memcpy(p, nc->l_array.la_array, len);
2868 				p += len;
2869 				namelen -= len;
2870 				nc = &ZAP_LEAF_CHUNK(&zl, nc->l_array.la_next);
2871 			}
2872 
2873 			/*
2874 			 * Assume the first eight bytes of the value are
2875 			 * a uint64_t.
2876 			 */
2877 			value = fzap_leaf_value(&zl, zc);
2878 
2879 			/* printf("%s 0x%jx\n", name, (uintmax_t)value); */
2880 			rc = callback((const char *)name, value);
2881 			if (rc != 0) {
2882 				free(zl.l_phys);
2883 				return (rc);
2884 			}
2885 		}
2886 	}
2887 
2888 	free(zl.l_phys);
2889 	return (0);
2890 }
2891 
2892 static int zfs_printf(const char *name, uint64_t value __unused)
2893 {
2894 
2895 	printf("%s\n", name);
2896 
2897 	return (0);
2898 }
2899 
2900 /*
2901  * List a zap directory.
2902  */
2903 static int
2904 zap_list(const spa_t *spa, const dnode_phys_t *dnode)
2905 {
2906 	zap_phys_t *zap;
2907 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2908 	int rc;
2909 
2910 	zap = malloc(size);
2911 	if (zap == NULL)
2912 		return (ENOMEM);
2913 
2914 	rc = dnode_read(spa, dnode, 0, zap, size);
2915 	if (rc == 0) {
2916 		if (zap->zap_block_type == ZBT_MICRO)
2917 			rc = mzap_list((const mzap_phys_t *)zap, size,
2918 			    zfs_printf);
2919 		else
2920 			rc = fzap_list(spa, dnode, zap, zfs_printf);
2921 	}
2922 	free(zap);
2923 	return (rc);
2924 }
2925 
2926 static int
2927 objset_get_dnode(const spa_t *spa, const objset_phys_t *os, uint64_t objnum,
2928     dnode_phys_t *dnode)
2929 {
2930 	off_t offset;
2931 
2932 	offset = objnum * sizeof (dnode_phys_t);
2933 	return (dnode_read(spa, &os->os_meta_dnode, offset,
2934 	    dnode, sizeof (dnode_phys_t)));
2935 }
2936 
2937 /*
2938  * Lookup a name in a microzap directory.
2939  */
2940 static int
2941 mzap_rlookup(const mzap_phys_t *mz, size_t size, char *name, uint64_t value)
2942 {
2943 	const mzap_ent_phys_t *mze;
2944 	int chunks, i;
2945 
2946 	/*
2947 	 * Microzap objects use exactly one block. Read the whole
2948 	 * thing.
2949 	 */
2950 	chunks = size / MZAP_ENT_LEN - 1;
2951 	for (i = 0; i < chunks; i++) {
2952 		mze = &mz->mz_chunk[i];
2953 		if (value == mze->mze_value) {
2954 			strcpy(name, mze->mze_name);
2955 			return (0);
2956 		}
2957 	}
2958 
2959 	return (ENOENT);
2960 }
2961 
2962 static void
2963 fzap_name_copy(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc, char *name)
2964 {
2965 	size_t namelen;
2966 	const zap_leaf_chunk_t *nc;
2967 	char *p;
2968 
2969 	namelen = zc->l_entry.le_name_numints;
2970 
2971 	nc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_name_chunk);
2972 	p = name;
2973 	while (namelen > 0) {
2974 		size_t len;
2975 		len = namelen;
2976 		if (len > ZAP_LEAF_ARRAY_BYTES)
2977 			len = ZAP_LEAF_ARRAY_BYTES;
2978 		memcpy(p, nc->l_array.la_array, len);
2979 		p += len;
2980 		namelen -= len;
2981 		nc = &ZAP_LEAF_CHUNK(zl, nc->l_array.la_next);
2982 	}
2983 
2984 	*p = '\0';
2985 }
2986 
2987 static int
2988 fzap_rlookup(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2989     char *name, uint64_t value)
2990 {
2991 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2992 	fat_zap_t z;
2993 	uint64_t i;
2994 	int j, rc;
2995 
2996 	if (zh->zap_magic != ZAP_MAGIC)
2997 		return (EIO);
2998 
2999 	z.zap_block_shift = ilog2(bsize);
3000 	z.zap_phys = zh;
3001 
3002 	/*
3003 	 * This assumes that the leaf blocks start at block 1. The
3004 	 * documentation isn't exactly clear on this.
3005 	 */
3006 	zap_leaf_t zl;
3007 	zl.l_bs = z.zap_block_shift;
3008 	zl.l_phys = malloc(bsize);
3009 	if (zl.l_phys == NULL)
3010 		return (ENOMEM);
3011 
3012 	for (i = 0; i < zh->zap_num_leafs; i++) {
3013 		off_t off = ((off_t)(i + 1)) << zl.l_bs;
3014 
3015 		rc = dnode_read(spa, dnode, off, zl.l_phys, bsize);
3016 		if (rc != 0)
3017 			goto done;
3018 
3019 		for (j = 0; j < ZAP_LEAF_NUMCHUNKS(&zl); j++) {
3020 			zap_leaf_chunk_t *zc;
3021 
3022 			zc = &ZAP_LEAF_CHUNK(&zl, j);
3023 			if (zc->l_entry.le_type != ZAP_CHUNK_ENTRY)
3024 				continue;
3025 			if (zc->l_entry.le_value_intlen != 8 ||
3026 			    zc->l_entry.le_value_numints != 1)
3027 				continue;
3028 
3029 			if (fzap_leaf_value(&zl, zc) == value) {
3030 				fzap_name_copy(&zl, zc, name);
3031 				goto done;
3032 			}
3033 		}
3034 	}
3035 
3036 	rc = ENOENT;
3037 done:
3038 	free(zl.l_phys);
3039 	return (rc);
3040 }
3041 
3042 static int
3043 zap_rlookup(const spa_t *spa, const dnode_phys_t *dnode, char *name,
3044     uint64_t value)
3045 {
3046 	zap_phys_t *zap;
3047 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
3048 	int rc;
3049 
3050 	zap = malloc(size);
3051 	if (zap == NULL)
3052 		return (ENOMEM);
3053 
3054 	rc = dnode_read(spa, dnode, 0, zap, size);
3055 	if (rc == 0) {
3056 		if (zap->zap_block_type == ZBT_MICRO)
3057 			rc = mzap_rlookup((const mzap_phys_t *)zap, size,
3058 			    name, value);
3059 		else
3060 			rc = fzap_rlookup(spa, dnode, zap, name, value);
3061 	}
3062 	free(zap);
3063 	return (rc);
3064 }
3065 
3066 static int
3067 zfs_rlookup(const spa_t *spa, uint64_t objnum, char *result)
3068 {
3069 	char name[256];
3070 	char component[256];
3071 	uint64_t dir_obj, parent_obj, child_dir_zapobj;
3072 	dnode_phys_t child_dir_zap, dataset, dir, parent;
3073 	dsl_dir_phys_t *dd;
3074 	dsl_dataset_phys_t *ds;
3075 	char *p;
3076 	int len;
3077 
3078 	p = &name[sizeof (name) - 1];
3079 	*p = '\0';
3080 
3081 	if (objset_get_dnode(spa, &spa->spa_mos, objnum, &dataset)) {
3082 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3083 		return (EIO);
3084 	}
3085 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3086 	dir_obj = ds->ds_dir_obj;
3087 
3088 	for (;;) {
3089 		if (objset_get_dnode(spa, &spa->spa_mos, dir_obj, &dir) != 0)
3090 			return (EIO);
3091 		dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3092 
3093 		/* Actual loop condition. */
3094 		parent_obj = dd->dd_parent_obj;
3095 		if (parent_obj == 0)
3096 			break;
3097 
3098 		if (objset_get_dnode(spa, &spa->spa_mos, parent_obj,
3099 		    &parent) != 0)
3100 			return (EIO);
3101 		dd = (dsl_dir_phys_t *)&parent.dn_bonus;
3102 		child_dir_zapobj = dd->dd_child_dir_zapobj;
3103 		if (objset_get_dnode(spa, &spa->spa_mos, child_dir_zapobj,
3104 		    &child_dir_zap) != 0)
3105 			return (EIO);
3106 		if (zap_rlookup(spa, &child_dir_zap, component, dir_obj) != 0)
3107 			return (EIO);
3108 
3109 		len = strlen(component);
3110 		p -= len;
3111 		memcpy(p, component, len);
3112 		--p;
3113 		*p = '/';
3114 
3115 		/* Actual loop iteration. */
3116 		dir_obj = parent_obj;
3117 	}
3118 
3119 	if (*p != '\0')
3120 		++p;
3121 	strcpy(result, p);
3122 
3123 	return (0);
3124 }
3125 
3126 static int
3127 zfs_lookup_dataset(const spa_t *spa, const char *name, uint64_t *objnum)
3128 {
3129 	char element[256];
3130 	uint64_t dir_obj, child_dir_zapobj;
3131 	dnode_phys_t child_dir_zap, dir;
3132 	dsl_dir_phys_t *dd;
3133 	const char *p, *q;
3134 
3135 	if (objset_get_dnode(spa, &spa->spa_mos,
3136 	    DMU_POOL_DIRECTORY_OBJECT, &dir))
3137 		return (EIO);
3138 	if (zap_lookup(spa, &dir, DMU_POOL_ROOT_DATASET, sizeof (dir_obj),
3139 	    1, &dir_obj))
3140 		return (EIO);
3141 
3142 	p = name;
3143 	for (;;) {
3144 		if (objset_get_dnode(spa, &spa->spa_mos, dir_obj, &dir))
3145 			return (EIO);
3146 		dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3147 
3148 		while (*p == '/')
3149 			p++;
3150 		/* Actual loop condition #1. */
3151 		if (*p == '\0')
3152 			break;
3153 
3154 		q = strchr(p, '/');
3155 		if (q) {
3156 			memcpy(element, p, q - p);
3157 			element[q - p] = '\0';
3158 			p = q + 1;
3159 		} else {
3160 			strcpy(element, p);
3161 			p += strlen(p);
3162 		}
3163 
3164 		child_dir_zapobj = dd->dd_child_dir_zapobj;
3165 		if (objset_get_dnode(spa, &spa->spa_mos, child_dir_zapobj,
3166 		    &child_dir_zap) != 0)
3167 			return (EIO);
3168 
3169 		/* Actual loop condition #2. */
3170 		if (zap_lookup(spa, &child_dir_zap, element, sizeof (dir_obj),
3171 		    1, &dir_obj) != 0)
3172 			return (ENOENT);
3173 	}
3174 
3175 	*objnum = dd->dd_head_dataset_obj;
3176 	return (0);
3177 }
3178 
3179 #pragma GCC diagnostic ignored "-Wstrict-aliasing"
3180 static int
3181 zfs_list_dataset(const spa_t *spa, uint64_t objnum)
3182 {
3183 	uint64_t dir_obj, child_dir_zapobj;
3184 	dnode_phys_t child_dir_zap, dir, dataset;
3185 	dsl_dataset_phys_t *ds;
3186 	dsl_dir_phys_t *dd;
3187 
3188 	if (objset_get_dnode(spa, &spa->spa_mos, objnum, &dataset)) {
3189 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3190 		return (EIO);
3191 	}
3192 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3193 	dir_obj = ds->ds_dir_obj;
3194 
3195 	if (objset_get_dnode(spa, &spa->spa_mos, dir_obj, &dir)) {
3196 		printf("ZFS: can't find dirobj %ju\n", (uintmax_t)dir_obj);
3197 		return (EIO);
3198 	}
3199 	dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3200 
3201 	child_dir_zapobj = dd->dd_child_dir_zapobj;
3202 	if (objset_get_dnode(spa, &spa->spa_mos, child_dir_zapobj,
3203 	    &child_dir_zap) != 0) {
3204 		printf("ZFS: can't find child zap %ju\n", (uintmax_t)dir_obj);
3205 		return (EIO);
3206 	}
3207 
3208 	return (zap_list(spa, &child_dir_zap) != 0);
3209 }
3210 
3211 int
3212 zfs_callback_dataset(const spa_t *spa, uint64_t objnum,
3213     int (*callback)(const char *, uint64_t))
3214 {
3215 	uint64_t dir_obj, child_dir_zapobj;
3216 	dnode_phys_t child_dir_zap, dir, dataset;
3217 	dsl_dataset_phys_t *ds;
3218 	dsl_dir_phys_t *dd;
3219 	zap_phys_t *zap;
3220 	size_t size;
3221 	int err;
3222 
3223 	err = objset_get_dnode(spa, &spa->spa_mos, objnum, &dataset);
3224 	if (err != 0) {
3225 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3226 		return (err);
3227 	}
3228 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3229 	dir_obj = ds->ds_dir_obj;
3230 
3231 	err = objset_get_dnode(spa, &spa->spa_mos, dir_obj, &dir);
3232 	if (err != 0) {
3233 		printf("ZFS: can't find dirobj %ju\n", (uintmax_t)dir_obj);
3234 		return (err);
3235 	}
3236 	dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3237 
3238 	child_dir_zapobj = dd->dd_child_dir_zapobj;
3239 	err = objset_get_dnode(spa, &spa->spa_mos, child_dir_zapobj,
3240 	    &child_dir_zap);
3241 	if (err != 0) {
3242 		printf("ZFS: can't find child zap %ju\n", (uintmax_t)dir_obj);
3243 		return (err);
3244 	}
3245 
3246 	size = child_dir_zap.dn_datablkszsec << SPA_MINBLOCKSHIFT;
3247 	zap = malloc(size);
3248 	if (zap != NULL) {
3249 		err = dnode_read(spa, &child_dir_zap, 0, zap, size);
3250 		if (err != 0)
3251 			goto done;
3252 
3253 		if (zap->zap_block_type == ZBT_MICRO)
3254 			err = mzap_list((const mzap_phys_t *)zap, size,
3255 			    callback);
3256 		else
3257 			err = fzap_list(spa, &child_dir_zap, zap, callback);
3258 	} else {
3259 		err = ENOMEM;
3260 	}
3261 done:
3262 	free(zap);
3263 	return (err);
3264 }
3265 
3266 /*
3267  * Find the object set given the object number of its dataset object
3268  * and return its details in *objset
3269  */
3270 static int
3271 zfs_mount_dataset(const spa_t *spa, uint64_t objnum, objset_phys_t *objset)
3272 {
3273 	dnode_phys_t dataset;
3274 	dsl_dataset_phys_t *ds;
3275 
3276 	if (objset_get_dnode(spa, &spa->spa_mos, objnum, &dataset)) {
3277 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3278 		return (EIO);
3279 	}
3280 
3281 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3282 	if (zio_read(spa, &ds->ds_bp, objset)) {
3283 		printf("ZFS: can't read object set for dataset %ju\n",
3284 		    (uintmax_t)objnum);
3285 		return (EIO);
3286 	}
3287 
3288 	return (0);
3289 }
3290 
3291 /*
3292  * Find the object set pointed to by the BOOTFS property or the root
3293  * dataset if there is none and return its details in *objset
3294  */
3295 static int
3296 zfs_get_root(const spa_t *spa, uint64_t *objid)
3297 {
3298 	dnode_phys_t dir, propdir;
3299 	uint64_t props, bootfs, root;
3300 
3301 	*objid = 0;
3302 
3303 	/*
3304 	 * Start with the MOS directory object.
3305 	 */
3306 	if (objset_get_dnode(spa, &spa->spa_mos,
3307 	    DMU_POOL_DIRECTORY_OBJECT, &dir)) {
3308 		printf("ZFS: can't read MOS object directory\n");
3309 		return (EIO);
3310 	}
3311 
3312 	/*
3313 	 * Lookup the pool_props and see if we can find a bootfs.
3314 	 */
3315 	if (zap_lookup(spa, &dir, DMU_POOL_PROPS,
3316 	    sizeof (props), 1, &props) == 0 &&
3317 	    objset_get_dnode(spa, &spa->spa_mos, props, &propdir) == 0 &&
3318 	    zap_lookup(spa, &propdir, "bootfs",
3319 	    sizeof (bootfs), 1, &bootfs) == 0 && bootfs != 0) {
3320 		*objid = bootfs;
3321 		return (0);
3322 	}
3323 	/*
3324 	 * Lookup the root dataset directory
3325 	 */
3326 	if (zap_lookup(spa, &dir, DMU_POOL_ROOT_DATASET,
3327 	    sizeof (root), 1, &root) ||
3328 	    objset_get_dnode(spa, &spa->spa_mos, root, &dir)) {
3329 		printf("ZFS: can't find root dsl_dir\n");
3330 		return (EIO);
3331 	}
3332 
3333 	/*
3334 	 * Use the information from the dataset directory's bonus buffer
3335 	 * to find the dataset object and from that the object set itself.
3336 	 */
3337 	dsl_dir_phys_t *dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3338 	*objid = dd->dd_head_dataset_obj;
3339 	return (0);
3340 }
3341 
3342 static int
3343 zfs_mount(const spa_t *spa, uint64_t rootobj, struct zfsmount *mnt)
3344 {
3345 
3346 	mnt->spa = spa;
3347 
3348 	/*
3349 	 * Find the root object set if not explicitly provided
3350 	 */
3351 	if (rootobj == 0 && zfs_get_root(spa, &rootobj)) {
3352 		printf("ZFS: can't find root filesystem\n");
3353 		return (EIO);
3354 	}
3355 
3356 	if (zfs_mount_dataset(spa, rootobj, &mnt->objset)) {
3357 		printf("ZFS: can't open root filesystem\n");
3358 		return (EIO);
3359 	}
3360 
3361 	mnt->rootobj = rootobj;
3362 
3363 	return (0);
3364 }
3365 
3366 /*
3367  * callback function for feature name checks.
3368  */
3369 static int
3370 check_feature(const char *name, uint64_t value)
3371 {
3372 	int i;
3373 
3374 	if (value == 0)
3375 		return (0);
3376 	if (name[0] == '\0')
3377 		return (0);
3378 
3379 	for (i = 0; features_for_read[i] != NULL; i++) {
3380 		if (strcmp(name, features_for_read[i]) == 0)
3381 			return (0);
3382 	}
3383 	printf("ZFS: unsupported feature: %s\n", name);
3384 	return (EIO);
3385 }
3386 
3387 /*
3388  * Checks whether the MOS features that are active are supported.
3389  */
3390 static int
3391 check_mos_features(const spa_t *spa)
3392 {
3393 	dnode_phys_t dir;
3394 	zap_phys_t *zap;
3395 	uint64_t objnum;
3396 	size_t size;
3397 	int rc;
3398 
3399 	if ((rc = objset_get_dnode(spa, &spa->spa_mos, DMU_OT_OBJECT_DIRECTORY,
3400 	    &dir)) != 0)
3401 		return (rc);
3402 	if ((rc = zap_lookup(spa, &dir, DMU_POOL_FEATURES_FOR_READ,
3403 	    sizeof (objnum), 1, &objnum)) != 0) {
3404 		/*
3405 		 * It is older pool without features. As we have already
3406 		 * tested the label, just return without raising the error.
3407 		 */
3408 		if (rc == ENOENT)
3409 			rc = 0;
3410 		return (rc);
3411 	}
3412 
3413 	if ((rc = objset_get_dnode(spa, &spa->spa_mos, objnum, &dir)) != 0)
3414 		return (rc);
3415 
3416 	if (dir.dn_type != DMU_OTN_ZAP_METADATA)
3417 		return (EIO);
3418 
3419 	size = dir.dn_datablkszsec << SPA_MINBLOCKSHIFT;
3420 	zap = malloc(size);
3421 	if (zap == NULL)
3422 		return (ENOMEM);
3423 
3424 	if (dnode_read(spa, &dir, 0, zap, size)) {
3425 		free(zap);
3426 		return (EIO);
3427 	}
3428 
3429 	if (zap->zap_block_type == ZBT_MICRO)
3430 		rc = mzap_list((const mzap_phys_t *)zap, size, check_feature);
3431 	else
3432 		rc = fzap_list(spa, &dir, zap, check_feature);
3433 
3434 	free(zap);
3435 	return (rc);
3436 }
3437 
3438 static int
3439 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
3440 {
3441 	dnode_phys_t dir;
3442 	size_t size;
3443 	int rc;
3444 	char *nv;
3445 
3446 	*value = NULL;
3447 	if ((rc = objset_get_dnode(spa, &spa->spa_mos, obj, &dir)) != 0)
3448 		return (rc);
3449 	if (dir.dn_type != DMU_OT_PACKED_NVLIST &&
3450 	    dir.dn_bonustype != DMU_OT_PACKED_NVLIST_SIZE) {
3451 		return (EIO);
3452 	}
3453 
3454 	if (dir.dn_bonuslen != sizeof (uint64_t))
3455 		return (EIO);
3456 
3457 	size = *(uint64_t *)DN_BONUS(&dir);
3458 	nv = malloc(size);
3459 	if (nv == NULL)
3460 		return (ENOMEM);
3461 
3462 	rc = dnode_read(spa, &dir, 0, nv, size);
3463 	if (rc != 0) {
3464 		free(nv);
3465 		nv = NULL;
3466 		return (rc);
3467 	}
3468 	*value = nvlist_import(nv, size);
3469 	free(nv);
3470 	return (rc);
3471 }
3472 
3473 static int
3474 zfs_spa_init(spa_t *spa)
3475 {
3476 	dnode_phys_t dir;
3477 	uint64_t config_object;
3478 	nvlist_t *nvlist;
3479 	int rc;
3480 
3481 	if (zio_read(spa, &spa->spa_uberblock.ub_rootbp, &spa->spa_mos)) {
3482 		printf("ZFS: can't read MOS of pool %s\n", spa->spa_name);
3483 		return (EIO);
3484 	}
3485 	if (spa->spa_mos.os_type != DMU_OST_META) {
3486 		printf("ZFS: corrupted MOS of pool %s\n", spa->spa_name);
3487 		return (EIO);
3488 	}
3489 
3490 	if (objset_get_dnode(spa, &spa->spa_mos, DMU_POOL_DIRECTORY_OBJECT,
3491 	    &dir)) {
3492 		printf("ZFS: failed to read pool %s directory object\n",
3493 		    spa->spa_name);
3494 		return (EIO);
3495 	}
3496 	/* this is allowed to fail, older pools do not have salt */
3497 	rc = zap_lookup(spa, &dir, DMU_POOL_CHECKSUM_SALT, 1,
3498 	    sizeof (spa->spa_cksum_salt.zcs_bytes),
3499 	    spa->spa_cksum_salt.zcs_bytes);
3500 
3501 	rc = check_mos_features(spa);
3502 	if (rc != 0) {
3503 		printf("ZFS: pool %s is not supported\n", spa->spa_name);
3504 		return (rc);
3505 	}
3506 
3507 	rc = zap_lookup(spa, &dir, DMU_POOL_CONFIG,
3508 	    sizeof (config_object), 1, &config_object);
3509 	if (rc != 0) {
3510 		printf("ZFS: can not read MOS %s\n", DMU_POOL_CONFIG);
3511 		return (EIO);
3512 	}
3513 	rc = load_nvlist(spa, config_object, &nvlist);
3514 	if (rc != 0)
3515 		return (rc);
3516 
3517 	/*
3518 	 * Update vdevs from MOS config. Note, we do skip encoding bytes
3519 	 * here. See also vdev_label_read_config().
3520 	 */
3521 	rc = vdev_init_from_nvlist(spa, nvlist);
3522 	nvlist_destroy(nvlist);
3523 	return (rc);
3524 }
3525 
3526 static int
3527 zfs_dnode_stat(const spa_t *spa, dnode_phys_t *dn, struct stat *sb)
3528 {
3529 
3530 	if (dn->dn_bonustype != DMU_OT_SA) {
3531 		znode_phys_t *zp = (znode_phys_t *)dn->dn_bonus;
3532 
3533 		sb->st_mode = zp->zp_mode;
3534 		sb->st_uid = zp->zp_uid;
3535 		sb->st_gid = zp->zp_gid;
3536 		sb->st_size = zp->zp_size;
3537 	} else {
3538 		sa_hdr_phys_t *sahdrp;
3539 		int hdrsize;
3540 		size_t size = 0;
3541 		void *buf = NULL;
3542 
3543 		if (dn->dn_bonuslen != 0)
3544 			sahdrp = (sa_hdr_phys_t *)DN_BONUS(dn);
3545 		else {
3546 			if ((dn->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0) {
3547 				blkptr_t *bp = DN_SPILL_BLKPTR(dn);
3548 				int error;
3549 
3550 				size = BP_GET_LSIZE(bp);
3551 				buf = malloc(size);
3552 				if (buf == NULL)
3553 					error = ENOMEM;
3554 				else
3555 					error = zio_read(spa, bp, buf);
3556 
3557 				if (error != 0) {
3558 					free(buf);
3559 					return (error);
3560 				}
3561 				sahdrp = buf;
3562 			} else {
3563 				return (EIO);
3564 			}
3565 		}
3566 		hdrsize = SA_HDR_SIZE(sahdrp);
3567 		sb->st_mode = *(uint64_t *)((char *)sahdrp + hdrsize +
3568 		    SA_MODE_OFFSET);
3569 		sb->st_uid = *(uint64_t *)((char *)sahdrp + hdrsize +
3570 		    SA_UID_OFFSET);
3571 		sb->st_gid = *(uint64_t *)((char *)sahdrp + hdrsize +
3572 		    SA_GID_OFFSET);
3573 		sb->st_size = *(uint64_t *)((char *)sahdrp + hdrsize +
3574 		    SA_SIZE_OFFSET);
3575 		free(buf);
3576 	}
3577 
3578 	return (0);
3579 }
3580 
3581 static int
3582 zfs_dnode_readlink(const spa_t *spa, dnode_phys_t *dn, char *path, size_t psize)
3583 {
3584 	int rc = 0;
3585 
3586 	if (dn->dn_bonustype == DMU_OT_SA) {
3587 		sa_hdr_phys_t *sahdrp = NULL;
3588 		size_t size = 0;
3589 		void *buf = NULL;
3590 		int hdrsize;
3591 		char *p;
3592 
3593 		if (dn->dn_bonuslen != 0) {
3594 			sahdrp = (sa_hdr_phys_t *)DN_BONUS(dn);
3595 		} else {
3596 			blkptr_t *bp;
3597 
3598 			if ((dn->dn_flags & DNODE_FLAG_SPILL_BLKPTR) == 0)
3599 				return (EIO);
3600 			bp = DN_SPILL_BLKPTR(dn);
3601 
3602 			size = BP_GET_LSIZE(bp);
3603 			buf = malloc(size);
3604 			if (buf == NULL)
3605 				rc = ENOMEM;
3606 			else
3607 				rc = zio_read(spa, bp, buf);
3608 			if (rc != 0) {
3609 				free(buf);
3610 				return (rc);
3611 			}
3612 			sahdrp = buf;
3613 		}
3614 		hdrsize = SA_HDR_SIZE(sahdrp);
3615 		p = (char *)((uintptr_t)sahdrp + hdrsize + SA_SYMLINK_OFFSET);
3616 		memcpy(path, p, psize);
3617 		free(buf);
3618 		return (0);
3619 	}
3620 	/*
3621 	 * Second test is purely to silence bogus compiler
3622 	 * warning about accessing past the end of dn_bonus.
3623 	 */
3624 	if (psize + sizeof (znode_phys_t) <= dn->dn_bonuslen &&
3625 	    sizeof (znode_phys_t) <= sizeof (dn->dn_bonus)) {
3626 		memcpy(path, &dn->dn_bonus[sizeof (znode_phys_t)], psize);
3627 	} else {
3628 		rc = dnode_read(spa, dn, 0, path, psize);
3629 	}
3630 	return (rc);
3631 }
3632 
3633 struct obj_list {
3634 	uint64_t		objnum;
3635 	STAILQ_ENTRY(obj_list)	entry;
3636 };
3637 
3638 /*
3639  * Lookup a file and return its dnode.
3640  */
3641 static int
3642 zfs_lookup(const struct zfsmount *mnt, const char *upath, dnode_phys_t *dnode)
3643 {
3644 	int rc;
3645 	uint64_t objnum;
3646 	const spa_t *spa;
3647 	dnode_phys_t dn;
3648 	const char *p, *q;
3649 	char element[256];
3650 	char path[1024];
3651 	int symlinks_followed = 0;
3652 	struct stat sb;
3653 	struct obj_list *entry, *tentry;
3654 	STAILQ_HEAD(, obj_list) on_cache = STAILQ_HEAD_INITIALIZER(on_cache);
3655 
3656 	spa = mnt->spa;
3657 	if (mnt->objset.os_type != DMU_OST_ZFS) {
3658 		printf("ZFS: unexpected object set type %ju\n",
3659 		    (uintmax_t)mnt->objset.os_type);
3660 		return (EIO);
3661 	}
3662 
3663 	if ((entry = malloc(sizeof (struct obj_list))) == NULL)
3664 		return (ENOMEM);
3665 
3666 	/*
3667 	 * Get the root directory dnode.
3668 	 */
3669 	rc = objset_get_dnode(spa, &mnt->objset, MASTER_NODE_OBJ, &dn);
3670 	if (rc) {
3671 		free(entry);
3672 		return (rc);
3673 	}
3674 
3675 	rc = zap_lookup(spa, &dn, ZFS_ROOT_OBJ, sizeof (objnum), 1, &objnum);
3676 	if (rc) {
3677 		free(entry);
3678 		return (rc);
3679 	}
3680 	entry->objnum = objnum;
3681 	STAILQ_INSERT_HEAD(&on_cache, entry, entry);
3682 
3683 	rc = objset_get_dnode(spa, &mnt->objset, objnum, &dn);
3684 	if (rc != 0)
3685 		goto done;
3686 
3687 	p = upath;
3688 	while (p && *p) {
3689 		rc = objset_get_dnode(spa, &mnt->objset, objnum, &dn);
3690 		if (rc != 0)
3691 			goto done;
3692 
3693 		while (*p == '/')
3694 			p++;
3695 		if (*p == '\0')
3696 			break;
3697 		q = p;
3698 		while (*q != '\0' && *q != '/')
3699 			q++;
3700 
3701 		/* skip dot */
3702 		if (p + 1 == q && p[0] == '.') {
3703 			p++;
3704 			continue;
3705 		}
3706 		/* double dot */
3707 		if (p + 2 == q && p[0] == '.' && p[1] == '.') {
3708 			p += 2;
3709 			if (STAILQ_FIRST(&on_cache) ==
3710 			    STAILQ_LAST(&on_cache, obj_list, entry)) {
3711 				rc = ENOENT;
3712 				goto done;
3713 			}
3714 			entry = STAILQ_FIRST(&on_cache);
3715 			STAILQ_REMOVE_HEAD(&on_cache, entry);
3716 			free(entry);
3717 			objnum = (STAILQ_FIRST(&on_cache))->objnum;
3718 			continue;
3719 		}
3720 		if (q - p + 1 > sizeof (element)) {
3721 			rc = ENAMETOOLONG;
3722 			goto done;
3723 		}
3724 		memcpy(element, p, q - p);
3725 		element[q - p] = 0;
3726 		p = q;
3727 
3728 		if ((rc = zfs_dnode_stat(spa, &dn, &sb)) != 0)
3729 			goto done;
3730 		if (!S_ISDIR(sb.st_mode)) {
3731 			rc = ENOTDIR;
3732 			goto done;
3733 		}
3734 
3735 		rc = zap_lookup(spa, &dn, element, sizeof (objnum), 1, &objnum);
3736 		if (rc)
3737 			goto done;
3738 		objnum = ZFS_DIRENT_OBJ(objnum);
3739 
3740 		if ((entry = malloc(sizeof (struct obj_list))) == NULL) {
3741 			rc = ENOMEM;
3742 			goto done;
3743 		}
3744 		entry->objnum = objnum;
3745 		STAILQ_INSERT_HEAD(&on_cache, entry, entry);
3746 		rc = objset_get_dnode(spa, &mnt->objset, objnum, &dn);
3747 		if (rc)
3748 			goto done;
3749 
3750 		/*
3751 		 * Check for symlink.
3752 		 */
3753 		rc = zfs_dnode_stat(spa, &dn, &sb);
3754 		if (rc)
3755 			goto done;
3756 		if (S_ISLNK(sb.st_mode)) {
3757 			if (symlinks_followed > 10) {
3758 				rc = EMLINK;
3759 				goto done;
3760 			}
3761 			symlinks_followed++;
3762 
3763 			/*
3764 			 * Read the link value and copy the tail of our
3765 			 * current path onto the end.
3766 			 */
3767 			if (sb.st_size + strlen(p) + 1 > sizeof (path)) {
3768 				rc = ENAMETOOLONG;
3769 				goto done;
3770 			}
3771 			strcpy(&path[sb.st_size], p);
3772 
3773 			rc = zfs_dnode_readlink(spa, &dn, path, sb.st_size);
3774 			if (rc != 0)
3775 				goto done;
3776 
3777 			/*
3778 			 * Restart with the new path, starting either at
3779 			 * the root or at the parent depending whether or
3780 			 * not the link is relative.
3781 			 */
3782 			p = path;
3783 			if (*p == '/') {
3784 				while (STAILQ_FIRST(&on_cache) !=
3785 				    STAILQ_LAST(&on_cache, obj_list, entry)) {
3786 					entry = STAILQ_FIRST(&on_cache);
3787 					STAILQ_REMOVE_HEAD(&on_cache, entry);
3788 					free(entry);
3789 				}
3790 			} else {
3791 				entry = STAILQ_FIRST(&on_cache);
3792 				STAILQ_REMOVE_HEAD(&on_cache, entry);
3793 				free(entry);
3794 			}
3795 			objnum = (STAILQ_FIRST(&on_cache))->objnum;
3796 		}
3797 	}
3798 
3799 	*dnode = dn;
3800 done:
3801 	STAILQ_FOREACH_SAFE(entry, &on_cache, entry, tentry)
3802 		free(entry);
3803 	return (rc);
3804 }
3805