xref: /freebsd/stand/libsa/zfs/zfsimpl.c (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 /*-
2  * Copyright (c) 2007 Doug Rabson
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 /*
31  *	Stand-alone ZFS file reader.
32  */
33 
34 #include <stdbool.h>
35 #include <sys/endian.h>
36 #include <sys/stat.h>
37 #include <sys/stdint.h>
38 #include <sys/list.h>
39 #include <sys/zfs_bootenv.h>
40 #include <machine/_inttypes.h>
41 
42 #include "zfsimpl.h"
43 #include "zfssubr.c"
44 
45 #ifdef HAS_ZSTD_ZFS
46 extern int zstd_init(void);
47 #endif
48 
49 struct zfsmount {
50 	const spa_t	*spa;
51 	objset_phys_t	objset;
52 	uint64_t	rootobj;
53 };
54 static struct zfsmount zfsmount __unused;
55 
56 /*
57  * The indirect_child_t represents the vdev that we will read from, when we
58  * need to read all copies of the data (e.g. for scrub or reconstruction).
59  * For plain (non-mirror) top-level vdevs (i.e. is_vdev is not a mirror),
60  * ic_vdev is the same as is_vdev.  However, for mirror top-level vdevs,
61  * ic_vdev is a child of the mirror.
62  */
63 typedef struct indirect_child {
64 	void *ic_data;
65 	vdev_t *ic_vdev;
66 } indirect_child_t;
67 
68 /*
69  * The indirect_split_t represents one mapped segment of an i/o to the
70  * indirect vdev. For non-split (contiguously-mapped) blocks, there will be
71  * only one indirect_split_t, with is_split_offset==0 and is_size==io_size.
72  * For split blocks, there will be several of these.
73  */
74 typedef struct indirect_split {
75 	list_node_t is_node; /* link on iv_splits */
76 
77 	/*
78 	 * is_split_offset is the offset into the i/o.
79 	 * This is the sum of the previous splits' is_size's.
80 	 */
81 	uint64_t is_split_offset;
82 
83 	vdev_t *is_vdev; /* top-level vdev */
84 	uint64_t is_target_offset; /* offset on is_vdev */
85 	uint64_t is_size;
86 	int is_children; /* number of entries in is_child[] */
87 
88 	/*
89 	 * is_good_child is the child that we are currently using to
90 	 * attempt reconstruction.
91 	 */
92 	int is_good_child;
93 
94 	indirect_child_t is_child[1]; /* variable-length */
95 } indirect_split_t;
96 
97 /*
98  * The indirect_vsd_t is associated with each i/o to the indirect vdev.
99  * It is the "Vdev-Specific Data" in the zio_t's io_vsd.
100  */
101 typedef struct indirect_vsd {
102 	boolean_t iv_split_block;
103 	boolean_t iv_reconstruct;
104 
105 	list_t iv_splits; /* list of indirect_split_t's */
106 } indirect_vsd_t;
107 
108 /*
109  * List of all vdevs, chained through v_alllink.
110  */
111 static vdev_list_t zfs_vdevs;
112 
113 /*
114  * List of ZFS features supported for read
115  */
116 static const char *features_for_read[] = {
117 	"org.illumos:lz4_compress",
118 	"com.delphix:hole_birth",
119 	"com.delphix:extensible_dataset",
120 	"com.delphix:embedded_data",
121 	"org.open-zfs:large_blocks",
122 	"org.illumos:sha512",
123 	"org.illumos:skein",
124 	"org.zfsonlinux:large_dnode",
125 	"com.joyent:multi_vdev_crash_dump",
126 	"com.delphix:spacemap_histogram",
127 	"com.delphix:zpool_checkpoint",
128 	"com.delphix:spacemap_v2",
129 	"com.datto:encryption",
130 	"com.datto:bookmark_v2",
131 	"org.zfsonlinux:allocation_classes",
132 	"com.datto:resilver_defer",
133 	"com.delphix:device_removal",
134 	"com.delphix:obsolete_counts",
135 	"com.intel:allocation_classes",
136 	"org.freebsd:zstd_compress",
137 	"com.delphix:bookmark_written",
138 	NULL
139 };
140 
141 /*
142  * List of all pools, chained through spa_link.
143  */
144 static spa_list_t zfs_pools;
145 
146 static const dnode_phys_t *dnode_cache_obj;
147 static uint64_t dnode_cache_bn;
148 static char *dnode_cache_buf;
149 
150 static int zio_read(const spa_t *spa, const blkptr_t *bp, void *buf);
151 static int zfs_get_root(const spa_t *spa, uint64_t *objid);
152 static int zfs_rlookup(const spa_t *spa, uint64_t objnum, char *result);
153 static int zap_lookup(const spa_t *spa, const dnode_phys_t *dnode,
154     const char *name, uint64_t integer_size, uint64_t num_integers,
155     void *value);
156 static int objset_get_dnode(const spa_t *, const objset_phys_t *, uint64_t,
157     dnode_phys_t *);
158 static int dnode_read(const spa_t *, const dnode_phys_t *, off_t, void *,
159     size_t);
160 static int vdev_indirect_read(vdev_t *, const blkptr_t *, void *, off_t,
161     size_t);
162 static int vdev_mirror_read(vdev_t *, const blkptr_t *, void *, off_t, size_t);
163 vdev_indirect_mapping_t *vdev_indirect_mapping_open(spa_t *, objset_phys_t *,
164     uint64_t);
165 vdev_indirect_mapping_entry_phys_t *
166     vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t *, uint64_t,
167     uint64_t, uint64_t *);
168 
169 static void
170 zfs_init(void)
171 {
172 	STAILQ_INIT(&zfs_vdevs);
173 	STAILQ_INIT(&zfs_pools);
174 
175 	dnode_cache_buf = malloc(SPA_MAXBLOCKSIZE);
176 
177 	zfs_init_crc();
178 #ifdef HAS_ZSTD_ZFS
179 	zstd_init();
180 #endif
181 }
182 
183 static int
184 nvlist_check_features_for_read(nvlist_t *nvl)
185 {
186 	nvlist_t *features = NULL;
187 	nvs_data_t *data;
188 	nvp_header_t *nvp;
189 	nv_string_t *nvp_name;
190 	int rc;
191 
192 	rc = nvlist_find(nvl, ZPOOL_CONFIG_FEATURES_FOR_READ,
193 	    DATA_TYPE_NVLIST, NULL, &features, NULL);
194 	if (rc != 0)
195 		return (rc);
196 
197 	data = (nvs_data_t *)features->nv_data;
198 	nvp = &data->nvl_pair;	/* first pair in nvlist */
199 
200 	while (nvp->encoded_size != 0 && nvp->decoded_size != 0) {
201 		int i, found;
202 
203 		nvp_name = (nv_string_t *)((uintptr_t)nvp + sizeof(*nvp));
204 		found = 0;
205 
206 		for (i = 0; features_for_read[i] != NULL; i++) {
207 			if (memcmp(nvp_name->nv_data, features_for_read[i],
208 			    nvp_name->nv_size) == 0) {
209 				found = 1;
210 				break;
211 			}
212 		}
213 
214 		if (!found) {
215 			printf("ZFS: unsupported feature: %.*s\n",
216 			    nvp_name->nv_size, nvp_name->nv_data);
217 			rc = EIO;
218 		}
219 		nvp = (nvp_header_t *)((uint8_t *)nvp + nvp->encoded_size);
220 	}
221 	nvlist_destroy(features);
222 
223 	return (rc);
224 }
225 
226 static int
227 vdev_read_phys(vdev_t *vdev, const blkptr_t *bp, void *buf,
228     off_t offset, size_t size)
229 {
230 	size_t psize;
231 	int rc;
232 
233 	if (vdev->v_phys_read == NULL)
234 		return (ENOTSUP);
235 
236 	if (bp) {
237 		psize = BP_GET_PSIZE(bp);
238 	} else {
239 		psize = size;
240 	}
241 
242 	rc = vdev->v_phys_read(vdev, vdev->v_priv, offset, buf, psize);
243 	if (rc == 0) {
244 		if (bp != NULL)
245 			rc = zio_checksum_verify(vdev->v_spa, bp, buf);
246 	}
247 
248 	return (rc);
249 }
250 
251 static int
252 vdev_write_phys(vdev_t *vdev, void *buf, off_t offset, size_t size)
253 {
254 	if (vdev->v_phys_write == NULL)
255 		return (ENOTSUP);
256 
257 	return (vdev->v_phys_write(vdev, offset, buf, size));
258 }
259 
260 typedef struct remap_segment {
261 	vdev_t *rs_vd;
262 	uint64_t rs_offset;
263 	uint64_t rs_asize;
264 	uint64_t rs_split_offset;
265 	list_node_t rs_node;
266 } remap_segment_t;
267 
268 static remap_segment_t *
269 rs_alloc(vdev_t *vd, uint64_t offset, uint64_t asize, uint64_t split_offset)
270 {
271 	remap_segment_t *rs = malloc(sizeof (remap_segment_t));
272 
273 	if (rs != NULL) {
274 		rs->rs_vd = vd;
275 		rs->rs_offset = offset;
276 		rs->rs_asize = asize;
277 		rs->rs_split_offset = split_offset;
278 	}
279 
280 	return (rs);
281 }
282 
283 vdev_indirect_mapping_t *
284 vdev_indirect_mapping_open(spa_t *spa, objset_phys_t *os,
285     uint64_t mapping_object)
286 {
287 	vdev_indirect_mapping_t *vim;
288 	vdev_indirect_mapping_phys_t *vim_phys;
289 	int rc;
290 
291 	vim = calloc(1, sizeof (*vim));
292 	if (vim == NULL)
293 		return (NULL);
294 
295 	vim->vim_dn = calloc(1, sizeof (*vim->vim_dn));
296 	if (vim->vim_dn == NULL) {
297 		free(vim);
298 		return (NULL);
299 	}
300 
301 	rc = objset_get_dnode(spa, os, mapping_object, vim->vim_dn);
302 	if (rc != 0) {
303 		free(vim->vim_dn);
304 		free(vim);
305 		return (NULL);
306 	}
307 
308 	vim->vim_spa = spa;
309 	vim->vim_phys = malloc(sizeof (*vim->vim_phys));
310 	if (vim->vim_phys == NULL) {
311 		free(vim->vim_dn);
312 		free(vim);
313 		return (NULL);
314 	}
315 
316 	vim_phys = (vdev_indirect_mapping_phys_t *)DN_BONUS(vim->vim_dn);
317 	*vim->vim_phys = *vim_phys;
318 
319 	vim->vim_objset = os;
320 	vim->vim_object = mapping_object;
321 	vim->vim_entries = NULL;
322 
323 	vim->vim_havecounts =
324 	    (vim->vim_dn->dn_bonuslen > VDEV_INDIRECT_MAPPING_SIZE_V0);
325 
326 	return (vim);
327 }
328 
329 /*
330  * Compare an offset with an indirect mapping entry; there are three
331  * possible scenarios:
332  *
333  *     1. The offset is "less than" the mapping entry; meaning the
334  *        offset is less than the source offset of the mapping entry. In
335  *        this case, there is no overlap between the offset and the
336  *        mapping entry and -1 will be returned.
337  *
338  *     2. The offset is "greater than" the mapping entry; meaning the
339  *        offset is greater than the mapping entry's source offset plus
340  *        the entry's size. In this case, there is no overlap between
341  *        the offset and the mapping entry and 1 will be returned.
342  *
343  *        NOTE: If the offset is actually equal to the entry's offset
344  *        plus size, this is considered to be "greater" than the entry,
345  *        and this case applies (i.e. 1 will be returned). Thus, the
346  *        entry's "range" can be considered to be inclusive at its
347  *        start, but exclusive at its end: e.g. [src, src + size).
348  *
349  *     3. The last case to consider is if the offset actually falls
350  *        within the mapping entry's range. If this is the case, the
351  *        offset is considered to be "equal to" the mapping entry and
352  *        0 will be returned.
353  *
354  *        NOTE: If the offset is equal to the entry's source offset,
355  *        this case applies and 0 will be returned. If the offset is
356  *        equal to the entry's source plus its size, this case does
357  *        *not* apply (see "NOTE" above for scenario 2), and 1 will be
358  *        returned.
359  */
360 static int
361 dva_mapping_overlap_compare(const void *v_key, const void *v_array_elem)
362 {
363 	const uint64_t *key = v_key;
364 	const vdev_indirect_mapping_entry_phys_t *array_elem =
365 	    v_array_elem;
366 	uint64_t src_offset = DVA_MAPPING_GET_SRC_OFFSET(array_elem);
367 
368 	if (*key < src_offset) {
369 		return (-1);
370 	} else if (*key < src_offset + DVA_GET_ASIZE(&array_elem->vimep_dst)) {
371 		return (0);
372 	} else {
373 		return (1);
374 	}
375 }
376 
377 /*
378  * Return array entry.
379  */
380 static vdev_indirect_mapping_entry_phys_t *
381 vdev_indirect_mapping_entry(vdev_indirect_mapping_t *vim, uint64_t index)
382 {
383 	uint64_t size;
384 	off_t offset = 0;
385 	int rc;
386 
387 	if (vim->vim_phys->vimp_num_entries == 0)
388 		return (NULL);
389 
390 	if (vim->vim_entries == NULL) {
391 		uint64_t bsize;
392 
393 		bsize = vim->vim_dn->dn_datablkszsec << SPA_MINBLOCKSHIFT;
394 		size = vim->vim_phys->vimp_num_entries *
395 		    sizeof (*vim->vim_entries);
396 		if (size > bsize) {
397 			size = bsize / sizeof (*vim->vim_entries);
398 			size *= sizeof (*vim->vim_entries);
399 		}
400 		vim->vim_entries = malloc(size);
401 		if (vim->vim_entries == NULL)
402 			return (NULL);
403 		vim->vim_num_entries = size / sizeof (*vim->vim_entries);
404 		offset = index * sizeof (*vim->vim_entries);
405 	}
406 
407 	/* We have data in vim_entries */
408 	if (offset == 0) {
409 		if (index >= vim->vim_entry_offset &&
410 		    index <= vim->vim_entry_offset + vim->vim_num_entries) {
411 			index -= vim->vim_entry_offset;
412 			return (&vim->vim_entries[index]);
413 		}
414 		offset = index * sizeof (*vim->vim_entries);
415 	}
416 
417 	vim->vim_entry_offset = index;
418 	size = vim->vim_num_entries * sizeof (*vim->vim_entries);
419 	rc = dnode_read(vim->vim_spa, vim->vim_dn, offset, vim->vim_entries,
420 	    size);
421 	if (rc != 0) {
422 		/* Read error, invalidate vim_entries. */
423 		free(vim->vim_entries);
424 		vim->vim_entries = NULL;
425 		return (NULL);
426 	}
427 	index -= vim->vim_entry_offset;
428 	return (&vim->vim_entries[index]);
429 }
430 
431 /*
432  * Returns the mapping entry for the given offset.
433  *
434  * It's possible that the given offset will not be in the mapping table
435  * (i.e. no mapping entries contain this offset), in which case, the
436  * return value value depends on the "next_if_missing" parameter.
437  *
438  * If the offset is not found in the table and "next_if_missing" is
439  * B_FALSE, then NULL will always be returned. The behavior is intended
440  * to allow consumers to get the entry corresponding to the offset
441  * parameter, iff the offset overlaps with an entry in the table.
442  *
443  * If the offset is not found in the table and "next_if_missing" is
444  * B_TRUE, then the entry nearest to the given offset will be returned,
445  * such that the entry's source offset is greater than the offset
446  * passed in (i.e. the "next" mapping entry in the table is returned, if
447  * the offset is missing from the table). If there are no entries whose
448  * source offset is greater than the passed in offset, NULL is returned.
449  */
450 static vdev_indirect_mapping_entry_phys_t *
451 vdev_indirect_mapping_entry_for_offset(vdev_indirect_mapping_t *vim,
452     uint64_t offset)
453 {
454 	ASSERT(vim->vim_phys->vimp_num_entries > 0);
455 
456 	vdev_indirect_mapping_entry_phys_t *entry;
457 
458 	uint64_t last = vim->vim_phys->vimp_num_entries - 1;
459 	uint64_t base = 0;
460 
461 	/*
462 	 * We don't define these inside of the while loop because we use
463 	 * their value in the case that offset isn't in the mapping.
464 	 */
465 	uint64_t mid;
466 	int result;
467 
468 	while (last >= base) {
469 		mid = base + ((last - base) >> 1);
470 
471 		entry = vdev_indirect_mapping_entry(vim, mid);
472 		if (entry == NULL)
473 			break;
474 		result = dva_mapping_overlap_compare(&offset, entry);
475 
476 		if (result == 0) {
477 			break;
478 		} else if (result < 0) {
479 			last = mid - 1;
480 		} else {
481 			base = mid + 1;
482 		}
483 	}
484 	return (entry);
485 }
486 
487 /*
488  * Given an indirect vdev and an extent on that vdev, it duplicates the
489  * physical entries of the indirect mapping that correspond to the extent
490  * to a new array and returns a pointer to it. In addition, copied_entries
491  * is populated with the number of mapping entries that were duplicated.
492  *
493  * Finally, since we are doing an allocation, it is up to the caller to
494  * free the array allocated in this function.
495  */
496 vdev_indirect_mapping_entry_phys_t *
497 vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t *vd, uint64_t offset,
498     uint64_t asize, uint64_t *copied_entries)
499 {
500 	vdev_indirect_mapping_entry_phys_t *duplicate_mappings = NULL;
501 	vdev_indirect_mapping_t *vim = vd->v_mapping;
502 	uint64_t entries = 0;
503 
504 	vdev_indirect_mapping_entry_phys_t *first_mapping =
505 	    vdev_indirect_mapping_entry_for_offset(vim, offset);
506 	ASSERT3P(first_mapping, !=, NULL);
507 
508 	vdev_indirect_mapping_entry_phys_t *m = first_mapping;
509 	while (asize > 0) {
510 		uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
511 		uint64_t inner_offset = offset - DVA_MAPPING_GET_SRC_OFFSET(m);
512 		uint64_t inner_size = MIN(asize, size - inner_offset);
513 
514 		offset += inner_size;
515 		asize -= inner_size;
516 		entries++;
517 		m++;
518 	}
519 
520 	size_t copy_length = entries * sizeof (*first_mapping);
521 	duplicate_mappings = malloc(copy_length);
522 	if (duplicate_mappings != NULL)
523 		bcopy(first_mapping, duplicate_mappings, copy_length);
524 	else
525 		entries = 0;
526 
527 	*copied_entries = entries;
528 
529 	return (duplicate_mappings);
530 }
531 
532 static vdev_t *
533 vdev_lookup_top(spa_t *spa, uint64_t vdev)
534 {
535 	vdev_t *rvd;
536 	vdev_list_t *vlist;
537 
538 	vlist = &spa->spa_root_vdev->v_children;
539 	STAILQ_FOREACH(rvd, vlist, v_childlink)
540 		if (rvd->v_id == vdev)
541 			break;
542 
543 	return (rvd);
544 }
545 
546 /*
547  * This is a callback for vdev_indirect_remap() which allocates an
548  * indirect_split_t for each split segment and adds it to iv_splits.
549  */
550 static void
551 vdev_indirect_gather_splits(uint64_t split_offset, vdev_t *vd, uint64_t offset,
552     uint64_t size, void *arg)
553 {
554 	int n = 1;
555 	zio_t *zio = arg;
556 	indirect_vsd_t *iv = zio->io_vsd;
557 
558 	if (vd->v_read == vdev_indirect_read)
559 		return;
560 
561 	if (vd->v_read == vdev_mirror_read)
562 		n = vd->v_nchildren;
563 
564 	indirect_split_t *is =
565 	    malloc(offsetof(indirect_split_t, is_child[n]));
566 	if (is == NULL) {
567 		zio->io_error = ENOMEM;
568 		return;
569 	}
570 	bzero(is, offsetof(indirect_split_t, is_child[n]));
571 
572 	is->is_children = n;
573 	is->is_size = size;
574 	is->is_split_offset = split_offset;
575 	is->is_target_offset = offset;
576 	is->is_vdev = vd;
577 
578 	/*
579 	 * Note that we only consider multiple copies of the data for
580 	 * *mirror* vdevs.  We don't for "replacing" or "spare" vdevs, even
581 	 * though they use the same ops as mirror, because there's only one
582 	 * "good" copy under the replacing/spare.
583 	 */
584 	if (vd->v_read == vdev_mirror_read) {
585 		int i = 0;
586 		vdev_t *kid;
587 
588 		STAILQ_FOREACH(kid, &vd->v_children, v_childlink) {
589 			is->is_child[i++].ic_vdev = kid;
590 		}
591 	} else {
592 		is->is_child[0].ic_vdev = vd;
593 	}
594 
595 	list_insert_tail(&iv->iv_splits, is);
596 }
597 
598 static void
599 vdev_indirect_remap(vdev_t *vd, uint64_t offset, uint64_t asize, void *arg)
600 {
601 	list_t stack;
602 	spa_t *spa = vd->v_spa;
603 	zio_t *zio = arg;
604 	remap_segment_t *rs;
605 
606 	list_create(&stack, sizeof (remap_segment_t),
607 	    offsetof(remap_segment_t, rs_node));
608 
609 	rs = rs_alloc(vd, offset, asize, 0);
610 	if (rs == NULL) {
611 		printf("vdev_indirect_remap: out of memory.\n");
612 		zio->io_error = ENOMEM;
613 	}
614 	for (; rs != NULL; rs = list_remove_head(&stack)) {
615 		vdev_t *v = rs->rs_vd;
616 		uint64_t num_entries = 0;
617 		/* vdev_indirect_mapping_t *vim = v->v_mapping; */
618 		vdev_indirect_mapping_entry_phys_t *mapping =
619 		    vdev_indirect_mapping_duplicate_adjacent_entries(v,
620 		    rs->rs_offset, rs->rs_asize, &num_entries);
621 
622 		if (num_entries == 0)
623 			zio->io_error = ENOMEM;
624 
625 		for (uint64_t i = 0; i < num_entries; i++) {
626 			vdev_indirect_mapping_entry_phys_t *m = &mapping[i];
627 			uint64_t size = DVA_GET_ASIZE(&m->vimep_dst);
628 			uint64_t dst_offset = DVA_GET_OFFSET(&m->vimep_dst);
629 			uint64_t dst_vdev = DVA_GET_VDEV(&m->vimep_dst);
630 			uint64_t inner_offset = rs->rs_offset -
631 			    DVA_MAPPING_GET_SRC_OFFSET(m);
632 			uint64_t inner_size =
633 			    MIN(rs->rs_asize, size - inner_offset);
634 			vdev_t *dst_v = vdev_lookup_top(spa, dst_vdev);
635 
636 			if (dst_v->v_read == vdev_indirect_read) {
637 				remap_segment_t *o;
638 
639 				o = rs_alloc(dst_v, dst_offset + inner_offset,
640 				    inner_size, rs->rs_split_offset);
641 				if (o == NULL) {
642 					printf("vdev_indirect_remap: "
643 					    "out of memory.\n");
644 					zio->io_error = ENOMEM;
645 					break;
646 				}
647 
648 				list_insert_head(&stack, o);
649 			}
650 			vdev_indirect_gather_splits(rs->rs_split_offset, dst_v,
651 			    dst_offset + inner_offset,
652 			    inner_size, arg);
653 
654 			/*
655 			 * vdev_indirect_gather_splits can have memory
656 			 * allocation error, we can not recover from it.
657 			 */
658 			if (zio->io_error != 0)
659 				break;
660 			rs->rs_offset += inner_size;
661 			rs->rs_asize -= inner_size;
662 			rs->rs_split_offset += inner_size;
663 		}
664 
665 		free(mapping);
666 		free(rs);
667 		if (zio->io_error != 0)
668 			break;
669 	}
670 
671 	list_destroy(&stack);
672 }
673 
674 static void
675 vdev_indirect_map_free(zio_t *zio)
676 {
677 	indirect_vsd_t *iv = zio->io_vsd;
678 	indirect_split_t *is;
679 
680 	while ((is = list_head(&iv->iv_splits)) != NULL) {
681 		for (int c = 0; c < is->is_children; c++) {
682 			indirect_child_t *ic = &is->is_child[c];
683 			free(ic->ic_data);
684 		}
685 		list_remove(&iv->iv_splits, is);
686 		free(is);
687 	}
688 	free(iv);
689 }
690 
691 static int
692 vdev_indirect_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
693     off_t offset, size_t bytes)
694 {
695 	zio_t zio;
696 	spa_t *spa = vdev->v_spa;
697 	indirect_vsd_t *iv;
698 	indirect_split_t *first;
699 	int rc = EIO;
700 
701 	iv = calloc(1, sizeof(*iv));
702 	if (iv == NULL)
703 		return (ENOMEM);
704 
705 	list_create(&iv->iv_splits,
706 	    sizeof (indirect_split_t), offsetof(indirect_split_t, is_node));
707 
708 	bzero(&zio, sizeof(zio));
709 	zio.io_spa = spa;
710 	zio.io_bp = (blkptr_t *)bp;
711 	zio.io_data = buf;
712 	zio.io_size = bytes;
713 	zio.io_offset = offset;
714 	zio.io_vd = vdev;
715 	zio.io_vsd = iv;
716 
717 	if (vdev->v_mapping == NULL) {
718 		vdev_indirect_config_t *vic;
719 
720 		vic = &vdev->vdev_indirect_config;
721 		vdev->v_mapping = vdev_indirect_mapping_open(spa,
722 		    spa->spa_mos, vic->vic_mapping_object);
723 	}
724 
725 	vdev_indirect_remap(vdev, offset, bytes, &zio);
726 	if (zio.io_error != 0)
727 		return (zio.io_error);
728 
729 	first = list_head(&iv->iv_splits);
730 	if (first->is_size == zio.io_size) {
731 		/*
732 		 * This is not a split block; we are pointing to the entire
733 		 * data, which will checksum the same as the original data.
734 		 * Pass the BP down so that the child i/o can verify the
735 		 * checksum, and try a different location if available
736 		 * (e.g. on a mirror).
737 		 *
738 		 * While this special case could be handled the same as the
739 		 * general (split block) case, doing it this way ensures
740 		 * that the vast majority of blocks on indirect vdevs
741 		 * (which are not split) are handled identically to blocks
742 		 * on non-indirect vdevs.  This allows us to be less strict
743 		 * about performance in the general (but rare) case.
744 		 */
745 		rc = first->is_vdev->v_read(first->is_vdev, zio.io_bp,
746 		    zio.io_data, first->is_target_offset, bytes);
747 	} else {
748 		iv->iv_split_block = B_TRUE;
749 		/*
750 		 * Read one copy of each split segment, from the
751 		 * top-level vdev.  Since we don't know the
752 		 * checksum of each split individually, the child
753 		 * zio can't ensure that we get the right data.
754 		 * E.g. if it's a mirror, it will just read from a
755 		 * random (healthy) leaf vdev.  We have to verify
756 		 * the checksum in vdev_indirect_io_done().
757 		 */
758 		for (indirect_split_t *is = list_head(&iv->iv_splits);
759 		    is != NULL; is = list_next(&iv->iv_splits, is)) {
760 			char *ptr = zio.io_data;
761 
762 			rc = is->is_vdev->v_read(is->is_vdev, zio.io_bp,
763 			    ptr + is->is_split_offset, is->is_target_offset,
764 			    is->is_size);
765 		}
766 		if (zio_checksum_verify(spa, zio.io_bp, zio.io_data))
767 			rc = ECKSUM;
768 		else
769 			rc = 0;
770 	}
771 
772 	vdev_indirect_map_free(&zio);
773 	if (rc == 0)
774 		rc = zio.io_error;
775 
776 	return (rc);
777 }
778 
779 static int
780 vdev_disk_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
781     off_t offset, size_t bytes)
782 {
783 
784 	return (vdev_read_phys(vdev, bp, buf,
785 	    offset + VDEV_LABEL_START_SIZE, bytes));
786 }
787 
788 static int
789 vdev_missing_read(vdev_t *vdev __unused, const blkptr_t *bp __unused,
790     void *buf __unused, off_t offset __unused, size_t bytes __unused)
791 {
792 
793 	return (ENOTSUP);
794 }
795 
796 static int
797 vdev_mirror_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
798     off_t offset, size_t bytes)
799 {
800 	vdev_t *kid;
801 	int rc;
802 
803 	rc = EIO;
804 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
805 		if (kid->v_state != VDEV_STATE_HEALTHY)
806 			continue;
807 		rc = kid->v_read(kid, bp, buf, offset, bytes);
808 		if (!rc)
809 			return (0);
810 	}
811 
812 	return (rc);
813 }
814 
815 static int
816 vdev_replacing_read(vdev_t *vdev, const blkptr_t *bp, void *buf,
817     off_t offset, size_t bytes)
818 {
819 	vdev_t *kid;
820 
821 	/*
822 	 * Here we should have two kids:
823 	 * First one which is the one we are replacing and we can trust
824 	 * only this one to have valid data, but it might not be present.
825 	 * Second one is that one we are replacing with. It is most likely
826 	 * healthy, but we can't trust it has needed data, so we won't use it.
827 	 */
828 	kid = STAILQ_FIRST(&vdev->v_children);
829 	if (kid == NULL)
830 		return (EIO);
831 	if (kid->v_state != VDEV_STATE_HEALTHY)
832 		return (EIO);
833 	return (kid->v_read(kid, bp, buf, offset, bytes));
834 }
835 
836 static vdev_t *
837 vdev_find(uint64_t guid)
838 {
839 	vdev_t *vdev;
840 
841 	STAILQ_FOREACH(vdev, &zfs_vdevs, v_alllink)
842 		if (vdev->v_guid == guid)
843 			return (vdev);
844 
845 	return (0);
846 }
847 
848 static vdev_t *
849 vdev_create(uint64_t guid, vdev_read_t *_read)
850 {
851 	vdev_t *vdev;
852 	vdev_indirect_config_t *vic;
853 
854 	vdev = calloc(1, sizeof(vdev_t));
855 	if (vdev != NULL) {
856 		STAILQ_INIT(&vdev->v_children);
857 		vdev->v_guid = guid;
858 		vdev->v_read = _read;
859 
860 		/*
861 		 * root vdev has no read function, we use this fact to
862 		 * skip setting up data we do not need for root vdev.
863 		 * We only point root vdev from spa.
864 		 */
865 		if (_read != NULL) {
866 			vic = &vdev->vdev_indirect_config;
867 			vic->vic_prev_indirect_vdev = UINT64_MAX;
868 			STAILQ_INSERT_TAIL(&zfs_vdevs, vdev, v_alllink);
869 		}
870 	}
871 
872 	return (vdev);
873 }
874 
875 static void
876 vdev_set_initial_state(vdev_t *vdev, const nvlist_t *nvlist)
877 {
878 	uint64_t is_offline, is_faulted, is_degraded, is_removed, isnt_present;
879 	uint64_t is_log;
880 
881 	is_offline = is_removed = is_faulted = is_degraded = isnt_present = 0;
882 	is_log = 0;
883 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_OFFLINE, DATA_TYPE_UINT64, NULL,
884 	    &is_offline, NULL);
885 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_REMOVED, DATA_TYPE_UINT64, NULL,
886 	    &is_removed, NULL);
887 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_FAULTED, DATA_TYPE_UINT64, NULL,
888 	    &is_faulted, NULL);
889 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_DEGRADED, DATA_TYPE_UINT64,
890 	    NULL, &is_degraded, NULL);
891 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_NOT_PRESENT, DATA_TYPE_UINT64,
892 	    NULL, &isnt_present, NULL);
893 	(void) nvlist_find(nvlist, ZPOOL_CONFIG_IS_LOG, DATA_TYPE_UINT64, NULL,
894 	    &is_log, NULL);
895 
896 	if (is_offline != 0)
897 		vdev->v_state = VDEV_STATE_OFFLINE;
898 	else if (is_removed != 0)
899 		vdev->v_state = VDEV_STATE_REMOVED;
900 	else if (is_faulted != 0)
901 		vdev->v_state = VDEV_STATE_FAULTED;
902 	else if (is_degraded != 0)
903 		vdev->v_state = VDEV_STATE_DEGRADED;
904 	else if (isnt_present != 0)
905 		vdev->v_state = VDEV_STATE_CANT_OPEN;
906 
907 	vdev->v_islog = is_log != 0;
908 }
909 
910 static int
911 vdev_init(uint64_t guid, const nvlist_t *nvlist, vdev_t **vdevp)
912 {
913 	uint64_t id, ashift, asize, nparity;
914 	const char *path;
915 	const char *type;
916 	int len, pathlen;
917 	char *name;
918 	vdev_t *vdev;
919 
920 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ID, DATA_TYPE_UINT64, NULL, &id,
921 	    NULL) ||
922 	    nvlist_find(nvlist, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING, NULL,
923 	    &type, &len)) {
924 		return (ENOENT);
925 	}
926 
927 	if (memcmp(type, VDEV_TYPE_MIRROR, len) != 0 &&
928 	    memcmp(type, VDEV_TYPE_DISK, len) != 0 &&
929 #ifdef ZFS_TEST
930 	    memcmp(type, VDEV_TYPE_FILE, len) != 0 &&
931 #endif
932 	    memcmp(type, VDEV_TYPE_RAIDZ, len) != 0 &&
933 	    memcmp(type, VDEV_TYPE_INDIRECT, len) != 0 &&
934 	    memcmp(type, VDEV_TYPE_REPLACING, len) != 0 &&
935 	    memcmp(type, VDEV_TYPE_HOLE, len) != 0) {
936 		printf("ZFS: can only boot from disk, mirror, raidz1, "
937 		    "raidz2 and raidz3 vdevs, got: %.*s\n", len, type);
938 		return (EIO);
939 	}
940 
941 	if (memcmp(type, VDEV_TYPE_MIRROR, len) == 0)
942 		vdev = vdev_create(guid, vdev_mirror_read);
943 	else if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0)
944 		vdev = vdev_create(guid, vdev_raidz_read);
945 	else if (memcmp(type, VDEV_TYPE_REPLACING, len) == 0)
946 		vdev = vdev_create(guid, vdev_replacing_read);
947 	else if (memcmp(type, VDEV_TYPE_INDIRECT, len) == 0) {
948 		vdev_indirect_config_t *vic;
949 
950 		vdev = vdev_create(guid, vdev_indirect_read);
951 		if (vdev != NULL) {
952 			vdev->v_state = VDEV_STATE_HEALTHY;
953 			vic = &vdev->vdev_indirect_config;
954 
955 			nvlist_find(nvlist,
956 			    ZPOOL_CONFIG_INDIRECT_OBJECT,
957 			    DATA_TYPE_UINT64,
958 			    NULL, &vic->vic_mapping_object, NULL);
959 			nvlist_find(nvlist,
960 			    ZPOOL_CONFIG_INDIRECT_BIRTHS,
961 			    DATA_TYPE_UINT64,
962 			    NULL, &vic->vic_births_object, NULL);
963 			nvlist_find(nvlist,
964 			    ZPOOL_CONFIG_PREV_INDIRECT_VDEV,
965 			    DATA_TYPE_UINT64,
966 			    NULL, &vic->vic_prev_indirect_vdev, NULL);
967 		}
968 	} else if (memcmp(type, VDEV_TYPE_HOLE, len) == 0) {
969 		vdev = vdev_create(guid, vdev_missing_read);
970 	} else {
971 		vdev = vdev_create(guid, vdev_disk_read);
972 	}
973 
974 	if (vdev == NULL)
975 		return (ENOMEM);
976 
977 	vdev_set_initial_state(vdev, nvlist);
978 	vdev->v_id = id;
979 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASHIFT,
980 	    DATA_TYPE_UINT64, NULL, &ashift, NULL) == 0)
981 		vdev->v_ashift = ashift;
982 
983 	if (nvlist_find(nvlist, ZPOOL_CONFIG_ASIZE,
984 	    DATA_TYPE_UINT64, NULL, &asize, NULL) == 0) {
985 		vdev->v_psize = asize +
986 		    VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
987 	}
988 
989 	if (nvlist_find(nvlist, ZPOOL_CONFIG_NPARITY,
990 	    DATA_TYPE_UINT64, NULL, &nparity, NULL) == 0)
991 		vdev->v_nparity = nparity;
992 
993 	if (nvlist_find(nvlist, ZPOOL_CONFIG_PATH,
994 	    DATA_TYPE_STRING, NULL, &path, &pathlen) == 0) {
995 		char prefix[] = "/dev/";
996 
997 		len = strlen(prefix);
998 		if (len < pathlen && memcmp(path, prefix, len) == 0) {
999 			path += len;
1000 			pathlen -= len;
1001 		}
1002 		name = malloc(pathlen + 1);
1003 		bcopy(path, name, pathlen);
1004 		name[pathlen] = '\0';
1005 		vdev->v_name = name;
1006 	} else {
1007 		name = NULL;
1008 		if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0) {
1009 			if (vdev->v_nparity < 1 ||
1010 			    vdev->v_nparity > 3) {
1011 				printf("ZFS: invalid raidz parity: %d\n",
1012 				    vdev->v_nparity);
1013 				return (EIO);
1014 			}
1015 			(void) asprintf(&name, "%.*s%d-%" PRIu64, len, type,
1016 			    vdev->v_nparity, id);
1017 		} else {
1018 			(void) asprintf(&name, "%.*s-%" PRIu64, len, type, id);
1019 		}
1020 		vdev->v_name = name;
1021 	}
1022 	*vdevp = vdev;
1023 	return (0);
1024 }
1025 
1026 /*
1027  * Find slot for vdev. We return either NULL to signal to use
1028  * STAILQ_INSERT_HEAD, or we return link element to be used with
1029  * STAILQ_INSERT_AFTER.
1030  */
1031 static vdev_t *
1032 vdev_find_previous(vdev_t *top_vdev, vdev_t *vdev)
1033 {
1034 	vdev_t *v, *previous;
1035 
1036 	if (STAILQ_EMPTY(&top_vdev->v_children))
1037 		return (NULL);
1038 
1039 	previous = NULL;
1040 	STAILQ_FOREACH(v, &top_vdev->v_children, v_childlink) {
1041 		if (v->v_id > vdev->v_id)
1042 			return (previous);
1043 
1044 		if (v->v_id == vdev->v_id)
1045 			return (v);
1046 
1047 		if (v->v_id < vdev->v_id)
1048 			previous = v;
1049 	}
1050 	return (previous);
1051 }
1052 
1053 static size_t
1054 vdev_child_count(vdev_t *vdev)
1055 {
1056 	vdev_t *v;
1057 	size_t count;
1058 
1059 	count = 0;
1060 	STAILQ_FOREACH(v, &vdev->v_children, v_childlink) {
1061 		count++;
1062 	}
1063 	return (count);
1064 }
1065 
1066 /*
1067  * Insert vdev into top_vdev children list. List is ordered by v_id.
1068  */
1069 static void
1070 vdev_insert(vdev_t *top_vdev, vdev_t *vdev)
1071 {
1072 	vdev_t *previous;
1073 	size_t count;
1074 
1075 	/*
1076 	 * The top level vdev can appear in random order, depending how
1077 	 * the firmware is presenting the disk devices.
1078 	 * However, we will insert vdev to create list ordered by v_id,
1079 	 * so we can use either STAILQ_INSERT_HEAD or STAILQ_INSERT_AFTER
1080 	 * as STAILQ does not have insert before.
1081 	 */
1082 	previous = vdev_find_previous(top_vdev, vdev);
1083 
1084 	if (previous == NULL) {
1085 		STAILQ_INSERT_HEAD(&top_vdev->v_children, vdev, v_childlink);
1086 	} else if (previous->v_id == vdev->v_id) {
1087 		/*
1088 		 * This vdev was configured from label config,
1089 		 * do not insert duplicate.
1090 		 */
1091 		return;
1092 	} else {
1093 		STAILQ_INSERT_AFTER(&top_vdev->v_children, previous, vdev,
1094 		    v_childlink);
1095 	}
1096 
1097 	count = vdev_child_count(top_vdev);
1098 	if (top_vdev->v_nchildren < count)
1099 		top_vdev->v_nchildren = count;
1100 }
1101 
1102 static int
1103 vdev_from_nvlist(spa_t *spa, uint64_t top_guid, const nvlist_t *nvlist)
1104 {
1105 	vdev_t *top_vdev, *vdev;
1106 	nvlist_t **kids = NULL;
1107 	int rc, nkids;
1108 
1109 	/* Get top vdev. */
1110 	top_vdev = vdev_find(top_guid);
1111 	if (top_vdev == NULL) {
1112 		rc = vdev_init(top_guid, nvlist, &top_vdev);
1113 		if (rc != 0)
1114 			return (rc);
1115 		top_vdev->v_spa = spa;
1116 		top_vdev->v_top = top_vdev;
1117 		vdev_insert(spa->spa_root_vdev, top_vdev);
1118 	}
1119 
1120 	/* Add children if there are any. */
1121 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1122 	    &nkids, &kids, NULL);
1123 	if (rc == 0) {
1124 		for (int i = 0; i < nkids; i++) {
1125 			uint64_t guid;
1126 
1127 			rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID,
1128 			    DATA_TYPE_UINT64, NULL, &guid, NULL);
1129 			if (rc != 0)
1130 				goto done;
1131 
1132 			rc = vdev_init(guid, kids[i], &vdev);
1133 			if (rc != 0)
1134 				goto done;
1135 
1136 			vdev->v_spa = spa;
1137 			vdev->v_top = top_vdev;
1138 			vdev_insert(top_vdev, vdev);
1139 		}
1140 	} else {
1141 		/*
1142 		 * When there are no children, nvlist_find() does return
1143 		 * error, reset it because leaf devices have no children.
1144 		 */
1145 		rc = 0;
1146 	}
1147 done:
1148 	if (kids != NULL) {
1149 		for (int i = 0; i < nkids; i++)
1150 			nvlist_destroy(kids[i]);
1151 		free(kids);
1152 	}
1153 
1154 	return (rc);
1155 }
1156 
1157 static int
1158 vdev_init_from_label(spa_t *spa, const nvlist_t *nvlist)
1159 {
1160 	uint64_t pool_guid, top_guid;
1161 	nvlist_t *vdevs;
1162 	int rc;
1163 
1164 	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
1165 	    NULL, &pool_guid, NULL) ||
1166 	    nvlist_find(nvlist, ZPOOL_CONFIG_TOP_GUID, DATA_TYPE_UINT64,
1167 	    NULL, &top_guid, NULL) ||
1168 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1169 	    NULL, &vdevs, NULL)) {
1170 		printf("ZFS: can't find vdev details\n");
1171 		return (ENOENT);
1172 	}
1173 
1174 	rc = vdev_from_nvlist(spa, top_guid, vdevs);
1175 	nvlist_destroy(vdevs);
1176 	return (rc);
1177 }
1178 
1179 static void
1180 vdev_set_state(vdev_t *vdev)
1181 {
1182 	vdev_t *kid;
1183 	int good_kids;
1184 	int bad_kids;
1185 
1186 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1187 		vdev_set_state(kid);
1188 	}
1189 
1190 	/*
1191 	 * A mirror or raidz is healthy if all its kids are healthy. A
1192 	 * mirror is degraded if any of its kids is healthy; a raidz
1193 	 * is degraded if at most nparity kids are offline.
1194 	 */
1195 	if (STAILQ_FIRST(&vdev->v_children)) {
1196 		good_kids = 0;
1197 		bad_kids = 0;
1198 		STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1199 			if (kid->v_state == VDEV_STATE_HEALTHY)
1200 				good_kids++;
1201 			else
1202 				bad_kids++;
1203 		}
1204 		if (bad_kids == 0) {
1205 			vdev->v_state = VDEV_STATE_HEALTHY;
1206 		} else {
1207 			if (vdev->v_read == vdev_mirror_read) {
1208 				if (good_kids) {
1209 					vdev->v_state = VDEV_STATE_DEGRADED;
1210 				} else {
1211 					vdev->v_state = VDEV_STATE_OFFLINE;
1212 				}
1213 			} else if (vdev->v_read == vdev_raidz_read) {
1214 				if (bad_kids > vdev->v_nparity) {
1215 					vdev->v_state = VDEV_STATE_OFFLINE;
1216 				} else {
1217 					vdev->v_state = VDEV_STATE_DEGRADED;
1218 				}
1219 			}
1220 		}
1221 	}
1222 }
1223 
1224 static int
1225 vdev_update_from_nvlist(uint64_t top_guid, const nvlist_t *nvlist)
1226 {
1227 	vdev_t *vdev;
1228 	nvlist_t **kids = NULL;
1229 	int rc, nkids;
1230 
1231 	/* Update top vdev. */
1232 	vdev = vdev_find(top_guid);
1233 	if (vdev != NULL)
1234 		vdev_set_initial_state(vdev, nvlist);
1235 
1236 	/* Update children if there are any. */
1237 	rc = nvlist_find(nvlist, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1238 	    &nkids, &kids, NULL);
1239 	if (rc == 0) {
1240 		for (int i = 0; i < nkids; i++) {
1241 			uint64_t guid;
1242 
1243 			rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID,
1244 			    DATA_TYPE_UINT64, NULL, &guid, NULL);
1245 			if (rc != 0)
1246 				break;
1247 
1248 			vdev = vdev_find(guid);
1249 			if (vdev != NULL)
1250 				vdev_set_initial_state(vdev, kids[i]);
1251 		}
1252 	} else {
1253 		rc = 0;
1254 	}
1255 	if (kids != NULL) {
1256 		for (int i = 0; i < nkids; i++)
1257 			nvlist_destroy(kids[i]);
1258 		free(kids);
1259 	}
1260 
1261 	return (rc);
1262 }
1263 
1264 static int
1265 vdev_init_from_nvlist(spa_t *spa, const nvlist_t *nvlist)
1266 {
1267 	uint64_t pool_guid, vdev_children;
1268 	nvlist_t *vdevs = NULL, **kids = NULL;
1269 	int rc, nkids;
1270 
1271 	if (nvlist_find(nvlist, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
1272 	    NULL, &pool_guid, NULL) ||
1273 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_CHILDREN, DATA_TYPE_UINT64,
1274 	    NULL, &vdev_children, NULL) ||
1275 	    nvlist_find(nvlist, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1276 	    NULL, &vdevs, NULL)) {
1277 		printf("ZFS: can't find vdev details\n");
1278 		return (ENOENT);
1279 	}
1280 
1281 	/* Wrong guid?! */
1282 	if (spa->spa_guid != pool_guid) {
1283 		nvlist_destroy(vdevs);
1284 		return (EINVAL);
1285 	}
1286 
1287 	spa->spa_root_vdev->v_nchildren = vdev_children;
1288 
1289 	rc = nvlist_find(vdevs, ZPOOL_CONFIG_CHILDREN, DATA_TYPE_NVLIST_ARRAY,
1290 	    &nkids, &kids, NULL);
1291 	nvlist_destroy(vdevs);
1292 
1293 	/*
1294 	 * MOS config has at least one child for root vdev.
1295 	 */
1296 	if (rc != 0)
1297 		return (rc);
1298 
1299 	for (int i = 0; i < nkids; i++) {
1300 		uint64_t guid;
1301 		vdev_t *vdev;
1302 
1303 		rc = nvlist_find(kids[i], ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
1304 		    NULL, &guid, NULL);
1305 		if (rc != 0)
1306 			break;
1307 		vdev = vdev_find(guid);
1308 		/*
1309 		 * Top level vdev is missing, create it.
1310 		 */
1311 		if (vdev == NULL)
1312 			rc = vdev_from_nvlist(spa, guid, kids[i]);
1313 		else
1314 			rc = vdev_update_from_nvlist(guid, kids[i]);
1315 		if (rc != 0)
1316 			break;
1317 	}
1318 	if (kids != NULL) {
1319 		for (int i = 0; i < nkids; i++)
1320 			nvlist_destroy(kids[i]);
1321 		free(kids);
1322 	}
1323 
1324 	/*
1325 	 * Re-evaluate top-level vdev state.
1326 	 */
1327 	vdev_set_state(spa->spa_root_vdev);
1328 
1329 	return (rc);
1330 }
1331 
1332 static spa_t *
1333 spa_find_by_guid(uint64_t guid)
1334 {
1335 	spa_t *spa;
1336 
1337 	STAILQ_FOREACH(spa, &zfs_pools, spa_link)
1338 		if (spa->spa_guid == guid)
1339 			return (spa);
1340 
1341 	return (NULL);
1342 }
1343 
1344 static spa_t *
1345 spa_find_by_name(const char *name)
1346 {
1347 	spa_t *spa;
1348 
1349 	STAILQ_FOREACH(spa, &zfs_pools, spa_link)
1350 		if (strcmp(spa->spa_name, name) == 0)
1351 			return (spa);
1352 
1353 	return (NULL);
1354 }
1355 
1356 static spa_t *
1357 spa_find_by_dev(struct zfs_devdesc *dev)
1358 {
1359 
1360 	if (dev->dd.d_dev->dv_type != DEVT_ZFS)
1361 		return (NULL);
1362 
1363 	if (dev->pool_guid == 0)
1364 		return (STAILQ_FIRST(&zfs_pools));
1365 
1366 	return (spa_find_by_guid(dev->pool_guid));
1367 }
1368 
1369 static spa_t *
1370 spa_create(uint64_t guid, const char *name)
1371 {
1372 	spa_t *spa;
1373 
1374 	if ((spa = calloc(1, sizeof(spa_t))) == NULL)
1375 		return (NULL);
1376 	if ((spa->spa_name = strdup(name)) == NULL) {
1377 		free(spa);
1378 		return (NULL);
1379 	}
1380 	spa->spa_uberblock = &spa->spa_uberblock_master;
1381 	spa->spa_mos = &spa->spa_mos_master;
1382 	spa->spa_guid = guid;
1383 	spa->spa_root_vdev = vdev_create(guid, NULL);
1384 	if (spa->spa_root_vdev == NULL) {
1385 		free(spa->spa_name);
1386 		free(spa);
1387 		return (NULL);
1388 	}
1389 	spa->spa_root_vdev->v_name = strdup("root");
1390 	STAILQ_INSERT_TAIL(&zfs_pools, spa, spa_link);
1391 
1392 	return (spa);
1393 }
1394 
1395 static const char *
1396 state_name(vdev_state_t state)
1397 {
1398 	static const char *names[] = {
1399 		"UNKNOWN",
1400 		"CLOSED",
1401 		"OFFLINE",
1402 		"REMOVED",
1403 		"CANT_OPEN",
1404 		"FAULTED",
1405 		"DEGRADED",
1406 		"ONLINE"
1407 	};
1408 	return (names[state]);
1409 }
1410 
1411 #ifdef BOOT2
1412 
1413 #define pager_printf printf
1414 
1415 #else
1416 
1417 static int
1418 pager_printf(const char *fmt, ...)
1419 {
1420 	char line[80];
1421 	va_list args;
1422 
1423 	va_start(args, fmt);
1424 	vsnprintf(line, sizeof(line), fmt, args);
1425 	va_end(args);
1426 	return (pager_output(line));
1427 }
1428 
1429 #endif
1430 
1431 #define	STATUS_FORMAT	"        %s %s\n"
1432 
1433 static int
1434 print_state(int indent, const char *name, vdev_state_t state)
1435 {
1436 	int i;
1437 	char buf[512];
1438 
1439 	buf[0] = 0;
1440 	for (i = 0; i < indent; i++)
1441 		strcat(buf, "  ");
1442 	strcat(buf, name);
1443 	return (pager_printf(STATUS_FORMAT, buf, state_name(state)));
1444 }
1445 
1446 static int
1447 vdev_status(vdev_t *vdev, int indent)
1448 {
1449 	vdev_t *kid;
1450 	int ret;
1451 
1452 	if (vdev->v_islog) {
1453 		(void) pager_output("        logs\n");
1454 		indent++;
1455 	}
1456 
1457 	ret = print_state(indent, vdev->v_name, vdev->v_state);
1458 	if (ret != 0)
1459 		return (ret);
1460 
1461 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1462 		ret = vdev_status(kid, indent + 1);
1463 		if (ret != 0)
1464 			return (ret);
1465 	}
1466 	return (ret);
1467 }
1468 
1469 static int
1470 spa_status(spa_t *spa)
1471 {
1472 	static char bootfs[ZFS_MAXNAMELEN];
1473 	uint64_t rootid;
1474 	vdev_list_t *vlist;
1475 	vdev_t *vdev;
1476 	int good_kids, bad_kids, degraded_kids, ret;
1477 	vdev_state_t state;
1478 
1479 	ret = pager_printf("  pool: %s\n", spa->spa_name);
1480 	if (ret != 0)
1481 		return (ret);
1482 
1483 	if (zfs_get_root(spa, &rootid) == 0 &&
1484 	    zfs_rlookup(spa, rootid, bootfs) == 0) {
1485 		if (bootfs[0] == '\0')
1486 			ret = pager_printf("bootfs: %s\n", spa->spa_name);
1487 		else
1488 			ret = pager_printf("bootfs: %s/%s\n", spa->spa_name,
1489 			    bootfs);
1490 		if (ret != 0)
1491 			return (ret);
1492 	}
1493 	ret = pager_printf("config:\n\n");
1494 	if (ret != 0)
1495 		return (ret);
1496 	ret = pager_printf(STATUS_FORMAT, "NAME", "STATE");
1497 	if (ret != 0)
1498 		return (ret);
1499 
1500 	good_kids = 0;
1501 	degraded_kids = 0;
1502 	bad_kids = 0;
1503 	vlist = &spa->spa_root_vdev->v_children;
1504 	STAILQ_FOREACH(vdev, vlist, v_childlink) {
1505 		if (vdev->v_state == VDEV_STATE_HEALTHY)
1506 			good_kids++;
1507 		else if (vdev->v_state == VDEV_STATE_DEGRADED)
1508 			degraded_kids++;
1509 		else
1510 			bad_kids++;
1511 	}
1512 
1513 	state = VDEV_STATE_CLOSED;
1514 	if (good_kids > 0 && (degraded_kids + bad_kids) == 0)
1515 		state = VDEV_STATE_HEALTHY;
1516 	else if ((good_kids + degraded_kids) > 0)
1517 		state = VDEV_STATE_DEGRADED;
1518 
1519 	ret = print_state(0, spa->spa_name, state);
1520 	if (ret != 0)
1521 		return (ret);
1522 
1523 	STAILQ_FOREACH(vdev, vlist, v_childlink) {
1524 		ret = vdev_status(vdev, 1);
1525 		if (ret != 0)
1526 			return (ret);
1527 	}
1528 	return (ret);
1529 }
1530 
1531 static int
1532 spa_all_status(void)
1533 {
1534 	spa_t *spa;
1535 	int first = 1, ret = 0;
1536 
1537 	STAILQ_FOREACH(spa, &zfs_pools, spa_link) {
1538 		if (!first) {
1539 			ret = pager_printf("\n");
1540 			if (ret != 0)
1541 				return (ret);
1542 		}
1543 		first = 0;
1544 		ret = spa_status(spa);
1545 		if (ret != 0)
1546 			return (ret);
1547 	}
1548 	return (ret);
1549 }
1550 
1551 static uint64_t
1552 vdev_label_offset(uint64_t psize, int l, uint64_t offset)
1553 {
1554 	uint64_t label_offset;
1555 
1556 	if (l < VDEV_LABELS / 2)
1557 		label_offset = 0;
1558 	else
1559 		label_offset = psize - VDEV_LABELS * sizeof (vdev_label_t);
1560 
1561 	return (offset + l * sizeof (vdev_label_t) + label_offset);
1562 }
1563 
1564 static int
1565 vdev_uberblock_compare(const uberblock_t *ub1, const uberblock_t *ub2)
1566 {
1567 	unsigned int seq1 = 0;
1568 	unsigned int seq2 = 0;
1569 	int cmp = AVL_CMP(ub1->ub_txg, ub2->ub_txg);
1570 
1571 	if (cmp != 0)
1572 		return (cmp);
1573 
1574 	cmp = AVL_CMP(ub1->ub_timestamp, ub2->ub_timestamp);
1575 	if (cmp != 0)
1576 		return (cmp);
1577 
1578 	if (MMP_VALID(ub1) && MMP_SEQ_VALID(ub1))
1579 		seq1 = MMP_SEQ(ub1);
1580 
1581 	if (MMP_VALID(ub2) && MMP_SEQ_VALID(ub2))
1582 		seq2 = MMP_SEQ(ub2);
1583 
1584 	return (AVL_CMP(seq1, seq2));
1585 }
1586 
1587 static int
1588 uberblock_verify(uberblock_t *ub)
1589 {
1590 	if (ub->ub_magic == BSWAP_64((uint64_t)UBERBLOCK_MAGIC)) {
1591 		byteswap_uint64_array(ub, sizeof (uberblock_t));
1592 	}
1593 
1594 	if (ub->ub_magic != UBERBLOCK_MAGIC ||
1595 	    !SPA_VERSION_IS_SUPPORTED(ub->ub_version))
1596 		return (EINVAL);
1597 
1598 	return (0);
1599 }
1600 
1601 static int
1602 vdev_label_read(vdev_t *vd, int l, void *buf, uint64_t offset,
1603     size_t size)
1604 {
1605 	blkptr_t bp;
1606 	off_t off;
1607 
1608 	off = vdev_label_offset(vd->v_psize, l, offset);
1609 
1610 	BP_ZERO(&bp);
1611 	BP_SET_LSIZE(&bp, size);
1612 	BP_SET_PSIZE(&bp, size);
1613 	BP_SET_CHECKSUM(&bp, ZIO_CHECKSUM_LABEL);
1614 	BP_SET_COMPRESS(&bp, ZIO_COMPRESS_OFF);
1615 	DVA_SET_OFFSET(BP_IDENTITY(&bp), off);
1616 	ZIO_SET_CHECKSUM(&bp.blk_cksum, off, 0, 0, 0);
1617 
1618 	return (vdev_read_phys(vd, &bp, buf, off, size));
1619 }
1620 
1621 /*
1622  * We do need to be sure we write to correct location.
1623  * Our vdev label does consist of 4 fields:
1624  * pad1 (8k), reserved.
1625  * bootenv (8k), checksummed, previously reserved, may contian garbage.
1626  * vdev_phys (112k), checksummed
1627  * uberblock ring (128k), checksummed.
1628  *
1629  * Since bootenv area may contain garbage, we can not reliably read it, as
1630  * we can get checksum errors.
1631  * Next best thing is vdev_phys - it is just after bootenv. It still may
1632  * be corrupted, but in such case we will miss this one write.
1633  */
1634 static int
1635 vdev_label_write_validate(vdev_t *vd, int l, uint64_t offset)
1636 {
1637 	uint64_t off, o_phys;
1638 	void *buf;
1639 	size_t size = VDEV_PHYS_SIZE;
1640 	int rc;
1641 
1642 	o_phys = offsetof(vdev_label_t, vl_vdev_phys);
1643 	off = vdev_label_offset(vd->v_psize, l, o_phys);
1644 
1645 	/* off should be 8K from bootenv */
1646 	if (vdev_label_offset(vd->v_psize, l, offset) + VDEV_PAD_SIZE != off)
1647 		return (EINVAL);
1648 
1649 	buf = malloc(size);
1650 	if (buf == NULL)
1651 		return (ENOMEM);
1652 
1653 	/* Read vdev_phys */
1654 	rc = vdev_label_read(vd, l, buf, o_phys, size);
1655 	free(buf);
1656 	return (rc);
1657 }
1658 
1659 static int
1660 vdev_label_write(vdev_t *vd, int l, vdev_boot_envblock_t *be, uint64_t offset)
1661 {
1662 	zio_checksum_info_t *ci;
1663 	zio_cksum_t cksum;
1664 	off_t off;
1665 	size_t size = VDEV_PAD_SIZE;
1666 	int rc;
1667 
1668 	if (vd->v_phys_write == NULL)
1669 		return (ENOTSUP);
1670 
1671 	off = vdev_label_offset(vd->v_psize, l, offset);
1672 
1673 	rc = vdev_label_write_validate(vd, l, offset);
1674 	if (rc != 0) {
1675 		return (rc);
1676 	}
1677 
1678 	ci = &zio_checksum_table[ZIO_CHECKSUM_LABEL];
1679 	be->vbe_zbt.zec_magic = ZEC_MAGIC;
1680 	zio_checksum_label_verifier(&be->vbe_zbt.zec_cksum, off);
1681 	ci->ci_func[0](be, size, NULL, &cksum);
1682 	be->vbe_zbt.zec_cksum = cksum;
1683 
1684 	return (vdev_write_phys(vd, be, off, size));
1685 }
1686 
1687 static int
1688 vdev_write_bootenv_impl(vdev_t *vdev, vdev_boot_envblock_t *be)
1689 {
1690 	vdev_t *kid;
1691 	int rv = 0, rc;
1692 
1693 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1694 		if (kid->v_state != VDEV_STATE_HEALTHY)
1695 			continue;
1696 		rc = vdev_write_bootenv_impl(kid, be);
1697 		if (rv == 0)
1698 			rv = rc;
1699 	}
1700 
1701 	/*
1702 	 * Non-leaf vdevs do not have v_phys_write.
1703 	 */
1704 	if (vdev->v_phys_write == NULL)
1705 		return (rv);
1706 
1707 	for (int l = 0; l < VDEV_LABELS; l++) {
1708 		rc = vdev_label_write(vdev, l, be,
1709 		    offsetof(vdev_label_t, vl_be));
1710 		if (rc != 0) {
1711 			printf("failed to write bootenv to %s label %d: %d\n",
1712 			    vdev->v_name ? vdev->v_name : "unknown", l, rc);
1713 			rv = rc;
1714 		}
1715 	}
1716 	return (rv);
1717 }
1718 
1719 int
1720 vdev_write_bootenv(vdev_t *vdev, nvlist_t *nvl)
1721 {
1722 	vdev_boot_envblock_t *be;
1723 	nvlist_t nv, *nvp;
1724 	uint64_t version;
1725 	int rv;
1726 
1727 	if (nvl->nv_size > sizeof(be->vbe_bootenv))
1728 		return (E2BIG);
1729 
1730 	version = VB_RAW;
1731 	nvp = vdev_read_bootenv(vdev);
1732 	if (nvp != NULL) {
1733 		nvlist_find(nvp, BOOTENV_VERSION, DATA_TYPE_UINT64, NULL,
1734 		    &version, NULL);
1735 		nvlist_destroy(nvp);
1736 	}
1737 
1738 	be = calloc(1, sizeof(*be));
1739 	if (be == NULL)
1740 		return (ENOMEM);
1741 
1742 	be->vbe_version = version;
1743 	switch (version) {
1744 	case VB_RAW:
1745 		/*
1746 		 * If there is no envmap, we will just wipe bootenv.
1747 		 */
1748 		nvlist_find(nvl, GRUB_ENVMAP, DATA_TYPE_STRING, NULL,
1749 		    be->vbe_bootenv, NULL);
1750 		rv = 0;
1751 		break;
1752 
1753 	case VB_NVLIST:
1754 		nv.nv_header = nvl->nv_header;
1755 		nv.nv_asize = nvl->nv_asize;
1756 		nv.nv_size = nvl->nv_size;
1757 
1758 		bcopy(&nv.nv_header, be->vbe_bootenv, sizeof(nv.nv_header));
1759 		nv.nv_data = be->vbe_bootenv + sizeof(nvs_header_t);
1760 		bcopy(nvl->nv_data, nv.nv_data, nv.nv_size);
1761 		rv = nvlist_export(&nv);
1762 		break;
1763 
1764 	default:
1765 		rv = EINVAL;
1766 		break;
1767 	}
1768 
1769 	if (rv == 0) {
1770 		be->vbe_version = htobe64(be->vbe_version);
1771 		rv = vdev_write_bootenv_impl(vdev, be);
1772 	}
1773 	free(be);
1774 	return (rv);
1775 }
1776 
1777 /*
1778  * Read the bootenv area from pool label, return the nvlist from it.
1779  * We return from first successful read.
1780  */
1781 nvlist_t *
1782 vdev_read_bootenv(vdev_t *vdev)
1783 {
1784 	vdev_t *kid;
1785 	nvlist_t *benv;
1786 	vdev_boot_envblock_t *be;
1787 	char *command;
1788 	bool ok;
1789 	int rv;
1790 
1791 	STAILQ_FOREACH(kid, &vdev->v_children, v_childlink) {
1792 		if (kid->v_state != VDEV_STATE_HEALTHY)
1793 			continue;
1794 
1795 		benv = vdev_read_bootenv(kid);
1796 		if (benv != NULL)
1797 			return (benv);
1798 	}
1799 
1800 	be = malloc(sizeof (*be));
1801 	if (be == NULL)
1802 		return (NULL);
1803 
1804 	rv = 0;
1805 	for (int l = 0; l < VDEV_LABELS; l++) {
1806 		rv = vdev_label_read(vdev, l, be,
1807 		    offsetof(vdev_label_t, vl_be),
1808 		    sizeof (*be));
1809 		if (rv == 0)
1810 			break;
1811 	}
1812 	if (rv != 0) {
1813 		free(be);
1814 		return (NULL);
1815 	}
1816 
1817 	be->vbe_version = be64toh(be->vbe_version);
1818 	switch (be->vbe_version) {
1819 	case VB_RAW:
1820 		/*
1821 		 * we have textual data in vbe_bootenv, create nvlist
1822 		 * with key "envmap".
1823 		 */
1824 		benv = nvlist_create(NV_UNIQUE_NAME);
1825 		if (benv != NULL) {
1826 			if (*be->vbe_bootenv == '\0') {
1827 				nvlist_add_uint64(benv, BOOTENV_VERSION,
1828 				    VB_NVLIST);
1829 				break;
1830 			}
1831 			nvlist_add_uint64(benv, BOOTENV_VERSION, VB_RAW);
1832 			be->vbe_bootenv[sizeof (be->vbe_bootenv) - 1] = '\0';
1833 			nvlist_add_string(benv, GRUB_ENVMAP, be->vbe_bootenv);
1834 		}
1835 		break;
1836 
1837 	case VB_NVLIST:
1838 		benv = nvlist_import(be->vbe_bootenv, sizeof(be->vbe_bootenv));
1839 		break;
1840 
1841 	default:
1842 		command = (char *)be;
1843 		ok = false;
1844 
1845 		/* Check for legacy zfsbootcfg command string */
1846 		for (int i = 0; command[i] != '\0'; i++) {
1847 			if (iscntrl(command[i])) {
1848 				ok = false;
1849 				break;
1850 			} else {
1851 				ok = true;
1852 			}
1853 		}
1854 		benv = nvlist_create(NV_UNIQUE_NAME);
1855 		if (benv != NULL) {
1856 			if (ok)
1857 				nvlist_add_string(benv, FREEBSD_BOOTONCE,
1858 				    command);
1859 			else
1860 				nvlist_add_uint64(benv, BOOTENV_VERSION,
1861 				    VB_NVLIST);
1862 		}
1863 		break;
1864 	}
1865 	free(be);
1866 	return (benv);
1867 }
1868 
1869 static uint64_t
1870 vdev_get_label_asize(nvlist_t *nvl)
1871 {
1872 	nvlist_t *vdevs;
1873 	uint64_t asize;
1874 	const char *type;
1875 	int len;
1876 
1877 	asize = 0;
1878 	/* Get vdev tree */
1879 	if (nvlist_find(nvl, ZPOOL_CONFIG_VDEV_TREE, DATA_TYPE_NVLIST,
1880 	    NULL, &vdevs, NULL) != 0)
1881 		return (asize);
1882 
1883 	/*
1884 	 * Get vdev type. We will calculate asize for raidz, mirror and disk.
1885 	 * For raidz, the asize is raw size of all children.
1886 	 */
1887 	if (nvlist_find(vdevs, ZPOOL_CONFIG_TYPE, DATA_TYPE_STRING,
1888 	    NULL, &type, &len) != 0)
1889 		goto done;
1890 
1891 	if (memcmp(type, VDEV_TYPE_MIRROR, len) != 0 &&
1892 	    memcmp(type, VDEV_TYPE_DISK, len) != 0 &&
1893 	    memcmp(type, VDEV_TYPE_RAIDZ, len) != 0)
1894 		goto done;
1895 
1896 	if (nvlist_find(vdevs, ZPOOL_CONFIG_ASIZE, DATA_TYPE_UINT64,
1897 	    NULL, &asize, NULL) != 0)
1898 		goto done;
1899 
1900 	if (memcmp(type, VDEV_TYPE_RAIDZ, len) == 0) {
1901 		nvlist_t **kids;
1902 		int nkids;
1903 
1904 		if (nvlist_find(vdevs, ZPOOL_CONFIG_CHILDREN,
1905 		    DATA_TYPE_NVLIST_ARRAY, &nkids, &kids, NULL) != 0) {
1906 			asize = 0;
1907 			goto done;
1908 		}
1909 
1910 		asize /= nkids;
1911 		for (int i = 0; i < nkids; i++)
1912 			nvlist_destroy(kids[i]);
1913 		free(kids);
1914 	}
1915 
1916 	asize += VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE;
1917 done:
1918 	nvlist_destroy(vdevs);
1919 	return (asize);
1920 }
1921 
1922 static nvlist_t *
1923 vdev_label_read_config(vdev_t *vd, uint64_t txg)
1924 {
1925 	vdev_phys_t *label;
1926 	uint64_t best_txg = 0;
1927 	uint64_t label_txg = 0;
1928 	uint64_t asize;
1929 	nvlist_t *nvl = NULL, *tmp;
1930 	int error;
1931 
1932 	label = malloc(sizeof (vdev_phys_t));
1933 	if (label == NULL)
1934 		return (NULL);
1935 
1936 	for (int l = 0; l < VDEV_LABELS; l++) {
1937 		if (vdev_label_read(vd, l, label,
1938 		    offsetof(vdev_label_t, vl_vdev_phys),
1939 		    sizeof (vdev_phys_t)))
1940 			continue;
1941 
1942 		tmp = nvlist_import(label->vp_nvlist,
1943 		    sizeof(label->vp_nvlist));
1944 		if (tmp == NULL)
1945 			continue;
1946 
1947 		error = nvlist_find(tmp, ZPOOL_CONFIG_POOL_TXG,
1948 		    DATA_TYPE_UINT64, NULL, &label_txg, NULL);
1949 		if (error != 0 || label_txg == 0) {
1950 			nvlist_destroy(nvl);
1951 			nvl = tmp;
1952 			goto done;
1953 		}
1954 
1955 		if (label_txg <= txg && label_txg > best_txg) {
1956 			best_txg = label_txg;
1957 			nvlist_destroy(nvl);
1958 			nvl = tmp;
1959 			tmp = NULL;
1960 
1961 			/*
1962 			 * Use asize from pool config. We need this
1963 			 * because we can get bad value from BIOS.
1964 			 */
1965 			asize = vdev_get_label_asize(nvl);
1966 			if (asize != 0) {
1967 				vd->v_psize = asize;
1968 			}
1969 		}
1970 		nvlist_destroy(tmp);
1971 	}
1972 
1973 	if (best_txg == 0) {
1974 		nvlist_destroy(nvl);
1975 		nvl = NULL;
1976 	}
1977 done:
1978 	free(label);
1979 	return (nvl);
1980 }
1981 
1982 static void
1983 vdev_uberblock_load(vdev_t *vd, uberblock_t *ub)
1984 {
1985 	uberblock_t *buf;
1986 
1987 	buf = malloc(VDEV_UBERBLOCK_SIZE(vd));
1988 	if (buf == NULL)
1989 		return;
1990 
1991 	for (int l = 0; l < VDEV_LABELS; l++) {
1992 		for (int n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
1993 			if (vdev_label_read(vd, l, buf,
1994 			    VDEV_UBERBLOCK_OFFSET(vd, n),
1995 			    VDEV_UBERBLOCK_SIZE(vd)))
1996 				continue;
1997 			if (uberblock_verify(buf) != 0)
1998 				continue;
1999 
2000 			if (vdev_uberblock_compare(buf, ub) > 0)
2001 				*ub = *buf;
2002 		}
2003 	}
2004 	free(buf);
2005 }
2006 
2007 static int
2008 vdev_probe(vdev_phys_read_t *_read, vdev_phys_write_t *_write, void *priv,
2009     spa_t **spap)
2010 {
2011 	vdev_t vtmp;
2012 	spa_t *spa;
2013 	vdev_t *vdev;
2014 	nvlist_t *nvl;
2015 	uint64_t val;
2016 	uint64_t guid, vdev_children;
2017 	uint64_t pool_txg, pool_guid;
2018 	const char *pool_name;
2019 	int rc, namelen;
2020 
2021 	/*
2022 	 * Load the vdev label and figure out which
2023 	 * uberblock is most current.
2024 	 */
2025 	memset(&vtmp, 0, sizeof(vtmp));
2026 	vtmp.v_phys_read = _read;
2027 	vtmp.v_phys_write = _write;
2028 	vtmp.v_priv = priv;
2029 	vtmp.v_psize = P2ALIGN(ldi_get_size(priv),
2030 	    (uint64_t)sizeof (vdev_label_t));
2031 
2032 	/* Test for minimum device size. */
2033 	if (vtmp.v_psize < SPA_MINDEVSIZE)
2034 		return (EIO);
2035 
2036 	nvl = vdev_label_read_config(&vtmp, UINT64_MAX);
2037 	if (nvl == NULL)
2038 		return (EIO);
2039 
2040 	if (nvlist_find(nvl, ZPOOL_CONFIG_VERSION, DATA_TYPE_UINT64,
2041 	    NULL, &val, NULL) != 0) {
2042 		nvlist_destroy(nvl);
2043 		return (EIO);
2044 	}
2045 
2046 	if (!SPA_VERSION_IS_SUPPORTED(val)) {
2047 		printf("ZFS: unsupported ZFS version %u (should be %u)\n",
2048 		    (unsigned)val, (unsigned)SPA_VERSION);
2049 		nvlist_destroy(nvl);
2050 		return (EIO);
2051 	}
2052 
2053 	/* Check ZFS features for read */
2054 	rc = nvlist_check_features_for_read(nvl);
2055 	if (rc != 0) {
2056 		nvlist_destroy(nvl);
2057 		return (EIO);
2058 	}
2059 
2060 	if (nvlist_find(nvl, ZPOOL_CONFIG_POOL_STATE, DATA_TYPE_UINT64,
2061 	    NULL, &val, NULL) != 0) {
2062 		nvlist_destroy(nvl);
2063 		return (EIO);
2064 	}
2065 
2066 	if (val == POOL_STATE_DESTROYED) {
2067 		/* We don't boot only from destroyed pools. */
2068 		nvlist_destroy(nvl);
2069 		return (EIO);
2070 	}
2071 
2072 	if (nvlist_find(nvl, ZPOOL_CONFIG_POOL_TXG, DATA_TYPE_UINT64,
2073 	    NULL, &pool_txg, NULL) != 0 ||
2074 	    nvlist_find(nvl, ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64,
2075 	    NULL, &pool_guid, NULL) != 0 ||
2076 	    nvlist_find(nvl, ZPOOL_CONFIG_POOL_NAME, DATA_TYPE_STRING,
2077 	    NULL, &pool_name, &namelen) != 0) {
2078 		/*
2079 		 * Cache and spare devices end up here - just ignore
2080 		 * them.
2081 		 */
2082 		nvlist_destroy(nvl);
2083 		return (EIO);
2084 	}
2085 
2086 	/*
2087 	 * Create the pool if this is the first time we've seen it.
2088 	 */
2089 	spa = spa_find_by_guid(pool_guid);
2090 	if (spa == NULL) {
2091 		char *name;
2092 
2093 		nvlist_find(nvl, ZPOOL_CONFIG_VDEV_CHILDREN,
2094 		    DATA_TYPE_UINT64, NULL, &vdev_children, NULL);
2095 		name = malloc(namelen + 1);
2096 		if (name == NULL) {
2097 			nvlist_destroy(nvl);
2098 			return (ENOMEM);
2099 		}
2100 		bcopy(pool_name, name, namelen);
2101 		name[namelen] = '\0';
2102 		spa = spa_create(pool_guid, name);
2103 		free(name);
2104 		if (spa == NULL) {
2105 			nvlist_destroy(nvl);
2106 			return (ENOMEM);
2107 		}
2108 		spa->spa_root_vdev->v_nchildren = vdev_children;
2109 	}
2110 	if (pool_txg > spa->spa_txg)
2111 		spa->spa_txg = pool_txg;
2112 
2113 	/*
2114 	 * Get the vdev tree and create our in-core copy of it.
2115 	 * If we already have a vdev with this guid, this must
2116 	 * be some kind of alias (overlapping slices, dangerously dedicated
2117 	 * disks etc).
2118 	 */
2119 	if (nvlist_find(nvl, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64,
2120 	    NULL, &guid, NULL) != 0) {
2121 		nvlist_destroy(nvl);
2122 		return (EIO);
2123 	}
2124 	vdev = vdev_find(guid);
2125 	/* Has this vdev already been inited? */
2126 	if (vdev && vdev->v_phys_read) {
2127 		nvlist_destroy(nvl);
2128 		return (EIO);
2129 	}
2130 
2131 	rc = vdev_init_from_label(spa, nvl);
2132 	nvlist_destroy(nvl);
2133 	if (rc != 0)
2134 		return (rc);
2135 
2136 	/*
2137 	 * We should already have created an incomplete vdev for this
2138 	 * vdev. Find it and initialise it with our read proc.
2139 	 */
2140 	vdev = vdev_find(guid);
2141 	if (vdev != NULL) {
2142 		vdev->v_phys_read = _read;
2143 		vdev->v_phys_write = _write;
2144 		vdev->v_priv = priv;
2145 		vdev->v_psize = vtmp.v_psize;
2146 		/*
2147 		 * If no other state is set, mark vdev healthy.
2148 		 */
2149 		if (vdev->v_state == VDEV_STATE_UNKNOWN)
2150 			vdev->v_state = VDEV_STATE_HEALTHY;
2151 	} else {
2152 		printf("ZFS: inconsistent nvlist contents\n");
2153 		return (EIO);
2154 	}
2155 
2156 	if (vdev->v_islog)
2157 		spa->spa_with_log = vdev->v_islog;
2158 
2159 	/*
2160 	 * Re-evaluate top-level vdev state.
2161 	 */
2162 	vdev_set_state(vdev->v_top);
2163 
2164 	/*
2165 	 * Ok, we are happy with the pool so far. Lets find
2166 	 * the best uberblock and then we can actually access
2167 	 * the contents of the pool.
2168 	 */
2169 	vdev_uberblock_load(vdev, spa->spa_uberblock);
2170 
2171 	if (spap != NULL)
2172 		*spap = spa;
2173 	return (0);
2174 }
2175 
2176 static int
2177 ilog2(int n)
2178 {
2179 	int v;
2180 
2181 	for (v = 0; v < 32; v++)
2182 		if (n == (1 << v))
2183 			return (v);
2184 	return (-1);
2185 }
2186 
2187 static int
2188 zio_read_gang(const spa_t *spa, const blkptr_t *bp, void *buf)
2189 {
2190 	blkptr_t gbh_bp;
2191 	zio_gbh_phys_t zio_gb;
2192 	char *pbuf;
2193 	int i;
2194 
2195 	/* Artificial BP for gang block header. */
2196 	gbh_bp = *bp;
2197 	BP_SET_PSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
2198 	BP_SET_LSIZE(&gbh_bp, SPA_GANGBLOCKSIZE);
2199 	BP_SET_CHECKSUM(&gbh_bp, ZIO_CHECKSUM_GANG_HEADER);
2200 	BP_SET_COMPRESS(&gbh_bp, ZIO_COMPRESS_OFF);
2201 	for (i = 0; i < SPA_DVAS_PER_BP; i++)
2202 		DVA_SET_GANG(&gbh_bp.blk_dva[i], 0);
2203 
2204 	/* Read gang header block using the artificial BP. */
2205 	if (zio_read(spa, &gbh_bp, &zio_gb))
2206 		return (EIO);
2207 
2208 	pbuf = buf;
2209 	for (i = 0; i < SPA_GBH_NBLKPTRS; i++) {
2210 		blkptr_t *gbp = &zio_gb.zg_blkptr[i];
2211 
2212 		if (BP_IS_HOLE(gbp))
2213 			continue;
2214 		if (zio_read(spa, gbp, pbuf))
2215 			return (EIO);
2216 		pbuf += BP_GET_PSIZE(gbp);
2217 	}
2218 
2219 	if (zio_checksum_verify(spa, bp, buf))
2220 		return (EIO);
2221 	return (0);
2222 }
2223 
2224 static int
2225 zio_read(const spa_t *spa, const blkptr_t *bp, void *buf)
2226 {
2227 	int cpfunc = BP_GET_COMPRESS(bp);
2228 	uint64_t align, size;
2229 	void *pbuf;
2230 	int i, error;
2231 
2232 	/*
2233 	 * Process data embedded in block pointer
2234 	 */
2235 	if (BP_IS_EMBEDDED(bp)) {
2236 		ASSERT(BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
2237 
2238 		size = BPE_GET_PSIZE(bp);
2239 		ASSERT(size <= BPE_PAYLOAD_SIZE);
2240 
2241 		if (cpfunc != ZIO_COMPRESS_OFF)
2242 			pbuf = malloc(size);
2243 		else
2244 			pbuf = buf;
2245 
2246 		if (pbuf == NULL)
2247 			return (ENOMEM);
2248 
2249 		decode_embedded_bp_compressed(bp, pbuf);
2250 		error = 0;
2251 
2252 		if (cpfunc != ZIO_COMPRESS_OFF) {
2253 			error = zio_decompress_data(cpfunc, pbuf,
2254 			    size, buf, BP_GET_LSIZE(bp));
2255 			free(pbuf);
2256 		}
2257 		if (error != 0)
2258 			printf("ZFS: i/o error - unable to decompress "
2259 			    "block pointer data, error %d\n", error);
2260 		return (error);
2261 	}
2262 
2263 	error = EIO;
2264 
2265 	for (i = 0; i < SPA_DVAS_PER_BP; i++) {
2266 		const dva_t *dva = &bp->blk_dva[i];
2267 		vdev_t *vdev;
2268 		vdev_list_t *vlist;
2269 		uint64_t vdevid;
2270 		off_t offset;
2271 
2272 		if (!dva->dva_word[0] && !dva->dva_word[1])
2273 			continue;
2274 
2275 		vdevid = DVA_GET_VDEV(dva);
2276 		offset = DVA_GET_OFFSET(dva);
2277 		vlist = &spa->spa_root_vdev->v_children;
2278 		STAILQ_FOREACH(vdev, vlist, v_childlink) {
2279 			if (vdev->v_id == vdevid)
2280 				break;
2281 		}
2282 		if (!vdev || !vdev->v_read)
2283 			continue;
2284 
2285 		size = BP_GET_PSIZE(bp);
2286 		if (vdev->v_read == vdev_raidz_read) {
2287 			align = 1ULL << vdev->v_ashift;
2288 			if (P2PHASE(size, align) != 0)
2289 				size = P2ROUNDUP(size, align);
2290 		}
2291 		if (size != BP_GET_PSIZE(bp) || cpfunc != ZIO_COMPRESS_OFF)
2292 			pbuf = malloc(size);
2293 		else
2294 			pbuf = buf;
2295 
2296 		if (pbuf == NULL) {
2297 			error = ENOMEM;
2298 			break;
2299 		}
2300 
2301 		if (DVA_GET_GANG(dva))
2302 			error = zio_read_gang(spa, bp, pbuf);
2303 		else
2304 			error = vdev->v_read(vdev, bp, pbuf, offset, size);
2305 		if (error == 0) {
2306 			if (cpfunc != ZIO_COMPRESS_OFF)
2307 				error = zio_decompress_data(cpfunc, pbuf,
2308 				    BP_GET_PSIZE(bp), buf, BP_GET_LSIZE(bp));
2309 			else if (size != BP_GET_PSIZE(bp))
2310 				bcopy(pbuf, buf, BP_GET_PSIZE(bp));
2311 		} else {
2312 			printf("zio_read error: %d\n", error);
2313 		}
2314 		if (buf != pbuf)
2315 			free(pbuf);
2316 		if (error == 0)
2317 			break;
2318 	}
2319 	if (error != 0)
2320 		printf("ZFS: i/o error - all block copies unavailable\n");
2321 
2322 	return (error);
2323 }
2324 
2325 static int
2326 dnode_read(const spa_t *spa, const dnode_phys_t *dnode, off_t offset,
2327     void *buf, size_t buflen)
2328 {
2329 	int ibshift = dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
2330 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2331 	int nlevels = dnode->dn_nlevels;
2332 	int i, rc;
2333 
2334 	if (bsize > SPA_MAXBLOCKSIZE) {
2335 		printf("ZFS: I/O error - blocks larger than %llu are not "
2336 		    "supported\n", SPA_MAXBLOCKSIZE);
2337 		return (EIO);
2338 	}
2339 
2340 	/*
2341 	 * Note: bsize may not be a power of two here so we need to do an
2342 	 * actual divide rather than a bitshift.
2343 	 */
2344 	while (buflen > 0) {
2345 		uint64_t bn = offset / bsize;
2346 		int boff = offset % bsize;
2347 		int ibn;
2348 		const blkptr_t *indbp;
2349 		blkptr_t bp;
2350 
2351 		if (bn > dnode->dn_maxblkid)
2352 			return (EIO);
2353 
2354 		if (dnode == dnode_cache_obj && bn == dnode_cache_bn)
2355 			goto cached;
2356 
2357 		indbp = dnode->dn_blkptr;
2358 		for (i = 0; i < nlevels; i++) {
2359 			/*
2360 			 * Copy the bp from the indirect array so that
2361 			 * we can re-use the scratch buffer for multi-level
2362 			 * objects.
2363 			 */
2364 			ibn = bn >> ((nlevels - i - 1) * ibshift);
2365 			ibn &= ((1 << ibshift) - 1);
2366 			bp = indbp[ibn];
2367 			if (BP_IS_HOLE(&bp)) {
2368 				memset(dnode_cache_buf, 0, bsize);
2369 				break;
2370 			}
2371 			rc = zio_read(spa, &bp, dnode_cache_buf);
2372 			if (rc)
2373 				return (rc);
2374 			indbp = (const blkptr_t *) dnode_cache_buf;
2375 		}
2376 		dnode_cache_obj = dnode;
2377 		dnode_cache_bn = bn;
2378 	cached:
2379 
2380 		/*
2381 		 * The buffer contains our data block. Copy what we
2382 		 * need from it and loop.
2383 		 */
2384 		i = bsize - boff;
2385 		if (i > buflen) i = buflen;
2386 		memcpy(buf, &dnode_cache_buf[boff], i);
2387 		buf = ((char *)buf) + i;
2388 		offset += i;
2389 		buflen -= i;
2390 	}
2391 
2392 	return (0);
2393 }
2394 
2395 /*
2396  * Lookup a value in a microzap directory.
2397  */
2398 static int
2399 mzap_lookup(const mzap_phys_t *mz, size_t size, const char *name,
2400     uint64_t *value)
2401 {
2402 	const mzap_ent_phys_t *mze;
2403 	int chunks, i;
2404 
2405 	/*
2406 	 * Microzap objects use exactly one block. Read the whole
2407 	 * thing.
2408 	 */
2409 	chunks = size / MZAP_ENT_LEN - 1;
2410 	for (i = 0; i < chunks; i++) {
2411 		mze = &mz->mz_chunk[i];
2412 		if (strcmp(mze->mze_name, name) == 0) {
2413 			*value = mze->mze_value;
2414 			return (0);
2415 		}
2416 	}
2417 
2418 	return (ENOENT);
2419 }
2420 
2421 /*
2422  * Compare a name with a zap leaf entry. Return non-zero if the name
2423  * matches.
2424  */
2425 static int
2426 fzap_name_equal(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc,
2427     const char *name)
2428 {
2429 	size_t namelen;
2430 	const zap_leaf_chunk_t *nc;
2431 	const char *p;
2432 
2433 	namelen = zc->l_entry.le_name_numints;
2434 
2435 	nc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_name_chunk);
2436 	p = name;
2437 	while (namelen > 0) {
2438 		size_t len;
2439 
2440 		len = namelen;
2441 		if (len > ZAP_LEAF_ARRAY_BYTES)
2442 			len = ZAP_LEAF_ARRAY_BYTES;
2443 		if (memcmp(p, nc->l_array.la_array, len))
2444 			return (0);
2445 		p += len;
2446 		namelen -= len;
2447 		nc = &ZAP_LEAF_CHUNK(zl, nc->l_array.la_next);
2448 	}
2449 
2450 	return (1);
2451 }
2452 
2453 /*
2454  * Extract a uint64_t value from a zap leaf entry.
2455  */
2456 static uint64_t
2457 fzap_leaf_value(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc)
2458 {
2459 	const zap_leaf_chunk_t *vc;
2460 	int i;
2461 	uint64_t value;
2462 	const uint8_t *p;
2463 
2464 	vc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_value_chunk);
2465 	for (i = 0, value = 0, p = vc->l_array.la_array; i < 8; i++) {
2466 		value = (value << 8) | p[i];
2467 	}
2468 
2469 	return (value);
2470 }
2471 
2472 static void
2473 stv(int len, void *addr, uint64_t value)
2474 {
2475 	switch (len) {
2476 	case 1:
2477 		*(uint8_t *)addr = value;
2478 		return;
2479 	case 2:
2480 		*(uint16_t *)addr = value;
2481 		return;
2482 	case 4:
2483 		*(uint32_t *)addr = value;
2484 		return;
2485 	case 8:
2486 		*(uint64_t *)addr = value;
2487 		return;
2488 	}
2489 }
2490 
2491 /*
2492  * Extract a array from a zap leaf entry.
2493  */
2494 static void
2495 fzap_leaf_array(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc,
2496     uint64_t integer_size, uint64_t num_integers, void *buf)
2497 {
2498 	uint64_t array_int_len = zc->l_entry.le_value_intlen;
2499 	uint64_t value = 0;
2500 	uint64_t *u64 = buf;
2501 	char *p = buf;
2502 	int len = MIN(zc->l_entry.le_value_numints, num_integers);
2503 	int chunk = zc->l_entry.le_value_chunk;
2504 	int byten = 0;
2505 
2506 	if (integer_size == 8 && len == 1) {
2507 		*u64 = fzap_leaf_value(zl, zc);
2508 		return;
2509 	}
2510 
2511 	while (len > 0) {
2512 		struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(zl, chunk).l_array;
2513 		int i;
2514 
2515 		ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(zl));
2516 		for (i = 0; i < ZAP_LEAF_ARRAY_BYTES && len > 0; i++) {
2517 			value = (value << 8) | la->la_array[i];
2518 			byten++;
2519 			if (byten == array_int_len) {
2520 				stv(integer_size, p, value);
2521 				byten = 0;
2522 				len--;
2523 				if (len == 0)
2524 					return;
2525 				p += integer_size;
2526 			}
2527 		}
2528 		chunk = la->la_next;
2529 	}
2530 }
2531 
2532 static int
2533 fzap_check_size(uint64_t integer_size, uint64_t num_integers)
2534 {
2535 
2536 	switch (integer_size) {
2537 	case 1:
2538 	case 2:
2539 	case 4:
2540 	case 8:
2541 		break;
2542 	default:
2543 		return (EINVAL);
2544 	}
2545 
2546 	if (integer_size * num_integers > ZAP_MAXVALUELEN)
2547 		return (E2BIG);
2548 
2549 	return (0);
2550 }
2551 
2552 static void
2553 zap_leaf_free(zap_leaf_t *leaf)
2554 {
2555 	free(leaf->l_phys);
2556 	free(leaf);
2557 }
2558 
2559 static int
2560 zap_get_leaf_byblk(fat_zap_t *zap, uint64_t blk, zap_leaf_t **lp)
2561 {
2562 	int bs = FZAP_BLOCK_SHIFT(zap);
2563 	int err;
2564 
2565 	*lp = malloc(sizeof(**lp));
2566 	if (*lp == NULL)
2567 		return (ENOMEM);
2568 
2569 	(*lp)->l_bs = bs;
2570 	(*lp)->l_phys = malloc(1 << bs);
2571 
2572 	if ((*lp)->l_phys == NULL) {
2573 		free(*lp);
2574 		return (ENOMEM);
2575 	}
2576 	err = dnode_read(zap->zap_spa, zap->zap_dnode, blk << bs, (*lp)->l_phys,
2577 	    1 << bs);
2578 	if (err != 0) {
2579 		zap_leaf_free(*lp);
2580 	}
2581 	return (err);
2582 }
2583 
2584 static int
2585 zap_table_load(fat_zap_t *zap, zap_table_phys_t *tbl, uint64_t idx,
2586     uint64_t *valp)
2587 {
2588 	int bs = FZAP_BLOCK_SHIFT(zap);
2589 	uint64_t blk = idx >> (bs - 3);
2590 	uint64_t off = idx & ((1 << (bs - 3)) - 1);
2591 	uint64_t *buf;
2592 	int rc;
2593 
2594 	buf = malloc(1 << zap->zap_block_shift);
2595 	if (buf == NULL)
2596 		return (ENOMEM);
2597 	rc = dnode_read(zap->zap_spa, zap->zap_dnode, (tbl->zt_blk + blk) << bs,
2598 	    buf, 1 << zap->zap_block_shift);
2599 	if (rc == 0)
2600 		*valp = buf[off];
2601 	free(buf);
2602 	return (rc);
2603 }
2604 
2605 static int
2606 zap_idx_to_blk(fat_zap_t *zap, uint64_t idx, uint64_t *valp)
2607 {
2608 	if (zap->zap_phys->zap_ptrtbl.zt_numblks == 0) {
2609 		*valp = ZAP_EMBEDDED_PTRTBL_ENT(zap, idx);
2610 		return (0);
2611 	} else {
2612 		return (zap_table_load(zap, &zap->zap_phys->zap_ptrtbl,
2613 		    idx, valp));
2614 	}
2615 }
2616 
2617 #define	ZAP_HASH_IDX(hash, n)	(((n) == 0) ? 0 : ((hash) >> (64 - (n))))
2618 static int
2619 zap_deref_leaf(fat_zap_t *zap, uint64_t h, zap_leaf_t **lp)
2620 {
2621 	uint64_t idx, blk;
2622 	int err;
2623 
2624 	idx = ZAP_HASH_IDX(h, zap->zap_phys->zap_ptrtbl.zt_shift);
2625 	err = zap_idx_to_blk(zap, idx, &blk);
2626 	if (err != 0)
2627 		return (err);
2628 	return (zap_get_leaf_byblk(zap, blk, lp));
2629 }
2630 
2631 #define	CHAIN_END	0xffff	/* end of the chunk chain */
2632 #define	LEAF_HASH(l, h) \
2633 	((ZAP_LEAF_HASH_NUMENTRIES(l)-1) & \
2634 	((h) >> \
2635 	(64 - ZAP_LEAF_HASH_SHIFT(l) - (l)->l_phys->l_hdr.lh_prefix_len)))
2636 #define	LEAF_HASH_ENTPTR(l, h)	(&(l)->l_phys->l_hash[LEAF_HASH(l, h)])
2637 
2638 static int
2639 zap_leaf_lookup(zap_leaf_t *zl, uint64_t hash, const char *name,
2640     uint64_t integer_size, uint64_t num_integers, void *value)
2641 {
2642 	int rc;
2643 	uint16_t *chunkp;
2644 	struct zap_leaf_entry *le;
2645 
2646 	/*
2647 	 * Make sure this chunk matches our hash.
2648 	 */
2649 	if (zl->l_phys->l_hdr.lh_prefix_len > 0 &&
2650 	    zl->l_phys->l_hdr.lh_prefix !=
2651 	    hash >> (64 - zl->l_phys->l_hdr.lh_prefix_len))
2652 		return (EIO);
2653 
2654 	rc = ENOENT;
2655 	for (chunkp = LEAF_HASH_ENTPTR(zl, hash);
2656 	    *chunkp != CHAIN_END; chunkp = &le->le_next) {
2657 		zap_leaf_chunk_t *zc;
2658 		uint16_t chunk = *chunkp;
2659 
2660 		le = ZAP_LEAF_ENTRY(zl, chunk);
2661 		if (le->le_hash != hash)
2662 			continue;
2663 		zc = &ZAP_LEAF_CHUNK(zl, chunk);
2664 		if (fzap_name_equal(zl, zc, name)) {
2665 			if (zc->l_entry.le_value_intlen > integer_size) {
2666 				rc = EINVAL;
2667 			} else {
2668 				fzap_leaf_array(zl, zc, integer_size,
2669 				    num_integers, value);
2670 				rc = 0;
2671 			}
2672 			break;
2673 		}
2674 	}
2675 	return (rc);
2676 }
2677 
2678 /*
2679  * Lookup a value in a fatzap directory.
2680  */
2681 static int
2682 fzap_lookup(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2683     const char *name, uint64_t integer_size, uint64_t num_integers,
2684     void *value)
2685 {
2686 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2687 	fat_zap_t z;
2688 	zap_leaf_t *zl;
2689 	uint64_t hash;
2690 	int rc;
2691 
2692 	if (zh->zap_magic != ZAP_MAGIC)
2693 		return (EIO);
2694 
2695 	if ((rc = fzap_check_size(integer_size, num_integers)) != 0) {
2696 		return (rc);
2697 	}
2698 
2699 	z.zap_block_shift = ilog2(bsize);
2700 	z.zap_phys = zh;
2701 	z.zap_spa = spa;
2702 	z.zap_dnode = dnode;
2703 
2704 	hash = zap_hash(zh->zap_salt, name);
2705 	rc = zap_deref_leaf(&z, hash, &zl);
2706 	if (rc != 0)
2707 		return (rc);
2708 
2709 	rc = zap_leaf_lookup(zl, hash, name, integer_size, num_integers, value);
2710 
2711 	zap_leaf_free(zl);
2712 	return (rc);
2713 }
2714 
2715 /*
2716  * Lookup a name in a zap object and return its value as a uint64_t.
2717  */
2718 static int
2719 zap_lookup(const spa_t *spa, const dnode_phys_t *dnode, const char *name,
2720     uint64_t integer_size, uint64_t num_integers, void *value)
2721 {
2722 	int rc;
2723 	zap_phys_t *zap;
2724 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2725 
2726 	zap = malloc(size);
2727 	if (zap == NULL)
2728 		return (ENOMEM);
2729 
2730 	rc = dnode_read(spa, dnode, 0, zap, size);
2731 	if (rc)
2732 		goto done;
2733 
2734 	switch (zap->zap_block_type) {
2735 	case ZBT_MICRO:
2736 		rc = mzap_lookup((const mzap_phys_t *)zap, size, name, value);
2737 		break;
2738 	case ZBT_HEADER:
2739 		rc = fzap_lookup(spa, dnode, zap, name, integer_size,
2740 		    num_integers, value);
2741 		break;
2742 	default:
2743 		printf("ZFS: invalid zap_type=%" PRIx64 "\n",
2744 		    zap->zap_block_type);
2745 		rc = EIO;
2746 	}
2747 done:
2748 	free(zap);
2749 	return (rc);
2750 }
2751 
2752 /*
2753  * List a microzap directory.
2754  */
2755 static int
2756 mzap_list(const mzap_phys_t *mz, size_t size,
2757     int (*callback)(const char *, uint64_t))
2758 {
2759 	const mzap_ent_phys_t *mze;
2760 	int chunks, i, rc;
2761 
2762 	/*
2763 	 * Microzap objects use exactly one block. Read the whole
2764 	 * thing.
2765 	 */
2766 	rc = 0;
2767 	chunks = size / MZAP_ENT_LEN - 1;
2768 	for (i = 0; i < chunks; i++) {
2769 		mze = &mz->mz_chunk[i];
2770 		if (mze->mze_name[0]) {
2771 			rc = callback(mze->mze_name, mze->mze_value);
2772 			if (rc != 0)
2773 				break;
2774 		}
2775 	}
2776 
2777 	return (rc);
2778 }
2779 
2780 /*
2781  * List a fatzap directory.
2782  */
2783 static int
2784 fzap_list(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2785     int (*callback)(const char *, uint64_t))
2786 {
2787 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2788 	fat_zap_t z;
2789 	uint64_t i;
2790 	int j, rc;
2791 
2792 	if (zh->zap_magic != ZAP_MAGIC)
2793 		return (EIO);
2794 
2795 	z.zap_block_shift = ilog2(bsize);
2796 	z.zap_phys = zh;
2797 
2798 	/*
2799 	 * This assumes that the leaf blocks start at block 1. The
2800 	 * documentation isn't exactly clear on this.
2801 	 */
2802 	zap_leaf_t zl;
2803 	zl.l_bs = z.zap_block_shift;
2804 	zl.l_phys = malloc(bsize);
2805 	if (zl.l_phys == NULL)
2806 		return (ENOMEM);
2807 
2808 	for (i = 0; i < zh->zap_num_leafs; i++) {
2809 		off_t off = ((off_t)(i + 1)) << zl.l_bs;
2810 		char name[256], *p;
2811 		uint64_t value;
2812 
2813 		if (dnode_read(spa, dnode, off, zl.l_phys, bsize)) {
2814 			free(zl.l_phys);
2815 			return (EIO);
2816 		}
2817 
2818 		for (j = 0; j < ZAP_LEAF_NUMCHUNKS(&zl); j++) {
2819 			zap_leaf_chunk_t *zc, *nc;
2820 			int namelen;
2821 
2822 			zc = &ZAP_LEAF_CHUNK(&zl, j);
2823 			if (zc->l_entry.le_type != ZAP_CHUNK_ENTRY)
2824 				continue;
2825 			namelen = zc->l_entry.le_name_numints;
2826 			if (namelen > sizeof(name))
2827 				namelen = sizeof(name);
2828 
2829 			/*
2830 			 * Paste the name back together.
2831 			 */
2832 			nc = &ZAP_LEAF_CHUNK(&zl, zc->l_entry.le_name_chunk);
2833 			p = name;
2834 			while (namelen > 0) {
2835 				int len;
2836 				len = namelen;
2837 				if (len > ZAP_LEAF_ARRAY_BYTES)
2838 					len = ZAP_LEAF_ARRAY_BYTES;
2839 				memcpy(p, nc->l_array.la_array, len);
2840 				p += len;
2841 				namelen -= len;
2842 				nc = &ZAP_LEAF_CHUNK(&zl, nc->l_array.la_next);
2843 			}
2844 
2845 			/*
2846 			 * Assume the first eight bytes of the value are
2847 			 * a uint64_t.
2848 			 */
2849 			value = fzap_leaf_value(&zl, zc);
2850 
2851 			/* printf("%s 0x%jx\n", name, (uintmax_t)value); */
2852 			rc = callback((const char *)name, value);
2853 			if (rc != 0) {
2854 				free(zl.l_phys);
2855 				return (rc);
2856 			}
2857 		}
2858 	}
2859 
2860 	free(zl.l_phys);
2861 	return (0);
2862 }
2863 
2864 static int zfs_printf(const char *name, uint64_t value __unused)
2865 {
2866 
2867 	printf("%s\n", name);
2868 
2869 	return (0);
2870 }
2871 
2872 /*
2873  * List a zap directory.
2874  */
2875 static int
2876 zap_list(const spa_t *spa, const dnode_phys_t *dnode)
2877 {
2878 	zap_phys_t *zap;
2879 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2880 	int rc;
2881 
2882 	zap = malloc(size);
2883 	if (zap == NULL)
2884 		return (ENOMEM);
2885 
2886 	rc = dnode_read(spa, dnode, 0, zap, size);
2887 	if (rc == 0) {
2888 		if (zap->zap_block_type == ZBT_MICRO)
2889 			rc = mzap_list((const mzap_phys_t *)zap, size,
2890 			    zfs_printf);
2891 		else
2892 			rc = fzap_list(spa, dnode, zap, zfs_printf);
2893 	}
2894 	free(zap);
2895 	return (rc);
2896 }
2897 
2898 static int
2899 objset_get_dnode(const spa_t *spa, const objset_phys_t *os, uint64_t objnum,
2900     dnode_phys_t *dnode)
2901 {
2902 	off_t offset;
2903 
2904 	offset = objnum * sizeof(dnode_phys_t);
2905 	return dnode_read(spa, &os->os_meta_dnode, offset,
2906 		dnode, sizeof(dnode_phys_t));
2907 }
2908 
2909 /*
2910  * Lookup a name in a microzap directory.
2911  */
2912 static int
2913 mzap_rlookup(const mzap_phys_t *mz, size_t size, char *name, uint64_t value)
2914 {
2915 	const mzap_ent_phys_t *mze;
2916 	int chunks, i;
2917 
2918 	/*
2919 	 * Microzap objects use exactly one block. Read the whole
2920 	 * thing.
2921 	 */
2922 	chunks = size / MZAP_ENT_LEN - 1;
2923 	for (i = 0; i < chunks; i++) {
2924 		mze = &mz->mz_chunk[i];
2925 		if (value == mze->mze_value) {
2926 			strcpy(name, mze->mze_name);
2927 			return (0);
2928 		}
2929 	}
2930 
2931 	return (ENOENT);
2932 }
2933 
2934 static void
2935 fzap_name_copy(const zap_leaf_t *zl, const zap_leaf_chunk_t *zc, char *name)
2936 {
2937 	size_t namelen;
2938 	const zap_leaf_chunk_t *nc;
2939 	char *p;
2940 
2941 	namelen = zc->l_entry.le_name_numints;
2942 
2943 	nc = &ZAP_LEAF_CHUNK(zl, zc->l_entry.le_name_chunk);
2944 	p = name;
2945 	while (namelen > 0) {
2946 		size_t len;
2947 		len = namelen;
2948 		if (len > ZAP_LEAF_ARRAY_BYTES)
2949 			len = ZAP_LEAF_ARRAY_BYTES;
2950 		memcpy(p, nc->l_array.la_array, len);
2951 		p += len;
2952 		namelen -= len;
2953 		nc = &ZAP_LEAF_CHUNK(zl, nc->l_array.la_next);
2954 	}
2955 
2956 	*p = '\0';
2957 }
2958 
2959 static int
2960 fzap_rlookup(const spa_t *spa, const dnode_phys_t *dnode, zap_phys_t *zh,
2961     char *name, uint64_t value)
2962 {
2963 	int bsize = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
2964 	fat_zap_t z;
2965 	uint64_t i;
2966 	int j, rc;
2967 
2968 	if (zh->zap_magic != ZAP_MAGIC)
2969 		return (EIO);
2970 
2971 	z.zap_block_shift = ilog2(bsize);
2972 	z.zap_phys = zh;
2973 
2974 	/*
2975 	 * This assumes that the leaf blocks start at block 1. The
2976 	 * documentation isn't exactly clear on this.
2977 	 */
2978 	zap_leaf_t zl;
2979 	zl.l_bs = z.zap_block_shift;
2980 	zl.l_phys = malloc(bsize);
2981 	if (zl.l_phys == NULL)
2982 		return (ENOMEM);
2983 
2984 	for (i = 0; i < zh->zap_num_leafs; i++) {
2985 		off_t off = ((off_t)(i + 1)) << zl.l_bs;
2986 
2987 		rc = dnode_read(spa, dnode, off, zl.l_phys, bsize);
2988 		if (rc != 0)
2989 			goto done;
2990 
2991 		for (j = 0; j < ZAP_LEAF_NUMCHUNKS(&zl); j++) {
2992 			zap_leaf_chunk_t *zc;
2993 
2994 			zc = &ZAP_LEAF_CHUNK(&zl, j);
2995 			if (zc->l_entry.le_type != ZAP_CHUNK_ENTRY)
2996 				continue;
2997 			if (zc->l_entry.le_value_intlen != 8 ||
2998 			    zc->l_entry.le_value_numints != 1)
2999 				continue;
3000 
3001 			if (fzap_leaf_value(&zl, zc) == value) {
3002 				fzap_name_copy(&zl, zc, name);
3003 				goto done;
3004 			}
3005 		}
3006 	}
3007 
3008 	rc = ENOENT;
3009 done:
3010 	free(zl.l_phys);
3011 	return (rc);
3012 }
3013 
3014 static int
3015 zap_rlookup(const spa_t *spa, const dnode_phys_t *dnode, char *name,
3016     uint64_t value)
3017 {
3018 	zap_phys_t *zap;
3019 	size_t size = dnode->dn_datablkszsec << SPA_MINBLOCKSHIFT;
3020 	int rc;
3021 
3022 	zap = malloc(size);
3023 	if (zap == NULL)
3024 		return (ENOMEM);
3025 
3026 	rc = dnode_read(spa, dnode, 0, zap, size);
3027 	if (rc == 0) {
3028 		if (zap->zap_block_type == ZBT_MICRO)
3029 			rc = mzap_rlookup((const mzap_phys_t *)zap, size,
3030 			    name, value);
3031 		else
3032 			rc = fzap_rlookup(spa, dnode, zap, name, value);
3033 	}
3034 	free(zap);
3035 	return (rc);
3036 }
3037 
3038 static int
3039 zfs_rlookup(const spa_t *spa, uint64_t objnum, char *result)
3040 {
3041 	char name[256];
3042 	char component[256];
3043 	uint64_t dir_obj, parent_obj, child_dir_zapobj;
3044 	dnode_phys_t child_dir_zap, dataset, dir, parent;
3045 	dsl_dir_phys_t *dd;
3046 	dsl_dataset_phys_t *ds;
3047 	char *p;
3048 	int len;
3049 
3050 	p = &name[sizeof(name) - 1];
3051 	*p = '\0';
3052 
3053 	if (objset_get_dnode(spa, spa->spa_mos, objnum, &dataset)) {
3054 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3055 		return (EIO);
3056 	}
3057 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3058 	dir_obj = ds->ds_dir_obj;
3059 
3060 	for (;;) {
3061 		if (objset_get_dnode(spa, spa->spa_mos, dir_obj, &dir) != 0)
3062 			return (EIO);
3063 		dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3064 
3065 		/* Actual loop condition. */
3066 		parent_obj = dd->dd_parent_obj;
3067 		if (parent_obj == 0)
3068 			break;
3069 
3070 		if (objset_get_dnode(spa, spa->spa_mos, parent_obj,
3071 		    &parent) != 0)
3072 			return (EIO);
3073 		dd = (dsl_dir_phys_t *)&parent.dn_bonus;
3074 		child_dir_zapobj = dd->dd_child_dir_zapobj;
3075 		if (objset_get_dnode(spa, spa->spa_mos, child_dir_zapobj,
3076 		    &child_dir_zap) != 0)
3077 			return (EIO);
3078 		if (zap_rlookup(spa, &child_dir_zap, component, dir_obj) != 0)
3079 			return (EIO);
3080 
3081 		len = strlen(component);
3082 		p -= len;
3083 		memcpy(p, component, len);
3084 		--p;
3085 		*p = '/';
3086 
3087 		/* Actual loop iteration. */
3088 		dir_obj = parent_obj;
3089 	}
3090 
3091 	if (*p != '\0')
3092 		++p;
3093 	strcpy(result, p);
3094 
3095 	return (0);
3096 }
3097 
3098 static int
3099 zfs_lookup_dataset(const spa_t *spa, const char *name, uint64_t *objnum)
3100 {
3101 	char element[256];
3102 	uint64_t dir_obj, child_dir_zapobj;
3103 	dnode_phys_t child_dir_zap, dir;
3104 	dsl_dir_phys_t *dd;
3105 	const char *p, *q;
3106 
3107 	if (objset_get_dnode(spa, spa->spa_mos,
3108 	    DMU_POOL_DIRECTORY_OBJECT, &dir))
3109 		return (EIO);
3110 	if (zap_lookup(spa, &dir, DMU_POOL_ROOT_DATASET, sizeof (dir_obj),
3111 	    1, &dir_obj))
3112 		return (EIO);
3113 
3114 	p = name;
3115 	for (;;) {
3116 		if (objset_get_dnode(spa, spa->spa_mos, dir_obj, &dir))
3117 			return (EIO);
3118 		dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3119 
3120 		while (*p == '/')
3121 			p++;
3122 		/* Actual loop condition #1. */
3123 		if (*p == '\0')
3124 			break;
3125 
3126 		q = strchr(p, '/');
3127 		if (q) {
3128 			memcpy(element, p, q - p);
3129 			element[q - p] = '\0';
3130 			p = q + 1;
3131 		} else {
3132 			strcpy(element, p);
3133 			p += strlen(p);
3134 		}
3135 
3136 		child_dir_zapobj = dd->dd_child_dir_zapobj;
3137 		if (objset_get_dnode(spa, spa->spa_mos, child_dir_zapobj,
3138 		    &child_dir_zap) != 0)
3139 			return (EIO);
3140 
3141 		/* Actual loop condition #2. */
3142 		if (zap_lookup(spa, &child_dir_zap, element, sizeof (dir_obj),
3143 		    1, &dir_obj) != 0)
3144 			return (ENOENT);
3145 	}
3146 
3147 	*objnum = dd->dd_head_dataset_obj;
3148 	return (0);
3149 }
3150 
3151 #ifndef BOOT2
3152 static int
3153 zfs_list_dataset(const spa_t *spa, uint64_t objnum/*, int pos, char *entry*/)
3154 {
3155 	uint64_t dir_obj, child_dir_zapobj;
3156 	dnode_phys_t child_dir_zap, dir, dataset;
3157 	dsl_dataset_phys_t *ds;
3158 	dsl_dir_phys_t *dd;
3159 
3160 	if (objset_get_dnode(spa, spa->spa_mos, objnum, &dataset)) {
3161 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3162 		return (EIO);
3163 	}
3164 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3165 	dir_obj = ds->ds_dir_obj;
3166 
3167 	if (objset_get_dnode(spa, spa->spa_mos, dir_obj, &dir)) {
3168 		printf("ZFS: can't find dirobj %ju\n", (uintmax_t)dir_obj);
3169 		return (EIO);
3170 	}
3171 	dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3172 
3173 	child_dir_zapobj = dd->dd_child_dir_zapobj;
3174 	if (objset_get_dnode(spa, spa->spa_mos, child_dir_zapobj,
3175 	    &child_dir_zap) != 0) {
3176 		printf("ZFS: can't find child zap %ju\n", (uintmax_t)dir_obj);
3177 		return (EIO);
3178 	}
3179 
3180 	return (zap_list(spa, &child_dir_zap) != 0);
3181 }
3182 
3183 int
3184 zfs_callback_dataset(const spa_t *spa, uint64_t objnum,
3185     int (*callback)(const char *, uint64_t))
3186 {
3187 	uint64_t dir_obj, child_dir_zapobj;
3188 	dnode_phys_t child_dir_zap, dir, dataset;
3189 	dsl_dataset_phys_t *ds;
3190 	dsl_dir_phys_t *dd;
3191 	zap_phys_t *zap;
3192 	size_t size;
3193 	int err;
3194 
3195 	err = objset_get_dnode(spa, spa->spa_mos, objnum, &dataset);
3196 	if (err != 0) {
3197 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3198 		return (err);
3199 	}
3200 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3201 	dir_obj = ds->ds_dir_obj;
3202 
3203 	err = objset_get_dnode(spa, spa->spa_mos, dir_obj, &dir);
3204 	if (err != 0) {
3205 		printf("ZFS: can't find dirobj %ju\n", (uintmax_t)dir_obj);
3206 		return (err);
3207 	}
3208 	dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3209 
3210 	child_dir_zapobj = dd->dd_child_dir_zapobj;
3211 	err = objset_get_dnode(spa, spa->spa_mos, child_dir_zapobj,
3212 	    &child_dir_zap);
3213 	if (err != 0) {
3214 		printf("ZFS: can't find child zap %ju\n", (uintmax_t)dir_obj);
3215 		return (err);
3216 	}
3217 
3218 	size = child_dir_zap.dn_datablkszsec << SPA_MINBLOCKSHIFT;
3219 	zap = malloc(size);
3220 	if (zap != NULL) {
3221 		err = dnode_read(spa, &child_dir_zap, 0, zap, size);
3222 		if (err != 0)
3223 			goto done;
3224 
3225 		if (zap->zap_block_type == ZBT_MICRO)
3226 			err = mzap_list((const mzap_phys_t *)zap, size,
3227 			    callback);
3228 		else
3229 			err = fzap_list(spa, &child_dir_zap, zap, callback);
3230 	} else {
3231 		err = ENOMEM;
3232 	}
3233 done:
3234 	free(zap);
3235 	return (err);
3236 }
3237 #endif
3238 
3239 /*
3240  * Find the object set given the object number of its dataset object
3241  * and return its details in *objset
3242  */
3243 static int
3244 zfs_mount_dataset(const spa_t *spa, uint64_t objnum, objset_phys_t *objset)
3245 {
3246 	dnode_phys_t dataset;
3247 	dsl_dataset_phys_t *ds;
3248 
3249 	if (objset_get_dnode(spa, spa->spa_mos, objnum, &dataset)) {
3250 		printf("ZFS: can't find dataset %ju\n", (uintmax_t)objnum);
3251 		return (EIO);
3252 	}
3253 
3254 	ds = (dsl_dataset_phys_t *)&dataset.dn_bonus;
3255 	if (zio_read(spa, &ds->ds_bp, objset)) {
3256 		printf("ZFS: can't read object set for dataset %ju\n",
3257 		    (uintmax_t)objnum);
3258 		return (EIO);
3259 	}
3260 
3261 	return (0);
3262 }
3263 
3264 /*
3265  * Find the object set pointed to by the BOOTFS property or the root
3266  * dataset if there is none and return its details in *objset
3267  */
3268 static int
3269 zfs_get_root(const spa_t *spa, uint64_t *objid)
3270 {
3271 	dnode_phys_t dir, propdir;
3272 	uint64_t props, bootfs, root;
3273 
3274 	*objid = 0;
3275 
3276 	/*
3277 	 * Start with the MOS directory object.
3278 	 */
3279 	if (objset_get_dnode(spa, spa->spa_mos,
3280 	    DMU_POOL_DIRECTORY_OBJECT, &dir)) {
3281 		printf("ZFS: can't read MOS object directory\n");
3282 		return (EIO);
3283 	}
3284 
3285 	/*
3286 	 * Lookup the pool_props and see if we can find a bootfs.
3287 	 */
3288 	if (zap_lookup(spa, &dir, DMU_POOL_PROPS,
3289 	    sizeof(props), 1, &props) == 0 &&
3290 	    objset_get_dnode(spa, spa->spa_mos, props, &propdir) == 0 &&
3291 	    zap_lookup(spa, &propdir, "bootfs",
3292 	    sizeof(bootfs), 1, &bootfs) == 0 && bootfs != 0) {
3293 		*objid = bootfs;
3294 		return (0);
3295 	}
3296 	/*
3297 	 * Lookup the root dataset directory
3298 	 */
3299 	if (zap_lookup(spa, &dir, DMU_POOL_ROOT_DATASET,
3300 	    sizeof(root), 1, &root) ||
3301 	    objset_get_dnode(spa, spa->spa_mos, root, &dir)) {
3302 		printf("ZFS: can't find root dsl_dir\n");
3303 		return (EIO);
3304 	}
3305 
3306 	/*
3307 	 * Use the information from the dataset directory's bonus buffer
3308 	 * to find the dataset object and from that the object set itself.
3309 	 */
3310 	dsl_dir_phys_t *dd = (dsl_dir_phys_t *)&dir.dn_bonus;
3311 	*objid = dd->dd_head_dataset_obj;
3312 	return (0);
3313 }
3314 
3315 static int
3316 zfs_mount(const spa_t *spa, uint64_t rootobj, struct zfsmount *mount)
3317 {
3318 
3319 	mount->spa = spa;
3320 
3321 	/*
3322 	 * Find the root object set if not explicitly provided
3323 	 */
3324 	if (rootobj == 0 && zfs_get_root(spa, &rootobj)) {
3325 		printf("ZFS: can't find root filesystem\n");
3326 		return (EIO);
3327 	}
3328 
3329 	if (zfs_mount_dataset(spa, rootobj, &mount->objset)) {
3330 		printf("ZFS: can't open root filesystem\n");
3331 		return (EIO);
3332 	}
3333 
3334 	mount->rootobj = rootobj;
3335 
3336 	return (0);
3337 }
3338 
3339 /*
3340  * callback function for feature name checks.
3341  */
3342 static int
3343 check_feature(const char *name, uint64_t value)
3344 {
3345 	int i;
3346 
3347 	if (value == 0)
3348 		return (0);
3349 	if (name[0] == '\0')
3350 		return (0);
3351 
3352 	for (i = 0; features_for_read[i] != NULL; i++) {
3353 		if (strcmp(name, features_for_read[i]) == 0)
3354 			return (0);
3355 	}
3356 	printf("ZFS: unsupported feature: %s\n", name);
3357 	return (EIO);
3358 }
3359 
3360 /*
3361  * Checks whether the MOS features that are active are supported.
3362  */
3363 static int
3364 check_mos_features(const spa_t *spa)
3365 {
3366 	dnode_phys_t dir;
3367 	zap_phys_t *zap;
3368 	uint64_t objnum;
3369 	size_t size;
3370 	int rc;
3371 
3372 	if ((rc = objset_get_dnode(spa, spa->spa_mos, DMU_OT_OBJECT_DIRECTORY,
3373 	    &dir)) != 0)
3374 		return (rc);
3375 	if ((rc = zap_lookup(spa, &dir, DMU_POOL_FEATURES_FOR_READ,
3376 	    sizeof (objnum), 1, &objnum)) != 0) {
3377 		/*
3378 		 * It is older pool without features. As we have already
3379 		 * tested the label, just return without raising the error.
3380 		 */
3381 		return (0);
3382 	}
3383 
3384 	if ((rc = objset_get_dnode(spa, spa->spa_mos, objnum, &dir)) != 0)
3385 		return (rc);
3386 
3387 	if (dir.dn_type != DMU_OTN_ZAP_METADATA)
3388 		return (EIO);
3389 
3390 	size = dir.dn_datablkszsec << SPA_MINBLOCKSHIFT;
3391 	zap = malloc(size);
3392 	if (zap == NULL)
3393 		return (ENOMEM);
3394 
3395 	if (dnode_read(spa, &dir, 0, zap, size)) {
3396 		free(zap);
3397 		return (EIO);
3398 	}
3399 
3400 	if (zap->zap_block_type == ZBT_MICRO)
3401 		rc = mzap_list((const mzap_phys_t *)zap, size, check_feature);
3402 	else
3403 		rc = fzap_list(spa, &dir, zap, check_feature);
3404 
3405 	free(zap);
3406 	return (rc);
3407 }
3408 
3409 static int
3410 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
3411 {
3412 	dnode_phys_t dir;
3413 	size_t size;
3414 	int rc;
3415 	char *nv;
3416 
3417 	*value = NULL;
3418 	if ((rc = objset_get_dnode(spa, spa->spa_mos, obj, &dir)) != 0)
3419 		return (rc);
3420 	if (dir.dn_type != DMU_OT_PACKED_NVLIST &&
3421 	    dir.dn_bonustype != DMU_OT_PACKED_NVLIST_SIZE) {
3422 		return (EIO);
3423 	}
3424 
3425 	if (dir.dn_bonuslen != sizeof (uint64_t))
3426 		return (EIO);
3427 
3428 	size = *(uint64_t *)DN_BONUS(&dir);
3429 	nv = malloc(size);
3430 	if (nv == NULL)
3431 		return (ENOMEM);
3432 
3433 	rc = dnode_read(spa, &dir, 0, nv, size);
3434 	if (rc != 0) {
3435 		free(nv);
3436 		nv = NULL;
3437 		return (rc);
3438 	}
3439 	*value = nvlist_import(nv, size);
3440 	free(nv);
3441 	return (rc);
3442 }
3443 
3444 static int
3445 zfs_spa_init(spa_t *spa)
3446 {
3447 	struct uberblock checkpoint;
3448 	dnode_phys_t dir;
3449 	uint64_t config_object;
3450 	nvlist_t *nvlist;
3451 	int rc;
3452 
3453 	if (zio_read(spa, &spa->spa_uberblock->ub_rootbp, spa->spa_mos)) {
3454 		printf("ZFS: can't read MOS of pool %s\n", spa->spa_name);
3455 		return (EIO);
3456 	}
3457 	if (spa->spa_mos->os_type != DMU_OST_META) {
3458 		printf("ZFS: corrupted MOS of pool %s\n", spa->spa_name);
3459 		return (EIO);
3460 	}
3461 
3462 	if (objset_get_dnode(spa, &spa->spa_mos_master,
3463 	    DMU_POOL_DIRECTORY_OBJECT, &dir)) {
3464 		printf("ZFS: failed to read pool %s directory object\n",
3465 		    spa->spa_name);
3466 		return (EIO);
3467 	}
3468 	/* this is allowed to fail, older pools do not have salt */
3469 	rc = zap_lookup(spa, &dir, DMU_POOL_CHECKSUM_SALT, 1,
3470 	    sizeof (spa->spa_cksum_salt.zcs_bytes),
3471 	    spa->spa_cksum_salt.zcs_bytes);
3472 
3473 	rc = check_mos_features(spa);
3474 	if (rc != 0) {
3475 		printf("ZFS: pool %s is not supported\n", spa->spa_name);
3476 		return (rc);
3477 	}
3478 
3479 	rc = zap_lookup(spa, &dir, DMU_POOL_CONFIG,
3480 	    sizeof (config_object), 1, &config_object);
3481 	if (rc != 0) {
3482 		printf("ZFS: can not read MOS %s\n", DMU_POOL_CONFIG);
3483 		return (EIO);
3484 	}
3485 	rc = load_nvlist(spa, config_object, &nvlist);
3486 	if (rc != 0)
3487 		return (rc);
3488 
3489 	rc = zap_lookup(spa, &dir, DMU_POOL_ZPOOL_CHECKPOINT,
3490 	    sizeof(uint64_t), sizeof(checkpoint) / sizeof(uint64_t),
3491 	    &checkpoint);
3492 	if (rc == 0 && checkpoint.ub_checkpoint_txg != 0) {
3493 		memcpy(&spa->spa_uberblock_checkpoint, &checkpoint,
3494 		    sizeof(checkpoint));
3495 		if (zio_read(spa, &spa->spa_uberblock_checkpoint.ub_rootbp,
3496 		    &spa->spa_mos_checkpoint)) {
3497 			printf("ZFS: can not read checkpoint data.\n");
3498 			return (EIO);
3499 		}
3500 	}
3501 
3502 	/*
3503 	 * Update vdevs from MOS config. Note, we do skip encoding bytes
3504 	 * here. See also vdev_label_read_config().
3505 	 */
3506 	rc = vdev_init_from_nvlist(spa, nvlist);
3507 	nvlist_destroy(nvlist);
3508 	return (rc);
3509 }
3510 
3511 static int
3512 zfs_dnode_stat(const spa_t *spa, dnode_phys_t *dn, struct stat *sb)
3513 {
3514 
3515 	if (dn->dn_bonustype != DMU_OT_SA) {
3516 		znode_phys_t *zp = (znode_phys_t *)dn->dn_bonus;
3517 
3518 		sb->st_mode = zp->zp_mode;
3519 		sb->st_uid = zp->zp_uid;
3520 		sb->st_gid = zp->zp_gid;
3521 		sb->st_size = zp->zp_size;
3522 	} else {
3523 		sa_hdr_phys_t *sahdrp;
3524 		int hdrsize;
3525 		size_t size = 0;
3526 		void *buf = NULL;
3527 
3528 		if (dn->dn_bonuslen != 0)
3529 			sahdrp = (sa_hdr_phys_t *)DN_BONUS(dn);
3530 		else {
3531 			if ((dn->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0) {
3532 				blkptr_t *bp = DN_SPILL_BLKPTR(dn);
3533 				int error;
3534 
3535 				size = BP_GET_LSIZE(bp);
3536 				buf = malloc(size);
3537 				if (buf == NULL)
3538 					error = ENOMEM;
3539 				else
3540 					error = zio_read(spa, bp, buf);
3541 
3542 				if (error != 0) {
3543 					free(buf);
3544 					return (error);
3545 				}
3546 				sahdrp = buf;
3547 			} else {
3548 				return (EIO);
3549 			}
3550 		}
3551 		hdrsize = SA_HDR_SIZE(sahdrp);
3552 		sb->st_mode = *(uint64_t *)((char *)sahdrp + hdrsize +
3553 		    SA_MODE_OFFSET);
3554 		sb->st_uid = *(uint64_t *)((char *)sahdrp + hdrsize +
3555 		    SA_UID_OFFSET);
3556 		sb->st_gid = *(uint64_t *)((char *)sahdrp + hdrsize +
3557 		    SA_GID_OFFSET);
3558 		sb->st_size = *(uint64_t *)((char *)sahdrp + hdrsize +
3559 		    SA_SIZE_OFFSET);
3560 		free(buf);
3561 	}
3562 
3563 	return (0);
3564 }
3565 
3566 static int
3567 zfs_dnode_readlink(const spa_t *spa, dnode_phys_t *dn, char *path, size_t psize)
3568 {
3569 	int rc = 0;
3570 
3571 	if (dn->dn_bonustype == DMU_OT_SA) {
3572 		sa_hdr_phys_t *sahdrp = NULL;
3573 		size_t size = 0;
3574 		void *buf = NULL;
3575 		int hdrsize;
3576 		char *p;
3577 
3578 		if (dn->dn_bonuslen != 0) {
3579 			sahdrp = (sa_hdr_phys_t *)DN_BONUS(dn);
3580 		} else {
3581 			blkptr_t *bp;
3582 
3583 			if ((dn->dn_flags & DNODE_FLAG_SPILL_BLKPTR) == 0)
3584 				return (EIO);
3585 			bp = DN_SPILL_BLKPTR(dn);
3586 
3587 			size = BP_GET_LSIZE(bp);
3588 			buf = malloc(size);
3589 			if (buf == NULL)
3590 				rc = ENOMEM;
3591 			else
3592 				rc = zio_read(spa, bp, buf);
3593 			if (rc != 0) {
3594 				free(buf);
3595 				return (rc);
3596 			}
3597 			sahdrp = buf;
3598 		}
3599 		hdrsize = SA_HDR_SIZE(sahdrp);
3600 		p = (char *)((uintptr_t)sahdrp + hdrsize + SA_SYMLINK_OFFSET);
3601 		memcpy(path, p, psize);
3602 		free(buf);
3603 		return (0);
3604 	}
3605 	/*
3606 	 * Second test is purely to silence bogus compiler
3607 	 * warning about accessing past the end of dn_bonus.
3608 	 */
3609 	if (psize + sizeof(znode_phys_t) <= dn->dn_bonuslen &&
3610 	    sizeof(znode_phys_t) <= sizeof(dn->dn_bonus)) {
3611 		memcpy(path, &dn->dn_bonus[sizeof(znode_phys_t)], psize);
3612 	} else {
3613 		rc = dnode_read(spa, dn, 0, path, psize);
3614 	}
3615 	return (rc);
3616 }
3617 
3618 struct obj_list {
3619 	uint64_t		objnum;
3620 	STAILQ_ENTRY(obj_list)	entry;
3621 };
3622 
3623 /*
3624  * Lookup a file and return its dnode.
3625  */
3626 static int
3627 zfs_lookup(const struct zfsmount *mount, const char *upath, dnode_phys_t *dnode)
3628 {
3629 	int rc;
3630 	uint64_t objnum;
3631 	const spa_t *spa;
3632 	dnode_phys_t dn;
3633 	const char *p, *q;
3634 	char element[256];
3635 	char path[1024];
3636 	int symlinks_followed = 0;
3637 	struct stat sb;
3638 	struct obj_list *entry, *tentry;
3639 	STAILQ_HEAD(, obj_list) on_cache = STAILQ_HEAD_INITIALIZER(on_cache);
3640 
3641 	spa = mount->spa;
3642 	if (mount->objset.os_type != DMU_OST_ZFS) {
3643 		printf("ZFS: unexpected object set type %ju\n",
3644 		    (uintmax_t)mount->objset.os_type);
3645 		return (EIO);
3646 	}
3647 
3648 	if ((entry = malloc(sizeof(struct obj_list))) == NULL)
3649 		return (ENOMEM);
3650 
3651 	/*
3652 	 * Get the root directory dnode.
3653 	 */
3654 	rc = objset_get_dnode(spa, &mount->objset, MASTER_NODE_OBJ, &dn);
3655 	if (rc) {
3656 		free(entry);
3657 		return (rc);
3658 	}
3659 
3660 	rc = zap_lookup(spa, &dn, ZFS_ROOT_OBJ, sizeof(objnum), 1, &objnum);
3661 	if (rc) {
3662 		free(entry);
3663 		return (rc);
3664 	}
3665 	entry->objnum = objnum;
3666 	STAILQ_INSERT_HEAD(&on_cache, entry, entry);
3667 
3668 	rc = objset_get_dnode(spa, &mount->objset, objnum, &dn);
3669 	if (rc != 0)
3670 		goto done;
3671 
3672 	p = upath;
3673 	while (p && *p) {
3674 		rc = objset_get_dnode(spa, &mount->objset, objnum, &dn);
3675 		if (rc != 0)
3676 			goto done;
3677 
3678 		while (*p == '/')
3679 			p++;
3680 		if (*p == '\0')
3681 			break;
3682 		q = p;
3683 		while (*q != '\0' && *q != '/')
3684 			q++;
3685 
3686 		/* skip dot */
3687 		if (p + 1 == q && p[0] == '.') {
3688 			p++;
3689 			continue;
3690 		}
3691 		/* double dot */
3692 		if (p + 2 == q && p[0] == '.' && p[1] == '.') {
3693 			p += 2;
3694 			if (STAILQ_FIRST(&on_cache) ==
3695 			    STAILQ_LAST(&on_cache, obj_list, entry)) {
3696 				rc = ENOENT;
3697 				goto done;
3698 			}
3699 			entry = STAILQ_FIRST(&on_cache);
3700 			STAILQ_REMOVE_HEAD(&on_cache, entry);
3701 			free(entry);
3702 			objnum = (STAILQ_FIRST(&on_cache))->objnum;
3703 			continue;
3704 		}
3705 		if (q - p + 1 > sizeof(element)) {
3706 			rc = ENAMETOOLONG;
3707 			goto done;
3708 		}
3709 		memcpy(element, p, q - p);
3710 		element[q - p] = 0;
3711 		p = q;
3712 
3713 		if ((rc = zfs_dnode_stat(spa, &dn, &sb)) != 0)
3714 			goto done;
3715 		if (!S_ISDIR(sb.st_mode)) {
3716 			rc = ENOTDIR;
3717 			goto done;
3718 		}
3719 
3720 		rc = zap_lookup(spa, &dn, element, sizeof (objnum), 1, &objnum);
3721 		if (rc)
3722 			goto done;
3723 		objnum = ZFS_DIRENT_OBJ(objnum);
3724 
3725 		if ((entry = malloc(sizeof(struct obj_list))) == NULL) {
3726 			rc = ENOMEM;
3727 			goto done;
3728 		}
3729 		entry->objnum = objnum;
3730 		STAILQ_INSERT_HEAD(&on_cache, entry, entry);
3731 		rc = objset_get_dnode(spa, &mount->objset, objnum, &dn);
3732 		if (rc)
3733 			goto done;
3734 
3735 		/*
3736 		 * Check for symlink.
3737 		 */
3738 		rc = zfs_dnode_stat(spa, &dn, &sb);
3739 		if (rc)
3740 			goto done;
3741 		if (S_ISLNK(sb.st_mode)) {
3742 			if (symlinks_followed > 10) {
3743 				rc = EMLINK;
3744 				goto done;
3745 			}
3746 			symlinks_followed++;
3747 
3748 			/*
3749 			 * Read the link value and copy the tail of our
3750 			 * current path onto the end.
3751 			 */
3752 			if (sb.st_size + strlen(p) + 1 > sizeof(path)) {
3753 				rc = ENAMETOOLONG;
3754 				goto done;
3755 			}
3756 			strcpy(&path[sb.st_size], p);
3757 
3758 			rc = zfs_dnode_readlink(spa, &dn, path, sb.st_size);
3759 			if (rc != 0)
3760 				goto done;
3761 
3762 			/*
3763 			 * Restart with the new path, starting either at
3764 			 * the root or at the parent depending whether or
3765 			 * not the link is relative.
3766 			 */
3767 			p = path;
3768 			if (*p == '/') {
3769 				while (STAILQ_FIRST(&on_cache) !=
3770 				    STAILQ_LAST(&on_cache, obj_list, entry)) {
3771 					entry = STAILQ_FIRST(&on_cache);
3772 					STAILQ_REMOVE_HEAD(&on_cache, entry);
3773 					free(entry);
3774 				}
3775 			} else {
3776 				entry = STAILQ_FIRST(&on_cache);
3777 				STAILQ_REMOVE_HEAD(&on_cache, entry);
3778 				free(entry);
3779 			}
3780 			objnum = (STAILQ_FIRST(&on_cache))->objnum;
3781 		}
3782 	}
3783 
3784 	*dnode = dn;
3785 done:
3786 	STAILQ_FOREACH_SAFE(entry, &on_cache, entry, tentry)
3787 		free(entry);
3788 	return (rc);
3789 }
3790