xref: /titanic_52/usr/src/lib/libbe/common/be_mount.c (revision 3608e2e0cad618747057828ec19600048ef41025)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25  * Copyright 2015 EveryCity Ltd.
26  * Copyright (c) 2015 by Delphix. All rights reserved.
27  */
28 
29 /*
30  * System includes
31  */
32 #include <assert.h>
33 #include <errno.h>
34 #include <libgen.h>
35 #include <libintl.h>
36 #include <libnvpair.h>
37 #include <libzfs.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sys/mntent.h>
42 #include <sys/mnttab.h>
43 #include <sys/mount.h>
44 #include <sys/stat.h>
45 #include <sys/types.h>
46 #include <sys/vfstab.h>
47 #include <sys/zone.h>
48 #include <sys/mkdev.h>
49 #include <unistd.h>
50 
51 #include <libbe.h>
52 #include <libbe_priv.h>
53 
54 #define	BE_TMP_MNTPNT		"/tmp/.be.XXXXXX"
55 
56 typedef struct dir_data {
57 	char *dir;
58 	char *ds;
59 } dir_data_t;
60 
61 /* Private function prototypes */
62 static int be_mount_callback(zfs_handle_t *, void *);
63 static int be_unmount_callback(zfs_handle_t *, void *);
64 static int be_get_legacy_fs_callback(zfs_handle_t *, void *);
65 static int fix_mountpoint(zfs_handle_t *);
66 static int fix_mountpoint_callback(zfs_handle_t *, void *);
67 static int get_mountpoint_from_vfstab(char *, const char *, char *, size_t,
68     boolean_t);
69 static int loopback_mount_shared_fs(zfs_handle_t *, be_mount_data_t *);
70 static int loopback_mount_zonepath(const char *, be_mount_data_t *);
71 static int iter_shared_fs_callback(zfs_handle_t *, void *);
72 static int zpool_shared_fs_callback(zpool_handle_t *, void *);
73 static int unmount_shared_fs(be_unmount_data_t *);
74 static int add_to_fs_list(be_fs_list_data_t *, const char *);
75 static int be_mount_root(zfs_handle_t *, char *);
76 static int be_unmount_root(zfs_handle_t *, be_unmount_data_t *);
77 static int be_mount_zones(zfs_handle_t *, be_mount_data_t *);
78 static int be_unmount_zones(be_unmount_data_t *);
79 static int be_mount_one_zone(zfs_handle_t *, be_mount_data_t *, char *, char *,
80     char *);
81 static int be_unmount_one_zone(be_unmount_data_t *, char *, char *, char *);
82 static int be_get_ds_from_dir_callback(zfs_handle_t *, void *);
83 
84 
85 /* ********************************************************************	*/
86 /*			Public Functions				*/
87 /* ********************************************************************	*/
88 
89 /*
90  * Function:	be_mount
91  * Description:	Mounts a BE and its subordinate datasets at a given mountpoint.
92  * Parameters:
93  *		be_attrs - pointer to nvlist_t of attributes being passed in.
94  *			The following attributes are used by this function:
95  *
96  *			BE_ATTR_ORIG_BE_NAME		*required
97  *			BE_ATTR_MOUNTPOINT		*required
98  *			BE_ATTR_MOUNT_FLAGS		*optional
99  * Return:
100  *		BE_SUCCESS - Success
101  *		be_errno_t - Failure
102  * Scope:
103  *		Public
104  */
105 int
106 be_mount(nvlist_t *be_attrs)
107 {
108 	char		*be_name = NULL;
109 	char		*mountpoint = NULL;
110 	uint16_t	flags = 0;
111 	int		ret = BE_SUCCESS;
112 
113 	/* Initialize libzfs handle */
114 	if (!be_zfs_init())
115 		return (BE_ERR_INIT);
116 
117 	/* Get original BE name */
118 	if (nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_NAME, &be_name)
119 	    != 0) {
120 		be_print_err(gettext("be_mount: failed to lookup "
121 		    "BE_ATTR_ORIG_BE_NAME attribute\n"));
122 		return (BE_ERR_INVAL);
123 	}
124 
125 	/* Validate original BE name */
126 	if (!be_valid_be_name(be_name)) {
127 		be_print_err(gettext("be_mount: invalid BE name %s\n"),
128 		    be_name);
129 		return (BE_ERR_INVAL);
130 	}
131 
132 	/* Get mountpoint */
133 	if (nvlist_lookup_string(be_attrs, BE_ATTR_MOUNTPOINT, &mountpoint)
134 	    != 0) {
135 		be_print_err(gettext("be_mount: failed to lookup "
136 		    "BE_ATTR_MOUNTPOINT attribute\n"));
137 		return (BE_ERR_INVAL);
138 	}
139 
140 	/* Get flags */
141 	if (nvlist_lookup_pairs(be_attrs, NV_FLAG_NOENTOK,
142 	    BE_ATTR_MOUNT_FLAGS, DATA_TYPE_UINT16, &flags, NULL) != 0) {
143 		be_print_err(gettext("be_mount: failed to lookup "
144 		    "BE_ATTR_MOUNT_FLAGS attribute\n"));
145 		return (BE_ERR_INVAL);
146 	}
147 
148 	ret = _be_mount(be_name, &mountpoint, flags);
149 
150 	be_zfs_fini();
151 
152 	return (ret);
153 }
154 
155 /*
156  * Function:	be_unmount
157  * Description:	Unmounts a BE and its subordinate datasets.
158  * Parameters:
159  *		be_attrs - pointer to nvlist_t of attributes being passed in.
160  *			The following attributes are used by this function:
161  *
162  *			BE_ATTR_ORIG_BE_NAME		*required
163  *			BE_ATTR_UNMOUNT_FLAGS		*optional
164  * Return:
165  *		BE_SUCCESS - Success
166  *		be_errno_t - Failure
167  * Scope:
168  *		Public
169  */
170 int
171 be_unmount(nvlist_t *be_attrs)
172 {
173 	char		*be_name = NULL;
174 	char		*be_name_mnt = NULL;
175 	char		*ds = NULL;
176 	uint16_t	flags = 0;
177 	int		ret = BE_SUCCESS;
178 
179 	/* Initialize libzfs handle */
180 	if (!be_zfs_init())
181 		return (BE_ERR_INIT);
182 
183 	/* Get original BE name */
184 	if (nvlist_lookup_string(be_attrs, BE_ATTR_ORIG_BE_NAME, &be_name)
185 	    != 0) {
186 		be_print_err(gettext("be_unmount: failed to lookup "
187 		    "BE_ATTR_ORIG_BE_NAME attribute\n"));
188 		return (BE_ERR_INVAL);
189 	}
190 
191 	/* Check if we have mountpoint argument instead of BE name */
192 	if (be_name[0] == '/') {
193 		if ((ds = be_get_ds_from_dir(be_name)) != NULL) {
194 			if ((be_name_mnt = strrchr(ds, '/')) != NULL) {
195 				be_name = be_name_mnt + 1;
196 			}
197 		} else {
198 			be_print_err(gettext("be_unmount: no datasets mounted "
199 			    "at '%s'\n"), be_name);
200 			return (BE_ERR_INVAL);
201 		}
202 	}
203 
204 	/* Validate original BE name */
205 	if (!be_valid_be_name(be_name)) {
206 		be_print_err(gettext("be_unmount: invalid BE name %s\n"),
207 		    be_name);
208 		return (BE_ERR_INVAL);
209 	}
210 
211 	/* Get unmount flags */
212 	if (nvlist_lookup_pairs(be_attrs, NV_FLAG_NOENTOK,
213 	    BE_ATTR_UNMOUNT_FLAGS, DATA_TYPE_UINT16, &flags, NULL) != 0) {
214 		be_print_err(gettext("be_unmount: failed to loookup "
215 		    "BE_ATTR_UNMOUNT_FLAGS attribute\n"));
216 		return (BE_ERR_INVAL);
217 	}
218 
219 	ret = _be_unmount(be_name, flags);
220 
221 	be_zfs_fini();
222 
223 	return (ret);
224 }
225 
226 /* ********************************************************************	*/
227 /*			Semi-Private Functions				*/
228 /* ******************************************************************** */
229 
230 /*
231  * Function:	_be_mount
232  * Description:	Mounts a BE.  If the altroot is not provided, this function
233  *		will generate a temporary mountpoint to mount the BE at.  It
234  *		will return this temporary mountpoint to the caller via the
235  *		altroot reference pointer passed in.  This returned value is
236  *		allocated on heap storage and is the repsonsibility of the
237  *		caller to free.
238  * Parameters:
239  *		be_name - pointer to name of BE to mount.
240  *		altroot - reference pointer to altroot of where to mount BE.
241  *		flags - flag indicating special handling for mounting the BE
242  * Return:
243  *		BE_SUCCESS - Success
244  *		be_errno_t - Failure
245  * Scope:
246  *		Semi-private (library wide use only)
247  */
248 int
249 _be_mount(char *be_name, char **altroot, int flags)
250 {
251 	be_transaction_data_t	bt = { 0 };
252 	be_mount_data_t	md = { 0 };
253 	zfs_handle_t	*zhp;
254 	char		obe_root_ds[MAXPATHLEN];
255 	char		*mp = NULL;
256 	char		*tmp_altroot = NULL;
257 	int		ret = BE_SUCCESS, err = 0;
258 	uuid_t		uu = { 0 };
259 	boolean_t	gen_tmp_altroot = B_FALSE;
260 
261 	if (be_name == NULL || altroot == NULL)
262 		return (BE_ERR_INVAL);
263 
264 	/* Set be_name as obe_name in bt structure */
265 	bt.obe_name = be_name;
266 
267 	/* Find which zpool obe_name lives in */
268 	if ((err = zpool_iter(g_zfs, be_find_zpool_callback, &bt)) == 0) {
269 		be_print_err(gettext("be_mount: failed to "
270 		    "find zpool for BE (%s)\n"), bt.obe_name);
271 		return (BE_ERR_BE_NOENT);
272 	} else if (err < 0) {
273 		be_print_err(gettext("be_mount: zpool_iter failed: %s\n"),
274 		    libzfs_error_description(g_zfs));
275 		return (zfs_err_to_be_err(g_zfs));
276 	}
277 
278 	/* Generate string for obe_name's root dataset */
279 	be_make_root_ds(bt.obe_zpool, bt.obe_name, obe_root_ds,
280 	    sizeof (obe_root_ds));
281 	bt.obe_root_ds = obe_root_ds;
282 
283 	/* Get handle to BE's root dataset */
284 	if ((zhp = zfs_open(g_zfs, bt.obe_root_ds, ZFS_TYPE_FILESYSTEM)) ==
285 	    NULL) {
286 		be_print_err(gettext("be_mount: failed to "
287 		    "open BE root dataset (%s): %s\n"), bt.obe_root_ds,
288 		    libzfs_error_description(g_zfs));
289 		return (zfs_err_to_be_err(g_zfs));
290 	}
291 
292 	/* Make sure BE's root dataset isn't already mounted somewhere */
293 	if (zfs_is_mounted(zhp, &mp)) {
294 		ZFS_CLOSE(zhp);
295 		be_print_err(gettext("be_mount: %s is already mounted "
296 		    "at %s\n"), bt.obe_name, mp != NULL ? mp : "");
297 		free(mp);
298 		return (BE_ERR_MOUNTED);
299 	}
300 
301 	/*
302 	 * Fix this BE's mountpoint if its root dataset isn't set to
303 	 * either 'legacy' or '/'.
304 	 */
305 	if ((ret = fix_mountpoint(zhp)) != BE_SUCCESS) {
306 		be_print_err(gettext("be_mount: mountpoint check "
307 		    "failed for %s\n"), bt.obe_root_ds);
308 		ZFS_CLOSE(zhp);
309 		return (ret);
310 	}
311 
312 	/*
313 	 * If altroot not provided, create a temporary alternate root
314 	 * to mount on
315 	 */
316 	if (*altroot == NULL) {
317 		if ((ret = be_make_tmp_mountpoint(&tmp_altroot))
318 		    != BE_SUCCESS) {
319 			be_print_err(gettext("be_mount: failed to "
320 			    "make temporary mountpoint\n"));
321 			ZFS_CLOSE(zhp);
322 			return (ret);
323 		}
324 		gen_tmp_altroot = B_TRUE;
325 	} else {
326 		tmp_altroot = *altroot;
327 	}
328 
329 	md.altroot = tmp_altroot;
330 	md.shared_fs = flags & BE_MOUNT_FLAG_SHARED_FS;
331 	md.shared_rw = flags & BE_MOUNT_FLAG_SHARED_RW;
332 
333 	/* Mount the BE's root file system */
334 	if (getzoneid() == GLOBAL_ZONEID) {
335 		if ((ret = be_mount_root(zhp, tmp_altroot)) != BE_SUCCESS) {
336 			be_print_err(gettext("be_mount: failed to "
337 			    "mount BE root file system\n"));
338 			if (gen_tmp_altroot)
339 				free(tmp_altroot);
340 			ZFS_CLOSE(zhp);
341 			return (ret);
342 		}
343 	} else {
344 		/* Legacy mount the zone root dataset */
345 		if ((ret = be_mount_zone_root(zhp, &md)) != BE_SUCCESS) {
346 			be_print_err(gettext("be_mount: failed to "
347 			    "mount BE zone root file system\n"));
348 			free(md.altroot);
349 			ZFS_CLOSE(zhp);
350 			return (ret);
351 		}
352 	}
353 
354 	/* Iterate through BE's children filesystems */
355 	if ((err = zfs_iter_filesystems(zhp, be_mount_callback,
356 	    tmp_altroot)) != 0) {
357 		be_print_err(gettext("be_mount: failed to "
358 		    "mount BE (%s) on %s\n"), bt.obe_name, tmp_altroot);
359 		if (gen_tmp_altroot)
360 			free(tmp_altroot);
361 		ZFS_CLOSE(zhp);
362 		return (err);
363 	}
364 
365 	/*
366 	 * Mount shared file systems if mount flag says so.
367 	 */
368 	if (md.shared_fs) {
369 		/*
370 		 * Mount all ZFS file systems not under the BE's root dataset
371 		 */
372 		(void) zpool_iter(g_zfs, zpool_shared_fs_callback, &md);
373 
374 		/* TODO: Mount all non-ZFS file systems - Not supported yet */
375 	}
376 
377 	/*
378 	 * If we're in the global zone and the global zone has a valid uuid,
379 	 * mount all supported non-global zones.
380 	 */
381 	if (getzoneid() == GLOBAL_ZONEID &&
382 	    !(flags & BE_MOUNT_FLAG_NO_ZONES) &&
383 	    be_get_uuid(bt.obe_root_ds, &uu) == BE_SUCCESS) {
384 		if (be_mount_zones(zhp, &md) != BE_SUCCESS) {
385 			ret = BE_ERR_NO_MOUNTED_ZONE;
386 		}
387 	}
388 
389 	ZFS_CLOSE(zhp);
390 
391 	/*
392 	 * If a NULL altroot was passed in, pass the generated altroot
393 	 * back to the caller in altroot.
394 	 */
395 	if (gen_tmp_altroot) {
396 		if (ret == BE_SUCCESS || ret == BE_ERR_NO_MOUNTED_ZONE)
397 			*altroot = tmp_altroot;
398 		else
399 			free(tmp_altroot);
400 	}
401 
402 	return (ret);
403 }
404 
405 /*
406  * Function:	_be_unmount
407  * Description:	Unmount a BE.
408  * Parameters:
409  *		be_name - pointer to name of BE to unmount.
410  *		flags - flags for unmounting the BE.
411  * Returns:
412  *		BE_SUCCESS - Success
413  *		be_errno_t - Failure
414  * Scope:
415  *		Semi-private (library wide use only)
416  */
417 int
418 _be_unmount(char *be_name, int flags)
419 {
420 	be_transaction_data_t	bt = { 0 };
421 	be_unmount_data_t	ud = { 0 };
422 	zfs_handle_t	*zhp;
423 	uuid_t		uu = { 0 };
424 	char		obe_root_ds[MAXPATHLEN];
425 	char		mountpoint[MAXPATHLEN];
426 	char		*mp = NULL;
427 	int		ret = BE_SUCCESS;
428 	int		zret = 0;
429 
430 	if (be_name == NULL)
431 		return (BE_ERR_INVAL);
432 
433 	/* Set be_name as obe_name in bt structure */
434 	bt.obe_name = be_name;
435 
436 	/* Find which zpool obe_name lives in */
437 	if ((zret = zpool_iter(g_zfs, be_find_zpool_callback, &bt)) == 0) {
438 		be_print_err(gettext("be_unmount: failed to "
439 		    "find zpool for BE (%s)\n"), bt.obe_name);
440 		return (BE_ERR_BE_NOENT);
441 	} else if (zret < 0) {
442 		be_print_err(gettext("be_unmount: "
443 		    "zpool_iter failed: %s\n"),
444 		    libzfs_error_description(g_zfs));
445 		ret = zfs_err_to_be_err(g_zfs);
446 		return (ret);
447 	}
448 
449 	/* Generate string for obe_name's root dataset */
450 	be_make_root_ds(bt.obe_zpool, bt.obe_name, obe_root_ds,
451 	    sizeof (obe_root_ds));
452 	bt.obe_root_ds = obe_root_ds;
453 
454 	/* Get handle to BE's root dataset */
455 	if ((zhp = zfs_open(g_zfs, bt.obe_root_ds, ZFS_TYPE_FILESYSTEM)) ==
456 	    NULL) {
457 		be_print_err(gettext("be_unmount: failed to "
458 		    "open BE root dataset (%s): %s\n"), bt.obe_root_ds,
459 		    libzfs_error_description(g_zfs));
460 		ret = zfs_err_to_be_err(g_zfs);
461 		return (ret);
462 	}
463 
464 	/* Make sure BE's root dataset is mounted somewhere */
465 	if (!zfs_is_mounted(zhp, &mp)) {
466 
467 		be_print_err(gettext("be_unmount: "
468 		    "(%s) not mounted\n"), bt.obe_name);
469 
470 		/*
471 		 * BE is not mounted, fix this BE's mountpoint if its root
472 		 * dataset isn't set to either 'legacy' or '/'.
473 		 */
474 		if ((ret = fix_mountpoint(zhp)) != BE_SUCCESS) {
475 			be_print_err(gettext("be_unmount: mountpoint check "
476 			    "failed for %s\n"), bt.obe_root_ds);
477 			ZFS_CLOSE(zhp);
478 			return (ret);
479 		}
480 
481 		ZFS_CLOSE(zhp);
482 		return (BE_ERR_NOTMOUNTED);
483 	}
484 
485 	/*
486 	 * If we didn't get a mountpoint from the zfs_is_mounted call,
487 	 * try and get it from its property.
488 	 */
489 	if (mp == NULL) {
490 		if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
491 		    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
492 			be_print_err(gettext("be_unmount: failed to "
493 			    "get mountpoint of (%s)\n"), bt.obe_name);
494 			ZFS_CLOSE(zhp);
495 			return (BE_ERR_ZFS);
496 		}
497 	} else {
498 		(void) strlcpy(mountpoint, mp, sizeof (mountpoint));
499 		free(mp);
500 	}
501 
502 	/* If BE mounted as current root, fail */
503 	if (strcmp(mountpoint, "/") == 0) {
504 		be_print_err(gettext("be_unmount: "
505 		    "cannot unmount currently running BE\n"));
506 		ZFS_CLOSE(zhp);
507 		return (BE_ERR_UMOUNT_CURR_BE);
508 	}
509 
510 	ud.altroot = mountpoint;
511 	ud.force = flags & BE_UNMOUNT_FLAG_FORCE;
512 
513 	/* Unmount all supported non-global zones if we're in the global zone */
514 	if (getzoneid() == GLOBAL_ZONEID &&
515 	    be_get_uuid(bt.obe_root_ds, &uu) == BE_SUCCESS) {
516 		if ((ret = be_unmount_zones(&ud)) != BE_SUCCESS) {
517 			ZFS_CLOSE(zhp);
518 			return (ret);
519 		}
520 	}
521 
522 	/* TODO: Unmount all non-ZFS file systems - Not supported yet */
523 
524 	/* Unmount all ZFS file systems not under the BE root dataset */
525 	if ((ret = unmount_shared_fs(&ud)) != BE_SUCCESS) {
526 		be_print_err(gettext("be_unmount: failed to "
527 		    "unmount shared file systems\n"));
528 		ZFS_CLOSE(zhp);
529 		return (ret);
530 	}
531 
532 	/* Unmount all children datasets under the BE's root dataset */
533 	if ((zret = zfs_iter_filesystems(zhp, be_unmount_callback,
534 	    &ud)) != 0) {
535 		be_print_err(gettext("be_unmount: failed to "
536 		    "unmount BE (%s)\n"), bt.obe_name);
537 		ZFS_CLOSE(zhp);
538 		return (zret);
539 	}
540 
541 	/* Unmount this BE's root filesystem */
542 	if (getzoneid() == GLOBAL_ZONEID) {
543 		if ((ret = be_unmount_root(zhp, &ud)) != BE_SUCCESS) {
544 			ZFS_CLOSE(zhp);
545 			return (ret);
546 		}
547 	} else {
548 		if ((ret = be_unmount_zone_root(zhp, &ud)) != BE_SUCCESS) {
549 			ZFS_CLOSE(zhp);
550 			return (ret);
551 		}
552 	}
553 
554 	ZFS_CLOSE(zhp);
555 
556 	return (BE_SUCCESS);
557 }
558 
559 /*
560  * Function:	be_mount_zone_root
561  * Description:	Mounts the zone root dataset for a zone.
562  * Parameters:
563  *		zfs - zfs_handle_t pointer to zone root dataset
564  *		md - be_mount_data_t pointer to data for zone to be mounted
565  * Returns:
566  *		BE_SUCCESS - Success
567  *		be_errno_t - Failure
568  * Scope:
569  *		Semi-private (library wide use only)
570  */
571 int
572 be_mount_zone_root(zfs_handle_t *zhp, be_mount_data_t *md)
573 {
574 	struct stat buf;
575 	char	mountpoint[MAXPATHLEN];
576 	int	err = 0;
577 
578 	/* Get mountpoint property of dataset */
579 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
580 	    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
581 		be_print_err(gettext("be_mount_zone_root: failed to "
582 		    "get mountpoint property for %s: %s\n"), zfs_get_name(zhp),
583 		    libzfs_error_description(g_zfs));
584 		return (zfs_err_to_be_err(g_zfs));
585 	}
586 
587 	/*
588 	 * Make sure zone's root dataset is set to 'legacy'.  This is
589 	 * currently a requirement in this implementation of zones
590 	 * support.
591 	 */
592 	if (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0) {
593 		be_print_err(gettext("be_mount_zone_root: "
594 		    "zone root dataset mountpoint is not 'legacy'\n"));
595 		return (BE_ERR_ZONE_ROOT_NOT_LEGACY);
596 	}
597 
598 	/* Create the mountpoint if it doesn't exist */
599 	if (lstat(md->altroot, &buf) != 0) {
600 		if (mkdirp(md->altroot, 0755) != 0) {
601 			err = errno;
602 			be_print_err(gettext("be_mount_zone_root: failed "
603 			    "to create mountpoint %s\n"), md->altroot);
604 			return (errno_to_be_err(err));
605 		}
606 	}
607 
608 	/*
609 	 * Legacy mount the zone root dataset.
610 	 *
611 	 * As a workaround for 6176743, we mount the zone's root with the
612 	 * MS_OVERLAY option in case an alternate BE is mounted, and we're
613 	 * mounting the root for the zone from the current BE here.  When an
614 	 * alternate BE is mounted, it ties up the zone's zoneroot directory
615 	 * for the current BE since the zone's zonepath is loopback mounted
616 	 * from the current BE.
617 	 *
618 	 * TODO: The MS_OVERLAY option needs to be removed when 6176743
619 	 * is fixed.
620 	 */
621 	if (mount(zfs_get_name(zhp), md->altroot, MS_OVERLAY, MNTTYPE_ZFS,
622 	    NULL, 0, NULL, 0) != 0) {
623 		err = errno;
624 		be_print_err(gettext("be_mount_zone_root: failed to "
625 		    "legacy mount zone root dataset (%s) at %s\n"),
626 		    zfs_get_name(zhp), md->altroot);
627 		return (errno_to_be_err(err));
628 	}
629 
630 	return (BE_SUCCESS);
631 }
632 
633 /*
634  * Function:	be_unmount_zone_root
635  * Description:	Unmounts the zone root dataset for a zone.
636  * Parameters:
637  *		zhp - zfs_handle_t pointer to zone root dataset
638  *		ud - be_unmount_data_t pointer to data for zone to be unmounted
639  * Returns:
640  *		BE_SUCCESS - Success
641  *		be_errno_t - Failure
642  * Scope:
643  *		Semi-private (library wise use only)
644  */
645 int
646 be_unmount_zone_root(zfs_handle_t *zhp, be_unmount_data_t *ud)
647 {
648 	char	mountpoint[MAXPATHLEN];
649 
650 	/* Unmount the dataset */
651 	if (zfs_unmount(zhp, NULL, ud->force ? MS_FORCE : 0) != 0) {
652 		be_print_err(gettext("be_unmount_zone_root: failed to "
653 		    "unmount zone root dataset %s: %s\n"), zfs_get_name(zhp),
654 		    libzfs_error_description(g_zfs));
655 		return (zfs_err_to_be_err(g_zfs));
656 	}
657 
658 	/* Get the current mountpoint property for the zone root dataset */
659 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
660 	    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
661 		be_print_err(gettext("be_unmount_zone_root: failed to "
662 		    "get mountpoint property for zone root dataset (%s): %s\n"),
663 		    zfs_get_name(zhp), libzfs_error_description(g_zfs));
664 		return (zfs_err_to_be_err(g_zfs));
665 	}
666 
667 	/* If mountpoint not already set to 'legacy', set it to 'legacy' */
668 	if (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0) {
669 		if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
670 		    ZFS_MOUNTPOINT_LEGACY) != 0) {
671 			be_print_err(gettext("be_unmount_zone_root: "
672 			    "failed to set mountpoint of zone root dataset "
673 			    "%s to 'legacy': %s\n"), zfs_get_name(zhp),
674 			    libzfs_error_description(g_zfs));
675 			return (zfs_err_to_be_err(g_zfs));
676 		}
677 	}
678 
679 	return (BE_SUCCESS);
680 }
681 
682 /*
683  * Function:	be_get_legacy_fs
684  * Description:	This function iterates through all non-shared file systems
685  *		of a BE and finds the ones with a legacy mountpoint.  For
686  *		those file systems, it reads the BE's vfstab to get the
687  *		mountpoint.  If found, it adds that file system to the
688  *		be_fs_list_data_t passed in.
689  *
690  *		This function can be used to gather legacy mounted file systems
691  *		for both global BEs and non-global zone BEs.  To get data for
692  *		a non-global zone BE, the zoneroot_ds and zoneroot parameters
693  *		will be specified, otherwise they should be set to NULL.
694  * Parameters:
695  *		be_name - global BE name from which to get legacy file
696  *			system list.
697  *		be_root_ds - root dataset of global BE.
698  *		zoneroot_ds - root dataset of zone.
699  *		zoneroot - zoneroot path of zone.
700  *		fld - be_fs_list_data_t pointer.
701  * Returns:
702  *		BE_SUCCESS - Success
703  *		be_errno_t - Failure
704  * Scope:
705  *		Semi-private (library wide use only)
706  */
707 int
708 be_get_legacy_fs(char *be_name, char *be_root_ds, char *zoneroot_ds,
709     char *zoneroot, be_fs_list_data_t *fld)
710 {
711 	zfs_handle_t		*zhp = NULL;
712 	char			mountpoint[MAXPATHLEN];
713 	boolean_t		mounted_here = B_FALSE;
714 	boolean_t		zone_mounted_here = B_FALSE;
715 	int			ret = BE_SUCCESS, err = 0;
716 
717 	if (be_name == NULL || be_root_ds == NULL || fld == NULL)
718 		return (BE_ERR_INVAL);
719 
720 	/* Get handle to BE's root dataset */
721 	if ((zhp = zfs_open(g_zfs, be_root_ds, ZFS_TYPE_FILESYSTEM))
722 	    == NULL) {
723 		be_print_err(gettext("be_get_legacy_fs: failed to "
724 		    "open BE root dataset (%s): %s\n"), be_root_ds,
725 		    libzfs_error_description(g_zfs));
726 		ret = zfs_err_to_be_err(g_zfs);
727 		return (ret);
728 	}
729 
730 	/* If BE is not already mounted, mount it. */
731 	if (!zfs_is_mounted(zhp, &fld->altroot)) {
732 		if ((ret = _be_mount(be_name, &fld->altroot,
733 		    zoneroot_ds ? BE_MOUNT_FLAG_NULL :
734 		    BE_MOUNT_FLAG_NO_ZONES)) != BE_SUCCESS) {
735 			be_print_err(gettext("be_get_legacy_fs: "
736 			    "failed to mount BE %s\n"), be_name);
737 			goto cleanup;
738 		}
739 
740 		mounted_here = B_TRUE;
741 	} else if (fld->altroot == NULL) {
742 		be_print_err(gettext("be_get_legacy_fs: failed to "
743 		    "get altroot of mounted BE %s: %s\n"),
744 		    be_name, libzfs_error_description(g_zfs));
745 		ret = zfs_err_to_be_err(g_zfs);
746 		goto cleanup;
747 	}
748 
749 	/*
750 	 * If a zone root dataset was passed in, we're wanting to get
751 	 * legacy mounted file systems for that zone, not the global
752 	 * BE.
753 	 */
754 	if (zoneroot_ds != NULL) {
755 		be_mount_data_t		zone_md = { 0 };
756 
757 		/* Close off handle to global BE's root dataset */
758 		ZFS_CLOSE(zhp);
759 
760 		/* Get handle to zone's root dataset */
761 		if ((zhp = zfs_open(g_zfs, zoneroot_ds,
762 		    ZFS_TYPE_FILESYSTEM)) == NULL) {
763 			be_print_err(gettext("be_get_legacy_fs: failed to "
764 			    "open zone BE root dataset (%s): %s\n"),
765 			    zoneroot_ds, libzfs_error_description(g_zfs));
766 			ret = zfs_err_to_be_err(g_zfs);
767 			goto cleanup;
768 		}
769 
770 		/* Make sure the zone we're looking for is mounted */
771 		if (!zfs_is_mounted(zhp, &zone_md.altroot)) {
772 			char	zone_altroot[MAXPATHLEN];
773 
774 			/* Generate alternate root path for zone */
775 			(void) snprintf(zone_altroot, sizeof (zone_altroot),
776 			    "%s%s", fld->altroot, zoneroot);
777 			if ((zone_md.altroot = strdup(zone_altroot)) == NULL) {
778 				be_print_err(gettext("be_get_legacy_fs: "
779 				    "memory allocation failed\n"));
780 				ret = BE_ERR_NOMEM;
781 				goto cleanup;
782 			}
783 
784 			if ((ret = be_mount_zone_root(zhp, &zone_md))
785 			    != BE_SUCCESS) {
786 				be_print_err(gettext("be_get_legacy_fs: "
787 				    "failed to mount zone root %s\n"),
788 				    zoneroot_ds);
789 				free(zone_md.altroot);
790 				zone_md.altroot = NULL;
791 				goto cleanup;
792 			}
793 			zone_mounted_here = B_TRUE;
794 		}
795 
796 		free(fld->altroot);
797 		fld->altroot = zone_md.altroot;
798 	}
799 
800 	/*
801 	 * If the root dataset is in the vfstab with a mountpoint of "/",
802 	 * add it to the list
803 	 */
804 	if (get_mountpoint_from_vfstab(fld->altroot, zfs_get_name(zhp),
805 	    mountpoint, sizeof (mountpoint), B_FALSE) == BE_SUCCESS) {
806 		if (strcmp(mountpoint, "/") == 0) {
807 			if (add_to_fs_list(fld, zfs_get_name(zhp))
808 			    != BE_SUCCESS) {
809 				be_print_err(gettext("be_get_legacy_fs: "
810 				    "failed to add %s to fs list\n"),
811 				    zfs_get_name(zhp));
812 				ret = BE_ERR_INVAL;
813 				goto cleanup;
814 			}
815 		}
816 	}
817 
818 	/* Iterate subordinate file systems looking for legacy mounts */
819 	if ((ret = zfs_iter_filesystems(zhp, be_get_legacy_fs_callback,
820 	    fld)) != 0) {
821 		be_print_err(gettext("be_get_legacy_fs: "
822 		    "failed to iterate  %s to get legacy mounts\n"),
823 		    zfs_get_name(zhp));
824 	}
825 
826 cleanup:
827 	/* If we mounted the zone BE, unmount it */
828 	if (zone_mounted_here) {
829 		be_unmount_data_t	zone_ud = { 0 };
830 
831 		zone_ud.altroot = fld->altroot;
832 		zone_ud.force = B_TRUE;
833 		if ((err = be_unmount_zone_root(zhp, &zone_ud)) != BE_SUCCESS) {
834 			be_print_err(gettext("be_get_legacy_fs: "
835 			    "failed to unmount zone root %s\n"),
836 			    zoneroot_ds);
837 			if (ret == BE_SUCCESS)
838 				ret = err;
839 		}
840 	}
841 
842 	/* If we mounted this BE, unmount it */
843 	if (mounted_here) {
844 		if ((err = _be_unmount(be_name, 0)) != BE_SUCCESS) {
845 			be_print_err(gettext("be_get_legacy_fs: "
846 			    "failed to unmount %s\n"), be_name);
847 			if (ret == BE_SUCCESS)
848 				ret = err;
849 		}
850 	}
851 
852 	ZFS_CLOSE(zhp);
853 
854 	free(fld->altroot);
855 	fld->altroot = NULL;
856 
857 	return (ret);
858 }
859 
860 /*
861  * Function:	be_free_fs_list
862  * Description:	Function used to free the members of a be_fs_list_data_t
863  *			structure.
864  * Parameters:
865  *		fld - be_fs_list_data_t pointer to free.
866  * Returns:
867  *		None
868  * Scope:
869  *		Semi-private (library wide use only)
870  */
871 void
872 be_free_fs_list(be_fs_list_data_t *fld)
873 {
874 	int	i;
875 
876 	if (fld == NULL)
877 		return;
878 
879 	free(fld->altroot);
880 
881 	if (fld->fs_list == NULL)
882 		return;
883 
884 	for (i = 0; i < fld->fs_num; i++)
885 		free(fld->fs_list[i]);
886 
887 	free(fld->fs_list);
888 }
889 
890 /*
891  * Function:	be_get_ds_from_dir(char *dir)
892  * Description:	Given a directory path, find the underlying dataset mounted
893  *		at that directory path if there is one.   The returned name
894  *		is allocated in heap storage, so the caller is responsible
895  *		for freeing it.
896  * Parameters:
897  *		dir - char pointer of directory to find.
898  * Returns:
899  *		NULL - if directory is not mounted from a dataset.
900  *		name of dataset mounted at dir.
901  * Scope:
902  *		Semi-private (library wide use only)
903  */
904 char *
905 be_get_ds_from_dir(char *dir)
906 {
907 	dir_data_t	dd = { 0 };
908 	char		resolved_dir[MAXPATHLEN];
909 
910 	/* Make sure length of dir is within the max length */
911 	if (dir == NULL || strlen(dir) >= MAXPATHLEN)
912 		return (NULL);
913 
914 	/* Resolve dir in case its lofs mounted */
915 	(void) strlcpy(resolved_dir, dir, sizeof (resolved_dir));
916 	z_resolve_lofs(resolved_dir, sizeof (resolved_dir));
917 
918 	dd.dir = resolved_dir;
919 
920 	(void) zfs_iter_root(g_zfs, be_get_ds_from_dir_callback, &dd);
921 
922 	return (dd.ds);
923 }
924 
925 /*
926  * Function:	be_make_tmp_mountpoint
927  * Description:	This function generates a random temporary mountpoint
928  *		and creates that mountpoint directory.  It returns the
929  *		mountpoint in heap storage, so the caller is responsible
930  *		for freeing it.
931  * Parameters:
932  *		tmp_mp - reference to pointer of where to store generated
933  *			temporary mountpoint.
934  * Returns:
935  *		BE_SUCCESS - Success
936  *		be_errno_t - Failure
937  * Scope:
938  *		Semi-private (library wide use only)
939  */
940 int
941 be_make_tmp_mountpoint(char **tmp_mp)
942 {
943 	int	err = 0;
944 
945 	if ((*tmp_mp = (char *)calloc(1, sizeof (BE_TMP_MNTPNT) + 1)) == NULL) {
946 		be_print_err(gettext("be_make_tmp_mountpoint: "
947 		    "malloc failed\n"));
948 		return (BE_ERR_NOMEM);
949 	}
950 	(void) strlcpy(*tmp_mp, BE_TMP_MNTPNT, sizeof (BE_TMP_MNTPNT) + 1);
951 	if (mkdtemp(*tmp_mp) == NULL) {
952 		err = errno;
953 		be_print_err(gettext("be_make_tmp_mountpoint: mkdtemp() failed "
954 		    "for %s: %s\n"), *tmp_mp, strerror(err));
955 		free(*tmp_mp);
956 		*tmp_mp = NULL;
957 		return (errno_to_be_err(err));
958 	}
959 
960 	return (BE_SUCCESS);
961 }
962 
963 /*
964  * Function:	be_mount_pool
965  * Description: This function determines if the pool's datase is mounted
966  *		and if not it is used to mount the pool's dataset. The
967  *		function returns the current mountpoint if we are able
968  *		to mount the dataset.
969  * Parameters:
970  *		zhp - handle to the pool's dataset
971  *		tmp_mntpnt - The temporary mountpoint that the pool's
972  *			      dataset is mounted on. This is set only
973  *			      if the attempt to mount the dataset at it's
974  *			      set mountpoint fails, and we've used a
975  *			      temporary mount point for this dataset. It
976  *			      is expected that the caller will free this
977  *			      memory.
978  *		orig_mntpnt - The original mountpoint for the pool. If a
979  *			      temporary mount point was needed this will
980  *			      be used to reset the mountpoint property to
981  *			      it's original mountpoint. It is expected that
982  *			      the caller will free this memory.
983  *		pool_mounted - This flag indicates that the pool was mounted
984  *			       in this function.
985  * Returns:
986  *		BE_SUCCESS - Success
987  *		be_errno_t - Failure
988  * Scope:
989  *		Semi-private (library wide use only)
990  */
991 int
992 be_mount_pool(
993 	zfs_handle_t *zhp,
994 	char **tmp_mntpnt,
995 	char **orig_mntpnt,
996 	boolean_t *pool_mounted)
997 {
998 
999 	char		mountpoint[MAXPATHLEN];
1000 	int		ret = 0;
1001 
1002 	*tmp_mntpnt = NULL;
1003 	*orig_mntpnt = NULL;
1004 	*pool_mounted = B_FALSE;
1005 
1006 	if (!zfs_is_mounted(zhp, NULL)) {
1007 		if (zfs_mount(zhp, NULL, 0) != 0) {
1008 			if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
1009 			    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
1010 				be_print_err(gettext("be_mount_pool: failed to "
1011 				    "get mountpoint of (%s): %s\n"),
1012 				    zfs_get_name(zhp),
1013 				    libzfs_error_description(g_zfs));
1014 				return (zfs_err_to_be_err(g_zfs));
1015 			}
1016 			if ((*orig_mntpnt = strdup(mountpoint)) == NULL) {
1017 				be_print_err(gettext("be_mount_pool: memory "
1018 				    "allocation failed\n"));
1019 				return (BE_ERR_NOMEM);
1020 			}
1021 			/*
1022 			 * attempt to mount on a temp mountpoint
1023 			 */
1024 			if ((ret = be_make_tmp_mountpoint(tmp_mntpnt))
1025 			    != BE_SUCCESS) {
1026 				be_print_err(gettext("be_mount_pool: failed "
1027 				    "to make temporary mountpoint\n"));
1028 				free(*orig_mntpnt);
1029 				*orig_mntpnt = NULL;
1030 				return (ret);
1031 			}
1032 
1033 			if (zfs_prop_set(zhp,
1034 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1035 			    *tmp_mntpnt) != 0) {
1036 				be_print_err(gettext("be_mount_pool: failed "
1037 				    "to set mountpoint of pool dataset %s to "
1038 				    "%s: %s\n"), zfs_get_name(zhp),
1039 				    *orig_mntpnt,
1040 				    libzfs_error_description(g_zfs));
1041 				free(*tmp_mntpnt);
1042 				free(*orig_mntpnt);
1043 				*orig_mntpnt = NULL;
1044 				*tmp_mntpnt = NULL;
1045 				return (zfs_err_to_be_err(g_zfs));
1046 			}
1047 
1048 			if (zfs_mount(zhp, NULL, 0) != 0) {
1049 				be_print_err(gettext("be_mount_pool: failed "
1050 				    "to mount dataset %s at %s: %s\n"),
1051 				    zfs_get_name(zhp), *tmp_mntpnt,
1052 				    libzfs_error_description(g_zfs));
1053 				ret = zfs_err_to_be_err(g_zfs);
1054 				if (zfs_prop_set(zhp,
1055 				    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1056 				    mountpoint) != 0) {
1057 					be_print_err(gettext("be_mount_pool: "
1058 					    "failed to set mountpoint of pool "
1059 					    "dataset %s to %s: %s\n"),
1060 					    zfs_get_name(zhp), *tmp_mntpnt,
1061 					    libzfs_error_description(g_zfs));
1062 				}
1063 				free(*tmp_mntpnt);
1064 				free(*orig_mntpnt);
1065 				*orig_mntpnt = NULL;
1066 				*tmp_mntpnt = NULL;
1067 				return (ret);
1068 			}
1069 		}
1070 		*pool_mounted = B_TRUE;
1071 	}
1072 
1073 	return (BE_SUCCESS);
1074 }
1075 
1076 /*
1077  * Function:	be_unmount_pool
1078  * Description: This function is used to unmount the pool's dataset if we
1079  *		mounted it previously using be_mount_pool().
1080  * Parameters:
1081  *		zhp - handle to the pool's dataset
1082  *		tmp_mntpnt - If a temprary mount point was used this will
1083  *			     be set. Since this was created in be_mount_pool
1084  *			     we will need to clean it up here.
1085  *		orig_mntpnt - The original mountpoint for the pool. This is
1086  *			      used to set the dataset mountpoint property
1087  *			      back to it's original value in the case where a
1088  *			      temporary mountpoint was used.
1089  * Returns:
1090  *		BE_SUCCESS - Success
1091  *		be_errno_t - Failure
1092  * Scope:
1093  *		Semi-private (library wide use only)
1094  */
1095 int
1096 be_unmount_pool(
1097 	zfs_handle_t *zhp,
1098 	char *tmp_mntpnt,
1099 	char *orig_mntpnt)
1100 {
1101 	if (zfs_unmount(zhp, NULL, 0) != 0) {
1102 		be_print_err(gettext("be_unmount_pool: failed to "
1103 		    "unmount pool (%s): %s\n"), zfs_get_name(zhp),
1104 		    libzfs_error_description(g_zfs));
1105 		return (zfs_err_to_be_err(g_zfs));
1106 	}
1107 	if (orig_mntpnt != NULL) {
1108 		if (tmp_mntpnt != NULL &&
1109 		    strcmp(orig_mntpnt, tmp_mntpnt) != 0) {
1110 			(void) rmdir(tmp_mntpnt);
1111 		}
1112 		if (zfs_prop_set(zhp,
1113 		    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1114 		    orig_mntpnt) != 0) {
1115 			be_print_err(gettext("be_unmount_pool: failed "
1116 			    "to set the mountpoint for dataset (%s) to "
1117 			    "%s: %s\n"), zfs_get_name(zhp), orig_mntpnt,
1118 			    libzfs_error_description(g_zfs));
1119 			return (zfs_err_to_be_err(g_zfs));
1120 		}
1121 	}
1122 
1123 	return (BE_SUCCESS);
1124 }
1125 
1126 /* ********************************************************************	*/
1127 /*			Private Functions				*/
1128 /* ********************************************************************	*/
1129 
1130 /*
1131  * Function:	be_mount_callback
1132  * Description:	Callback function used to iterate through all of a BE's
1133  *		subordinate file systems and to mount them accordingly.
1134  * Parameters:
1135  *		zhp - zfs_handle_t pointer to current file system being
1136  *			processed.
1137  *		data - pointer to the altroot of where to mount BE.
1138  * Returns:
1139  *		0 - Success
1140  *		be_errno_t - Failure
1141  * Scope:
1142  *		Private
1143  */
1144 static int
1145 be_mount_callback(zfs_handle_t *zhp, void *data)
1146 {
1147 	zprop_source_t	sourcetype;
1148 	const char	*fs_name = zfs_get_name(zhp);
1149 	char		source[ZFS_MAX_DATASET_NAME_LEN];
1150 	char		*altroot = data;
1151 	char		zhp_mountpoint[MAXPATHLEN];
1152 	char		mountpoint[MAXPATHLEN];
1153 	int		ret = 0;
1154 
1155 	/* Get dataset's mountpoint and source values */
1156 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, zhp_mountpoint,
1157 	    sizeof (zhp_mountpoint), &sourcetype, source, sizeof (source),
1158 	    B_FALSE) != 0) {
1159 		be_print_err(gettext("be_mount_callback: failed to "
1160 		    "get mountpoint and sourcetype for %s\n"),
1161 		    fs_name);
1162 		ZFS_CLOSE(zhp);
1163 		return (BE_ERR_ZFS);
1164 	}
1165 
1166 	/*
1167 	 * Set this filesystem's 'canmount' property to 'noauto' just incase
1168 	 * it's been set 'on'.  We do this so that when we change its
1169 	 * mountpoint zfs won't immediately try to mount it.
1170 	 */
1171 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_CANMOUNT), "noauto")) {
1172 		be_print_err(gettext("be_mount_callback: failed to "
1173 		    "set canmount to 'noauto' (%s)\n"), fs_name);
1174 		ZFS_CLOSE(zhp);
1175 		return (BE_ERR_ZFS);
1176 	}
1177 
1178 	/*
1179 	 * If the mountpoint is none, there's nothing to do, goto next.
1180 	 * If the mountpoint is legacy, legacy mount it with mount(2).
1181 	 * If the mountpoint is inherited, its mountpoint should
1182 	 * already be set.  If it's not, then explicitly fix-up
1183 	 * the mountpoint now by appending its explicitly set
1184 	 * mountpoint value to the BE mountpoint.
1185 	 */
1186 	if (strcmp(zhp_mountpoint, ZFS_MOUNTPOINT_NONE) == 0) {
1187 		goto next;
1188 	} else if (strcmp(zhp_mountpoint, ZFS_MOUNTPOINT_LEGACY) == 0) {
1189 		/*
1190 		 * If the mountpoint is set to 'legacy', we need to
1191 		 * dig into this BE's vfstab to figure out where to
1192 		 * mount it, and just mount it via mount(2).
1193 		 */
1194 		if (get_mountpoint_from_vfstab(altroot, fs_name,
1195 		    mountpoint, sizeof (mountpoint), B_TRUE) == BE_SUCCESS) {
1196 
1197 			/* Legacy mount the file system */
1198 			if (mount(fs_name, mountpoint, MS_DATA,
1199 			    MNTTYPE_ZFS, NULL, 0, NULL, 0) != 0) {
1200 				be_print_err(
1201 				    gettext("be_mount_callback: "
1202 				    "failed to mount %s on %s\n"),
1203 				    fs_name, mountpoint);
1204 			}
1205 		} else {
1206 			be_print_err(
1207 			    gettext("be_mount_callback: "
1208 			    "no entry for %s in vfstab, "
1209 			    "skipping ...\n"), fs_name);
1210 		}
1211 
1212 		goto next;
1213 
1214 	} else if (sourcetype & ZPROP_SRC_INHERITED) {
1215 		/*
1216 		 * If the mountpoint is inherited, its parent should have
1217 		 * already been processed so its current mountpoint value
1218 		 * is what its mountpoint ought to be.
1219 		 */
1220 		(void) strlcpy(mountpoint, zhp_mountpoint, sizeof (mountpoint));
1221 	} else if (sourcetype & ZPROP_SRC_LOCAL) {
1222 		/*
1223 		 * Else process dataset with explicitly set mountpoint.
1224 		 */
1225 		(void) snprintf(mountpoint, sizeof (mountpoint),
1226 		    "%s%s", altroot, zhp_mountpoint);
1227 
1228 		/* Set the new mountpoint for the dataset */
1229 		if (zfs_prop_set(zhp,
1230 		    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1231 		    mountpoint)) {
1232 			be_print_err(gettext("be_mount_callback: "
1233 			    "failed to set mountpoint for %s to "
1234 			    "%s\n"), fs_name, mountpoint);
1235 			ZFS_CLOSE(zhp);
1236 			return (BE_ERR_ZFS);
1237 		}
1238 	} else {
1239 		be_print_err(gettext("be_mount_callback: "
1240 		    "mountpoint sourcetype of %s is %d, skipping ...\n"),
1241 		    fs_name, sourcetype);
1242 
1243 		goto next;
1244 	}
1245 
1246 	/* Mount this filesystem */
1247 	if (zfs_mount(zhp, NULL, 0) != 0) {
1248 		be_print_err(gettext("be_mount_callback: failed to "
1249 		    "mount dataset %s at %s: %s\n"), fs_name, mountpoint,
1250 		    libzfs_error_description(g_zfs));
1251 		/*
1252 		 * Set this filesystem's 'mountpoint' property back to what
1253 		 * it was
1254 		 */
1255 		if (sourcetype & ZPROP_SRC_LOCAL &&
1256 		    strcmp(zhp_mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0) {
1257 			(void) zfs_prop_set(zhp,
1258 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1259 			    zhp_mountpoint);
1260 		}
1261 
1262 		ZFS_CLOSE(zhp);
1263 		return (BE_ERR_MOUNT);
1264 	}
1265 
1266 next:
1267 	/* Iterate through this dataset's children and mount them */
1268 	if ((ret = zfs_iter_filesystems(zhp, be_mount_callback,
1269 	    altroot)) != 0) {
1270 		ZFS_CLOSE(zhp);
1271 		return (ret);
1272 	}
1273 
1274 
1275 	ZFS_CLOSE(zhp);
1276 	return (0);
1277 }
1278 
1279 /*
1280  * Function:	be_unmount_callback
1281  * Description:	Callback function used to iterate through all of a BE's
1282  *		subordinate file systems and to unmount them.
1283  * Parameters:
1284  *		zhp - zfs_handle_t pointer to current file system being
1285  *			processed.
1286  *		data - pointer to the mountpoint of where BE is mounted.
1287  * Returns:
1288  *		0 - Success
1289  *		be_errno_t - Failure
1290  * Scope:
1291  *		Private
1292  */
1293 static int
1294 be_unmount_callback(zfs_handle_t *zhp, void *data)
1295 {
1296 	be_unmount_data_t	*ud = data;
1297 	zprop_source_t	sourcetype;
1298 	const char	*fs_name = zfs_get_name(zhp);
1299 	char		source[ZFS_MAX_DATASET_NAME_LEN];
1300 	char		mountpoint[MAXPATHLEN];
1301 	char		*zhp_mountpoint;
1302 	int		ret = 0;
1303 
1304 	/* Iterate down this dataset's children first */
1305 	if (zfs_iter_filesystems(zhp, be_unmount_callback, ud)) {
1306 		ret = BE_ERR_UMOUNT;
1307 		goto done;
1308 	}
1309 
1310 	/* Is dataset even mounted ? */
1311 	if (!zfs_is_mounted(zhp, NULL))
1312 		goto done;
1313 
1314 	/* Unmount this file system */
1315 	if (zfs_unmount(zhp, NULL, ud->force ? MS_FORCE : 0) != 0) {
1316 		be_print_err(gettext("be_unmount_callback: "
1317 		    "failed to unmount %s: %s\n"), fs_name,
1318 		    libzfs_error_description(g_zfs));
1319 		ret = zfs_err_to_be_err(g_zfs);
1320 		goto done;
1321 	}
1322 
1323 	/* Get dataset's current mountpoint and source value */
1324 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
1325 	    sizeof (mountpoint), &sourcetype, source, sizeof (source),
1326 	    B_FALSE) != 0) {
1327 		be_print_err(gettext("be_unmount_callback: "
1328 		    "failed to get mountpoint and sourcetype for %s: %s\n"),
1329 		    fs_name, libzfs_error_description(g_zfs));
1330 		ret = zfs_err_to_be_err(g_zfs);
1331 		goto done;
1332 	}
1333 
1334 	if (sourcetype & ZPROP_SRC_INHERITED) {
1335 		/*
1336 		 * If the mountpoint is inherited we don't need to
1337 		 * do anything.  When its parent gets processed
1338 		 * its mountpoint will be set accordingly.
1339 		 */
1340 		goto done;
1341 	} else if (sourcetype & ZPROP_SRC_LOCAL) {
1342 
1343 		if (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) == 0) {
1344 			/*
1345 			 * If the mountpoint is set to 'legacy', its already
1346 			 * been unmounted (from above call to zfs_unmount), and
1347 			 * we don't need to do anything else with it.
1348 			 */
1349 			goto done;
1350 
1351 		} else {
1352 			/*
1353 			 * Else process dataset with explicitly set mountpoint.
1354 			 */
1355 
1356 			/*
1357 			 * Get this dataset's mountpoint relative to
1358 			 * the BE's mountpoint.
1359 			 */
1360 			if ((strncmp(mountpoint, ud->altroot,
1361 			    strlen(ud->altroot)) == 0) &&
1362 			    (mountpoint[strlen(ud->altroot)] == '/')) {
1363 
1364 				zhp_mountpoint = mountpoint +
1365 				    strlen(ud->altroot);
1366 
1367 				/* Set this dataset's mountpoint value */
1368 				if (zfs_prop_set(zhp,
1369 				    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1370 				    zhp_mountpoint)) {
1371 					be_print_err(
1372 					    gettext("be_unmount_callback: "
1373 					    "failed to set mountpoint for "
1374 					    "%s to %s: %s\n"), fs_name,
1375 					    zhp_mountpoint,
1376 					    libzfs_error_description(g_zfs));
1377 					ret = zfs_err_to_be_err(g_zfs);
1378 				}
1379 			} else {
1380 				be_print_err(
1381 				    gettext("be_unmount_callback: "
1382 				    "%s not mounted under BE's altroot %s, "
1383 				    "skipping ...\n"), fs_name, ud->altroot);
1384 				/*
1385 				 * fs_name is mounted but not under the
1386 				 * root for this BE.
1387 				 */
1388 				ret = BE_ERR_INVALMOUNTPOINT;
1389 			}
1390 		}
1391 	} else {
1392 		be_print_err(gettext("be_unmount_callback: "
1393 		    "mountpoint sourcetype of %s is %d, skipping ...\n"),
1394 		    fs_name, sourcetype);
1395 		ret = BE_ERR_ZFS;
1396 	}
1397 
1398 done:
1399 	/* Set this filesystem's 'canmount' property to 'noauto' */
1400 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_CANMOUNT), "noauto")) {
1401 		be_print_err(gettext("be_unmount_callback: "
1402 		    "failed to set canmount to 'noauto' (%s)\n"), fs_name);
1403 		if (ret == 0)
1404 			ret = BE_ERR_ZFS;
1405 	}
1406 
1407 	ZFS_CLOSE(zhp);
1408 	return (ret);
1409 }
1410 
1411 /*
1412  * Function:	be_get_legacy_fs_callback
1413  * Description:	The callback function is used to iterate through all
1414  *		non-shared file systems of a BE, finding ones that have
1415  *		a legacy mountpoint and an entry in the BE's vfstab.
1416  *		It adds these file systems to the callback data.
1417  * Parameters:
1418  *		zhp - zfs_handle_t pointer to current file system being
1419  *			processed.
1420  *		data - be_fs_list_data_t pointer
1421  * Returns:
1422  *		0 - Success
1423  *		be_errno_t - Failure
1424  * Scope:
1425  *		Private
1426  */
1427 static int
1428 be_get_legacy_fs_callback(zfs_handle_t *zhp, void *data)
1429 {
1430 	be_fs_list_data_t	*fld = data;
1431 	const char		*fs_name = zfs_get_name(zhp);
1432 	char			zhp_mountpoint[MAXPATHLEN];
1433 	char			mountpoint[MAXPATHLEN];
1434 	int			ret = 0;
1435 
1436 	/* Get this dataset's mountpoint property */
1437 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, zhp_mountpoint,
1438 	    sizeof (zhp_mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
1439 		be_print_err(gettext("be_get_legacy_fs_callback: "
1440 		    "failed to get mountpoint for %s: %s\n"),
1441 		    fs_name, libzfs_error_description(g_zfs));
1442 		ret = zfs_err_to_be_err(g_zfs);
1443 		ZFS_CLOSE(zhp);
1444 		return (ret);
1445 	}
1446 
1447 	/*
1448 	 * If mountpoint is legacy, try to get its mountpoint from this BE's
1449 	 * vfstab.  If it exists in the vfstab, add this file system to the
1450 	 * callback data.
1451 	 */
1452 	if (strcmp(zhp_mountpoint, ZFS_MOUNTPOINT_LEGACY) == 0) {
1453 		if (get_mountpoint_from_vfstab(fld->altroot, fs_name,
1454 		    mountpoint, sizeof (mountpoint), B_FALSE) != BE_SUCCESS) {
1455 			be_print_err(gettext("be_get_legacy_fs_callback: "
1456 			    "no entry for %s in vfstab, "
1457 			    "skipping ...\n"), fs_name);
1458 
1459 			goto next;
1460 		}
1461 
1462 		/* Record file system into the callback data. */
1463 		if (add_to_fs_list(fld, zfs_get_name(zhp)) != BE_SUCCESS) {
1464 			be_print_err(gettext("be_get_legacy_fs_callback: "
1465 			    "failed to add %s to fs list\n"), mountpoint);
1466 			ZFS_CLOSE(zhp);
1467 			return (BE_ERR_NOMEM);
1468 		}
1469 	}
1470 
1471 next:
1472 	/* Iterate through this dataset's children file systems */
1473 	if ((ret = zfs_iter_filesystems(zhp, be_get_legacy_fs_callback,
1474 	    fld)) != 0) {
1475 		ZFS_CLOSE(zhp);
1476 		return (ret);
1477 	}
1478 	ZFS_CLOSE(zhp);
1479 	return (0);
1480 }
1481 
1482 /*
1483  * Function:	add_to_fs_list
1484  * Description:	Function used to add a file system to the fs_list array in
1485  *			a be_fs_list_data_t structure.
1486  * Parameters:
1487  *		fld - be_fs_list_data_t pointer
1488  *		fs - file system to add
1489  * Returns:
1490  *		BE_SUCCESS - Success
1491  *		1 - Failure
1492  * Scope:
1493  *		Private
1494  */
1495 static int
1496 add_to_fs_list(be_fs_list_data_t *fld, const char *fs)
1497 {
1498 	if (fld == NULL || fs == NULL)
1499 		return (1);
1500 
1501 	if ((fld->fs_list = (char **)realloc(fld->fs_list,
1502 	    sizeof (char *)*(fld->fs_num + 1))) == NULL) {
1503 		be_print_err(gettext("add_to_fs_list: "
1504 		    "memory allocation failed\n"));
1505 		return (1);
1506 	}
1507 
1508 	if ((fld->fs_list[fld->fs_num++] = strdup(fs)) == NULL) {
1509 		be_print_err(gettext("add_to_fs_list: "
1510 		    "memory allocation failed\n"));
1511 		return (1);
1512 	}
1513 
1514 	return (BE_SUCCESS);
1515 }
1516 
1517 /*
1518  * Function:	zpool_shared_fs_callback
1519  * Description:	Callback function used to iterate through all existing pools
1520  *		to find and mount all shared filesystems.  This function
1521  *		processes the pool's "pool data" dataset, then uses
1522  *		iter_shared_fs_callback to iterate through the pool's
1523  *		datasets.
1524  * Parameters:
1525  *		zlp - zpool_handle_t pointer to the current pool being
1526  *			looked at.
1527  *		data - be_mount_data_t pointer
1528  * Returns:
1529  *		0 - Success
1530  *		be_errno_t - Failure
1531  * Scope:
1532  *		Private
1533  */
1534 static int
1535 zpool_shared_fs_callback(zpool_handle_t *zlp, void *data)
1536 {
1537 	be_mount_data_t	*md = data;
1538 	zfs_handle_t	*zhp = NULL;
1539 	const char	*zpool = zpool_get_name(zlp);
1540 	int		ret = 0;
1541 
1542 	/*
1543 	 * Get handle to pool's "pool data" dataset
1544 	 */
1545 	if ((zhp = zfs_open(g_zfs, zpool, ZFS_TYPE_FILESYSTEM)) == NULL) {
1546 		be_print_err(gettext("zpool_shared_fs: "
1547 		    "failed to open pool dataset %s: %s\n"), zpool,
1548 		    libzfs_error_description(g_zfs));
1549 		ret = zfs_err_to_be_err(g_zfs);
1550 		zpool_close(zlp);
1551 		return (ret);
1552 	}
1553 
1554 	/* Process this pool's "pool data" dataset */
1555 	(void) loopback_mount_shared_fs(zhp, md);
1556 
1557 	/* Interate through this pool's children */
1558 	(void) zfs_iter_filesystems(zhp, iter_shared_fs_callback, md);
1559 
1560 	ZFS_CLOSE(zhp);
1561 	zpool_close(zlp);
1562 
1563 	return (0);
1564 }
1565 
1566 /*
1567  * Function:	iter_shared_fs_callback
1568  * Description:	Callback function used to iterate through a pool's datasets
1569  *		to find and mount all shared filesystems.  It makes sure to
1570  *		find the BE container dataset of the pool, if it exists, and
1571  *		does not process and iterate down that path.
1572  *
1573  *		Note - This function iterates linearly down the
1574  *		hierarchical dataset paths and mounts things as it goes
1575  *		along.  It does not make sure that something deeper down
1576  *		a dataset path has an interim mountpoint for something
1577  *		processed earlier.
1578  *
1579  * Parameters:
1580  *		zhp - zfs_handle_t pointer to the current dataset being
1581  *			processed.
1582  *		data - be_mount_data_t pointer
1583  * Returns:
1584  *		0 - Success
1585  *		be_errno_t - Failure
1586  * Scope:
1587  *		Private
1588  */
1589 static int
1590 iter_shared_fs_callback(zfs_handle_t *zhp, void *data)
1591 {
1592 	be_mount_data_t	*md = data;
1593 	const char	*name = zfs_get_name(zhp);
1594 	char		container_ds[MAXPATHLEN];
1595 	char		tmp_name[MAXPATHLEN];
1596 	char		*pool;
1597 
1598 	/* Get the pool's name */
1599 	(void) strlcpy(tmp_name, name, sizeof (tmp_name));
1600 	pool = strtok(tmp_name, "/");
1601 
1602 	if (pool) {
1603 		/* Get the name of this pool's container dataset */
1604 		be_make_container_ds(pool, container_ds,
1605 		    sizeof (container_ds));
1606 
1607 		/*
1608 		 * If what we're processing is this pool's BE container
1609 		 * dataset, skip it.
1610 		 */
1611 		if (strcmp(name, container_ds) == 0) {
1612 			ZFS_CLOSE(zhp);
1613 			return (0);
1614 		}
1615 	} else {
1616 		/* Getting the pool name failed, return error */
1617 		be_print_err(gettext("iter_shared_fs_callback: "
1618 		    "failed to get pool name from %s\n"), name);
1619 		ZFS_CLOSE(zhp);
1620 		return (BE_ERR_POOL_NOENT);
1621 	}
1622 
1623 	/* Mount this shared filesystem */
1624 	(void) loopback_mount_shared_fs(zhp, md);
1625 
1626 	/* Iterate this dataset's children file systems */
1627 	(void) zfs_iter_filesystems(zhp, iter_shared_fs_callback, md);
1628 	ZFS_CLOSE(zhp);
1629 
1630 	return (0);
1631 }
1632 
1633 /*
1634  * Function:	loopback_mount_shared_fs
1635  * Description:	This function loopback mounts a file system into the altroot
1636  *		area of the BE being mounted.  Since these are shared file
1637  *		systems, they are expected to be already mounted for the
1638  *		current BE, and this function just loopback mounts them into
1639  *		the BE mountpoint.  If they are not mounted for the current
1640  *		live system, they are skipped and not mounted into the BE
1641  *		we're mounting.
1642  * Parameters:
1643  *		zhp - zfs_handle_t pointer to the dataset to loopback mount
1644  *		md - be_mount_data_t pointer
1645  * Returns:
1646  *		BE_SUCCESS - Success
1647  *		be_errno_t - Failure
1648  * Scope:
1649  *		Private
1650  */
1651 static int
1652 loopback_mount_shared_fs(zfs_handle_t *zhp, be_mount_data_t *md)
1653 {
1654 	char		zhp_mountpoint[MAXPATHLEN];
1655 	char		mountpoint[MAXPATHLEN];
1656 	char		*mp = NULL;
1657 	char		optstr[MAX_MNTOPT_STR];
1658 	int		mflag = MS_OPTIONSTR;
1659 	int		err;
1660 
1661 	/*
1662 	 * Check if file system is currently mounted and not delegated
1663 	 * to a non-global zone (if we're in the global zone)
1664 	 */
1665 	if (zfs_is_mounted(zhp, &mp) && (getzoneid() != GLOBAL_ZONEID ||
1666 	    !zfs_prop_get_int(zhp, ZFS_PROP_ZONED))) {
1667 		/*
1668 		 * If we didn't get a mountpoint from the zfs_is_mounted call,
1669 		 * get it from the mountpoint property.
1670 		 */
1671 		if (mp == NULL) {
1672 			if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT,
1673 			    zhp_mountpoint, sizeof (zhp_mountpoint), NULL,
1674 			    NULL, 0, B_FALSE) != 0) {
1675 				be_print_err(
1676 				    gettext("loopback_mount_shared_fs: "
1677 				    "failed to get mountpoint property\n"));
1678 				return (BE_ERR_ZFS);
1679 			}
1680 		} else {
1681 			(void) strlcpy(zhp_mountpoint, mp,
1682 			    sizeof (zhp_mountpoint));
1683 			free(mp);
1684 		}
1685 
1686 		(void) snprintf(mountpoint, sizeof (mountpoint), "%s%s",
1687 		    md->altroot, zhp_mountpoint);
1688 
1689 		/* Mount it read-only if read-write was not requested */
1690 		if (!md->shared_rw) {
1691 			mflag |= MS_RDONLY;
1692 		}
1693 
1694 		/* Add the "nosub" option to the mount options string */
1695 		(void) strlcpy(optstr, MNTOPT_NOSUB, sizeof (optstr));
1696 
1697 		/* Loopback mount this dataset at the altroot */
1698 		if (mount(zhp_mountpoint, mountpoint, mflag, MNTTYPE_LOFS,
1699 		    NULL, 0, optstr, sizeof (optstr)) != 0) {
1700 			err = errno;
1701 			be_print_err(gettext("loopback_mount_shared_fs: "
1702 			    "failed to loopback mount %s at %s: %s\n"),
1703 			    zhp_mountpoint, mountpoint, strerror(err));
1704 			return (BE_ERR_MOUNT);
1705 		}
1706 	}
1707 
1708 	return (BE_SUCCESS);
1709 }
1710 
1711 /*
1712  * Function:	loopback_mount_zonepath
1713  * Description:	This function loopback mounts a zonepath into the altroot
1714  *		area of the BE being mounted.
1715  * Parameters:
1716  *		zonepath - pointer to zone path in the current BE
1717  *		md - be_mount_data_t pointer
1718  * Returns:
1719  *		BE_SUCCESS - Success
1720  *		be_errno_t - Failure
1721  * Scope:
1722  *		Private
1723  */
1724 static int
1725 loopback_mount_zonepath(const char *zonepath, be_mount_data_t *md)
1726 {
1727 	FILE		*fp = (FILE *)NULL;
1728 	struct stat	st;
1729 	char		*p;
1730 	char		*p1;
1731 	char		*parent_dir;
1732 	struct extmnttab	extmtab;
1733 	dev_t		dev = NODEV;
1734 	char		*parentmnt;
1735 	char		alt_parentmnt[MAXPATHLEN];
1736 	struct mnttab	mntref;
1737 	char		altzonepath[MAXPATHLEN];
1738 	char		optstr[MAX_MNTOPT_STR];
1739 	int		mflag = MS_OPTIONSTR;
1740 	int		ret;
1741 	int		err;
1742 
1743 	fp = fopen(MNTTAB, "r");
1744 	if (fp == NULL) {
1745 		err = errno;
1746 		be_print_err(gettext("loopback_mount_zonepath: "
1747 		    "failed to open /etc/mnttab\n"));
1748 		return (errno_to_be_err(err));
1749 	}
1750 
1751 	/*
1752 	 * before attempting the loopback mount of zonepath under altroot,
1753 	 * we need to make sure that all intermediate file systems in the
1754 	 * zone path are also mounted under altroot
1755 	 */
1756 
1757 	/* get the parent directory for zonepath */
1758 	p = strrchr(zonepath, '/');
1759 	if (p != NULL && p != zonepath) {
1760 		if ((parent_dir = (char *)calloc(sizeof (char),
1761 		    p - zonepath + 1)) == NULL) {
1762 			ret = BE_ERR_NOMEM;
1763 			goto done;
1764 		}
1765 		(void) strlcpy(parent_dir, zonepath, p - zonepath + 1);
1766 		if (stat(parent_dir, &st) < 0) {
1767 			ret = errno_to_be_err(errno);
1768 			be_print_err(gettext("loopback_mount_zonepath: "
1769 			    "failed to stat %s"),
1770 			    parent_dir);
1771 			free(parent_dir);
1772 			goto done;
1773 		}
1774 		free(parent_dir);
1775 
1776 		/*
1777 		 * After the above stat call, st.st_dev contains ID of the
1778 		 * device over which parent dir resides.
1779 		 * Now, search mnttab and find mount point of parent dir device.
1780 		 */
1781 
1782 		resetmnttab(fp);
1783 		while (getextmntent(fp, &extmtab, sizeof (extmtab)) == 0) {
1784 			dev = makedev(extmtab.mnt_major, extmtab.mnt_minor);
1785 			if (st.st_dev == dev && strcmp(extmtab.mnt_fstype,
1786 			    MNTTYPE_ZFS) == 0) {
1787 				p1 = strchr(extmtab.mnt_special, '/');
1788 				if (p1 == NULL || strncmp(p1 + 1,
1789 				    BE_CONTAINER_DS_NAME, 4) != 0 ||
1790 				    (*(p1 + 5) != '/' && *(p1 + 5) != '\0')) {
1791 					/*
1792 					 * if parent dir is in a shared file
1793 					 * system, check whether it is already
1794 					 * loopback mounted under altroot or
1795 					 * not.  It would have been mounted
1796 					 * already under altroot if it is in
1797 					 * a non-shared filesystem.
1798 					 */
1799 					parentmnt = strdup(extmtab.mnt_mountp);
1800 					(void) snprintf(alt_parentmnt,
1801 					    sizeof (alt_parentmnt), "%s%s",
1802 					    md->altroot, parentmnt);
1803 					mntref.mnt_mountp = alt_parentmnt;
1804 					mntref.mnt_special = parentmnt;
1805 					mntref.mnt_fstype = MNTTYPE_LOFS;
1806 					mntref.mnt_mntopts = NULL;
1807 					mntref.mnt_time = NULL;
1808 					resetmnttab(fp);
1809 					if (getmntany(fp, (struct mnttab *)
1810 					    &extmtab, &mntref) != 0) {
1811 						ret = loopback_mount_zonepath(
1812 						    parentmnt, md);
1813 						if (ret != BE_SUCCESS) {
1814 							free(parentmnt);
1815 							goto done;
1816 						}
1817 					}
1818 					free(parentmnt);
1819 				}
1820 				break;
1821 			}
1822 		}
1823 	}
1824 
1825 
1826 	if (!md->shared_rw) {
1827 		mflag |= MS_RDONLY;
1828 	}
1829 
1830 	(void) snprintf(altzonepath, sizeof (altzonepath), "%s%s",
1831 	    md->altroot, zonepath);
1832 
1833 	/* Add the "nosub" option to the mount options string */
1834 	(void) strlcpy(optstr, MNTOPT_NOSUB, sizeof (optstr));
1835 
1836 	/* Loopback mount this dataset at the altroot */
1837 	if (mount(zonepath, altzonepath, mflag, MNTTYPE_LOFS,
1838 	    NULL, 0, optstr, sizeof (optstr)) != 0) {
1839 		err = errno;
1840 		be_print_err(gettext("loopback_mount_zonepath: "
1841 		    "failed to loopback mount %s at %s: %s\n"),
1842 		    zonepath, altzonepath, strerror(err));
1843 		ret = BE_ERR_MOUNT;
1844 		goto done;
1845 	}
1846 	ret = BE_SUCCESS;
1847 
1848 done :
1849 	(void) fclose(fp);
1850 	return (ret);
1851 }
1852 
1853 /*
1854  * Function:	unmount_shared_fs
1855  * Description:	This function iterates through the mnttab and finds all
1856  *		loopback mount entries that reside within the altroot of
1857  *		where the BE is mounted, and unmounts it.
1858  * Parameters:
1859  *		ud - be_unmount_data_t pointer
1860  * Returns:
1861  *		BE_SUCCESS - Success
1862  *		be_errno_t - Failure
1863  * Scope:
1864  *		Private
1865  */
1866 static int
1867 unmount_shared_fs(be_unmount_data_t *ud)
1868 {
1869 	FILE		*fp = NULL;
1870 	struct mnttab	*table = NULL;
1871 	struct mnttab	ent;
1872 	struct mnttab	*entp = NULL;
1873 	size_t		size = 0;
1874 	int		read_chunk = 32;
1875 	int		i;
1876 	int		altroot_len;
1877 	int		err = 0;
1878 
1879 	errno = 0;
1880 
1881 	/* Read in the mnttab into a table */
1882 	if ((fp = fopen(MNTTAB, "r")) == NULL) {
1883 		err = errno;
1884 		be_print_err(gettext("unmount_shared_fs: "
1885 		    "failed to open mnttab\n"));
1886 		return (errno_to_be_err(err));
1887 	}
1888 
1889 	while (getmntent(fp, &ent) == 0) {
1890 		if (size % read_chunk == 0) {
1891 			table = (struct mnttab *)realloc(table,
1892 			    (size + read_chunk) * sizeof (ent));
1893 		}
1894 		entp = &table[size++];
1895 
1896 		/*
1897 		 * Copy over the current mnttab entry into our table,
1898 		 * copying only the fields that we care about.
1899 		 */
1900 		(void) memset(entp, 0, sizeof (*entp));
1901 		if ((entp->mnt_mountp = strdup(ent.mnt_mountp)) == NULL ||
1902 		    (entp->mnt_fstype = strdup(ent.mnt_fstype)) == NULL) {
1903 			be_print_err(gettext("unmount_shared_fs: "
1904 			    "memory allocation failed\n"));
1905 			return (BE_ERR_NOMEM);
1906 		}
1907 	}
1908 	(void) fclose(fp);
1909 
1910 	/*
1911 	 * Process the mnttab entries in reverse order, looking for
1912 	 * loopback mount entries mounted under our altroot.
1913 	 */
1914 	altroot_len = strlen(ud->altroot);
1915 	for (i = size; i > 0; i--) {
1916 		entp = &table[i - 1];
1917 
1918 		/* If not of type lofs, skip */
1919 		if (strcmp(entp->mnt_fstype, MNTTYPE_LOFS) != 0)
1920 			continue;
1921 
1922 		/* If inside the altroot, unmount it */
1923 		if (strncmp(entp->mnt_mountp, ud->altroot, altroot_len) == 0 &&
1924 		    entp->mnt_mountp[altroot_len] == '/') {
1925 			if (umount(entp->mnt_mountp) != 0) {
1926 				err = errno;
1927 				if (err == EBUSY) {
1928 					(void) sleep(1);
1929 					err = errno = 0;
1930 					if (umount(entp->mnt_mountp) != 0)
1931 						err = errno;
1932 				}
1933 				if (err != 0) {
1934 					be_print_err(gettext(
1935 					    "unmount_shared_fs: "
1936 					    "failed to unmount shared file "
1937 					    "system %s: %s\n"),
1938 					    entp->mnt_mountp, strerror(err));
1939 					return (errno_to_be_err(err));
1940 				}
1941 			}
1942 		}
1943 	}
1944 
1945 	return (BE_SUCCESS);
1946 }
1947 
1948 /*
1949  * Function:	get_mountpoint_from_vfstab
1950  * Description:	This function digs into the vfstab in the given altroot,
1951  *		and searches for an entry for the fs passed in.  If found,
1952  *		it returns the mountpoint of that fs in the mountpoint
1953  *		buffer passed in.  If the get_alt_mountpoint flag is set,
1954  *		it returns the mountpoint with the altroot prepended.
1955  * Parameters:
1956  *		altroot - pointer to the alternate root location
1957  *		fs - pointer to the file system name to look for in the
1958  *			vfstab in altroot
1959  *		mountpoint - pointer to buffer of where the mountpoint of
1960  *			fs will be returned.
1961  *		size_mp - size of mountpoint argument
1962  *		get_alt_mountpoint - flag to indicate whether or not the
1963  *			mountpoint should be populated with the altroot
1964  *			prepended.
1965  * Returns:
1966  *		BE_SUCCESS - Success
1967  *		1 - Failure
1968  * Scope:
1969  *		Private
1970  */
1971 static int
1972 get_mountpoint_from_vfstab(char *altroot, const char *fs, char *mountpoint,
1973     size_t size_mp, boolean_t get_alt_mountpoint)
1974 {
1975 	struct vfstab	vp;
1976 	FILE		*fp = NULL;
1977 	char		alt_vfstab[MAXPATHLEN];
1978 
1979 	/* Generate path to alternate root vfstab */
1980 	(void) snprintf(alt_vfstab, sizeof (alt_vfstab), "%s/etc/vfstab",
1981 	    altroot);
1982 
1983 	/* Open alternate root vfstab */
1984 	if ((fp = fopen(alt_vfstab, "r")) == NULL) {
1985 		be_print_err(gettext("get_mountpoint_from_vfstab: "
1986 		    "failed to open vfstab (%s)\n"), alt_vfstab);
1987 		return (1);
1988 	}
1989 
1990 	if (getvfsspec(fp, &vp, (char *)fs) == 0) {
1991 		/*
1992 		 * Found entry for fs, grab its mountpoint.
1993 		 * If the flag to prepend the altroot into the mountpoint
1994 		 * is set, prepend it.  Otherwise, just return the mountpoint.
1995 		 */
1996 		if (get_alt_mountpoint) {
1997 			(void) snprintf(mountpoint, size_mp, "%s%s", altroot,
1998 			    vp.vfs_mountp);
1999 		} else {
2000 			(void) strlcpy(mountpoint, vp.vfs_mountp, size_mp);
2001 		}
2002 	} else {
2003 		(void) fclose(fp);
2004 		return (1);
2005 	}
2006 
2007 	(void) fclose(fp);
2008 
2009 	return (BE_SUCCESS);
2010 }
2011 
2012 /*
2013  * Function:	fix_mountpoint_callback
2014  * Description:	This callback function is used to iterate through a BE's
2015  *		children filesystems to check if its mountpoint is currently
2016  *		set to be mounted at some specified altroot.  If so, fix it by
2017  *		removing altroot from the beginning of its mountpoint.
2018  *
2019  *		Note - There's no way to tell if a child filesystem's
2020  *		mountpoint isn't broken, and just happens to begin with
2021  *		the altroot we're looking for.  In this case, this function
2022  *		will errantly remove the altroot portion from the beginning
2023  *		of this filesystem's mountpoint.
2024  *
2025  * Parameters:
2026  *		zhp - zfs_handle_t pointer to filesystem being processed.
2027  *		data - altroot of where BE is to be mounted.
2028  * Returns:
2029  *		0 - Success
2030  *		be_errno_t - Failure
2031  * Scope:
2032  *		Private
2033  */
2034 static int
2035 fix_mountpoint_callback(zfs_handle_t *zhp, void *data)
2036 {
2037 	zprop_source_t	sourcetype;
2038 	char		source[ZFS_MAX_DATASET_NAME_LEN];
2039 	char		mountpoint[MAXPATHLEN];
2040 	char		*zhp_mountpoint = NULL;
2041 	char		*altroot = data;
2042 	int		ret = 0;
2043 
2044 	/* Get dataset's mountpoint and source values */
2045 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
2046 	    sizeof (mountpoint), &sourcetype, source, sizeof (source),
2047 	    B_FALSE) != 0) {
2048 		be_print_err(gettext("fix_mountpoint_callback: "
2049 		    "failed to get mountpoint and sourcetype for %s\n"),
2050 		    zfs_get_name(zhp));
2051 		ZFS_CLOSE(zhp);
2052 		return (BE_ERR_ZFS);
2053 	}
2054 
2055 	/*
2056 	 * If the mountpoint is not inherited and the mountpoint is not
2057 	 * 'legacy', this file system potentially needs its mountpoint
2058 	 * fixed.
2059 	 */
2060 	if (!(sourcetype & ZPROP_SRC_INHERITED) &&
2061 	    strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0) {
2062 
2063 		/*
2064 		 * Check if this file system's current mountpoint is
2065 		 * under the altroot we're fixing it against.
2066 		 */
2067 		if (strncmp(mountpoint, altroot, strlen(altroot)) == 0 &&
2068 		    mountpoint[strlen(altroot)] == '/') {
2069 
2070 			/*
2071 			 * Get this dataset's mountpoint relative to the
2072 			 * altroot.
2073 			 */
2074 			zhp_mountpoint = mountpoint + strlen(altroot);
2075 
2076 			/* Fix this dataset's mountpoint value */
2077 			if (zfs_prop_set(zhp,
2078 			    zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
2079 			    zhp_mountpoint)) {
2080 				be_print_err(gettext("fix_mountpoint_callback: "
2081 				    "failed to set mountpoint for %s to "
2082 				    "%s: %s\n"), zfs_get_name(zhp),
2083 				    zhp_mountpoint,
2084 				    libzfs_error_description(g_zfs));
2085 				ret = zfs_err_to_be_err(g_zfs);
2086 				ZFS_CLOSE(zhp);
2087 				return (ret);
2088 			}
2089 		}
2090 	}
2091 
2092 	/* Iterate through this dataset's children and fix them */
2093 	if ((ret = zfs_iter_filesystems(zhp, fix_mountpoint_callback,
2094 	    altroot)) != 0) {
2095 		ZFS_CLOSE(zhp);
2096 		return (ret);
2097 	}
2098 
2099 
2100 	ZFS_CLOSE(zhp);
2101 	return (0);
2102 }
2103 
2104 /*
2105  * Function:	be_mount_root
2106  * Description:	This function mounts the root dataset of a BE at the
2107  *		specified altroot.
2108  * Parameters:
2109  *		zhp - zfs_handle_t pointer to root dataset of a BE that is
2110  *		to be mounted at altroot.
2111  *		altroot - location of where to mount the BE root.
2112  * Return:
2113  *		BE_SUCCESS - Success
2114  *		be_errno_t - Failure
2115  * Scope:
2116  *		Private
2117  */
2118 static int
2119 be_mount_root(zfs_handle_t *zhp, char *altroot)
2120 {
2121 	char		mountpoint[MAXPATHLEN];
2122 
2123 	/* Get mountpoint property of dataset */
2124 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
2125 	    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
2126 		be_print_err(gettext("be_mount_root: failed to "
2127 		    "get mountpoint property for %s: %s\n"), zfs_get_name(zhp),
2128 		    libzfs_error_description(g_zfs));
2129 		return (zfs_err_to_be_err(g_zfs));
2130 	}
2131 
2132 	/*
2133 	 * Set the canmount property for the BE's root dataset to 'noauto' just
2134 	 * in case it's been set to 'on'.  We do this so that when we change its
2135 	 * mountpoint, zfs won't immediately try to mount it.
2136 	 */
2137 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_CANMOUNT), "noauto")
2138 	    != 0) {
2139 		be_print_err(gettext("be_mount_root: failed to "
2140 		    "set canmount property to 'noauto' (%s): %s\n"),
2141 		    zfs_get_name(zhp), libzfs_error_description(g_zfs));
2142 		return (zfs_err_to_be_err(g_zfs));
2143 	}
2144 
2145 	/* Set mountpoint for BE's root filesystem */
2146 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT), altroot)
2147 	    != 0) {
2148 		be_print_err(gettext("be_mount_root: failed to "
2149 		    "set mountpoint of %s to %s: %s\n"),
2150 		    zfs_get_name(zhp), altroot,
2151 		    libzfs_error_description(g_zfs));
2152 		return (zfs_err_to_be_err(g_zfs));
2153 	}
2154 
2155 	/* Mount the BE's root filesystem */
2156 	if (zfs_mount(zhp, NULL, 0) != 0) {
2157 		be_print_err(gettext("be_mount_root: failed to "
2158 		    "mount dataset %s at %s: %s\n"), zfs_get_name(zhp),
2159 		    altroot, libzfs_error_description(g_zfs));
2160 		/*
2161 		 * Set this BE's root filesystem 'mountpoint' property
2162 		 * back to what it was before.
2163 		 */
2164 		(void) zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
2165 		    mountpoint);
2166 		return (zfs_err_to_be_err(g_zfs));
2167 	}
2168 
2169 	return (BE_SUCCESS);
2170 }
2171 
2172 /*
2173  * Function:	be_unmount_root
2174  * Description:	This function unmounts the root dataset of a BE, but before
2175  *		unmounting, it looks at the BE's vfstab to determine
2176  *		if the root dataset mountpoint should be left as 'legacy'
2177  *		or '/'.  If the vfstab contains an entry for this root
2178  *		dataset with a mountpoint of '/', it sets the mountpoint
2179  *		property to 'legacy'.
2180  *
2181  * Parameters:
2182  *		zhp - zfs_handle_t pointer of the BE root dataset that
2183  *		is currently mounted.
2184  *		ud - be_unmount_data_t pointer providing unmount data
2185  *		for the given BE root dataset.
2186  * Returns:
2187  *		BE_SUCCESS - Success
2188  *		be_errno_t - Failure
2189  * Scope:
2190  *		Private
2191  */
2192 static int
2193 be_unmount_root(zfs_handle_t *zhp, be_unmount_data_t *ud)
2194 {
2195 	char		mountpoint[MAXPATHLEN];
2196 	boolean_t	is_legacy = B_FALSE;
2197 
2198 	/* See if this is a legacy mounted root */
2199 	if (get_mountpoint_from_vfstab(ud->altroot, zfs_get_name(zhp),
2200 	    mountpoint, sizeof (mountpoint), B_FALSE) == BE_SUCCESS &&
2201 	    strcmp(mountpoint, "/") == 0) {
2202 		is_legacy = B_TRUE;
2203 	}
2204 
2205 	/* Unmount the dataset */
2206 	if (zfs_unmount(zhp, NULL, ud->force ? MS_FORCE : 0) != 0) {
2207 		be_print_err(gettext("be_unmount_root: failed to "
2208 		    "unmount BE root dataset %s: %s\n"), zfs_get_name(zhp),
2209 		    libzfs_error_description(g_zfs));
2210 		return (zfs_err_to_be_err(g_zfs));
2211 	}
2212 
2213 	/* Set canmount property for this BE's root filesystem to noauto */
2214 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_CANMOUNT), "noauto")
2215 	    != 0) {
2216 		be_print_err(gettext("be_unmount_root: failed to "
2217 		    "set canmount property for %s to 'noauto': %s\n"),
2218 		    zfs_get_name(zhp), libzfs_error_description(g_zfs));
2219 		return (zfs_err_to_be_err(g_zfs));
2220 	}
2221 
2222 	/*
2223 	 * Set mountpoint for BE's root dataset back to '/', or 'legacy'
2224 	 * if its a legacy mounted root.
2225 	 */
2226 	if (zfs_prop_set(zhp, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
2227 	    is_legacy ? ZFS_MOUNTPOINT_LEGACY : "/") != 0) {
2228 		be_print_err(gettext("be_unmount_root: failed to "
2229 		    "set mountpoint of %s to %s\n"), zfs_get_name(zhp),
2230 		    is_legacy ? ZFS_MOUNTPOINT_LEGACY : "/");
2231 		return (zfs_err_to_be_err(g_zfs));
2232 	}
2233 
2234 	return (BE_SUCCESS);
2235 }
2236 
2237 /*
2238  * Function:	fix_mountpoint
2239  * Description:	This function checks the mountpoint of an unmounted BE to make
2240  *		sure that it is set to either 'legacy' or '/'.  If it's not,
2241  *		then we're in a situation where an unmounted BE has some random
2242  *		mountpoint set for it.  (This could happen if the system was
2243  *		rebooted while an inactive BE was mounted).  This function
2244  *		attempts to fix its mountpoints.
2245  * Parameters:
2246  *		zhp - zfs_handle_t pointer to root dataset of the BE
2247  *		whose mountpoint needs to be checked.
2248  * Return:
2249  *		BE_SUCCESS - Success
2250  *		be_errno_t - Failure
2251  * Scope:
2252  *		Private
2253  */
2254 static int
2255 fix_mountpoint(zfs_handle_t *zhp)
2256 {
2257 	be_unmount_data_t	ud = { 0 };
2258 	char	*altroot = NULL;
2259 	char	mountpoint[MAXPATHLEN];
2260 	int	ret = BE_SUCCESS;
2261 
2262 	/*
2263 	 * Record what this BE's root dataset mountpoint property is currently
2264 	 * set to.
2265 	 */
2266 	if (zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint,
2267 	    sizeof (mountpoint), NULL, NULL, 0, B_FALSE) != 0) {
2268 		be_print_err(gettext("fix_mountpoint: failed to get "
2269 		    "mountpoint property of (%s): %s\n"), zfs_get_name(zhp),
2270 		    libzfs_error_description(g_zfs));
2271 		return (BE_ERR_ZFS);
2272 	}
2273 
2274 	/*
2275 	 * If the root dataset mountpoint is set to 'legacy' or '/', we're okay.
2276 	 */
2277 	if (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) == 0 ||
2278 	    strcmp(mountpoint, "/") == 0) {
2279 		return (BE_SUCCESS);
2280 	}
2281 
2282 	/*
2283 	 * Iterate through this BE's children datasets and fix
2284 	 * them if they need fixing.
2285 	 */
2286 	if (zfs_iter_filesystems(zhp, fix_mountpoint_callback, mountpoint)
2287 	    != 0) {
2288 		return (BE_ERR_ZFS);
2289 	}
2290 
2291 	/*
2292 	 * The process of mounting and unmounting the root file system
2293 	 * will fix its mountpoint to correctly be either 'legacy' or '/'
2294 	 * since be_unmount_root will do the right thing by looking at
2295 	 * its vfstab.
2296 	 */
2297 
2298 	/* Generate temporary altroot to mount the root file system */
2299 	if ((ret = be_make_tmp_mountpoint(&altroot)) != BE_SUCCESS) {
2300 		be_print_err(gettext("fix_mountpoint: failed to "
2301 		    "make temporary mountpoint\n"));
2302 		return (ret);
2303 	}
2304 
2305 	/* Mount and unmount the root. */
2306 	if ((ret = be_mount_root(zhp, altroot)) != BE_SUCCESS) {
2307 		be_print_err(gettext("fix_mountpoint: failed to "
2308 		    "mount BE root file system\n"));
2309 		goto cleanup;
2310 	}
2311 	ud.altroot = altroot;
2312 	if ((ret = be_unmount_root(zhp, &ud)) != BE_SUCCESS) {
2313 		be_print_err(gettext("fix_mountpoint: failed to "
2314 		    "unmount BE root file system\n"));
2315 		goto cleanup;
2316 	}
2317 
2318 cleanup:
2319 	free(altroot);
2320 
2321 	return (ret);
2322 }
2323 
2324 /*
2325  * Function:	be_mount_zones
2326  * Description:	This function finds all supported non-global zones in the
2327  *		given global BE and mounts them with respect to where the
2328  *		global BE is currently mounted.  The global BE datasets
2329  *		(including its shared datasets) are expected to already
2330  *		be mounted.
2331  * Parameters:
2332  *		be_zhp - zfs_handle_t pointer to the root dataset of the
2333  *			global BE.
2334  *		md - be_mount_data_t pointer to data for global BE.
2335  * Returns:
2336  *		BE_SUCCESS - Success
2337  *		be_errno_t - Failure
2338  * Scope:
2339  *		Private
2340  */
2341 static int
2342 be_mount_zones(zfs_handle_t *be_zhp, be_mount_data_t *md)
2343 {
2344 	zoneBrandList_t	*brands = NULL;
2345 	zoneList_t	zlst = NULL;
2346 	char		*zonename = NULL;
2347 	char		*zonepath = NULL;
2348 	char		*zonepath_ds = NULL;
2349 	int		k;
2350 	int		ret = BE_SUCCESS;
2351 
2352 	z_set_zone_root(md->altroot);
2353 
2354 	if ((brands = be_get_supported_brandlist()) == NULL) {
2355 		be_print_err(gettext("be_mount_zones: "
2356 		    "no supported brands\n"));
2357 		return (BE_SUCCESS);
2358 	}
2359 
2360 	zlst = z_get_nonglobal_zone_list_by_brand(brands);
2361 	if (zlst == NULL) {
2362 		z_free_brand_list(brands);
2363 		return (BE_SUCCESS);
2364 	}
2365 
2366 	for (k = 0; (zonename = z_zlist_get_zonename(zlst, k)) != NULL; k++) {
2367 		if (z_zlist_get_current_state(zlst, k) ==
2368 		    ZONE_STATE_INSTALLED) {
2369 			zonepath = z_zlist_get_zonepath(zlst, k);
2370 
2371 			/*
2372 			 * Get the dataset of this zonepath in current BE.
2373 			 * If its not a dataset, skip it.
2374 			 */
2375 			if ((zonepath_ds = be_get_ds_from_dir(zonepath))
2376 			    == NULL)
2377 				continue;
2378 
2379 			/*
2380 			 * Check if this zone is supported based on
2381 			 * the dataset of its zonepath
2382 			 */
2383 			if (!be_zone_supported(zonepath_ds)) {
2384 				free(zonepath_ds);
2385 				zonepath_ds = NULL;
2386 				continue;
2387 			}
2388 
2389 			/*
2390 			 * if BE's shared file systems are already mounted,
2391 			 * zone path dataset would have already been lofs
2392 			 * mounted under altroot. Otherwise, we need to do
2393 			 * it here.
2394 			 */
2395 			if (!md->shared_fs) {
2396 				ret = loopback_mount_zonepath(zonepath, md);
2397 				if (ret != BE_SUCCESS)
2398 					goto done;
2399 			}
2400 
2401 
2402 			/* Mount this zone */
2403 			ret = be_mount_one_zone(be_zhp, md, zonename,
2404 			    zonepath, zonepath_ds);
2405 
2406 			free(zonepath_ds);
2407 			zonepath_ds = NULL;
2408 
2409 			if (ret != BE_SUCCESS) {
2410 				be_print_err(gettext("be_mount_zones: "
2411 				    "failed to mount zone %s under "
2412 				    "altroot %s\n"), zonename, md->altroot);
2413 				goto done;
2414 			}
2415 		}
2416 	}
2417 
2418 done:
2419 	z_free_brand_list(brands);
2420 	z_free_zone_list(zlst);
2421 	/*
2422 	 * libinstzones caches mnttab and uses cached version for resolving lofs
2423 	 * mounts when we call z_resolve_lofs. It creates the cached version
2424 	 * when the first call to z_resolve_lofs happens. So, library's cached
2425 	 * mnttab doesn't contain entries for lofs mounts created in the above
2426 	 * loop. Because of this, subsequent calls to z_resolve_lofs would fail
2427 	 * to resolve these lofs mounts. So, here we destroy library's cached
2428 	 * mnttab to force its recreation when the next call to z_resolve_lofs
2429 	 * happens.
2430 	 */
2431 	z_destroyMountTable();
2432 	return (ret);
2433 }
2434 
2435 /*
2436  * Function:	be_unmount_zones
2437  * Description:	This function finds all supported non-global zones in the
2438  *		given mounted global BE and unmounts them.
2439  * Parameters:
2440  *		ud - unmount_data_t pointer data for the global BE.
2441  * Returns:
2442  *		BE_SUCCESS - Success
2443  *		be_errno_t - Failure
2444  * Scope:
2445  *		Private
2446  */
2447 static int
2448 be_unmount_zones(be_unmount_data_t *ud)
2449 {
2450 	zoneBrandList_t		*brands = NULL;
2451 	zoneList_t		zlst = NULL;
2452 	char			*zonename = NULL;
2453 	char			*zonepath = NULL;
2454 	char			alt_zonepath[MAXPATHLEN];
2455 	char			*zonepath_ds = NULL;
2456 	int			k;
2457 	int			ret = BE_SUCCESS;
2458 
2459 	z_set_zone_root(ud->altroot);
2460 
2461 	if ((brands = be_get_supported_brandlist()) == NULL) {
2462 		be_print_err(gettext("be_unmount_zones: "
2463 		    "no supported brands\n"));
2464 		return (BE_SUCCESS);
2465 	}
2466 
2467 	zlst = z_get_nonglobal_zone_list_by_brand(brands);
2468 	if (zlst == NULL) {
2469 		z_free_brand_list(brands);
2470 		return (BE_SUCCESS);
2471 	}
2472 
2473 	for (k = 0; (zonename = z_zlist_get_zonename(zlst, k)) != NULL; k++) {
2474 		if (z_zlist_get_current_state(zlst, k) ==
2475 		    ZONE_STATE_INSTALLED) {
2476 			zonepath = z_zlist_get_zonepath(zlst, k);
2477 
2478 			/* Build zone's zonepath wrt the global BE altroot */
2479 			(void) snprintf(alt_zonepath, sizeof (alt_zonepath),
2480 			    "%s%s", ud->altroot, zonepath);
2481 
2482 			/*
2483 			 * Get the dataset of this zonepath.  If its not
2484 			 * a dataset, skip it.
2485 			 */
2486 			if ((zonepath_ds = be_get_ds_from_dir(alt_zonepath))
2487 			    == NULL)
2488 				continue;
2489 
2490 			/*
2491 			 * Check if this zone is supported based on the
2492 			 * dataset of its zonepath.
2493 			 */
2494 			if (!be_zone_supported(zonepath_ds)) {
2495 				free(zonepath_ds);
2496 				zonepath_ds = NULL;
2497 				continue;
2498 			}
2499 
2500 			/* Unmount this zone */
2501 			ret = be_unmount_one_zone(ud, zonename, zonepath,
2502 			    zonepath_ds);
2503 
2504 			free(zonepath_ds);
2505 			zonepath_ds = NULL;
2506 
2507 			if (ret != BE_SUCCESS) {
2508 				be_print_err(gettext("be_unmount_zones:"
2509 				    " failed to unmount zone %s from "
2510 				    "altroot %s\n"), zonename, ud->altroot);
2511 				goto done;
2512 			}
2513 		}
2514 	}
2515 
2516 done:
2517 	z_free_brand_list(brands);
2518 	z_free_zone_list(zlst);
2519 	return (ret);
2520 }
2521 
2522 /*
2523  * Function:	be_mount_one_zone
2524  * Description:	This function is called to mount one zone for a given
2525  *		global BE.
2526  * Parameters:
2527  *		be_zhp - zfs_handle_t pointer to the root dataset of the
2528  *			global BE
2529  *		md - be_mount_data_t pointer to data for global BE
2530  *		zonename - name of zone to mount
2531  *		zonepath - zonepath of zone to mount
2532  *		zonepath_ds - dataset for the zonepath
2533  * Returns:
2534  *		BE_SUCCESS - Success
2535  *		be_errno_t - Failure
2536  * Scope:
2537  *		Private
2538  */
2539 static int
2540 be_mount_one_zone(zfs_handle_t *be_zhp, be_mount_data_t *md, char *zonename,
2541     char *zonepath, char *zonepath_ds)
2542 {
2543 	be_mount_data_t	zone_md = { 0 };
2544 	zfs_handle_t	*zone_zhp = NULL;
2545 	char		zone_altroot[MAXPATHLEN];
2546 	char		zoneroot[MAXPATHLEN];
2547 	char		zoneroot_ds[MAXPATHLEN];
2548 	int		ret = BE_SUCCESS;
2549 
2550 	/* Find the active zone root dataset for this zone for this BE */
2551 	if ((ret = be_find_active_zone_root(be_zhp, zonepath_ds, zoneroot_ds,
2552 	    sizeof (zoneroot_ds))) == BE_ERR_ZONE_NO_ACTIVE_ROOT) {
2553 		be_print_err(gettext("be_mount_one_zone: did not "
2554 		    "find active zone root for zone %s, skipping ...\n"),
2555 		    zonename);
2556 		return (BE_SUCCESS);
2557 	} else if (ret != BE_SUCCESS) {
2558 		be_print_err(gettext("be_mount_one_zone: failed to "
2559 		    "find active zone root for zone %s\n"), zonename);
2560 		return (ret);
2561 	}
2562 
2563 	/* Get handle to active zoneroot dataset */
2564 	if ((zone_zhp = zfs_open(g_zfs, zoneroot_ds, ZFS_TYPE_FILESYSTEM))
2565 	    == NULL) {
2566 		be_print_err(gettext("be_mount_one_zone: failed to "
2567 		    "open zone root dataset (%s): %s\n"), zoneroot_ds,
2568 		    libzfs_error_description(g_zfs));
2569 		return (zfs_err_to_be_err(g_zfs));
2570 	}
2571 
2572 	/* Generate string for zone's altroot path */
2573 	be_make_zoneroot(zonepath, zoneroot, sizeof (zoneroot));
2574 	(void) strlcpy(zone_altroot, md->altroot, sizeof (zone_altroot));
2575 	(void) strlcat(zone_altroot, zoneroot, sizeof (zone_altroot));
2576 
2577 	/* Build mount_data for the zone */
2578 	zone_md.altroot = zone_altroot;
2579 	zone_md.shared_fs = md->shared_fs;
2580 	zone_md.shared_rw = md->shared_rw;
2581 
2582 	/* Mount the zone's root file system */
2583 	if ((ret = be_mount_zone_root(zone_zhp, &zone_md)) != BE_SUCCESS) {
2584 		be_print_err(gettext("be_mount_one_zone: failed to "
2585 		    "mount zone root file system at %s\n"), zone_altroot);
2586 		goto done;
2587 	}
2588 
2589 	/* Iterate through zone's children filesystems */
2590 	if ((ret = zfs_iter_filesystems(zone_zhp, be_mount_callback,
2591 	    zone_altroot)) != 0) {
2592 		be_print_err(gettext("be_mount_one_zone: failed to "
2593 		    "mount zone subordinate file systems at %s\n"),
2594 		    zone_altroot);
2595 		goto done;
2596 	}
2597 
2598 	/* TODO: Mount all shared file systems for this zone */
2599 
2600 done:
2601 	ZFS_CLOSE(zone_zhp);
2602 	return (ret);
2603 }
2604 
2605 /*
2606  * Function:	be_unmount_one_zone
2607  * Description:	This function unmount one zone for a give global BE.
2608  * Parameters:
2609  *		ud - be_unmount_data_t pointer to data for global BE
2610  *		zonename - name of zone to unmount
2611  *		zonepath - zonepath of the zone to unmount
2612  *		zonepath_ds - dataset for the zonepath
2613  * Returns:
2614  *		BE_SUCCESS - Success
2615  *		be_errno_t - Failure
2616  * Scope:
2617  *		Private
2618  */
2619 static int
2620 be_unmount_one_zone(be_unmount_data_t *ud, char *zonename, char *zonepath,
2621     char *zonepath_ds)
2622 {
2623 	be_unmount_data_t	zone_ud = { 0 };
2624 	zfs_handle_t	*zone_zhp = NULL;
2625 	char		zone_altroot[MAXPATHLEN];
2626 	char		zoneroot[MAXPATHLEN];
2627 	char		zoneroot_ds[MAXPATHLEN];
2628 	int		ret = BE_SUCCESS;
2629 
2630 	/* Generate string for zone's alternate root path */
2631 	be_make_zoneroot(zonepath, zoneroot, sizeof (zoneroot));
2632 	(void) strlcpy(zone_altroot, ud->altroot, sizeof (zone_altroot));
2633 	(void) strlcat(zone_altroot, zoneroot, sizeof (zone_altroot));
2634 
2635 	/* Build be_unmount_data for zone */
2636 	zone_ud.altroot = zone_altroot;
2637 	zone_ud.force = ud->force;
2638 
2639 	/* Find the mounted zone root dataset for this zone for this BE */
2640 	if ((ret = be_find_mounted_zone_root(zone_altroot, zonepath_ds,
2641 	    zoneroot_ds, sizeof (zoneroot_ds))) == BE_ERR_NO_MOUNTED_ZONE) {
2642 		be_print_err(gettext("be_unmount_one_zone: did not "
2643 		    "find any zone root mounted for zone %s\n"), zonename);
2644 		return (BE_SUCCESS);
2645 	} else if (ret != BE_SUCCESS) {
2646 		be_print_err(gettext("be_unmount_one_zone: failed to "
2647 		    "find mounted zone root for zone %s\n"), zonename);
2648 		return (ret);
2649 	}
2650 
2651 	/* Get handle to zoneroot dataset mounted for this BE */
2652 	if ((zone_zhp = zfs_open(g_zfs, zoneroot_ds, ZFS_TYPE_FILESYSTEM))
2653 	    == NULL) {
2654 		be_print_err(gettext("be_unmount_one_zone: failed to "
2655 		    "open mounted zone root dataset (%s): %s\n"), zoneroot_ds,
2656 		    libzfs_error_description(g_zfs));
2657 		return (zfs_err_to_be_err(g_zfs));
2658 	}
2659 
2660 	/* TODO: Unmount all shared file systems for this zone */
2661 
2662 	/* Iterate through zone's children filesystems and unmount them */
2663 	if ((ret = zfs_iter_filesystems(zone_zhp, be_unmount_callback,
2664 	    &zone_ud)) != 0) {
2665 		be_print_err(gettext("be_unmount_one_zone: failed to "
2666 		    "unmount zone subordinate file systems at %s\n"),
2667 		    zone_altroot);
2668 		goto done;
2669 	}
2670 
2671 	/* Unmount the zone's root filesystem */
2672 	if ((ret = be_unmount_zone_root(zone_zhp, &zone_ud)) != BE_SUCCESS) {
2673 		be_print_err(gettext("be_unmount_one_zone: failed to "
2674 		    "unmount zone root file system at %s\n"), zone_altroot);
2675 		goto done;
2676 	}
2677 
2678 done:
2679 	ZFS_CLOSE(zone_zhp);
2680 	return (ret);
2681 }
2682 
2683 /*
2684  * Function:	be_get_ds_from_dir_callback
2685  * Description:	This is a callback function used to iterate all datasets
2686  *		to find the one that is currently mounted at the directory
2687  *		being searched for.  If matched, the name of the dataset is
2688  *		returned in heap storage, so the caller is responsible for
2689  *		freeing it.
2690  * Parameters:
2691  *		zhp - zfs_handle_t pointer to current dataset being processed.
2692  *		data - dir_data_t pointer providing name of directory being
2693  *			searched for.
2694  * Returns:
2695  *		1 - This dataset is mounted at directory being searched for.
2696  *		0 - This dataset is not mounted at directory being searched for.
2697  * Scope:
2698  *		Private
2699  */
2700 static int
2701 be_get_ds_from_dir_callback(zfs_handle_t *zhp, void *data)
2702 {
2703 	dir_data_t	*dd = data;
2704 	char		*mp = NULL;
2705 	int		zret = 0;
2706 
2707 	if (zfs_get_type(zhp) != ZFS_TYPE_FILESYSTEM) {
2708 		ZFS_CLOSE(zhp);
2709 		return (0);
2710 	}
2711 
2712 	if (zfs_is_mounted(zhp, &mp) && mp != NULL &&
2713 	    strcmp(mp, dd->dir) == 0) {
2714 		if ((dd->ds = strdup(zfs_get_name(zhp))) == NULL) {
2715 			be_print_err(gettext("be_get_ds_from_dir_callback: "
2716 			    "memory allocation failed\n"));
2717 			ZFS_CLOSE(zhp);
2718 			return (0);
2719 		}
2720 		ZFS_CLOSE(zhp);
2721 		return (1);
2722 	}
2723 
2724 	zret = zfs_iter_filesystems(zhp, be_get_ds_from_dir_callback, dd);
2725 
2726 	ZFS_CLOSE(zhp);
2727 
2728 	return (zret);
2729 }
2730