1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2013 by Delphix. All rights reserved.
25 */
26
27 /*
28 * Functions to convert between a list of vdevs and an nvlist representing the
29 * configuration. Each entry in the list can be one of:
30 *
31 * Device vdevs
32 * disk=(path=..., devid=...)
33 * file=(path=...)
34 *
35 * Group vdevs
36 * raidz[1|2]=(...)
37 * mirror=(...)
38 *
39 * Hot spares
40 *
41 * While the underlying implementation supports it, group vdevs cannot contain
42 * other group vdevs. All userland verification of devices is contained within
43 * this file. If successful, the nvlist returned can be passed directly to the
44 * kernel; we've done as much verification as possible in userland.
45 *
46 * Hot spares are a special case, and passed down as an array of disk vdevs, at
47 * the same level as the root of the vdev tree.
48 *
49 * The only function exported by this file is 'make_root_vdev'. The
50 * function performs several passes:
51 *
52 * 1. Construct the vdev specification. Performs syntax validation and
53 * makes sure each device is valid.
54 * 2. Check for devices in use. Using libdiskmgt, makes sure that no
55 * devices are also in use. Some can be overridden using the 'force'
56 * flag, others cannot.
57 * 3. Check for replication errors if the 'force' flag is not specified.
58 * validates that the replication level is consistent across the
59 * entire pool.
60 * 4. Call libzfs to label any whole disks with an EFI label.
61 */
62
63 #include <assert.h>
64 #include <devid.h>
65 #include <errno.h>
66 #include <fcntl.h>
67 #include <libdiskmgt.h>
68 #include <libintl.h>
69 #include <libnvpair.h>
70 #include <limits.h>
71 #include <stdio.h>
72 #include <string.h>
73 #include <unistd.h>
74 #include <sys/efi_partition.h>
75 #include <sys/stat.h>
76 #include <sys/vtoc.h>
77 #include <sys/mntent.h>
78
79 #include "zpool_util.h"
80
81 #define DISK_ROOT "/dev/dsk"
82 #define RDISK_ROOT "/dev/rdsk"
83 #define BACKUP_SLICE "s2"
84
85 /*
86 * For any given vdev specification, we can have multiple errors. The
87 * vdev_error() function keeps track of whether we have seen an error yet, and
88 * prints out a header if its the first error we've seen.
89 */
90 boolean_t error_seen;
91 boolean_t is_force;
92
93 /*PRINTFLIKE1*/
94 static void
vdev_error(const char * fmt,...)95 vdev_error(const char *fmt, ...)
96 {
97 va_list ap;
98
99 if (!error_seen) {
100 (void) fprintf(stderr, gettext("invalid vdev specification\n"));
101 if (!is_force)
102 (void) fprintf(stderr, gettext("use '-f' to override "
103 "the following errors:\n"));
104 else
105 (void) fprintf(stderr, gettext("the following errors "
106 "must be manually repaired:\n"));
107 error_seen = B_TRUE;
108 }
109
110 va_start(ap, fmt);
111 (void) vfprintf(stderr, fmt, ap);
112 va_end(ap);
113 }
114
115 static void
libdiskmgt_error(int error)116 libdiskmgt_error(int error)
117 {
118 /*
119 * ENXIO/ENODEV is a valid error message if the device doesn't live in
120 * /dev/dsk. Don't bother printing an error message in this case.
121 */
122 if (error == ENXIO || error == ENODEV)
123 return;
124
125 (void) fprintf(stderr, gettext("warning: device in use checking "
126 "failed: %s\n"), strerror(error));
127 }
128
129 /*
130 * Validate a device, passing the bulk of the work off to libdiskmgt.
131 */
132 static int
check_slice(const char * path,int force,boolean_t wholedisk,boolean_t isspare)133 check_slice(const char *path, int force, boolean_t wholedisk, boolean_t isspare)
134 {
135 char *msg;
136 int error = 0;
137 dm_who_type_t who;
138
139 if (force)
140 who = DM_WHO_ZPOOL_FORCE;
141 else if (isspare)
142 who = DM_WHO_ZPOOL_SPARE;
143 else
144 who = DM_WHO_ZPOOL;
145
146 if (dm_inuse((char *)path, &msg, who, &error) || error) {
147 if (error != 0) {
148 libdiskmgt_error(error);
149 return (0);
150 } else {
151 vdev_error("%s", msg);
152 free(msg);
153 return (-1);
154 }
155 }
156
157 /*
158 * If we're given a whole disk, ignore overlapping slices since we're
159 * about to label it anyway.
160 */
161 error = 0;
162 if (!wholedisk && !force &&
163 (dm_isoverlapping((char *)path, &msg, &error) || error)) {
164 if (error == 0) {
165 /* dm_isoverlapping returned -1 */
166 vdev_error(gettext("%s overlaps with %s\n"), path, msg);
167 free(msg);
168 return (-1);
169 } else if (error != ENODEV) {
170 /* libdiskmgt's devcache only handles physical drives */
171 libdiskmgt_error(error);
172 return (0);
173 }
174 }
175
176 return (0);
177 }
178
179
180 /*
181 * Validate a whole disk. Iterate over all slices on the disk and make sure
182 * that none is in use by calling check_slice().
183 */
184 static int
check_disk(const char * name,dm_descriptor_t disk,int force,int isspare)185 check_disk(const char *name, dm_descriptor_t disk, int force, int isspare)
186 {
187 dm_descriptor_t *drive, *media, *slice;
188 int err = 0;
189 int i;
190 int ret;
191
192 /*
193 * Get the drive associated with this disk. This should never fail,
194 * because we already have an alias handle open for the device.
195 */
196 if ((drive = dm_get_associated_descriptors(disk, DM_DRIVE,
197 &err)) == NULL || *drive == NULL) {
198 if (err)
199 libdiskmgt_error(err);
200 return (0);
201 }
202
203 if ((media = dm_get_associated_descriptors(*drive, DM_MEDIA,
204 &err)) == NULL) {
205 dm_free_descriptors(drive);
206 if (err)
207 libdiskmgt_error(err);
208 return (0);
209 }
210
211 dm_free_descriptors(drive);
212
213 /*
214 * It is possible that the user has specified a removable media drive,
215 * and the media is not present.
216 */
217 if (*media == NULL) {
218 dm_free_descriptors(media);
219 vdev_error(gettext("'%s' has no media in drive\n"), name);
220 return (-1);
221 }
222
223 if ((slice = dm_get_associated_descriptors(*media, DM_SLICE,
224 &err)) == NULL) {
225 dm_free_descriptors(media);
226 if (err)
227 libdiskmgt_error(err);
228 return (0);
229 }
230
231 dm_free_descriptors(media);
232
233 ret = 0;
234
235 /*
236 * Iterate over all slices and report any errors. We don't care about
237 * overlapping slices because we are using the whole disk.
238 */
239 for (i = 0; slice[i] != NULL; i++) {
240 char *name = dm_get_name(slice[i], &err);
241
242 if (check_slice(name, force, B_TRUE, isspare) != 0)
243 ret = -1;
244
245 dm_free_name(name);
246 }
247
248 dm_free_descriptors(slice);
249 return (ret);
250 }
251
252 /*
253 * Validate a device.
254 */
255 static int
check_device(const char * path,boolean_t force,boolean_t isspare)256 check_device(const char *path, boolean_t force, boolean_t isspare)
257 {
258 dm_descriptor_t desc;
259 int err;
260 char *dev;
261
262 /*
263 * For whole disks, libdiskmgt does not include the leading dev path.
264 */
265 dev = strrchr(path, '/');
266 assert(dev != NULL);
267 dev++;
268 if ((desc = dm_get_descriptor_by_name(DM_ALIAS, dev, &err)) != NULL) {
269 err = check_disk(path, desc, force, isspare);
270 dm_free_descriptor(desc);
271 return (err);
272 }
273
274 return (check_slice(path, force, B_FALSE, isspare));
275 }
276
277 /*
278 * Check that a file is valid. All we can do in this case is check that it's
279 * not in use by another pool, and not in use by swap.
280 */
281 static int
check_file(const char * file,boolean_t force,boolean_t isspare)282 check_file(const char *file, boolean_t force, boolean_t isspare)
283 {
284 char *name;
285 int fd;
286 int ret = 0;
287 int err;
288 pool_state_t state;
289 boolean_t inuse;
290
291 if (dm_inuse_swap(file, &err)) {
292 if (err)
293 libdiskmgt_error(err);
294 else
295 vdev_error(gettext("%s is currently used by swap. "
296 "Please see swap(1M).\n"), file);
297 return (-1);
298 }
299
300 if ((fd = open(file, O_RDONLY)) < 0)
301 return (0);
302
303 if (zpool_in_use(g_zfs, fd, &state, &name, &inuse) == 0 && inuse) {
304 const char *desc;
305
306 switch (state) {
307 case POOL_STATE_ACTIVE:
308 desc = gettext("active");
309 break;
310
311 case POOL_STATE_EXPORTED:
312 desc = gettext("exported");
313 break;
314
315 case POOL_STATE_POTENTIALLY_ACTIVE:
316 desc = gettext("potentially active");
317 break;
318
319 default:
320 desc = gettext("unknown");
321 break;
322 }
323
324 /*
325 * Allow hot spares to be shared between pools.
326 */
327 if (state == POOL_STATE_SPARE && isspare)
328 return (0);
329
330 if (state == POOL_STATE_ACTIVE ||
331 state == POOL_STATE_SPARE || !force) {
332 switch (state) {
333 case POOL_STATE_SPARE:
334 vdev_error(gettext("%s is reserved as a hot "
335 "spare for pool %s\n"), file, name);
336 break;
337 default:
338 vdev_error(gettext("%s is part of %s pool "
339 "'%s'\n"), file, desc, name);
340 break;
341 }
342 ret = -1;
343 }
344
345 free(name);
346 }
347
348 (void) close(fd);
349 return (ret);
350 }
351
352
353 /*
354 * By "whole disk" we mean an entire physical disk (something we can
355 * label, toggle the write cache on, etc.) as opposed to the full
356 * capacity of a pseudo-device such as lofi or did. We act as if we
357 * are labeling the disk, which should be a pretty good test of whether
358 * it's a viable device or not. Returns B_TRUE if it is and B_FALSE if
359 * it isn't.
360 */
361 static boolean_t
is_whole_disk(const char * arg)362 is_whole_disk(const char *arg)
363 {
364 struct dk_gpt *label;
365 int fd;
366 char path[MAXPATHLEN];
367
368 (void) snprintf(path, sizeof (path), "%s%s%s",
369 RDISK_ROOT, strrchr(arg, '/'), BACKUP_SLICE);
370 if ((fd = open(path, O_RDWR | O_NDELAY)) < 0)
371 return (B_FALSE);
372 if (efi_alloc_and_init(fd, EFI_NUMPAR, &label) != 0) {
373 (void) close(fd);
374 return (B_FALSE);
375 }
376 efi_free(label);
377 (void) close(fd);
378 return (B_TRUE);
379 }
380
381 /*
382 * Create a leaf vdev. Determine if this is a file or a device. If it's a
383 * device, fill in the device id to make a complete nvlist. Valid forms for a
384 * leaf vdev are:
385 *
386 * /dev/dsk/xxx Complete disk path
387 * /xxx Full path to file
388 * xxx Shorthand for /dev/dsk/xxx
389 */
390 static nvlist_t *
make_leaf_vdev(const char * arg,uint64_t is_log)391 make_leaf_vdev(const char *arg, uint64_t is_log)
392 {
393 char path[MAXPATHLEN];
394 struct stat64 statbuf;
395 nvlist_t *vdev = NULL;
396 char *type = NULL;
397 boolean_t wholedisk = B_FALSE;
398
399 /*
400 * Determine what type of vdev this is, and put the full path into
401 * 'path'. We detect whether this is a device of file afterwards by
402 * checking the st_mode of the file.
403 */
404 if (arg[0] == '/') {
405 /*
406 * Complete device or file path. Exact type is determined by
407 * examining the file descriptor afterwards.
408 */
409 wholedisk = is_whole_disk(arg);
410 if (!wholedisk && (stat64(arg, &statbuf) != 0)) {
411 (void) fprintf(stderr,
412 gettext("cannot open '%s': %s\n"),
413 arg, strerror(errno));
414 return (NULL);
415 }
416
417 (void) strlcpy(path, arg, sizeof (path));
418 } else {
419 /*
420 * This may be a short path for a device, or it could be total
421 * gibberish. Check to see if it's a known device in
422 * /dev/dsk/. As part of this check, see if we've been given a
423 * an entire disk (minus the slice number).
424 */
425 (void) snprintf(path, sizeof (path), "%s/%s", DISK_ROOT,
426 arg);
427 wholedisk = is_whole_disk(path);
428 if (!wholedisk && (stat64(path, &statbuf) != 0)) {
429 /*
430 * If we got ENOENT, then the user gave us
431 * gibberish, so try to direct them with a
432 * reasonable error message. Otherwise,
433 * regurgitate strerror() since it's the best we
434 * can do.
435 */
436 if (errno == ENOENT) {
437 (void) fprintf(stderr,
438 gettext("cannot open '%s': no such "
439 "device in %s\n"), arg, DISK_ROOT);
440 (void) fprintf(stderr,
441 gettext("must be a full path or "
442 "shorthand device name\n"));
443 return (NULL);
444 } else {
445 (void) fprintf(stderr,
446 gettext("cannot open '%s': %s\n"),
447 path, strerror(errno));
448 return (NULL);
449 }
450 }
451 }
452
453 /*
454 * Determine whether this is a device or a file.
455 */
456 if (wholedisk || S_ISBLK(statbuf.st_mode)) {
457 type = VDEV_TYPE_DISK;
458 } else if (S_ISREG(statbuf.st_mode)) {
459 type = VDEV_TYPE_FILE;
460 } else {
461 (void) fprintf(stderr, gettext("cannot use '%s': must be a "
462 "block device or regular file\n"), path);
463 return (NULL);
464 }
465
466 /*
467 * Finally, we have the complete device or file, and we know that it is
468 * acceptable to use. Construct the nvlist to describe this vdev. All
469 * vdevs have a 'path' element, and devices also have a 'devid' element.
470 */
471 verify(nvlist_alloc(&vdev, NV_UNIQUE_NAME, 0) == 0);
472 verify(nvlist_add_string(vdev, ZPOOL_CONFIG_PATH, path) == 0);
473 verify(nvlist_add_string(vdev, ZPOOL_CONFIG_TYPE, type) == 0);
474 verify(nvlist_add_uint64(vdev, ZPOOL_CONFIG_IS_LOG, is_log) == 0);
475 if (strcmp(type, VDEV_TYPE_DISK) == 0)
476 verify(nvlist_add_uint64(vdev, ZPOOL_CONFIG_WHOLE_DISK,
477 (uint64_t)wholedisk) == 0);
478
479 /*
480 * For a whole disk, defer getting its devid until after labeling it.
481 */
482 if (S_ISBLK(statbuf.st_mode) && !wholedisk) {
483 /*
484 * Get the devid for the device.
485 */
486 int fd;
487 ddi_devid_t devid;
488 char *minor = NULL, *devid_str = NULL;
489
490 if ((fd = open(path, O_RDONLY)) < 0) {
491 (void) fprintf(stderr, gettext("cannot open '%s': "
492 "%s\n"), path, strerror(errno));
493 nvlist_free(vdev);
494 return (NULL);
495 }
496
497 if (devid_get(fd, &devid) == 0) {
498 if (devid_get_minor_name(fd, &minor) == 0 &&
499 (devid_str = devid_str_encode(devid, minor)) !=
500 NULL) {
501 verify(nvlist_add_string(vdev,
502 ZPOOL_CONFIG_DEVID, devid_str) == 0);
503 }
504 if (devid_str != NULL)
505 devid_str_free(devid_str);
506 if (minor != NULL)
507 devid_str_free(minor);
508 devid_free(devid);
509 }
510
511 (void) close(fd);
512 }
513
514 return (vdev);
515 }
516
517 /*
518 * Go through and verify the replication level of the pool is consistent.
519 * Performs the following checks:
520 *
521 * For the new spec, verifies that devices in mirrors and raidz are the
522 * same size.
523 *
524 * If the current configuration already has inconsistent replication
525 * levels, ignore any other potential problems in the new spec.
526 *
527 * Otherwise, make sure that the current spec (if there is one) and the new
528 * spec have consistent replication levels.
529 */
530 typedef struct replication_level {
531 char *zprl_type;
532 uint64_t zprl_children;
533 uint64_t zprl_parity;
534 } replication_level_t;
535
536 #define ZPOOL_FUZZ (16 * 1024 * 1024)
537
538 /*
539 * Given a list of toplevel vdevs, return the current replication level. If
540 * the config is inconsistent, then NULL is returned. If 'fatal' is set, then
541 * an error message will be displayed for each self-inconsistent vdev.
542 */
543 static replication_level_t *
get_replication(nvlist_t * nvroot,boolean_t fatal)544 get_replication(nvlist_t *nvroot, boolean_t fatal)
545 {
546 nvlist_t **top;
547 uint_t t, toplevels;
548 nvlist_t **child;
549 uint_t c, children;
550 nvlist_t *nv;
551 char *type;
552 replication_level_t lastrep, rep, *ret;
553 boolean_t dontreport;
554
555 ret = safe_malloc(sizeof (replication_level_t));
556
557 verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
558 &top, &toplevels) == 0);
559
560 lastrep.zprl_type = NULL;
561 for (t = 0; t < toplevels; t++) {
562 uint64_t is_log = B_FALSE;
563
564 nv = top[t];
565
566 /*
567 * For separate logs we ignore the top level vdev replication
568 * constraints.
569 */
570 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG, &is_log);
571 if (is_log)
572 continue;
573
574 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE,
575 &type) == 0);
576 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
577 &child, &children) != 0) {
578 /*
579 * This is a 'file' or 'disk' vdev.
580 */
581 rep.zprl_type = type;
582 rep.zprl_children = 1;
583 rep.zprl_parity = 0;
584 } else {
585 uint64_t vdev_size;
586
587 /*
588 * This is a mirror or RAID-Z vdev. Go through and make
589 * sure the contents are all the same (files vs. disks),
590 * keeping track of the number of elements in the
591 * process.
592 *
593 * We also check that the size of each vdev (if it can
594 * be determined) is the same.
595 */
596 rep.zprl_type = type;
597 rep.zprl_children = 0;
598
599 if (strcmp(type, VDEV_TYPE_RAIDZ) == 0) {
600 verify(nvlist_lookup_uint64(nv,
601 ZPOOL_CONFIG_NPARITY,
602 &rep.zprl_parity) == 0);
603 assert(rep.zprl_parity != 0);
604 } else {
605 rep.zprl_parity = 0;
606 }
607
608 /*
609 * The 'dontreport' variable indicates that we've
610 * already reported an error for this spec, so don't
611 * bother doing it again.
612 */
613 type = NULL;
614 dontreport = 0;
615 vdev_size = -1ULL;
616 for (c = 0; c < children; c++) {
617 nvlist_t *cnv = child[c];
618 char *path;
619 struct stat64 statbuf;
620 uint64_t size = -1ULL;
621 char *childtype;
622 int fd, err;
623
624 rep.zprl_children++;
625
626 verify(nvlist_lookup_string(cnv,
627 ZPOOL_CONFIG_TYPE, &childtype) == 0);
628
629 /*
630 * If this is a replacing or spare vdev, then
631 * get the real first child of the vdev.
632 */
633 if (strcmp(childtype,
634 VDEV_TYPE_REPLACING) == 0 ||
635 strcmp(childtype, VDEV_TYPE_SPARE) == 0) {
636 nvlist_t **rchild;
637 uint_t rchildren;
638
639 verify(nvlist_lookup_nvlist_array(cnv,
640 ZPOOL_CONFIG_CHILDREN, &rchild,
641 &rchildren) == 0);
642 assert(rchildren == 2);
643 cnv = rchild[0];
644
645 verify(nvlist_lookup_string(cnv,
646 ZPOOL_CONFIG_TYPE,
647 &childtype) == 0);
648 }
649
650 verify(nvlist_lookup_string(cnv,
651 ZPOOL_CONFIG_PATH, &path) == 0);
652
653 /*
654 * If we have a raidz/mirror that combines disks
655 * with files, report it as an error.
656 */
657 if (!dontreport && type != NULL &&
658 strcmp(type, childtype) != 0) {
659 if (ret != NULL)
660 free(ret);
661 ret = NULL;
662 if (fatal)
663 vdev_error(gettext(
664 "mismatched replication "
665 "level: %s contains both "
666 "files and devices\n"),
667 rep.zprl_type);
668 else
669 return (NULL);
670 dontreport = B_TRUE;
671 }
672
673 /*
674 * According to stat(2), the value of 'st_size'
675 * is undefined for block devices and character
676 * devices. But there is no effective way to
677 * determine the real size in userland.
678 *
679 * Instead, we'll take advantage of an
680 * implementation detail of spec_size(). If the
681 * device is currently open, then we (should)
682 * return a valid size.
683 *
684 * If we still don't get a valid size (indicated
685 * by a size of 0 or MAXOFFSET_T), then ignore
686 * this device altogether.
687 */
688 if ((fd = open(path, O_RDONLY)) >= 0) {
689 err = fstat64(fd, &statbuf);
690 (void) close(fd);
691 } else {
692 err = stat64(path, &statbuf);
693 }
694
695 if (err != 0 ||
696 statbuf.st_size == 0 ||
697 statbuf.st_size == MAXOFFSET_T)
698 continue;
699
700 size = statbuf.st_size;
701
702 /*
703 * Also make sure that devices and
704 * slices have a consistent size. If
705 * they differ by a significant amount
706 * (~16MB) then report an error.
707 */
708 if (!dontreport &&
709 (vdev_size != -1ULL &&
710 (labs(size - vdev_size) >
711 ZPOOL_FUZZ))) {
712 if (ret != NULL)
713 free(ret);
714 ret = NULL;
715 if (fatal)
716 vdev_error(gettext(
717 "%s contains devices of "
718 "different sizes\n"),
719 rep.zprl_type);
720 else
721 return (NULL);
722 dontreport = B_TRUE;
723 }
724
725 type = childtype;
726 vdev_size = size;
727 }
728 }
729
730 /*
731 * At this point, we have the replication of the last toplevel
732 * vdev in 'rep'. Compare it to 'lastrep' to see if its
733 * different.
734 */
735 if (lastrep.zprl_type != NULL) {
736 if (strcmp(lastrep.zprl_type, rep.zprl_type) != 0) {
737 if (ret != NULL)
738 free(ret);
739 ret = NULL;
740 if (fatal)
741 vdev_error(gettext(
742 "mismatched replication level: "
743 "both %s and %s vdevs are "
744 "present\n"),
745 lastrep.zprl_type, rep.zprl_type);
746 else
747 return (NULL);
748 } else if (lastrep.zprl_parity != rep.zprl_parity) {
749 if (ret)
750 free(ret);
751 ret = NULL;
752 if (fatal)
753 vdev_error(gettext(
754 "mismatched replication level: "
755 "both %llu and %llu device parity "
756 "%s vdevs are present\n"),
757 lastrep.zprl_parity,
758 rep.zprl_parity,
759 rep.zprl_type);
760 else
761 return (NULL);
762 } else if (lastrep.zprl_children != rep.zprl_children) {
763 if (ret)
764 free(ret);
765 ret = NULL;
766 if (fatal)
767 vdev_error(gettext(
768 "mismatched replication level: "
769 "both %llu-way and %llu-way %s "
770 "vdevs are present\n"),
771 lastrep.zprl_children,
772 rep.zprl_children,
773 rep.zprl_type);
774 else
775 return (NULL);
776 }
777 }
778 lastrep = rep;
779 }
780
781 if (ret != NULL)
782 *ret = rep;
783
784 return (ret);
785 }
786
787 /*
788 * Check the replication level of the vdev spec against the current pool. Calls
789 * get_replication() to make sure the new spec is self-consistent. If the pool
790 * has a consistent replication level, then we ignore any errors. Otherwise,
791 * report any difference between the two.
792 */
793 static int
check_replication(nvlist_t * config,nvlist_t * newroot)794 check_replication(nvlist_t *config, nvlist_t *newroot)
795 {
796 nvlist_t **child;
797 uint_t children;
798 replication_level_t *current = NULL, *new;
799 int ret;
800
801 /*
802 * If we have a current pool configuration, check to see if it's
803 * self-consistent. If not, simply return success.
804 */
805 if (config != NULL) {
806 nvlist_t *nvroot;
807
808 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
809 &nvroot) == 0);
810 if ((current = get_replication(nvroot, B_FALSE)) == NULL)
811 return (0);
812 }
813 /*
814 * for spares there may be no children, and therefore no
815 * replication level to check
816 */
817 if ((nvlist_lookup_nvlist_array(newroot, ZPOOL_CONFIG_CHILDREN,
818 &child, &children) != 0) || (children == 0)) {
819 free(current);
820 return (0);
821 }
822
823 /*
824 * If all we have is logs then there's no replication level to check.
825 */
826 if (num_logs(newroot) == children) {
827 free(current);
828 return (0);
829 }
830
831 /*
832 * Get the replication level of the new vdev spec, reporting any
833 * inconsistencies found.
834 */
835 if ((new = get_replication(newroot, B_TRUE)) == NULL) {
836 free(current);
837 return (-1);
838 }
839
840 /*
841 * Check to see if the new vdev spec matches the replication level of
842 * the current pool.
843 */
844 ret = 0;
845 if (current != NULL) {
846 if (strcmp(current->zprl_type, new->zprl_type) != 0) {
847 vdev_error(gettext(
848 "mismatched replication level: pool uses %s "
849 "and new vdev is %s\n"),
850 current->zprl_type, new->zprl_type);
851 ret = -1;
852 } else if (current->zprl_parity != new->zprl_parity) {
853 vdev_error(gettext(
854 "mismatched replication level: pool uses %llu "
855 "device parity and new vdev uses %llu\n"),
856 current->zprl_parity, new->zprl_parity);
857 ret = -1;
858 } else if (current->zprl_children != new->zprl_children) {
859 vdev_error(gettext(
860 "mismatched replication level: pool uses %llu-way "
861 "%s and new vdev uses %llu-way %s\n"),
862 current->zprl_children, current->zprl_type,
863 new->zprl_children, new->zprl_type);
864 ret = -1;
865 }
866 }
867
868 free(new);
869 if (current != NULL)
870 free(current);
871
872 return (ret);
873 }
874
875 /*
876 * Go through and find any whole disks in the vdev specification, labelling them
877 * as appropriate. When constructing the vdev spec, we were unable to open this
878 * device in order to provide a devid. Now that we have labelled the disk and
879 * know that slice 0 is valid, we can construct the devid now.
880 *
881 * If the disk was already labeled with an EFI label, we will have gotten the
882 * devid already (because we were able to open the whole disk). Otherwise, we
883 * need to get the devid after we label the disk.
884 */
885 static int
make_disks(zpool_handle_t * zhp,nvlist_t * nv)886 make_disks(zpool_handle_t *zhp, nvlist_t *nv)
887 {
888 nvlist_t **child;
889 uint_t c, children;
890 char *type, *path, *diskname;
891 char buf[MAXPATHLEN];
892 uint64_t wholedisk;
893 int fd;
894 int ret;
895 ddi_devid_t devid;
896 char *minor = NULL, *devid_str = NULL;
897
898 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
899
900 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
901 &child, &children) != 0) {
902
903 if (strcmp(type, VDEV_TYPE_DISK) != 0)
904 return (0);
905
906 /*
907 * We have a disk device. Get the path to the device
908 * and see if it's a whole disk by appending the backup
909 * slice and stat()ing the device.
910 */
911 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
912 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
913 &wholedisk) != 0 || !wholedisk)
914 return (0);
915
916 diskname = strrchr(path, '/');
917 assert(diskname != NULL);
918 diskname++;
919 if (zpool_label_disk(g_zfs, zhp, diskname) == -1)
920 return (-1);
921
922 /*
923 * Fill in the devid, now that we've labeled the disk.
924 */
925 (void) snprintf(buf, sizeof (buf), "%ss0", path);
926 if ((fd = open(buf, O_RDONLY)) < 0) {
927 (void) fprintf(stderr,
928 gettext("cannot open '%s': %s\n"),
929 buf, strerror(errno));
930 return (-1);
931 }
932
933 if (devid_get(fd, &devid) == 0) {
934 if (devid_get_minor_name(fd, &minor) == 0 &&
935 (devid_str = devid_str_encode(devid, minor)) !=
936 NULL) {
937 verify(nvlist_add_string(nv,
938 ZPOOL_CONFIG_DEVID, devid_str) == 0);
939 }
940 if (devid_str != NULL)
941 devid_str_free(devid_str);
942 if (minor != NULL)
943 devid_str_free(minor);
944 devid_free(devid);
945 }
946
947 /*
948 * Update the path to refer to the 's0' slice. The presence of
949 * the 'whole_disk' field indicates to the CLI that we should
950 * chop off the slice number when displaying the device in
951 * future output.
952 */
953 verify(nvlist_add_string(nv, ZPOOL_CONFIG_PATH, buf) == 0);
954
955 (void) close(fd);
956
957 return (0);
958 }
959
960 for (c = 0; c < children; c++)
961 if ((ret = make_disks(zhp, child[c])) != 0)
962 return (ret);
963
964 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
965 &child, &children) == 0)
966 for (c = 0; c < children; c++)
967 if ((ret = make_disks(zhp, child[c])) != 0)
968 return (ret);
969
970 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
971 &child, &children) == 0)
972 for (c = 0; c < children; c++)
973 if ((ret = make_disks(zhp, child[c])) != 0)
974 return (ret);
975
976 return (0);
977 }
978
979 /*
980 * Determine if the given path is a hot spare within the given configuration.
981 */
982 static boolean_t
is_spare(nvlist_t * config,const char * path)983 is_spare(nvlist_t *config, const char *path)
984 {
985 int fd;
986 pool_state_t state;
987 char *name = NULL;
988 nvlist_t *label;
989 uint64_t guid, spareguid;
990 nvlist_t *nvroot;
991 nvlist_t **spares;
992 uint_t i, nspares;
993 boolean_t inuse;
994
995 if ((fd = open(path, O_RDONLY)) < 0)
996 return (B_FALSE);
997
998 if (zpool_in_use(g_zfs, fd, &state, &name, &inuse) != 0 ||
999 !inuse ||
1000 state != POOL_STATE_SPARE ||
1001 zpool_read_label(fd, &label) != 0) {
1002 free(name);
1003 (void) close(fd);
1004 return (B_FALSE);
1005 }
1006 free(name);
1007 (void) close(fd);
1008
1009 verify(nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) == 0);
1010 nvlist_free(label);
1011
1012 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1013 &nvroot) == 0);
1014 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1015 &spares, &nspares) == 0) {
1016 for (i = 0; i < nspares; i++) {
1017 verify(nvlist_lookup_uint64(spares[i],
1018 ZPOOL_CONFIG_GUID, &spareguid) == 0);
1019 if (spareguid == guid)
1020 return (B_TRUE);
1021 }
1022 }
1023
1024 return (B_FALSE);
1025 }
1026
1027 /*
1028 * Go through and find any devices that are in use. We rely on libdiskmgt for
1029 * the majority of this task.
1030 */
1031 static boolean_t
is_device_in_use(nvlist_t * config,nvlist_t * nv,boolean_t force,boolean_t replacing,boolean_t isspare)1032 is_device_in_use(nvlist_t *config, nvlist_t *nv, boolean_t force,
1033 boolean_t replacing, boolean_t isspare)
1034 {
1035 nvlist_t **child;
1036 uint_t c, children;
1037 char *type, *path;
1038 int ret;
1039 char buf[MAXPATHLEN];
1040 uint64_t wholedisk;
1041 boolean_t anyinuse = B_FALSE;
1042
1043 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1044
1045 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1046 &child, &children) != 0) {
1047
1048 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1049
1050 /*
1051 * As a generic check, we look to see if this is a replace of a
1052 * hot spare within the same pool. If so, we allow it
1053 * regardless of what libdiskmgt or zpool_in_use() says.
1054 */
1055 if (replacing) {
1056 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
1057 &wholedisk) == 0 && wholedisk)
1058 (void) snprintf(buf, sizeof (buf), "%ss0",
1059 path);
1060 else
1061 (void) strlcpy(buf, path, sizeof (buf));
1062
1063 if (is_spare(config, buf))
1064 return (B_FALSE);
1065 }
1066
1067 if (strcmp(type, VDEV_TYPE_DISK) == 0)
1068 ret = check_device(path, force, isspare);
1069 else if (strcmp(type, VDEV_TYPE_FILE) == 0)
1070 ret = check_file(path, force, isspare);
1071
1072 return (ret != 0);
1073 }
1074
1075 for (c = 0; c < children; c++)
1076 if (is_device_in_use(config, child[c], force, replacing,
1077 B_FALSE))
1078 anyinuse = B_TRUE;
1079
1080 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1081 &child, &children) == 0)
1082 for (c = 0; c < children; c++)
1083 if (is_device_in_use(config, child[c], force, replacing,
1084 B_TRUE))
1085 anyinuse = B_TRUE;
1086
1087 if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1088 &child, &children) == 0)
1089 for (c = 0; c < children; c++)
1090 if (is_device_in_use(config, child[c], force, replacing,
1091 B_FALSE))
1092 anyinuse = B_TRUE;
1093
1094 return (anyinuse);
1095 }
1096
1097 static const char *
is_grouping(const char * type,int * mindev,int * maxdev)1098 is_grouping(const char *type, int *mindev, int *maxdev)
1099 {
1100 if (strncmp(type, "raidz", 5) == 0) {
1101 const char *p = type + 5;
1102 char *end;
1103 long nparity;
1104
1105 if (*p == '\0') {
1106 nparity = 1;
1107 } else if (*p == '0') {
1108 return (NULL); /* no zero prefixes allowed */
1109 } else {
1110 errno = 0;
1111 nparity = strtol(p, &end, 10);
1112 if (errno != 0 || nparity < 1 || nparity >= 255 ||
1113 *end != '\0')
1114 return (NULL);
1115 }
1116
1117 if (mindev != NULL)
1118 *mindev = nparity + 1;
1119 if (maxdev != NULL)
1120 *maxdev = 255;
1121 return (VDEV_TYPE_RAIDZ);
1122 }
1123
1124 if (maxdev != NULL)
1125 *maxdev = INT_MAX;
1126
1127 if (strcmp(type, "mirror") == 0) {
1128 if (mindev != NULL)
1129 *mindev = 2;
1130 return (VDEV_TYPE_MIRROR);
1131 }
1132
1133 if (strcmp(type, "spare") == 0) {
1134 if (mindev != NULL)
1135 *mindev = 1;
1136 return (VDEV_TYPE_SPARE);
1137 }
1138
1139 if (strcmp(type, "log") == 0) {
1140 if (mindev != NULL)
1141 *mindev = 1;
1142 return (VDEV_TYPE_LOG);
1143 }
1144
1145 if (strcmp(type, "cache") == 0) {
1146 if (mindev != NULL)
1147 *mindev = 1;
1148 return (VDEV_TYPE_L2CACHE);
1149 }
1150
1151 return (NULL);
1152 }
1153
1154 /*
1155 * Construct a syntactically valid vdev specification,
1156 * and ensure that all devices and files exist and can be opened.
1157 * Note: we don't bother freeing anything in the error paths
1158 * because the program is just going to exit anyway.
1159 */
1160 nvlist_t *
construct_spec(int argc,char ** argv)1161 construct_spec(int argc, char **argv)
1162 {
1163 nvlist_t *nvroot, *nv, **top, **spares, **l2cache;
1164 int t, toplevels, mindev, maxdev, nspares, nlogs, nl2cache;
1165 const char *type;
1166 uint64_t is_log;
1167 boolean_t seen_logs;
1168
1169 top = NULL;
1170 toplevels = 0;
1171 spares = NULL;
1172 l2cache = NULL;
1173 nspares = 0;
1174 nlogs = 0;
1175 nl2cache = 0;
1176 is_log = B_FALSE;
1177 seen_logs = B_FALSE;
1178
1179 while (argc > 0) {
1180 nv = NULL;
1181
1182 /*
1183 * If it's a mirror or raidz, the subsequent arguments are
1184 * its leaves -- until we encounter the next mirror or raidz.
1185 */
1186 if ((type = is_grouping(argv[0], &mindev, &maxdev)) != NULL) {
1187 nvlist_t **child = NULL;
1188 int c, children = 0;
1189
1190 if (strcmp(type, VDEV_TYPE_SPARE) == 0) {
1191 if (spares != NULL) {
1192 (void) fprintf(stderr,
1193 gettext("invalid vdev "
1194 "specification: 'spare' can be "
1195 "specified only once\n"));
1196 return (NULL);
1197 }
1198 is_log = B_FALSE;
1199 }
1200
1201 if (strcmp(type, VDEV_TYPE_LOG) == 0) {
1202 if (seen_logs) {
1203 (void) fprintf(stderr,
1204 gettext("invalid vdev "
1205 "specification: 'log' can be "
1206 "specified only once\n"));
1207 return (NULL);
1208 }
1209 seen_logs = B_TRUE;
1210 is_log = B_TRUE;
1211 argc--;
1212 argv++;
1213 /*
1214 * A log is not a real grouping device.
1215 * We just set is_log and continue.
1216 */
1217 continue;
1218 }
1219
1220 if (strcmp(type, VDEV_TYPE_L2CACHE) == 0) {
1221 if (l2cache != NULL) {
1222 (void) fprintf(stderr,
1223 gettext("invalid vdev "
1224 "specification: 'cache' can be "
1225 "specified only once\n"));
1226 return (NULL);
1227 }
1228 is_log = B_FALSE;
1229 }
1230
1231 if (is_log) {
1232 if (strcmp(type, VDEV_TYPE_MIRROR) != 0) {
1233 (void) fprintf(stderr,
1234 gettext("invalid vdev "
1235 "specification: unsupported 'log' "
1236 "device: %s\n"), type);
1237 return (NULL);
1238 }
1239 nlogs++;
1240 }
1241
1242 for (c = 1; c < argc; c++) {
1243 if (is_grouping(argv[c], NULL, NULL) != NULL)
1244 break;
1245 children++;
1246 child = realloc(child,
1247 children * sizeof (nvlist_t *));
1248 if (child == NULL)
1249 zpool_no_memory();
1250 if ((nv = make_leaf_vdev(argv[c], B_FALSE))
1251 == NULL)
1252 return (NULL);
1253 child[children - 1] = nv;
1254 }
1255
1256 if (children < mindev) {
1257 (void) fprintf(stderr, gettext("invalid vdev "
1258 "specification: %s requires at least %d "
1259 "devices\n"), argv[0], mindev);
1260 return (NULL);
1261 }
1262
1263 if (children > maxdev) {
1264 (void) fprintf(stderr, gettext("invalid vdev "
1265 "specification: %s supports no more than "
1266 "%d devices\n"), argv[0], maxdev);
1267 return (NULL);
1268 }
1269
1270 argc -= c;
1271 argv += c;
1272
1273 if (strcmp(type, VDEV_TYPE_SPARE) == 0) {
1274 spares = child;
1275 nspares = children;
1276 continue;
1277 } else if (strcmp(type, VDEV_TYPE_L2CACHE) == 0) {
1278 l2cache = child;
1279 nl2cache = children;
1280 continue;
1281 } else {
1282 verify(nvlist_alloc(&nv, NV_UNIQUE_NAME,
1283 0) == 0);
1284 verify(nvlist_add_string(nv, ZPOOL_CONFIG_TYPE,
1285 type) == 0);
1286 verify(nvlist_add_uint64(nv,
1287 ZPOOL_CONFIG_IS_LOG, is_log) == 0);
1288 if (strcmp(type, VDEV_TYPE_RAIDZ) == 0) {
1289 verify(nvlist_add_uint64(nv,
1290 ZPOOL_CONFIG_NPARITY,
1291 mindev - 1) == 0);
1292 }
1293 verify(nvlist_add_nvlist_array(nv,
1294 ZPOOL_CONFIG_CHILDREN, child,
1295 children) == 0);
1296
1297 for (c = 0; c < children; c++)
1298 nvlist_free(child[c]);
1299 free(child);
1300 }
1301 } else {
1302 /*
1303 * We have a device. Pass off to make_leaf_vdev() to
1304 * construct the appropriate nvlist describing the vdev.
1305 */
1306 if ((nv = make_leaf_vdev(argv[0], is_log)) == NULL)
1307 return (NULL);
1308 if (is_log)
1309 nlogs++;
1310 argc--;
1311 argv++;
1312 }
1313
1314 toplevels++;
1315 top = realloc(top, toplevels * sizeof (nvlist_t *));
1316 if (top == NULL)
1317 zpool_no_memory();
1318 top[toplevels - 1] = nv;
1319 }
1320
1321 if (toplevels == 0 && nspares == 0 && nl2cache == 0) {
1322 (void) fprintf(stderr, gettext("invalid vdev "
1323 "specification: at least one toplevel vdev must be "
1324 "specified\n"));
1325 return (NULL);
1326 }
1327
1328 if (seen_logs && nlogs == 0) {
1329 (void) fprintf(stderr, gettext("invalid vdev specification: "
1330 "log requires at least 1 device\n"));
1331 return (NULL);
1332 }
1333
1334 /*
1335 * Finally, create nvroot and add all top-level vdevs to it.
1336 */
1337 verify(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, 0) == 0);
1338 verify(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE,
1339 VDEV_TYPE_ROOT) == 0);
1340 verify(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
1341 top, toplevels) == 0);
1342 if (nspares != 0)
1343 verify(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1344 spares, nspares) == 0);
1345 if (nl2cache != 0)
1346 verify(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
1347 l2cache, nl2cache) == 0);
1348
1349 for (t = 0; t < toplevels; t++)
1350 nvlist_free(top[t]);
1351 for (t = 0; t < nspares; t++)
1352 nvlist_free(spares[t]);
1353 for (t = 0; t < nl2cache; t++)
1354 nvlist_free(l2cache[t]);
1355 if (spares)
1356 free(spares);
1357 if (l2cache)
1358 free(l2cache);
1359 free(top);
1360
1361 return (nvroot);
1362 }
1363
1364 nvlist_t *
split_mirror_vdev(zpool_handle_t * zhp,char * newname,nvlist_t * props,splitflags_t flags,int argc,char ** argv)1365 split_mirror_vdev(zpool_handle_t *zhp, char *newname, nvlist_t *props,
1366 splitflags_t flags, int argc, char **argv)
1367 {
1368 nvlist_t *newroot = NULL, **child;
1369 uint_t c, children;
1370
1371 if (argc > 0) {
1372 if ((newroot = construct_spec(argc, argv)) == NULL) {
1373 (void) fprintf(stderr, gettext("Unable to build a "
1374 "pool from the specified devices\n"));
1375 return (NULL);
1376 }
1377
1378 if (!flags.dryrun && make_disks(zhp, newroot) != 0) {
1379 nvlist_free(newroot);
1380 return (NULL);
1381 }
1382
1383 /* avoid any tricks in the spec */
1384 verify(nvlist_lookup_nvlist_array(newroot,
1385 ZPOOL_CONFIG_CHILDREN, &child, &children) == 0);
1386 for (c = 0; c < children; c++) {
1387 char *path;
1388 const char *type;
1389 int min, max;
1390
1391 verify(nvlist_lookup_string(child[c],
1392 ZPOOL_CONFIG_PATH, &path) == 0);
1393 if ((type = is_grouping(path, &min, &max)) != NULL) {
1394 (void) fprintf(stderr, gettext("Cannot use "
1395 "'%s' as a device for splitting\n"), type);
1396 nvlist_free(newroot);
1397 return (NULL);
1398 }
1399 }
1400 }
1401
1402 if (zpool_vdev_split(zhp, newname, &newroot, props, flags) != 0) {
1403 if (newroot != NULL)
1404 nvlist_free(newroot);
1405 return (NULL);
1406 }
1407
1408 return (newroot);
1409 }
1410
1411 /*
1412 * Get and validate the contents of the given vdev specification. This ensures
1413 * that the nvlist returned is well-formed, that all the devices exist, and that
1414 * they are not currently in use by any other known consumer. The 'poolconfig'
1415 * parameter is the current configuration of the pool when adding devices
1416 * existing pool, and is used to perform additional checks, such as changing the
1417 * replication level of the pool. It can be 'NULL' to indicate that this is a
1418 * new pool. The 'force' flag controls whether devices should be forcefully
1419 * added, even if they appear in use.
1420 */
1421 nvlist_t *
make_root_vdev(zpool_handle_t * zhp,int force,int check_rep,boolean_t replacing,boolean_t dryrun,int argc,char ** argv)1422 make_root_vdev(zpool_handle_t *zhp, int force, int check_rep,
1423 boolean_t replacing, boolean_t dryrun, int argc, char **argv)
1424 {
1425 nvlist_t *newroot;
1426 nvlist_t *poolconfig = NULL;
1427 is_force = force;
1428
1429 /*
1430 * Construct the vdev specification. If this is successful, we know
1431 * that we have a valid specification, and that all devices can be
1432 * opened.
1433 */
1434 if ((newroot = construct_spec(argc, argv)) == NULL)
1435 return (NULL);
1436
1437 if (zhp && ((poolconfig = zpool_get_config(zhp, NULL)) == NULL))
1438 return (NULL);
1439
1440 /*
1441 * Validate each device to make sure that its not shared with another
1442 * subsystem. We do this even if 'force' is set, because there are some
1443 * uses (such as a dedicated dump device) that even '-f' cannot
1444 * override.
1445 */
1446 if (is_device_in_use(poolconfig, newroot, force, replacing, B_FALSE)) {
1447 nvlist_free(newroot);
1448 return (NULL);
1449 }
1450
1451 /*
1452 * Check the replication level of the given vdevs and report any errors
1453 * found. We include the existing pool spec, if any, as we need to
1454 * catch changes against the existing replication level.
1455 */
1456 if (check_rep && check_replication(poolconfig, newroot) != 0) {
1457 nvlist_free(newroot);
1458 return (NULL);
1459 }
1460
1461 /*
1462 * Run through the vdev specification and label any whole disks found.
1463 */
1464 if (!dryrun && make_disks(zhp, newroot) != 0) {
1465 nvlist_free(newroot);
1466 return (NULL);
1467 }
1468
1469 return (newroot);
1470 }
1471