xref: /freebsd/usr.sbin/mfiutil/mfi_config.c (revision 8833b15f6382203d094cd6eb69068709b467079d)
1 /*-
2  * Copyright (c) 2008, 2009 Yahoo!, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. The names of the authors may not be used to endorse or promote
14  *    products derived from this software without specific prior written
15  *    permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/param.h>
33 #ifdef DEBUG
34 #include <sys/sysctl.h>
35 #endif
36 #include <err.h>
37 #include <errno.h>
38 #include <libutil.h>
39 #ifdef DEBUG
40 #include <stdint.h>
41 #endif
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <string.h>
45 #include <unistd.h>
46 #include "mfiutil.h"
47 
48 #ifdef DEBUG
49 static void	dump_config(int fd, struct mfi_config_data *config);
50 #endif
51 
52 static int	add_spare(int ac, char **av);
53 static int	remove_spare(int ac, char **av);
54 
55 static long
56 dehumanize(const char *value)
57 {
58         char    *vtp;
59         long    iv;
60 
61         if (value == NULL)
62                 return (0);
63         iv = strtoq(value, &vtp, 0);
64         if (vtp == value || (vtp[0] != '\0' && vtp[1] != '\0')) {
65                 return (0);
66         }
67         switch (vtp[0]) {
68         case 't': case 'T':
69                 iv *= 1024;
70         case 'g': case 'G':
71                 iv *= 1024;
72         case 'm': case 'M':
73                 iv *= 1024;
74         case 'k': case 'K':
75                 iv *= 1024;
76         case '\0':
77                 break;
78         default:
79                 return (0);
80         }
81         return (iv);
82 }
83 int
84 mfi_config_read(int fd, struct mfi_config_data **configp)
85 {
86 	struct mfi_config_data *config;
87 	uint32_t config_size;
88 	int error;
89 
90 	/*
91 	 * Keep fetching the config in a loop until we have a large enough
92 	 * buffer to hold the entire configuration.
93 	 */
94 	config = NULL;
95 	config_size = 1024;
96 fetch:
97 	config = reallocf(config, config_size);
98 	if (config == NULL)
99 		return (-1);
100 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_READ, config,
101 	    config_size, NULL, 0, NULL) < 0) {
102 		error = errno;
103 		free(config);
104 		errno = error;
105 		return (-1);
106 	}
107 
108 	if (config->size > config_size) {
109 		config_size = config->size;
110 		goto fetch;
111 	}
112 
113 	*configp = config;
114 	return (0);
115 }
116 
117 static struct mfi_array *
118 mfi_config_lookup_array(struct mfi_config_data *config, uint16_t array_ref)
119 {
120 	struct mfi_array *ar;
121 	char *p;
122 	int i;
123 
124 	p = (char *)config->array;
125 	for (i = 0; i < config->array_count; i++) {
126 		ar = (struct mfi_array *)p;
127 		if (ar->array_ref == array_ref)
128 			return (ar);
129 		p += config->array_size;
130 	}
131 
132 	return (NULL);
133 }
134 
135 static struct mfi_ld_config *
136 mfi_config_lookup_volume(struct mfi_config_data *config, uint8_t target_id)
137 {
138 	struct mfi_ld_config *ld;
139 	char *p;
140 	int i;
141 
142 	p = (char *)config->array + config->array_count * config->array_size;
143 	for (i = 0; i < config->log_drv_count; i++) {
144 		ld = (struct mfi_ld_config *)p;
145 		if (ld->properties.ld.v.target_id == target_id)
146 			return (ld);
147 		p += config->log_drv_size;
148 	}
149 
150 	return (NULL);
151 }
152 
153 static int
154 clear_config(int ac, char **av)
155 {
156 	struct mfi_ld_list list;
157 	int ch, error, fd;
158 	u_int i;
159 
160 	fd = mfi_open(mfi_unit);
161 	if (fd < 0) {
162 		error = errno;
163 		warn("mfi_open");
164 		return (error);
165 	}
166 
167 	if (!mfi_reconfig_supported()) {
168 		warnx("The current mfi(4) driver does not support "
169 		    "configuration changes.");
170 		close(fd);
171 		return (EOPNOTSUPP);
172 	}
173 
174 	if (mfi_ld_get_list(fd, &list, NULL) < 0) {
175 		error = errno;
176 		warn("Failed to get volume list");
177 		close(fd);
178 		return (error);
179 	}
180 
181 	for (i = 0; i < list.ld_count; i++) {
182 		if (mfi_volume_busy(fd, list.ld_list[i].ld.v.target_id)) {
183 			warnx("Volume %s is busy and cannot be deleted",
184 			    mfi_volume_name(fd, list.ld_list[i].ld.v.target_id));
185 			close(fd);
186 			return (EBUSY);
187 		}
188 	}
189 
190 	printf(
191 	    "Are you sure you wish to clear the configuration on mfi%u? [y/N] ",
192 	    mfi_unit);
193 	ch = getchar();
194 	if (ch != 'y' && ch != 'Y') {
195 		printf("\nAborting\n");
196 		close(fd);
197 		return (0);
198 	}
199 
200 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_CLEAR, NULL, 0, NULL, 0, NULL) < 0) {
201 		error = errno;
202 		warn("Failed to clear configuration");
203 		close(fd);
204 		return (error);
205 	}
206 
207 	printf("mfi%d: Configuration cleared\n", mfi_unit);
208 	close(fd);
209 
210 	return (0);
211 }
212 MFI_COMMAND(top, clear, clear_config);
213 
214 #define MAX_DRIVES_PER_ARRAY MFI_MAX_ROW_SIZE
215 #define MFI_ARRAY_SIZE sizeof(struct mfi_array)
216 
217 #define	RT_RAID0	0
218 #define	RT_RAID1	1
219 #define	RT_RAID5	2
220 #define	RT_RAID6	3
221 #define	RT_JBOD		4
222 #define	RT_CONCAT	5
223 #define	RT_RAID10	6
224 #define	RT_RAID50	7
225 #define	RT_RAID60	8
226 
227 static int
228 compare_int(const void *one, const void *two)
229 {
230 	int first, second;
231 
232 	first = *(const int *)one;
233 	second = *(const int *)two;
234 
235 	return (first - second);
236 }
237 
238 static struct raid_type_entry {
239 	const char *name;
240 	int	raid_type;
241 } raid_type_table[] = {
242 	{ "raid0",	RT_RAID0 },
243 	{ "raid-0",	RT_RAID0 },
244 	{ "raid1",	RT_RAID1 },
245 	{ "raid-1",	RT_RAID1 },
246 	{ "mirror",	RT_RAID1 },
247 	{ "raid5",	RT_RAID5 },
248 	{ "raid-5",	RT_RAID5 },
249 	{ "raid6",	RT_RAID6 },
250 	{ "raid-6",	RT_RAID6 },
251 	{ "jbod",	RT_JBOD },
252 	{ "concat",	RT_CONCAT },
253 	{ "raid10",	RT_RAID10 },
254 	{ "raid1+0",	RT_RAID10 },
255 	{ "raid-10",	RT_RAID10 },
256 	{ "raid-1+0",	RT_RAID10 },
257 	{ "raid50",	RT_RAID50 },
258 	{ "raid5+0",	RT_RAID50 },
259 	{ "raid-50",	RT_RAID50 },
260 	{ "raid-5+0",	RT_RAID50 },
261 	{ "raid60",	RT_RAID60 },
262 	{ "raid6+0",	RT_RAID60 },
263 	{ "raid-60",	RT_RAID60 },
264 	{ "raid-6+0",	RT_RAID60 },
265 	{ NULL,		0 },
266 };
267 
268 struct config_id_state {
269 	int	array_count;
270 	int	log_drv_count;
271 	int	*arrays;
272 	int	*volumes;
273 	uint16_t array_ref;
274 	uint8_t	target_id;
275 };
276 
277 struct array_info {
278 	int	drive_count;
279 	struct mfi_pd_info *drives;
280 	struct mfi_array *array;
281 };
282 
283 /* Parse a comma-separated list of drives for an array. */
284 static int
285 parse_array(int fd, int raid_type, char *array_str, struct array_info *info)
286 {
287 	struct mfi_pd_info *pinfo;
288 	uint16_t device_id;
289 	char *cp;
290 	u_int count;
291 	int error;
292 
293 	cp = array_str;
294 	for (count = 0; cp != NULL; count++) {
295 		cp = strchr(cp, ',');
296 		if (cp != NULL) {
297 			cp++;
298 			if (*cp == ',') {
299 				warnx("Invalid drive list '%s'", array_str);
300 				return (EINVAL);
301 			}
302 		}
303 	}
304 
305 	/* Validate the number of drives for this array. */
306 	if (count >= MAX_DRIVES_PER_ARRAY) {
307 		warnx("Too many drives for a single array: max is %d",
308 		    MAX_DRIVES_PER_ARRAY);
309 		return (EINVAL);
310 	}
311 	switch (raid_type) {
312 	case RT_RAID1:
313 	case RT_RAID10:
314 		if (count % 2 != 0) {
315 			warnx("RAID1 and RAID10 require an even number of "
316 			    "drives in each array");
317 			return (EINVAL);
318 		}
319 		break;
320 	case RT_RAID5:
321 	case RT_RAID50:
322 		if (count < 3) {
323 			warnx("RAID5 and RAID50 require at least 3 drives in "
324 			    "each array");
325 			return (EINVAL);
326 		}
327 		break;
328 	case RT_RAID6:
329 	case RT_RAID60:
330 		if (count < 4) {
331 			warnx("RAID6 and RAID60 require at least 4 drives in "
332 			    "each array");
333 			return (EINVAL);
334 		}
335 		break;
336 	}
337 
338 	/* Validate each drive. */
339 	info->drives = calloc(count, sizeof(struct mfi_pd_info));
340 	if (info->drives == NULL) {
341 		warnx("malloc failed");
342 		return (ENOMEM);
343 	}
344 	info->drive_count = count;
345 	for (pinfo = info->drives; (cp = strsep(&array_str, ",")) != NULL;
346 	     pinfo++) {
347 		error = mfi_lookup_drive(fd, cp, &device_id);
348 		if (error) {
349 			free(info->drives);
350 			info->drives = NULL;
351 			return (error);
352 		}
353 
354 		if (mfi_pd_get_info(fd, device_id, pinfo, NULL) < 0) {
355 			error = errno;
356 			warn("Failed to fetch drive info for drive %s", cp);
357 			free(info->drives);
358 			info->drives = NULL;
359 			return (error);
360 		}
361 
362 		if (pinfo->fw_state != MFI_PD_STATE_UNCONFIGURED_GOOD) {
363 			warnx("Drive %u is not available", device_id);
364 			free(info->drives);
365 			info->drives = NULL;
366 			return (EINVAL);
367 		}
368 	}
369 
370 	return (0);
371 }
372 
373 /*
374  * Find the next free array ref assuming that 'array_ref' is the last
375  * one used.  'array_ref' should be 0xffff for the initial test.
376  */
377 static uint16_t
378 find_next_array(struct config_id_state *state)
379 {
380 	int i;
381 
382 	/* Assume the current one is used. */
383 	state->array_ref++;
384 
385 	/* Find the next free one. */
386 	for (i = 0; i < state->array_count; i++)
387 		if (state->arrays[i] == state->array_ref)
388 			state->array_ref++;
389 	return (state->array_ref);
390 }
391 
392 /*
393  * Find the next free volume ID assuming that 'target_id' is the last
394  * one used.  'target_id' should be 0xff for the initial test.
395  */
396 static uint8_t
397 find_next_volume(struct config_id_state *state)
398 {
399 	int i;
400 
401 	/* Assume the current one is used. */
402 	state->target_id++;
403 
404 	/* Find the next free one. */
405 	for (i = 0; i < state->log_drv_count; i++)
406 		if (state->volumes[i] == state->target_id)
407 			state->target_id++;
408 	return (state->target_id);
409 }
410 
411 /* Populate an array with drives. */
412 static void
413 build_array(int fd, char *arrayp, struct array_info *array_info,
414     struct config_id_state *state, int verbose)
415 {
416 	struct mfi_array *ar = (struct mfi_array *)arrayp;
417 	int i;
418 
419 	ar->size = array_info->drives[0].coerced_size;
420 	ar->num_drives = array_info->drive_count;
421 	ar->array_ref = find_next_array(state);
422 	for (i = 0; i < array_info->drive_count; i++) {
423 		if (verbose)
424 			printf("Adding drive %s to array %u\n",
425 			    mfi_drive_name(NULL,
426 			    array_info->drives[i].ref.v.device_id,
427 			    MFI_DNAME_DEVICE_ID|MFI_DNAME_HONOR_OPTS),
428 			    ar->array_ref);
429 		if (ar->size > array_info->drives[i].coerced_size)
430 			ar->size = array_info->drives[i].coerced_size;
431 		ar->pd[i].ref = array_info->drives[i].ref;
432 		ar->pd[i].fw_state = MFI_PD_STATE_ONLINE;
433 	}
434 	array_info->array = ar;
435 }
436 
437 /*
438  * Create a volume that spans one or more arrays.
439  */
440 static void
441 build_volume(char *volumep, int narrays, struct array_info *arrays,
442     int raid_type, long stripe_size, struct config_id_state *state, int verbose)
443 {
444 	struct mfi_ld_config *ld = (struct mfi_ld_config *)volumep;
445 	struct mfi_array *ar;
446 	int i;
447 
448 	/* properties */
449 	ld->properties.ld.v.target_id = find_next_volume(state);
450 	ld->properties.ld.v.seq = 0;
451 	ld->properties.default_cache_policy = MR_LD_CACHE_ALLOW_WRITE_CACHE |
452 	    MR_LD_CACHE_WRITE_BACK;
453 	ld->properties.access_policy = MFI_LD_ACCESS_RW;
454 	ld->properties.disk_cache_policy = MR_PD_CACHE_UNCHANGED;
455 	ld->properties.current_cache_policy = MR_LD_CACHE_ALLOW_WRITE_CACHE |
456 	    MR_LD_CACHE_WRITE_BACK;
457 	ld->properties.no_bgi = 0;
458 
459 	/* params */
460 	switch (raid_type) {
461 	case RT_RAID0:
462 	case RT_JBOD:
463 		ld->params.primary_raid_level = DDF_RAID0;
464 		ld->params.raid_level_qualifier = 0;
465 		ld->params.secondary_raid_level = 0;
466 		break;
467 	case RT_RAID1:
468 		ld->params.primary_raid_level = DDF_RAID1;
469 		ld->params.raid_level_qualifier = 0;
470 		ld->params.secondary_raid_level = 0;
471 		break;
472 	case RT_RAID5:
473 		ld->params.primary_raid_level = DDF_RAID5;
474 		ld->params.raid_level_qualifier = 3;
475 		ld->params.secondary_raid_level = 0;
476 		break;
477 	case RT_RAID6:
478 		ld->params.primary_raid_level = DDF_RAID6;
479 		ld->params.raid_level_qualifier = 3;
480 		ld->params.secondary_raid_level = 0;
481 		break;
482 	case RT_CONCAT:
483 		ld->params.primary_raid_level = DDF_CONCAT;
484 		ld->params.raid_level_qualifier = 0;
485 		ld->params.secondary_raid_level = 0;
486 		break;
487 	case RT_RAID10:
488 		ld->params.primary_raid_level = DDF_RAID1;
489 		ld->params.raid_level_qualifier = 0;
490 		ld->params.secondary_raid_level = 3; /* XXX? */
491 		break;
492 	case RT_RAID50:
493 		/*
494 		 * XXX: This appears to work though the card's BIOS
495 		 * complains that the configuration is foreign.  The
496 		 * BIOS setup does not allow for creation of RAID-50
497 		 * or RAID-60 arrays.  The only nested array
498 		 * configuration it allows for is RAID-10.
499 		 */
500 		ld->params.primary_raid_level = DDF_RAID5;
501 		ld->params.raid_level_qualifier = 3;
502 		ld->params.secondary_raid_level = 3; /* XXX? */
503 		break;
504 	case RT_RAID60:
505 		ld->params.primary_raid_level = DDF_RAID6;
506 		ld->params.raid_level_qualifier = 3;
507 		ld->params.secondary_raid_level = 3; /* XXX? */
508 		break;
509 	}
510 
511 	/*
512 	 * Stripe size is encoded as (2 ^ N) * 512 = stripe_size.  Use
513 	 * ffs() to simulate log2(stripe_size).
514 	 */
515 	ld->params.stripe_size = ffs(stripe_size) - 1 - 9;
516 	ld->params.num_drives = arrays[0].array->num_drives;
517 	ld->params.span_depth = narrays;
518 	ld->params.state = MFI_LD_STATE_OPTIMAL;
519 	ld->params.init_state = MFI_LD_PARAMS_INIT_NO;
520 	ld->params.is_consistent = 0;
521 
522 	/* spans */
523 	for (i = 0; i < narrays; i++) {
524 		ar = arrays[i].array;
525 		if (verbose)
526 			printf("Adding array %u to volume %u\n", ar->array_ref,
527 			    ld->properties.ld.v.target_id);
528 		ld->span[i].start_block = 0;
529 		ld->span[i].num_blocks = ar->size;
530 		ld->span[i].array_ref = ar->array_ref;
531 	}
532 }
533 
534 static int
535 create_volume(int ac, char **av)
536 {
537 	struct mfi_config_data *config;
538 	struct mfi_array *ar;
539 	struct mfi_ld_config *ld;
540 	struct config_id_state state;
541 	size_t config_size;
542 	char *p, *cfg_arrays, *cfg_volumes;
543 	int error, fd, i, raid_type;
544 	int narrays, nvolumes, arrays_per_volume;
545 	struct array_info *arrays;
546 	long stripe_size;
547 #ifdef DEBUG
548 	int dump;
549 #endif
550 	int ch, verbose;
551 
552 	/*
553 	 * Backwards compat.  Map 'create volume' to 'create' and
554 	 * 'create spare' to 'add'.
555 	 */
556 	if (ac > 1) {
557 		if (strcmp(av[1], "volume") == 0) {
558 			av++;
559 			ac--;
560 		} else if (strcmp(av[1], "spare") == 0) {
561 			av++;
562 			ac--;
563 			return (add_spare(ac, av));
564 		}
565 	}
566 
567 	if (ac < 2) {
568 		warnx("create volume: volume type required");
569 		return (EINVAL);
570 	}
571 
572 	bzero(&state, sizeof(state));
573 	config = NULL;
574 	arrays = NULL;
575 	narrays = 0;
576 	error = 0;
577 
578 	fd = mfi_open(mfi_unit);
579 	if (fd < 0) {
580 		error = errno;
581 		warn("mfi_open");
582 		return (error);
583 	}
584 
585 	if (!mfi_reconfig_supported()) {
586 		warnx("The current mfi(4) driver does not support "
587 		    "configuration changes.");
588 		error = EOPNOTSUPP;
589 		goto error;
590 	}
591 
592 	/* Lookup the RAID type first. */
593 	raid_type = -1;
594 	for (i = 0; raid_type_table[i].name != NULL; i++)
595 		if (strcasecmp(raid_type_table[i].name, av[1]) == 0) {
596 			raid_type = raid_type_table[i].raid_type;
597 			break;
598 		}
599 
600 	if (raid_type == -1) {
601 		warnx("Unknown or unsupported volume type %s", av[1]);
602 		error = EINVAL;
603 		goto error;
604 	}
605 
606 	/* Parse any options. */
607 	optind = 2;
608 #ifdef DEBUG
609 	dump = 0;
610 #endif
611 	verbose = 0;
612 	stripe_size = 64 * 1024;
613 
614 	while ((ch = getopt(ac, av, "ds:v")) != -1) {
615 		switch (ch) {
616 #ifdef DEBUG
617 		case 'd':
618 			dump = 1;
619 			break;
620 #endif
621 		case 's':
622 			stripe_size = dehumanize(optarg);
623 			if ((stripe_size < 512) || (!powerof2(stripe_size)))
624 				stripe_size = 64 * 1024;
625 			break;
626 		case 'v':
627 			verbose = 1;
628 			break;
629 		case '?':
630 		default:
631 			error = EINVAL;
632 			goto error;
633 		}
634 	}
635 	ac -= optind;
636 	av += optind;
637 
638 	/* Parse all the arrays. */
639 	narrays = ac;
640 	if (narrays == 0) {
641 		warnx("At least one drive list is required");
642 		error = EINVAL;
643 		goto error;
644 	}
645 	switch (raid_type) {
646 	case RT_RAID0:
647 	case RT_RAID1:
648 	case RT_RAID5:
649 	case RT_RAID6:
650 	case RT_CONCAT:
651 		if (narrays != 1) {
652 			warnx("Only one drive list can be specified");
653 			error = EINVAL;
654 			goto error;
655 		}
656 		break;
657 	case RT_RAID10:
658 	case RT_RAID50:
659 	case RT_RAID60:
660 		if (narrays < 1) {
661 			warnx("RAID10, RAID50, and RAID60 require at least "
662 			    "two drive lists");
663 			error = EINVAL;
664 			goto error;
665 		}
666 		if (narrays > MFI_MAX_SPAN_DEPTH) {
667 			warnx("Volume spans more than %d arrays",
668 			    MFI_MAX_SPAN_DEPTH);
669 			error = EINVAL;
670 			goto error;
671 		}
672 		break;
673 	}
674 	arrays = calloc(narrays, sizeof(*arrays));
675 	if (arrays == NULL) {
676 		warnx("malloc failed");
677 		error = ENOMEM;
678 		goto error;
679 	}
680 	for (i = 0; i < narrays; i++) {
681 		error = parse_array(fd, raid_type, av[i], &arrays[i]);
682 		if (error)
683 			goto error;
684 	}
685 
686 	switch (raid_type) {
687 	case RT_RAID10:
688 	case RT_RAID50:
689 	case RT_RAID60:
690 		for (i = 1; i < narrays; i++) {
691 			if (arrays[i].drive_count != arrays[0].drive_count) {
692 				warnx("All arrays must contain the same "
693 				    "number of drives");
694 				error = EINVAL;
695 				goto error;
696 			}
697 		}
698 		break;
699 	}
700 
701 	/*
702 	 * Fetch the current config and build sorted lists of existing
703 	 * array and volume identifiers.
704 	 */
705 	if (mfi_config_read(fd, &config) < 0) {
706 		error = errno;
707 		warn("Failed to read configuration");
708 		goto error;
709 	}
710 	p = (char *)config->array;
711 	state.array_ref = 0xffff;
712 	state.target_id = 0xff;
713 	state.array_count = config->array_count;
714 	if (config->array_count > 0) {
715 		state.arrays = calloc(config->array_count, sizeof(int));
716 		if (state.arrays == NULL) {
717 			warnx("malloc failed");
718 			error = ENOMEM;
719 			goto error;
720 		}
721 		for (i = 0; i < config->array_count; i++) {
722 			ar = (struct mfi_array *)p;
723 			state.arrays[i] = ar->array_ref;
724 			p += config->array_size;
725 		}
726 		qsort(state.arrays, config->array_count, sizeof(int),
727 		    compare_int);
728 	} else
729 		state.arrays = NULL;
730 	state.log_drv_count = config->log_drv_count;
731 	if (config->log_drv_count) {
732 		state.volumes = calloc(config->log_drv_count, sizeof(int));
733 		if (state.volumes == NULL) {
734 			warnx("malloc failed");
735 			error = ENOMEM;
736 			goto error;
737 		}
738 		for (i = 0; i < config->log_drv_count; i++) {
739 			ld = (struct mfi_ld_config *)p;
740 			state.volumes[i] = ld->properties.ld.v.target_id;
741 			p += config->log_drv_size;
742 		}
743 		qsort(state.volumes, config->log_drv_count, sizeof(int),
744 		    compare_int);
745 	} else
746 		state.volumes = NULL;
747 	free(config);
748 
749 	/* Determine the size of the configuration we will build. */
750 	switch (raid_type) {
751 	case RT_RAID0:
752 	case RT_RAID1:
753 	case RT_RAID5:
754 	case RT_RAID6:
755 	case RT_CONCAT:
756 	case RT_JBOD:
757 		/* Each volume spans a single array. */
758 		nvolumes = narrays;
759 		break;
760 	case RT_RAID10:
761 	case RT_RAID50:
762 	case RT_RAID60:
763 		/* A single volume spans multiple arrays. */
764 		nvolumes = 1;
765 		break;
766 	default:
767 		/* Pacify gcc. */
768 		abort();
769 	}
770 
771 	config_size = sizeof(struct mfi_config_data) +
772 	    sizeof(struct mfi_ld_config) * nvolumes + MFI_ARRAY_SIZE * narrays;
773 	config = calloc(1, config_size);
774 	if (config == NULL) {
775 		warnx("malloc failed");
776 		error = ENOMEM;
777 		goto error;
778 	}
779 	config->size = config_size;
780 	config->array_count = narrays;
781 	config->array_size = MFI_ARRAY_SIZE;	/* XXX: Firmware hardcode */
782 	config->log_drv_count = nvolumes;
783 	config->log_drv_size = sizeof(struct mfi_ld_config);
784 	config->spares_count = 0;
785 	config->spares_size = 40;		/* XXX: Firmware hardcode */
786 	cfg_arrays = (char *)config->array;
787 	cfg_volumes = cfg_arrays + config->array_size * narrays;
788 
789 	/* Build the arrays. */
790 	for (i = 0; i < narrays; i++) {
791 		build_array(fd, cfg_arrays, &arrays[i], &state, verbose);
792 		cfg_arrays += config->array_size;
793 	}
794 
795 	/* Now build the volume(s). */
796 	arrays_per_volume = narrays / nvolumes;
797 	for (i = 0; i < nvolumes; i++) {
798 		build_volume(cfg_volumes, arrays_per_volume,
799 		    &arrays[i * arrays_per_volume], raid_type, stripe_size,
800 		    &state, verbose);
801 		cfg_volumes += config->log_drv_size;
802 	}
803 
804 #ifdef DEBUG
805 	if (dump)
806 		dump_config(fd, config);
807 #endif
808 
809 	/* Send the new config to the controller. */
810 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_ADD, config, config_size,
811 	    NULL, 0, NULL) < 0) {
812 		error = errno;
813 		warn("Failed to add volume");
814 		/* FALLTHROUGH */
815 	}
816 
817 error:
818 	/* Clean up. */
819 	free(config);
820 	free(state.volumes);
821 	free(state.arrays);
822 	if (arrays != NULL) {
823 		for (i = 0; i < narrays; i++)
824 			free(arrays[i].drives);
825 		free(arrays);
826 	}
827 	close(fd);
828 
829 	return (error);
830 }
831 MFI_COMMAND(top, create, create_volume);
832 
833 static int
834 delete_volume(int ac, char **av)
835 {
836 	struct mfi_ld_info info;
837 	int error, fd;
838 	uint8_t target_id, mbox[4];
839 
840 	/*
841 	 * Backwards compat.  Map 'delete volume' to 'delete' and
842 	 * 'delete spare' to 'remove'.
843 	 */
844 	if (ac > 1) {
845 		if (strcmp(av[1], "volume") == 0) {
846 			av++;
847 			ac--;
848 		} else if (strcmp(av[1], "spare") == 0) {
849 			av++;
850 			ac--;
851 			return (remove_spare(ac, av));
852 		}
853 	}
854 
855 	if (ac != 2) {
856 		warnx("delete volume: volume required");
857 		return (EINVAL);
858 	}
859 
860 	fd = mfi_open(mfi_unit);
861 	if (fd < 0) {
862 		error = errno;
863 		warn("mfi_open");
864 		return (error);
865 	}
866 
867 	if (!mfi_reconfig_supported()) {
868 		warnx("The current mfi(4) driver does not support "
869 		    "configuration changes.");
870 		close(fd);
871 		return (EOPNOTSUPP);
872 	}
873 
874 	if (mfi_lookup_volume(fd, av[1], &target_id) < 0) {
875 		error = errno;
876 		warn("Invalid volume %s", av[1]);
877 		close(fd);
878 		return (error);
879 	}
880 
881 	if (mfi_ld_get_info(fd, target_id, &info, NULL) < 0) {
882 		error = errno;
883 		warn("Failed to get info for volume %d", target_id);
884 		close(fd);
885 		return (error);
886 	}
887 
888 	if (mfi_volume_busy(fd, target_id)) {
889 		warnx("Volume %s is busy and cannot be deleted",
890 		    mfi_volume_name(fd, target_id));
891 		close(fd);
892 		return (EBUSY);
893 	}
894 
895 	mbox_store_ldref(mbox, &info.ld_config.properties.ld);
896 	if (mfi_dcmd_command(fd, MFI_DCMD_LD_DELETE, NULL, 0, mbox,
897 	    sizeof(mbox), NULL) < 0) {
898 		error = errno;
899 		warn("Failed to delete volume");
900 		close(fd);
901 		return (error);
902 	}
903 
904 	close(fd);
905 
906 	return (0);
907 }
908 MFI_COMMAND(top, delete, delete_volume);
909 
910 static int
911 add_spare(int ac, char **av)
912 {
913 	struct mfi_pd_info info;
914 	struct mfi_config_data *config;
915 	struct mfi_array *ar;
916 	struct mfi_ld_config *ld;
917 	struct mfi_spare *spare;
918 	uint16_t device_id;
919 	uint8_t target_id;
920 	char *p;
921 	int error, fd, i;
922 
923 	if (ac < 2) {
924 		warnx("add spare: drive required");
925 		return (EINVAL);
926 	}
927 
928 	fd = mfi_open(mfi_unit);
929 	if (fd < 0) {
930 		error = errno;
931 		warn("mfi_open");
932 		return (error);
933 	}
934 
935 	config = NULL;
936 	spare = NULL;
937 	error = mfi_lookup_drive(fd, av[1], &device_id);
938 	if (error)
939 		goto error;
940 
941 	if (mfi_pd_get_info(fd, device_id, &info, NULL) < 0) {
942 		error = errno;
943 		warn("Failed to fetch drive info");
944 		goto error;
945 	}
946 
947 	if (info.fw_state != MFI_PD_STATE_UNCONFIGURED_GOOD) {
948 		warnx("Drive %u is not available", device_id);
949 		error = EINVAL;
950 		goto error;
951 	}
952 
953 	if (ac > 2) {
954 		if (mfi_lookup_volume(fd, av[2], &target_id) < 0) {
955 			error = errno;
956 			warn("Invalid volume %s", av[2]);
957 			goto error;
958 		}
959 	}
960 
961 	if (mfi_config_read(fd, &config) < 0) {
962 		error = errno;
963 		warn("Failed to read configuration");
964 		goto error;
965 	}
966 
967 	spare = malloc(sizeof(struct mfi_spare) + sizeof(uint16_t) *
968 	    config->array_count);
969 	if (spare == NULL) {
970 		warnx("malloc failed");
971 		error = ENOMEM;
972 		goto error;
973 	}
974 	bzero(spare, sizeof(struct mfi_spare));
975 	spare->ref = info.ref;
976 
977 	if (ac == 2) {
978 		/* Global spare backs all arrays. */
979 		p = (char *)config->array;
980 		for (i = 0; i < config->array_count; i++) {
981 			ar = (struct mfi_array *)p;
982 			if (ar->size > info.coerced_size) {
983 				warnx("Spare isn't large enough for array %u",
984 				    ar->array_ref);
985 				error = EINVAL;
986 				goto error;
987 			}
988 			p += config->array_size;
989 		}
990 		spare->array_count = 0;
991 	} else  {
992 		/*
993 		 * Dedicated spares only back the arrays for a
994 		 * specific volume.
995 		 */
996 		ld = mfi_config_lookup_volume(config, target_id);
997 		if (ld == NULL) {
998 			warnx("Did not find volume %d", target_id);
999 			error = EINVAL;
1000 			goto error;
1001 		}
1002 
1003 		spare->spare_type |= MFI_SPARE_DEDICATED;
1004 		spare->array_count = ld->params.span_depth;
1005 		for (i = 0; i < ld->params.span_depth; i++) {
1006 			ar = mfi_config_lookup_array(config,
1007 			    ld->span[i].array_ref);
1008 			if (ar == NULL) {
1009 				warnx("Missing array; inconsistent config?");
1010 				error = ENXIO;
1011 				goto error;
1012 			}
1013 			if (ar->size > info.coerced_size) {
1014 				warnx("Spare isn't large enough for array %u",
1015 				    ar->array_ref);
1016 				error = EINVAL;
1017 				goto error;
1018 			}
1019 			spare->array_ref[i] = ar->array_ref;
1020 		}
1021 	}
1022 
1023 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_MAKE_SPARE, spare,
1024 	    sizeof(struct mfi_spare) + sizeof(uint16_t) * spare->array_count,
1025 	    NULL, 0, NULL) < 0) {
1026 		error = errno;
1027 		warn("Failed to assign spare");
1028 		/* FALLTHROUGH. */
1029 	}
1030 
1031 error:
1032 	free(spare);
1033 	free(config);
1034 	close(fd);
1035 
1036 	return (error);
1037 }
1038 MFI_COMMAND(top, add, add_spare);
1039 
1040 static int
1041 remove_spare(int ac, char **av)
1042 {
1043 	struct mfi_pd_info info;
1044 	int error, fd;
1045 	uint16_t device_id;
1046 	uint8_t mbox[4];
1047 
1048 	if (ac != 2) {
1049 		warnx("remove spare: drive required");
1050 		return (EINVAL);
1051 	}
1052 
1053 	fd = mfi_open(mfi_unit);
1054 	if (fd < 0) {
1055 		error = errno;
1056 		warn("mfi_open");
1057 		return (error);
1058 	}
1059 
1060 	error = mfi_lookup_drive(fd, av[1], &device_id);
1061 	if (error) {
1062 		close(fd);
1063 		return (error);
1064 	}
1065 
1066 	/* Get the info for this drive. */
1067 	if (mfi_pd_get_info(fd, device_id, &info, NULL) < 0) {
1068 		error = errno;
1069 		warn("Failed to fetch info for drive %u", device_id);
1070 		close(fd);
1071 		return (error);
1072 	}
1073 
1074 	if (info.fw_state != MFI_PD_STATE_HOT_SPARE) {
1075 		warnx("Drive %u is not a hot spare", device_id);
1076 		close(fd);
1077 		return (EINVAL);
1078 	}
1079 
1080 	mbox_store_pdref(mbox, &info.ref);
1081 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_REMOVE_SPARE, NULL, 0, mbox,
1082 	    sizeof(mbox), NULL) < 0) {
1083 		error = errno;
1084 		warn("Failed to delete spare");
1085 		close(fd);
1086 		return (error);
1087 	}
1088 
1089 	close(fd);
1090 
1091 	return (0);
1092 }
1093 MFI_COMMAND(top, remove, remove_spare);
1094 
1095 #ifdef DEBUG
1096 /* Display raw data about a config. */
1097 static void
1098 dump_config(int fd, struct mfi_config_data *config)
1099 {
1100 	struct mfi_array *ar;
1101 	struct mfi_ld_config *ld;
1102 	struct mfi_spare *sp;
1103 	struct mfi_pd_info pinfo;
1104 	uint16_t device_id;
1105 	char *p;
1106 	int i, j;
1107 
1108 	printf(
1109 	    "mfi%d Configuration (Debug): %d arrays, %d volumes, %d spares\n",
1110 	    mfi_unit, config->array_count, config->log_drv_count,
1111 	    config->spares_count);
1112 	printf("  array size: %u\n", config->array_size);
1113 	printf("  volume size: %u\n", config->log_drv_size);
1114 	printf("  spare size: %u\n", config->spares_size);
1115 	p = (char *)config->array;
1116 
1117 	for (i = 0; i < config->array_count; i++) {
1118 		ar = (struct mfi_array *)p;
1119 		printf("    array %u of %u drives:\n", ar->array_ref,
1120 		    ar->num_drives);
1121 		printf("      size = %ju\n", (uintmax_t)ar->size);
1122 		for (j = 0; j < ar->num_drives; j++) {
1123 			device_id = ar->pd[j].ref.v.device_id;
1124 			if (device_id == 0xffff)
1125 				printf("        drive MISSING\n");
1126 			else {
1127 				printf("        drive %u %s\n", device_id,
1128 				    mfi_pdstate(ar->pd[j].fw_state));
1129 				if (mfi_pd_get_info(fd, device_id, &pinfo,
1130 				    NULL) >= 0) {
1131 					printf("          raw size: %ju\n",
1132 					    (uintmax_t)pinfo.raw_size);
1133 					printf("          non-coerced size: %ju\n",
1134 					    (uintmax_t)pinfo.non_coerced_size);
1135 					printf("          coerced size: %ju\n",
1136 					    (uintmax_t)pinfo.coerced_size);
1137 				}
1138 			}
1139 		}
1140 		p += config->array_size;
1141 	}
1142 
1143 	for (i = 0; i < config->log_drv_count; i++) {
1144 		ld = (struct mfi_ld_config *)p;
1145 		printf("    volume %s ",
1146 		    mfi_volume_name(fd, ld->properties.ld.v.target_id));
1147 		printf("%s %s",
1148 		    mfi_raid_level(ld->params.primary_raid_level,
1149 			ld->params.secondary_raid_level),
1150 		    mfi_ldstate(ld->params.state));
1151 		if (ld->properties.name[0] != '\0')
1152 			printf(" <%s>", ld->properties.name);
1153 		printf("\n");
1154 		printf("      primary raid level: %u\n",
1155 		    ld->params.primary_raid_level);
1156 		printf("      raid level qualifier: %u\n",
1157 		    ld->params.raid_level_qualifier);
1158 		printf("      secondary raid level: %u\n",
1159 		    ld->params.secondary_raid_level);
1160 		printf("      stripe size: %u\n", ld->params.stripe_size);
1161 		printf("      num drives: %u\n", ld->params.num_drives);
1162 		printf("      init state: %u\n", ld->params.init_state);
1163 		printf("      consistent: %u\n", ld->params.is_consistent);
1164 		printf("      no bgi: %u\n", ld->properties.no_bgi);
1165 		printf("      spans:\n");
1166 		for (j = 0; j < ld->params.span_depth; j++) {
1167 			printf("        array %u @ ", ld->span[j].array_ref);
1168 			printf("%ju : %ju\n",
1169 			    (uintmax_t)ld->span[j].start_block,
1170 			    (uintmax_t)ld->span[j].num_blocks);
1171 		}
1172 		p += config->log_drv_size;
1173 	}
1174 
1175 	for (i = 0; i < config->spares_count; i++) {
1176 		sp = (struct mfi_spare *)p;
1177 		printf("    %s spare %u ",
1178 		    sp->spare_type & MFI_SPARE_DEDICATED ? "dedicated" :
1179 		    "global", sp->ref.v.device_id);
1180 		printf("%s", mfi_pdstate(MFI_PD_STATE_HOT_SPARE));
1181 		printf(" backs:\n");
1182 		for (j = 0; j < sp->array_count; j++)
1183 			printf("        array %u\n", sp->array_ref[j]);
1184 		p += config->spares_size;
1185 	}
1186 }
1187 
1188 static int
1189 debug_config(int ac, char **av)
1190 {
1191 	struct mfi_config_data *config;
1192 	int error, fd;
1193 
1194 	if (ac != 1) {
1195 		warnx("debug: extra arguments");
1196 		return (EINVAL);
1197 	}
1198 
1199 	fd = mfi_open(mfi_unit);
1200 	if (fd < 0) {
1201 		error = errno;
1202 		warn("mfi_open");
1203 		return (error);
1204 	}
1205 
1206 	/* Get the config from the controller. */
1207 	if (mfi_config_read(fd, &config) < 0) {
1208 		error = errno;
1209 		warn("Failed to get config");
1210 		close(fd);
1211 		return (error);
1212 	}
1213 
1214 	/* Dump out the configuration. */
1215 	dump_config(fd, config);
1216 	free(config);
1217 	close(fd);
1218 
1219 	return (0);
1220 }
1221 MFI_COMMAND(top, debug, debug_config);
1222 
1223 static int
1224 dump(int ac, char **av)
1225 {
1226 	struct mfi_config_data *config;
1227 	char buf[64];
1228 	size_t len;
1229 	int error, fd;
1230 
1231 	if (ac != 1) {
1232 		warnx("dump: extra arguments");
1233 		return (EINVAL);
1234 	}
1235 
1236 	fd = mfi_open(mfi_unit);
1237 	if (fd < 0) {
1238 		error = errno;
1239 		warn("mfi_open");
1240 		return (error);
1241 	}
1242 
1243 	/* Get the stashed copy of the last dcmd from the driver. */
1244 	snprintf(buf, sizeof(buf), "dev.mfi.%d.debug_command", mfi_unit);
1245 	if (sysctlbyname(buf, NULL, &len, NULL, 0) < 0) {
1246 		error = errno;
1247 		warn("Failed to read debug command");
1248 		if (error == ENOENT)
1249 			error = EOPNOTSUPP;
1250 		close(fd);
1251 		return (error);
1252 	}
1253 
1254 	config = malloc(len);
1255 	if (config == NULL) {
1256 		warnx("malloc failed");
1257 		close(fd);
1258 		return (ENOMEM);
1259 	}
1260 	if (sysctlbyname(buf, config, &len, NULL, 0) < 0) {
1261 		error = errno;
1262 		warn("Failed to read debug command");
1263 		free(config);
1264 		close(fd);
1265 		return (error);
1266 	}
1267 	dump_config(fd, config);
1268 	free(config);
1269 	close(fd);
1270 
1271 	return (0);
1272 }
1273 MFI_COMMAND(top, dump, dump);
1274 #endif
1275