xref: /freebsd/usr.sbin/mfiutil/mfi_config.c (revision 1d8f043a7485a83ae2417337e80d69468c14d8fe)
1 /*-
2  * Copyright (c) 2008, 2009 Yahoo!, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. The names of the authors may not be used to endorse or promote
14  *    products derived from this software without specific prior written
15  *    permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/param.h>
33 #ifdef DEBUG
34 #include <sys/sysctl.h>
35 #endif
36 #include <err.h>
37 #include <errno.h>
38 #include <libutil.h>
39 #ifdef DEBUG
40 #include <stdint.h>
41 #endif
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <string.h>
45 #include <unistd.h>
46 #include "mfiutil.h"
47 
48 #ifdef DEBUG
49 static void	dump_config(int fd, struct mfi_config_data *config);
50 #endif
51 
52 static int	add_spare(int ac, char **av);
53 static int	remove_spare(int ac, char **av);
54 
55 static long
56 dehumanize(const char *value)
57 {
58         char    *vtp;
59         long    iv;
60 
61         if (value == NULL)
62                 return (0);
63         iv = strtoq(value, &vtp, 0);
64         if (vtp == value || (vtp[0] != '\0' && vtp[1] != '\0')) {
65                 return (0);
66         }
67         switch (vtp[0]) {
68         case 't': case 'T':
69                 iv *= 1024;
70         case 'g': case 'G':
71                 iv *= 1024;
72         case 'm': case 'M':
73                 iv *= 1024;
74         case 'k': case 'K':
75                 iv *= 1024;
76         case '\0':
77                 break;
78         default:
79                 return (0);
80         }
81         return (iv);
82 }
83 int
84 mfi_config_read(int fd, struct mfi_config_data **configp)
85 {
86 	struct mfi_config_data *config;
87 	uint32_t config_size;
88 	int error;
89 
90 	/*
91 	 * Keep fetching the config in a loop until we have a large enough
92 	 * buffer to hold the entire configuration.
93 	 */
94 	config = NULL;
95 	config_size = 1024;
96 fetch:
97 	config = reallocf(config, config_size);
98 	if (config == NULL)
99 		return (-1);
100 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_READ, config,
101 	    config_size, NULL, 0, NULL) < 0) {
102 		error = errno;
103 		free(config);
104 		errno = error;
105 		return (-1);
106 	}
107 
108 	if (config->size > config_size) {
109 		config_size = config->size;
110 		goto fetch;
111 	}
112 
113 	*configp = config;
114 	return (0);
115 }
116 
117 static struct mfi_array *
118 mfi_config_lookup_array(struct mfi_config_data *config, uint16_t array_ref)
119 {
120 	struct mfi_array *ar;
121 	char *p;
122 	int i;
123 
124 	p = (char *)config->array;
125 	for (i = 0; i < config->array_count; i++) {
126 		ar = (struct mfi_array *)p;
127 		if (ar->array_ref == array_ref)
128 			return (ar);
129 		p += config->array_size;
130 	}
131 
132 	return (NULL);
133 }
134 
135 static struct mfi_ld_config *
136 mfi_config_lookup_volume(struct mfi_config_data *config, uint8_t target_id)
137 {
138 	struct mfi_ld_config *ld;
139 	char *p;
140 	int i;
141 
142 	p = (char *)config->array + config->array_count * config->array_size;
143 	for (i = 0; i < config->log_drv_count; i++) {
144 		ld = (struct mfi_ld_config *)p;
145 		if (ld->properties.ld.v.target_id == target_id)
146 			return (ld);
147 		p += config->log_drv_size;
148 	}
149 
150 	return (NULL);
151 }
152 
153 static int
154 clear_config(int ac, char **av)
155 {
156 	struct mfi_ld_list list;
157 	int ch, error, fd;
158 	u_int i;
159 
160 	fd = mfi_open(mfi_unit);
161 	if (fd < 0) {
162 		error = errno;
163 		warn("mfi_open");
164 		return (error);
165 	}
166 
167 	if (!mfi_reconfig_supported()) {
168 		warnx("The current mfi(4) driver does not support "
169 		    "configuration changes.");
170 		close(fd);
171 		return (EOPNOTSUPP);
172 	}
173 
174 	if (mfi_ld_get_list(fd, &list, NULL) < 0) {
175 		error = errno;
176 		warn("Failed to get volume list");
177 		close(fd);
178 		return (error);
179 	}
180 
181 	for (i = 0; i < list.ld_count; i++) {
182 		if (mfi_volume_busy(fd, list.ld_list[i].ld.v.target_id)) {
183 			warnx("Volume %s is busy and cannot be deleted",
184 			    mfi_volume_name(fd, list.ld_list[i].ld.v.target_id));
185 			close(fd);
186 			return (EBUSY);
187 		}
188 	}
189 
190 	printf(
191 	    "Are you sure you wish to clear the configuration on mfi%u? [y/N] ",
192 	    mfi_unit);
193 	ch = getchar();
194 	if (ch != 'y' && ch != 'Y') {
195 		printf("\nAborting\n");
196 		close(fd);
197 		return (0);
198 	}
199 
200 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_CLEAR, NULL, 0, NULL, 0, NULL) < 0) {
201 		error = errno;
202 		warn("Failed to clear configuration");
203 		close(fd);
204 		return (error);
205 	}
206 
207 	printf("mfi%d: Configuration cleared\n", mfi_unit);
208 	close(fd);
209 
210 	return (0);
211 }
212 MFI_COMMAND(top, clear, clear_config);
213 
214 #define MAX_DRIVES_PER_ARRAY MFI_MAX_ROW_SIZE
215 #define MFI_ARRAY_SIZE sizeof(struct mfi_array)
216 
217 #define	RT_RAID0	0
218 #define	RT_RAID1	1
219 #define	RT_RAID5	2
220 #define	RT_RAID6	3
221 #define	RT_JBOD		4
222 #define	RT_CONCAT	5
223 #define	RT_RAID10	6
224 #define	RT_RAID50	7
225 #define	RT_RAID60	8
226 
227 static int
228 compare_int(const void *one, const void *two)
229 {
230 	int first, second;
231 
232 	first = *(const int *)one;
233 	second = *(const int *)two;
234 
235 	return (first - second);
236 }
237 
238 static struct raid_type_entry {
239 	const char *name;
240 	int	raid_type;
241 } raid_type_table[] = {
242 	{ "raid0",	RT_RAID0 },
243 	{ "raid-0",	RT_RAID0 },
244 	{ "raid1",	RT_RAID1 },
245 	{ "raid-1",	RT_RAID1 },
246 	{ "mirror",	RT_RAID1 },
247 	{ "raid5",	RT_RAID5 },
248 	{ "raid-5",	RT_RAID5 },
249 	{ "raid6",	RT_RAID6 },
250 	{ "raid-6",	RT_RAID6 },
251 	{ "jbod",	RT_JBOD },
252 	{ "concat",	RT_CONCAT },
253 	{ "raid10",	RT_RAID10 },
254 	{ "raid1+0",	RT_RAID10 },
255 	{ "raid-10",	RT_RAID10 },
256 	{ "raid-1+0",	RT_RAID10 },
257 	{ "raid50",	RT_RAID50 },
258 	{ "raid5+0",	RT_RAID50 },
259 	{ "raid-50",	RT_RAID50 },
260 	{ "raid-5+0",	RT_RAID50 },
261 	{ "raid60",	RT_RAID60 },
262 	{ "raid6+0",	RT_RAID60 },
263 	{ "raid-60",	RT_RAID60 },
264 	{ "raid-6+0",	RT_RAID60 },
265 	{ NULL,		0 },
266 };
267 
268 struct config_id_state {
269 	int	array_count;
270 	int	log_drv_count;
271 	int	*arrays;
272 	int	*volumes;
273 	uint16_t array_ref;
274 	uint8_t	target_id;
275 };
276 
277 struct array_info {
278 	int	drive_count;
279 	struct mfi_pd_info *drives;
280 	struct mfi_array *array;
281 };
282 
283 /* Parse a comma-separated list of drives for an array. */
284 static int
285 parse_array(int fd, int raid_type, char *array_str, struct array_info *info)
286 {
287 	struct mfi_pd_info *pinfo;
288 	uint16_t device_id;
289 	char *cp;
290 	u_int count;
291 	int error;
292 
293 	cp = array_str;
294 	for (count = 0; cp != NULL; count++) {
295 		cp = strchr(cp, ',');
296 		if (cp != NULL) {
297 			cp++;
298 			if (*cp == ',') {
299 				warnx("Invalid drive list '%s'", array_str);
300 				return (EINVAL);
301 			}
302 		}
303 	}
304 
305 	/* Validate the number of drives for this array. */
306 	if (count >= MAX_DRIVES_PER_ARRAY) {
307 		warnx("Too many drives for a single array: max is %d",
308 		    MAX_DRIVES_PER_ARRAY);
309 		return (EINVAL);
310 	}
311 	switch (raid_type) {
312 	case RT_RAID1:
313 	case RT_RAID10:
314 		if (count % 2 != 0) {
315 			warnx("RAID1 and RAID10 require an even number of "
316 			    "drives in each array");
317 			return (EINVAL);
318 		}
319 		break;
320 	case RT_RAID5:
321 	case RT_RAID50:
322 		if (count < 3) {
323 			warnx("RAID5 and RAID50 require at least 3 drives in "
324 			    "each array");
325 			return (EINVAL);
326 		}
327 		break;
328 	case RT_RAID6:
329 	case RT_RAID60:
330 		if (count < 4) {
331 			warnx("RAID6 and RAID60 require at least 4 drives in "
332 			    "each array");
333 			return (EINVAL);
334 		}
335 		break;
336 	}
337 
338 	/* Validate each drive. */
339 	info->drives = calloc(count, sizeof(struct mfi_pd_info));
340 	if (info->drives == NULL) {
341 		warnx("malloc failed");
342 		return (ENOMEM);
343 	}
344 	info->drive_count = count;
345 	for (pinfo = info->drives; (cp = strsep(&array_str, ",")) != NULL;
346 	     pinfo++) {
347 		error = mfi_lookup_drive(fd, cp, &device_id);
348 		if (error) {
349 			free(info->drives);
350 			return (error);
351 		}
352 
353 		if (mfi_pd_get_info(fd, device_id, pinfo, NULL) < 0) {
354 			error = errno;
355 			warn("Failed to fetch drive info for drive %s", cp);
356 			free(info->drives);
357 			return (error);
358 		}
359 
360 		if (pinfo->fw_state != MFI_PD_STATE_UNCONFIGURED_GOOD) {
361 			warnx("Drive %u is not available", device_id);
362 			free(info->drives);
363 			return (EINVAL);
364 		}
365 	}
366 
367 	return (0);
368 }
369 
370 /*
371  * Find the next free array ref assuming that 'array_ref' is the last
372  * one used.  'array_ref' should be 0xffff for the initial test.
373  */
374 static uint16_t
375 find_next_array(struct config_id_state *state)
376 {
377 	int i;
378 
379 	/* Assume the current one is used. */
380 	state->array_ref++;
381 
382 	/* Find the next free one. */
383 	for (i = 0; i < state->array_count; i++)
384 		if (state->arrays[i] == state->array_ref)
385 			state->array_ref++;
386 	return (state->array_ref);
387 }
388 
389 /*
390  * Find the next free volume ID assuming that 'target_id' is the last
391  * one used.  'target_id' should be 0xff for the initial test.
392  */
393 static uint8_t
394 find_next_volume(struct config_id_state *state)
395 {
396 	int i;
397 
398 	/* Assume the current one is used. */
399 	state->target_id++;
400 
401 	/* Find the next free one. */
402 	for (i = 0; i < state->log_drv_count; i++)
403 		if (state->volumes[i] == state->target_id)
404 			state->target_id++;
405 	return (state->target_id);
406 }
407 
408 /* Populate an array with drives. */
409 static void
410 build_array(int fd, char *arrayp, struct array_info *array_info,
411     struct config_id_state *state, int verbose)
412 {
413 	struct mfi_array *ar = (struct mfi_array *)arrayp;
414 	int i;
415 
416 	ar->size = array_info->drives[0].coerced_size;
417 	ar->num_drives = array_info->drive_count;
418 	ar->array_ref = find_next_array(state);
419 	for (i = 0; i < array_info->drive_count; i++) {
420 		if (verbose)
421 			printf("Adding drive %s to array %u\n",
422 			    mfi_drive_name(NULL,
423 			    array_info->drives[i].ref.v.device_id,
424 			    MFI_DNAME_DEVICE_ID|MFI_DNAME_HONOR_OPTS),
425 			    ar->array_ref);
426 		if (ar->size > array_info->drives[i].coerced_size)
427 			ar->size = array_info->drives[i].coerced_size;
428 		ar->pd[i].ref = array_info->drives[i].ref;
429 		ar->pd[i].fw_state = MFI_PD_STATE_ONLINE;
430 	}
431 	array_info->array = ar;
432 }
433 
434 /*
435  * Create a volume that spans one or more arrays.
436  */
437 static void
438 build_volume(char *volumep, int narrays, struct array_info *arrays,
439     int raid_type, long stripe_size, struct config_id_state *state, int verbose)
440 {
441 	struct mfi_ld_config *ld = (struct mfi_ld_config *)volumep;
442 	struct mfi_array *ar;
443 	int i;
444 
445 	/* properties */
446 	ld->properties.ld.v.target_id = find_next_volume(state);
447 	ld->properties.ld.v.seq = 0;
448 	ld->properties.default_cache_policy = MR_LD_CACHE_ALLOW_WRITE_CACHE |
449 	    MR_LD_CACHE_WRITE_BACK;
450 	ld->properties.access_policy = MFI_LD_ACCESS_RW;
451 	ld->properties.disk_cache_policy = MR_PD_CACHE_UNCHANGED;
452 	ld->properties.current_cache_policy = MR_LD_CACHE_ALLOW_WRITE_CACHE |
453 	    MR_LD_CACHE_WRITE_BACK;
454 	ld->properties.no_bgi = 0;
455 
456 	/* params */
457 	switch (raid_type) {
458 	case RT_RAID0:
459 	case RT_JBOD:
460 		ld->params.primary_raid_level = DDF_RAID0;
461 		ld->params.raid_level_qualifier = 0;
462 		ld->params.secondary_raid_level = 0;
463 		break;
464 	case RT_RAID1:
465 		ld->params.primary_raid_level = DDF_RAID1;
466 		ld->params.raid_level_qualifier = 0;
467 		ld->params.secondary_raid_level = 0;
468 		break;
469 	case RT_RAID5:
470 		ld->params.primary_raid_level = DDF_RAID5;
471 		ld->params.raid_level_qualifier = 3;
472 		ld->params.secondary_raid_level = 0;
473 		break;
474 	case RT_RAID6:
475 		ld->params.primary_raid_level = DDF_RAID6;
476 		ld->params.raid_level_qualifier = 3;
477 		ld->params.secondary_raid_level = 0;
478 		break;
479 	case RT_CONCAT:
480 		ld->params.primary_raid_level = DDF_CONCAT;
481 		ld->params.raid_level_qualifier = 0;
482 		ld->params.secondary_raid_level = 0;
483 		break;
484 	case RT_RAID10:
485 		ld->params.primary_raid_level = DDF_RAID1;
486 		ld->params.raid_level_qualifier = 0;
487 		ld->params.secondary_raid_level = 3; /* XXX? */
488 		break;
489 	case RT_RAID50:
490 		/*
491 		 * XXX: This appears to work though the card's BIOS
492 		 * complains that the configuration is foreign.  The
493 		 * BIOS setup does not allow for creation of RAID-50
494 		 * or RAID-60 arrays.  The only nested array
495 		 * configuration it allows for is RAID-10.
496 		 */
497 		ld->params.primary_raid_level = DDF_RAID5;
498 		ld->params.raid_level_qualifier = 3;
499 		ld->params.secondary_raid_level = 3; /* XXX? */
500 		break;
501 	case RT_RAID60:
502 		ld->params.primary_raid_level = DDF_RAID6;
503 		ld->params.raid_level_qualifier = 3;
504 		ld->params.secondary_raid_level = 3; /* XXX? */
505 		break;
506 	}
507 
508 	/*
509 	 * Stripe size is encoded as (2 ^ N) * 512 = stripe_size.  Use
510 	 * ffs() to simulate log2(stripe_size).
511 	 */
512 	ld->params.stripe_size = ffs(stripe_size) - 1 - 9;
513 	ld->params.num_drives = arrays[0].array->num_drives;
514 	ld->params.span_depth = narrays;
515 	ld->params.state = MFI_LD_STATE_OPTIMAL;
516 	ld->params.init_state = MFI_LD_PARAMS_INIT_NO;
517 	ld->params.is_consistent = 0;
518 
519 	/* spans */
520 	for (i = 0; i < narrays; i++) {
521 		ar = arrays[i].array;
522 		if (verbose)
523 			printf("Adding array %u to volume %u\n", ar->array_ref,
524 			    ld->properties.ld.v.target_id);
525 		ld->span[i].start_block = 0;
526 		ld->span[i].num_blocks = ar->size;
527 		ld->span[i].array_ref = ar->array_ref;
528 	}
529 }
530 
531 static int
532 create_volume(int ac, char **av)
533 {
534 	struct mfi_config_data *config;
535 	struct mfi_array *ar;
536 	struct mfi_ld_config *ld;
537 	struct config_id_state state;
538 	size_t config_size;
539 	char *p, *cfg_arrays, *cfg_volumes;
540 	int error, fd, i, raid_type;
541 	int narrays, nvolumes, arrays_per_volume;
542 	struct array_info *arrays;
543 	long stripe_size;
544 #ifdef DEBUG
545 	int dump;
546 #endif
547 	int ch, verbose;
548 
549 	/*
550 	 * Backwards compat.  Map 'create volume' to 'create' and
551 	 * 'create spare' to 'add'.
552 	 */
553 	if (ac > 1) {
554 		if (strcmp(av[1], "volume") == 0) {
555 			av++;
556 			ac--;
557 		} else if (strcmp(av[1], "spare") == 0) {
558 			av++;
559 			ac--;
560 			return (add_spare(ac, av));
561 		}
562 	}
563 
564 	if (ac < 2) {
565 		warnx("create volume: volume type required");
566 		return (EINVAL);
567 	}
568 
569 	bzero(&state, sizeof(state));
570 	config = NULL;
571 	arrays = NULL;
572 	narrays = 0;
573 	error = 0;
574 
575 	fd = mfi_open(mfi_unit);
576 	if (fd < 0) {
577 		error = errno;
578 		warn("mfi_open");
579 		return (error);
580 	}
581 
582 	if (!mfi_reconfig_supported()) {
583 		warnx("The current mfi(4) driver does not support "
584 		    "configuration changes.");
585 		error = EOPNOTSUPP;
586 		goto error;
587 	}
588 
589 	/* Lookup the RAID type first. */
590 	raid_type = -1;
591 	for (i = 0; raid_type_table[i].name != NULL; i++)
592 		if (strcasecmp(raid_type_table[i].name, av[1]) == 0) {
593 			raid_type = raid_type_table[i].raid_type;
594 			break;
595 		}
596 
597 	if (raid_type == -1) {
598 		warnx("Unknown or unsupported volume type %s", av[1]);
599 		error = EINVAL;
600 		goto error;
601 	}
602 
603 	/* Parse any options. */
604 	optind = 2;
605 #ifdef DEBUG
606 	dump = 0;
607 #endif
608 	verbose = 0;
609 	stripe_size = 64 * 1024;
610 
611 	while ((ch = getopt(ac, av, "ds:v")) != -1) {
612 		switch (ch) {
613 #ifdef DEBUG
614 		case 'd':
615 			dump = 1;
616 			break;
617 #endif
618 		case 's':
619 			stripe_size = dehumanize(optarg);
620 			if ((stripe_size < 512) || (!powerof2(stripe_size)))
621 				stripe_size = 64 * 1024;
622 			break;
623 		case 'v':
624 			verbose = 1;
625 			break;
626 		case '?':
627 		default:
628 			error = EINVAL;
629 			goto error;
630 		}
631 	}
632 	ac -= optind;
633 	av += optind;
634 
635 	/* Parse all the arrays. */
636 	narrays = ac;
637 	if (narrays == 0) {
638 		warnx("At least one drive list is required");
639 		error = EINVAL;
640 		goto error;
641 	}
642 	switch (raid_type) {
643 	case RT_RAID0:
644 	case RT_RAID1:
645 	case RT_RAID5:
646 	case RT_RAID6:
647 	case RT_CONCAT:
648 		if (narrays != 1) {
649 			warnx("Only one drive list can be specified");
650 			error = EINVAL;
651 			goto error;
652 		}
653 		break;
654 	case RT_RAID10:
655 	case RT_RAID50:
656 	case RT_RAID60:
657 		if (narrays < 1) {
658 			warnx("RAID10, RAID50, and RAID60 require at least "
659 			    "two drive lists");
660 			error = EINVAL;
661 			goto error;
662 		}
663 		if (narrays > MFI_MAX_SPAN_DEPTH) {
664 			warnx("Volume spans more than %d arrays",
665 			    MFI_MAX_SPAN_DEPTH);
666 			error = EINVAL;
667 			goto error;
668 		}
669 		break;
670 	}
671 	arrays = calloc(narrays, sizeof(*arrays));
672 	if (arrays == NULL) {
673 		warnx("malloc failed");
674 		error = ENOMEM;
675 		goto error;
676 	}
677 	for (i = 0; i < narrays; i++) {
678 		error = parse_array(fd, raid_type, av[i], &arrays[i]);
679 		if (error)
680 			goto error;
681 	}
682 
683 	switch (raid_type) {
684 	case RT_RAID10:
685 	case RT_RAID50:
686 	case RT_RAID60:
687 		for (i = 1; i < narrays; i++) {
688 			if (arrays[i].drive_count != arrays[0].drive_count) {
689 				warnx("All arrays must contain the same "
690 				    "number of drives");
691 				error = EINVAL;
692 				goto error;
693 			}
694 		}
695 		break;
696 	}
697 
698 	/*
699 	 * Fetch the current config and build sorted lists of existing
700 	 * array and volume identifiers.
701 	 */
702 	if (mfi_config_read(fd, &config) < 0) {
703 		error = errno;
704 		warn("Failed to read configuration");
705 		goto error;
706 	}
707 	p = (char *)config->array;
708 	state.array_ref = 0xffff;
709 	state.target_id = 0xff;
710 	state.array_count = config->array_count;
711 	if (config->array_count > 0) {
712 		state.arrays = calloc(config->array_count, sizeof(int));
713 		if (state.arrays == NULL) {
714 			warnx("malloc failed");
715 			error = ENOMEM;
716 			goto error;
717 		}
718 		for (i = 0; i < config->array_count; i++) {
719 			ar = (struct mfi_array *)p;
720 			state.arrays[i] = ar->array_ref;
721 			p += config->array_size;
722 		}
723 		qsort(state.arrays, config->array_count, sizeof(int),
724 		    compare_int);
725 	} else
726 		state.arrays = NULL;
727 	state.log_drv_count = config->log_drv_count;
728 	if (config->log_drv_count) {
729 		state.volumes = calloc(config->log_drv_count, sizeof(int));
730 		if (state.volumes == NULL) {
731 			warnx("malloc failed");
732 			error = ENOMEM;
733 			goto error;
734 		}
735 		for (i = 0; i < config->log_drv_count; i++) {
736 			ld = (struct mfi_ld_config *)p;
737 			state.volumes[i] = ld->properties.ld.v.target_id;
738 			p += config->log_drv_size;
739 		}
740 		qsort(state.volumes, config->log_drv_count, sizeof(int),
741 		    compare_int);
742 	} else
743 		state.volumes = NULL;
744 	free(config);
745 
746 	/* Determine the size of the configuration we will build. */
747 	switch (raid_type) {
748 	case RT_RAID0:
749 	case RT_RAID1:
750 	case RT_RAID5:
751 	case RT_RAID6:
752 	case RT_CONCAT:
753 	case RT_JBOD:
754 		/* Each volume spans a single array. */
755 		nvolumes = narrays;
756 		break;
757 	case RT_RAID10:
758 	case RT_RAID50:
759 	case RT_RAID60:
760 		/* A single volume spans multiple arrays. */
761 		nvolumes = 1;
762 		break;
763 	default:
764 		/* Pacify gcc. */
765 		abort();
766 	}
767 
768 	config_size = sizeof(struct mfi_config_data) +
769 	    sizeof(struct mfi_ld_config) * nvolumes + MFI_ARRAY_SIZE * narrays;
770 	config = calloc(1, config_size);
771 	if (config == NULL) {
772 		warnx("malloc failed");
773 		error = ENOMEM;
774 		goto error;
775 	}
776 	config->size = config_size;
777 	config->array_count = narrays;
778 	config->array_size = MFI_ARRAY_SIZE;	/* XXX: Firmware hardcode */
779 	config->log_drv_count = nvolumes;
780 	config->log_drv_size = sizeof(struct mfi_ld_config);
781 	config->spares_count = 0;
782 	config->spares_size = 40;		/* XXX: Firmware hardcode */
783 	cfg_arrays = (char *)config->array;
784 	cfg_volumes = cfg_arrays + config->array_size * narrays;
785 
786 	/* Build the arrays. */
787 	for (i = 0; i < narrays; i++) {
788 		build_array(fd, cfg_arrays, &arrays[i], &state, verbose);
789 		cfg_arrays += config->array_size;
790 	}
791 
792 	/* Now build the volume(s). */
793 	arrays_per_volume = narrays / nvolumes;
794 	for (i = 0; i < nvolumes; i++) {
795 		build_volume(cfg_volumes, arrays_per_volume,
796 		    &arrays[i * arrays_per_volume], raid_type, stripe_size,
797 		    &state, verbose);
798 		cfg_volumes += config->log_drv_size;
799 	}
800 
801 #ifdef DEBUG
802 	if (dump)
803 		dump_config(fd, config);
804 #endif
805 
806 	/* Send the new config to the controller. */
807 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_ADD, config, config_size,
808 	    NULL, 0, NULL) < 0) {
809 		error = errno;
810 		warn("Failed to add volume");
811 		/* FALLTHROUGH */
812 	}
813 
814 error:
815 	/* Clean up. */
816 	free(config);
817 	free(state.volumes);
818 	free(state.arrays);
819 	for (i = 0; i < narrays; i++)
820 		free(arrays[i].drives);
821 	free(arrays);
822 	close(fd);
823 
824 	return (error);
825 }
826 MFI_COMMAND(top, create, create_volume);
827 
828 static int
829 delete_volume(int ac, char **av)
830 {
831 	struct mfi_ld_info info;
832 	int error, fd;
833 	uint8_t target_id, mbox[4];
834 
835 	/*
836 	 * Backwards compat.  Map 'delete volume' to 'delete' and
837 	 * 'delete spare' to 'remove'.
838 	 */
839 	if (ac > 1) {
840 		if (strcmp(av[1], "volume") == 0) {
841 			av++;
842 			ac--;
843 		} else if (strcmp(av[1], "spare") == 0) {
844 			av++;
845 			ac--;
846 			return (remove_spare(ac, av));
847 		}
848 	}
849 
850 	if (ac != 2) {
851 		warnx("delete volume: volume required");
852 		return (EINVAL);
853 	}
854 
855 	fd = mfi_open(mfi_unit);
856 	if (fd < 0) {
857 		error = errno;
858 		warn("mfi_open");
859 		return (error);
860 	}
861 
862 	if (!mfi_reconfig_supported()) {
863 		warnx("The current mfi(4) driver does not support "
864 		    "configuration changes.");
865 		close(fd);
866 		return (EOPNOTSUPP);
867 	}
868 
869 	if (mfi_lookup_volume(fd, av[1], &target_id) < 0) {
870 		error = errno;
871 		warn("Invalid volume %s", av[1]);
872 		close(fd);
873 		return (error);
874 	}
875 
876 	if (mfi_ld_get_info(fd, target_id, &info, NULL) < 0) {
877 		error = errno;
878 		warn("Failed to get info for volume %d", target_id);
879 		close(fd);
880 		return (error);
881 	}
882 
883 	if (mfi_volume_busy(fd, target_id)) {
884 		warnx("Volume %s is busy and cannot be deleted",
885 		    mfi_volume_name(fd, target_id));
886 		close(fd);
887 		return (EBUSY);
888 	}
889 
890 	mbox_store_ldref(mbox, &info.ld_config.properties.ld);
891 	if (mfi_dcmd_command(fd, MFI_DCMD_LD_DELETE, NULL, 0, mbox,
892 	    sizeof(mbox), NULL) < 0) {
893 		error = errno;
894 		warn("Failed to delete volume");
895 		close(fd);
896 		return (error);
897 	}
898 
899 	close(fd);
900 
901 	return (0);
902 }
903 MFI_COMMAND(top, delete, delete_volume);
904 
905 static int
906 add_spare(int ac, char **av)
907 {
908 	struct mfi_pd_info info;
909 	struct mfi_config_data *config;
910 	struct mfi_array *ar;
911 	struct mfi_ld_config *ld;
912 	struct mfi_spare *spare;
913 	uint16_t device_id;
914 	uint8_t target_id;
915 	char *p;
916 	int error, fd, i;
917 
918 	if (ac < 2) {
919 		warnx("add spare: drive required");
920 		return (EINVAL);
921 	}
922 
923 	fd = mfi_open(mfi_unit);
924 	if (fd < 0) {
925 		error = errno;
926 		warn("mfi_open");
927 		return (error);
928 	}
929 
930 	config = NULL;
931 	spare = NULL;
932 	error = mfi_lookup_drive(fd, av[1], &device_id);
933 	if (error)
934 		goto error;
935 
936 	if (mfi_pd_get_info(fd, device_id, &info, NULL) < 0) {
937 		error = errno;
938 		warn("Failed to fetch drive info");
939 		goto error;
940 	}
941 
942 	if (info.fw_state != MFI_PD_STATE_UNCONFIGURED_GOOD) {
943 		warnx("Drive %u is not available", device_id);
944 		error = EINVAL;
945 		goto error;
946 	}
947 
948 	if (ac > 2) {
949 		if (mfi_lookup_volume(fd, av[2], &target_id) < 0) {
950 			error = errno;
951 			warn("Invalid volume %s", av[2]);
952 			goto error;
953 		}
954 	}
955 
956 	if (mfi_config_read(fd, &config) < 0) {
957 		error = errno;
958 		warn("Failed to read configuration");
959 		goto error;
960 	}
961 
962 	spare = malloc(sizeof(struct mfi_spare) + sizeof(uint16_t) *
963 	    config->array_count);
964 	if (spare == NULL) {
965 		warnx("malloc failed");
966 		error = ENOMEM;
967 		goto error;
968 	}
969 	bzero(spare, sizeof(struct mfi_spare));
970 	spare->ref = info.ref;
971 
972 	if (ac == 2) {
973 		/* Global spare backs all arrays. */
974 		p = (char *)config->array;
975 		for (i = 0; i < config->array_count; i++) {
976 			ar = (struct mfi_array *)p;
977 			if (ar->size > info.coerced_size) {
978 				warnx("Spare isn't large enough for array %u",
979 				    ar->array_ref);
980 				error = EINVAL;
981 				goto error;
982 			}
983 			p += config->array_size;
984 		}
985 		spare->array_count = 0;
986 	} else  {
987 		/*
988 		 * Dedicated spares only back the arrays for a
989 		 * specific volume.
990 		 */
991 		ld = mfi_config_lookup_volume(config, target_id);
992 		if (ld == NULL) {
993 			warnx("Did not find volume %d", target_id);
994 			error = EINVAL;
995 			goto error;
996 		}
997 
998 		spare->spare_type |= MFI_SPARE_DEDICATED;
999 		spare->array_count = ld->params.span_depth;
1000 		for (i = 0; i < ld->params.span_depth; i++) {
1001 			ar = mfi_config_lookup_array(config,
1002 			    ld->span[i].array_ref);
1003 			if (ar == NULL) {
1004 				warnx("Missing array; inconsistent config?");
1005 				error = ENXIO;
1006 				goto error;
1007 			}
1008 			if (ar->size > info.coerced_size) {
1009 				warnx("Spare isn't large enough for array %u",
1010 				    ar->array_ref);
1011 				error = EINVAL;
1012 				goto error;
1013 			}
1014 			spare->array_ref[i] = ar->array_ref;
1015 		}
1016 	}
1017 
1018 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_MAKE_SPARE, spare,
1019 	    sizeof(struct mfi_spare) + sizeof(uint16_t) * spare->array_count,
1020 	    NULL, 0, NULL) < 0) {
1021 		error = errno;
1022 		warn("Failed to assign spare");
1023 		/* FALLTHROUGH. */
1024 	}
1025 
1026 error:
1027 	free(spare);
1028 	free(config);
1029 	close(fd);
1030 
1031 	return (error);
1032 }
1033 MFI_COMMAND(top, add, add_spare);
1034 
1035 static int
1036 remove_spare(int ac, char **av)
1037 {
1038 	struct mfi_pd_info info;
1039 	int error, fd;
1040 	uint16_t device_id;
1041 	uint8_t mbox[4];
1042 
1043 	if (ac != 2) {
1044 		warnx("remove spare: drive required");
1045 		return (EINVAL);
1046 	}
1047 
1048 	fd = mfi_open(mfi_unit);
1049 	if (fd < 0) {
1050 		error = errno;
1051 		warn("mfi_open");
1052 		return (error);
1053 	}
1054 
1055 	error = mfi_lookup_drive(fd, av[1], &device_id);
1056 	if (error) {
1057 		close(fd);
1058 		return (error);
1059 	}
1060 
1061 	/* Get the info for this drive. */
1062 	if (mfi_pd_get_info(fd, device_id, &info, NULL) < 0) {
1063 		error = errno;
1064 		warn("Failed to fetch info for drive %u", device_id);
1065 		close(fd);
1066 		return (error);
1067 	}
1068 
1069 	if (info.fw_state != MFI_PD_STATE_HOT_SPARE) {
1070 		warnx("Drive %u is not a hot spare", device_id);
1071 		close(fd);
1072 		return (EINVAL);
1073 	}
1074 
1075 	mbox_store_pdref(mbox, &info.ref);
1076 	if (mfi_dcmd_command(fd, MFI_DCMD_CFG_REMOVE_SPARE, NULL, 0, mbox,
1077 	    sizeof(mbox), NULL) < 0) {
1078 		error = errno;
1079 		warn("Failed to delete spare");
1080 		close(fd);
1081 		return (error);
1082 	}
1083 
1084 	close(fd);
1085 
1086 	return (0);
1087 }
1088 MFI_COMMAND(top, remove, remove_spare);
1089 
1090 #ifdef DEBUG
1091 /* Display raw data about a config. */
1092 static void
1093 dump_config(int fd, struct mfi_config_data *config)
1094 {
1095 	struct mfi_array *ar;
1096 	struct mfi_ld_config *ld;
1097 	struct mfi_spare *sp;
1098 	struct mfi_pd_info pinfo;
1099 	uint16_t device_id;
1100 	char *p;
1101 	int i, j;
1102 
1103 	printf(
1104 	    "mfi%d Configuration (Debug): %d arrays, %d volumes, %d spares\n",
1105 	    mfi_unit, config->array_count, config->log_drv_count,
1106 	    config->spares_count);
1107 	printf("  array size: %u\n", config->array_size);
1108 	printf("  volume size: %u\n", config->log_drv_size);
1109 	printf("  spare size: %u\n", config->spares_size);
1110 	p = (char *)config->array;
1111 
1112 	for (i = 0; i < config->array_count; i++) {
1113 		ar = (struct mfi_array *)p;
1114 		printf("    array %u of %u drives:\n", ar->array_ref,
1115 		    ar->num_drives);
1116 		printf("      size = %ju\n", (uintmax_t)ar->size);
1117 		for (j = 0; j < ar->num_drives; j++) {
1118 			device_id = ar->pd[j].ref.v.device_id;
1119 			if (device_id == 0xffff)
1120 				printf("        drive MISSING\n");
1121 			else {
1122 				printf("        drive %u %s\n", device_id,
1123 				    mfi_pdstate(ar->pd[j].fw_state));
1124 				if (mfi_pd_get_info(fd, device_id, &pinfo,
1125 				    NULL) >= 0) {
1126 					printf("          raw size: %ju\n",
1127 					    (uintmax_t)pinfo.raw_size);
1128 					printf("          non-coerced size: %ju\n",
1129 					    (uintmax_t)pinfo.non_coerced_size);
1130 					printf("          coerced size: %ju\n",
1131 					    (uintmax_t)pinfo.coerced_size);
1132 				}
1133 			}
1134 		}
1135 		p += config->array_size;
1136 	}
1137 
1138 	for (i = 0; i < config->log_drv_count; i++) {
1139 		ld = (struct mfi_ld_config *)p;
1140 		printf("    volume %s ",
1141 		    mfi_volume_name(fd, ld->properties.ld.v.target_id));
1142 		printf("%s %s",
1143 		    mfi_raid_level(ld->params.primary_raid_level,
1144 			ld->params.secondary_raid_level),
1145 		    mfi_ldstate(ld->params.state));
1146 		if (ld->properties.name[0] != '\0')
1147 			printf(" <%s>", ld->properties.name);
1148 		printf("\n");
1149 		printf("      primary raid level: %u\n",
1150 		    ld->params.primary_raid_level);
1151 		printf("      raid level qualifier: %u\n",
1152 		    ld->params.raid_level_qualifier);
1153 		printf("      secondary raid level: %u\n",
1154 		    ld->params.secondary_raid_level);
1155 		printf("      stripe size: %u\n", ld->params.stripe_size);
1156 		printf("      num drives: %u\n", ld->params.num_drives);
1157 		printf("      init state: %u\n", ld->params.init_state);
1158 		printf("      consistent: %u\n", ld->params.is_consistent);
1159 		printf("      no bgi: %u\n", ld->properties.no_bgi);
1160 		printf("      spans:\n");
1161 		for (j = 0; j < ld->params.span_depth; j++) {
1162 			printf("        array %u @ ", ld->span[j].array_ref);
1163 			printf("%ju : %ju\n",
1164 			    (uintmax_t)ld->span[j].start_block,
1165 			    (uintmax_t)ld->span[j].num_blocks);
1166 		}
1167 		p += config->log_drv_size;
1168 	}
1169 
1170 	for (i = 0; i < config->spares_count; i++) {
1171 		sp = (struct mfi_spare *)p;
1172 		printf("    %s spare %u ",
1173 		    sp->spare_type & MFI_SPARE_DEDICATED ? "dedicated" :
1174 		    "global", sp->ref.v.device_id);
1175 		printf("%s", mfi_pdstate(MFI_PD_STATE_HOT_SPARE));
1176 		printf(" backs:\n");
1177 		for (j = 0; j < sp->array_count; j++)
1178 			printf("        array %u\n", sp->array_ref[j]);
1179 		p += config->spares_size;
1180 	}
1181 }
1182 
1183 static int
1184 debug_config(int ac, char **av)
1185 {
1186 	struct mfi_config_data *config;
1187 	int error, fd;
1188 
1189 	if (ac != 1) {
1190 		warnx("debug: extra arguments");
1191 		return (EINVAL);
1192 	}
1193 
1194 	fd = mfi_open(mfi_unit);
1195 	if (fd < 0) {
1196 		error = errno;
1197 		warn("mfi_open");
1198 		return (error);
1199 	}
1200 
1201 	/* Get the config from the controller. */
1202 	if (mfi_config_read(fd, &config) < 0) {
1203 		error = errno;
1204 		warn("Failed to get config");
1205 		close(fd);
1206 		return (error);
1207 	}
1208 
1209 	/* Dump out the configuration. */
1210 	dump_config(fd, config);
1211 	free(config);
1212 	close(fd);
1213 
1214 	return (0);
1215 }
1216 MFI_COMMAND(top, debug, debug_config);
1217 
1218 static int
1219 dump(int ac, char **av)
1220 {
1221 	struct mfi_config_data *config;
1222 	char buf[64];
1223 	size_t len;
1224 	int error, fd;
1225 
1226 	if (ac != 1) {
1227 		warnx("dump: extra arguments");
1228 		return (EINVAL);
1229 	}
1230 
1231 	fd = mfi_open(mfi_unit);
1232 	if (fd < 0) {
1233 		error = errno;
1234 		warn("mfi_open");
1235 		return (error);
1236 	}
1237 
1238 	/* Get the stashed copy of the last dcmd from the driver. */
1239 	snprintf(buf, sizeof(buf), "dev.mfi.%d.debug_command", mfi_unit);
1240 	if (sysctlbyname(buf, NULL, &len, NULL, 0) < 0) {
1241 		error = errno;
1242 		warn("Failed to read debug command");
1243 		if (error == ENOENT)
1244 			error = EOPNOTSUPP;
1245 		close(fd);
1246 		return (error);
1247 	}
1248 
1249 	config = malloc(len);
1250 	if (config == NULL) {
1251 		warnx("malloc failed");
1252 		close(fd);
1253 		return (ENOMEM);
1254 	}
1255 	if (sysctlbyname(buf, config, &len, NULL, 0) < 0) {
1256 		error = errno;
1257 		warn("Failed to read debug command");
1258 		free(config);
1259 		close(fd);
1260 		return (error);
1261 	}
1262 	dump_config(fd, config);
1263 	free(config);
1264 	close(fd);
1265 
1266 	return (0);
1267 }
1268 MFI_COMMAND(top, dump, dump);
1269 #endif
1270