xref: /titanic_41/usr/src/cmd/avs/rdc/sndrsyncd.c (revision 71269a2275bf5a143dad6461eee2710a344e7261)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/wait.h>
28 #include <stdio.h>
29 #include <errno.h>
30 #include <limits.h>
31 #include <fcntl.h>
32 #include <strings.h>
33 #include <stdlib.h>
34 #include <unistd.h>
35 #include <pthread.h>
36 #include <thread.h>
37 
38 #include <locale.h>
39 #include <langinfo.h>
40 #include <libintl.h>
41 #include <stdarg.h>
42 
43 #include <sys/nsctl/rdc_io.h>
44 #include <sys/nsctl/rdc_ioctl.h>
45 #include <sys/nsctl/rdc_prot.h>
46 
47 #include <sys/nsctl/cfg.h>
48 
49 #include <sys/unistat/spcs_s.h>
50 #include <sys/unistat/spcs_s_u.h>
51 #include <sys/unistat/spcs_errors.h>
52 
53 #include <sys/nsctl/librdc.h>
54 
55 #include "rdcadm.h"
56 
57 
58 #define	RDCADM "/usr/sbin/sndradm"
59 #define	IIADM "/usr/sbin/iiadm"
60 
61 #define	UPDATE "update"
62 #define	NOUPDATE "noupdate"
63 
64 #define	RESYNC_SLEEP	(3 * 60)	/* Three minutes */
65 #define	MAIN_SLEEP	(5 * 60)	/* Five minutes */
66 #define	CFG_WAIT_SLEEP	(5)		/* 5 sec */
67 
68 #define	MAXHOSTS 1024
69 mutex_t cfglock = DEFAULTMUTEX;
70 #define	LOCKCFG() (void) mutex_lock(&cfglock);
71 #define	UNLOCKCFG() (void) mutex_unlock(&cfglock);
72 
73 typedef struct host_list_s {
74 	char *hosts[MAXHOSTS];
75 	int numhosts;
76 	int configured[MAXHOSTS];
77 	mutex_t hosts_mutex;
78 } host_list_t;
79 
80 host_list_t *host_list;
81 
82 extern char *basename(char *);
83 int rdc_maxsets;
84 char *program;
85 
86 static int clustered = 0;
87 
88 int isnewhost(char *host);
89 void *wait_sync_event();
90 void *wait_link_down(void *host);
91 void rdc_sync(char *tohost);
92 void remove_from_hostlist(char *host);
93 void sync_start(char *master);
94 void sync_complete(char *master);
95 void cleanup_hostlist();
96 void group_start(char *group);
97 void group_complete(char *group);
98 
99 
100 void
101 init_host_list(void)
102 {
103 	host_list = calloc(1, sizeof (host_list_t));
104 	if (host_list == NULL) {
105 		spcs_log("sndr", NULL,
106 		    gettext("host list not initialized, cannot run"));
107 		rdc_err(NULL, gettext("host list not initialized, cannot run"));
108 	}
109 	(void) mutex_init(&host_list->hosts_mutex, USYNC_THREAD, NULL);
110 }
111 
112 /* ARGSUSED */
113 #ifdef lint
114 void
115 sndrsyncd_lintmain(argc, argv)
116 #else
117 int
118 main(argc, argv)
119 #endif
120 int argc;
121 char **argv;
122 {
123 	rdc_status_t *rdc_info;
124 	int size;
125 	int i;
126 	pid_t pid;
127 	spcs_s_info_t ustatus;
128 	int rc, trc;
129 	int first = 0;
130 	char *required;
131 
132 	(void) setlocale(LC_ALL, "");
133 	(void) textdomain("rdc");
134 
135 	ustatus = spcs_s_ucreate();
136 
137 	program = basename(argv[0]);
138 
139 	init_host_list();
140 
141 	rc = rdc_check_release(&required);
142 	if (rc < 0) {
143 		rdc_err(NULL,
144 		    gettext("unable to determine the current "
145 		    "Solaris release: %s\n"), strerror(errno));
146 		/* NOTREACHED */
147 	} else if (rc == FALSE) {
148 		rdc_err(NULL,
149 		    gettext("incorrect Solaris release (requires %s)\n"),
150 		    required);
151 		/* NOTREACHED */
152 	}
153 
154 	clustered = cfg_iscluster();
155 	if (clustered < 0) {
156 		rdc_err(NULL, gettext("unable to ascertain environment"));
157 	}
158 
159 	rdc_maxsets = rdc_get_maxsets();
160 	if (rdc_maxsets == -1) {
161 		spcs_log("sndr", NULL,
162 		    gettext("%s: unable to get maxsets value from kernel"),
163 		    program);
164 		rdc_err(NULL,
165 		    gettext("unable to get maxsets value from kernel"));
166 	}
167 	size = sizeof (rdc_status_t) + (sizeof (rdc_set_t) * (rdc_maxsets - 1));
168 	rdc_info = malloc(size);
169 	if (rdc_info == NULL) {
170 		spcs_log("sndr", NULL,
171 		    gettext("%s: unable to allocate %ld bytes"),
172 		    program, size);
173 		rdc_err(NULL,
174 			gettext("unable to allocate %ld bytes"), size);
175 	}
176 	bzero(rdc_info, size);
177 
178 	rdc_info->nset = rdc_maxsets;
179 
180 	/*
181 	 * Fork off a child that becomes the daemon.
182 	 */
183 	if ((pid = fork()) > 0)
184 		exit(0);
185 	else if (pid < 0) {
186 		spcs_log("sndr", NULL,
187 		    gettext("%s: cannot fork: %s"),
188 		    program, strerror(errno));
189 		rdc_err(NULL, gettext("cannot fork: %s\n"),
190 		    strerror(errno));
191 	}
192 
193 	/*
194 	 * In child - become daemon.
195 	 */
196 
197 	for (i = 0; i < 3; i++)
198 		(void) close(i);
199 
200 	(void) open("/dev/console", O_WRONLY|O_APPEND);
201 	(void) dup(0);
202 	(void) dup(0);
203 	(void) close(0);
204 
205 	(void) setpgrp();
206 
207 	(void) setlocale(LC_ALL, "");
208 	(void) textdomain("rdc");
209 
210 	/* launch a thread to wait for sync start and sync stop events */
211 
212 	if ((trc = thr_create(NULL, 0, wait_sync_event, NULL,
213 	    THR_BOUND|THR_DETACHED, NULL)) != 0) {
214 		spcs_log("sndr", NULL,
215 		    gettext("%s: unable to create thread wait_sync_event"),
216 		    program);
217 		rdc_warn(NULL,
218 		    gettext("%s unable to create thread wait_sync_event"),
219 		    program);
220 	} else {
221 #ifdef DEBUG
222 		spcs_log("sndr", NULL,
223 		    gettext("%s: thread wait_sync_event started"), program);
224 #endif
225 		;
226 	}
227 
228 	for (;;) {
229 		if (!first) {
230 			first++;
231 			(void) sleep(15);
232 		} else
233 			(void) sleep(MAIN_SLEEP);
234 
235 		bzero(rdc_info, size);
236 		rdc_info->nset = rdc_maxsets;
237 		if (RDC_IOCTL(RDC_STATUS, rdc_info, 0, 0, 0, 0, ustatus)
238 		    != SPCS_S_OK) {
239 			spcs_log("sndr", &ustatus,
240 			    gettext("%s: status ioctl"),
241 			    program);
242 			rdc_warn(&ustatus, gettext("status ioctl"));
243 			continue;
244 		}
245 
246 		cleanup_hostlist(rdc_info); /* remove non-existent hosts */
247 
248 		/*
249 		 * Check all enabled sets to see if a new remote host has
250 		 * appeared.
251 		 */
252 		for (i = 0; i < rdc_maxsets; i++) {
253 			if (!(rdc_info->rdc_set[i].flags & RDC_ENABLED))
254 				continue;
255 			/* spawn a new thread for each new host found */
256 			if (isnewhost(rdc_info->rdc_set[i].secondary.intf)) {
257 				/*
258 				 * right now, we could be here before
259 				 * the database did the write for this set
260 				 * I could check the lock on the database
261 				 * but I am just going to give up some time here
262 				 * instead. Why do the allocations etc, etc
263 				 * if the set is enabled in the kernel and not
264 				 * in the config, we know that this set has the
265 				 * lock. Why bother adding more contention to
266 				 * the lock.
267 				 * this is a daemon, afterall. its got time
268 				 */
269 				(void) sleep(CFG_WAIT_SLEEP);
270 
271 				spcs_log("sndr", NULL,
272 				    gettext("%s: new host found (%s) starting "
273 				    "its autosync thread"), program,
274 				    rdc_info->rdc_set[i].secondary.intf);
275 
276 				trc = thr_create(NULL, 0, wait_link_down,
277 				    (void *) rdc_info->rdc_set[i].\
278 secondary.intf, THR_BOUND|THR_DETACHED, NULL);
279 
280 				if (trc != 0) {
281 					spcs_log("sndr", NULL,
282 					    gettext(
283 					    "%s create new autosync "
284 					    "thread failed"), program);
285 					rdc_warn(NULL, gettext(
286 					    "%s create new autosync "
287 					    "thread failed"), program);
288 				}
289 			}
290 		}
291 	}
292 	/* NOTREACHED */
293 }
294 
295 
296 /*
297  * The kernel wakes up this function every time it detects the link to the
298  * specified host has dropped.
299  */
300 void *
301 wait_link_down(void *thehost)
302 {
303 	char *host = (char *)thehost;
304 	char tmphost[MAX_RDC_HOST_SIZE] = { '\0' };
305 	spcs_s_info_t ustatus;
306 
307 	if (host)
308 		strncpy(tmphost, host, MAX_RDC_HOST_SIZE);
309 
310 	ustatus = spcs_s_ucreate();
311 
312 	/* Never give up */
313 	for (;;) {
314 #ifdef DEBUG
315 		spcs_log("sndr", NULL,
316 		    gettext("%s: awaiting link down ioctl for %s"),
317 		    program, host[0] == '\0' ? tmphost : host);
318 #endif
319 		if (RDC_IOCTL(RDC_LINK_DOWN, host, 0, 0, 0, 0, ustatus)
320 		    != SPCS_S_OK) {
321 			spcs_log("sndr", &ustatus,
322 			    gettext("%s: link down ioctl"),
323 			    program);
324 			rdc_warn(&ustatus, gettext("link down ioctl"));
325 			continue;
326 		}
327 #ifdef DEBUG
328 
329 		spcs_log("sndr", NULL,
330 		    gettext("%s: received link down ioctl for %s"),
331 		    program, host[0] == '\0' ? tmphost : host);
332 #endif
333 		rdc_sync(host[0] == '\0' ? tmphost : host);
334 	}
335 	/* LINTED */
336 }
337 
338 
339 /*
340  * Called when the link to the specified host has dropped.
341  * For all Remote Mirror sets using the link that have autosync on,
342  * issue rdcadm -u commands until they complete successfully.
343  */
344 void
345 rdc_sync(char *tohost)
346 {
347 	rdc_set_t *rdc_set = NULL;
348 	int *sync_done = NULL;
349 	int sets = 0;
350 	int syncs_done = 0;
351 	char cmd[256];
352 	rdc_config_t parms = { 0 };
353 	spcs_s_info_t ustatus;
354 	int i;
355 	int setnumber;
356 	int numfound = 0;
357 	char buf[CFG_MAX_BUF];
358 	char key[CFG_MAX_KEY];
359 	CFGFILE *cfg = NULL;
360 	int size;
361 	int first = 0;
362 	int death = 0;
363 	int cfglocked = 0;
364 
365 	ustatus = spcs_s_ucreate();
366 
367 	size = sizeof (rdc_set_t) * rdc_maxsets;
368 	rdc_set = malloc(size);
369 	if (rdc_set == NULL) {
370 		spcs_log("sndr", NULL,
371 		    gettext("%s: unable to allocate %ld bytes"),
372 		    program, size);
373 		rdc_warn(NULL,
374 			gettext("unable to allocate %ld bytes"), size);
375 		goto done;
376 	}
377 	bzero(rdc_set, size);
378 	size = sizeof (int) * rdc_maxsets;
379 	sync_done = malloc(size);
380 	if (sync_done == NULL) {
381 		spcs_log("sndr", NULL,
382 		    gettext("%s: unable to allocate %ld bytes"),
383 		    program, size);
384 		rdc_warn(NULL,
385 			gettext("unable to allocate %ld bytes"), size);
386 		goto done;
387 	}
388 	bzero(sync_done, size);
389 
390 	/*
391 	 * Get all sndr entries with shost matching tohost, and save the
392 	 * details in an array.
393 	 */
394 	for (i = 0; i < rdc_maxsets; i++) {
395 		setnumber = i + 1;
396 		bzero(buf, sizeof (buf));
397 		bzero(key, sizeof (key));
398 
399 		(void) snprintf(key, sizeof (key), "sndr.set%d.shost",
400 		    setnumber);
401 
402 		if (!cfglocked) {
403 			LOCKCFG();
404 			if ((cfg = cfg_open(NULL)) == NULL) {
405 				spcs_log("sndr", NULL,
406 				    gettext("%s: error opening config"),
407 				    program);
408 
409 				rdc_warn(NULL,
410 				    gettext("error opening config"));
411 				UNLOCKCFG();
412 				goto done;
413 			}
414 
415 			if (!cfg_lock(cfg, CFG_RDLOCK)) {
416 				spcs_log("sndr", NULL,
417 				    gettext("%s: error locking config"),
418 				    program);
419 				rdc_warn(NULL, gettext("error locking config"));
420 				goto done;
421 			}
422 		}
423 
424 		cfglocked = 1;
425 
426 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
427 			if (numfound == 0) /* no matching hosts */
428 				death = 1; /* thread will exit */
429 			break;
430 		}
431 		if (strcmp(buf, tohost) != 0)
432 			continue;
433 
434 		numfound++;
435 		strncpy(rdc_set[sets].secondary.intf, buf, MAX_RDC_HOST_SIZE);
436 
437 		/* Got a matching entry */
438 
439 		(void) snprintf(key, sizeof (key), "sndr.set%d.phost",
440 		    setnumber);
441 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
442 			break;
443 		strncpy(rdc_set[sets].primary.intf, buf, MAX_RDC_HOST_SIZE);
444 
445 		(void) snprintf(key, sizeof (key), "sndr.set%d.primary",
446 		    setnumber);
447 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
448 			break;
449 		strncpy(rdc_set[sets].primary.file, buf, NSC_MAXPATH);
450 
451 		(void) snprintf(key, sizeof (key), "sndr.set%d.secondary",
452 		    setnumber);
453 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
454 			break;
455 		strncpy(rdc_set[sets].secondary.file, buf, NSC_MAXPATH);
456 
457 		parms.command = RDC_CMD_STATUS;
458 		bcopy((void *)(&rdc_set[sets]), (void *)(&parms.rdc_set[0]),
459 		    sizeof (rdc_set_t));
460 
461 		/*
462 		 * release cfg before diving into the kernel
463 		 * this prevents a possible deadlock when doing
464 		 * a reverse sync whick will wake up the sync_event
465 		 * thread which will try and iiadm -c and hang
466 		 * because we still have the cfg_lock. the timed
467 		 * wait cv in the kernel will fail the sync and things
468 		 * will undeadlock.
469 		 */
470 
471 		cfg_close(cfg);
472 		cfg = NULL;
473 		cfglocked = 0;
474 		UNLOCKCFG();
475 
476 		if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
477 			continue;
478 		}
479 		if ((parms.rdc_set[0].autosync == 0) ||
480 		    (!(parms.rdc_set[0].flags & RDC_LOGGING))) {
481 			continue;
482 		}
483 
484 		/* Found a suitable set with autosync on, in logging mode */
485 		sets++;
486 	}
487 
488 	if (cfg) {
489 		cfg_close(cfg);
490 		cfg = NULL;
491 		UNLOCKCFG();
492 	}
493 
494 	if (sets == 0) {
495 #ifdef DEBUG
496 		spcs_log("sndr", NULL,
497 		    gettext("%s: no sets requiring autosync found for %s"),
498 		    program, tohost);
499 #endif
500 		if (death) {
501 			spcs_log("sndr", NULL,
502 			    gettext("%s: autosync thread stopping for %s "
503 			    "(host deconfigured)"), program, tohost);
504 		}
505 		goto done;
506 	}
507 
508 	/* Keep issuing rdcadm -u commands until they have all completed */
509 	for (;;) {
510 		if (!first)
511 			first++;
512 		else
513 			(void) sleep(RESYNC_SLEEP);
514 
515 		/* Issue rdcadm -u commands for all remaining sets */
516 		for (i = 0; i < sets; i++) {
517 			if (sync_done[i])
518 				continue;
519 
520 			/*
521 			 * Need to check if autosync was turned off for a set
522 			 * while we were sleeping. We could have the case where
523 			 * an update sync failed and autosync was disabled
524 			 * while we were sleeping and didn't detect the disable.
525 			 * See BugID 4814213.
526 			 */
527 			parms.command = RDC_CMD_STATUS;
528 			bcopy((void *)(&rdc_set[i]),
529 			    (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
530 			if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
531 			    ustatus) < 0) {
532 				spcs_log("sndr", &ustatus, gettext("%s: "
533 				    "status not available for %s:%s, stopping "
534 				    "this autosync attempt"), program, tohost,
535 				    rdc_set[i].secondary.file);
536 				sync_done[i] = 1;
537 				syncs_done++;
538 				continue;
539 			}
540 			if (!(parms.rdc_set[0].autosync)) {
541 #ifdef DEBUG
542 	spcs_log("sndr", NULL, gettext("%s: autosync disabled during sleep, "
543 	    "stopping attempt for set %s:%s"), program, tohost,
544 	    rdc_set[i].secondary.file);
545 #endif
546 				sync_done[i] = 1;
547 				syncs_done++;
548 				continue;
549 			}
550 
551 			(void) sprintf(cmd, "%s -un %s:%s", RDCADM, tohost,
552 			    rdc_set[i].secondary.file);
553 			spcs_log("sndr", NULL,
554 			    gettext("%s: issuing update sync for %s:%s"),
555 			    program, tohost, rdc_set[i].secondary.file);
556 			(void) system(cmd);
557 		}
558 
559 		/* Issue rdcadm -w commands to wait for updates to finish */
560 		for (i = 0; i < sets; i++) {
561 			if (sync_done[i])
562 				continue;
563 
564 			(void) sprintf(cmd, "%s -wn %s:%s", RDCADM, tohost,
565 			    rdc_set[i].secondary.file);
566 			spcs_log("sndr", NULL,
567 			    gettext("%s: issuing wait for %s:%s"),
568 			    program, tohost, rdc_set[i].secondary.file);
569 
570 			(void) system(cmd);
571 
572 			parms.command = RDC_CMD_STATUS;
573 			bcopy((void *)(&rdc_set[i]),
574 			    (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
575 
576 			if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
577 			    ustatus) < 0) {
578 				spcs_log("sndr", &ustatus,
579 				    gettext("%s: status not available for "
580 				    "%s:%s, stopping this autosync attempt"),
581 				    program, tohost, rdc_set[i].secondary.file);
582 				sync_done[i] = 1;
583 				syncs_done++;
584 				continue;
585 			}
586 			/* Check if completed OK, failed or autosync off */
587 			if (!(parms.rdc_set[0].autosync) ||
588 			    !(parms.rdc_set[0].flags & RDC_LOGGING) &&
589 			    !(parms.rdc_set[0].flags & RDC_SYNCING)) {
590 				sync_done[i] = 1;
591 				syncs_done++;
592 			}
593 		}
594 
595 		if (syncs_done == sets)
596 			break;		/* All completed OK */
597 	}
598 
599 done:
600 	if (cfg) {
601 		cfg_close(cfg);
602 		UNLOCKCFG();
603 	}
604 	spcs_s_ufree(&ustatus);
605 	if (sync_done)
606 		free(sync_done);
607 	if (rdc_set)
608 		free(rdc_set);
609 	if (death) { /* bye bye */
610 		/*
611 		 * if perhaps we lost some race, lets remove this entry from
612 		 * the list. Then, if something did go wrong, and we did kill
613 		 * a valid thread, it will be detected on the next go around
614 		 * of the thread who is looking for new hosts to spawn threads
615 		 */
616 
617 		remove_from_hostlist(tohost);
618 		thr_exit(0);
619 	}
620 
621 	(void) sleep(RESYNC_SLEEP);
622 }
623 
624 /*
625  * Wait for notification by the kernel of a sync start or a sync completed OK
626  */
627 void *
628 wait_sync_event()
629 {
630 	spcs_s_info_t ustatus;
631 	char master[NSC_MAXPATH];
632 	char group[NSC_MAXPATH];
633 	int state;
634 
635 	ustatus = spcs_s_ucreate();
636 
637 	master[0] = '\0';
638 	group[0] = '\0';
639 
640 	/* Never give up */
641 	for (;;) {
642 		/* Kernel tells us which volume and group the event is for */
643 		state = RDC_IOCTL(RDC_SYNC_EVENT, master, group, 0, 0, 0,
644 		    ustatus);
645 		if (state < SPCS_S_OK) {
646 			if (errno != EAGAIN) {
647 				spcs_log("sndr", &ustatus,
648 				    gettext("%s: update ioctl"),
649 				    program);
650 				rdc_warn(&ustatus, gettext("update ioctl"));
651 				continue;
652 			}
653 			master[0] = '\0';
654 			continue;
655 		}
656 
657 		/*
658 		 * If target is mounted at the start of a sync or reverse sync,
659 		 * return a negative ack.
660 		 */
661 		if ((state == RDC_SYNC_START || state == RDC_RSYNC_START) &&
662 		    mounted(master)) {
663 			spcs_log("sndr", NULL,
664 			    gettext("%s: %s has a file system mounted"),
665 			    program, master);
666 			rdc_warn(NULL,
667 			    gettext("%s has a file system mounted"),
668 			    master);
669 			master[0] = '\0';	/* negative ack */
670 			continue;
671 		}
672 
673 		switch (state) {
674 		case RDC_SYNC_START:
675 			if (group[0])
676 				group_start(group);
677 			else
678 				sync_start(master);
679 			break;
680 
681 		case RDC_SYNC_DONE:
682 			if (group[0])
683 				group_complete(group);
684 			else
685 				sync_complete(master);
686 			break;
687 
688 		default:
689 			break;
690 		}
691 	}
692 	/* LINTED */
693 }
694 
695 
696 /*
697  * A sync has completed OK to a volume not belonging to a group.
698  * Set the state of the ndr_ii config entry to "update".
699  */
700 void
701 sync_complete(char *master)
702 {
703 	CFGFILE *cfg = NULL;
704 	char buf[CFG_MAX_BUF];
705 	char key[CFG_MAX_KEY];
706 	int i;
707 	int setnumber;
708 	int sev;
709 
710 	LOCKCFG();
711 	if ((cfg = cfg_open(NULL)) == NULL) {
712 		spcs_log("sndr", NULL,
713 		    gettext("%s: error opening config"),
714 		    program);
715 		rdc_warn(NULL, gettext("error opening config"));
716 		UNLOCKCFG();
717 		return;
718 	}
719 	if (!cfg_lock(cfg, CFG_WRLOCK)) {
720 		spcs_log("sndr", NULL,
721 		    gettext("%s: error locking config"),
722 		    program);
723 		rdc_warn(NULL, gettext("error locking config"));
724 		cfg_close(cfg);
725 		UNLOCKCFG();
726 		return;
727 	}
728 
729 	/* get ndr_ii entries until a match is found */
730 	for (i = 0; ; i++) {
731 		setnumber = i + 1;
732 
733 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
734 		    setnumber);
735 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
736 			break;
737 		if (strcmp(buf, master) != 0)
738 			continue;
739 
740 		/* Found the matching entry */
741 
742 		/*
743 		 * Set state to "update" so that starting another sync will
744 		 * cause a new Point-in-Time Copy snapshot to be taken.
745 		 */
746 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
747 		    setnumber);
748 		if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
749 		    (cfg_commit(cfg) < 0)) {
750 			spcs_log("sndr", NULL,
751 			    gettext("%s: unable to update \"%s\" "
752 			    "in configuration storage: %s"),
753 			    program, buf, cfg_error(&sev));
754 			rdc_warn(NULL,
755 			    gettext("unable to update \"%s\" "
756 			    "in configuration storage: %s"),
757 			    buf, cfg_error(&sev));
758 		}
759 		break;
760 	}
761 
762 	cfg_close(cfg);
763 	UNLOCKCFG();
764 }
765 
766 
767 /*
768  * Starting a sync to the specified master volume.
769  * Check the ndr_ii config entries to see if a Point-in-Time Copy
770  * snapshot should be taken.
771  */
772 void
773 sync_start(char *master)
774 {
775 	char cmd[256];
776 	char buf[CFG_MAX_BUF];
777 	char key[CFG_MAX_KEY];
778 	CFGFILE *cfg = NULL;
779 	int i;
780 	int setnumber;
781 	int found;
782 	int sev;
783 	char shadow[NSC_MAXPATH];
784 	char bitmap[NSC_MAXPATH];
785 	char *ctag = NULL;
786 
787 	LOCKCFG();
788 	if ((cfg = cfg_open(NULL)) == NULL) {
789 		spcs_log("sndr", NULL,
790 		    gettext("%s: error opening config"),
791 		    program);
792 		rdc_warn(NULL,
793 		    gettext("error opening config"));
794 		UNLOCKCFG();
795 		return;
796 	}
797 	if (!cfg_lock(cfg, CFG_RDLOCK)) {
798 		spcs_log("sndr", NULL,
799 		    gettext("%s: error locking config"),
800 		    program);
801 		rdc_warn(NULL, gettext("error locking config"));
802 		cfg_close(cfg);
803 		UNLOCKCFG();
804 		return;
805 	}
806 
807 	found = 0;
808 	/* get ndr_ii entries until a match is found */
809 	for (i = 0; ; i++) {
810 		setnumber = i + 1;
811 
812 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
813 		    setnumber);
814 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
815 			break;
816 		if (strcmp(buf, master) != 0)
817 			continue;
818 
819 		/* Got a matching entry */
820 
821 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
822 		    setnumber);
823 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
824 			break;
825 		strncpy(shadow, buf, NSC_MAXPATH);
826 
827 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
828 		    setnumber);
829 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
830 			break;
831 		strncpy(bitmap, buf, NSC_MAXPATH);
832 
833 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
834 		    setnumber);
835 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
836 			break;
837 
838 		/*
839 		 * If an PIT snapshot has already been taken, and syncing did
840 		 * not complete, the state will be "noupdate", to indicate we
841 		 * should not take another one at this point.
842 		 */
843 		if (strcmp(buf, NOUPDATE) != 0)
844 			found = 1;
845 
846 		break;
847 	}
848 
849 	if (!found) {
850 		cfg_close(cfg);
851 		UNLOCKCFG();
852 		return;
853 	}
854 
855 	found = 0;
856 	/* get ii entries until a match is found */
857 	for (i = 0; ; i++) {
858 		setnumber = i + 1;
859 
860 		(void) snprintf(key, sizeof (key), "ii.set%d.shadow",
861 		    setnumber);
862 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
863 			break;
864 		if (strcmp(buf, shadow) != 0)
865 			continue;
866 
867 		/* Matching shadow found, so ii already enabled */
868 		found = 1;
869 		break;
870 	}
871 
872 	if (found) {
873 		/* Already PIT enabled, so just take a snapshot */
874 
875 		/* Get cluster tag of matching entry */
876 		(void) snprintf(key, sizeof (key), "ii.set%d.cnode", setnumber);
877 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
878 			if ((strlen(buf) == 0) || (buf[0] == '-'))
879 					ctag = "-C local";
880 				else
881 					ctag = "";
882 		(void) sprintf(cmd, "%s %s -u s %s", IIADM, ctag, shadow);
883 	} else {
884 		/*
885 		 * If clustered, need to enable PIT Copy sets in the same
886 		 * cluster as the Remote Mirror set
887 		 */
888 
889 		if (clustered) {
890 			/* Find a RM set with master as the local volume */
891 
892 			for (i = 0; i < rdc_maxsets; i++) {
893 				setnumber = i + 1;
894 				(void) snprintf(key, sizeof (key),
895 				    "sndr.set%d.phost", setnumber);
896 				if (cfg_get_cstring(cfg, key, buf,
897 				    CFG_MAX_BUF) < 0)
898 					break;
899 
900 				if (self_check(buf))
901 					(void) snprintf(key, sizeof (key),
902 					    "sndr.set%d.primary", setnumber);
903 				else
904 					(void) snprintf(key, sizeof (key),
905 					    "sndr.set%d.secondary", setnumber);
906 				if (cfg_get_cstring(cfg, key, buf,
907 				    CFG_MAX_BUF) < 0)
908 					break;
909 
910 				if (strcmp(buf, master) != 0)
911 					continue;
912 
913 				/* Get cluster tag of matching entry */
914 
915 				(void) snprintf(key, sizeof (key),
916 				    "sndr.set%d.cnode", setnumber);
917 				if (cfg_get_cstring(cfg, key, buf,
918 				    CFG_MAX_BUF) < 0)
919 					break;
920 				if ((strlen(buf) == 0) || (buf[0] == '-'))
921 					ctag = strdup("local");
922 				else
923 					ctag = strdup(buf);
924 				break;
925 			}
926 		}
927 
928 		/* Not already enabled, so enable a dependent */
929 		if (ctag) {
930 			(void) sprintf(cmd, "%s -C %s -e dep %s %s %s", IIADM,
931 			    ctag, master, shadow, bitmap);
932 			free(ctag);
933 		} else
934 			(void) sprintf(cmd, "%s -e dep %s %s %s", IIADM, master,
935 			    shadow, bitmap);
936 	}
937 
938 	cfg_close(cfg);
939 
940 	if (system(cmd) != 0) {
941 		spcs_log("sndr", NULL,
942 		    gettext("Point-in-Time Copy snapshot failed for %s %s %s."
943 		    " Please check validity of ndr_ii entry"),
944 		    master, shadow, bitmap);
945 		cfg_close(cfg);
946 		UNLOCKCFG();
947 		return;
948 	}
949 
950 	/*
951 	 * PIT Copy enable or update was fine, so update the ndr_ii entry
952 	 * to "noupdate", to prevent invalid point in time copies.
953 	 */
954 
955 	if ((cfg = cfg_open(NULL)) == NULL) {
956 		spcs_log("sndr", NULL,
957 		    gettext("%s: error opening config"),
958 		    program);
959 		rdc_warn(NULL,
960 		    gettext("error opening config"));
961 		UNLOCKCFG();
962 		return;
963 	}
964 	if (!cfg_lock(cfg, CFG_WRLOCK)) {
965 		spcs_log("sndr", NULL,
966 		    gettext("%s: error locking config"),
967 		    program);
968 		rdc_warn(NULL, gettext("error locking config"));
969 		cfg_close(cfg);
970 		UNLOCKCFG();
971 		return;
972 	}
973 
974 	/* get ndr_ii entries until a match is found */
975 	for (i = 0; ; i++) {
976 		setnumber = i + 1;
977 
978 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
979 		    setnumber);
980 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
981 			break;
982 		if (strcmp(buf, shadow) != 0)
983 			continue;
984 
985 		/* Found the matching entry */
986 
987 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
988 		    setnumber);
989 		if ((cfg_put_cstring(cfg, key, NOUPDATE,
990 			strlen(NOUPDATE)) < 0) || (cfg_commit(cfg) < 0)) {
991 			spcs_log("sndr", NULL,
992 			    gettext("%s: unable to update \"%s\" "
993 			    "in configuration storage: %s"),
994 			    program, buf, cfg_error(&sev));
995 			rdc_warn(NULL,
996 			    gettext("unable to update \"%s\" "
997 			    "in configuration storage: %s"),
998 			    buf, cfg_error(&sev));
999 		}
1000 		break;
1001 	}
1002 	cfg_close(cfg);
1003 	UNLOCKCFG();
1004 }
1005 
1006 void
1007 cleanup_hostlist(rdc_status_t *rdc_info)
1008 {
1009 	int i, j, k;
1010 	char *host, *exhost;
1011 
1012 
1013 	(void) mutex_lock(&host_list->hosts_mutex);
1014 	for (i = 0; i < host_list->numhosts; i++) {
1015 		int found = 0;
1016 		for (j = 0; (j < rdc_maxsets) && !found; j++) {
1017 			if (!rdc_info->rdc_set[j].flags & RDC_ENABLED)
1018 				continue;
1019 			if ((!host_list->configured[i]) ||
1020 			    (host_list->hosts[i] == '\0')) {
1021 				(void) mutex_unlock(&host_list->hosts_mutex);
1022 				return;
1023 			}
1024 
1025 			host = rdc_info->rdc_set[j].secondary.intf;
1026 			if (strcmp(host_list->hosts[i], host) == 0)
1027 				found++;
1028 		}
1029 		if (j == rdc_maxsets) {
1030 			/*
1031 			 * this set is not in the kernel, so remove from list
1032 			 */
1033 			exhost = host_list->hosts[i];
1034 			if (exhost) {
1035 				free(exhost);
1036 				exhost = NULL;
1037 			}
1038 
1039 			k = i;
1040 			while (k < host_list->numhosts) {
1041 			    host_list->hosts[k] = k < host_list->numhosts - 1 ?
1042 				host_list->hosts[k+1] : NULL;
1043 			    k++;
1044 			}
1045 			host_list->numhosts--;
1046 
1047 			bcopy(&host_list->configured[i+1],
1048 			    &host_list->configured[i],
1049 			    (MAXHOSTS - i + 1) * sizeof (int));
1050 			host_list->configured[MAXHOSTS - 1] = 0;
1051 		}
1052 	}
1053 	(void) mutex_unlock(&host_list->hosts_mutex);
1054 }
1055 
1056 /*
1057  * explicity remove a host from the host list
1058  * also update the configured array
1059  * called in rdc_sync, just before exiting a thread.
1060  */
1061 void
1062 remove_from_hostlist(char *host)
1063 {
1064 	int i, k;
1065 	char *exhost;
1066 
1067 	/* why bother? */
1068 	if ((!host) || (host[0] == '\0'))
1069 		return;
1070 
1071 	(void) mutex_lock(&host_list->hosts_mutex);
1072 	for (i = 0; i < host_list->numhosts; i++) {
1073 		if (strcmp(host, host_list->hosts[i]) == 0) { /* found it */
1074 			exhost = host_list->hosts[i];
1075 			if (exhost) {
1076 				free(exhost);
1077 				exhost = NULL;
1078 			}
1079 			k = i;
1080 			while (k < host_list->numhosts) {
1081 			    host_list->hosts[k] = k < host_list->numhosts - 1 ?
1082 				host_list->hosts[k+1] : NULL;
1083 			    k++;
1084 			}
1085 			host_list->numhosts--;
1086 			bcopy(&host_list->configured[i+1],
1087 			    &host_list->configured[i],
1088 			    (MAXHOSTS - i + 1) * sizeof (int));
1089 			host_list->configured[MAXHOSTS - 1] = 0;
1090 		}
1091 
1092 	}
1093 	(void) mutex_unlock(&host_list->hosts_mutex);
1094 }
1095 /*
1096  * Check to see if this host isn't in our list, so needs a new rdcsyncd proc
1097  */
1098 int
1099 isnewhost(char *host)
1100 {
1101 	int i;
1102 	int new;
1103 
1104 	if (self_check(host)) {
1105 		return (0);
1106 	}
1107 
1108 	(void) mutex_lock(&host_list->hosts_mutex);
1109 	new = 1;
1110 	for (i = 0; i < MAXHOSTS; i++) {
1111 		if (host_list->configured[i] == 0) {
1112 			host_list->configured[i] = 1;
1113 			host_list->hosts[i] = strdup(host);
1114 			host_list->numhosts++;
1115 			break;
1116 		}
1117 		if (strcmp(host, host_list->hosts[i]) == 0) {
1118 			new = 0;
1119 			break;
1120 		}
1121 	}
1122 	(void) mutex_unlock(&host_list->hosts_mutex);
1123 	if (i == MAXHOSTS)
1124 		new = 0;
1125 	return (new);
1126 }
1127 
1128 
1129 /*
1130  * Look for a matching volume name in our remembered list.
1131  */
1132 int
1133 volume_match(char *buf, char **volume_list, int volumes)
1134 {
1135 	int i;
1136 	char *vol;
1137 
1138 	for (i = 0; i < volumes; i++) {
1139 		vol = volume_list[i];
1140 		if (strcmp(buf, vol) == 0) {
1141 			return (1);
1142 		}
1143 	}
1144 	return (0);
1145 }
1146 
1147 
1148 /*
1149  * A sync has completed to a group. We can only update the ndr_ii entries
1150  * if all the members of the group have completed their syncs OK.
1151  * It would be bad to allow some members of the group to have PIT Copy snapshots
1152  * taken and others not, as they need to be consistent.
1153  */
1154 void
1155 group_complete(char *group)
1156 {
1157 	char **volumes = NULL;
1158 	spcs_s_info_t ustatus;
1159 	rdc_config_t parms = { 0 };
1160 	char buf[CFG_MAX_BUF];
1161 	char key[CFG_MAX_KEY];
1162 	CFGFILE *cfg = NULL;
1163 	int i;
1164 	int setnumber;
1165 	int found;
1166 	int replicating = 0;
1167 	char primary[NSC_MAXPATH];
1168 	char secondary[NSC_MAXPATH];
1169 	char phost[MAX_RDC_HOST_SIZE];
1170 	char shost[MAX_RDC_HOST_SIZE];
1171 	rdc_set_t *rdc_set;
1172 	int sev;
1173 	char *local_file;
1174 	int size;
1175 
1176 	ustatus = spcs_s_ucreate();
1177 
1178 	size = sizeof (char *) * rdc_maxsets;
1179 	volumes = malloc(size);
1180 	if (volumes == NULL) {
1181 		spcs_log("sndr", NULL,
1182 		    gettext("%s: unable to allocate %ld bytes"),
1183 		    program, size);
1184 		rdc_warn(NULL,
1185 			gettext("unable to allocate %ld bytes"), size);
1186 		goto done;
1187 	}
1188 	bzero(volumes, size);
1189 
1190 	/*
1191 	 * If all members of this group are replicating
1192 	 * set ii_ndr state to "update". Otherwise leave them alone.
1193 	 */
1194 	LOCKCFG();
1195 	if ((cfg = cfg_open(NULL)) == NULL) {
1196 		spcs_log("sndr", NULL,
1197 		    gettext("%s: error opening lconfig"),
1198 		    program);
1199 		rdc_warn(NULL, gettext("error opening config"));
1200 		UNLOCKCFG();
1201 		goto done;
1202 	}
1203 
1204 	if (!cfg_lock(cfg, CFG_RDLOCK)) {
1205 		spcs_log("sndr", NULL,
1206 		    gettext("%s: error locking config"),
1207 		    program);
1208 		rdc_warn(NULL, gettext("error locking config"));
1209 		goto done;
1210 	}
1211 
1212 	found = 0;
1213 
1214 	/* get all RM entries, with a matching group, that are replicating */
1215 	for (i = 0; i < rdc_maxsets; i++) {
1216 		setnumber = i + 1;
1217 
1218 		(void) snprintf(key, sizeof (key),
1219 		    "sndr.set%d.group", setnumber);
1220 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1221 			break;
1222 
1223 		if (strcmp(buf, group) != 0)
1224 			continue;
1225 
1226 		/* Found a matching entry */
1227 
1228 		(void) snprintf(key, sizeof (key),
1229 		    "sndr.set%d.primary", setnumber);
1230 		if (cfg_get_cstring(cfg, key, primary, sizeof (primary)) < 0)
1231 			break;
1232 		strcpy(parms.rdc_set->primary.file, primary);
1233 
1234 		(void) snprintf(key, sizeof (key),
1235 		    "sndr.set%d.phost", setnumber);
1236 		if (cfg_get_cstring(cfg, key, phost, sizeof (phost)) < 0)
1237 			break;
1238 		strcpy(parms.rdc_set->primary.intf, phost);
1239 
1240 		(void) snprintf(key, sizeof (key),
1241 		    "sndr.set%d.secondary", setnumber);
1242 		if (cfg_get_cstring(cfg, key, secondary,
1243 				sizeof (secondary)) < 0)
1244 			break;
1245 		strcpy(parms.rdc_set->secondary.file, secondary);
1246 
1247 		(void) snprintf(key, sizeof (key),
1248 		    "sndr.set%d.shost", setnumber);
1249 		if (cfg_get_cstring(cfg, key, shost, sizeof (shost)) < 0)
1250 			break;
1251 		strcpy(parms.rdc_set->secondary.intf, shost);
1252 
1253 		parms.command = RDC_CMD_STATUS;
1254 		if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
1255 			continue;
1256 		}
1257 
1258 		/* We found a matching set */
1259 		found++;
1260 
1261 		if (self_check(phost))
1262 			local_file = primary;
1263 		else
1264 			local_file = secondary;
1265 
1266 		rdc_set = &parms.rdc_set[0];
1267 		if (!(rdc_set->flags & RDC_LOGGING) &&
1268 		    !(rdc_set->flags & RDC_SYNCING)) {
1269 			volumes[replicating] = strdup(local_file);
1270 			if (volumes[replicating] == NULL) {
1271 				size = strlen(local_file);
1272 				spcs_log("sndr", NULL,
1273 				    gettext("%s: unable to allocate %ld bytes"),
1274 				    program, size);
1275 				rdc_warn(NULL,
1276 				    gettext("unable to allocate %ld bytes"),
1277 				    size);
1278 				goto done;
1279 			}
1280 			/* We remember all replicating sets */
1281 			replicating++;
1282 		} else
1283 			break;		/* Not all replicating, so done */
1284 	}
1285 
1286 	if (found != replicating)
1287 		goto done;
1288 
1289 	/* All replicating, so update ndr_ii state fields */
1290 
1291 	cfg_unlock(cfg);
1292 
1293 	if (!cfg_lock(cfg, CFG_WRLOCK)) {
1294 		spcs_log("sndr", NULL,
1295 		    gettext("%s: error locking lconfig"),
1296 		    program);
1297 		rdc_warn(NULL, gettext("error locking config"));
1298 		goto done;
1299 	}
1300 
1301 	/*
1302 	 * Search through the ndr_ii entries for entries
1303 	 * that match the saved secondary volume names.
1304 	 * Set state to "update".
1305 	 */
1306 
1307 	for (i = 0; ; i++) {
1308 		setnumber = i + 1;
1309 
1310 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1311 		    setnumber);
1312 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1313 			break;
1314 
1315 		if (!volume_match(buf, volumes, found)) {
1316 			continue;
1317 		}
1318 
1319 		/* Got a matching entry */
1320 
1321 		(void) snprintf(key, sizeof (key),
1322 		    "ndr_ii.set%d.state", setnumber);
1323 		if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
1324 		    (cfg_commit(cfg) < 0)) {
1325 			spcs_log("sndr", NULL,
1326 			    gettext("%s: unable to update \"%s\" "
1327 			    "in configuration storage: %s"),
1328 			    program, buf, cfg_error(&sev));
1329 			rdc_warn(NULL,
1330 			    gettext("unable to update \"%s\" "
1331 			    "in configuration storage: %s"),
1332 			    buf, cfg_error(&sev));
1333 		}
1334 	}
1335 
1336 
1337 done:
1338 	if (cfg) {
1339 		cfg_close(cfg);
1340 		UNLOCKCFG();
1341 	}
1342 	spcs_s_ufree(&ustatus);
1343 	if (volumes) {
1344 		for (i = 0; i < replicating; i++)
1345 			free(volumes[i]);
1346 		free(volumes);
1347 	}
1348 }
1349 
1350 
1351 /*
1352  * Sync started to a member of a group.
1353  * If all members of the group are in ndr_ii state "update" then take an PIT
1354  * snapshot on all of them. This will provide a consistent point-in-time
1355  * copy until whatever syncs take place are all completed.
1356  */
1357 void
1358 group_start(char *group)
1359 {
1360 	char **masters = NULL;
1361 	char **shadows = NULL;
1362 	char **bitmaps = NULL;
1363 	char cmd[256];
1364 	char buf[CFG_MAX_BUF];
1365 	char key[CFG_MAX_KEY];
1366 	CFGFILE *cfg = NULL;
1367 	int i;
1368 	int j;
1369 	int setnumber;
1370 	int found;
1371 	int sndr_sets = 0;
1372 	int update_needed = 0;
1373 	int sev;
1374 	char *ctag = NULL;
1375 	int commit = 0;
1376 	int size;
1377 
1378 	size = sizeof (char *) * rdc_maxsets;
1379 	masters = malloc(size);
1380 	if (masters == NULL) {
1381 		spcs_log("sndr", NULL,
1382 		    gettext("%s: unable to allocate %ld bytes"),
1383 		    program, size);
1384 		rdc_warn(NULL,
1385 			gettext("unable to allocate %ld bytes"), size);
1386 		goto done;
1387 	}
1388 	bzero(masters, size);
1389 	shadows = malloc(size);
1390 	if (shadows == NULL) {
1391 		spcs_log("sndr", NULL,
1392 		    gettext("%s: unable to allocate %ld bytes"),
1393 		    program, size);
1394 		rdc_warn(NULL,
1395 			gettext("unable to allocate %ld bytes"), size);
1396 		goto done;
1397 	}
1398 	bzero(shadows, size);
1399 	bitmaps = malloc(size);
1400 	if (bitmaps == NULL) {
1401 		spcs_log("sndr", NULL,
1402 		    gettext("%s: unable to allocate %ld bytes"),
1403 		    program, size);
1404 		rdc_warn(NULL,
1405 			gettext("unable to allocate %ld bytes"), size);
1406 		goto done;
1407 	}
1408 	bzero(bitmaps, size);
1409 
1410 	LOCKCFG();
1411 	if ((cfg = cfg_open(NULL)) == NULL) {
1412 		spcs_log("sndr", NULL,
1413 		    gettext("%s: error opening config"),
1414 		    program);
1415 		rdc_warn(NULL,
1416 		    gettext("error opening config"));
1417 		UNLOCKCFG();
1418 		goto done;
1419 	}
1420 
1421 	if (!cfg_lock(cfg, CFG_WRLOCK)) {
1422 		spcs_log("sndr", NULL,
1423 		    gettext("%s: error locking config"),
1424 		    program);
1425 		rdc_warn(NULL, gettext("error locking config"));
1426 		goto done;
1427 	}
1428 
1429 	/* Now get all Remote Mirror entries with a matching group */
1430 	for (i = 0; i < rdc_maxsets; i++) {
1431 		setnumber = i + 1;
1432 
1433 		(void) snprintf(key, sizeof (key),
1434 		    "sndr.set%d.group", setnumber);
1435 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1436 			break;
1437 
1438 		if (strcmp(buf, group) != 0)
1439 			continue;
1440 
1441 		/* Found a matching entry */
1442 
1443 		(void) snprintf(key, sizeof (key),
1444 		    "sndr.set%d.phost", setnumber);
1445 		if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1446 			break;
1447 
1448 		if (self_check(buf)) {
1449 			(void) snprintf(key, sizeof (key), "sndr.set%d.primary",
1450 			    setnumber);
1451 		} else {
1452 			(void) snprintf(key, sizeof (key),
1453 			    "sndr.set%d.secondary", setnumber);
1454 		}
1455 		if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1456 			break;
1457 
1458 		masters[sndr_sets] = strdup(buf);
1459 		if (masters[sndr_sets] == NULL) {
1460 			size = strlen(buf);
1461 			spcs_log("sndr", NULL,
1462 			    gettext("%s: unable to allocate %ld bytes"),
1463 			    program, size);
1464 			rdc_warn(NULL,
1465 				gettext("unable to allocate %ld bytes"), size);
1466 			goto done;
1467 		}
1468 		sndr_sets++;
1469 
1470 		if (ctag == NULL && clustered) {
1471 			/* Get cluster tag of matching entry */
1472 
1473 			(void) snprintf(key, sizeof (key), "sndr.set%d.cnode",
1474 			    setnumber);
1475 			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
1476 				ctag = strdup(buf);
1477 		}
1478 	}
1479 
1480 	/*
1481 	 * Search through the ndr_ii entries for entries
1482 	 * that match the saved local volume names and are in "update" state.
1483 	 */
1484 
1485 	update_needed = 0;
1486 
1487 	for (i = 0; ; i++) {
1488 		setnumber = i + 1;
1489 
1490 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1491 		    setnumber);
1492 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1493 			break;
1494 
1495 		if (!volume_match(buf, masters, sndr_sets))
1496 			continue;
1497 
1498 		/* Got a matching entry */
1499 
1500 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
1501 		    setnumber);
1502 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1503 			break;
1504 		shadows[update_needed] = strdup(buf);
1505 		if (shadows[update_needed] == NULL) {
1506 			size = strlen(buf);
1507 			spcs_log("sndr", NULL,
1508 			    gettext("%s: unable to allocate %ld bytes"),
1509 			    program, size);
1510 			rdc_warn(NULL,
1511 				gettext("unable to allocate %ld bytes"), size);
1512 			goto done;
1513 		}
1514 
1515 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
1516 		    setnumber);
1517 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1518 			break;
1519 		}
1520 		bitmaps[update_needed] = strdup(buf);
1521 		if (bitmaps[update_needed] == NULL) {
1522 			size = strlen(buf);
1523 			spcs_log("sndr", NULL,
1524 			    gettext("%s: unable to allocate %ld bytes"),
1525 			    program, size);
1526 			rdc_warn(NULL,
1527 				gettext("unable to allocate %ld bytes"), size);
1528 			goto done;
1529 		}
1530 
1531 		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1532 		    setnumber);
1533 		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1534 			break;
1535 		}
1536 		if (strcmp(buf, UPDATE) != 0) {
1537 			break;
1538 		}
1539 
1540 		update_needed++;
1541 	}
1542 
1543 	if (update_needed != sndr_sets) {
1544 #ifdef DEBUG
1545 		spcs_log("sndr", NULL,
1546 		    gettext("%s: group sync: no Point-in-Time Copy snapshot "
1547 			    "for %s"), program, group);
1548 #endif
1549 		goto done;
1550 	}
1551 
1552 	/* All RM sets in the group have an ndr_ii entry in "update" state */
1553 
1554 	/* Issue PIT Copy snapshot commands for all sets in the group */
1555 	for (j = 0; j < sndr_sets; j++) {
1556 		found = 0;
1557 
1558 		/* get ii entries until a match is found */
1559 		for (i = 0; ; i++) {
1560 			setnumber = i + 1;
1561 
1562 			(void) snprintf(key, sizeof (key), "ii.set%d.shadow",
1563 			    setnumber);
1564 			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1565 				break;
1566 			if (strcmp(buf, shadows[j]) != 0)
1567 				continue;
1568 
1569 			/* Matching shadow found, so ii already enabled */
1570 			found = 1;
1571 			break;
1572 		}
1573 
1574 		if (commit)
1575 			if (cfg_commit(cfg) < 0)
1576 				rdc_warn(NULL, gettext("commit config error"));
1577 		cfg_close(cfg);
1578 
1579 		if (found) {
1580 			(void) sprintf(cmd, "%s -u s %s", IIADM, shadows[j]);
1581 		} else {
1582 			if (ctag) {
1583 				(void) sprintf(cmd, "%s -C %s -e dep %s %s %s",
1584 				    IIADM, ctag, masters[j], shadows[j],
1585 				    bitmaps[j]);
1586 				free(ctag);
1587 				ctag = NULL;
1588 			} else
1589 				(void) sprintf(cmd, "%s -e dep %s %s %s", IIADM,
1590 				    masters[j], shadows[j], bitmaps[j]);
1591 		}
1592 
1593 		if (system(cmd) != 0) {
1594 			spcs_log("sndr", NULL,
1595 			    gettext("%s: group sync: Point-in-Time Copy"
1596 			    " snapshot failed for %s"),
1597 			    program, masters[j]);
1598 
1599 			goto done;
1600 		}
1601 
1602 		if ((cfg = cfg_open(NULL)) == NULL) {
1603 			spcs_log("sndr", NULL,
1604 			    gettext("%s: error opening config"),
1605 			    program);
1606 			rdc_warn(NULL,
1607 			    gettext("error opening config"));
1608 			goto done;
1609 		}
1610 		if (!cfg_lock(cfg, CFG_WRLOCK)) {
1611 			spcs_log("sndr", NULL,
1612 			    gettext("%s: error locking config"),
1613 			    program);
1614 			rdc_warn(NULL, gettext("error locking config"));
1615 			goto done;
1616 		}
1617 		commit = 0;
1618 
1619 		/* PIT enable or update was fine, so update the ndr_ii entry */
1620 
1621 		/* get ndr_ii entries until a match is found */
1622 		for (i = 0; ; i++) {
1623 			setnumber = i + 1;
1624 
1625 			(void) snprintf(key, sizeof (key),
1626 			    "ndr_ii.set%d.shadow", setnumber);
1627 			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1628 				break;
1629 			if (strcmp(buf, shadows[j]) != 0)
1630 				continue;
1631 
1632 			/* Found the matching entry */
1633 
1634 			(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1635 			    setnumber);
1636 			if (cfg_put_cstring(cfg, key, NOUPDATE,
1637 			    strlen(NOUPDATE)) < 0) {
1638 				spcs_log("sndr", NULL,
1639 				    gettext("%s: unable to update \"%s\" "
1640 				    "in configuration storage: %s"),
1641 				    program, buf, cfg_error(&sev));
1642 				rdc_warn(NULL,
1643 				    gettext("unable to update \"%s\" "
1644 				    "in configuration storage: %s"),
1645 				    buf, cfg_error(&sev));
1646 			} else
1647 				commit = 1;
1648 			break;
1649 		}
1650 	}
1651 
1652 	if (commit)
1653 		if (cfg_commit(cfg) < 0)
1654 			rdc_warn(NULL, gettext("commit config error"));
1655 
1656 	spcs_log("sndr", NULL,
1657 	    gettext("%s: group sync: Point-in-Time Copy snapshots completed "
1658 		    "for %s"), program, group);
1659 
1660 done:
1661 	if (ctag)
1662 		free(ctag);
1663 
1664 	if (cfg) {
1665 		cfg_close(cfg);
1666 		UNLOCKCFG();
1667 	}
1668 
1669 	if (masters) {
1670 		for (i = 0; i < sndr_sets; i++) {
1671 			if (masters[i])
1672 				free(masters[i]);
1673 		}
1674 		free(masters);
1675 	}
1676 
1677 	if (shadows) {
1678 		for (i = 0; i < update_needed; i++) {
1679 			if (shadows[i])
1680 				free(shadows[i]);
1681 		}
1682 		free(shadows);
1683 	}
1684 
1685 	if (bitmaps) {
1686 		for (i = 0; i < update_needed; i++) {
1687 			if (bitmaps[i])
1688 				free(bitmaps[i]);
1689 		}
1690 		free(bitmaps);
1691 	}
1692 }
1693