xref: /titanic_52/usr/src/uts/common/avs/ns/rdc/rdc.c (revision 49d3bc91e27cd871b950d56c01398fa2f2e12ab4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #define	_RDC_
27 #include <sys/types.h>
28 #include <sys/ksynch.h>
29 #include <sys/kmem.h>
30 #include <sys/errno.h>
31 #include <sys/conf.h>
32 #include <sys/cmn_err.h>
33 #include <sys/modctl.h>
34 #include <sys/cred.h>
35 #include <sys/ddi.h>
36 #include <sys/unistat/spcs_s.h>
37 #include <sys/unistat/spcs_s_k.h>
38 #include <sys/unistat/spcs_errors.h>
39 
40 #include <sys/nsc_thread.h>
41 #ifdef DS_DDICT
42 #include "../contract.h"
43 #endif
44 #include <sys/nsctl/nsctl.h>
45 #include <sys/nsctl/nsvers.h>
46 
47 #include <sys/sdt.h>		/* dtrace is S10 or later */
48 
49 #include "rdc.h"
50 #include "rdc_io.h"
51 #include "rdc_bitmap.h"
52 #include "rdc_ioctl.h"
53 #include "rdcsrv.h"
54 #include "rdc_diskq.h"
55 
56 #define	DIDINIT		0x01
57 #define	DIDNODES	0x02
58 #define	DIDCONFIG	0x04
59 
60 static int rdcopen(dev_t *devp, int flag, int otyp, cred_t *crp);
61 static int rdcclose(dev_t dev, int flag, int otyp, cred_t *crp);
62 static int rdcprint(dev_t dev, char *str);
63 static int rdcioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *crp,
64 	int *rvp);
65 static int rdcattach(dev_info_t *dip, ddi_attach_cmd_t cmd);
66 static int rdcdetach(dev_info_t *dip, ddi_detach_cmd_t cmd);
67 static int rdcgetinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg,
68 	void **result);
69 #ifdef	DEBUG
70 static int rdc_clrkstat(void *);
71 #endif
72 
73 /*
74  * kstat interface
75  */
76 static kstat_t *sndr_kstats;
77 
78 int sndr_info_stats_update(kstat_t *ksp, int rw);
79 
80 static sndr_m_stats_t sndr_info_stats = {
81 	{RDC_MKSTAT_MAXSETS,			KSTAT_DATA_ULONG},
82 	{RDC_MKSTAT_MAXFBAS,			KSTAT_DATA_ULONG},
83 	{RDC_MKSTAT_RPC_TIMEOUT,		KSTAT_DATA_ULONG},
84 	{RDC_MKSTAT_HEALTH_THRES,		KSTAT_DATA_ULONG},
85 	{RDC_MKSTAT_BITMAP_WRITES,		KSTAT_DATA_ULONG},
86 	{RDC_MKSTAT_CLNT_COTS_CALLS,		KSTAT_DATA_ULONG},
87 	{RDC_MKSTAT_CLNT_CLTS_CALLS,		KSTAT_DATA_ULONG},
88 	{RDC_MKSTAT_SVC_COTS_CALLS,		KSTAT_DATA_ULONG},
89 	{RDC_MKSTAT_SVC_CLTS_CALLS,		KSTAT_DATA_ULONG},
90 	{RDC_MKSTAT_BITMAP_REF_DELAY,		KSTAT_DATA_ULONG}
91 };
92 
93 int rdc_info_stats_update(kstat_t *ksp, int rw);
94 
95 static rdc_info_stats_t rdc_info_stats = {
96 	{RDC_IKSTAT_FLAGS,		KSTAT_DATA_ULONG},
97 	{RDC_IKSTAT_SYNCFLAGS,		KSTAT_DATA_ULONG},
98 	{RDC_IKSTAT_BMPFLAGS,		KSTAT_DATA_ULONG},
99 	{RDC_IKSTAT_SYNCPOS,		KSTAT_DATA_ULONG},
100 	{RDC_IKSTAT_VOLSIZE,		KSTAT_DATA_ULONG},
101 	{RDC_IKSTAT_BITSSET,		KSTAT_DATA_ULONG},
102 	{RDC_IKSTAT_AUTOSYNC,		KSTAT_DATA_ULONG},
103 	{RDC_IKSTAT_MAXQFBAS,		KSTAT_DATA_ULONG},
104 	{RDC_IKSTAT_MAXQITEMS,		KSTAT_DATA_ULONG},
105 	{RDC_IKSTAT_FILE,		KSTAT_DATA_STRING},
106 	{RDC_IKSTAT_SECFILE,		KSTAT_DATA_STRING},
107 	{RDC_IKSTAT_BITMAP,		KSTAT_DATA_STRING},
108 	{RDC_IKSTAT_PRIMARY_HOST,	KSTAT_DATA_STRING},
109 	{RDC_IKSTAT_SECONDARY_HOST,	KSTAT_DATA_STRING},
110 	{RDC_IKSTAT_TYPE_FLAG,		KSTAT_DATA_ULONG},
111 	{RDC_IKSTAT_BMP_SIZE,		KSTAT_DATA_ULONG},
112 	{RDC_IKSTAT_DISK_STATUS,	KSTAT_DATA_ULONG},
113 	{RDC_IKSTAT_IF_DOWN,		KSTAT_DATA_ULONG},
114 	{RDC_IKSTAT_IF_RPC_VERSION,	KSTAT_DATA_ULONG},
115 	{RDC_IKSTAT_ASYNC_BLOCK_HWM,	KSTAT_DATA_ULONG},
116 	{RDC_IKSTAT_ASYNC_ITEM_HWM,	KSTAT_DATA_ULONG},
117 	{RDC_IKSTAT_ASYNC_THROTTLE_DELAY,	KSTAT_DATA_ULONG},
118 	{RDC_IKSTAT_ASYNC_ITEMS,	KSTAT_DATA_ULONG},
119 	{RDC_IKSTAT_ASYNC_BLOCKS,	KSTAT_DATA_ULONG},
120 	{RDC_IKSTAT_QUEUE_TYPE,		KSTAT_DATA_CHAR}
121 };
122 
123 static struct cb_ops rdc_cb_ops = {
124 	rdcopen,
125 	rdcclose,
126 	nulldev,		/* no strategy */
127 	rdcprint,
128 	nodev,			/* no dump */
129 	nodev,			/* no read */
130 	nodev,			/* no write */
131 	rdcioctl,
132 	nodev,			/* no devmap */
133 	nodev,			/* no mmap */
134 	nodev,			/* no segmap */
135 	nochpoll,
136 	ddi_prop_op,
137 	NULL,			/* not STREAMS */
138 	D_NEW | D_MP | D_64BIT,
139 	CB_REV,
140 	nodev,			/* no aread */
141 	nodev,			/* no awrite */
142 };
143 
144 static struct dev_ops rdc_ops = {
145 	DEVO_REV,
146 	0,
147 	rdcgetinfo,
148 	nulldev,		/* identify */
149 	nulldev,		/* probe */
150 	rdcattach,
151 	rdcdetach,
152 	nodev,			/* no reset */
153 	&rdc_cb_ops,
154 	(struct bus_ops *)NULL
155 };
156 
157 static struct modldrv rdc_ldrv = {
158 	&mod_driverops,
159 	"nws:Remote Mirror:" ISS_VERSION_STR,
160 	&rdc_ops
161 };
162 
163 static struct modlinkage rdc_modlinkage = {
164 	MODREV_1,
165 	&rdc_ldrv,
166 	NULL
167 };
168 
169 const	int sndr_major_rev = ISS_VERSION_MAJ;
170 const	int sndr_minor_rev = ISS_VERSION_MIN;
171 const	int sndr_micro_rev = ISS_VERSION_MIC;
172 const	int sndr_baseline_rev = ISS_VERSION_NUM;
173 static	char sndr_version[16];
174 
175 static void *rdc_dip;
176 
177 extern int _rdc_init_dev();
178 extern void _rdc_deinit_dev();
179 extern void rdc_link_down_free();
180 
181 int rdc_bitmap_mode;
182 int rdc_auto_sync;
183 int rdc_max_sets;
184 extern int rdc_health_thres;
185 
186 kmutex_t rdc_sync_mutex;
187 rdc_sync_event_t rdc_sync_event;
188 clock_t rdc_sync_event_timeout;
189 
190 static void
191 rdc_sync_event_init()
192 {
193 	mutex_init(&rdc_sync_mutex, NULL, MUTEX_DRIVER, NULL);
194 	mutex_init(&rdc_sync_event.mutex, NULL, MUTEX_DRIVER, NULL);
195 	cv_init(&rdc_sync_event.cv, NULL, CV_DRIVER, NULL);
196 	cv_init(&rdc_sync_event.done_cv, NULL, CV_DRIVER, NULL);
197 	rdc_sync_event.master[0] = 0;
198 	rdc_sync_event.lbolt = (clock_t)0;
199 	rdc_sync_event_timeout = RDC_SYNC_EVENT_TIMEOUT;
200 }
201 
202 
203 static void
204 rdc_sync_event_destroy()
205 {
206 	mutex_destroy(&rdc_sync_mutex);
207 	mutex_destroy(&rdc_sync_event.mutex);
208 	cv_destroy(&rdc_sync_event.cv);
209 	cv_destroy(&rdc_sync_event.done_cv);
210 }
211 
212 
213 
214 int
215 _init(void)
216 {
217 	return (mod_install(&rdc_modlinkage));
218 }
219 
220 int
221 _fini(void)
222 {
223 	return (mod_remove(&rdc_modlinkage));
224 }
225 
226 int
227 _info(struct modinfo *modinfop)
228 {
229 	return (mod_info(&rdc_modlinkage, modinfop));
230 }
231 
232 static int
233 rdcattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
234 {
235 	intptr_t flags;
236 	int instance;
237 	int i;
238 
239 	/*CONSTCOND*/
240 	ASSERT(sizeof (u_longlong_t) == 8);
241 
242 	if (cmd != DDI_ATTACH)
243 		return (DDI_FAILURE);
244 
245 	(void) strncpy(sndr_version, _VERSION_, sizeof (sndr_version));
246 
247 	instance = ddi_get_instance(dip);
248 	rdc_dip = dip;
249 
250 	flags = 0;
251 
252 	rdc_sync_event_init();
253 
254 	/*
255 	 * rdc_max_sets must be set before calling _rdc_load().
256 	 */
257 
258 	rdc_max_sets = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
259 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "rdc_max_sets", 64);
260 
261 	if (_rdc_init_dev()) {
262 		cmn_err(CE_WARN, "!rdc: _rdc_init_dev failed");
263 		goto out;
264 	}
265 	flags |= DIDINIT;
266 
267 	if (_rdc_load() != 0) {
268 		cmn_err(CE_WARN, "!rdc: _rdc_load failed");
269 		goto out;
270 	}
271 
272 	if (_rdc_configure()) {
273 		cmn_err(CE_WARN, "!rdc: _rdc_configure failed");
274 		goto out;
275 	}
276 	flags |= DIDCONFIG;
277 
278 	if (ddi_create_minor_node(dip, "rdc", S_IFCHR, instance, DDI_PSEUDO, 0)
279 	    != DDI_SUCCESS) {
280 		cmn_err(CE_WARN, "!rdc: could not create node.");
281 		goto out;
282 	}
283 	flags |= DIDNODES;
284 
285 	rdc_bitmap_mode = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
286 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
287 	    "rdc_bitmap_mode", 0);
288 
289 	switch (rdc_bitmap_mode) {
290 	case RDC_BMP_AUTO:		/* 0 */
291 		break;
292 	case RDC_BMP_ALWAYS:		/* 1 */
293 		break;
294 	case RDC_BMP_NEVER:		/* 2 */
295 		cmn_err(CE_NOTE, "!SNDR bitmap mode override");
296 		cmn_err(CE_CONT,
297 		    "!SNDR: bitmaps will only be written on shutdown\n");
298 		break;
299 	default:			/* unknown */
300 		cmn_err(CE_NOTE,
301 		    "!SNDR: unknown bitmap mode %d - autodetecting mode",
302 		    rdc_bitmap_mode);
303 		rdc_bitmap_mode = RDC_BMP_AUTO;
304 		break;
305 	}
306 
307 	rdc_bitmap_init();
308 
309 	rdc_auto_sync = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
310 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
311 	    "rdc_auto_sync", 0);
312 
313 	i = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
314 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
315 	    "rdc_health_thres", RDC_HEALTH_THRESHOLD);
316 	if (i >= RDC_MIN_HEALTH_THRES)
317 		rdc_health_thres = i;
318 	else
319 		cmn_err(CE_WARN, "!value rdc_heath_thres from rdc.conf ignored "
320 		    "as it is smaller than the min value of %d",
321 		    RDC_MIN_HEALTH_THRES);
322 
323 	ddi_set_driver_private(dip, (caddr_t)flags);
324 	ddi_report_dev(dip);
325 
326 	sndr_kstats = kstat_create(RDC_KSTAT_MODULE, 0,
327 	    RDC_KSTAT_MINFO, RDC_KSTAT_CLASS, KSTAT_TYPE_NAMED,
328 	    sizeof (sndr_m_stats_t) / sizeof (kstat_named_t),
329 	    KSTAT_FLAG_VIRTUAL);
330 
331 	if (sndr_kstats) {
332 		sndr_kstats->ks_data = &sndr_info_stats;
333 		sndr_kstats->ks_update = sndr_info_stats_update;
334 		sndr_kstats->ks_private = &rdc_k_info[0];
335 		kstat_install(sndr_kstats);
336 	} else
337 			cmn_err(CE_WARN, "!SNDR: module kstats failed");
338 
339 	return (DDI_SUCCESS);
340 
341 out:
342 	DTRACE_PROBE(rdc_attach_failed);
343 	ddi_set_driver_private(dip, (caddr_t)flags);
344 	(void) rdcdetach(dip, DDI_DETACH);
345 	return (DDI_FAILURE);
346 }
347 
348 static int
349 rdcdetach(dev_info_t *dip, ddi_detach_cmd_t cmd)
350 {
351 	rdc_k_info_t *krdc;
352 	rdc_u_info_t *urdc;
353 	int rdcd;
354 	intptr_t flags;
355 
356 
357 	if (cmd != DDI_DETACH) {
358 		DTRACE_PROBE(rdc_detach_unknown_cmd);
359 		return (DDI_FAILURE);
360 	}
361 
362 	if (rdc_k_info == NULL || rdc_u_info == NULL)
363 		goto cleanup;
364 
365 	mutex_enter(&rdc_conf_lock);
366 
367 	for (rdcd = 0; rdcd < rdc_max_sets; rdcd++) {
368 		krdc = &rdc_k_info[rdcd];
369 		urdc = &rdc_u_info[rdcd];
370 
371 		if (IS_ENABLED(urdc) || krdc->devices) {
372 #ifdef DEBUG
373 			cmn_err(CE_WARN,
374 			    "!rdc: cannot detach, rdcd %d still in use", rdcd);
375 #endif
376 			mutex_exit(&rdc_conf_lock);
377 			DTRACE_PROBE(rdc_detach_err_busy);
378 			return (DDI_FAILURE);
379 		}
380 	}
381 
382 	mutex_exit(&rdc_conf_lock);
383 
384 cleanup:
385 	flags = (intptr_t)ddi_get_driver_private(dip);
386 
387 	if (flags & DIDNODES)
388 		ddi_remove_minor_node(dip, NULL);
389 
390 	if (sndr_kstats) {
391 		kstat_delete(sndr_kstats);
392 	}
393 	if (flags & DIDINIT)
394 		_rdc_deinit_dev();
395 
396 	if (flags & DIDCONFIG) {
397 		(void) _rdc_deconfigure();
398 		(void) _rdc_unload();
399 		rdcsrv_unload();
400 	}
401 
402 	rdc_sync_event_destroy();
403 	rdc_link_down_free();
404 
405 	rdc_dip = NULL;
406 	return (DDI_SUCCESS);
407 }
408 
409 /* ARGSUSED */
410 static int
411 rdcgetinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
412 {
413 	int rc = DDI_FAILURE;
414 
415 	switch (infocmd) {
416 
417 	case DDI_INFO_DEVT2DEVINFO:
418 		*result = rdc_dip;
419 		rc = DDI_SUCCESS;
420 		break;
421 
422 	case DDI_INFO_DEVT2INSTANCE:
423 		/* We only have a single instance */
424 		*result = 0;
425 		rc = DDI_SUCCESS;
426 		break;
427 
428 	default:
429 		break;
430 	}
431 
432 	return (rc);
433 }
434 
435 
436 /* ARGSUSED */
437 
438 static int
439 rdcopen(dev_t *devp, int flag, int otyp, cred_t *crp)
440 {
441 	return (0);
442 }
443 
444 
445 /* ARGSUSED */
446 
447 static int
448 rdcclose(dev_t dev, int flag, int otyp, cred_t *crp)
449 {
450 	return (0);
451 }
452 
453 /* ARGSUSED */
454 
455 static int
456 rdcprint(dev_t dev, char *str)
457 {
458 	int instance = 0;
459 
460 	cmn_err(CE_WARN, "!rdc%d: %s", instance, str);
461 	return (0);
462 }
463 
464 
465 static int
466 convert_ioctl_args(int cmd, intptr_t arg, int mode, _rdc_ioctl_t *args)
467 {
468 	_rdc_ioctl32_t args32;
469 
470 	if (ddi_copyin((void *)arg, &args32, sizeof (_rdc_ioctl32_t), mode))
471 		return (EFAULT);
472 
473 	bzero((void *)args, sizeof (_rdc_ioctl_t));
474 
475 	switch (cmd) {
476 	case RDC_CONFIG:
477 		args->arg0 = (uint32_t)args32.arg0; /* _rdc_config_t * */
478 		args->arg1 = (uint32_t)args32.arg1; /* pointer */
479 		args->arg2 = (uint32_t)args32.arg2; /* size */
480 		args->ustatus = (spcs_s_info_t)args32.ustatus;
481 		break;
482 
483 	case RDC_STATUS:
484 		args->arg0 = (uint32_t)args32.arg0; /* pointer */
485 		args->ustatus = (spcs_s_info_t)args32.ustatus;
486 		break;
487 
488 	case RDC_ENABLE_SVR:
489 		args->arg0 = (uint32_t)args32.arg0; /* _rdc_svc_args *  */
490 		break;
491 
492 	case RDC_VERSION:
493 		args->arg0 = (uint32_t)args32.arg0; /* _rdc_version_t *  */
494 		args->ustatus = (spcs_s_info_t)args32.ustatus;
495 		break;
496 
497 	case RDC_SYNC_EVENT:
498 		args->arg0 = (uint32_t)args32.arg0; /* char *  */
499 		args->arg1 = (uint32_t)args32.arg1; /* char *  */
500 		args->ustatus = (spcs_s_info_t)args32.ustatus;
501 		break;
502 
503 	case RDC_LINK_DOWN:
504 		args->arg0 = (uint32_t)args32.arg0; /* char *  */
505 		args->ustatus = (spcs_s_info_t)args32.ustatus;
506 		break;
507 	case RDC_POOL_CREATE:
508 		args->arg0 = (uint32_t)args32.arg0; /* svcpool_args *  */
509 		break;
510 	case RDC_POOL_WAIT:
511 		args->arg0 = (uint32_t)args32.arg0; /* int */
512 		break;
513 	case RDC_POOL_RUN:
514 		args->arg0 = (uint32_t)args32.arg0; /* int */
515 		break;
516 
517 	default:
518 		return (EINVAL);
519 	}
520 
521 	return (0);
522 }
523 
524 
525 /*
526  * Yet another standard thing that is not standard ...
527  */
528 #ifndef	offsetof
529 #define	offsetof(s, m)	((size_t)(&((s *)0)->m))
530 #endif
531 
532 /*
533  * Build a 32bit rdc_set structure and copyout to the user level.
534  */
535 int
536 rdc_status_copy32(const void *arg, void *usetp, size_t size, int mode)
537 {
538 	rdc_u_info_t *urdc = (rdc_u_info_t *)arg;
539 	struct rdc_set32 set32;
540 	size_t tailsize;
541 #ifdef DEBUG
542 	size_t tailsize32;
543 #endif
544 
545 	bzero(&set32, sizeof (set32));
546 
547 	tailsize = sizeof (struct rdc_addr32) -
548 	    offsetof(struct rdc_addr32, intf);
549 
550 	/* primary address structure, avoiding netbuf */
551 	bcopy(&urdc->primary.intf[0], &set32.primary.intf[0], tailsize);
552 
553 	/* secondary address structure, avoiding netbuf */
554 	bcopy(&urdc->secondary.intf[0], &set32.secondary.intf[0], tailsize);
555 
556 	/*
557 	 * the rest, avoiding netconfig
558 	 * note: the tail must be the same size in both structures
559 	 */
560 	tailsize = sizeof (struct rdc_set) - offsetof(struct rdc_set, flags);
561 #ifdef DEBUG
562 	/*
563 	 * ASSERT is calling for debug reason, and tailsize32 is only declared
564 	 * for ASSERT, put them under debug to avoid lint warning.
565 	 */
566 	tailsize32 = sizeof (struct rdc_set32) -
567 	    offsetof(struct rdc_set32, flags);
568 	ASSERT(tailsize == tailsize32);
569 #endif
570 
571 	bcopy(&urdc->flags, &set32.flags, tailsize);
572 
573 	/* copyout to user level */
574 	return (ddi_copyout(&set32, usetp, size, mode));
575 }
576 
577 
578 /*
579  * Status ioctl.
580  */
581 static int
582 rdcstatus(_rdc_ioctl_t *args, int mode)
583 {
584 	int (*copyout)(const void *, void *, size_t, int);
585 	rdc_u_info_t *urdc;
586 	rdc_k_info_t *krdc;
587 	disk_queue *dqp;
588 	char *usetp;			/* pointer to user rdc_set structure */
589 	size_t size;			/* sizeof user rdc_set structure */
590 	int32_t *maxsetsp;		/* address of status->maxsets; */
591 	int nset, max, i, j;
592 
593 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
594 		struct rdc_status32 status32;
595 
596 		if (ddi_copyin((void *)args->arg0, &status32,
597 		    sizeof (status32), mode)) {
598 			return (EFAULT);
599 		}
600 
601 		usetp = ((char *)args->arg0) +
602 		    offsetof(struct rdc_status32, rdc_set);
603 		maxsetsp = (int32_t *)((char *)args->arg0 +
604 		    offsetof(struct rdc_status32, maxsets));
605 		nset = status32.nset;
606 
607 		size = sizeof (struct rdc_set32);
608 		copyout = rdc_status_copy32;
609 	} else {
610 		struct rdc_status status;
611 
612 		if (ddi_copyin((void *)args->arg0, &status,
613 		    sizeof (status), mode)) {
614 			return (EFAULT);
615 		}
616 
617 		usetp = ((char *)args->arg0) +
618 		    offsetof(struct rdc_status, rdc_set);
619 		maxsetsp = (int32_t *)((char *)args->arg0 +
620 		    offsetof(struct rdc_status, maxsets));
621 		nset = status.nset;
622 
623 		size = sizeof (struct rdc_set);
624 		copyout = ddi_copyout;
625 	}
626 
627 	max = min(nset, rdc_max_sets);
628 
629 	for (i = 0, j = 0; i < max; i++) {
630 		urdc = &rdc_u_info[i];
631 		krdc = &rdc_k_info[i];
632 
633 		if (!IS_ENABLED(urdc))
634 			continue;
635 
636 		/*
637 		 * sneak out qstate in urdc->flags
638 		 * this is harmless because it's value is not used
639 		 * in urdc->flags. the real qstate is kept in
640 		 * group->diskq->disk_hdr.h.state
641 		 */
642 		if (RDC_IS_DISKQ(krdc->group)) {
643 			dqp = &krdc->group->diskq;
644 			if (IS_QSTATE(dqp, RDC_QNOBLOCK))
645 				urdc->flags |= RDC_QNOBLOCK;
646 		}
647 
648 		j++;
649 		if ((*copyout)(urdc, usetp, size, mode) != 0)
650 			return (EFAULT);
651 
652 		urdc->flags &= ~RDC_QNOBLOCK; /* clear qstate */
653 		usetp += size;
654 	}
655 
656 	/* copyout rdc_max_sets value */
657 
658 	if (ddi_copyout(&rdc_max_sets, maxsetsp, sizeof (*maxsetsp), mode) != 0)
659 		return (EFAULT);
660 
661 	/* copyout number of sets manipulated */
662 
663 	/*CONSTCOND*/
664 	ASSERT(offsetof(struct rdc_status32, nset) == 0);
665 	/*CONSTCOND*/
666 	ASSERT(offsetof(struct rdc_status, nset) == 0);
667 
668 	return (ddi_copyout(&j, (void *)args->arg0, sizeof (int), mode));
669 }
670 
671 
672 /* ARGSUSED */
673 
674 static int
675 rdcioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *crp, int *rvp)
676 {
677 	spcs_s_info_t kstatus = NULL;
678 	_rdc_ioctl_t args;
679 	int error;
680 	int rc = 0;
681 
682 	if (cmd != RDC_STATUS) {
683 		if ((error = drv_priv(crp)) != 0)
684 			return (error);
685 	}
686 #ifdef	DEBUG
687 	if (cmd == RDC_ASYNC6) {
688 		rc = rdc_async6((void *)arg, mode, rvp);
689 		return (rc);
690 	}
691 
692 	if (cmd == RDC_CLRKSTAT) {
693 		rc = rdc_clrkstat((void *)arg);
694 		return (rc);
695 	}
696 
697 	if (cmd == RDC_STALL0) {
698 		if (((int)arg > 1) || ((int)arg < 0))
699 			return (EINVAL);
700 		rdc_stallzero((int)arg);
701 		return (0);
702 	}
703 	if (cmd == RDC_READGEN) {
704 		rc = rdc_readgen((void *)arg, mode, rvp);
705 		return (rc);
706 	}
707 #endif
708 	if (cmd == RDC_BITMAPOP) {
709 		rdc_bitmap_op_t bmop;
710 		rdc_bitmap_op32_t bmop32;
711 
712 		if (ddi_model_convert_from(mode & FMODELS)
713 		    == DDI_MODEL_ILP32) {
714 			if (ddi_copyin((void *)arg, &bmop32, sizeof (bmop32),
715 			    mode))
716 				return (EFAULT);
717 			bmop.offset = bmop32.offset;
718 			bmop.op = bmop32.op;
719 			(void) strncpy(bmop.sechost, bmop32.sechost,
720 			    MAX_RDC_HOST_SIZE);
721 			(void) strncpy(bmop.secfile, bmop32.secfile,
722 			    NSC_MAXPATH);
723 			bmop.len = bmop32.len;
724 			bmop.addr = (unsigned long)bmop32.addr;
725 		} else {
726 			if (ddi_copyin((void *)arg, &bmop, sizeof (bmop),
727 			    mode))
728 				return (EFAULT);
729 		}
730 		rc = rdc_bitmapset(bmop.op, bmop.sechost, bmop.secfile,
731 		    (void *)bmop.addr, bmop.len, bmop.offset, mode);
732 		return (rc);
733 	}
734 
735 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
736 		if ((rc = convert_ioctl_args(cmd, arg, mode, &args)) != 0)
737 			return (rc);
738 	} else {
739 		if (ddi_copyin((void *)arg, &args,
740 		    sizeof (_rdc_ioctl_t), mode)) {
741 			return (EFAULT);
742 		}
743 	}
744 
745 	kstatus = spcs_s_kcreate();
746 	if (!kstatus) {
747 		return (ENOMEM);
748 	}
749 
750 
751 	switch (cmd) {
752 
753 	case RDC_POOL_CREATE: {
754 		struct svcpool_args p;
755 
756 		if (ddi_copyin((void *)arg, &p, sizeof (p), mode)) {
757 			spcs_s_kfree(kstatus);
758 			return (EFAULT);
759 		}
760 		error = svc_pool_create(&p);
761 
762 		break;
763 	}
764 	case RDC_POOL_WAIT: {
765 		int id;
766 
767 		if (ddi_copyin((void *)arg, &id, sizeof (id), mode)) {
768 			spcs_s_kfree(kstatus);
769 			return (EFAULT);
770 		}
771 
772 		error = svc_wait(id);
773 		break;
774 	}
775 	case RDC_POOL_RUN: {
776 		int id;
777 
778 		if (ddi_copyin((void *)arg, &id, sizeof (id), mode)) {
779 			spcs_s_kfree(kstatus);
780 			return (EFAULT);
781 		}
782 		error = svc_do_run(id);
783 		break;
784 	}
785 	case RDC_ENABLE_SVR:
786 		{
787 			STRUCT_DECL(rdc_svc_args, parms);
788 
789 			STRUCT_INIT(parms, mode);
790 			/* Only used by sndrd which does not use unistat */
791 
792 			if (ddi_copyin((void *)args.arg0, STRUCT_BUF(parms),
793 			    STRUCT_SIZE(parms), mode)) {
794 				spcs_s_kfree(kstatus);
795 				return (EFAULT);
796 			}
797 			rc = rdc_start_server(STRUCT_BUF(parms), mode);
798 		}
799 		break;
800 
801 	case RDC_STATUS:
802 		rc = rdcstatus(&args, mode);
803 		break;
804 
805 	case RDC_CONFIG:
806 		rc = _rdc_config((void *)args.arg0, mode, kstatus, rvp);
807 		spcs_s_copyoutf(&kstatus, args.ustatus);
808 		return (rc);
809 
810 	case RDC_VERSION:
811 		{
812 			STRUCT_DECL(rdc_version, parms);
813 
814 			STRUCT_INIT(parms, mode);
815 
816 			STRUCT_FSET(parms, major, sndr_major_rev);
817 			STRUCT_FSET(parms, minor, sndr_minor_rev);
818 			STRUCT_FSET(parms, micro, sndr_micro_rev);
819 			STRUCT_FSET(parms, baseline, sndr_baseline_rev);
820 
821 			if (ddi_copyout(STRUCT_BUF(parms), (void *)args.arg0,
822 			    STRUCT_SIZE(parms), mode)) {
823 				spcs_s_kfree(kstatus);
824 				return (EFAULT);
825 			}
826 			break;
827 		}
828 
829 	case RDC_LINK_DOWN:
830 		/* char *host from user */
831 		rc = _rdc_link_down((void *)args.arg0, mode, kstatus, rvp);
832 		spcs_s_copyoutf(&kstatus, args.ustatus);
833 
834 		return (rc);
835 
836 	case RDC_SYNC_EVENT:
837 		rc = _rdc_sync_event_wait((void *)args.arg0, (void *)args.arg1,
838 		    mode, kstatus, rvp);
839 		spcs_s_copyoutf(&kstatus, args.ustatus);
840 
841 		return (rc);
842 
843 
844 	default:
845 		rc = EINVAL;
846 		break;
847 	}
848 
849 	spcs_s_kfree(kstatus);
850 	return (rc);
851 }
852 
853 int
854 sndr_info_stats_update(kstat_t *ksp, int rw)
855 {
856 	extern int rdc_rpc_tmout;
857 	extern int rdc_health_thres;
858 	extern int rdc_bitmap_delay;
859 	extern long rdc_clnt_count;
860 	extern long rdc_svc_count;
861 	sndr_m_stats_t	*info_stats;
862 	rdc_k_info_t	*krdc;
863 
864 	info_stats = (sndr_m_stats_t *)(ksp->ks_data);
865 	krdc = (rdc_k_info_t *)(ksp->ks_private);
866 
867 	/* no writes currently allowed */
868 
869 	if (rw == KSTAT_WRITE) {
870 		return (EACCES);
871 	}
872 
873 	/* default to READ */
874 	info_stats->m_maxsets.value.ul = rdc_max_sets;
875 	info_stats->m_maxfbas.value.ul = krdc->maxfbas;
876 	info_stats->m_rpc_timeout.value.ul = rdc_rpc_tmout;
877 	info_stats->m_health_thres.value.ul = rdc_health_thres;
878 	info_stats->m_bitmap_writes.value.ul = krdc->bitmap_write;
879 	info_stats->m_bitmap_ref_delay.value.ul = rdc_bitmap_delay;
880 
881 	/* clts counters not implemented yet */
882 	info_stats->m_clnt_cots_calls.value.ul = rdc_clnt_count;
883 	info_stats->m_clnt_clts_calls.value.ul = 0;
884 	info_stats->m_svc_cots_calls.value.ul = rdc_svc_count;
885 	info_stats->m_svc_clts_calls.value.ul = 0;
886 
887 	return (0);
888 }
889 
890 /*
891  * copy tailsize-1 bytes of tail of s to s1.
892  */
893 void
894 rdc_str_tail_cpy(char *s1, char *s, size_t tailsize)
895 {
896 	/* To avoid un-terminated string, max size is 16 - 1 */
897 	ssize_t offset = strlen(s) - (tailsize - 1);
898 
899 	offset = (offset > 0) ? offset : 0;
900 
901 	/* ensure it's null terminated */
902 	(void) strlcpy(s1, (const char *)(s + offset), tailsize);
903 }
904 
905 int
906 rdc_info_stats_update(kstat_t *ksp, int rw)
907 {
908 	rdc_info_stats_t	*rdc_info_stats;
909 	rdc_k_info_t		*krdc;
910 	rdc_u_info_t		*urdc;
911 
912 	rdc_info_stats = (rdc_info_stats_t *)(ksp->ks_data);
913 	krdc = (rdc_k_info_t *)(ksp->ks_private);
914 	urdc = &rdc_u_info[krdc->index];
915 
916 	/* no writes currently allowed */
917 
918 	if (rw == KSTAT_WRITE) {
919 		return (EACCES);
920 	}
921 
922 	/* default to READ */
923 	rdc_info_stats->s_flags.value.ul = urdc->flags;
924 	rdc_info_stats->s_syncflags.value.ul =
925 	    urdc->sync_flags;
926 	rdc_info_stats->s_bmpflags.value.ul =
927 	    urdc->bmap_flags;
928 	rdc_info_stats->s_syncpos.value.ul =
929 	    urdc->sync_pos;
930 	rdc_info_stats->s_volsize.value.ul =
931 	    urdc->volume_size;
932 	rdc_info_stats->s_bits_set.value.ul =
933 	    urdc->bits_set;
934 	rdc_info_stats->s_autosync.value.ul =
935 	    urdc->autosync;
936 	rdc_info_stats->s_maxqfbas.value.ul =
937 	    urdc->maxqfbas;
938 	rdc_info_stats->s_maxqitems.value.ul =
939 	    urdc->maxqitems;
940 
941 	kstat_named_setstr(&rdc_info_stats->s_primary_vol,
942 	    urdc->primary.file);
943 
944 	kstat_named_setstr(&rdc_info_stats->s_secondary_vol,
945 	    urdc->secondary.file);
946 
947 	if (rdc_get_vflags(urdc) & RDC_PRIMARY) {
948 		kstat_named_setstr(&rdc_info_stats->s_bitmap,
949 		    urdc->primary.bitmap);
950 	} else {
951 		kstat_named_setstr(&rdc_info_stats->s_bitmap,
952 		    urdc->secondary.bitmap);
953 	}
954 
955 	kstat_named_setstr(&rdc_info_stats->s_primary_intf,
956 	    urdc->primary.intf);
957 
958 	kstat_named_setstr(&rdc_info_stats->s_secondary_intf,
959 	    urdc->secondary.intf);
960 
961 	rdc_info_stats->s_type_flag.value.ul = krdc->type_flag;
962 	rdc_info_stats->s_bitmap_size.value.ul = krdc->bitmap_size;
963 	rdc_info_stats->s_disk_status.value.ul = krdc->disk_status;
964 
965 	if (krdc->intf) {
966 		rdc_info_stats->s_if_if_down.value.ul = krdc->intf->if_down;
967 		rdc_info_stats->s_if_rpc_version.value.ul =
968 		    krdc->intf->rpc_version;
969 	}
970 
971 	/* the type can change without disable/re-enable so... */
972 	bzero(rdc_info_stats->s_aqueue_type.value.c, KSTAT_DATA_CHAR_LEN);
973 	if (RDC_IS_MEMQ(krdc->group)) {
974 		(void) strcpy(rdc_info_stats->s_aqueue_type.value.c, "memory");
975 		rdc_info_stats->s_aqueue_blk_hwm.value.ul =
976 		    krdc->group->ra_queue.blocks_hwm;
977 		rdc_info_stats->s_aqueue_itm_hwm.value.ul =
978 		    krdc->group->ra_queue.nitems_hwm;
979 		rdc_info_stats->s_aqueue_throttle.value.ul =
980 		    krdc->group->ra_queue.throttle_delay;
981 		rdc_info_stats->s_aqueue_items.value.ul =
982 		    krdc->group->ra_queue.nitems;
983 		rdc_info_stats->s_aqueue_blocks.value.ul =
984 		    krdc->group->ra_queue.blocks;
985 
986 	} else if (RDC_IS_DISKQ(krdc->group)) {
987 		disk_queue *q = &krdc->group->diskq;
988 		rdc_info_stats->s_aqueue_blk_hwm.value.ul =
989 		    krdc->group->diskq.blocks_hwm;
990 		rdc_info_stats->s_aqueue_itm_hwm.value.ul =
991 		    krdc->group->diskq.nitems_hwm;
992 		rdc_info_stats->s_aqueue_throttle.value.ul =
993 		    krdc->group->diskq.throttle_delay;
994 		rdc_info_stats->s_aqueue_items.value.ul = QNITEMS(q);
995 		rdc_info_stats->s_aqueue_blocks.value.ul = QBLOCKS(q);
996 		(void) strcpy(rdc_info_stats->s_aqueue_type.value.c, "disk");
997 	}
998 
999 	return (0);
1000 }
1001 
1002 void
1003 rdc_kstat_create(int index)
1004 {
1005 	int j = index;
1006 	rdc_k_info_t *krdc = &rdc_k_info[index];
1007 	rdc_u_info_t *urdc = &rdc_u_info[krdc->index];
1008 	size_t varsize;
1009 
1010 	if (!krdc->set_kstats) {
1011 		krdc->set_kstats = kstat_create(RDC_KSTAT_MODULE, j,
1012 		    RDC_KSTAT_INFO, RDC_KSTAT_CLASS, KSTAT_TYPE_NAMED,
1013 		    sizeof (rdc_info_stats_t) / sizeof (kstat_named_t),
1014 		    KSTAT_FLAG_VIRTUAL);
1015 #ifdef DEBUG
1016 		if (!krdc->set_kstats)
1017 			cmn_err(CE_NOTE, "!krdc:u_kstat null");
1018 #endif
1019 
1020 		if (krdc->set_kstats) {
1021 			/* calculate exact size of KSTAT_DATA_STRINGs */
1022 			varsize = strlen(urdc->primary.file) + 1
1023 			    + strlen(urdc->secondary.file) + 1
1024 			    + strlen(urdc->primary.intf) + 1
1025 			    + strlen(urdc->secondary.intf) + 1;
1026 			if (rdc_get_vflags(urdc) & RDC_PRIMARY) {
1027 				varsize += strlen(urdc->primary.bitmap) + 1;
1028 			} else {
1029 				varsize += strlen(urdc->secondary.bitmap) + 1;
1030 			}
1031 
1032 			krdc->set_kstats->ks_data_size += varsize;
1033 			krdc->set_kstats->ks_data = &rdc_info_stats;
1034 			krdc->set_kstats->ks_update = rdc_info_stats_update;
1035 			krdc->set_kstats->ks_private = &rdc_k_info[j];
1036 			kstat_install(krdc->set_kstats);
1037 		} else
1038 			cmn_err(CE_WARN, "!SNDR: k-kstats failed");
1039 	}
1040 
1041 	krdc->io_kstats = kstat_create(RDC_KSTAT_MODULE, j, NULL,
1042 	    "disk", KSTAT_TYPE_IO, 1, 0);
1043 	if (krdc->io_kstats) {
1044 		krdc->io_kstats->ks_lock = &krdc->kstat_mutex;
1045 		kstat_install(krdc->io_kstats);
1046 	}
1047 	krdc->bmp_kstats = kstat_create("sndrbmp", j, NULL,
1048 	    "disk", KSTAT_TYPE_IO, 1, 0);
1049 	if (krdc->bmp_kstats) {
1050 		krdc->bmp_kstats->ks_lock = &krdc->bmp_kstat_mutex;
1051 		kstat_install(krdc->bmp_kstats);
1052 	}
1053 }
1054 
1055 void
1056 rdc_kstat_delete(int index)
1057 {
1058 	rdc_k_info_t *krdc = &rdc_k_info[index];
1059 
1060 	if (krdc->set_kstats) {
1061 		kstat_delete(krdc->set_kstats);
1062 		krdc->set_kstats = NULL;
1063 	}
1064 
1065 	if (krdc->io_kstats) {
1066 		kstat_delete(krdc->io_kstats);
1067 		krdc->io_kstats = NULL;
1068 	}
1069 	if (krdc->bmp_kstats) {
1070 		kstat_delete(krdc->bmp_kstats);
1071 		krdc->bmp_kstats = NULL;
1072 	}
1073 }
1074 
1075 #ifdef	DEBUG
1076 /*
1077  * Reset the io_kstat structure of the krdc specified
1078  * by the arg index.
1079  */
1080 static int
1081 rdc_clrkstat(void *arg)
1082 {
1083 	int index;
1084 	rdc_k_info_t *krdc;
1085 
1086 	index = (int)(unsigned long)arg;
1087 	if ((index < 0) || (index >= rdc_max_sets)) {
1088 		return (EINVAL);
1089 	}
1090 	krdc = &rdc_k_info[index];
1091 	if (krdc->io_kstats) {
1092 		kstat_delete(krdc->io_kstats);
1093 		krdc->io_kstats = NULL;
1094 	} else {
1095 		return (EINVAL);
1096 	}
1097 	krdc->io_kstats = kstat_create(RDC_KSTAT_MODULE, index, NULL,
1098 	    "disk", KSTAT_TYPE_IO, 1, 0);
1099 	if (krdc->io_kstats) {
1100 		krdc->io_kstats->ks_lock = &krdc->kstat_mutex;
1101 		kstat_install(krdc->io_kstats);
1102 	} else {
1103 		return (EINVAL);
1104 	}
1105 	/*
1106 	 * clear the high water marks and throttle.
1107 	 */
1108 	if (krdc->group) {
1109 		krdc->group->ra_queue.nitems_hwm = 0;
1110 		krdc->group->ra_queue.blocks_hwm = 0;
1111 		krdc->group->ra_queue.throttle_delay = 0;
1112 	}
1113 	return (0);
1114 }
1115 #endif
1116