xref: /titanic_41/usr/src/uts/sun4u/starfire/io/idn.c (revision 489b7c4ab76ae8df137fbfcc2214f7baa52883a0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 
27 #include <sys/types.h>
28 #include <sys/sysmacros.h>
29 #include <sys/open.h>
30 #include <sys/param.h>
31 #include <sys/machparam.h>
32 #include <sys/systm.h>
33 #include <sys/signal.h>
34 #include <sys/cred.h>
35 #include <sys/user.h>
36 #include <sys/proc.h>
37 #include <sys/vnode.h>
38 #include <sys/uio.h>
39 #include <sys/buf.h>
40 #include <sys/file.h>
41 #include <sys/kmem.h>
42 #include <sys/stat.h>
43 #include <sys/stream.h>
44 #include <sys/stropts.h>
45 #include <sys/strsubr.h>
46 #include <sys/strsun.h>
47 #include <inet/common.h>
48 #include <inet/mi.h>
49 #include <inet/nd.h>
50 #include <sys/poll.h>
51 #include <sys/utsname.h>
52 #include <sys/debug.h>
53 #include <sys/conf.h>
54 #include <sys/ddi.h>
55 #include <sys/sunddi.h>
56 #include <sys/errno.h>
57 #include <sys/modctl.h>
58 #include <sys/machsystm.h>
59 #include <sys/promif.h>
60 #include <sys/prom_plat.h>
61 #include <sys/obpdefs.h>
62 #include <vm/seg_kmem.h>
63 #include <vm/seg_kp.h>
64 #include <sys/kstat.h>
65 #include <sys/membar.h>
66 #include <sys/ivintr.h>
67 #include <sys/vm_machparam.h>
68 #include <sys/x_call.h>
69 #include <sys/cpuvar.h>
70 #include <sys/archsystm.h>
71 #include <sys/dmv.h>
72 
73 #include <sys/idn.h>
74 #include <sys/idn_xf.h>
75 #include <sys/cpu_sgnblk_defs.h>
76 #include <sys/cpu_sgn.h>
77 
78 struct idn_gkstat	sg_kstat;
79 
80 #define	MBXTBL_PART_REPORT	((caddr_t)1)
81 #define	MBXTBL_FULL_REPORT	((caddr_t)2)
82 
83 idn_domain_t	idn_domain[MAX_DOMAINS];
84 idn_global_t	idn;
85 int		idn_debug;
86 int		idn_snoop;
87 int		idn_history;
88 
89 typedef enum {
90 	IDN_GPROPS_OKAY,
91 	IDN_GPROPS_UNCHECKED,
92 	IDN_GPROPS_ERROR
93 } idn_gprops_t;
94 
95 struct idn_history	idnhlog;
96 
97 /*
98  * IDN "tunables".
99  */
100 int		idn_smr_size;
101 int		idn_nwr_size;
102 int		idn_lowat;
103 int		idn_hiwat;
104 int		idn_protocol_nservers;
105 int		idn_awolmsg_interval;
106 int		idn_smr_bufsize;
107 int		idn_slab_bufcount;
108 int		idn_slab_prealloc;
109 int		idn_slab_maxperdomain;
110 int		idn_slab_mintotal;
111 int		idn_window_max;
112 int		idn_window_incr;
113 int		idn_window_emax;
114 int		idn_reclaim_min;
115 int		idn_reclaim_max;
116 int		idn_mbox_per_net;
117 int		idn_max_nets;
118 
119 int		idn_netsvr_spin_count;
120 int		idn_netsvr_wait_min;
121 int		idn_netsvr_wait_max;
122 int		idn_netsvr_wait_shift;
123 
124 int		idn_checksum;
125 
126 int		idn_msgwait_nego;
127 int		idn_msgwait_cfg;
128 int		idn_msgwait_con;
129 int		idn_msgwait_fin;
130 int		idn_msgwait_cmd;
131 int		idn_msgwait_data;
132 
133 int		idn_retryfreq_nego;
134 int		idn_retryfreq_con;
135 int		idn_retryfreq_fin;
136 
137 int		idn_window_emax;	/* calculated */
138 int		idn_slab_maxperdomain;	/* calculated */
139 
140 /*
141  * DMV interrupt support.
142  */
143 int		idn_pil;
144 int		idn_dmv_pending_max;
145 idn_dmv_msg_t	*idn_iv_queue[NCPU];
146 int		idn_intr_index[NCPU];	/* idn_handler ONLY */
147 static idn_dmv_data_t	*idn_dmv_data;
148 
149 int		idn_sigbpil;
150 
151 idnparam_t	idn_param_arr[] = {
152 { 0,		1,		0,		/* 0 */ "idn_modunloadable" },
153 };
154 
155 /*
156  * Parameters that are only accessible in a DEBUG driver.
157  */
158 static char *idn_param_debug_only[] = {
159 #if 0
160 	"idn_checksum",
161 #endif /* 0 */
162 	0
163 };
164 
165 /*
166  * Parameters that are READ-ONLY.
167  */
168 static char *idn_param_read_only[] = {
169 #if 0
170 	"idn_window_emax",
171 	"idn_slab_maxperdomain",
172 #endif /* 0 */
173 	0
174 };
175 
176 static struct idn_global_props {
177 	int		p_min, p_max, p_def;
178 	char		*p_string;
179 	int		*p_var;
180 } idn_global_props[] = {
181 { 0,	0,	0,	"idn_debug",		&idn_debug		},
182 { 0,	1,	0,	"idn_history",		&idn_history		},
183 { 0,	IDN_SMR_MAXSIZE,
184 		0,	"idn_smr_size",		&idn_smr_size		},
185 { 0,	IDN_SMR_MAXSIZE,
186 		0,	"idn_nwr_size",		&idn_nwr_size		},
187 { 1,	512*1024,
188 		1,	"idn_lowat",		&idn_lowat		},
189 { 1*1024,
190 	1*1024*1024,
191 		256*1024,
192 			"idn_hiwat",		&idn_hiwat		},
193 { IDN_SMR_BUFSIZE_MIN,
194 	IDN_SMR_BUFSIZE_MAX,
195 		IDN_SMR_BUFSIZE_DEF,
196 			"idn_smr_bufsize",	&idn_smr_bufsize	},
197 { 4,	1024,	32,	"idn_slab_bufcount",	&idn_slab_bufcount	},
198 { 0,	10,	0,	"idn_slab_prealloc",	&idn_slab_prealloc	},
199 { 2,	MAX_DOMAINS,
200 		8,	"idn_slab_mintotal",	&idn_slab_mintotal	},
201 { 8,	256,	64,	"idn_window_max",	&idn_window_max		},
202 { 0,	32,	8,	"idn_window_incr",	&idn_window_incr	},
203 { 1,	128,	5,	"idn_reclaim_min",	&idn_reclaim_min	},
204 { 0,	128,	0,	"idn_reclaim_max",	&idn_reclaim_max	},
205 { 1,	IDN_MAXMAX_NETS,
206 		8,	"idn_max_nets",		&idn_max_nets		},
207 { 31,	511,	127,	"idn_mbox_per_net",	&idn_mbox_per_net	},
208 { 0,	1,	1,	"idn_checksum",		&idn_checksum		},
209 { 0,	10000,	500,	"idn_netsvr_spin_count",
210 						&idn_netsvr_spin_count	},
211 { 0,	30*100,	40,	"idn_netsvr_wait_min",	&idn_netsvr_wait_min	},
212 { 0,	60*100,	16*100,	"idn_netsvr_wait_max",	&idn_netsvr_wait_max	},
213 { 1,	5,	1,	"idn_netsvr_wait_shift",
214 						&idn_netsvr_wait_shift	},
215 { 1,	MAX_DOMAINS,
216 		IDN_PROTOCOL_NSERVERS,
217 			"idn_protocol_nservers",
218 						&idn_protocol_nservers	},
219 { 0,	3600,	IDN_AWOLMSG_INTERVAL,
220 			"idn_awolmsg_interval",	&idn_awolmsg_interval	},
221 { 10,	300,	IDN_MSGWAIT_NEGO,
222 			"idn_msgwait_nego",	&idn_msgwait_nego	},
223 { 10,	300,	IDN_MSGWAIT_CFG,
224 			"idn_msgwait_cfg",	&idn_msgwait_cfg	},
225 { 10,	300,	IDN_MSGWAIT_CON,
226 			"idn_msgwait_con",	&idn_msgwait_con	},
227 { 10,	300,	IDN_MSGWAIT_FIN,
228 			"idn_msgwait_fin",	&idn_msgwait_fin	},
229 { 10,	300,	IDN_MSGWAIT_CMD,
230 			"idn_msgwait_cmd",	&idn_msgwait_cmd	},
231 { 10,	300,	IDN_MSGWAIT_DATA,
232 			"idn_msgwait_data",	&idn_msgwait_data	},
233 { 1,	60,	IDN_RETRYFREQ_NEGO,
234 			"idn_retryfreq_nego",	&idn_retryfreq_nego	},
235 { 1,	60,	IDN_RETRYFREQ_CON,
236 			"idn_retryfreq_con",	&idn_retryfreq_con	},
237 { 1,	60,	IDN_RETRYFREQ_FIN,
238 			"idn_retryfreq_fin",	&idn_retryfreq_fin	},
239 { 1,	9,	IDN_PIL,
240 			"idn_pil",		&idn_pil		},
241 { 1,	9,	IDN_SIGBPIL,
242 			"idn_sigbpil",		&idn_sigbpil		},
243 { 8,	512,	IDN_DMV_PENDING_MAX,
244 			"idn_dmv_pending_max",	&idn_dmv_pending_max	},
245 { 0,	0,	0,	NULL,			NULL			}
246 };
247 
248 struct idn	*idn_i2s_table[IDN_MAXMAX_NETS << 1];
249 clock_t		idn_msg_waittime[IDN_NUM_MSGTYPES];
250 clock_t		idn_msg_retrytime[(int)IDN_NUM_RETRYTYPES];
251 
252 static caddr_t	idn_ndlist;	/* head of 'named dispatch' var list */
253 
254 static int	idnattach(dev_info_t *, ddi_attach_cmd_t);
255 static int	idndetach(dev_info_t *, ddi_detach_cmd_t);
256 static int	idnopen(register queue_t *, dev_t *, int, int, cred_t *);
257 static int	idnclose(queue_t *, int, cred_t *);
258 static int	idnwput(queue_t *, mblk_t *);
259 static int	idnwsrv(queue_t *);
260 static int	idnrput(queue_t *, mblk_t *);
261 static void	idnioctl(queue_t *, mblk_t *);
262 static idn_gprops_t	idn_check_conf(dev_info_t *dip, processorid_t *cpuid);
263 static int	idn_size_check();
264 static void	idn_xmit_monitor_init();
265 static void	idn_xmit_monitor_deinit();
266 static void	idn_init_msg_waittime();
267 static void	idn_init_msg_retrytime();
268 static void	idn_sigb_setup(cpu_sgnblk_t *sigbp, void *arg);
269 static int	idn_init(dev_info_t *dip);
270 static int	idn_deinit();
271 static void	idn_sigbhandler_create();
272 static void	idn_sigbhandler_kill();
273 static uint_t	idn_sigbhandler_wakeup(caddr_t arg1, caddr_t arg2);
274 static void	idn_sigbhandler_thread(struct sigbintr **sbpp);
275 static void	idn_sigbhandler(processorid_t cpuid, cpu_sgnblk_t *sgnblkp);
276 static int	idn_info(idnsb_info_t *sfp);
277 static int	idn_init_smr();
278 static void	idn_deinit_smr();
279 static int	idn_prom_getsmr(uint_t *smrsz, uint64_t *paddrp,
280 				uint64_t *sizep);
281 static int	idn_init_handler();
282 static void	idn_deinit_handler();
283 static uint_t	idn_handler(caddr_t unused, caddr_t unused2);
284 /*
285  * ioctl services
286  */
287 static int	idnioc_link(idnop_t *idnop);
288 static int	idnioc_unlink(idnop_t *idnop);
289 static int	idn_rw_mem(idnop_t *idnop);
290 static int	idn_send_ping(idnop_t *idnop);
291 
292 static void 	idn_domains_init(struct hwconfig *local_hw);
293 static void 	idn_domains_deinit();
294 static void	idn_retrytask_init();
295 static void	idn_retrytask_deinit();
296 static void	idn_gkstat_init();
297 static void	idn_gkstat_deinit();
298 static int	idn_gkstat_update();
299 static void	idn_timercache_init();
300 static void	idn_timercache_deinit();
301 static void	idn_dopers_init();
302 static void	idn_dopers_deinit();
303 
304 static void	idn_param_cleanup();
305 static int	idn_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr);
306 static int	idn_param_set(queue_t *q, mblk_t *mp, char *value, caddr_t cp,
307 					cred_t *cr);
308 static int	idn_param_register(register idnparam_t *idnpa, int count);
309 static int	idn_slabpool_report(queue_t *wq, mblk_t *mp, caddr_t cp,
310 					cred_t *cr);
311 static int	idn_buffer_report(queue_t *wq, mblk_t *mp, caddr_t cp,
312 					cred_t *cr);
313 static int	idn_mboxtbl_report(queue_t *wq, mblk_t *mp, caddr_t cp,
314 					cred_t *cr);
315 static int	idn_mainmbox_report(queue_t *wq, mblk_t *mp, caddr_t cp,
316 					cred_t *cr);
317 static void	idn_mainmbox_domain_report(queue_t *wq, mblk_t *mp, int domid,
318 					idn_mainmbox_t *mmp, char *mbxtype);
319 static int	idn_global_report(queue_t *wq, mblk_t *mp, caddr_t cp,
320 					cred_t *cr);
321 static int	idn_domain_report(queue_t *wq, mblk_t *mp, caddr_t cp,
322 					cred_t *cr);
323 static int	idn_get_net_binding(queue_t *wq, mblk_t *mp, caddr_t cp,
324 					cred_t *cr);
325 static int	idn_set_net_binding(queue_t *wq, mblk_t *mp, char *value,
326 					caddr_t cp, cred_t *cr);
327 
328 /*
329  * String definitions used for DEBUG and non-DEBUG.
330  */
331 const char	*idnm_str[] = {
332 /*  0 */	"null",
333 /*  1 */	"nego",
334 /*  2 */	"con",
335 /*  3 */	"cfg",
336 /*  4 */	"fin",
337 /*  5 */	"cmd",
338 /*  6 */	"data",
339 };
340 
341 const char	*idnds_str[] = {
342 /*  0 */	"CLOSED",
343 /*  1 */	"NEGO_PEND",
344 /*  2 */	"NEGO_SENT",
345 /*  3 */	"NEGO_RCVD",
346 /*  4 */	"CONFIG",
347 /*  5 */	"CON_PEND",
348 /*  6 */	"CON_SENT",
349 /*  7 */	"CON_RCVD",
350 /*  8 */	"CON_READY",
351 /*  9 */	"CONNECTED",
352 /* 10 */	"FIN_PEND",
353 /* 11 */	"FIN_SENT",
354 /* 12 */	"FIN_RCVD",
355 /* 13 */	"DMAP"
356 };
357 
358 const char	*idnxs_str[] = {
359 /* 0 */		"PEND",
360 /* 1 */		"SENT",
361 /* 2 */		"RCVD",
362 /* 3 */		"FINAL",
363 /* 4 */		"NIL"
364 };
365 
366 const char	*idngs_str[] = {
367 /*  0 */	"OFFLINE",
368 /*  1 */	"CONNECT",
369 /*  2 */	"ONLINE",
370 /*  3 */	"DISCONNECT",
371 /*  4 */	"RECONFIG",
372 /*  5 */	"unknown",
373 /*  6 */	"unknown",
374 /*  7 */	"unknown",
375 /*  8 */	"unknown",
376 /*  9 */	"unknown",
377 /* 10 */	"IGNORE"
378 };
379 
380 const char	*idncmd_str[] = {
381 /*  0 */	"unknown",
382 /*  1 */	"SLABALLOC",
383 /*  2 */	"SLABFREE",
384 /*  3 */	"SLABREAP",
385 /*  4 */	"NODENAME"
386 };
387 
388 const char	*idncon_str[] = {
389 /*  0 */	"OFF",
390 /*  1 */	"NORMAL",
391 /*  2 */	"QUERY"
392 };
393 
394 const char	*idnfin_str[] = {
395 /*  0 */	"OFF",
396 /*  1 */	"NORMAL",
397 /*  2 */	"FORCE_SOFT",
398 /*  3 */	"FORCE_HARD",
399 /*  4 */	"QUERY"
400 };
401 
402 const char	*idnfinopt_str[] = {
403 /*  0 */	"NONE",
404 /*  1 */	"UNLINK",
405 /*  2 */	"RELINK"
406 };
407 
408 const char	*idnfinarg_str[] = {
409 /*  0 */	"NONE",
410 /*  1 */	"SMRBAD",
411 /*  2 */	"CPUCFG",
412 /*  3 */	"HWERR",
413 /*  4 */	"CFGERR_FATAL",
414 /*  5 */	"CFGERR_MTU",
415 /*  6 */	"CFGERR_BUF",
416 /*  7 */	"CFGERR_SLAB",
417 /*  8 */	"CFGERR_NWR",
418 /*  9 */	"CFGERR_NETS",
419 /* 10 */	"CFGERR_MBOX",
420 /* 11 */	"CFGERR_NMCADR",
421 /* 12 */	"CFGERR_MCADR",
422 /* 13 */	"CFGERR_CKSUM",
423 /* 14 */	"CFGERR_SMR",
424 };
425 
426 const char	*idnsync_str[] = {
427 /*  0 */	"NIL",
428 /*  1 */	"CONNECT",
429 /*  2 */	"DISCONNECT"
430 };
431 
432 const char	*idnreg_str[] = {
433 /*  0 */	"REG",
434 /*  1 */	"NEW",
435 /*  2 */	"QUERY"
436 };
437 
438 const char	*idnnack_str[] = {
439 /*  0 */	"unknown",
440 /*  1 */	"NOCONN",
441 /*  2 */	"BADCHAN",
442 /*  3 */	"BADCFG",
443 /*  4 */	"BADCMD",
444 /*  5 */	"RETRY",
445 /*  6 */	"DUP",
446 /*  7 */	"EXIT",
447 /*  8 */	"--reserved1",
448 /*  9 */	"--reserved2",
449 /* 10 */	"--reserved3"
450 };
451 
452 const char	*idnop_str[] = {
453 /*  0 */	"DISCONNECTED",
454 /*  1 */	"CONNECTED",
455 /*  2 */	"ERROR"
456 };
457 
458 const char	*chanop_str[] = {
459 /*  0 */	"OPEN",
460 /*  1 */	"SOFT_CLOSE",
461 /*  2 */	"HARD_CLOSE",
462 /*  3 */	"OFFLINE",
463 /*  4 */	"ONLINE"
464 };
465 
466 const char	*chanaction_str[] = {
467 /*  0 */	"DETACH",
468 /*  1 */	"STOP",
469 /*  2 */	"SUSPEND",
470 /*  3 */	"RESUME",
471 /*  4 */	"RESTART",
472 /*  5 */	"ATTACH"
473 };
474 
475 const char	*timer_str[] = {
476 /* 0 */		"NIL",
477 /* 1 */		"MSG"
478 };
479 
480 static struct module_info idnrinfo = {
481 	IDNIDNUM,		/* mi_idnum */
482 	IDNNAME,		/* mi_idname */
483 	IDNMINPSZ,		/* mi_minpsz */
484 	IDNMAXPSZ,		/* mi_maxpsz */
485 	0,			/* mi_hiwat - see IDN_HIWAT */
486 	0			/* mi_lowat - see IDN_LOWAT */
487 };
488 
489 static struct module_info idnwinfo = {
490 	IDNIDNUM,		/* mi_idnum */
491 	IDNNAME,		/* mi_idname */
492 	IDNMINPSZ,		/* mi_minpsz */
493 	IDNMAXPSZ,		/* mi_maxpsz */
494 	0,			/* mi_hiwat - see IDN_HIWAT */
495 	0			/* mi_lowat - see IDN_LOWAT */
496 };
497 
498 static struct qinit idnrinit = {
499 	idnrput,		/* qi_putp */
500 	NULL,			/* qi_srvp */
501 	idnopen,		/* qi_qopen */
502 	idnclose,		/* qi_qclose */
503 	NULL,			/* qi_qadmin */
504 	&idnrinfo,		/* qi_minfo */
505 	NULL,			/* qi_mstat */
506 	NULL,			/* qi_rwp */
507 	NULL,			/* qi_infop */
508 	STRUIOT_DONTCARE	/* qi_struiot */
509 };
510 
511 static struct qinit idnwinit = {
512 	idnwput,		/* qi_putp */
513 	idnwsrv,		/* qi_srvp */
514 	NULL,			/* qi_qopen */
515 	NULL,			/* qi_qclose */
516 	NULL,			/* qi_qadmin */
517 	&idnwinfo,		/* qi_minfo */
518 	NULL,			/* qi_mstat */
519 	NULL,			/* qi_rwp */
520 	NULL,			/* qi_infop */
521 	STRUIOT_DONTCARE	/* qi_struiot */
522 };
523 
524 struct streamtab idninfo = {
525 	&idnrinit,		/* st_rdinit */
526 	&idnwinit,		/* st_wrinit */
527 	NULL,			/* st_muxrinit */
528 	NULL,			/* st_muxwinit */
529 };
530 
531 /*
532  * Module linkage information (cb_ops & dev_ops) for the kernel.
533  */
534 
535 static struct cb_ops cb_idnops = {
536 	nulldev,		/* cb_open */
537 	nulldev,		/* cb_close */
538 	nodev,			/* cb_strategy */
539 	nodev,			/* cb_print */
540 	nodev,			/* cb_dump */
541 	nodev,			/* cb_read */
542 	nodev,			/* cb_write */
543 	nodev,			/* cb_ioctl */
544 	nodev,			/* cb_devmap */
545 	nodev,			/* cb_mmap */
546 	nodev,			/* cb_segmap */
547 	nochpoll,		/* cb_chpoll */
548 	ddi_prop_op,		/* cb_prop_op */
549 	&idninfo,		/* cb_stream */
550 	D_MP,			/* cb_flag */
551 	CB_REV,			/* cb_rev */
552 	nodev,			/* cb_aread */
553 	nodev,			/* cb_awrite */
554 };
555 
556 static struct dev_ops idnops = {
557 	DEVO_REV,		/* devo_rev */
558 	0,			/* devo_refcnt */
559 	ddi_no_info,		/* devo_getinfo */
560 	nulldev,		/* devo_identify */
561 	nulldev,		/* devo_probe */
562 	idnattach,		/* devo_attach */
563 	idndetach,		/* devo_detach */
564 	nodev,			/* devo_reset */
565 	&cb_idnops,		/* devo_cb_ops */
566 	(struct bus_ops *)NULL,	/* devo_bus_ops */
567 	NULL,			/* devo_power */
568 	ddi_quiesce_not_needed,		/* quiesce */
569 };
570 
571 extern cpuset_t	cpu_ready_set;
572 
573 static struct modldrv modldrv = {
574 	&mod_driverops,		/* This module is a pseudo driver */
575 	IDNDESC " 1.58",
576 	&idnops
577 };
578 
579 static struct modlinkage modlinkage = {
580 	MODREV_1,
581 	&modldrv,
582 	NULL
583 };
584 
585 /*
586  * --------------------------------------------------
587  */
588 int
589 _init(void)
590 {
591 	idn.version = IDN_VERSION;
592 
593 	return (mod_install(&modlinkage));
594 }
595 
596 int
597 _fini(void)
598 {
599 	return (mod_remove(&modlinkage));
600 }
601 
602 int
603 _info(struct modinfo *modinfop)
604 {
605 	return (mod_info(&modlinkage, modinfop));
606 }
607 
608 /*
609  * ----------------------------------------------
610  */
611 static int
612 idnattach(dev_info_t *dip, ddi_attach_cmd_t cmd)
613 {
614 	int		instance;
615 	int		doinit = 0;
616 	processorid_t	bcpuid;
617 	struct idn	*sip;
618 	struct idnstr	*stp;
619 	procname_t	proc = "idnattach";
620 
621 
622 #ifndef	lint
623 	ASSERT(sizeof (idnsb_t) == IDNSB_SIZE);
624 	ASSERT((uint_t)&((struct idnsb *)0)->id_hwchkpt[0] == 0x40);
625 #endif	/* lint */
626 
627 	switch (cmd) {
628 	case DDI_RESUME:
629 		sip = ddi_get_driver_private(dip);
630 		/*
631 		 * sip may have not yet been set if the
632 		 * OBP environment variable (idn-smr-size)
633 		 * was not set.
634 		 */
635 		if (sip == NULL)
636 			return (DDI_FAILURE);
637 		/*
638 		 * RESUME IDN services.
639 		 */
640 		IDN_GLOCK_SHARED();
641 		if (idn.state != IDNGS_OFFLINE) {
642 			cmn_err(CE_WARN,
643 			    "IDN: 101: not in expected OFFLINE state "
644 			    "for DDI_RESUME");
645 			ASSERT(0);
646 		}
647 		IDN_GUNLOCK();
648 
649 		/*
650 		 * RESUME DLPI services.
651 		 */
652 		sip->si_flags &= ~IDNSUSPENDED;
653 
654 		rw_enter(&idn.struprwlock, RW_READER);
655 		for (stp = idn.strup; stp; stp = stp->ss_nextp)
656 			if (stp->ss_sip == sip) {
657 				doinit = 1;
658 				break;
659 			}
660 		rw_exit(&idn.struprwlock);
661 		if (doinit)
662 			(void) idndl_init(sip);
663 
664 		return (DDI_SUCCESS);
665 
666 	case DDI_ATTACH:
667 		break;
668 
669 	default:
670 		return (DDI_FAILURE);
671 	}
672 
673 	instance = ddi_get_instance(dip);
674 
675 	PR_DRV("%s: instance = %d\n", proc, instance);
676 
677 	if (idn_check_conf(dip, &bcpuid) == IDN_GPROPS_ERROR)
678 		return (DDI_FAILURE);
679 
680 	mutex_enter(&idn.siplock);
681 
682 	if (ddi_create_minor_node(dip, IDNNAME, S_IFCHR, instance,
683 	    DDI_NT_NET, CLONE_DEV) == DDI_FAILURE) {
684 		mutex_exit(&idn.siplock);
685 		return (DDI_FAILURE);
686 	}
687 
688 	if (idn.smr.ready == 0) {
689 		if (idn_init_smr() == 0) {
690 			idn.enabled = 1;
691 #ifdef DEBUG
692 			cmn_err(CE_NOTE, "!IDN: Driver enabled");
693 #endif /* DEBUG */
694 		} else {
695 			cmn_err(CE_NOTE,
696 			    "!IDN: 102: driver disabled "
697 			    "- check OBP environment "
698 			    "(idn-smr-size)");
699 			mutex_exit(&idn.siplock);
700 			return (DDI_SUCCESS);
701 		}
702 	}
703 
704 	ASSERT(idn.smr.ready || idn.enabled);
705 
706 	if (idn.dip == NULL) {
707 		doinit = 1;
708 
709 		if (idn_size_check()) {
710 			idn_deinit_smr();
711 			ddi_remove_minor_node(dip, NULL);
712 			mutex_exit(&idn.siplock);
713 			return (DDI_FAILURE);
714 		}
715 
716 		if (idn_init(dip)) {
717 			idn_deinit_smr();
718 			ddi_remove_minor_node(dip, NULL);
719 			mutex_exit(&idn.siplock);
720 			return (DDI_FAILURE);
721 		}
722 	}
723 
724 	ASSERT(idn.dip);
725 
726 	/*
727 	 * This must occur _after_ idn_init() since
728 	 * it assumes idn_chanservers_init() has been
729 	 * called.
730 	 */
731 	idn_chanserver_bind(ddi_get_instance(dip), bcpuid);
732 
733 	/*
734 	 * DLPI supporting stuff.
735 	 */
736 	sip = GETSTRUCT(struct idn, 1);
737 	sip->si_dip = dip;
738 	ddi_set_driver_private(dip, sip);
739 	sip->si_nextp = idn.sip;
740 	idn.sip = sip;
741 	IDN_SET_INST2SIP(instance, sip);
742 	mutex_exit(&idn.siplock);
743 
744 	if (doinit)
745 		idndl_dlpi_init();	/* initializes idninfoack */
746 	/*
747 	 * Get our local IDN ethernet address.
748 	 */
749 	idndl_localetheraddr(sip, &sip->si_ouraddr);
750 	idndl_statinit(sip);
751 
752 	if (doinit) {
753 		idn_gkstat_init();
754 		/*
755 		 * Add our sigblock SSP interrupt handler.
756 		 */
757 		mutex_enter(&idn.sigbintr.sb_mutex);
758 		idn_sigbhandler_create();
759 		mutex_exit(&idn.sigbintr.sb_mutex);
760 
761 		if (sgnblk_poll_register(idn_sigbhandler) == 0) {
762 			mutex_enter(&idn.sigbintr.sb_mutex);
763 			idn_sigbhandler_kill();
764 			idn.sigbintr.sb_cpuid = (uchar_t)-1;
765 			idn.sigbintr.sb_busy = IDNSIGB_INACTIVE;
766 			mutex_exit(&idn.sigbintr.sb_mutex);
767 
768 			idn_gkstat_deinit();
769 
770 			mutex_enter(&idn.siplock);
771 			(void) idn_deinit();
772 			IDN_SET_INST2SIP(instance, NULL);
773 			idn.sip = sip->si_nextp;
774 			mutex_exit(&idn.siplock);
775 
776 			ddi_remove_minor_node(dip, NULL);
777 
778 			return (DDI_FAILURE);
779 		}
780 		/*
781 		 * We require sigblkp[cpu0] to be mapped for hardware
782 		 * configuration determination and also auto-linking
783 		 * on bootup.
784 		 */
785 		if (sgnblk_poll_reference(idn_sigb_setup, NULL) != 0) {
786 			sgnblk_poll_unregister(idn_sigbhandler);
787 			mutex_enter(&idn.sigbintr.sb_mutex);
788 			idn_sigbhandler_kill();
789 			idn.sigbintr.sb_cpuid = (uchar_t)-1;
790 			idn.sigbintr.sb_busy = IDNSIGB_INACTIVE;
791 			mutex_exit(&idn.sigbintr.sb_mutex);
792 
793 			idn_gkstat_deinit();
794 
795 			mutex_enter(&idn.siplock);
796 			(void) idn_deinit();
797 			IDN_SET_INST2SIP(instance, NULL);
798 			idn.sip = sip->si_nextp;
799 			mutex_exit(&idn.siplock);
800 
801 			ddi_remove_minor_node(dip, NULL);
802 
803 			cmn_err(CE_WARN,
804 			    "IDN: 103: unable to reference sigblock area");
805 
806 			return (DDI_FAILURE);
807 		}
808 
809 		idn_init_autolink();
810 	}
811 
812 	ddi_report_dev(dip);
813 
814 	return (DDI_SUCCESS);
815 }
816 
817 /*
818  * ----------------------------------------------
819  */
820 static int
821 idndetach(dev_info_t *dip, ddi_detach_cmd_t cmd)
822 {
823 	int		err = 0;
824 	int		instance;
825 	struct idn	*sip, *hsip, *tsip;
826 	procname_t	proc = "idndetach";
827 
828 	sip = ddi_get_driver_private(dip);
829 	instance = ddi_get_instance(dip);
830 
831 	switch (cmd) {
832 	case DDI_SUSPEND:
833 		if (sip == NULL)
834 			return (DDI_FAILURE);
835 		/*
836 		 * SUSPEND IDN services.
837 		 * - Actually don't suspend anything, we just
838 		 *   make sure we're not connected per DR protocol.
839 		 *   If we really wanted to suspend it should
840 		 *   be done _after_ DLPI is suspended so that
841 		 *   we're not competing with that traffic.
842 		 */
843 		IDN_GLOCK_SHARED();
844 
845 		if (idn.state != IDNGS_OFFLINE) {
846 			int	d;
847 
848 			cmn_err(CE_WARN,
849 			    "IDN: 104: cannot suspend while active "
850 			    "(state = %s)",
851 			    idngs_str[idn.state]);
852 
853 			for (d = 0; d < MAX_DOMAINS; d++) {
854 				idn_domain_t	*dp;
855 
856 				dp = &idn_domain[d];
857 				if (dp->dcpu < 0)
858 					continue;
859 
860 				cmn_err(CE_CONT,
861 				    "IDN: 121: domain %d (CPU %d, name "
862 				    "\"%s\", state %s)\n",
863 				    d, dp->dcpu, dp->dname,
864 				    idnds_str[dp->dstate]);
865 			}
866 			err = 1;
867 		}
868 
869 		IDN_GUNLOCK();
870 
871 		if (err)
872 			return (DDI_FAILURE);
873 		/*
874 		 * SUSPEND DLPI services.
875 		 */
876 		sip->si_flags |= IDNSUSPENDED;
877 
878 		idndl_uninit(sip);
879 
880 		return (DDI_FAILURE);
881 
882 	case DDI_DETACH:
883 		if (idn.enabled == 0) {
884 			ddi_remove_minor_node(dip, NULL);
885 			ASSERT(idn.dip == NULL);
886 			return (DDI_SUCCESS);
887 		}
888 		if (!IDN_MODUNLOADABLE)
889 			return (DDI_FAILURE);
890 		break;
891 
892 	default:
893 		return (DDI_FAILURE);
894 	}
895 
896 	PR_DRV("%s: instance = %d\n", proc, instance);
897 
898 	if (sip == NULL) {
899 		/*
900 		 * No resources allocated.
901 		 */
902 		return (DDI_SUCCESS);
903 	}
904 
905 	mutex_enter(&idn.siplock);
906 	if (idn.sip && (idn.sip->si_nextp == NULL)) {
907 		/*
908 		 * This is our last stream connection
909 		 * going away.  Time to deinit and flag
910 		 * the SSP we're (IDN) DOWN.
911 		 */
912 		if (idn_deinit()) {
913 			/*
914 			 * Must still be active.
915 			 */
916 			mutex_exit(&idn.siplock);
917 			return (DDI_FAILURE);
918 		}
919 		idn_deinit_autolink();
920 		/*
921 		 * Remove our sigblock SSP interrupt handler.
922 		 */
923 		sgnblk_poll_unregister(idn_sigbhandler);
924 		mutex_enter(&idn.sigbintr.sb_mutex);
925 		idn_sigbhandler_kill();
926 		idn.sigbintr.sb_cpuid = (uchar_t)-1;
927 		idn.sigbintr.sb_busy = IDNSIGB_NOTREADY;
928 		mutex_exit(&idn.sigbintr.sb_mutex);
929 		/*
930 		 * Remove our reference to the sigblock area.
931 		 */
932 		sgnblk_poll_unreference(idn_sigb_setup);
933 		idn_gkstat_deinit();
934 	}
935 
936 	ddi_remove_minor_node(dip, NULL);
937 
938 	/*
939 	 * Remove this instance from our linked list.
940 	 */
941 	IDN_SET_INST2SIP(instance, NULL);
942 	if ((hsip = tsip = idn.sip) == sip) {
943 		idn.sip = sip->si_nextp;
944 	} else {
945 		for (; hsip && (sip != hsip); tsip = hsip,
946 		    hsip = hsip->si_nextp)
947 			;
948 		if (hsip)
949 			tsip->si_nextp = hsip->si_nextp;
950 	}
951 	mutex_exit(&idn.siplock);
952 	if (sip->si_ksp)
953 		kstat_delete(sip->si_ksp);
954 
955 	ddi_set_driver_private(dip, NULL);
956 
957 	FREESTRUCT(sip, struct idn, 1);
958 
959 	return (DDI_SUCCESS);
960 }
961 
962 /*
963  * ----------------------------------------------
964  */
965 static idn_gprops_t
966 idn_check_conf(dev_info_t *dip, processorid_t *cpuid)
967 {
968 	static idn_gprops_t	global_props = IDN_GPROPS_UNCHECKED;
969 
970 	if (global_props == IDN_GPROPS_UNCHECKED) {
971 		int		p;
972 
973 		global_props = IDN_GPROPS_OKAY;
974 
975 		for (p = 0; idn_global_props[p].p_string; p++) {
976 			char	*str;
977 			int	*var;
978 			int	val, v_min, v_max, v_def;
979 
980 			str = idn_global_props[p].p_string;
981 			var = (int *)idn_global_props[p].p_var;
982 			v_min = idn_global_props[p].p_min;
983 			v_max = idn_global_props[p].p_max;
984 			v_def = idn_global_props[p].p_def;
985 			ASSERT(str && var);
986 
987 			val = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
988 			    DDI_PROP_DONTPASS |
989 			    DDI_PROP_NOTPROM,
990 			    str, v_def);
991 			if ((v_min != v_max) &&
992 			    ((val < v_min) || (val > v_max))) {
993 				cmn_err(CE_WARN,
994 				    "IDN: 105: driver parameter "
995 				    "(%s) specified (%d) out of "
996 				    "range [%d - %d]",
997 				    str, val, v_min, v_max);
998 				global_props = IDN_GPROPS_ERROR;
999 			} else {
1000 				*var = val;
1001 			}
1002 		}
1003 	}
1004 
1005 	*cpuid = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1006 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
1007 	    "bind_cpu", -1);
1008 
1009 	return (global_props);
1010 }
1011 
1012 static int
1013 idn_size_check()
1014 {
1015 	int		i, cnt;
1016 	int		rv = 0;
1017 	ulong_t		mboxareasize;
1018 	int		max_num_slabs;
1019 	procname_t	proc = "idn_size_check";
1020 
1021 	if (IDN_NWR_SIZE == 0)
1022 		IDN_NWR_SIZE = IDN_SMR_SIZE;
1023 
1024 	if (IDN_NWR_SIZE > IDN_SMR_SIZE) {
1025 		cmn_err(CE_WARN,
1026 		    "IDN: 106: idn_nwr_size(%d) > idn_smr_size(%d)"
1027 		    " - Limiting to %d MB",
1028 		    IDN_NWR_SIZE, IDN_SMR_SIZE, IDN_SMR_SIZE);
1029 		IDN_NWR_SIZE = IDN_SMR_SIZE;
1030 	}
1031 
1032 	if (MB2B(IDN_NWR_SIZE) < IDN_SLAB_SIZE) {
1033 		cmn_err(CE_WARN,
1034 		    "IDN: 107: memory region(%lu) < slab size(%u)",
1035 		    MB2B(IDN_NWR_SIZE), IDN_SLAB_SIZE);
1036 		rv = -1;
1037 	}
1038 
1039 	if (IDN_LOWAT >= IDN_HIWAT) {
1040 		cmn_err(CE_WARN,
1041 		    "IDN: 108: idn_lowat(%d) >= idn_hiwat(%d)",
1042 		    IDN_LOWAT, IDN_HIWAT);
1043 		rv = -1;
1044 	}
1045 
1046 	mboxareasize = (ulong_t)(IDN_MBOXAREA_SIZE + (IDN_SMR_BUFSIZE - 1));
1047 	mboxareasize &= ~((ulong_t)IDN_SMR_BUFSIZE - 1);
1048 #ifdef DEBUG
1049 	if ((ulong_t)IDN_SLAB_SIZE < mboxareasize) {
1050 		PR_DRV("%s: slab size(%d) < mailbox area(%ld)",
1051 		    proc, IDN_SLAB_SIZE, mboxareasize);
1052 		/* not fatal */
1053 	}
1054 #endif /* DEBUG */
1055 
1056 	if ((mboxareasize + (ulong_t)IDN_SLAB_SIZE) > MB2B(IDN_NWR_SIZE)) {
1057 		cmn_err(CE_WARN,
1058 		    "IDN: 109: mailbox area(%lu) + slab size(%u) "
1059 		    "> nwr region(%lu)",
1060 		    mboxareasize, IDN_SLAB_SIZE,
1061 		    MB2B(IDN_NWR_SIZE));
1062 		rv = -1;
1063 	}
1064 
1065 	max_num_slabs = (int)((MB2B(IDN_NWR_SIZE) - mboxareasize) /
1066 	    (ulong_t)IDN_SLAB_SIZE);
1067 	if (max_num_slabs < IDN_SLAB_MINTOTAL) {
1068 		cmn_err(CE_WARN,
1069 		    "IDN: 110: maximum number of slabs(%d) < "
1070 		    "minimum required(%d)",
1071 		    max_num_slabs, IDN_SLAB_MINTOTAL);
1072 		rv = -1;
1073 	} else {
1074 		IDN_SLAB_MAXPERDOMAIN = max_num_slabs / IDN_SLAB_MINTOTAL;
1075 	}
1076 
1077 #if 0
1078 	if ((IDN_MTU + sizeof (struct ether_header)) > IDN_DATA_SIZE) {
1079 		cmn_err(CE_WARN,
1080 		    "IDN: (IDN_MTU(%d) + ether_header(%d)) "
1081 		    "> IDN_DATA_SIZE(%lu)",
1082 		    IDN_MTU, sizeof (struct ether_header),
1083 		    IDN_DATA_SIZE);
1084 		rv = -1;
1085 	}
1086 #endif /* 0 */
1087 
1088 	if (IDN_SMR_BUFSIZE & (IDN_ALIGNSIZE - 1)) {
1089 		cmn_err(CE_WARN,
1090 		    "IDN: 111: idn_smr_bufsize(%d) not on a "
1091 		    "64 byte boundary", IDN_SMR_BUFSIZE);
1092 		rv = -1;
1093 	}
1094 
1095 	for (i = cnt = 0;
1096 	    (cnt <= 1) && (((ulong_t)1 << i) < MB2B(IDN_NWR_SIZE));
1097 	    i++)
1098 		if ((1 << i) & IDN_SMR_BUFSIZE)
1099 			cnt++;
1100 	if ((i > 0) && (!cnt || (cnt > 1))) {
1101 		cmn_err(CE_WARN,
1102 		    "IDN: 112: idn_smr_bufsize(%d) not a power of 2",
1103 		    IDN_SMR_BUFSIZE);
1104 		rv = -1;
1105 	}
1106 
1107 	if ((IDN_MBOX_PER_NET & 1) == 0) {
1108 		cmn_err(CE_WARN,
1109 		    "IDN: 113: idn_mbox_per_net(%d) must be an "
1110 		    "odd number", IDN_MBOX_PER_NET);
1111 		rv = -1;
1112 	}
1113 
1114 	if (idn.nchannels > 0)
1115 		IDN_WINDOW_EMAX = IDN_WINDOW_MAX +
1116 		    ((idn.nchannels - 1) * IDN_WINDOW_INCR);
1117 
1118 	if (IDN_NETSVR_WAIT_MIN > IDN_NETSVR_WAIT_MAX) {
1119 		cmn_err(CE_WARN,
1120 		    "IDN: 115: idn_netsvr_wait_min(%d) cannot be "
1121 		    "greater than idn_netsvr_wait_max(%d)",
1122 		    IDN_NETSVR_WAIT_MIN,
1123 		    IDN_NETSVR_WAIT_MAX);
1124 		rv = -1;
1125 	}
1126 
1127 	return (rv);
1128 }
1129 
1130 static int
1131 idn_init_smr()
1132 {
1133 	uint64_t	obp_paddr;
1134 	uint64_t	obp_size;	/* in Bytes */
1135 	uint_t		smr_size;	/* in MBytes */
1136 	pgcnt_t		npages;
1137 	procname_t	proc = "idn_init_smr";
1138 
1139 	if (idn.smr.ready)
1140 		return (0);
1141 
1142 	if (idn_prom_getsmr(&smr_size, &obp_paddr, &obp_size) < 0)
1143 		return (-1);
1144 
1145 	PR_PROTO("%s: smr_size = %d, obp_paddr = 0x%lx, obp_size = 0x%lx\n",
1146 	    proc, smr_size, obp_paddr, obp_size);
1147 
1148 	if (IDN_SMR_SIZE)
1149 		smr_size = MIN(smr_size, IDN_SMR_SIZE);
1150 
1151 	npages = btopr(MB2B(smr_size));
1152 
1153 	idn.smr.prom_paddr = obp_paddr;
1154 	idn.smr.prom_size = obp_size;
1155 	idn.smr.vaddr = vmem_alloc(heap_arena, ptob(npages), VM_SLEEP);
1156 	ASSERT(((ulong_t)idn.smr.vaddr & MMU_PAGEOFFSET) == 0);
1157 	idn.smr.locpfn = (pfn_t)(obp_paddr >> MMU_PAGESHIFT);
1158 	idn.smr.rempfn = idn.smr.rempfnlim = PFN_INVALID;
1159 	IDN_SMR_SIZE = smr_size;
1160 
1161 	PR_PROTO("%s: smr vaddr = %p\n", proc, idn.smr.vaddr);
1162 
1163 	smr_remap(&kas, idn.smr.vaddr, idn.smr.locpfn, IDN_SMR_SIZE);
1164 
1165 	idn.localid = PADDR_TO_DOMAINID(obp_paddr);
1166 
1167 	idn.smr.ready = 1;
1168 
1169 	return (0);
1170 }
1171 
1172 static void
1173 idn_deinit_smr()
1174 {
1175 	pgcnt_t		npages;
1176 
1177 	if (idn.smr.ready == 0)
1178 		return;
1179 
1180 	smr_remap(&kas, idn.smr.vaddr, PFN_INVALID, IDN_SMR_SIZE);
1181 
1182 	npages = btopr(MB2B(IDN_SMR_SIZE));
1183 
1184 	vmem_free(heap_arena, idn.smr.vaddr, ptob(npages));
1185 
1186 	idn.localid = IDN_NIL_DOMID;
1187 
1188 	IDN_SMR_SIZE = 0;
1189 
1190 	idn.smr.ready = 0;
1191 }
1192 
1193 /*ARGSUSED1*/
1194 static void
1195 idn_sigb_setup(cpu_sgnblk_t *sigbp, void *arg)
1196 {
1197 	procname_t	proc = "idn_sigb_setup";
1198 
1199 	PR_PROTO("%s: Setting sigb to %p\n", proc, sigbp);
1200 
1201 	mutex_enter(&idn.idnsb_mutex);
1202 	if (sigbp == NULL) {
1203 		idn.idnsb = NULL;
1204 		idn.idnsb_eventp = NULL;
1205 		mutex_exit(&idn.idnsb_mutex);
1206 		return;
1207 	}
1208 	idn.idnsb_eventp = (idnsb_event_t *)sigbp->sigb_idn;
1209 	idn.idnsb = (idnsb_t *)&idn.idnsb_eventp->idn_reserved1;
1210 	mutex_exit(&idn.idnsb_mutex);
1211 }
1212 
1213 static int
1214 idn_init(dev_info_t *dip)
1215 {
1216 	struct hwconfig	local_hw;
1217 	procname_t	proc = "idn_init";
1218 
1219 
1220 	ASSERT(MUTEX_HELD(&idn.siplock));
1221 
1222 	if (!idn.enabled) {
1223 		cmn_err(CE_WARN,
1224 		    "IDN: 117: IDN not enabled");
1225 		return (-1);
1226 	}
1227 
1228 	if (idn.dip != NULL) {
1229 		PR_DRV("%s: already initialized (dip = 0x%p)\n",
1230 		    proc, idn.dip);
1231 		return (0);
1232 	}
1233 
1234 	/*
1235 	 * Determine our local domain's hardware configuration.
1236 	 */
1237 	if (get_hw_config(&local_hw)) {
1238 		cmn_err(CE_WARN,
1239 		    "IDN: 118: hardware config not appropriate");
1240 		return (-1);
1241 	}
1242 
1243 	PR_DRV("%s: locpfn = 0x%lx\n", proc, idn.smr.locpfn);
1244 	PR_DRV("%s: rempfn = 0x%lx\n", proc, idn.smr.rempfn);
1245 	PR_DRV("%s: smrsize = %d MB\n", proc, IDN_SMR_SIZE);
1246 
1247 	rw_init(&idn.grwlock, NULL, RW_DEFAULT, NULL);
1248 	rw_init(&idn.struprwlock, NULL, RW_DEFAULT, NULL);
1249 	mutex_init(&idn.sync.sz_mutex, NULL, MUTEX_DEFAULT, NULL);
1250 	mutex_init(&idn.sipwenlock, NULL, MUTEX_DEFAULT, NULL);
1251 
1252 	/*
1253 	 * Calculate proper value for idn.bframe_shift.
1254 	 * Kind of hokey as it assume knowledge of the format
1255 	 * of the idnparam_t structure.
1256 	 */
1257 	{
1258 		int		s;
1259 
1260 		for (s = 0; (1 << s) < IDN_SMR_BUFSIZE_MIN; s++)
1261 			;
1262 		idn.bframe_shift = s;
1263 		PR_DRV("%s: idn.bframe_shift = %d, minbuf = %d\n",
1264 		    proc, idn.bframe_shift, IDN_SMR_BUFSIZE_MIN);
1265 
1266 		ASSERT((uint_t)IDN_OFFSET2BFRAME(MB2B(idn_smr_size)) <
1267 		    (1 << 24));
1268 	}
1269 
1270 	idn_xmit_monitor_init();
1271 
1272 	/*
1273 	 * Initialize the domain op (dopers) stuff.
1274 	 */
1275 	idn_dopers_init();
1276 
1277 	/*
1278 	 * Initialize the timer (kmem) cache used for timeout
1279 	 * structures.
1280 	 */
1281 	idn_timercache_init();
1282 
1283 	/*
1284 	 * Initialize the slab waiting areas.
1285 	 */
1286 	(void) smr_slabwaiter_init();
1287 
1288 	/*
1289 	 * Initialize retryjob kmem cache.
1290 	 */
1291 	idn_retrytask_init();
1292 
1293 	idn_init_msg_waittime();
1294 	idn_init_msg_retrytime();
1295 
1296 	/*
1297 	 * Initialize idn_domains[] and local domains information
1298 	 * include idn_global information.
1299 	 */
1300 	idn_domains_init(&local_hw);
1301 
1302 	/*
1303 	 * Start up IDN protocol servers.
1304 	 */
1305 	if (idn_protocol_init(idn_protocol_nservers) <= 0) {
1306 		cmn_err(CE_WARN,
1307 		    "IDN: 119: failed to initialize %d protocol servers",
1308 		    idn_protocol_nservers);
1309 		idn_domains_deinit();
1310 		idn_retrytask_deinit();
1311 		smr_slabwaiter_deinit();
1312 		idn_timercache_deinit();
1313 		idn_dopers_deinit();
1314 		idn_xmit_monitor_deinit();
1315 		mutex_destroy(&idn.sipwenlock);
1316 		mutex_destroy(&idn.sync.sz_mutex);
1317 		rw_destroy(&idn.grwlock);
1318 		rw_destroy(&idn.struprwlock);
1319 		return (-1);
1320 	}
1321 
1322 	/*
1323 	 * Initialize chan_servers array.
1324 	 */
1325 	(void) idn_chanservers_init();
1326 
1327 	/*
1328 	 * Need to register the IDN handler with the DMV subsystem.
1329 	 *
1330 	 * Need to prevent the IDN driver from being unloaded
1331 	 * once loaded since DMV's may come in at any time.
1332 	 * If the driver is not loaded and the idn_dmv_handler
1333 	 * has been registered with the DMV, system will crash.
1334 	 */
1335 	(void) idn_init_handler();
1336 
1337 	idn.dip = dip;
1338 	IDN_GLOCK_EXCL();
1339 	IDN_GSTATE_TRANSITION(IDNGS_OFFLINE);
1340 	IDN_GUNLOCK();
1341 
1342 	return (0);
1343 }
1344 
1345 static int
1346 idn_deinit()
1347 {
1348 	procname_t	proc = "idn_deinit";
1349 
1350 	ASSERT(MUTEX_HELD(&idn.siplock));
1351 
1352 	IDN_GLOCK_EXCL();
1353 
1354 	if (idn.state != IDNGS_OFFLINE) {
1355 		int	d;
1356 
1357 		cmn_err(CE_WARN,
1358 		    "IDN: 120: cannot deinit while active "
1359 		    "(state = %s)", idngs_str[idn.state]);
1360 
1361 		for (d = 0; d < MAX_DOMAINS; d++) {
1362 			idn_domain_t	*dp;
1363 
1364 			dp = &idn_domain[d];
1365 			if (dp->dcpu < 0)
1366 				continue;
1367 
1368 			cmn_err(CE_CONT,
1369 			    "IDN: 121: domain %d (CPU %d, "
1370 			    "name \"%s\", state %s)\n",
1371 			    d, dp->dcpu, dp->dname,
1372 			    idnds_str[dp->dstate]);
1373 		}
1374 		IDN_GUNLOCK();
1375 		return (-1);
1376 	}
1377 
1378 	if (idn.dip == NULL) {
1379 		PR_DRV("%s: already deinitialized\n", proc);
1380 		IDN_GUNLOCK();
1381 		return (0);
1382 	}
1383 
1384 	IDN_GSTATE_TRANSITION(IDNGS_IGNORE);
1385 
1386 	IDN_GUNLOCK();
1387 
1388 	idn_xmit_monitor_deinit();
1389 
1390 	idn_deinit_handler();
1391 
1392 	idn_chanservers_deinit();
1393 
1394 	idn.nchannels = 0;
1395 	ASSERT(idn.chan_servers == NULL);
1396 
1397 	smr_slabpool_deinit();
1398 
1399 	idn_protocol_deinit();
1400 
1401 	idn_domains_deinit();
1402 
1403 	smr_slabwaiter_deinit();
1404 
1405 	idn_retrytask_deinit();
1406 
1407 	idn_timercache_deinit();
1408 
1409 	idn_dopers_deinit();
1410 
1411 	ASSERT(idn.localid == IDN_NIL_DOMID);
1412 
1413 	IDN_SET_MASTERID(IDN_NIL_DOMID);
1414 
1415 	idn_deinit_smr();
1416 
1417 	mutex_destroy(&idn.sipwenlock);
1418 	mutex_destroy(&idn.sync.sz_mutex);
1419 	rw_destroy(&idn.grwlock);
1420 	rw_destroy(&idn.struprwlock);
1421 
1422 	idn.dip = NULL;
1423 
1424 	return (0);
1425 }
1426 
1427 static void
1428 idn_xmit_monitor_init()
1429 {
1430 	mutex_init(&idn.xmit_lock, NULL, MUTEX_DEFAULT, NULL);
1431 	idn.xmit_tid = (timeout_id_t)NULL;
1432 	CHANSET_ZERO(idn.xmit_chanset_wanted);
1433 }
1434 
1435 static void
1436 idn_xmit_monitor_deinit()
1437 {
1438 	timeout_id_t	tid;
1439 
1440 	mutex_enter(&idn.xmit_lock);
1441 	CHANSET_ZERO(idn.xmit_chanset_wanted);
1442 	if ((tid = idn.xmit_tid) != (timeout_id_t)NULL) {
1443 		idn.xmit_tid = (timeout_id_t)NULL;
1444 		mutex_exit(&idn.xmit_lock);
1445 		(void) untimeout(tid);
1446 	} else {
1447 		mutex_exit(&idn.xmit_lock);
1448 	}
1449 	mutex_destroy(&idn.xmit_lock);
1450 }
1451 
1452 static void
1453 idn_init_msg_waittime()
1454 {
1455 	idn_msg_waittime[IDNP_NULL] = -1;
1456 	idn_msg_waittime[IDNP_NEGO] = idn_msgwait_nego * hz;
1457 	idn_msg_waittime[IDNP_CFG]  = idn_msgwait_cfg * hz;
1458 	idn_msg_waittime[IDNP_CON]  = idn_msgwait_con * hz;
1459 	idn_msg_waittime[IDNP_FIN]  = idn_msgwait_fin * hz;
1460 	idn_msg_waittime[IDNP_CMD]  = idn_msgwait_cmd * hz;
1461 	idn_msg_waittime[IDNP_DATA] = idn_msgwait_data * hz;
1462 }
1463 
1464 static void
1465 idn_init_msg_retrytime()
1466 {
1467 	idn_msg_retrytime[(int)IDNRETRY_NIL]	 = -1;
1468 	idn_msg_retrytime[(int)IDNRETRY_NEGO]	 = idn_retryfreq_nego * hz;
1469 	idn_msg_retrytime[(int)IDNRETRY_CON]	 = idn_retryfreq_con * hz;
1470 	idn_msg_retrytime[(int)IDNRETRY_CONQ]	 = idn_retryfreq_con * hz;
1471 	idn_msg_retrytime[(int)IDNRETRY_FIN]	 = idn_retryfreq_fin * hz;
1472 	idn_msg_retrytime[(int)IDNRETRY_FINQ]	 = idn_retryfreq_fin * hz;
1473 }
1474 
1475 /*
1476  * ----------------------------------------------
1477  */
1478 /*ARGSUSED*/
1479 static int
1480 idnopen(register queue_t *rq, dev_t *devp, int flag, int sflag, cred_t *crp)
1481 {
1482 	register int	err = 0;
1483 	int		minordev;
1484 	struct idnstr	*stp, **pstp;
1485 	procname_t	proc = "idnopen";
1486 
1487 	ASSERT(sflag != MODOPEN);
1488 
1489 	IDN_GLOCK_EXCL();
1490 
1491 	rw_enter(&idn.struprwlock, RW_WRITER);
1492 	mutex_enter(&idn.sipwenlock);
1493 	pstp = &idn.strup;
1494 
1495 	if (idn.enabled == 0) {
1496 		PR_DRV("%s: Driver disabled (check OBP:idn-smr-size)\n",
1497 		    proc);
1498 		mutex_exit(&idn.sipwenlock);
1499 		rw_exit(&idn.struprwlock);
1500 		IDN_GUNLOCK();
1501 		return (EACCES);
1502 	}
1503 
1504 	if (!idn_ndlist &&
1505 	    idn_param_register(idn_param_arr, A_CNT(idn_param_arr))) {
1506 		PR_DRV("%s: failed to register ndd parameters\n", proc);
1507 		mutex_exit(&idn.sipwenlock);
1508 		rw_exit(&idn.struprwlock);
1509 		IDN_GUNLOCK();
1510 		return (ENOMEM);
1511 	}
1512 	IDN_GUNLOCK();
1513 
1514 	if (sflag == CLONEOPEN) {
1515 		minordev = 0;
1516 		for (stp = *pstp; stp; pstp = &stp->ss_nextp, stp = *pstp) {
1517 			if (minordev < stp->ss_minor)
1518 				break;
1519 			minordev++;
1520 		}
1521 		*devp = makedevice(getmajor(*devp), minordev);
1522 	} else {
1523 		minordev = getminor(*devp);
1524 	}
1525 	if (rq->q_ptr)
1526 		goto done;
1527 
1528 	stp = GETSTRUCT(struct idnstr, 1);
1529 	stp->ss_rq = rq;
1530 	stp->ss_minor = minordev;
1531 	rw_init(&stp->ss_rwlock, NULL, RW_DEFAULT, NULL);
1532 	/*
1533 	 * DLPI stuff
1534 	 */
1535 	stp->ss_sip = NULL;
1536 	stp->ss_state = DL_UNATTACHED;
1537 	stp->ss_sap = 0;
1538 	stp->ss_flags = 0;
1539 	stp->ss_mccount = 0;
1540 	stp->ss_mctab = NULL;
1541 
1542 	/*
1543 	 * Link new entry into list of actives.
1544 	 */
1545 	stp->ss_nextp = *pstp;
1546 	*pstp = stp;
1547 
1548 	WR(rq)->q_ptr = rq->q_ptr = (void *)stp;
1549 	/*
1550 	 * Disable automatic enabling of our write service
1551 	 * procedure.  We control this explicitly.
1552 	 */
1553 	noenable(WR(rq));
1554 
1555 	/*
1556 	 * Set our STREAMs queue maximum packet size that
1557 	 * we'll accept and our high/low water marks.
1558 	 */
1559 	(void) strqset(WR(rq), QMAXPSZ, 0, IDN_DATA_SIZE);
1560 	(void) strqset(WR(rq), QLOWAT,  0, IDN_LOWAT);
1561 	(void) strqset(WR(rq), QHIWAT,  0, IDN_HIWAT);
1562 	(void) strqset(rq, QMAXPSZ, 0, IDN_DATA_SIZE);
1563 	(void) strqset(rq, QLOWAT,  0, IDN_LOWAT);
1564 	(void) strqset(rq, QHIWAT,  0, IDN_HIWAT);
1565 
1566 done:
1567 	mutex_exit(&idn.sipwenlock);
1568 	rw_exit(&idn.struprwlock);
1569 
1570 	(void) qassociate(rq, -1);
1571 	qprocson(rq);
1572 
1573 	return (err);
1574 }
1575 
1576 /*
1577  * ----------------------------------------------
1578  */
1579 /*ARGSUSED1*/
1580 static int
1581 idnclose(queue_t *rq, int flag, cred_t *crp)
1582 {
1583 	struct idnstr	*stp, **pstp;
1584 
1585 	ASSERT(rq->q_ptr);
1586 
1587 	qprocsoff(rq);
1588 	/*
1589 	 * Guaranteed to be single threaded with respect
1590 	 * to this stream at this point.
1591 	 */
1592 
1593 	stp = (struct idnstr *)rq->q_ptr;
1594 
1595 	if (stp->ss_sip)
1596 		idndl_dodetach(stp);
1597 
1598 	rw_enter(&idn.struprwlock, RW_WRITER);
1599 	mutex_enter(&idn.sipwenlock);
1600 	pstp = &idn.strup;
1601 	for (stp = *pstp; stp; pstp = &stp->ss_nextp, stp = *pstp)
1602 		if (stp == (struct idnstr *)rq->q_ptr)
1603 			break;
1604 	ASSERT(stp);
1605 	ASSERT(stp->ss_rq == rq);
1606 	*pstp = stp->ss_nextp;
1607 
1608 	rw_destroy(&stp->ss_rwlock);
1609 	FREESTRUCT(stp, struct idnstr, 1);
1610 
1611 	WR(rq)->q_ptr = rq->q_ptr = NULL;
1612 	mutex_exit(&idn.sipwenlock);
1613 	rw_exit(&idn.struprwlock);
1614 
1615 	idn_param_cleanup();
1616 	(void) qassociate(rq, -1);
1617 
1618 	return (0);
1619 }
1620 
1621 /*
1622  * ----------------------------------------------
1623  */
1624 static int
1625 idnwput(register queue_t *wq, register mblk_t *mp)
1626 {
1627 	register struct idnstr	*stp;
1628 	struct idn		*sip;
1629 	procname_t	proc = "idnwput";
1630 
1631 	stp = (struct idnstr *)wq->q_ptr;
1632 	sip = stp->ss_sip;
1633 
1634 	switch (DB_TYPE(mp)) {
1635 	case M_IOCTL:
1636 		idnioctl(wq, mp);
1637 		break;
1638 
1639 	case M_DATA:
1640 		if (((stp->ss_flags & (IDNSFAST|IDNSRAW)) == 0) ||
1641 		    (stp->ss_state != DL_IDLE) ||
1642 		    (sip == NULL)) {
1643 			PR_DLPI("%s: fl=0x%x, st=0x%x, ret(EPROTO)\n",
1644 			    proc, stp->ss_flags, stp->ss_state);
1645 			merror(wq, mp, EPROTO);
1646 
1647 		} else if (wq->q_first) {
1648 			if (putq(wq, mp) == 0)
1649 				freemsg(mp);
1650 			/*
1651 			 * We're only holding the reader lock,
1652 			 * but that's okay since this field
1653 			 * is just a soft-flag.
1654 			 */
1655 			sip->si_wantw = 1;
1656 			qenable(wq);
1657 
1658 		} else if (sip->si_flags & IDNPROMISC) {
1659 			if (putq(wq, mp) == 0) {
1660 				PR_DLPI("%s: putq failed\n", proc);
1661 				freemsg(mp);
1662 			} else {
1663 				PR_DLPI("%s: putq succeeded\n", proc);
1664 			}
1665 			qenable(wq);
1666 
1667 		} else {
1668 			PR_DLPI("%s: idndl_start(sip=0x%p)\n",
1669 			    proc, sip);
1670 			rw_enter(&stp->ss_rwlock, RW_READER);
1671 			(void) idndl_start(wq, mp, sip);
1672 			rw_exit(&stp->ss_rwlock);
1673 		}
1674 		break;
1675 
1676 	case M_PROTO:
1677 	case M_PCPROTO:
1678 		/*
1679 		 * Break the association between the current thread
1680 		 * and the thread that calls idndl_proto() to resolve
1681 		 * the problem of idn_chan_server() threads which
1682 		 * loop back around to call idndl_proto and try to
1683 		 * recursively acquire internal locks.
1684 		 */
1685 		if (putq(wq, mp) == 0)
1686 			freemsg(mp);
1687 		qenable(wq);
1688 		break;
1689 
1690 	case M_FLUSH:
1691 		PR_STR("%s: M_FLUSH request (flush = %d)\n",
1692 		    proc, (int)*mp->b_rptr);
1693 		if (*mp->b_rptr & FLUSHW) {
1694 			flushq(wq, FLUSHALL);
1695 			*mp->b_rptr &= ~FLUSHW;
1696 		}
1697 		if (*mp->b_rptr & FLUSHR)
1698 			qreply(wq, mp);
1699 		else
1700 			freemsg(mp);
1701 		break;
1702 
1703 	default:
1704 		PR_STR("%s: unexpected DB_TYPE 0x%x\n",
1705 		    proc, DB_TYPE(mp));
1706 		freemsg(mp);
1707 		break;
1708 	}
1709 
1710 	return (0);
1711 }
1712 
1713 /*
1714  * ----------------------------------------------
1715  */
1716 static int
1717 idnwsrv(queue_t *wq)
1718 {
1719 	mblk_t		*mp;
1720 	int		err = 0;
1721 	struct idnstr	*stp;
1722 	struct idn	*sip;
1723 	procname_t	proc = "idnwsrv";
1724 
1725 	stp = (struct idnstr *)wq->q_ptr;
1726 	sip = stp->ss_sip;
1727 
1728 	while (mp = getq(wq)) {
1729 		switch (DB_TYPE(mp)) {
1730 		case M_DATA:
1731 			if (sip) {
1732 				PR_DLPI("%s: idndl_start(sip=0x%p)\n",
1733 				    proc, sip);
1734 				rw_enter(&stp->ss_rwlock, RW_READER);
1735 				err = idndl_start(wq, mp, sip);
1736 				rw_exit(&stp->ss_rwlock);
1737 				if (err)
1738 					goto done;
1739 			} else {
1740 				PR_DLPI("%s: NO sip to start msg\n", proc);
1741 				freemsg(mp);
1742 			}
1743 			break;
1744 
1745 		case M_PROTO:
1746 		case M_PCPROTO:
1747 			idndl_proto(wq, mp);
1748 			break;
1749 
1750 		default:
1751 			ASSERT(0);
1752 			PR_STR("%s: unexpected db_type (%d)\n",
1753 			    proc, DB_TYPE(mp));
1754 			freemsg(mp);
1755 			break;
1756 		}
1757 	}
1758 done:
1759 	return (0);
1760 }
1761 
1762 /*
1763  * ----------------------------------------------
1764  */
1765 static int
1766 idnrput(register queue_t *rq, register mblk_t *mp)
1767 {
1768 	register int	err = 0;
1769 	procname_t	proc = "idnrput";
1770 
1771 	switch (DB_TYPE(mp)) {
1772 	case M_DATA:
1773 		/*
1774 		 * Should not reach here with data packets
1775 		 * if running DLPI.
1776 		 */
1777 		cmn_err(CE_WARN,
1778 		    "IDN: 123: unexpected M_DATA packets for "
1779 		    "q_stream 0x%p", rq->q_stream);
1780 		freemsg(mp);
1781 		err = ENXIO;
1782 		break;
1783 
1784 	case M_FLUSH:
1785 		PR_STR("%s: M_FLUSH request (flush = %d)\n",
1786 		    proc, (int)*mp->b_rptr);
1787 		if (*mp->b_rptr & FLUSHR)
1788 			flushq(rq, FLUSHALL);
1789 		(void) putnext(rq, mp);
1790 		break;
1791 
1792 	case M_ERROR:
1793 		PR_STR("%s: M_ERROR (error = %d) coming through\n",
1794 		    proc, (int)*mp->b_rptr);
1795 		(void) putnext(rq, mp);
1796 		break;
1797 	default:
1798 		PR_STR("%s: unexpected DB_TYPE 0x%x\n",
1799 		    proc, DB_TYPE(mp));
1800 		freemsg(mp);
1801 		err = ENXIO;
1802 		break;
1803 	}
1804 
1805 	return (err);
1806 }
1807 
1808 /*
1809  * ----------------------------------------------
1810  * Not allowed to enqueue messages!  Only M_DATA messages
1811  * can be enqueued on the write stream.
1812  * ----------------------------------------------
1813  */
1814 static void
1815 idnioctl(register queue_t *wq, register mblk_t *mp)
1816 {
1817 	register struct iocblk	*iocp;
1818 	register int	cmd;
1819 	idnop_t		*idnop = NULL;
1820 	int		error = 0;
1821 	int		argsize;
1822 	procname_t	proc = "idnioctl";
1823 
1824 	iocp = (struct iocblk *)mp->b_rptr;
1825 	cmd  = iocp->ioc_cmd;
1826 
1827 	/*
1828 	 * Intercept DLPI ioctl's.
1829 	 */
1830 	if (VALID_DLPIOP(cmd)) {
1831 		PR_STR("%s: DLPI ioctl(%d)\n", proc, cmd);
1832 		error = idnioc_dlpi(wq, mp, &argsize);
1833 		goto done;
1834 	}
1835 
1836 	/*
1837 	 * Validate expected arguments.
1838 	 */
1839 	if (!VALID_IDNIOCTL(cmd)) {
1840 		PR_STR("%s: invalid cmd (0x%x)\n", proc, cmd);
1841 		error = EINVAL;
1842 		goto done;
1843 
1844 	} else if (!VALID_NDOP(cmd)) {
1845 		error = miocpullup(mp, sizeof (idnop_t));
1846 		if (error != 0) {
1847 			PR_STR("%s: idnioc(cmd = 0x%x) miocpullup "
1848 			    "failed (%d)\n", proc, cmd, error);
1849 			goto done;
1850 		}
1851 	}
1852 
1853 	argsize = mp->b_cont->b_wptr - mp->b_cont->b_rptr;
1854 	idnop = (idnop_t *)mp->b_cont->b_rptr;
1855 
1856 	switch (cmd) {
1857 	case IDNIOC_LINK:
1858 		error = idnioc_link(idnop);
1859 		break;
1860 
1861 	case IDNIOC_UNLINK:
1862 		error = idnioc_unlink(idnop);
1863 		break;
1864 
1865 	case IDNIOC_MEM_RW:
1866 		error = idn_rw_mem(idnop);
1867 		break;
1868 
1869 	case IDNIOC_PING:
1870 		error = idn_send_ping(idnop);
1871 		break;
1872 
1873 	case ND_SET:
1874 		IDN_GLOCK_EXCL();
1875 		if (!nd_getset(wq, idn_ndlist, mp)) {
1876 			IDN_GUNLOCK();
1877 			error = ENOENT;
1878 			break;
1879 		}
1880 		IDN_GUNLOCK();
1881 		qreply(wq, mp);
1882 		return;
1883 
1884 	case ND_GET:
1885 		IDN_GLOCK_SHARED();
1886 		if (!nd_getset(wq, idn_ndlist, mp)) {
1887 			IDN_GUNLOCK();
1888 			error = ENOENT;
1889 			break;
1890 		}
1891 		IDN_GUNLOCK();
1892 		qreply(wq, mp);
1893 		return;
1894 
1895 	default:
1896 		PR_STR("%s: invalid cmd 0x%x\n", proc, cmd);
1897 		error = EINVAL;
1898 		break;
1899 	}
1900 
1901 done:
1902 	if (error == 0)
1903 		miocack(wq, mp, argsize, 0);
1904 	else
1905 		miocnak(wq, mp, 0, error);
1906 }
1907 
1908 /*
1909  * This thread actually services the SSI_LINK/UNLINK calls
1910  * asynchronously that come via BBSRAM.  This is necessary
1911  * since we can't process them from within the context of
1912  * the interrupt handler in which idn_sigbhandler() is
1913  * called.
1914  */
1915 static void
1916 idn_sigbhandler_thread(struct sigbintr **sbpp)
1917 {
1918 	int		d, pri, rv;
1919 	struct sigbintr	*sbp;
1920 	sigbmbox_t	*mbp;
1921 	idn_fin_t	fintype;
1922 	idnsb_data_t	*sdp;
1923 	idnsb_info_t	*sfp;
1924 	idnsb_error_t	*sep;
1925 	idn_domain_t	*dp;
1926 	procname_t	proc = "idn_sigbhandler_thread";
1927 
1928 
1929 	sbp = *sbpp;
1930 
1931 	PR_PROTO("%s: KICKED OFF (sigbintr pointer = 0x%p)\n",
1932 	    proc, sbp);
1933 
1934 	ASSERT(sbp == &idn.sigbintr);
1935 
1936 	mutex_enter(&idn.sigbintr.sb_mutex);
1937 
1938 	while (sbp->sb_busy != IDNSIGB_DIE) {
1939 		cpu_sgnblk_t	*sigbp;
1940 
1941 		while ((sbp->sb_busy != IDNSIGB_ACTIVE) &&
1942 		    (sbp->sb_busy != IDNSIGB_DIE)) {
1943 			cv_wait(&sbp->sb_cv, &idn.sigbintr.sb_mutex);
1944 			PR_PROTO("%s: AWAKENED (busy = %d)\n",
1945 			    proc, (int)sbp->sb_busy);
1946 		}
1947 		if (sbp->sb_busy == IDNSIGB_DIE) {
1948 			PR_PROTO("%s: DIE REQUESTED\n", proc);
1949 			break;
1950 		}
1951 
1952 		if ((sigbp = cpu_sgnblkp[sbp->sb_cpuid]) == NULL) {
1953 			cmn_err(CE_WARN,
1954 			    "IDN: 124: sigblk for CPU ID %d "
1955 			    "is NULL", sbp->sb_cpuid);
1956 			sbp->sb_busy = IDNSIGB_INACTIVE;
1957 			continue;
1958 		}
1959 
1960 		mbp = &sigbp->sigb_host_mbox;
1961 
1962 		if (mbp->flag != SIGB_MBOX_BUSY) {
1963 			PR_PROTO("%s: sigblk mbox flag (%d) != BUSY (%d)\n",
1964 			    proc, mbp->flag, SIGB_MBOX_BUSY);
1965 			sbp->sb_busy = IDNSIGB_INACTIVE;
1966 			continue;
1967 		}
1968 		/*
1969 		 * The sb_busy bit is set and the mailbox flag
1970 		 * indicates BUSY also, so we effectively have things locked.
1971 		 * So, we can drop the critical sb_mutex which we want to
1972 		 * do since it pushes us to PIL 14 while we hold it and we
1973 		 * don't want to run at PIL 14 across IDN code.
1974 		 */
1975 		mutex_exit(&idn.sigbintr.sb_mutex);
1976 
1977 		sdp = (idnsb_data_t *)mbp->data;
1978 		sep = (idnsb_error_t *)&sdp->ssb_error;
1979 		INIT_IDNKERR(sep);
1980 
1981 		if (mbp->len != sizeof (idnsb_data_t)) {
1982 			PR_PROTO("%s: sigblk mbox length (%d) != "
1983 			    "expected (%lu)\n", proc, mbp->len,
1984 			    sizeof (idnsb_data_t));
1985 			SET_IDNKERR_ERRNO(sep, EINVAL);
1986 			SET_IDNKERR_IDNERR(sep, IDNKERR_DATA_LEN);
1987 			SET_IDNKERR_PARAM0(sep, sizeof (idnsb_data_t));
1988 
1989 			goto sberr;
1990 
1991 		}
1992 		if (idn.enabled == 0) {
1993 #ifdef DEBUG
1994 			cmn_err(CE_NOTE,
1995 			    "IDN: 102: driver disabled "
1996 			    "- check OBP environment "
1997 			    "(idn-smr-size)");
1998 #else /* DEBUG */
1999 			cmn_err(CE_NOTE,
2000 			    "!IDN: 102: driver disabled "
2001 			    "- check OBP environment "
2002 			    "(idn-smr-size)");
2003 #endif /* DEBUG */
2004 			SET_IDNKERR_ERRNO(sep, EACCES);
2005 			SET_IDNKERR_IDNERR(sep, IDNKERR_DRV_DISABLED);
2006 
2007 			goto sberr;
2008 
2009 		}
2010 
2011 		switch (mbp->cmd) {
2012 
2013 		case SSI_LINK:
2014 		{
2015 			idnsb_link_t	slp;
2016 
2017 			bcopy(&sdp->ssb_link, &slp, sizeof (slp));
2018 
2019 			if (slp.master_pri < 0) {
2020 				pri = IDNVOTE_MINPRI;
2021 			} else if (slp.master_pri > 0) {
2022 				/*
2023 				 * If I'm already in a IDN network,
2024 				 * then my vote priority is set to
2025 				 * the max, otherwise it's one-less.
2026 				 */
2027 				pri = IDNVOTE_MAXPRI;
2028 				IDN_GLOCK_SHARED();
2029 				if (idn.ndomains <= 1)
2030 					pri--;
2031 				IDN_GUNLOCK();
2032 			} else {
2033 				pri = IDNVOTE_DEFPRI;
2034 			}
2035 
2036 			PR_PROTO("%s: SSI_LINK(cpuid = %d, domid = %d, "
2037 			    "pri = %d (req = %d), t/o = %d)\n",
2038 			    proc, slp.cpuid, slp.domid, pri,
2039 			    slp.master_pri, slp.timeout);
2040 
2041 			rv = idn_link(slp.domid, slp.cpuid, pri,
2042 			    slp.timeout, sep);
2043 			SET_IDNKERR_ERRNO(sep, rv);
2044 			(void) idn_info(&sdp->ssb_info);
2045 			break;
2046 		}
2047 
2048 		case SSI_UNLINK:
2049 		{
2050 			idnsb_unlink_t	sup;
2051 			idn_domain_t	*xdp;
2052 			domainset_t	domset;
2053 
2054 			bcopy(&sdp->ssb_unlink, &sup, sizeof (sup));
2055 
2056 			PR_PROTO("%s: SSI_UNLINK(c = %d, d = %d, bs = 0x%x, "
2057 			    "f = %d, is = 0x%x, t/o = %d)\n",
2058 			    proc, sup.cpuid, sup.domid, sup.boardset,
2059 			    sup.force, sup.idnset, sup.timeout);
2060 
2061 			domset = idn.domset.ds_trans_on |
2062 			    idn.domset.ds_connected |
2063 			    idn.domset.ds_trans_off |
2064 			    idn.domset.ds_awol |
2065 			    idn.domset.ds_relink;
2066 
2067 			if (VALID_DOMAINID(sup.domid)) {
2068 				dp = &idn_domain[sup.domid];
2069 			} else if (VALID_CPUID(sup.cpuid)) {
2070 				for (d = 0; d < MAX_DOMAINS; d++) {
2071 					xdp = &idn_domain[d];
2072 
2073 					if ((xdp->dcpu == IDN_NIL_DCPU) &&
2074 					    !DOMAIN_IN_SET(domset, d))
2075 						continue;
2076 
2077 					if (CPU_IN_SET(xdp->dcpuset,
2078 					    sup.cpuid))
2079 						break;
2080 				}
2081 				dp = (d == MAX_DOMAINS) ? NULL : xdp;
2082 			}
2083 			if ((dp == NULL) && sup.boardset) {
2084 				for (d = 0; d < MAX_DOMAINS; d++) {
2085 					xdp = &idn_domain[d];
2086 
2087 					if ((xdp->dcpu == IDN_NIL_DCPU) &&
2088 					    !DOMAIN_IN_SET(domset, d))
2089 						continue;
2090 
2091 					if (xdp->dhw.dh_boardset &
2092 					    sup.boardset)
2093 						break;
2094 				}
2095 				dp = (d == MAX_DOMAINS) ? NULL : xdp;
2096 			}
2097 			if (dp == NULL) {
2098 				SET_IDNKERR_ERRNO(sep, EINVAL);
2099 				SET_IDNKERR_IDNERR(sep, IDNKERR_INVALID_DOMAIN);
2100 				SET_IDNKERR_PARAM0(sep, sup.domid);
2101 				SET_IDNKERR_PARAM1(sep, sup.cpuid);
2102 				(void) idn_info(&sdp->ssb_info);
2103 				goto sberr;
2104 			} else {
2105 				sup.domid = dp->domid;
2106 			}
2107 
2108 			switch (sup.force) {
2109 			case SSIFORCE_OFF:
2110 				fintype = IDNFIN_NORMAL;
2111 				break;
2112 
2113 			case SSIFORCE_SOFT:
2114 				fintype = IDNFIN_FORCE_SOFT;
2115 				break;
2116 
2117 			case SSIFORCE_HARD:
2118 				fintype = IDNFIN_FORCE_HARD;
2119 				break;
2120 			default:
2121 				SET_IDNKERR_ERRNO(sep, EINVAL);
2122 				SET_IDNKERR_IDNERR(sep, IDNKERR_INVALID_FORCE);
2123 				SET_IDNKERR_PARAM0(sep, sup.force);
2124 				(void) idn_info(&sdp->ssb_info);
2125 				goto sberr;
2126 			}
2127 
2128 			rv = idn_unlink(sup.domid, sup.idnset, fintype,
2129 			    IDNFIN_OPT_UNLINK, sup.timeout, sep);
2130 			SET_IDNKERR_ERRNO(sep, rv);
2131 			(void) idn_info(&sdp->ssb_info);
2132 			break;
2133 		}
2134 
2135 		case SSI_INFO:
2136 			sfp = &sdp->ssb_info;
2137 
2138 			PR_PROTO("%s: SSI_INFO\n", proc);
2139 
2140 			rv = idn_info(sfp);
2141 			SET_IDNKERR_ERRNO(sep, rv);
2142 			if (rv != 0) {
2143 				SET_IDNKERR_IDNERR(sep, IDNKERR_INFO_FAILED);
2144 			}
2145 			break;
2146 
2147 		default:
2148 			ASSERT(0);
2149 			SET_IDNKERR_ERRNO(sep, EINVAL);
2150 			SET_IDNKERR_IDNERR(sep, IDNKERR_INVALID_CMD);
2151 			SET_IDNKERR_PARAM0(sep, mbp->cmd);
2152 			break;
2153 		}
2154 
2155 sberr:
2156 
2157 		if (GET_IDNKERR_ERRNO(sep) != 0) {
2158 			cmn_err(CE_WARN,
2159 #ifdef DEBUG
2160 			    "IDN: 125: op (%s) failed, returning "
2161 			    "(%d/0x%x [%d, %d, %d])",
2162 #else /* DEBUG */
2163 			    "!IDN: 125: op (%s) failed, returning "
2164 			    "(%d/0x%x [%d, %d, %d])",
2165 #endif /* DEBUG */
2166 			    (mbp->cmd == SSI_LINK) ? "LINK" :
2167 			    (mbp->cmd == SSI_UNLINK) ? "UNLINK" :
2168 			    (mbp->cmd == SSI_INFO) ?
2169 			    "INFO" : "UNKNOWN",
2170 			    GET_IDNKERR_ERRNO(sep),
2171 			    GET_IDNKERR_IDNERR(sep),
2172 			    GET_IDNKERR_PARAM0(sep),
2173 			    GET_IDNKERR_PARAM1(sep),
2174 			    GET_IDNKERR_PARAM2(sep));
2175 		}
2176 
2177 		PR_PROTO("%s: returning errno = %d, idnerr = %d, "
2178 		    "params = [%d, %d, %d]\n",
2179 		    proc, GET_IDNKERR_ERRNO(sep), GET_IDNKERR_IDNERR(sep),
2180 		    GET_IDNKERR_PARAM0(sep), GET_IDNKERR_PARAM1(sep),
2181 		    GET_IDNKERR_PARAM2(sep));
2182 
2183 		mutex_enter(&idn.sigbintr.sb_mutex);
2184 		ASSERT((sbp->sb_busy == IDNSIGB_ACTIVE) ||
2185 		    (sbp->sb_busy == IDNSIGB_DIE));
2186 		mbp->cmd |= SSI_ACK;
2187 		if (sbp->sb_busy == IDNSIGB_ACTIVE)
2188 			sbp->sb_busy = IDNSIGB_INACTIVE;
2189 		/*
2190 		 * Set flag which kicks off response to SSP.
2191 		 */
2192 		membar_stst_ldst();
2193 		mbp->flag = HOST_TO_CBS;
2194 	}
2195 
2196 	/*
2197 	 * Wake up the dude that killed us!
2198 	 */
2199 	idn.sigb_threadp = NULL;
2200 	cv_signal(&sbp->sb_cv);
2201 	mutex_exit(&idn.sigbintr.sb_mutex);
2202 	thread_exit();
2203 }
2204 
2205 /*
2206  * Create the thread that will service sigb interrupts.
2207  */
2208 static void
2209 idn_sigbhandler_create()
2210 {
2211 	struct sigbintr	*sbp;
2212 
2213 	if (idn.sigb_threadp) {
2214 		cmn_err(CE_WARN,
2215 		    "IDN: 126: sigbhandler thread already "
2216 		    "exists (0x%p)", idn.sigb_threadp);
2217 		return;
2218 	}
2219 	cv_init(&idn.sigbintr.sb_cv, NULL, CV_DEFAULT, NULL);
2220 	sbp = &idn.sigbintr;
2221 	sbp->sb_busy = IDNSIGB_INACTIVE;
2222 	idn.sigb_threadp = thread_create(NULL, 0,
2223 	    idn_sigbhandler_thread, &sbp, sizeof (sbp), &p0,
2224 	    TS_RUN, minclsyspri);
2225 	sbp->sb_inum = add_softintr((uint_t)idn_sigbpil,
2226 	    idn_sigbhandler_wakeup, 0, SOFTINT_ST);
2227 }
2228 
2229 static void
2230 idn_sigbhandler_kill()
2231 {
2232 	if (idn.sigb_threadp) {
2233 		struct sigbintr	*sbp;
2234 
2235 		sbp = &idn.sigbintr;
2236 		if (sbp->sb_inum != 0)
2237 			(void) rem_softintr(sbp->sb_inum);
2238 		sbp->sb_inum = 0;
2239 		sbp->sb_busy = IDNSIGB_DIE;
2240 		cv_signal(&sbp->sb_cv);
2241 		while (idn.sigb_threadp != NULL)
2242 			cv_wait(&sbp->sb_cv, &idn.sigbintr.sb_mutex);
2243 		sbp->sb_busy = IDNSIGB_INACTIVE;
2244 		cv_destroy(&sbp->sb_cv);
2245 	}
2246 }
2247 
2248 /*ARGSUSED0*/
2249 static uint_t
2250 idn_sigbhandler_wakeup(caddr_t arg1, caddr_t arg2)
2251 {
2252 	mutex_enter(&idn.sigbintr.sb_mutex);
2253 	if (idn.sigbintr.sb_busy == IDNSIGB_STARTED) {
2254 		idn.sigbintr.sb_busy = IDNSIGB_ACTIVE;
2255 		cv_signal(&idn.sigbintr.sb_cv);
2256 	}
2257 	mutex_exit(&idn.sigbintr.sb_mutex);
2258 
2259 	return (DDI_INTR_CLAIMED);
2260 }
2261 
2262 static void
2263 idn_sigbhandler(processorid_t cpuid, cpu_sgnblk_t *sgnblkp)
2264 {
2265 	struct sigbintr	*sbp = &idn.sigbintr;
2266 	sigbmbox_t	*mbp;
2267 	idnsb_data_t	*sdp;
2268 	idnsb_error_t	*sep;
2269 	uint32_t	cmd;
2270 	int		sigb_lock = 0;
2271 
2272 	ASSERT(sgnblkp);
2273 
2274 	mbp = &sgnblkp->sigb_host_mbox;
2275 	sdp = (idnsb_data_t *)mbp->data;
2276 	sep = &sdp->ssb_error;
2277 	cmd = mbp->cmd;
2278 
2279 	if ((mbp->flag != CBS_TO_HOST) || !VALID_IDNSIGBCMD(cmd)) {
2280 		/*
2281 		 * Not a valid IDN command.  Just bail out.
2282 		 */
2283 		return;
2284 	}
2285 
2286 	mbp->flag = SIGB_MBOX_BUSY;
2287 	SET_IDNKERR_ERRNO(sep, 0);
2288 
2289 	if (cmd & SSI_ACK) {
2290 		/*
2291 		 * Hmmm...weird, the ACK bit is set.
2292 		 */
2293 		SET_IDNKERR_ERRNO(sep, EPROTO);
2294 		SET_IDNKERR_IDNERR(sep, IDNKERR_INVALID_CMD);
2295 		SET_IDNKERR_PARAM0(sep, cmd);
2296 		goto sigb_done;
2297 	}
2298 
2299 	if (!mutex_tryenter(&idn.sigbintr.sb_mutex)) {
2300 		/*
2301 		 * Couldn't get the lock.  Driver is either
2302 		 * not quite all the way up or is shutting down
2303 		 * for some reason.  Caller should spin again.
2304 		 */
2305 		cmd |= SSI_ACK;
2306 		SET_IDNKERR_ERRNO(sep, EBUSY);
2307 		SET_IDNKERR_IDNERR(sep, IDNKERR_SIGBINTR_LOCKED);
2308 		goto sigb_done;
2309 	}
2310 	sigb_lock = 1;
2311 
2312 	if ((idn.sigb_threadp == NULL) ||
2313 	    (sbp->sb_busy == IDNSIGB_NOTREADY)) {
2314 		cmd |= SSI_ACK;
2315 		SET_IDNKERR_ERRNO(sep, EAGAIN);
2316 		SET_IDNKERR_IDNERR(sep, IDNKERR_SIGBINTR_NOTRDY);
2317 		goto sigb_done;
2318 	}
2319 
2320 	if (sbp->sb_busy != IDNSIGB_INACTIVE) {
2321 		cmd |= SSI_ACK;
2322 		SET_IDNKERR_ERRNO(sep, EBUSY);
2323 		SET_IDNKERR_IDNERR(sep, IDNKERR_SIGBINTR_BUSY);
2324 		goto sigb_done;
2325 	}
2326 
2327 	sbp->sb_cpuid = (uchar_t)cpuid & 0xff;
2328 	membar_stst_ldst();
2329 	sbp->sb_busy = IDNSIGB_STARTED;
2330 	/*
2331 	 * The sb_busy bit is set and the mailbox flag
2332 	 * indicates BUSY also, so we effectively have things locked.
2333 	 * So, we can drop the critical sb_mutex which we want to
2334 	 * do since it pushes us to PIL 14 while we hold it and we
2335 	 * don't want to run at PIL 14 across IDN code.
2336 	 *
2337 	 * Send interrupt to cause idn_sigbhandler_thread to wakeup.
2338 	 * We cannot do wakeup (cv_signal) directly from here since
2339 	 * we're executing from a high-level (14) interrupt.
2340 	 */
2341 	setsoftint(sbp->sb_inum);
2342 
2343 sigb_done:
2344 
2345 	if (GET_IDNKERR_ERRNO(sep) != 0) {
2346 		mbp->len = sizeof (idnsb_data_t);
2347 		mbp->cmd = cmd;
2348 		membar_stst_ldst();
2349 		mbp->flag = HOST_TO_CBS;
2350 	}
2351 
2352 	if (sigb_lock)
2353 		mutex_exit(&idn.sigbintr.sb_mutex);
2354 }
2355 
2356 static int
2357 idn_info(idnsb_info_t *sfp)
2358 {
2359 	int		count, d;
2360 	idn_domain_t	*dp;
2361 	idnsb_info_t	sinfo;
2362 	int		local_id, master_id;
2363 	procname_t	proc = "idn_info";
2364 
2365 	bzero(&sinfo, sizeof (sinfo));
2366 	sinfo.master_index = (uchar_t)-1;
2367 	sinfo.master_cpuid = (uchar_t)-1;
2368 	sinfo.local_index  = (uchar_t)-1;
2369 	sinfo.local_cpuid  = (uchar_t)-1;
2370 
2371 	IDN_GLOCK_SHARED();
2372 
2373 	sinfo.idn_state = (uchar_t)idn.state;
2374 
2375 	switch (idn.state) {
2376 	case IDNGS_OFFLINE:
2377 		sinfo.idn_active = SSISTATE_INACTIVE;
2378 		PR_PROTO("%s: idn_state (%s) = INACTIVE\n",
2379 		    proc, idngs_str[idn.state]);
2380 		break;
2381 
2382 	case IDNGS_IGNORE:
2383 		PR_PROTO("%s: IGNORING IDN_INFO call...\n", proc);
2384 		IDN_GUNLOCK();
2385 		return (EIO);
2386 
2387 	default:
2388 		sinfo.idn_active = SSISTATE_ACTIVE;
2389 		PR_PROTO("%s: idn_state (%s) = ACTIVE\n",
2390 		    proc, idngs_str[idn.state]);
2391 		break;
2392 	}
2393 	master_id = IDN_GET_MASTERID();
2394 	local_id = idn.localid;
2395 
2396 	/*
2397 	 * Need to drop idn.grwlock before acquiring domain locks.
2398 	 */
2399 	IDN_GUNLOCK();
2400 
2401 	IDN_SYNC_LOCK();
2402 
2403 	sinfo.awol_domset = (ushort_t)idn.domset.ds_awol;
2404 	sinfo.conn_domset = (ushort_t)(idn.domset.ds_connected &
2405 	    ~idn.domset.ds_trans_on);
2406 	DOMAINSET_ADD(sinfo.conn_domset, idn.localid);
2407 
2408 	count = 0;
2409 	for (d = 0; d < MAX_DOMAINS; d++) {
2410 		dp = &idn_domain[d];
2411 
2412 		if (dp->dcpu == IDN_NIL_DCPU)
2413 			continue;
2414 
2415 		IDN_DLOCK_SHARED(d);
2416 		if ((dp->dcpu == IDN_NIL_DCPU) ||
2417 		    (dp->dstate == IDNDS_CLOSED)) {
2418 			IDN_DUNLOCK(d);
2419 			continue;
2420 		}
2421 
2422 		count++;
2423 		if (d == local_id) {
2424 			sinfo.local_index = (uchar_t)d;
2425 			sinfo.local_cpuid = (uchar_t)dp->dcpu;
2426 			PR_PROTO("%s: domid %d is LOCAL (cpuid = %d)\n",
2427 			    proc, d, dp->dcpu);
2428 		}
2429 		if (d == master_id) {
2430 			sinfo.master_index = (uchar_t)d;
2431 			sinfo.master_cpuid = (uchar_t)dp->dcpu;
2432 			PR_PROTO("%s: domid %d is MASTER (cpuid = %d)\n",
2433 			    proc, d, dp->dcpu);
2434 		}
2435 
2436 		sinfo.domain_boardset[d] = (ushort_t)dp->dhw.dh_boardset;
2437 
2438 		IDN_DUNLOCK(d);
2439 	}
2440 
2441 	IDN_SYNC_UNLOCK();
2442 
2443 	bcopy(&sinfo, sfp, sizeof (*sfp));
2444 
2445 	PR_PROTO("%s: Found %d domains within IDNnet\n", proc, count);
2446 
2447 	return (0);
2448 }
2449 
2450 /*
2451  * ----------------------------------------------
2452  * ndd param support routines.
2453  * - Borrowed from tcp.
2454  * ----------------------------------------------
2455  */
2456 static void
2457 idn_param_cleanup()
2458 {
2459 	IDN_GLOCK_EXCL();
2460 	if (!idn.strup && idn_ndlist)
2461 		nd_free(&idn_ndlist);
2462 	IDN_GUNLOCK();
2463 }
2464 
2465 /*ARGSUSED*/
2466 static int
2467 idn_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr)
2468 {
2469 	idnparam_t	*idnpa = (idnparam_t *)cp;
2470 
2471 	/*
2472 	 * lock grabbed before calling nd_getset.
2473 	 */
2474 	ASSERT(IDN_GLOCK_IS_HELD());
2475 
2476 	(void) mi_mpprintf(mp, "%ld", idnpa->sp_val);
2477 
2478 	return (0);
2479 }
2480 
2481 /*ARGSUSED*/
2482 static int
2483 idn_param_set(queue_t *q, mblk_t *mp, char *value, caddr_t cp, cred_t *cr)
2484 {
2485 	char		*end;
2486 	ulong_t		new_value;
2487 	idnparam_t	*idnpa = (idnparam_t *)cp;
2488 
2489 	/*
2490 	 * lock grabbed before calling nd_getset.
2491 	 */
2492 	ASSERT(IDN_GLOCK_IS_EXCL());
2493 
2494 	new_value = (ulong_t)mi_strtol(value, &end, 10);
2495 
2496 	if ((end == value) ||
2497 	    (new_value < idnpa->sp_min) ||
2498 	    (new_value > idnpa->sp_max))
2499 		return (EINVAL);
2500 
2501 	if (idn.enabled == 0) {
2502 #ifdef DEBUG
2503 		cmn_err(CE_NOTE,
2504 		    "IDN: 102: driver disabled "
2505 		    "- check OBP environment "
2506 		    "(idn-smr-size)");
2507 #else /* DEBUG */
2508 		cmn_err(CE_NOTE,
2509 		    "!IDN: 102: driver disabled "
2510 		    "- check OBP environment "
2511 		    "(idn-smr-size)");
2512 #endif /* DEBUG */
2513 		return (EACCES);
2514 	}
2515 
2516 	idnpa->sp_val = new_value;
2517 
2518 	return (0);
2519 }
2520 
2521 static int
2522 idn_param_register(register idnparam_t *idnpa, int count)
2523 {
2524 	ASSERT(IDN_GLOCK_IS_EXCL());
2525 
2526 	for (; count > 0; count--, idnpa++) {
2527 		if (idnpa->sp_name && idnpa->sp_name[0]) {
2528 			register int	i;
2529 			ndsetf_t	set_func;
2530 			char		*p;
2531 			/*
2532 			 * Don't advertise in non-DEBUG parameters.
2533 			 */
2534 			for (i = 0; idn_param_debug_only[i]; i++) {
2535 				p = idn_param_debug_only[i];
2536 				if (strcmp(idnpa->sp_name, p) == 0)
2537 					break;
2538 			}
2539 			if (idn_param_debug_only[i])
2540 				continue;
2541 
2542 			/*
2543 			 * Do not register a "set" function for
2544 			 * read-only parameters.
2545 			 */
2546 			for (i = 0; idn_param_read_only[i]; i++) {
2547 				p = idn_param_read_only[i];
2548 				if (strcmp(idnpa->sp_name, p) == 0)
2549 					break;
2550 			}
2551 			if (idn_param_read_only[i])
2552 				set_func = NULL;
2553 			else
2554 				set_func = idn_param_set;
2555 
2556 			if (!nd_load(&idn_ndlist, idnpa->sp_name,
2557 			    idn_param_get, set_func,
2558 			    (caddr_t)idnpa)) {
2559 				nd_free(&idn_ndlist);
2560 				return (-1);
2561 			}
2562 		}
2563 	}
2564 	if (!nd_load(&idn_ndlist, "idn_slabpool", idn_slabpool_report,
2565 	    NULL, NULL)) {
2566 		nd_free(&idn_ndlist);
2567 		return (-1);
2568 	}
2569 	if (!nd_load(&idn_ndlist, "idn_buffers", idn_buffer_report,
2570 	    NULL, NULL)) {
2571 		nd_free(&idn_ndlist);
2572 		return (-1);
2573 	}
2574 	if (!nd_load(&idn_ndlist, "idn_mboxtbl", idn_mboxtbl_report,
2575 	    NULL, MBXTBL_PART_REPORT)) {
2576 		nd_free(&idn_ndlist);
2577 		return (-1);
2578 	}
2579 	if (!nd_load(&idn_ndlist, "idn_mboxtbl_all", idn_mboxtbl_report,
2580 	    NULL, MBXTBL_FULL_REPORT)) {
2581 		nd_free(&idn_ndlist);
2582 		return (-1);
2583 	}
2584 	if (!nd_load(&idn_ndlist, "idn_mainmbox", idn_mainmbox_report,
2585 	    NULL, NULL)) {
2586 		nd_free(&idn_ndlist);
2587 		return (-1);
2588 	}
2589 	if (!nd_load(&idn_ndlist, "idn_global", idn_global_report,
2590 	    NULL, NULL)) {
2591 		nd_free(&idn_ndlist);
2592 		return (-1);
2593 	}
2594 	if (!nd_load(&idn_ndlist, "idn_domain", idn_domain_report,
2595 	    NULL, (caddr_t)0)) {
2596 		nd_free(&idn_ndlist);
2597 		return (-1);
2598 	}
2599 	if (!nd_load(&idn_ndlist, "idn_domain_all", idn_domain_report,
2600 	    NULL, (caddr_t)1)) {
2601 		nd_free(&idn_ndlist);
2602 		return (-1);
2603 	}
2604 	if (!nd_load(&idn_ndlist, "idn_bind_net", idn_get_net_binding,
2605 	    idn_set_net_binding, NULL)) {
2606 		nd_free(&idn_ndlist);
2607 		return (-1);
2608 	}
2609 
2610 	return (0);
2611 }
2612 
2613 /*ARGSUSED*/
2614 static int
2615 idn_set_net_binding(queue_t *q, mblk_t *mp, char *value, caddr_t cp, cred_t *cr)
2616 {
2617 	char		*end, *cpup;
2618 	long		net;
2619 	processorid_t	cpuid;
2620 
2621 	/*
2622 	 * lock grabbed before calling nd_getset.
2623 	 */
2624 	ASSERT(IDN_GLOCK_IS_EXCL());
2625 
2626 	if ((cpup = strchr(value, '=')) == NULL)
2627 		return (EINVAL);
2628 
2629 	*cpup++ = '\0';
2630 
2631 	net = mi_strtol(value, &end, 10);
2632 	if ((end == value) || (net < 0) || (net >= IDN_MAX_NETS) ||
2633 	    !CHAN_IN_SET(idn.chanset, net))
2634 		return (EINVAL);
2635 
2636 	cpuid = (processorid_t)mi_strtol(cpup, &end, 10);
2637 	if ((end == cpup) || ((cpuid != -1) &&
2638 	    (!VALID_CPUID(cpuid) ||
2639 	    !CPU_IN_SET(cpu_ready_set, cpuid))))
2640 		return (EINVAL);
2641 
2642 	idn_chanserver_bind(net, cpuid);
2643 
2644 	return (0);
2645 }
2646 
2647 /*ARGSUSED*/
2648 static int
2649 idn_get_net_binding(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr)
2650 {
2651 	int	c;
2652 
2653 	/*
2654 	 * lock grabbed before calling nd_getset.
2655 	 */
2656 	ASSERT(IDN_GLOCK_IS_HELD());
2657 
2658 	(void) mi_mpprintf(mp,
2659 	    "IDN network interfaces/channels active = %d",
2660 	    idn.nchannels);
2661 
2662 	if (idn.nchannels == 0)
2663 		return (0);
2664 
2665 	mi_mpprintf(mp, "Net    Cpu");
2666 
2667 	for (c = 0; c < IDN_MAX_NETS; c++) {
2668 		int		bc;
2669 		idn_chansvr_t	*csp;
2670 
2671 		if (!CHAN_IN_SET(idn.chanset, c))
2672 			continue;
2673 
2674 		csp = &idn.chan_servers[c];
2675 
2676 		if ((bc = csp->ch_bound_cpuid) == -1)
2677 			bc = csp->ch_bound_cpuid_pending;
2678 
2679 		if (c < 10)
2680 			mi_mpprintf(mp, " %d      %d", c, bc);
2681 		else
2682 			mi_mpprintf(mp, " %d     %d", c, bc);
2683 	}
2684 
2685 	return (0);
2686 }
2687 
2688 static int
2689 idnioc_link(idnop_t *idnop)
2690 {
2691 	int		rv;
2692 	int		pri;
2693 	idnsb_error_t	err;
2694 	procname_t	proc = "idnioc_link";
2695 
2696 	if (idnop->link.master < 0)
2697 		pri = IDNVOTE_MINPRI;
2698 	else if (idnop->link.master > 0)
2699 		pri = IDNVOTE_MAXPRI;
2700 	else
2701 		pri = IDNVOTE_DEFPRI;
2702 
2703 	PR_DRV("%s: domid = %d, cpuid = %d, pri = %d\n",
2704 	    proc, idnop->link.domid, idnop->link.cpuid, pri);
2705 
2706 	rv = idn_link(idnop->link.domid, idnop->link.cpuid,
2707 	    pri, idnop->link.wait, &err);
2708 
2709 	return (rv);
2710 }
2711 
2712 static int
2713 idnioc_unlink(idnop_t *idnop)
2714 {
2715 	int		d, cpuid, domid, rv;
2716 	boardset_t	idnset;
2717 	idn_fin_t	fintype;
2718 	idn_domain_t	*dp, *xdp;
2719 	idnsb_error_t	err;
2720 	procname_t	proc = "idnioc_unlink";
2721 
2722 	PR_DRV("%s: domid = %d, cpuid = %d, force = %d\n",
2723 	    proc, idnop->unlink.domid, idnop->unlink.cpuid,
2724 	    idnop->unlink.force);
2725 
2726 	idnset = BOARDSET_ALL;
2727 	domid = idnop->unlink.domid;
2728 	cpuid = idnop->unlink.cpuid;
2729 	dp = NULL;
2730 
2731 	if (domid == IDN_NIL_DOMID)
2732 		domid = idn.localid;
2733 
2734 	if (VALID_DOMAINID(domid)) {
2735 		dp = &idn_domain[domid];
2736 		if (VALID_CPUID(cpuid) && (dp->dcpu != IDN_NIL_DCPU) &&
2737 		    !CPU_IN_SET(dp->dcpuset, cpuid)) {
2738 			dp = NULL;
2739 			PR_PROTO("%s: ERROR: invalid cpuid "
2740 			    "(%d) for domain (%d) [cset = 0x%x.x%x]\n",
2741 			    proc, cpuid, domid,
2742 			    UPPER32_CPUMASK(dp->dcpuset),
2743 			    LOWER32_CPUMASK(dp->dcpuset));
2744 		}
2745 	} else if (VALID_CPUID(cpuid)) {
2746 		for (d = 0; d < MAX_DOMAINS; d++) {
2747 			xdp = &idn_domain[d];
2748 
2749 			if (xdp->dcpu == IDN_NIL_DCPU)
2750 				continue;
2751 
2752 			if (CPU_IN_SET(xdp->dcpuset, cpuid))
2753 				break;
2754 		}
2755 		dp = (d == MAX_DOMAINS) ? NULL : xdp;
2756 	}
2757 
2758 	if ((dp == NULL) || (dp->dcpu == IDN_NIL_DCPU))
2759 		return (0);
2760 
2761 	domid = dp->domid;
2762 
2763 	switch (idnop->unlink.force) {
2764 	case SSIFORCE_OFF:
2765 		fintype = IDNFIN_NORMAL;
2766 		break;
2767 
2768 	case SSIFORCE_SOFT:
2769 		fintype = IDNFIN_FORCE_SOFT;
2770 		break;
2771 
2772 	case SSIFORCE_HARD:
2773 		fintype = IDNFIN_FORCE_HARD;
2774 		break;
2775 	default:
2776 		PR_PROTO("%s: invalid force parameter \"%d\"",
2777 		    proc, idnop->unlink.force);
2778 		return (EINVAL);
2779 	}
2780 
2781 	rv = idn_unlink(domid, idnset, fintype, IDNFIN_OPT_UNLINK,
2782 	    idnop->unlink.wait, &err);
2783 
2784 	return (rv);
2785 }
2786 
2787 static int
2788 idn_send_ping(idnop_t *idnop)
2789 {
2790 	int		domid = idnop->ping.domid;
2791 	int		cpuid = idnop->ping.cpuid;
2792 	int		ocpuid;
2793 	idn_domain_t	*dp;
2794 	idn_msgtype_t	mt;
2795 	procname_t	proc = "idn_send_ping";
2796 
2797 	if ((domid == IDN_NIL_DOMID) && (cpuid == IDN_NIL_DCPU)) {
2798 		cmn_err(CE_WARN,
2799 		    "IDN: %s: no valid domain ID or CPU ID given",
2800 		    proc);
2801 		return (EINVAL);
2802 	}
2803 	if (domid == IDN_NIL_DOMID)
2804 		domid = MAX_DOMAINS - 1;
2805 
2806 	dp = &idn_domain[domid];
2807 	IDN_DLOCK_EXCL(domid);
2808 	if ((dp->dcpu == IDN_NIL_DCPU) && (cpuid == IDN_NIL_DCPU)) {
2809 		cmn_err(CE_WARN,
2810 		    "IDN: %s: no valid target CPU specified",
2811 		    proc);
2812 		IDN_DUNLOCK(domid);
2813 		return (EINVAL);
2814 	}
2815 	if (cpuid == IDN_NIL_DCPU)
2816 		cpuid = dp->dcpu;
2817 
2818 	ocpuid = dp->dcpu;
2819 	dp->dcpu = cpuid;
2820 
2821 	/*
2822 	 * XXX - Need a special PING IDN command.
2823 	 */
2824 	mt.mt_mtype = IDNP_DATA | IDNP_ACK;
2825 	mt.mt_atype = 0;
2826 
2827 	(void) IDNXDC(domid, &mt, 0x100, 0x200, 0x300, 0x400);
2828 
2829 	dp->dcpu = ocpuid;
2830 	IDN_DUNLOCK(domid);
2831 
2832 	return (0);
2833 }
2834 
2835 /*
2836  * ----------------------------------------------
2837  */
2838 static void
2839 idn_dopers_init()
2840 {
2841 	int		i;
2842 	dop_waitlist_t	*dwl;
2843 
2844 	if (idn.dopers)
2845 		return;
2846 
2847 	idn.dopers = GETSTRUCT(struct dopers, 1);
2848 
2849 	bzero(idn.dopers, sizeof (struct dopers));
2850 
2851 	dwl = &idn.dopers->_dop_wcache[0];
2852 	for (i = 0; i < (IDNOP_CACHE_SIZE-1); i++)
2853 		dwl[i].dw_next = &dwl[i+1];
2854 	dwl[i].dw_next = NULL;
2855 
2856 	idn.dopers->dop_freelist = &idn.dopers->_dop_wcache[0];
2857 	idn.dopers->dop_waitcount = 0;
2858 	idn.dopers->dop_domset = 0;
2859 	idn.dopers->dop_waitlist = NULL;
2860 
2861 	cv_init(&idn.dopers->dop_cv, NULL, CV_DEFAULT, NULL);
2862 	mutex_init(&idn.dopers->dop_mutex, NULL, MUTEX_DEFAULT, NULL);
2863 }
2864 
2865 static void
2866 idn_dopers_deinit()
2867 {
2868 	dop_waitlist_t	*dwl, *next_dwl;
2869 
2870 
2871 	if (idn.dopers == NULL)
2872 		return;
2873 
2874 	for (dwl = idn.dopers->dop_waitlist; dwl; dwl = next_dwl) {
2875 		next_dwl = dwl->dw_next;
2876 		if (!IDNOP_IN_CACHE(dwl))
2877 			FREESTRUCT(dwl, dop_waitlist_t, 1);
2878 	}
2879 
2880 	cv_destroy(&idn.dopers->dop_cv);
2881 	mutex_destroy(&idn.dopers->dop_mutex);
2882 
2883 	FREESTRUCT(idn.dopers, struct dopers, 1);
2884 	idn.dopers = NULL;
2885 }
2886 
2887 /*
2888  * Reset the dop_errset field in preparation for an
2889  * IDN operation attempt.  This is only called from
2890  * idn_link() and idn_unlink().
2891  */
2892 void *
2893 idn_init_op(idn_opflag_t opflag, domainset_t domset, idnsb_error_t *sep)
2894 {
2895 	dop_waitlist_t	*dwl;
2896 	/*
2897 	 * Clear any outstanding error ops in preparation
2898 	 * for an IDN (link/unlink) operation.
2899 	 */
2900 	mutex_enter(&idn.dopers->dop_mutex);
2901 	if ((dwl = idn.dopers->dop_freelist) == NULL) {
2902 		dwl = GETSTRUCT(dop_waitlist_t, 1);
2903 	} else {
2904 		idn.dopers->dop_freelist = dwl->dw_next;
2905 		bzero(dwl, sizeof (*dwl));
2906 	}
2907 	dwl->dw_op = opflag;
2908 	dwl->dw_reqset = domset;
2909 	dwl->dw_idnerr = sep;
2910 	dwl->dw_next = idn.dopers->dop_waitlist;
2911 
2912 	idn.dopers->dop_waitlist = dwl;
2913 	idn.dopers->dop_waitcount++;
2914 	idn.dopers->dop_domset |= domset;
2915 	mutex_exit(&idn.dopers->dop_mutex);
2916 
2917 	return (dwl);
2918 }
2919 
2920 /*
2921  * Anybody waiting on a opflag operation for any one
2922  * of the domains in domset, needs to be updated to
2923  * additionally wait for new domains in domset.
2924  * This is used, for example, when needing to connect
2925  * to more domains than known at the time of the
2926  * original request.
2927  */
2928 void
2929 idn_add_op(idn_opflag_t opflag, domainset_t domset)
2930 {
2931 	dop_waitlist_t	*dwl;
2932 
2933 	mutex_enter(&idn.dopers->dop_mutex);
2934 	if ((idn.dopers->dop_waitcount == 0) ||
2935 	    ((idn.dopers->dop_domset & domset) == 0)) {
2936 		mutex_exit(&idn.dopers->dop_mutex);
2937 		return;
2938 	}
2939 	for (dwl = idn.dopers->dop_waitlist; dwl; dwl = dwl->dw_next)
2940 		if ((dwl->dw_op == opflag) && (dwl->dw_reqset & domset))
2941 			dwl->dw_reqset |= domset;
2942 	mutex_exit(&idn.dopers->dop_mutex);
2943 }
2944 
2945 /*
2946  * Mechanism to wakeup any potential users which may be waiting
2947  * for a link/unlink operation to complete.  If an error occurred
2948  * don't update dop_errset unless there was no previous error.
2949  */
2950 void
2951 idn_update_op(idn_opflag_t opflag, domainset_t domset, idnsb_error_t *sep)
2952 {
2953 	int		do_wakeup = 0;
2954 	dop_waitlist_t	*dw;
2955 	procname_t	proc = "idn_update_op";
2956 
2957 	mutex_enter(&idn.dopers->dop_mutex);
2958 	/*
2959 	 * If there are no waiters, or nobody is waiting for
2960 	 * the particular domainset in question, then
2961 	 * just bail.
2962 	 */
2963 	if ((idn.dopers->dop_waitcount == 0) ||
2964 	    ((idn.dopers->dop_domset & domset) == 0)) {
2965 		mutex_exit(&idn.dopers->dop_mutex);
2966 		PR_PROTO("%s: NO waiters exist (domset=0x%x)\n",
2967 		    proc, domset);
2968 		return;
2969 	}
2970 
2971 	for (dw = idn.dopers->dop_waitlist; dw; dw = dw->dw_next) {
2972 		int		d;
2973 		domainset_t	dset, rset;
2974 
2975 		if ((dset = dw->dw_reqset & domset) == 0)
2976 			continue;
2977 
2978 		if (opflag == IDNOP_ERROR) {
2979 			dw->dw_errset |= dset;
2980 			if (sep) {
2981 				for (d = 0; d < MAX_DOMAINS; d++) {
2982 					if (!DOMAIN_IN_SET(dset, d))
2983 						continue;
2984 
2985 					dw->dw_errors[d] =
2986 					    (short)GET_IDNKERR_ERRNO(sep);
2987 				}
2988 				bcopy(sep, dw->dw_idnerr, sizeof (*sep));
2989 			}
2990 		} else if (opflag == dw->dw_op) {
2991 			dw->dw_domset |= dset;
2992 		}
2993 
2994 		/*
2995 		 * Check if all the domains are spoken for that
2996 		 * a particular waiter may have been waiting for.
2997 		 * If there's at least one, we'll need to broadcast.
2998 		 */
2999 		rset = (dw->dw_errset | dw->dw_domset) & dw->dw_reqset;
3000 		if (rset == dw->dw_reqset)
3001 			do_wakeup++;
3002 	}
3003 
3004 	PR_PROTO("%s: found %d waiters ready for wakeup\n", proc, do_wakeup);
3005 
3006 	if (do_wakeup > 0)
3007 		cv_broadcast(&idn.dopers->dop_cv);
3008 
3009 	mutex_exit(&idn.dopers->dop_mutex);
3010 }
3011 
3012 void
3013 idn_deinit_op(void *cookie)
3014 {
3015 	domainset_t	domset;
3016 	dop_waitlist_t	*hw, *tw;
3017 	dop_waitlist_t	*dwl = (dop_waitlist_t *)cookie;
3018 
3019 	mutex_enter(&idn.dopers->dop_mutex);
3020 
3021 	ASSERT(idn.dopers->dop_waitlist);
3022 
3023 	if (dwl == idn.dopers->dop_waitlist) {
3024 		idn.dopers->dop_waitlist = dwl->dw_next;
3025 		if (IDNOP_IN_CACHE(dwl)) {
3026 			dwl->dw_next = idn.dopers->dop_freelist;
3027 			idn.dopers->dop_freelist = dwl;
3028 		} else {
3029 			FREESTRUCT(dwl, dop_waitlist_t, 1);
3030 		}
3031 	} else {
3032 		for (tw = idn.dopers->dop_waitlist, hw = tw->dw_next;
3033 		    hw;
3034 		    tw = hw, hw = hw->dw_next) {
3035 			if (dwl == hw)
3036 				break;
3037 		}
3038 		ASSERT(hw);
3039 
3040 		tw->dw_next = hw->dw_next;
3041 	}
3042 
3043 	/*
3044 	 * Recompute domainset for which waiters might be waiting.
3045 	 * It's possible there may be other waiters waiting for
3046 	 * the same domainset that the current waiter that's leaving
3047 	 * may have been waiting for, so we can't simply delete
3048 	 * the leaving waiter's domainset from dop_domset.
3049 	 */
3050 	for (hw = idn.dopers->dop_waitlist, domset = 0; hw; hw = hw->dw_next)
3051 		domset |= hw->dw_reqset;
3052 
3053 	idn.dopers->dop_waitcount--;
3054 	idn.dopers->dop_domset = domset;
3055 
3056 	mutex_exit(&idn.dopers->dop_mutex);
3057 }
3058 
3059 /*
3060  * Wait until the specified operation succeeds or fails with
3061  * respect to the given domains.  Note the function terminates
3062  * if at least one error occurs.
3063  * This process is necessary since link/unlink operations occur
3064  * asynchronously and we need some way of waiting to find out
3065  * if it indeed completed.
3066  * Timeout value is received indirectly from the SSP and
3067  * represents seconds.
3068  */
3069 int
3070 idn_wait_op(void *cookie, domainset_t *domsetp, int wait_timeout)
3071 {
3072 	int	d, rv, err = 0;
3073 	dop_waitlist_t	*dwl;
3074 
3075 
3076 	dwl = (dop_waitlist_t *)cookie;
3077 
3078 	ASSERT(wait_timeout > 0);
3079 	ASSERT((dwl->dw_op == IDNOP_CONNECTED) ||
3080 	    (dwl->dw_op == IDNOP_DISCONNECTED));
3081 
3082 	mutex_enter(&idn.dopers->dop_mutex);
3083 
3084 	while (((dwl->dw_domset | dwl->dw_errset) != dwl->dw_reqset) && !err) {
3085 		rv = cv_timedwait_sig(&idn.dopers->dop_cv,
3086 		    &idn.dopers->dop_mutex,
3087 		    lbolt + (wait_timeout * hz));
3088 
3089 		if ((dwl->dw_domset | dwl->dw_errset) == dwl->dw_reqset)
3090 			break;
3091 
3092 		switch (rv) {
3093 		case -1:
3094 			/*
3095 			 * timed out
3096 			 */
3097 			cmn_err(CE_WARN,
3098 			    "!IDN: 129: %s operation timed out",
3099 			    (dwl->dw_op == IDNOP_CONNECTED) ? "LINK" :
3100 			    (dwl->dw_op == IDNOP_DISCONNECTED) ? "UNLINK" :
3101 			    "UNKNOWN");
3102 			/*FALLTHROUGH*/
3103 		case 0:
3104 			/*
3105 			 * signal, e.g. kill(2)
3106 			 */
3107 			err = 1;
3108 			break;
3109 
3110 		default:
3111 			break;
3112 		}
3113 	}
3114 
3115 	if (dwl->dw_domset == dwl->dw_reqset) {
3116 		rv = 0;
3117 	} else {
3118 		/*
3119 		 * Op failed for some domains or we were awakened.
3120 		 */
3121 		for (d = rv = 0; (d < MAX_DOMAINS) && !rv; d++)
3122 			rv = dwl->dw_errors[d];
3123 	}
3124 	*domsetp = dwl->dw_domset;
3125 
3126 	mutex_exit(&idn.dopers->dop_mutex);
3127 
3128 	idn_deinit_op(cookie);
3129 
3130 	return (rv);
3131 }
3132 
3133 /*
3134  * --------------------------------------------------
3135  * Return any valid (& ready) cpuid for the given board based on
3136  * the given cpuset.
3137  * --------------------------------------------------
3138  */
3139 int
3140 board_to_ready_cpu(int board, cpuset_t cpuset)
3141 {
3142 	int	base_cpuid;
3143 	int	ncpu_board = MAX_CPU_PER_BRD;
3144 
3145 	board *= ncpu_board;
3146 	for (base_cpuid = board;
3147 	    base_cpuid < (board + ncpu_board);
3148 	    base_cpuid++)
3149 		if (CPU_IN_SET(cpuset, base_cpuid))
3150 			return (base_cpuid);
3151 
3152 	return (-1);
3153 }
3154 
3155 void
3156 idn_domain_resetentry(idn_domain_t *dp)
3157 {
3158 	register int		i;
3159 	procname_t	proc = "idn_domain_resetentry";
3160 
3161 	ASSERT(dp);
3162 	ASSERT(dp->dstate == IDNDS_CLOSED);
3163 	ASSERT(IDN_DLOCK_IS_EXCL(dp->domid));
3164 	ASSERT(IDN_GLOCK_IS_EXCL());
3165 
3166 	ASSERT(dp->domid == (dp - &idn_domain[0]));
3167 
3168 	IDN_FSTATE_TRANSITION(dp, IDNFIN_OFF);
3169 	dp->dname[0]	= '\0';
3170 	dp->dnetid	= (ushort_t)-1;
3171 	dp->dmtu	= 0;
3172 	dp->dbufsize	= 0;
3173 	dp->dslabsize	= 0;
3174 	dp->dnwrsize	= 0;
3175 	dp->dncpus	= 0;
3176 	dp->dcpuindex   = 0;
3177 	CPUSET_ZERO(dp->dcpuset);
3178 	dp->dcpu	= dp->dcpu_last = dp->dcpu_save = IDN_NIL_DCPU;
3179 	dp->dvote.ticket = 0;
3180 	dp->dslab	= NULL;
3181 	dp->dslab_state = DSLAB_STATE_UNKNOWN;
3182 	dp->dnslabs	= 0;
3183 	dp->dio		= 0;
3184 	dp->dioerr	= 0;
3185 	lock_clear(&dp->diowanted);
3186 	bzero(&dp->dhw, sizeof (dp->dhw));
3187 	dp->dxp		= NULL;
3188 	IDN_XSTATE_TRANSITION(dp, IDNXS_NIL);
3189 	dp->dsync.s_cmd = IDNSYNC_NIL;
3190 	dp->dfin_sync   = IDNFIN_SYNC_OFF;
3191 	IDN_RESET_COOKIES(dp->domid);
3192 	dp->dcookie_err = 0;
3193 	bzero(&dp->dawol, sizeof (dp->dawol));
3194 	dp->dtmp = -1;
3195 
3196 	if (dp->dtimerq.tq_queue != NULL) {
3197 		PR_PROTO("%s: WARNING: MSG timerq not empty (count = %d)\n",
3198 		    proc, dp->dtimerq.tq_count);
3199 		IDN_MSGTIMER_STOP(dp->domid, 0, 0);
3200 	}
3201 
3202 	for (i = 0; i < NCPU; i++)
3203 		dp->dcpumap[i] = (uchar_t)-1;
3204 }
3205 
3206 int
3207 idn_open_domain(int domid, int cpuid, uint_t ticket)
3208 {
3209 	int		c, new_cpuid;
3210 	idn_domain_t	*dp, *ldp;
3211 	procname_t	proc = "idn_open_domain";
3212 
3213 	ASSERT(IDN_SYNC_IS_LOCKED());
3214 	ASSERT(IDN_DLOCK_IS_EXCL(domid));
3215 
3216 	if (!VALID_DOMAINID(domid)) {
3217 		PR_PROTO("%s: INVALID domainid (%d) "
3218 		    "[cpuid = %d, ticket = 0x%x]\n",
3219 		    proc, domid, cpuid, ticket);
3220 		return (-1);
3221 	}
3222 
3223 	dp = &idn_domain[domid];
3224 	ldp = &idn_domain[idn.localid];
3225 
3226 	if (dp->dcpu >= 0) {
3227 		PR_PROTO("%s:%d: domain already OPEN (state = %s)\n",
3228 		    proc, domid, idnds_str[dp->dstate]);
3229 		return (1);
3230 	}
3231 
3232 	if (DOMAIN_IN_SET(idn.domset.ds_relink, domid)) {
3233 		if (dp->dcpu_save == IDN_NIL_DCPU)
3234 			new_cpuid = cpuid;
3235 		else
3236 			new_cpuid = dp->dcpu_save;
3237 	} else {
3238 		new_cpuid = cpuid;
3239 	}
3240 
3241 	if (new_cpuid == IDN_NIL_DCPU) {
3242 		PR_PROTO("%s:%d: WARNING: invalid cpuid (%d) specified\n",
3243 		    proc, domid, new_cpuid);
3244 		return (-1);
3245 	}
3246 
3247 	IDN_GLOCK_EXCL();
3248 
3249 	idn_domain_resetentry(dp);
3250 
3251 	PR_STATE("%s:%d: requested cpuid %d, assigning cpuid %d\n",
3252 	    proc, domid, cpuid, new_cpuid);
3253 
3254 	idn_assign_cookie(domid);
3255 
3256 	dp->dcpu = dp->dcpu_save = new_cpuid;
3257 	dp->dvote.ticket = ticket;
3258 	CPUSET_ADD(dp->dcpuset, new_cpuid);
3259 	dp->dncpus = 1;
3260 	for (c = 0; c < NCPU; c++)
3261 		dp->dcpumap[c] = (uchar_t)new_cpuid;
3262 	dp->dhw.dh_nboards = 1;
3263 	dp->dhw.dh_boardset = BOARDSET(CPUID_TO_BOARDID(new_cpuid));
3264 
3265 	if (domid != idn.localid)
3266 		IDN_DLOCK_EXCL(idn.localid);
3267 
3268 	if (idn.ndomains == 1) {
3269 		struct hwconfig	local_hw;
3270 
3271 		/*
3272 		 * We're attempting to connect to our first domain.
3273 		 * Recheck our local hardware configuration before
3274 		 * we go any further in case it changed due to a DR,
3275 		 * and update any structs dependent on this.
3276 		 * ASSUMPTION:
3277 		 *	IDN is unlinked before performing any DRs.
3278 		 */
3279 		PR_PROTO("%s: RECHECKING local HW config.\n", proc);
3280 		if (get_hw_config(&local_hw)) {
3281 			dp->dcpu = IDN_NIL_DCPU;
3282 			cmn_err(CE_WARN,
3283 			    "IDN: 118: hardware config not appropriate");
3284 			if (domid != idn.localid)
3285 				IDN_DUNLOCK(idn.localid);
3286 			IDN_GUNLOCK();
3287 			return (-1);
3288 		}
3289 		(void) update_local_hw_config(ldp, &local_hw);
3290 	}
3291 
3292 	idn.ndomains++;
3293 
3294 	if (domid != idn.localid)
3295 		IDN_DUNLOCK(idn.localid);
3296 	IDN_GUNLOCK();
3297 
3298 	IDN_MBOX_LOCK(domid);
3299 	dp->dmbox.m_tbl = NULL;
3300 
3301 	if (domid != idn.localid) {
3302 		dp->dmbox.m_send = idn_mainmbox_init(domid,
3303 		    IDNMMBOX_TYPE_SEND);
3304 		dp->dmbox.m_recv = idn_mainmbox_init(domid,
3305 		    IDNMMBOX_TYPE_RECV);
3306 	} else {
3307 		/*
3308 		 * The local domain does not need send/recv
3309 		 * mailboxes in its idn_domain[] entry.
3310 		 */
3311 		dp->dmbox.m_send = NULL;
3312 		dp->dmbox.m_recv = NULL;
3313 	}
3314 	IDN_MBOX_UNLOCK(domid);
3315 
3316 	PR_PROTO("%s:%d: new domain (cpu = %d, vote = 0x%x)\n",
3317 	    proc, domid, dp->dcpu, dp->dvote.ticket);
3318 
3319 	return (0);
3320 }
3321 
3322 /*
3323  * The local domain never "closes" itself unless the driver
3324  * is doing a idndetach.  It will be reopened during idnattach
3325  * when idn_domains_init is called.
3326  */
3327 void
3328 idn_close_domain(int domid)
3329 {
3330 	uint_t		token;
3331 	idn_domain_t	*dp;
3332 	procname_t	proc = "idn_close_domain";
3333 
3334 	ASSERT(IDN_SYNC_IS_LOCKED());
3335 	ASSERT(IDN_DLOCK_IS_EXCL(domid));
3336 
3337 	dp = &idn_domain[domid];
3338 
3339 	ASSERT(dp->dstate == IDNDS_CLOSED);
3340 
3341 	if (dp->dcpu == IDN_NIL_DCPU) {
3342 		PR_PROTO("%s:%d: DOMAIN ALREADY CLOSED!\n",
3343 		    proc, domid);
3344 		return;
3345 	}
3346 
3347 	token = IDN_RETRY_TOKEN(domid, IDN_RETRY_TYPEALL);
3348 
3349 	(void) idn_retry_terminate(token);
3350 
3351 	DOMAINSET_DEL(idn.domset.ds_trans_on, domid);
3352 	DOMAINSET_DEL(idn.domset.ds_ready_on, domid);
3353 	DOMAINSET_DEL(idn.domset.ds_connected, domid);
3354 	DOMAINSET_DEL(idn.domset.ds_trans_off, domid);
3355 	DOMAINSET_DEL(idn.domset.ds_ready_off, domid);
3356 	DOMAINSET_DEL(idn.domset.ds_hwlinked, domid);
3357 	DOMAINSET_DEL(idn.domset.ds_flush, domid);
3358 
3359 	idn_sync_exit(domid, IDNSYNC_CONNECT);
3360 	idn_sync_exit(domid, IDNSYNC_DISCONNECT);
3361 
3362 	IDN_GLOCK_EXCL();
3363 
3364 	if (DOMAIN_IN_SET(idn.domset.ds_awol, domid))
3365 		idn_clear_awol(domid);
3366 
3367 	idn.ndomains--;
3368 
3369 	IDN_GUNLOCK();
3370 
3371 	IDN_MBOX_LOCK(domid);
3372 	dp->dmbox.m_tbl = NULL;
3373 
3374 	if (dp->dmbox.m_send)  {
3375 		idn_mainmbox_deinit(domid, dp->dmbox.m_send);
3376 		dp->dmbox.m_send = NULL;
3377 	}
3378 
3379 	if (dp->dmbox.m_recv) {
3380 		idn_mainmbox_deinit(domid, dp->dmbox.m_recv);
3381 		dp->dmbox.m_recv = NULL;
3382 	}
3383 
3384 	IDN_MBOX_UNLOCK(domid);
3385 
3386 	cmn_err(CE_NOTE,
3387 	    "!IDN: 142: link (domain %d, CPU %d) disconnected",
3388 	    dp->domid, dp->dcpu);
3389 
3390 	dp->dcpu = IDN_NIL_DCPU;	/* ultimate demise */
3391 
3392 	IDN_RESET_COOKIES(domid);
3393 
3394 	ASSERT(dp->dio <= 0);
3395 	ASSERT(dp->dioerr == 0);
3396 	ASSERT(dp->dslab == NULL);
3397 	ASSERT(dp->dnslabs == 0);
3398 
3399 	IDN_GKSTAT_GLOBAL_EVENT(gk_unlinks, gk_unlink_last);
3400 }
3401 
3402 
3403 /*
3404  * -----------------------------------------------------------------------
3405  */
3406 static void
3407 idn_domains_init(struct hwconfig *local_hw)
3408 {
3409 	register int		i, d;
3410 	idn_domain_t		*ldp;
3411 	uchar_t			*cpumap;
3412 
3413 	ASSERT(local_hw != NULL);
3414 
3415 	cpumap = GETSTRUCT(uchar_t, NCPU * MAX_DOMAINS);
3416 
3417 	for (d = 0; d < MAX_DOMAINS; d++) {
3418 		register idn_domain_t	*dp;
3419 
3420 		dp = &idn_domain[d];
3421 
3422 		dp->domid = d;
3423 
3424 		rw_init(&dp->drwlock, NULL, RW_DEFAULT, NULL);
3425 
3426 		IDN_TIMERQ_INIT(&dp->dtimerq);
3427 
3428 		dp->dstate = IDNDS_CLOSED;
3429 
3430 		mutex_init(&dp->dmbox.m_mutex, NULL, MUTEX_DEFAULT, NULL);
3431 
3432 		dp->dcpumap = cpumap;
3433 
3434 		rw_init(&dp->dslab_rwlock, NULL, RW_DEFAULT, NULL);
3435 
3436 		IDN_DLOCK_EXCL(d);
3437 		IDN_GLOCK_EXCL();
3438 
3439 		idn_domain_resetentry(dp);
3440 
3441 		IDN_GUNLOCK();
3442 
3443 		IDNSB_DOMAIN_UPDATE(dp);
3444 
3445 		IDN_DUNLOCK(d);
3446 
3447 		cpumap += NCPU;
3448 	}
3449 
3450 	IDN_SYNC_LOCK();
3451 
3452 	/*
3453 	 * Update local domain information.
3454 	 */
3455 	ASSERT(idn.smr.locpfn);
3456 	ASSERT(local_hw->dh_nboards && local_hw->dh_boardset);
3457 
3458 	idn.ndomains = 0;	/* note that open_domain will get us to 1 */
3459 
3460 	IDN_DLOCK_EXCL(idn.localid);
3461 	d = idn_open_domain(idn.localid, (int)CPU->cpu_id, 0);
3462 	ASSERT(d == 0);
3463 	IDN_GLOCK_EXCL();
3464 	IDN_SET_MASTERID(IDN_NIL_DOMID);
3465 	IDN_SET_NEW_MASTERID(IDN_NIL_DOMID);
3466 
3467 	ldp = &idn_domain[idn.localid];
3468 
3469 	strncpy(ldp->dname, utsname.nodename, MAXDNAME - 1);
3470 	ldp->dname[MAXDNAME-1] = '\0';
3471 	bcopy(local_hw, &ldp->dhw, sizeof (ldp->dhw));
3472 	ASSERT(idn.ndomains == 1);
3473 	ASSERT((ldp->dhw.dh_nboards > 0) &&
3474 	    (ldp->dhw.dh_nboards <= MAX_BOARDS));
3475 	ldp->dnetid	= IDN_DOMID2NETID(ldp->domid);
3476 	ldp->dmtu	= IDN_MTU;
3477 	ldp->dbufsize	= IDN_SMR_BUFSIZE;
3478 	ldp->dslabsize	= (short)IDN_SLAB_BUFCOUNT;
3479 	ldp->dnwrsize	= (short)IDN_NWR_SIZE;
3480 	ldp->dcpuset	= cpu_ready_set;
3481 	ldp->dncpus	= (short)ncpus;
3482 	ldp->dvote.ticket	= IDNVOTE_INITIAL_TICKET;
3483 	ldp->dvote.v.master	= 0;
3484 	ldp->dvote.v.nmembrds	= ldp->dhw.dh_nmcadr - 1;
3485 	ldp->dvote.v.ncpus	= (int)ldp->dncpus - 1;
3486 	ldp->dvote.v.board	= CPUID_TO_BOARDID(ldp->dcpu);
3487 	i = -1;
3488 	for (d = 0; d < NCPU; d++) {
3489 		BUMP_INDEX(ldp->dcpuset, i);
3490 		ldp->dcpumap[d] = (uchar_t)i;
3491 	}
3492 
3493 	CPUSET_ZERO(idn.dc_cpuset);
3494 	CPUSET_OR(idn.dc_cpuset, ldp->dcpuset);
3495 	idn.dc_boardset = ldp->dhw.dh_boardset;
3496 
3497 	/*
3498 	 * Setting the state for ourselves is only relevant
3499 	 * for loopback performance testing.  Anyway, it
3500 	 * makes sense that we always have an established
3501 	 * connection with ourself regardless of IDN :-o
3502 	 */
3503 	IDN_DSTATE_TRANSITION(ldp, IDNDS_CONNECTED);
3504 
3505 	IDN_GUNLOCK();
3506 	IDN_DUNLOCK(idn.localid);
3507 	IDN_SYNC_UNLOCK();
3508 }
3509 
3510 static void
3511 idn_domains_deinit()
3512 {
3513 	register int	d;
3514 
3515 	IDN_SYNC_LOCK();
3516 	IDN_DLOCK_EXCL(idn.localid);
3517 	IDN_DSTATE_TRANSITION(&idn_domain[idn.localid], IDNDS_CLOSED);
3518 	idn_close_domain(idn.localid);
3519 	IDN_DUNLOCK(idn.localid);
3520 	IDN_SYNC_UNLOCK();
3521 	idn.localid = IDN_NIL_DOMID;
3522 
3523 	FREESTRUCT(idn_domain[0].dcpumap, uchar_t, NCPU * MAX_DOMAINS);
3524 
3525 	for (d = 0; d < MAX_DOMAINS; d++) {
3526 		idn_domain_t	*dp;
3527 
3528 		dp = &idn_domain[d];
3529 
3530 		rw_destroy(&dp->dslab_rwlock);
3531 		mutex_destroy(&dp->dmbox.m_mutex);
3532 		rw_destroy(&dp->drwlock);
3533 		IDN_TIMERQ_DEINIT(&dp->dtimerq);
3534 		dp->dcpumap = NULL;
3535 	}
3536 }
3537 
3538 /*
3539  * -----------------------------------------------------------------------
3540  */
3541 static void
3542 idn_retrytask_init()
3543 {
3544 	ASSERT(idn.retryqueue.rq_cache == NULL);
3545 
3546 	mutex_init(&idn.retryqueue.rq_mutex, NULL, MUTEX_DEFAULT, NULL);
3547 	idn.retryqueue.rq_cache = kmem_cache_create("idn_retryjob_cache",
3548 	    sizeof (idn_retry_job_t),
3549 	    0, NULL, NULL, NULL,
3550 	    NULL, NULL, 0);
3551 }
3552 
3553 static void
3554 idn_retrytask_deinit()
3555 {
3556 	if (idn.retryqueue.rq_cache == NULL)
3557 		return;
3558 
3559 	kmem_cache_destroy(idn.retryqueue.rq_cache);
3560 	mutex_destroy(&idn.retryqueue.rq_mutex);
3561 
3562 	bzero(&idn.retryqueue, sizeof (idn.retryqueue));
3563 }
3564 
3565 /*
3566  * -----------------------------------------------------------------------
3567  */
3568 static void
3569 idn_timercache_init()
3570 {
3571 	ASSERT(idn.timer_cache == NULL);
3572 
3573 	idn.timer_cache = kmem_cache_create("idn_timer_cache",
3574 	    sizeof (idn_timer_t),
3575 	    0, NULL, NULL, NULL,
3576 	    NULL, NULL, 0);
3577 }
3578 
3579 static void
3580 idn_timercache_deinit()
3581 {
3582 	if (idn.timer_cache == NULL)
3583 		return;
3584 
3585 	kmem_cache_destroy(idn.timer_cache);
3586 	idn.timer_cache = NULL;
3587 }
3588 
3589 idn_timer_t *
3590 idn_timer_alloc()
3591 {
3592 	idn_timer_t	*tp;
3593 
3594 	tp = kmem_cache_alloc(idn.timer_cache, KM_SLEEP);
3595 	bzero(tp, sizeof (*tp));
3596 	tp->t_forw = tp->t_back = tp;
3597 
3598 	return (tp);
3599 }
3600 
3601 void
3602 idn_timer_free(idn_timer_t *tp)
3603 {
3604 	if (tp == NULL)
3605 		return;
3606 	kmem_cache_free(idn.timer_cache, tp);
3607 }
3608 
3609 void
3610 idn_timerq_init(idn_timerq_t *tq)
3611 {
3612 	mutex_init(&tq->tq_mutex, NULL, MUTEX_DEFAULT, NULL);
3613 	tq->tq_count = 0;
3614 	tq->tq_queue = NULL;
3615 }
3616 
3617 void
3618 idn_timerq_deinit(idn_timerq_t *tq)
3619 {
3620 	ASSERT(tq->tq_queue == NULL);
3621 	mutex_destroy(&tq->tq_mutex);
3622 }
3623 
3624 /*
3625  * Dequeue all the timers of the given subtype from the
3626  * given timerQ.  If subtype is 0, then dequeue all the
3627  * timers.
3628  */
3629 idn_timer_t *
3630 idn_timer_get(idn_timerq_t *tq, int type, ushort_t tcookie)
3631 {
3632 	register idn_timer_t	*tp, *tphead;
3633 
3634 	ASSERT(IDN_TIMERQ_IS_LOCKED(tq));
3635 
3636 	if ((tp = tq->tq_queue) == NULL)
3637 		return (NULL);
3638 
3639 	if (!type) {
3640 		tq->tq_queue = NULL;
3641 		tq->tq_count = 0;
3642 		tphead = tp;
3643 	} else {
3644 		int		count;
3645 		idn_timer_t	*tpnext;
3646 
3647 		tphead = NULL;
3648 		count = tq->tq_count;
3649 		do {
3650 			tpnext = tp->t_forw;
3651 			if ((tp->t_type == type) &&
3652 			    (!tcookie || (tp->t_cookie == tcookie))) {
3653 				tp->t_forw->t_back = tp->t_back;
3654 				tp->t_back->t_forw = tp->t_forw;
3655 				if (tphead == NULL) {
3656 					tp->t_forw = tp->t_back = tp;
3657 				} else {
3658 					tp->t_forw = tphead;
3659 					tp->t_back = tphead->t_back;
3660 					tp->t_back->t_forw = tp;
3661 					tphead->t_back = tp;
3662 				}
3663 				tphead = tp;
3664 				if (--(tq->tq_count) == 0)
3665 					tq->tq_queue = NULL;
3666 				else if (tq->tq_queue == tp)
3667 					tq->tq_queue = tpnext;
3668 			}
3669 			tp = tpnext;
3670 		} while (--count > 0);
3671 	}
3672 
3673 	if (tphead) {
3674 		tphead->t_back->t_forw = NULL;
3675 
3676 		for (tp = tphead; tp; tp = tp->t_forw)
3677 			tp->t_onq = 0;
3678 	}
3679 
3680 	return (tphead);
3681 }
3682 
3683 ushort_t
3684 idn_timer_start(idn_timerq_t *tq, idn_timer_t *tp, clock_t tval)
3685 {
3686 	idn_timer_t	*otp;
3687 	ushort_t	tcookie;
3688 	procname_t	proc = "idn_timer_start";
3689 	STRING(str);
3690 
3691 	ASSERT(tq && tp && (tval > 0));
3692 	ASSERT((tp->t_forw == tp) && (tp->t_back == tp));
3693 	ASSERT(tp->t_type != 0);
3694 
3695 	IDN_TIMERQ_LOCK(tq);
3696 	/*
3697 	 * Assign a unique non-zero 8-bit cookie to this timer
3698 	 * if the caller hasn't already preassigned one.
3699 	 */
3700 	while ((tcookie = tp->t_cookie) == 0) {
3701 		tp->t_cookie = (tp->t_type & 0xf) |
3702 		    ((++tq->tq_cookie & 0xf) << 4);
3703 		/*
3704 		 * Calculated cookie must never conflict
3705 		 * with the public timer cookie.
3706 		 */
3707 		ASSERT(tp->t_cookie != IDN_TIMER_PUBLIC_COOKIE);
3708 	}
3709 
3710 	/*
3711 	 * First have to remove old timers of the
3712 	 * same type and cookie, and get rid of them.
3713 	 */
3714 	otp = idn_timer_get(tq, tp->t_type, tcookie);
3715 
3716 	tq->tq_count++;
3717 
3718 	if (tq->tq_queue == NULL) {
3719 		tq->tq_queue = tp;
3720 		ASSERT((tp->t_forw == tp) && (tp->t_back == tp));
3721 	} else {
3722 		/*
3723 		 * Put me at the end of the list.
3724 		 */
3725 		tp->t_forw = tq->tq_queue;
3726 		tp->t_back = tq->tq_queue->t_back;
3727 		tp->t_back->t_forw = tp;
3728 		tp->t_forw->t_back = tp;
3729 	}
3730 
3731 	tp->t_onq = 1;
3732 	tp->t_q = tq;
3733 	tp->t_id = timeout(idn_timer_expired, (caddr_t)tp, tval);
3734 
3735 
3736 	INUM2STR(tp->t_type, str);
3737 	PR_TIMER("%s: started %s timer (domain = %d, cookie = 0x%x)\n",
3738 	    proc, str, tp->t_domid, tcookie);
3739 
3740 	IDN_TIMERQ_UNLOCK(tq);
3741 
3742 	if (otp)
3743 		(void) idn_timer_stopall(otp);
3744 
3745 	return (tcookie);
3746 }
3747 
3748 /*
3749  * Stop all timers of the given subtype.
3750  * If subtype is 0, then stop all timers
3751  * in this timerQ.
3752  */
3753 void
3754 idn_timer_stop(idn_timerq_t *tq, int type, ushort_t tcookie)
3755 {
3756 	idn_timer_t	*tphead;
3757 	procname_t	proc = "idn_timer_stop";
3758 	STRING(str);
3759 
3760 	ASSERT(tq);
3761 
3762 	INUM2STR(type, str);
3763 
3764 	IDN_TIMERQ_LOCK(tq);
3765 
3766 	if (tq->tq_count == 0) {
3767 		PR_TIMER("%s: found no %s timers (count=0)\n", proc, str);
3768 		IDN_TIMERQ_UNLOCK(tq);
3769 		return;
3770 	}
3771 	tphead = idn_timer_get(tq, type, tcookie);
3772 #ifdef DEBUG
3773 	if (tphead == NULL)
3774 		PR_TIMER("%s: found no %s (cookie = 0x%x) "
3775 		    "timers (count=%d)!!\n",
3776 		    proc, str, tcookie, tq->tq_count);
3777 #endif /* DEBUG */
3778 	IDN_TIMERQ_UNLOCK(tq);
3779 
3780 	if (tphead)
3781 		(void) idn_timer_stopall(tphead);
3782 }
3783 
3784 int
3785 idn_timer_stopall(idn_timer_t *tp)
3786 {
3787 	int		count = 0;
3788 	int		nonactive;
3789 	uint_t		type;
3790 	idn_timer_t	*ntp;
3791 	procname_t	proc = "idn_timer_stopall";
3792 	STRING(str);
3793 
3794 	nonactive = 0;
3795 
3796 	if (tp) {
3797 		/*
3798 		 * Circle should have been broken.
3799 		 */
3800 		ASSERT(tp->t_back->t_forw == NULL);
3801 		type = tp->t_type;
3802 		INUM2STR(type, str);
3803 	}
3804 
3805 	for (; tp; tp = ntp) {
3806 		ntp = tp->t_forw;
3807 		count++;
3808 		ASSERT(tp->t_id != (timeout_id_t)0);
3809 		if (untimeout(tp->t_id) < 0) {
3810 			nonactive++;
3811 			PR_TIMER("%s: bad %s untimeout (domain=%d)\n",
3812 			    proc, str, tp->t_domid);
3813 		} else {
3814 			PR_TIMER("%s: good %s untimeout (domain=%d)\n",
3815 			    proc, str, tp->t_domid);
3816 		}
3817 		/*
3818 		 * There are two possible outcomes from
3819 		 * the untimeout().  Each ultimately result
3820 		 * in us having to free the timeout structure.
3821 		 *
3822 		 * 1. We successfully aborted a timeout call.
3823 		 *
3824 		 * 2. We failed to find the given timer.  He
3825 		 *    probably just fired off.
3826 		 */
3827 		idn_timer_free(tp);
3828 	}
3829 	PR_TIMER("%s: stopped %d of %d %s timers\n",
3830 	    proc, count - nonactive, count, str);
3831 
3832 	return (count);
3833 }
3834 
3835 void
3836 idn_timer_dequeue(idn_timerq_t *tq, idn_timer_t *tp)
3837 {
3838 	ASSERT(tq && tp);
3839 	ASSERT(IDN_TIMERQ_IS_LOCKED(tq));
3840 
3841 	ASSERT(tp->t_q == tq);
3842 
3843 	if (tp->t_onq == 0) {
3844 		/*
3845 		 * We've already been dequeued.
3846 		 */
3847 		ASSERT(tp == tp->t_forw);
3848 		ASSERT(tp == tp->t_back);
3849 	} else {
3850 		/*
3851 		 * We're still in the queue, get out.
3852 		 */
3853 		if (tq->tq_queue == tp)
3854 			tq->tq_queue = tp->t_forw;
3855 		tp->t_forw->t_back = tp->t_back;
3856 		tp->t_back->t_forw = tp->t_forw;
3857 		tp->t_onq = 0;
3858 		if (--(tq->tq_count) == 0) {
3859 			ASSERT(tq->tq_queue == tp);
3860 			tq->tq_queue = NULL;
3861 		}
3862 		tp->t_forw = tp->t_back = tp;
3863 	}
3864 }
3865 
3866 /*
3867  * -----------------------------------------------------------------------
3868  */
3869 /*ARGSUSED*/
3870 static int
3871 idn_slabpool_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
3872 {
3873 	register int	p, nfree;
3874 	char		dsetstr[128];
3875 
3876 	ASSERT(IDN_GLOCK_IS_HELD());
3877 
3878 	if (idn.slabpool == NULL) {
3879 		(void) mi_mpprintf(mp,
3880 		    "IDN slabpool not initialized (masterid = %d)",
3881 		    IDN_GET_MASTERID());
3882 		return (0);
3883 	}
3884 
3885 	for (p = nfree = 0; p < idn.slabpool->npools; p++)
3886 		nfree += idn.slabpool->pool[p].nfree;
3887 
3888 	(void) mi_mpprintf(mp,
3889 	    "IDN slabpool (ntotal_slabs = %d, nalloc = %d, "
3890 	    "npools = %d)",
3891 	    idn.slabpool->ntotslabs,
3892 	    idn.slabpool->ntotslabs - nfree,
3893 	    idn.slabpool->npools);
3894 
3895 	(void) mi_mpprintf(mp, "pool  nslabs  nfree domains");
3896 
3897 	for (p = 0; p < idn.slabpool->npools; p++) {
3898 		register int	d, s;
3899 		uint_t		domset;
3900 
3901 		domset = 0;
3902 		for (s = 0; s < idn.slabpool->pool[p].nslabs; s++) {
3903 			short	dd;
3904 
3905 			dd = idn.slabpool->pool[p].sarray[s].sl_domid;
3906 			if (dd != (short)IDN_NIL_DOMID)
3907 				DOMAINSET_ADD(domset, dd);
3908 		}
3909 		dsetstr[0] = '\0';
3910 		if (domset) {
3911 			for (d = 0; d < MAX_DOMAINS; d++) {
3912 				if (!DOMAIN_IN_SET(domset, d))
3913 					continue;
3914 
3915 				if (dsetstr[0] == '\0')
3916 					(void) sprintf(dsetstr, "%d", d);
3917 				else
3918 					(void) sprintf(dsetstr, "%s %d",
3919 					    dsetstr, d);
3920 			}
3921 		}
3922 
3923 		if (p < 10)
3924 			(void) mi_mpprintf(mp, "  %d     %d       %d    %s",
3925 			    p, idn.slabpool->pool[p].nslabs,
3926 			    idn.slabpool->pool[p].nfree,
3927 			    dsetstr);
3928 		else
3929 			(void) mi_mpprintf(mp, " %d     %d       %d    %s",
3930 			    p, idn.slabpool->pool[p].nslabs,
3931 			    idn.slabpool->pool[p].nfree,
3932 			    dsetstr);
3933 	}
3934 	return (0);
3935 }
3936 
3937 /*ARGSUSED*/
3938 static int
3939 idn_buffer_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
3940 {
3941 	smr_slab_t	*sp;
3942 	register int	d, cnt;
3943 	int		bufcount[MAX_DOMAINS];
3944 	int		spl;
3945 
3946 	ASSERT(IDN_GLOCK_IS_HELD());
3947 
3948 	if (idn.localid == IDN_NIL_DOMID) {
3949 		(void) mi_mpprintf(mp, "IDN not initialized (localid = %d)",
3950 		    idn.localid);
3951 		return (0);
3952 	}
3953 
3954 	(void) mi_mpprintf(mp, "Local domain has %d slabs allocated.",
3955 	    idn_domain[idn.localid].dnslabs);
3956 
3957 	DSLAB_LOCK_SHARED(idn.localid);
3958 	if ((sp = idn_domain[idn.localid].dslab) == NULL) {
3959 		DSLAB_UNLOCK(idn.localid);
3960 		return (0);
3961 	}
3962 
3963 	bzero(bufcount, sizeof (bufcount));
3964 	cnt = 0;
3965 
3966 	spl = splhi();
3967 	for (; sp; sp = sp->sl_next) {
3968 		smr_slabbuf_t	*bp;
3969 
3970 		while (!lock_try(&sp->sl_lock))
3971 			;
3972 		for (bp = sp->sl_inuse; bp; bp = bp->sb_next) {
3973 			bufcount[bp->sb_domid]++;
3974 			cnt++;
3975 		}
3976 		lock_clear(&sp->sl_lock);
3977 	}
3978 	splx(spl);
3979 
3980 	DSLAB_UNLOCK(idn.localid);
3981 
3982 	(void) mi_mpprintf(mp, "Local domain has %d buffers outstanding.", cnt);
3983 	if (cnt == 0)
3984 		return (0);
3985 
3986 	(void) mi_mpprintf(mp, "Domain  nbufs");
3987 	for (d = 0; d < MAX_DOMAINS; d++)
3988 		if (bufcount[d]) {
3989 			if (d < 10)
3990 				(void) mi_mpprintf(mp, "   %d      %d",
3991 				    d, bufcount[d]);
3992 			else
3993 				(void) mi_mpprintf(mp, "  %d      %d",
3994 				    d, bufcount[d]);
3995 		}
3996 
3997 	return (0);
3998 }
3999 
4000 static const char *
4001 _get_spaces(int w, int s, int W)
4002 {
4003 	static const char *const _spaces[] = {
4004 		"",			/* 0 */
4005 		" ",			/* 1 */
4006 		"  ", 			/* 2 */
4007 		"   ", 			/* 3 */
4008 		"    ", 		/* 4 */
4009 		"     ", 		/* 5 */
4010 		"      ", 		/* 6 */
4011 		"       ", 		/* 7 */
4012 		"        ", 		/* 8 */
4013 		"         ", 		/* 9 */
4014 		"          ", 		/* 10 */
4015 		"           ", 		/* 11 */
4016 		"            ", 	/* 12 */
4017 		"             ", 	/* 13 */
4018 		"              ", 	/* 14 */
4019 		"               ", 	/* 15 */
4020 		"                ", 	/* 16 */
4021 		"                 ", 	/* 17 */
4022 		"                  ", 	/* 18 */
4023 		"                   ", 	/* 19 */
4024 	};
4025 	return (_spaces[w+s-W]);
4026 }
4027 
4028 #define	_SSS(X, W, w, s) \
4029 	(((w) >= (W)) && (X)) ? _get_spaces((w), (s), (W))
4030 
4031 static const char *
4032 _hexspace(uint64_t v, int sz, int width, int padding)
4033 {
4034 	int	maxnbl = 16;
4035 	int	diff;
4036 	uchar_t	*np;
4037 
4038 	diff = sizeof (uint64_t) - sz;
4039 	np = (uchar_t *)&v + diff;
4040 	maxnbl -= diff << 1;
4041 	while (sz-- > 0) {
4042 		if ((*np & 0xf0) && (width >= maxnbl))
4043 			return (_get_spaces(width, padding, maxnbl));
4044 		maxnbl--;
4045 		if ((*np & 0x0f) && (width >= maxnbl))
4046 			return (_get_spaces(width, padding, maxnbl));
4047 		maxnbl--;
4048 		np++;
4049 	}
4050 	return (_get_spaces(width, padding, 1));
4051 }
4052 
4053 #define	HEXSPACE(v, t, w, s)	_hexspace((uint64_t)(v), sizeof (t), (w), (s))
4054 
4055 #define	DECSPACE(n, w, s) \
4056 	(_SSS((uint_t)(n) >= 10000000, 8, (w), (s)) : \
4057 	_SSS((uint_t)(n) >= 1000000, 7, (w), (s)) : \
4058 	_SSS((uint_t)(n) >= 100000, 6, (w), (s)) : \
4059 	_SSS((uint_t)(n) >= 10000, 5, (w), (s)) : \
4060 	_SSS((uint_t)(n) >= 1000, 4, (w), (s)) : \
4061 	_SSS((uint_t)(n) >= 100, 3, (w), (s)) : \
4062 	_SSS((uint_t)(n) >= 10, 2, (w), (s)) : \
4063 	_get_spaces((w), (s), 1))
4064 
4065 #define	DECSPACE16(n, w, s) \
4066 	(_SSS((n) >= 10000, 5, (w), (s)) : \
4067 	_SSS((n) >= 1000, 4, (w), (s)) : \
4068 	_SSS((n) >= 100, 3, (w), (s)) : \
4069 	_SSS((n) >= 10, 2, (w), (s)) : \
4070 	_get_spaces((w), (s), 1))
4071 
4072 #define	MBXINFO(mtp) \
4073 	&mtp->mt_header, \
4074 		HEXSPACE(&mtp->mt_header, &mtp->mt_header, 16, 2), \
4075 	mtp->mt_header.mh_svr_ready_ptr, \
4076 		HEXSPACE(mtp->mt_header.mh_svr_ready_ptr, \
4077 			mtp->mt_header.mh_svr_ready_ptr, 8, 1), \
4078 	mtp->mt_header.mh_svr_active_ptr, \
4079 		HEXSPACE(mtp->mt_header.mh_svr_active_ptr, \
4080 			mtp->mt_header.mh_svr_active_ptr, 8, 2), \
4081 	*(ushort_t *)(IDN_OFFSET2ADDR(mtp->mt_header.mh_svr_ready_ptr)), \
4082 	DECSPACE16(*(ushort_t *) \
4083 			(IDN_OFFSET2ADDR(mtp->mt_header.mh_svr_ready_ptr)), \
4084 			1, 1), \
4085 	*(ushort_t *)(IDN_OFFSET2ADDR(mtp->mt_header.mh_svr_active_ptr)), \
4086 	DECSPACE16(*(ushort_t *) \
4087 			(IDN_OFFSET2ADDR(mtp->mt_header.mh_svr_active_ptr)), \
4088 			1, 5), \
4089 	mtp->mt_header.mh_cookie, \
4090 		HEXSPACE(mtp->mt_header.mh_cookie, \
4091 			mtp->mt_header.mh_cookie, 8, 2), \
4092 	&mtp->mt_queue[0], \
4093 		HEXSPACE(&mtp->mt_queue[0], &mtp->mt_queue[0], 16, 2)
4094 
4095 /*ARGSUSED*/
4096 static int
4097 idn_mboxtbl_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
4098 {
4099 	register int		c, n, domid, subdomid;
4100 	register idn_mboxtbl_t	*mtp;
4101 	register idn_mboxmsg_t	*msp;
4102 	idn_mboxtbl_t		*map, *mtbasep;
4103 
4104 
4105 	ASSERT((cp == MBXTBL_PART_REPORT) || (cp == MBXTBL_FULL_REPORT));
4106 
4107 	if (IDN_GLOCK_TRY_SHARED() == 0) {
4108 		(void) mi_mpprintf(mp, "Local domain busy, try again.");
4109 		return (0);
4110 	}
4111 
4112 	if ((map = idn.mboxarea) == NULL) {
4113 		(void) mi_mpprintf(mp,
4114 		    "WARNING: Local domain is not master, "
4115 		    "ASSUMING idn.smr.vaddr.");
4116 		map = (idn_mboxtbl_t *)idn.smr.vaddr;
4117 	}
4118 
4119 	if (map) {
4120 		(void) mi_mpprintf(mp, "Mailbox Area starts @ 0x%p",
4121 		    map);
4122 	} else {
4123 		(void) mi_mpprintf(mp, "Mailbox Area not found.");
4124 		goto repdone;
4125 	}
4126 
4127 	if (!idn.nchannels) {
4128 		(void) mi_mpprintf(mp, "No OPEN channels found");
4129 		goto repdone;
4130 	}
4131 
4132 	for (c = 0; c < IDN_MAX_NETS; c++) {
4133 
4134 		IDN_CHAN_LOCK_GLOBAL(&idn.chan_servers[c]);
4135 		if (!IDN_CHANNEL_IS_ATTACHED(&idn.chan_servers[c])) {
4136 			IDN_CHAN_UNLOCK_GLOBAL(&idn.chan_servers[c]);
4137 			continue;
4138 		}
4139 
4140 		(void) mi_mpprintf(mp,
4141 		    "Channel %d ---------------------------"
4142 		    "--------------------------"
4143 		    "-----------------------------", c);
4144 		(void) mi_mpprintf(mp,
4145 		    "  Domain   Header	    "
4146 		    "Ready/Active Ptrs    "
4147 		    "rdy/actv  cookie    Queue	     "
4148 		    "busy");
4149 
4150 		for (domid = 0; domid < MAX_DOMAINS; domid++) {
4151 			register int	busy_count;
4152 
4153 			if ((cp == MBXTBL_PART_REPORT) &&
4154 			    (idn_domain[domid].dcpu == IDN_NIL_DCPU))
4155 				continue;
4156 
4157 			mtbasep = IDN_MBOXAREA_BASE(map, domid);
4158 
4159 			for (subdomid = 0; subdomid < MAX_DOMAINS;
4160 			    subdomid++) {
4161 				mtp = IDN_MBOXTBL_PTR(mtbasep, subdomid);
4162 				mtp = IDN_MBOXTBL_PTR_CHAN(mtp, c);
4163 
4164 				if (subdomid == domid) {
4165 					if (subdomid == 0)
4166 						(void) mi_mpprintf(mp,
4167 						    "   %x.%x-%d%s%s",
4168 						    domid, subdomid, c,
4169 							/*CONSTCOND*/
4170 						    DECSPACE(c, 2, 2),
4171 						    "-- unused --");
4172 					else
4173 						(void) mi_mpprintf(mp,
4174 						    "    .%x-%d%s%s",
4175 						    subdomid, c,
4176 							/*CONSTCOND*/
4177 						    DECSPACE(c, 2, 2),
4178 						    "-- unused --");
4179 					continue;
4180 				}
4181 				busy_count = 0;
4182 				msp = &mtp->mt_queue[0];
4183 				for (n = 0; n < IDN_MMBOX_NUMENTRIES; n++) {
4184 					if (msp[n].ms_owner)
4185 						busy_count++;
4186 				}
4187 				if (subdomid == 0) {
4188 					(void) mi_mpprintf(mp,
4189 					    "   %x.%x-%d%s%p%s%x%s/ %x%s"
4190 					    "%d%s/ %d%s%x%s%p%s%d%s",
4191 					    domid, subdomid, c,
4192 						/*CONSTCOND*/
4193 					    DECSPACE(c, 2, 2),
4194 					    MBXINFO(mtp), busy_count,
4195 					    busy_count ? " <<<<<":"");
4196 				} else {
4197 					(void) mi_mpprintf(mp,
4198 					    "    .%x-%d%s%p%s%x%s/ %x%s"
4199 					    "%d%s/ %d%s%x%s%p%s%d%s",
4200 					    subdomid, c,
4201 						/*CONSTCOND*/
4202 					    DECSPACE(c, 2, 2),
4203 					    MBXINFO(mtp), busy_count,
4204 					    busy_count ? " <<<<<":"");
4205 				}
4206 			}
4207 		}
4208 		IDN_CHAN_UNLOCK_GLOBAL(&idn.chan_servers[c]);
4209 	}
4210 
4211 repdone:
4212 	IDN_GUNLOCK();
4213 
4214 	return (0);
4215 }
4216 
4217 /*ARGSUSED*/
4218 static void
4219 idn_mainmbox_domain_report(queue_t *wq, mblk_t *mp, int domid,
4220 					idn_mainmbox_t *mmp, char *mbxtype)
4221 {
4222 	register int	c;
4223 
4224 	if (mmp == NULL) {
4225 		(void) mi_mpprintf(mp, " %x.%s  -- none --", domid, mbxtype);
4226 		return;
4227 	}
4228 
4229 	for (c = 0; c < IDN_MAX_NETS; mmp++, c++) {
4230 		int	mm_count;
4231 
4232 		IDN_CHAN_LOCK_GLOBAL(&idn.chan_servers[c]);
4233 		if (IDN_CHANNEL_IS_DETACHED(&idn.chan_servers[c])) {
4234 			(void) mi_mpprintf(mp, " %x.%s  %u  -- not open --",
4235 			    domid, mbxtype, (int)mmp->mm_channel);
4236 			IDN_CHAN_UNLOCK_GLOBAL(&idn.chan_servers[c]);
4237 			continue;
4238 		}
4239 
4240 		mm_count = ((mmp->mm_count < 0) ? 0 : mmp->mm_count) / 1000;
4241 
4242 		(void) mi_mpprintf(mp, " %x.%s  %d%s%d%s%d%s%p%s%p%s%p%s%d/%d",
4243 		    domid, mbxtype,
4244 		    (int)mmp->mm_channel,
4245 					/*CONSTCOND*/
4246 		    DECSPACE((int)mmp->mm_channel, 5, 2),
4247 		    mm_count, DECSPACE(mm_count, 8, 2),
4248 		    mmp->mm_dropped,
4249 		    DECSPACE(mmp->mm_dropped, 8, 2),
4250 		    mmp->mm_smr_mboxp,
4251 		    HEXSPACE(mmp->mm_smr_mboxp,
4252 		    mmp->mm_smr_mboxp, 16, 2),
4253 		    mmp->mm_smr_readyp,
4254 		    HEXSPACE(mmp->mm_smr_readyp,
4255 		    mmp->mm_smr_readyp, 16, 2),
4256 		    mmp->mm_smr_activep,
4257 		    HEXSPACE(mmp->mm_smr_activep,
4258 		    mmp->mm_smr_activep, 16, 2),
4259 		    mmp->mm_qiget, mmp->mm_qiput);
4260 		IDN_CHAN_UNLOCK_GLOBAL(&idn.chan_servers[c]);
4261 	}
4262 }
4263 
4264 /*ARGSUSED2*/
4265 static int
4266 idn_mainmbox_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
4267 {
4268 	int	domid;
4269 	int	header = 0;
4270 
4271 	/*
4272 	 * Domain 0 never has a send/recv mainmbox so
4273 	 * don't bother printing him.
4274 	 */
4275 	for (domid = 1; domid < MAX_DOMAINS; domid++) {
4276 		idn_domain_t	*dp;
4277 
4278 		dp = &idn_domain[domid];
4279 
4280 		if (dp->dcpu == IDN_NIL_DCPU)
4281 			continue;
4282 		IDN_DLOCK_SHARED(domid);
4283 		if (dp->dcpu == IDN_NIL_DCPU) {
4284 			IDN_DUNLOCK(domid);
4285 			continue;
4286 		}
4287 		if (!header) {
4288 			(void) mi_mpprintf(mp,
4289 			    "Domain  Chan   PktCntK   "
4290 			    "PktDrop   SMRMbox	   "
4291 			    "ReadyPtr	  "
4292 			    "ActvPtr	  Miget/Miput");
4293 			header = 1;
4294 		}
4295 
4296 		mutex_enter(&dp->dmbox.m_mutex);
4297 		idn_mainmbox_domain_report(wq, mp, domid,
4298 		    idn_domain[domid].dmbox.m_send,
4299 		    "snd");
4300 		idn_mainmbox_domain_report(wq, mp, domid,
4301 		    idn_domain[domid].dmbox.m_recv,
4302 		    "rcv");
4303 		mutex_exit(&dp->dmbox.m_mutex);
4304 
4305 		IDN_DUNLOCK(domid);
4306 
4307 		(void) mi_mpprintf(mp,
4308 		    "  ---------------------------------------"
4309 		    "------------------------"
4310 		    "----------------------------");
4311 	}
4312 
4313 	if (!header)
4314 		(void) mi_mpprintf(mp, "No ACTIVE domain connections exist");
4315 
4316 	return (0);
4317 }
4318 
4319 /*ARGSUSED*/
4320 static int
4321 idn_global_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
4322 {
4323 	int		i, nactive, masterid, nretry;
4324 	uint_t		locpfn_upper, locpfn_lower,
4325 	    rempfn_upper, rempfn_lower;
4326 	uint_t		marea_upper, marea_lower,
4327 	    iarea_upper, iarea_lower;
4328 	char		alt_dbuffer[64];
4329 	idn_retry_job_t	*rp;
4330 	domainset_t	retryset;
4331 	domainset_t	connected;
4332 	idn_synczone_t	*zp;
4333 	idn_syncop_t	*sp;
4334 	idn_domain_t	*dp;
4335 	char		*dbp, *dbuffer;
4336 
4337 	if (IDN_SYNC_TRYLOCK() == 0) {
4338 		(void) mi_mpprintf(mp, "Sync lock busy, try again.");
4339 		return (0);
4340 	}
4341 
4342 	if (IDN_GLOCK_TRY_SHARED() == 0) {
4343 		(void) mi_mpprintf(mp, "Local domain busy, try again.");
4344 		IDN_SYNC_UNLOCK();
4345 		return (0);
4346 	}
4347 	if ((dbp = dbuffer = ALLOC_DISPSTRING()) == NULL)
4348 		dbp = alt_dbuffer;
4349 
4350 	(void) mi_mpprintf(mp, "IDN\n    Global State = %s (%d)",
4351 	    idngs_str[idn.state], idn.state);
4352 
4353 	(void) mi_mpprintf(mp, "SMR");
4354 	(void) mi_mpprintf(mp, "    vaddr                ");
4355 	(void) mi_mpprintf(mp, "    0x%p", idn.smr.vaddr);
4356 
4357 	(void) mi_mpprintf(mp, "    paddr-local     paddr-remote");
4358 	masterid = IDN_GET_MASTERID();
4359 	locpfn_upper = (uint_t)(idn.smr.locpfn >> (32 - PAGESHIFT));
4360 	locpfn_lower = (uint_t)(idn.smr.locpfn << PAGESHIFT);
4361 	if (idn.smr.rempfn == PFN_INVALID) {
4362 		rempfn_upper = rempfn_lower = 0;
4363 	} else {
4364 		rempfn_upper = (uint_t)(idn.smr.rempfn >> (32 - PAGESHIFT));
4365 		rempfn_lower = (uint_t)(idn.smr.rempfn << PAGESHIFT);
4366 	}
4367 	(void) mi_mpprintf(mp, "    0x%x.%x%s0x%x.%x",
4368 	    locpfn_upper, locpfn_lower,
4369 	    HEXSPACE(locpfn_lower, locpfn_lower, 8,
4370 	    (locpfn_upper < 0x10) ? 4 : 3),
4371 	    rempfn_upper, rempfn_lower);
4372 
4373 	(void) mi_mpprintf(mp, "    SMR length  = %d MBytes", IDN_SMR_SIZE);
4374 	(void) mi_mpprintf(mp, "    SMR bufsize = %d Bytes", IDN_SMR_BUFSIZE);
4375 	(void) mi_mpprintf(mp, "    NWR length  = %d MBytes", IDN_NWR_SIZE);
4376 	marea_upper = (uint_t)((uint64_t)IDN_MBOXAREA_SIZE >> 32);
4377 	marea_lower = (uint_t)((uint64_t)IDN_MBOXAREA_SIZE & 0xffffffff);
4378 	iarea_upper = (uint_t)((uint64_t)(MB2B(IDN_NWR_SIZE) -
4379 	    (size_t)IDN_MBOXAREA_SIZE) >> 32);
4380 	iarea_lower = (uint_t)((MB2B(IDN_NWR_SIZE) -
4381 	    (size_t)IDN_MBOXAREA_SIZE) & 0xffffffff);
4382 	(void) mi_mpprintf(mp,
4383 	    "    [ mbox area = 0x%x.%x Bytes, "
4384 	    "iobuf area = 0x%x.%x Bytes ]",
4385 	    marea_upper, marea_lower, iarea_upper, iarea_lower);
4386 
4387 	(void) mi_mpprintf(mp,
4388 	    "\nIDNnet (local domain [id:%d] [name:%s] is %s)",
4389 	    idn.localid,
4390 	    idn_domain[idn.localid].dname,
4391 	    (masterid == IDN_NIL_DOMID) ? "IDLE" :
4392 	    (idn.localid == masterid) ? "MASTER" :
4393 	    "SLAVE");
4394 	nactive = 0;
4395 	for (i = 0; i < IDN_MAX_NETS; i++) {
4396 		IDN_CHAN_LOCK_GLOBAL(&idn.chan_servers[i]);
4397 		if (IDN_CHANNEL_IS_ACTIVE(&idn.chan_servers[i]))
4398 			nactive++;
4399 		IDN_CHAN_UNLOCK_GLOBAL(&idn.chan_servers[i]);
4400 	}
4401 	(void) mi_mpprintf(mp, "    I/O Networks: (Open = %d, "
4402 	    "Active = %d, Max = %d)",
4403 	    idn.nchannels, nactive, IDN_MAX_NETS);
4404 	(void) mi_mpprintf(mp, "    Number of Domains  = %d", idn.ndomains);
4405 	(void) mi_mpprintf(mp, "    Number of AWOLs    = %d", idn.nawols);
4406 	/*
4407 	 * During connect domains can possibly be in ds_connected
4408 	 * while still in ds_trans_on.  Only once they leave ds_trans_on
4409 	 * are they really connected.
4410 	 */
4411 	connected = idn.domset.ds_connected & ~idn.domset.ds_trans_on;
4412 	DOMAINSET_ADD(connected, idn.localid);
4413 	boardset2str(connected, dbp);
4414 	(void) mi_mpprintf(mp, "    Connected Domains      = %s", dbp);
4415 	domainset2str(idn.domset.ds_trans_on, dbp);
4416 	(void) mi_mpprintf(mp, "    Pending Domain Links   = %s",
4417 	    idn.domset.ds_trans_on ? dbp : "<>");
4418 	domainset2str(idn.domset.ds_trans_off, dbp);
4419 	(void) mi_mpprintf(mp, "    Pending Domain Unlinks = %s",
4420 	    idn.domset.ds_trans_off ? dbp : "<>");
4421 	mutex_enter(&idn.retryqueue.rq_mutex);
4422 	nretry = idn.retryqueue.rq_count;
4423 	retryset = 0;
4424 	for (i = 0, rp = idn.retryqueue.rq_jobs; i < nretry; i++,
4425 	    rp = rp->rj_next) {
4426 		int	domid;
4427 
4428 		domid = IDN_RETRY_TOKEN2DOMID(rp->rj_token);
4429 		if (VALID_DOMAINID(domid)) {
4430 			DOMAINSET_ADD(retryset, domid);
4431 		}
4432 	}
4433 	mutex_exit(&idn.retryqueue.rq_mutex);
4434 	domainset2str(retryset, dbp);
4435 	(void) mi_mpprintf(mp, "    Retry Jobs:Domains     = %d:%s",
4436 	    nretry, retryset ? dbp : "<>");
4437 	domainset2str(idn.domset.ds_hitlist, dbp);
4438 	(void) mi_mpprintf(mp, "    Hitlist Domains        = %s",
4439 	    idn.domset.ds_hitlist ? dbp : "<>");
4440 	domainset2str(idn.domset.ds_relink, dbp);
4441 	(void) mi_mpprintf(mp, "    Reconfig Domains       = %s",
4442 	    idn.domset.ds_relink ? dbp : "<>");
4443 	if (idn.domset.ds_relink)
4444 		(void) mi_mpprintf(mp, "         new master id = %d",
4445 		    IDN_GET_NEW_MASTERID());
4446 	if (masterid == IDN_NIL_DOMID) {
4447 		(void) mi_mpprintf(mp, "    Master Domain: no master");
4448 	} else {
4449 		idn_domain_t	*mdp;
4450 
4451 		mdp = &idn_domain[masterid];
4452 
4453 		(void) mi_mpprintf(mp,
4454 		    "    Master Domain (id:name/brds - state):");
4455 
4456 		if (strlen(mdp->dname) > 0)
4457 			strcpy(dbp, mdp->dname);
4458 		else
4459 			boardset2str(mdp->dhw.dh_boardset, dbp);
4460 		if (masterid < 10)
4461 			(void) mi_mpprintf(mp, "         %d: %s - %s",
4462 			    masterid, dbp,
4463 			    idnds_str[mdp->dstate]);
4464 		else
4465 			(void) mi_mpprintf(mp, "        %d: %s - %s",
4466 			    masterid, dbp,
4467 			    idnds_str[mdp->dstate]);
4468 	}
4469 	if (idn.ndomains <= 1) {
4470 		(void) mi_mpprintf(mp, "    Slave Domains: none");
4471 	} else {
4472 		int	d;
4473 
4474 		(void) mi_mpprintf(mp,
4475 		    "    Slave Domains (id:name/brds - state):");
4476 		for (d = 0; d < MAX_DOMAINS; d++) {
4477 			dp = &idn_domain[d];
4478 
4479 			if ((dp->dcpu == IDN_NIL_DCPU) || (d == masterid))
4480 				continue;
4481 
4482 			if (strlen(dp->dname) > 0)
4483 				strcpy(dbp, dp->dname);
4484 			else
4485 				boardset2str(dp->dhw.dh_boardset, dbp);
4486 			if (d < 10)
4487 				(void) mi_mpprintf(mp, "         %d: %s - %s",
4488 				    d, dbp,
4489 				    idnds_str[dp->dstate]);
4490 			else
4491 				(void) mi_mpprintf(mp, "        %d: %s - %s",
4492 				    d, dbp,
4493 				    idnds_str[dp->dstate]);
4494 		}
4495 	}
4496 
4497 	if (idn.nawols == 0) {
4498 		(void) mi_mpprintf(mp, "    AWOL Domains: none");
4499 	} else {
4500 		int	d;
4501 
4502 		(void) mi_mpprintf(mp, "    AWOL Domains (id:name/brds):");
4503 		for (d = 0; d < MAX_DOMAINS; d++) {
4504 			dp = &idn_domain[d];
4505 
4506 			if (!DOMAIN_IN_SET(idn.domset.ds_awol, d) ||
4507 			    (dp->dcpu == IDN_NIL_DCPU))
4508 				continue;
4509 
4510 			if (strlen(dp->dname) > 0)
4511 				strcpy(dbp, dp->dname);
4512 			else
4513 				boardset2str(dp->dhw.dh_boardset, dbp);
4514 			if (d < 10)
4515 				(void) mi_mpprintf(mp, "         %d: %s",
4516 				    d, dbp);
4517 			else
4518 				(void) mi_mpprintf(mp, "        %d: %s",
4519 				    d, dbp);
4520 		}
4521 	}
4522 
4523 	/*CONSTCOND*/
4524 	i = IDN_SYNC_GETZONE(IDNSYNC_CONNECT);
4525 	zp = &idn.sync.sz_zone[i];
4526 	if (zp->sc_cnt == 0) {
4527 		(void) mi_mpprintf(mp, "    Sync Zone (con): [empty]");
4528 	} else {
4529 		(void) mi_mpprintf(mp, "    Sync Zone (con): [%d domains]",
4530 		    zp->sc_cnt);
4531 		sp = zp->sc_op;
4532 		for (i = 0; (i < zp->sc_cnt) && sp; i++) {
4533 			(void) mi_mpprintf(mp,
4534 			    "	             "
4535 			    "%x: x_set =%s0x%x, r_set =%s0x%x",
4536 			    sp->s_domid,
4537 			    HEXSPACE(sp->s_set_exp,
4538 			    sp->s_set_exp, 4, 1),
4539 			    sp->s_set_exp,
4540 			    HEXSPACE(sp->s_set_rdy,
4541 			    sp->s_set_rdy, 4, 1),
4542 			    sp->s_set_rdy);
4543 			sp = sp->s_next;
4544 		}
4545 	}
4546 	/*CONSTCOND*/
4547 	i = IDN_SYNC_GETZONE(IDNSYNC_DISCONNECT);
4548 	zp = &idn.sync.sz_zone[i];
4549 	if (zp->sc_cnt == 0) {
4550 		(void) mi_mpprintf(mp, "    Sync Zone (dis): [empty]");
4551 	} else {
4552 		(void) mi_mpprintf(mp, "    Sync Zone (dis): [%d domains]",
4553 		    zp->sc_cnt);
4554 		sp = zp->sc_op;
4555 		for (i = 0; (i < zp->sc_cnt) && sp; i++) {
4556 			(void) mi_mpprintf(mp,
4557 			    "	             "
4558 			    "%x: x_set =%s0x%x, r_set =%s0x%x",
4559 			    sp->s_domid,
4560 			    HEXSPACE(sp->s_set_exp,
4561 			    sp->s_set_exp, 4, 1),
4562 			    sp->s_set_exp,
4563 			    HEXSPACE(sp->s_set_rdy,
4564 			    sp->s_set_rdy, 4, 1),
4565 			    sp->s_set_rdy);
4566 			sp = sp->s_next;
4567 		}
4568 	}
4569 
4570 	IDN_GUNLOCK();
4571 	IDN_SYNC_UNLOCK();
4572 
4573 	if (dbuffer) {
4574 		FREE_DISPSTRING(dbuffer);
4575 	}
4576 
4577 	return (0);
4578 }
4579 
4580 /*ARGSUSED*/
4581 static int
4582 idn_domain_report(queue_t *wq, mblk_t *mp, caddr_t cp, cred_t *cr)
4583 {
4584 	int		d, nchan;
4585 	uint_t		domset;
4586 	idn_chanset_t	chanset;
4587 	idn_domain_t	*dp;
4588 	uint_t		pset_upper, pset_lower;
4589 	char		*dbuffer, *dbp;
4590 	char		alt_dbuffer[64];
4591 
4592 
4593 	if (IDN_SYNC_TRYLOCK() == 0) {
4594 		(void) mi_mpprintf(mp, "Sync lock busy, try again.");
4595 		return (0);
4596 	}
4597 
4598 	if (IDN_GLOCK_TRY_SHARED() == 0) {
4599 		(void) mi_mpprintf(mp, "Local domain busy, try again.");
4600 		IDN_SYNC_UNLOCK();
4601 		return (0);
4602 	}
4603 
4604 	if ((dbp = dbuffer = ALLOC_DISPSTRING()) == NULL)
4605 		dbp = alt_dbuffer;
4606 
4607 	if (cp == NULL)
4608 		domset = DOMAINSET(idn.localid);
4609 	else
4610 		domset = DOMAINSET_ALL;
4611 
4612 	for (d = 0; d < MAX_DOMAINS; d++) {
4613 
4614 		if (DOMAIN_IN_SET(domset, d) == 0)
4615 			continue;
4616 
4617 		dp = &idn_domain[d];
4618 
4619 		if (dp->dcpu == IDN_NIL_DCPU)
4620 			continue;
4621 
4622 		if (IDN_DLOCK_TRY_SHARED(d) == 0) {
4623 			if (d < 10)
4624 				(void) mi_mpprintf(mp,
4625 				    "Domain %d   (0x%p) busy...",
4626 				    d, dp);
4627 			else
4628 				(void) mi_mpprintf(mp,
4629 				    "Domain %d  (0x%p) busy...",
4630 				    d, dp);
4631 			continue;
4632 		}
4633 		if (dp->dcpu == IDN_NIL_DCPU) {
4634 			IDN_DUNLOCK(d);
4635 			continue;
4636 		}
4637 		if (d < 10)
4638 			(void) mi_mpprintf(mp, "%sDomain %d   (0x%p)",
4639 			    (d && (idn.ndomains > 1)) ? "\n" : "",
4640 			    d, dp);
4641 		else
4642 			(void) mi_mpprintf(mp, "%sDomain %d  (0x%p)",
4643 			    (d && (idn.ndomains > 1)) ? "\n" : "",
4644 			    d, dp);
4645 
4646 		if (d == idn.localid)
4647 			(void) mi_mpprintf(mp, "  (local)  State = %s (%d)",
4648 			    idnds_str[dp->dstate], dp->dstate);
4649 		else
4650 			(void) mi_mpprintf(mp, "           State = %s (%d)",
4651 			    idnds_str[dp->dstate], dp->dstate);
4652 		(void) mi_mpprintf(mp, "           Name = %s, Netid = %d",
4653 		    (strlen(dp->dname) > 0) ? dp->dname : "<>",
4654 		    (int)dp->dnetid);
4655 
4656 		CHANSET_ZERO(chanset);
4657 		nchan = idn_domain_is_registered(d, -1, &chanset);
4658 		if (dbuffer)
4659 			mask2str(chanset, dbp, 32);
4660 		else
4661 			(void) sprintf(dbp, "0x%x", chanset);
4662 		(void) mi_mpprintf(mp, "           Nchans = %d, Chanset = %s",
4663 		    nchan, nchan ? dbp : "<>");
4664 		pset_upper = UPPER32_CPUMASK(dp->dcpuset);
4665 		pset_lower = LOWER32_CPUMASK(dp->dcpuset);
4666 		if (dbuffer)
4667 			boardset2str(dp->dhw.dh_boardset, dbp);
4668 		else
4669 			(void) sprintf(dbp, "0x%x", dp->dhw.dh_boardset);
4670 
4671 		(void) mi_mpprintf(mp, "           Nboards = %d, Brdset = %s",
4672 		    dp->dhw.dh_nboards,
4673 		    dp->dhw.dh_nboards ? dbp : "<>");
4674 		(void) sprintf(dbp, "0x%x.%x", pset_upper, pset_lower);
4675 		(void) mi_mpprintf(mp, "           Ncpus = %d, Cpuset = %s",
4676 		    dp->dncpus, dp->dncpus ? dbp : "<>");
4677 		(void) mi_mpprintf(mp, "           Nmcadr = %d",
4678 		    dp->dhw.dh_nmcadr);
4679 		(void) mi_mpprintf(mp,
4680 		    "	   MsgTimer = %s  (cnt = %d)",
4681 		    (dp->dtimerq.tq_count > 0)
4682 		    ? "active" : "idle",
4683 		    dp->dtimerq.tq_count);
4684 		(void) mi_mpprintf(mp, "           Dcpu = %d  "
4685 		    "(lastcpu = %d, cpuindex = %d)",
4686 		    dp->dcpu, dp->dcpu_last, dp->dcpuindex);
4687 		(void) mi_mpprintf(mp, "           Dio = %d  "
4688 		    "(ioerr = %d, iochk = %d, iowanted = %d)",
4689 		    dp->dio, dp->dioerr, dp->diocheck ? 1 : 0,
4690 		    dp->diowanted ? 1 : 0);
4691 		if (dp->dsync.s_cmd == IDNSYNC_NIL) {
4692 			(void) mi_mpprintf(mp, "           Dsync = %s",
4693 			    idnsync_str[IDNSYNC_NIL]);
4694 		} else {
4695 			(void) mi_mpprintf(mp,
4696 			    "	   Dsync = %s "
4697 			    "(x_set = 0x%x, r_set = 0x%x)",
4698 			    idnsync_str[dp->dsync.s_cmd],
4699 			    (uint_t)dp->dsync.s_set_exp,
4700 			    (uint_t)dp->dsync.s_set_rdy);
4701 		}
4702 		(void) mi_mpprintf(mp, "           Dvote = 0x%x",
4703 		    dp->dvote.ticket);
4704 		(void) mi_mpprintf(mp, "           Dfin = %s (Sync = %s)",
4705 		    idnfin_str[dp->dfin],
4706 		    (dp->dfin_sync == IDNFIN_SYNC_OFF) ? "OFF" :
4707 		    (dp->dfin_sync == IDNFIN_SYNC_YES) ? "YES" :
4708 		    "NO");
4709 		(void) mi_mpprintf(mp, "           Dcookie_err = %s (cnt = %d)",
4710 		    dp->dcookie_err ? "YES" : "NO",
4711 		    dp->dcookie_errcnt);
4712 		IDN_DUNLOCK(d);
4713 	}
4714 
4715 	IDN_GUNLOCK();
4716 
4717 	if (dbuffer) {
4718 		FREE_DISPSTRING(dbuffer);
4719 	}
4720 
4721 	IDN_SYNC_UNLOCK();
4722 
4723 	return (0);
4724 }
4725 
4726 #define	SNOOP_ENTRIES	2048	/* power of 2 */
4727 
4728 struct snoop_buffer {
4729 /*  0 */	char	io;
4730 /*  1 */	char	board;
4731 /*  2 */	char	trans[14];
4732 
4733 /* 10 */	uint_t	xargs[4];
4734 } *snoop_data, snoop_buffer[SNOOP_ENTRIES+1];
4735 
4736 
4737 int		snoop_index;
4738 kmutex_t	snoop_mutex;
4739 static char	_bd2hexascii[] = {
4740 	'0', '1', '2', '3', '4', '5', '6', '7',
4741 	'8', '9', 'A', 'B', 'C', 'D', 'E', 'F'
4742 };
4743 
4744 #define	SNOOP_IDN(in, tr, bd, arg1, arg2, arg3, arg4) \
4745 { \
4746 	if (idn_snoop) { \
4747 		mutex_enter(&snoop_mutex); \
4748 		if (snoop_data == NULL) { \
4749 			snoop_data = (struct snoop_buffer *) \
4750 				(((uint_t)snoop_buffer + 0xf) & ~0xf); \
4751 		} \
4752 		snoop_data[snoop_index].io = ((in) == 0) ? 'o' : 'i'; \
4753 		snoop_data[snoop_index].board = \
4754 				((bd) == -1) ? 'X' : _bd2hexascii[bd]; \
4755 		(void) strncpy(snoop_data[snoop_index].trans, (tr), 14); \
4756 		snoop_data[snoop_index].xargs[0] = (arg1); \
4757 		snoop_data[snoop_index].xargs[1] = (arg2); \
4758 		snoop_data[snoop_index].xargs[2] = (arg3); \
4759 		snoop_data[snoop_index].xargs[3] = (arg4); \
4760 		snoop_index++; \
4761 		snoop_index &= SNOOP_ENTRIES - 1; \
4762 		mutex_exit(&snoop_mutex); \
4763 	} \
4764 }
4765 
4766 /*
4767  * Allocate the circular buffers to be used for
4768  * DMV interrupt processing.
4769  */
4770 static int
4771 idn_init_handler()
4772 {
4773 	int		i, c;
4774 	size_t		len;
4775 	idn_dmv_msg_t	*basep, *ivp;
4776 	uint32_t	ivp_offset;
4777 	procname_t	proc = "idn_init_handler";
4778 
4779 	if (idn.intr.dmv_data != NULL) {
4780 		cmn_err(CE_WARN,
4781 		    "IDN: 130: IDN DMV handler already initialized");
4782 		return (-1);
4783 	}
4784 
4785 	/*
4786 	 * This memory will be touched by the low-level
4787 	 * DMV trap handler for IDN.
4788 	 */
4789 	len = sizeof (idn_dmv_data_t);
4790 	len = roundup(len, sizeof (uint64_t));
4791 	len += NCPU * idn_dmv_pending_max * sizeof (idn_dmv_msg_t);
4792 	len = roundup(len, PAGESIZE);
4793 
4794 	PR_PROTO("%s: sizeof (idn_dmv_data_t) = %lu\n",
4795 	    proc, sizeof (idn_dmv_data_t));
4796 	PR_PROTO("%s: allocating %lu bytes for dmv data area\n", proc, len);
4797 
4798 	idn.intr.dmv_data_len = len;
4799 	idn.intr.dmv_data = kmem_zalloc(len, KM_SLEEP);
4800 
4801 	PR_PROTO("%s: DMV data area = %p\n", proc, idn.intr.dmv_data);
4802 
4803 	idn_dmv_data = (idn_dmv_data_t *)idn.intr.dmv_data;
4804 	basep = (idn_dmv_msg_t *)roundup((size_t)idn.intr.dmv_data +
4805 	    sizeof (idn_dmv_data_t),
4806 	    sizeof (uint64_t));
4807 	idn_dmv_data->idn_dmv_qbase = (uint64_t)basep;
4808 
4809 	ivp = basep;
4810 	ivp_offset = 0;
4811 	/*
4812 	 * The buffer queues are allocated per-cpu.
4813 	 */
4814 	for (c = 0, ivp = basep; c < NCPU; ivp++, c++) {
4815 		idn_dmv_data->idn_dmv_cpu[c].idn_dmv_current = ivp_offset;
4816 		idn_iv_queue[c] = ivp;
4817 		ivp_offset += sizeof (idn_dmv_msg_t);
4818 		for (i = 1; i < idn_dmv_pending_max; ivp++, i++) {
4819 			ivp->iv_next = ivp_offset;
4820 			ivp->iv_ready = 0;
4821 			lock_set(&ivp->iv_ready);
4822 			ivp_offset += sizeof (idn_dmv_msg_t);
4823 		}
4824 		ivp->iv_next = idn_dmv_data->idn_dmv_cpu[c].idn_dmv_current;
4825 		ivp->iv_ready = 0;
4826 		lock_set(&ivp->iv_ready);
4827 	}
4828 
4829 	idn.intr.dmv_inum = STARFIRE_DMV_IDN_BASE;
4830 	idn.intr.soft_inum = add_softintr((uint_t)idn_pil, idn_handler, 0,
4831 	    SOFTINT_ST);
4832 	idn_dmv_data->idn_soft_inum = idn.intr.soft_inum;
4833 	/*
4834 	 * Make sure everything is out there before
4835 	 * we effectively set it free for use.
4836 	 */
4837 	membar_stld_stst();
4838 
4839 	if (dmv_add_intr(idn.intr.dmv_inum, idn_dmv_handler,
4840 	    (caddr_t)idn_dmv_data)) {
4841 		idn_deinit_handler();
4842 		cmn_err(CE_WARN, "IDN: 132: failed to add IDN DMV handler");
4843 		return (-1);
4844 	}
4845 
4846 	return (0);
4847 }
4848 
4849 static void
4850 idn_deinit_handler()
4851 {
4852 	if (idn.intr.dmv_data == NULL)
4853 		return;
4854 
4855 	(void) dmv_rem_intr(idn.intr.dmv_inum);
4856 	(void) rem_softintr(idn.intr.soft_inum);
4857 	kmem_free(idn.intr.dmv_data, idn.intr.dmv_data_len);
4858 	idn.intr.dmv_data = NULL;
4859 }
4860 
4861 /*
4862  * High-level (soft interrupt) handler for DMV interrupts
4863  */
4864 /*ARGSUSED0*/
4865 static uint_t
4866 idn_handler(caddr_t unused, caddr_t unused2)
4867 {
4868 #ifdef DEBUG
4869 	int		count = 0;
4870 #endif /* DEBUG */
4871 	int		cpuid = (int)CPU->cpu_id;
4872 	ushort_t	mtype, atype;
4873 	idn_dmv_msg_t	*xp, *xplimit;
4874 	procname_t	proc = "idn_handler";
4875 
4876 	ASSERT(getpil() >= idn_pil);
4877 	flush_windows();
4878 
4879 	/*
4880 	 * Clear the synchronization flag to indicate that
4881 	 * processing has started.  As long as idn_dmv_active
4882 	 * is non-zero, idn_dmv_handler will queue work without
4883 	 * initiating a soft interrupt.  Since we clear it
4884 	 * first thing at most one pil-interrupt for IDN will
4885 	 * queue up behind the currently active one.  We don't
4886 	 * want to clear this flag at the end because it leaves
4887 	 * a window where an interrupt could get lost (unless it's
4888 	 * pushed by a subsequent interrupt).  The objective in
4889 	 * doing this is to prevent exhausting a cpu's intr_vec
4890 	 * structures with interrupts of the same pil level.
4891 	 */
4892 	lock_clear(&idn_dmv_data->idn_dmv_cpu[cpuid].idn_dmv_active);
4893 
4894 	xp = idn_iv_queue[cpuid];
4895 	xplimit = xp + idn_dmv_pending_max;
4896 	xp += idn_intr_index[cpuid];
4897 	/*
4898 	 * As long as there's stuff that's READY in the
4899 	 * queue, keep processing.
4900 	 */
4901 	while (lock_try(&xp->iv_ready)) {
4902 
4903 		ASSERT(lock_try(&xp->iv_inuse) == 0);
4904 
4905 		mtype = (ushort_t)xp->iv_mtype;
4906 		mtype &= IDNP_MSGTYPE_MASK | IDNP_ACKNACK_MASK;
4907 		atype = (ushort_t)xp->iv_atype;
4908 
4909 		if (((int)xp->iv_ver == idn.version) && mtype) {
4910 			idn_protojob_t	*jp;
4911 #ifdef DEBUG
4912 			STRING(mstr);
4913 			STRING(astr);
4914 
4915 			INUM2STR(mtype, mstr);
4916 			if ((mtype & IDNP_MSGTYPE_MASK) == 0) {
4917 				INUM2STR(atype, astr);
4918 				strcat(mstr, "/");
4919 				strcat(mstr, astr);
4920 			}
4921 
4922 			count++;
4923 
4924 			PR_XDC("%s:%d:%d RECV: scpu = %d, msg = 0x%x(%s)\n",
4925 			    proc, (int)xp->iv_domid, count,
4926 			    (int)xp->iv_cpuid, mtype, mstr);
4927 			PR_XDC("%s:%d:%d R-DATA: a0 = 0x%x, a1 = 0x%x\n",
4928 			    proc, (int)xp->iv_domid, count,
4929 			    xp->iv_xargs0, xp->iv_xargs1);
4930 			PR_XDC("%s:%d:%d R-DATA: a2 = 0x%x, a3 = 0x%x\n",
4931 			    proc, (int)xp->iv_domid, count,
4932 			    xp->iv_xargs2, xp->iv_xargs3);
4933 #endif /* DEBUG */
4934 
4935 			if (mtype == IDNP_DATA) {
4936 				jp = NULL;
4937 				/*
4938 				 * The only time we receive pure
4939 				 * data messages at this level is
4940 				 * to wake up the channel server.
4941 				 * Since this is often an urgent
4942 				 * request we'll do it from here
4943 				 * instead of waiting for a proto
4944 				 * server to do it.
4945 				 */
4946 				idn_signal_data_server((int)xp->iv_domid,
4947 				    (ushort_t)xp->iv_xargs0);
4948 			} else {
4949 				jp = idn_protojob_alloc(KM_NOSLEEP);
4950 				/*
4951 				 * If the allocation fails, just drop
4952 				 * the message and get on with life.
4953 				 * If memory pressure is this great then
4954 				 * dropping this message is probably
4955 				 * the least of our worries!
4956 				 */
4957 				if (jp) {
4958 					jp->j_msg.m_domid = (int)xp->iv_domid;
4959 					jp->j_msg.m_cpuid = (int)xp->iv_cpuid;
4960 					jp->j_msg.m_msgtype = mtype;
4961 					jp->j_msg.m_acktype = atype;
4962 					jp->j_msg.m_cookie = xp->iv_cookie;
4963 					SET_XARGS(jp->j_msg.m_xargs,
4964 					    xp->iv_xargs0, xp->iv_xargs1,
4965 					    xp->iv_xargs2, xp->iv_xargs3);
4966 				}
4967 
4968 			}
4969 			membar_ldst_stst();
4970 
4971 			lock_clear(&xp->iv_inuse);
4972 
4973 			if (jp)
4974 				idn_protojob_submit(jp->j_msg.m_domid, jp);
4975 		} else {
4976 			membar_ldst_stst();
4977 			IDN_GKSTAT_INC(gk_dropped_intrs);
4978 			lock_clear(&xp->iv_inuse);
4979 		}
4980 
4981 		if (++xp == xplimit)
4982 			xp = idn_iv_queue[cpuid];
4983 	}
4984 
4985 	idn_intr_index[cpuid] = xp - idn_iv_queue[cpuid];
4986 
4987 	return (DDI_INTR_CLAIMED);
4988 }
4989 
4990 void
4991 idn_awol_event_set(boardset_t boardset)
4992 {
4993 	idnsb_event_t	*sbp;
4994 	procname_t	proc = "idn_awol_event_set";
4995 
4996 	ASSERT(IDN_GLOCK_IS_EXCL());
4997 
4998 	mutex_enter(&idn.idnsb_mutex);
4999 	sbp = idn.idnsb_eventp;
5000 	if (sbp == NULL) {
5001 		cmn_err(CE_WARN, "IDN: 133: sigblock event area missing");
5002 		cmn_err(CE_CONT,
5003 		    "IDN: 134: unable to mark boardset (0x%x) AWOL\n",
5004 		    boardset);
5005 		mutex_exit(&idn.idnsb_mutex);
5006 		return;
5007 	}
5008 
5009 	if (boardset == 0) {
5010 		PR_PROTO("%s: AWOL BOARDSET is 0, NO EVENT <<<<<<<<<<<<<<<\n",
5011 		    proc);
5012 		mutex_exit(&idn.idnsb_mutex);
5013 		return;
5014 	} else {
5015 		PR_PROTO("%s: MARKING BOARDSET (0x%x) AWOL\n", proc, boardset);
5016 	}
5017 	SSIEVENT_ADD(sbp, SSIEVENT_AWOL, boardset);
5018 	mutex_exit(&idn.idnsb_mutex);
5019 }
5020 
5021 void
5022 idn_awol_event_clear(boardset_t boardset)
5023 {
5024 	idnsb_event_t	*sbp;
5025 	procname_t	proc = "idn_awol_event_clear";
5026 
5027 	ASSERT(IDN_GLOCK_IS_EXCL());
5028 
5029 	mutex_enter(&idn.idnsb_mutex);
5030 	sbp = idn.idnsb_eventp;
5031 	if (sbp == NULL) {
5032 		cmn_err(CE_WARN, "IDN: 133: sigblock event area missing");
5033 		cmn_err(CE_CONT,
5034 		    "IDN: 134: unable to mark boardset (0x%x) AWOL\n",
5035 		    boardset);
5036 		mutex_exit(&idn.idnsb_mutex);
5037 		return;
5038 	}
5039 
5040 	if (boardset == 0) {
5041 		PR_PROTO("%s: AWOL BOARDSET is 0, NO EVENT <<<<<<<<<<<<<<<\n",
5042 		    proc);
5043 		mutex_exit(&idn.idnsb_mutex);
5044 		return;
5045 	} else {
5046 		PR_PROTO("%s: CLEARING BOARDSET (0x%x) AWOL\n", proc, boardset);
5047 	}
5048 	SSIEVENT_DEL(sbp, SSIEVENT_AWOL, boardset);
5049 	mutex_exit(&idn.idnsb_mutex);
5050 }
5051 
5052 static void
5053 idn_gkstat_init()
5054 {
5055 	struct	kstat			*ksp;
5056 	struct	idn_gkstat_named	*sgkp;
5057 
5058 #ifdef	kstat
5059 	if ((ksp = kstat_create(IDNNAME, ddi_get_instance(idn.dip),
5060 	    IDNNAME, "net", KSTAT_TYPE_NAMED,
5061 	    sizeof (struct idn_gkstat_named) / sizeof (kstat_named_t),
5062 	    KSTAT_FLAG_PERSISTENT)) == NULL) {
5063 #else
5064 	if ((ksp = kstat_create(IDNNAME, ddi_get_instance(idn.dip),
5065 	    IDNNAME, "net", KSTAT_TYPE_NAMED,
5066 	    sizeof (struct idn_gkstat_named) /
5067 	    sizeof (kstat_named_t), 0)) == NULL) {
5068 #endif /* kstat */
5069 		cmn_err(CE_CONT, "IDN: 135: %s: %s\n",
5070 		    IDNNAME, "kstat_create failed");
5071 		return;
5072 	}
5073 
5074 	idn.ksp = ksp;
5075 	sgkp = (struct idn_gkstat_named *)(ksp->ks_data);
5076 	kstat_named_init(&sgkp->sk_curtime,		"curtime",
5077 	    KSTAT_DATA_ULONG);
5078 	kstat_named_init(&sgkp->sk_reconfigs,		"reconfigs",
5079 	    KSTAT_DATA_ULONG);
5080 	kstat_named_init(&sgkp->sk_reconfig_last,	"reconfig_last",
5081 	    KSTAT_DATA_ULONG);
5082 	kstat_named_init(&sgkp->sk_reaps,		"reaps",
5083 	    KSTAT_DATA_ULONG);
5084 	kstat_named_init(&sgkp->sk_reap_last,		"reap_last",
5085 	    KSTAT_DATA_ULONG);
5086 	kstat_named_init(&sgkp->sk_links,		"links",
5087 	    KSTAT_DATA_ULONG);
5088 	kstat_named_init(&sgkp->sk_link_last,		"link_last",
5089 	    KSTAT_DATA_ULONG);
5090 	kstat_named_init(&sgkp->sk_unlinks,		"unlinks",
5091 	    KSTAT_DATA_ULONG);
5092 	kstat_named_init(&sgkp->sk_unlink_last,		"unlink_last",
5093 	    KSTAT_DATA_ULONG);
5094 	kstat_named_init(&sgkp->sk_buffail,		"buf_fail",
5095 	    KSTAT_DATA_ULONG);
5096 	kstat_named_init(&sgkp->sk_buffail_last,	"buf_fail_last",
5097 	    KSTAT_DATA_ULONG);
5098 	kstat_named_init(&sgkp->sk_slabfail,		"slab_fail",
5099 	    KSTAT_DATA_ULONG);
5100 	kstat_named_init(&sgkp->sk_slabfail_last,	"slab_fail_last",
5101 	    KSTAT_DATA_ULONG);
5102 	kstat_named_init(&sgkp->sk_slabfail_last,	"slab_fail_last",
5103 	    KSTAT_DATA_ULONG);
5104 	kstat_named_init(&sgkp->sk_reap_count,		"reap_count",
5105 	    KSTAT_DATA_ULONG);
5106 	kstat_named_init(&sgkp->sk_dropped_intrs,	"dropped_intrs",
5107 	    KSTAT_DATA_ULONG);
5108 	ksp->ks_update = idn_gkstat_update;
5109 	ksp->ks_private = (void *)NULL;
5110 	kstat_install(ksp);
5111 }
5112 
5113 static void
5114 idn_gkstat_deinit()
5115 {
5116 	if (idn.ksp)
5117 		kstat_delete(idn.ksp);
5118 	idn.ksp = NULL;
5119 }
5120 
5121 static int
5122 idn_gkstat_update(kstat_t *ksp, int rw)
5123 {
5124 	struct idn_gkstat_named	*sgkp;
5125 
5126 	sgkp = (struct idn_gkstat_named *)ksp->ks_data;
5127 
5128 	if (rw == KSTAT_WRITE) {
5129 		sg_kstat.gk_reconfigs	    = sgkp->sk_reconfigs.value.ul;
5130 		sg_kstat.gk_reconfig_last   = sgkp->sk_reconfig_last.value.ul;
5131 		sg_kstat.gk_reaps	    = sgkp->sk_reaps.value.ul;
5132 		sg_kstat.gk_reap_last	    = sgkp->sk_reap_last.value.ul;
5133 		sg_kstat.gk_links	    = sgkp->sk_links.value.ul;
5134 		sg_kstat.gk_link_last	    = sgkp->sk_link_last.value.ul;
5135 		sg_kstat.gk_unlinks	    = sgkp->sk_unlinks.value.ul;
5136 		sg_kstat.gk_unlink_last	    = sgkp->sk_unlink_last.value.ul;
5137 		sg_kstat.gk_buffail	    = sgkp->sk_buffail.value.ul;
5138 		sg_kstat.gk_buffail_last    = sgkp->sk_buffail_last.value.ul;
5139 		sg_kstat.gk_slabfail	    = sgkp->sk_slabfail.value.ul;
5140 		sg_kstat.gk_slabfail_last   = sgkp->sk_slabfail_last.value.ul;
5141 		sg_kstat.gk_reap_count	    = sgkp->sk_reap_count.value.ul;
5142 		sg_kstat.gk_dropped_intrs   = sgkp->sk_dropped_intrs.value.ul;
5143 	} else {
5144 		sgkp->sk_curtime.value.ul	  = lbolt;
5145 		sgkp->sk_reconfigs.value.ul	  = sg_kstat.gk_reconfigs;
5146 		sgkp->sk_reconfig_last.value.ul	  = sg_kstat.gk_reconfig_last;
5147 		sgkp->sk_reaps.value.ul		  = sg_kstat.gk_reaps;
5148 		sgkp->sk_reap_last.value.ul	  = sg_kstat.gk_reap_last;
5149 		sgkp->sk_links.value.ul		  = sg_kstat.gk_links;
5150 		sgkp->sk_link_last.value.ul	  = sg_kstat.gk_link_last;
5151 		sgkp->sk_unlinks.value.ul	  = sg_kstat.gk_unlinks;
5152 		sgkp->sk_unlink_last.value.ul	  = sg_kstat.gk_unlink_last;
5153 		sgkp->sk_buffail.value.ul	  = sg_kstat.gk_buffail;
5154 		sgkp->sk_buffail_last.value.ul    = sg_kstat.gk_buffail_last;
5155 		sgkp->sk_slabfail.value.ul	  = sg_kstat.gk_slabfail;
5156 		sgkp->sk_slabfail_last.value.ul   = sg_kstat.gk_slabfail_last;
5157 		sgkp->sk_reap_count.value.ul	  = sg_kstat.gk_reap_count;
5158 		sgkp->sk_dropped_intrs.value.ul	  = sg_kstat.gk_dropped_intrs;
5159 	}
5160 
5161 	return (0);
5162 }
5163 
5164 #ifdef DEBUG
5165 #define	RW_HISTORY	100
5166 static uint_t	rw_history[NCPU][RW_HISTORY];
5167 static int	rw_index[NCPU];
5168 #endif /* DEBUG */
5169 
5170 static int
5171 idn_rw_mem(idnop_t *idnop)
5172 {
5173 	uint_t		lo_off, hi_off;
5174 	int		rw, blksize, num;
5175 	int		cpuid;
5176 	register int	n, idx;
5177 	char 		*ibuf, *obuf;
5178 	char		*smraddr;
5179 	struct seg	*segp;
5180 	ulong_t		randx;
5181 	kmutex_t	slock;
5182 	kcondvar_t	scv;
5183 	static int	orig_gstate = IDNGS_IGNORE;
5184 	extern struct  seg	ktextseg;
5185 
5186 #define	RANDOM_INIT()	(randx = lbolt)
5187 #define	RANDOM(a, b)	\
5188 	(((a) >= (b)) ? \
5189 	(a) : (((randx = randx * 1103515245L + 12345) % ((b)-(a))) + (a)))
5190 
5191 	RANDOM_INIT();
5192 
5193 	lo_off  = idnop->rwmem.lo_off;
5194 	hi_off  = idnop->rwmem.hi_off;
5195 	blksize = idnop->rwmem.blksize;
5196 	num	= idnop->rwmem.num;
5197 	rw	= idnop->rwmem.rw;	/* 0 = rd, 1 = wr, 2 = rd/wr */
5198 
5199 	if (((hi_off > (uint_t)MB2B(IDN_SMR_SIZE)) || (lo_off >= hi_off) ||
5200 	    (blksize <= 0) || (blksize > (hi_off - lo_off)) || (num <= 0)) &&
5201 	    (idnop->rwmem.goawol == -1)) {
5202 		return (EINVAL);
5203 	}
5204 
5205 	if (idnop->rwmem.goawol && (orig_gstate == IDNGS_IGNORE)) {
5206 		IDN_GLOCK_EXCL();
5207 		cmn_err(CE_WARN, "IDN: Local domain going into IGNORE MODE!!");
5208 		orig_gstate = idn.state;
5209 		IDN_GSTATE_TRANSITION(IDNGS_IGNORE);
5210 		IDN_GUNLOCK();
5211 
5212 	} else if (!idnop->rwmem.goawol && (orig_gstate != IDNGS_IGNORE)) {
5213 		IDN_GLOCK_EXCL();
5214 		cmn_err(CE_WARN,
5215 		    "IDN: Local domain restoring original state %s(%d)",
5216 		    idngs_str[orig_gstate], (int)orig_gstate);
5217 		IDN_GSTATE_TRANSITION(orig_gstate);
5218 		orig_gstate = IDNGS_IGNORE;
5219 		IDN_GUNLOCK();
5220 	}
5221 	/*
5222 	 * Just requested AWOL.
5223 	 */
5224 	if (num == 0)
5225 		return (0);
5226 	/*
5227 	 * Default READ only.
5228 	 */
5229 	ibuf = (char *)kmem_alloc(blksize, KM_SLEEP);
5230 	if (rw == 1) {
5231 		/*
5232 		 * WRITE only.
5233 		 */
5234 		obuf = ibuf;
5235 		ibuf = NULL;
5236 	} else if (rw == 2) {
5237 		/*
5238 		 * READ/WRITE.
5239 		 */
5240 		obuf = (char *)kmem_alloc(blksize, KM_SLEEP);
5241 		for (segp = &ktextseg; segp; segp = AS_SEGNEXT(&kas, segp)) {
5242 			if (segp->s_size >= blksize)
5243 				break;
5244 		}
5245 		if (segp == NULL) {
5246 			cmn_err(CE_WARN,
5247 			    "IDN: blksize (%d) too large", blksize);
5248 			return (EINVAL);
5249 		}
5250 		bcopy(segp->s_base, obuf, blksize);
5251 	}
5252 
5253 	mutex_init(&slock, NULL, MUTEX_DEFAULT, NULL);
5254 	cv_init(&scv, NULL, CV_DEFAULT, NULL);
5255 
5256 	cmn_err(CE_NOTE,
5257 	    "IDN: starting %s of %d blocks of %d bytes each...",
5258 	    (rw == 1) ? "W-ONLY" : (rw == 2) ? "RW" : "R-ONLY",
5259 	    num, blksize);
5260 
5261 	for (n = 0; n < num; n++) {
5262 		uint_t	rpos;
5263 
5264 		if ((hi_off - lo_off) > blksize)
5265 			rpos = RANDOM(lo_off, (hi_off - blksize));
5266 		else
5267 			rpos = lo_off;
5268 		smraddr = IDN_OFFSET2ADDR(rpos);
5269 
5270 		cpuid = (int)CPU->cpu_id;
5271 #ifdef DEBUG
5272 		idx = rw_index[cpuid]++ % RW_HISTORY;
5273 		rw_history[cpuid][idx] = rpos;
5274 #endif /* DEBUG */
5275 
5276 		switch (rw) {
5277 		case 0:
5278 			bcopy(smraddr, ibuf, blksize);
5279 			break;
5280 		case 1:
5281 			bcopy(obuf, smraddr, blksize);
5282 			break;
5283 		case 2:
5284 			if (n & 1)
5285 				bcopy(obuf, smraddr, blksize);
5286 			else
5287 				bcopy(smraddr, ibuf, blksize);
5288 			break;
5289 		default:
5290 			break;
5291 		}
5292 		if (!(n % 1000)) {
5293 			int	rv;
5294 
5295 			mutex_enter(&slock);
5296 			rv = cv_timedwait_sig(&scv, &slock, lbolt+hz);
5297 			mutex_exit(&slock);
5298 			if (rv == 0)
5299 				break;
5300 		}
5301 	}
5302 
5303 	cv_destroy(&scv);
5304 	mutex_destroy(&slock);
5305 
5306 	if (ibuf)
5307 		kmem_free(ibuf, blksize);
5308 	if (obuf)
5309 		kmem_free(obuf, blksize);
5310 
5311 	return (0);
5312 }
5313 
5314 void
5315 inum2str(uint_t inum, char str[])
5316 {
5317 	uint_t	acknack;
5318 
5319 	str[0] = '\0';
5320 
5321 	acknack = (inum & IDNP_ACKNACK_MASK);
5322 	inum &= ~IDNP_ACKNACK_MASK;
5323 
5324 	if (!inum && !acknack) {
5325 		strcpy(str, idnm_str[0]);
5326 		return;
5327 	}
5328 
5329 	if (inum == 0) {
5330 		strcpy(str, (acknack & IDNP_ACK) ? "ack" : "nack");
5331 	} else {
5332 		if (inum < IDN_NUM_MSGTYPES)
5333 			strcpy(str, idnm_str[inum]);
5334 		else
5335 			sprintf(str, "0x%x?", inum);
5336 		if (acknack) {
5337 			if (acknack & IDNP_ACK)
5338 				strcat(str, "+ack");
5339 			else
5340 				strcat(str, "+nack");
5341 		}
5342 	}
5343 }
5344 
5345 boardset_t
5346 cpuset2boardset(cpuset_t portset)
5347 {
5348 	register int		c;
5349 	register boardset_t	bset;
5350 
5351 	bset = 0;
5352 	for (c = 0; c < NCPU; )
5353 		if (CPU_IN_SET(portset, c)) {
5354 			BOARDSET_ADD(bset, CPUID_TO_BOARDID(c));
5355 			c = (c + 4) & ~3;
5356 		} else {
5357 			c++;
5358 		}
5359 
5360 	return (bset);
5361 }
5362 
5363 void
5364 cpuset2str(cpuset_t cset, char buffer[])
5365 {
5366 	register int	c, n;
5367 
5368 	buffer[0] = '\0';
5369 	for (c = n = 0; c < NCPU; c++) {
5370 		if (!CPU_IN_SET(cset, c))
5371 			continue;
5372 #ifdef DEBUG
5373 		if (strlen(buffer) >= _DSTRLEN) {
5374 			PR_PROTO("************* WARNING WARNING WARNING\n");
5375 			PR_PROTO("cpuset2str(cpu = %d) buffer "
5376 			    "OVERFLOW <<<<<<\n", c);
5377 			PR_PROTO("*******************************\n");
5378 			(void) sprintf(&buffer[_DSTRLEN-6], "*OVER");
5379 			return;
5380 		}
5381 #endif /* DEBUG */
5382 		if (n == 0)
5383 			(void) sprintf(buffer, "%d", c);
5384 		else
5385 			(void) sprintf(buffer, "%s, %d", buffer, c);
5386 		n++;
5387 	}
5388 }
5389 
5390 void
5391 domainset2str(domainset_t dset, char buffer[])
5392 {
5393 	/*
5394 	 * Since domainset_t and boardset_t are the
5395 	 * same (max = MAX_DOMAINS = MAX_BOARDS) we
5396 	 * can just overload boardset2str().
5397 	 */
5398 	mask2str((uint_t)dset, buffer, MAX_DOMAINS);
5399 }
5400 
5401 void
5402 boardset2str(boardset_t bset, char buffer[])
5403 {
5404 	mask2str((uint_t)bset, buffer, MAX_BOARDS);
5405 }
5406 
5407 void
5408 mask2str(uint_t mask, char buffer[], int maxnum)
5409 {
5410 	int	n, i;
5411 
5412 	buffer[0] = '\0';
5413 	for (i = n = 0; i < maxnum; i++) {
5414 		if ((mask & (1 << i)) == 0)
5415 			continue;
5416 		if (n == 0)
5417 			(void) sprintf(buffer, "%d", i);
5418 		else
5419 			(void) sprintf(buffer, "%s, %d", buffer, i);
5420 		n++;
5421 	}
5422 }
5423 
5424 int
5425 idnxdc(int domid, idn_msgtype_t *mtp,
5426 	uint_t arg1, uint_t arg2,
5427 	uint_t arg3, uint_t arg4)
5428 {
5429 	int		rv, cpuid, tcpuid;
5430 	uint_t		cookie;
5431 	uint64_t	pdata;
5432 	uint64_t	dmv_word0, dmv_word1, dmv_word2;
5433 	idn_domain_t	*dp = &idn_domain[domid];
5434 	extern kmutex_t	xc_sys_mutex;
5435 	extern int	xc_spl_enter[];
5436 	procname_t	proc = "idnxdc";
5437 
5438 
5439 	if (idn_snoop) {
5440 		int	bd;
5441 		STRING(str);
5442 		STRING(mstr);
5443 		STRING(astr);
5444 
5445 		INUM2STR(mtp->mt_mtype, mstr);
5446 		if ((mtp->mt_mtype & IDNP_MSGTYPE_MASK) == 0) {
5447 			INUM2STR(arg1, astr);
5448 			sprintf(str, "%s/%s", mstr, astr);
5449 		} else {
5450 			strcpy(str, mstr);
5451 		}
5452 		if (dp->dcpu == IDN_NIL_DCPU)
5453 			bd = -1;
5454 		else
5455 			bd = CPUID_TO_BOARDID(dp->dcpu);
5456 		SNOOP_IDN(0, str, bd, arg1, arg2, arg3, arg4);
5457 	}
5458 
5459 	/*
5460 	 * For NEGO messages we send the remote domain the cookie we
5461 	 * expect it to use in subsequent messages that it sends
5462 	 * to us (dcookie_recv).
5463 	 * For other messages, we must use the cookie that the
5464 	 * remote domain assigned to us for sending (dcookie_send).
5465 	 */
5466 	if ((mtp->mt_mtype & IDNP_MSGTYPE_MASK) == IDNP_NEGO)
5467 		cookie = IDN_MAKE_COOKIE(dp->dcookie_recv, mtp->mt_cookie);
5468 	else
5469 		cookie = IDN_MAKE_COOKIE(dp->dcookie_send, mtp->mt_cookie);
5470 
5471 	pdata = IDN_MAKE_PDATA(mtp->mt_mtype, mtp->mt_atype, cookie);
5472 
5473 	dmv_word0 = DMV_MAKE_DMV(idn.intr.dmv_inum, pdata);
5474 	dmv_word1 = ((uint64_t)arg1 << 32) | (uint64_t)arg2;
5475 	dmv_word2 = ((uint64_t)arg3 << 32) | (uint64_t)arg4;
5476 
5477 	ASSERT((dp->dcpu != IDN_NIL_DCPU) ||
5478 	    (dp->dcpu_last != IDN_NIL_DCPU));
5479 
5480 	tcpuid = (dp->dcpu == IDN_NIL_DCPU) ?
5481 	    dp->dcpu_last : dp->dcpu;
5482 
5483 	if (tcpuid == IDN_NIL_DCPU) {
5484 		PR_PROTO("%s:%d: cpu/cpu_last == NIL_DCPU\n",
5485 		    proc, domid);
5486 		return (-1);
5487 	}
5488 
5489 	mutex_enter(&xc_sys_mutex);
5490 	cpuid = (int)CPU->cpu_id;
5491 	xc_spl_enter[cpuid] = 1;
5492 
5493 	idnxf_init_mondo(dmv_word0, dmv_word1, dmv_word2);
5494 
5495 	rv = idnxf_send_mondo(STARFIRE_UPAID2HWMID(tcpuid));
5496 
5497 	xc_spl_enter[cpuid] = 0;
5498 	mutex_exit(&xc_sys_mutex);
5499 
5500 	return (rv);
5501 }
5502 
5503 void
5504 idnxdc_broadcast(domainset_t domset, idn_msgtype_t *mtp,
5505 		uint_t arg1, uint_t arg2,
5506 		uint_t arg3, uint_t arg4)
5507 {
5508 	int	d;
5509 
5510 	for (d = 0; d < MAX_DOMAINS; d++) {
5511 		idn_domain_t	*dp;
5512 
5513 		if (!DOMAIN_IN_SET(domset, d))
5514 			continue;
5515 
5516 		dp = &idn_domain[d];
5517 		if (dp->dcpu == IDN_NIL_DCPU)
5518 			continue;
5519 
5520 		(void) IDNXDC(d, mtp, arg1, arg2, arg3, arg4);
5521 	}
5522 }
5523 
5524 #define	PROM_SMRSIZE	0x1
5525 #define	PROM_SMRADDR	0x2
5526 #define	PROM_SMRPROPS	(PROM_SMRSIZE | PROM_SMRADDR)
5527 /*
5528  * Locate the idn-smr-size property to determine the size of the SMR
5529  * region for the SSI.  Value inherently enables/disables SSI capability.
5530  */
5531 static int
5532 idn_prom_getsmr(uint_t *smrsz, uint64_t *paddrp, uint64_t *sizep)
5533 {
5534 	pnode_t		nodeid;
5535 	int		found = 0;
5536 	int		len;
5537 	uint_t		smrsize = 0;
5538 	uint64_t	obpaddr, obpsize;
5539 	struct smraddr {
5540 		uint32_t	hi_addr;
5541 		uint32_t	lo_addr;
5542 		uint32_t	hi_size;
5543 		uint32_t	lo_size;
5544 	} smraddr;
5545 	procname_t	proc = "idn_prom_getsmr";
5546 
5547 	bzero(&smraddr, sizeof (smraddr));
5548 	/*
5549 	 * idn-smr-size is a property of the "memory" node and
5550 	 * is defined in megabytes.
5551 	 */
5552 	nodeid = prom_finddevice("/memory");
5553 
5554 	if (nodeid != OBP_NONODE) {
5555 		len = prom_getproplen(nodeid, IDN_PROP_SMRSIZE);
5556 		if (len == sizeof (smrsize)) {
5557 			(void) prom_getprop(nodeid, IDN_PROP_SMRSIZE,
5558 			    (caddr_t)&smrsize);
5559 			found |= PROM_SMRSIZE;
5560 		}
5561 		len = prom_getproplen(nodeid, IDN_PROP_SMRADDR);
5562 		if (len  == sizeof (smraddr)) {
5563 			(void) prom_getprop(nodeid, IDN_PROP_SMRADDR,
5564 			    (caddr_t)&smraddr);
5565 			found |= PROM_SMRADDR;
5566 		}
5567 	}
5568 
5569 	if (found != PROM_SMRPROPS) {
5570 		if ((found & PROM_SMRSIZE) == 0)
5571 			cmn_err(CE_WARN,
5572 			    "IDN: 136: \"%s\" property not found, "
5573 			    "disabling IDN",
5574 			    IDN_PROP_SMRSIZE);
5575 		if (smrsize && ((found & PROM_SMRADDR) == 0))
5576 			cmn_err(CE_WARN,
5577 			    "IDN: 136: \"%s\" property not found, "
5578 			    "disabling IDN",
5579 			    IDN_PROP_SMRADDR);
5580 		return (-1);
5581 	}
5582 
5583 	if (smrsize == 0) {
5584 		PR_SMR("%s: IDN DISABLED (idn_smr_size = 0)\n", proc);
5585 		cmn_err(CE_NOTE, "!IDN: 137: SMR size is 0, disabling IDN");
5586 
5587 	} else if (smrsize > IDN_SMR_MAXSIZE) {
5588 		PR_SMR("%s: IDN DISABLED (idn_smr_size too big %d > %d MB)\n",
5589 		    proc, smrsize, IDN_SMR_MAXSIZE);
5590 		cmn_err(CE_WARN,
5591 		    "!IDN: 138: SMR size (%dMB) is too big (max = %dMB), "
5592 		    "disabling IDN",
5593 		    smrsize, IDN_SMR_MAXSIZE);
5594 		smrsize = 0;
5595 	} else {
5596 		*smrsz = smrsize;
5597 		found &= ~PROM_SMRSIZE;
5598 	}
5599 
5600 	obpaddr = ((uint64_t)smraddr.hi_addr << 32) |
5601 	    (uint64_t)smraddr.lo_addr;
5602 	obpsize = ((uint64_t)smraddr.hi_size << 32) |
5603 	    (uint64_t)smraddr.lo_size;
5604 
5605 	if (obpsize == 0) {
5606 		if (smrsize > 0) {
5607 			cmn_err(CE_WARN, "!IDN: 139: OBP region for "
5608 			    "SMR is 0 length");
5609 		}
5610 	} else if (obpsize < (uint64_t)MB2B(smrsize)) {
5611 		cmn_err(CE_WARN,
5612 		    "!IDN: 140: OBP region (%ld B) smaller "
5613 		    "than requested size (%ld B)",
5614 		    obpsize, MB2B(smrsize));
5615 	} else if ((obpaddr & ((uint64_t)IDN_SMR_ALIGN - 1)) != 0) {
5616 		cmn_err(CE_WARN,
5617 		    "!IDN: 141: OBP region (0x%lx) not on (0x%x) "
5618 		    "boundary", obpaddr, IDN_SMR_ALIGN);
5619 	} else {
5620 		*sizep = obpsize;
5621 		*paddrp = obpaddr;
5622 		found &= ~PROM_SMRADDR;
5623 	}
5624 
5625 	return (found ? -1 : 0);
5626 }
5627 
5628 void
5629 idn_init_autolink()
5630 {
5631 	idnsb_event_t	*sbp;
5632 	procname_t	proc = "idn_init_autolink";
5633 
5634 	mutex_enter(&idn.idnsb_mutex);
5635 	if ((sbp = idn.idnsb_eventp) == NULL) {
5636 		PR_PROTO("%s: IDN private sigb (event) area is NULL\n", proc);
5637 		mutex_exit(&idn.idnsb_mutex);
5638 		return;
5639 	}
5640 
5641 	PR_PROTO("%s: marking domain IDN ready.\n", proc);
5642 
5643 	bzero(sbp, sizeof (*sbp));
5644 
5645 	sbp->idn_version = (uchar_t)idn.version;
5646 	SSIEVENT_SET(sbp, SSIEVENT_BOOT, 0);
5647 	(void) strncpy(sbp->idn_cookie_str, SSIEVENT_COOKIE,
5648 	    SSIEVENT_COOKIE_LEN);
5649 	mutex_exit(&idn.idnsb_mutex);
5650 }
5651 
5652 void
5653 idn_deinit_autolink()
5654 {
5655 	idnsb_event_t	*sbp;
5656 	procname_t	proc = "idn_deinit_autolink";
5657 
5658 	mutex_enter(&idn.idnsb_mutex);
5659 	if ((sbp = idn.idnsb_eventp) == NULL) {
5660 		PR_PROTO("%s: IDN private sigb (event) area is NULL\n", proc);
5661 		mutex_exit(&idn.idnsb_mutex);
5662 		return;
5663 	}
5664 
5665 	PR_PROTO("%s: marking domain IDN unavailable.\n", proc);
5666 
5667 	sbp->idn_version = (uchar_t)idn.version;
5668 	SSIEVENT_CLEAR(sbp, SSIEVENT_BOOT, 0);
5669 	(void) strncpy(sbp->idn_cookie_str, SSIEVENT_COOKIE,
5670 	    SSIEVENT_COOKIE_LEN);
5671 	mutex_exit(&idn.idnsb_mutex);
5672 }
5673 
5674 void
5675 _make64cpumask(cpuset_t *csetp, uint_t upper, uint_t lower)
5676 {
5677 	int	c;
5678 
5679 	CPUSET_ZERO(*csetp);
5680 
5681 	for (c = 0; c < 32; c++) {
5682 		if (lower & (1 << c)) {
5683 			CPUSET_ADD(*csetp, c);
5684 		}
5685 		if (upper & (1 << (c + 32))) {
5686 			CPUSET_ADD(*csetp, c + 32);
5687 		}
5688 	}
5689 }
5690 
5691 uint_t
5692 _lower32cpumask(cpuset_t cset)
5693 {
5694 	int	c;
5695 	uint_t	set = 0;
5696 
5697 	for (c = 0; c < 32; c++)
5698 		if (CPU_IN_SET(cset, c))
5699 			set |= 1 << c;
5700 
5701 	return (set);
5702 }
5703 
5704 uint_t
5705 _upper32cpumask(cpuset_t cset)
5706 {
5707 	int	c;
5708 	uint_t	set = 0;
5709 
5710 	for (c = 32; c < NCPU; c++)
5711 		if (CPU_IN_SET(cset, c))
5712 			set |= 1 << (c - 32);
5713 
5714 	return (set);
5715 }
5716 
5717 #ifdef DEBUG
5718 int
5719 debug_idnxdc(char *f, int domid, idn_msgtype_t *mtp,
5720 		uint_t a1, uint_t a2, uint_t a3, uint_t a4)
5721 {
5722 	idn_domain_t	*dp = &idn_domain[domid];
5723 	int		rv, cpuid, bd;
5724 	static int	xx = 0;
5725 	STRING(str);
5726 	STRING(mstr);
5727 	STRING(astr);
5728 
5729 	xx++;
5730 	INUM2STR(mtp->mt_mtype, mstr);
5731 	if ((mtp->mt_mtype & IDNP_MSGTYPE_MASK) == 0) {
5732 		INUM2STR(a1, astr);
5733 		sprintf(str, "%s/%s", mstr, astr);
5734 	} else {
5735 		strcpy(str, mstr);
5736 	}
5737 
5738 	if ((cpuid = dp->dcpu) == IDN_NIL_DCPU)
5739 		bd = -1;
5740 	else
5741 		bd = CPUID_TO_BOARDID(cpuid);
5742 
5743 	SNOOP_IDN(0, str, bd, a1, a2, a3, a4);
5744 
5745 	PR_XDC("%s:%d:%d SENT: scpu = %d, msg = 0x%x(%s)\n",
5746 	    f, domid, xx, cpuid, mtp->mt_mtype, str);
5747 	PR_XDC("%s:%d:%d S-DATA: a1 = 0x%x, a2 = 0x%x\n",
5748 	    f, domid, xx, a1, a2);
5749 	PR_XDC("%s:%d:%d S-DATA: a3 = 0x%x, a4 = 0x%x\n",
5750 	    f, domid, xx, a3, a4);
5751 
5752 	rv = idnxdc(domid, mtp, a1, a2, a3, a4);
5753 	if (rv != 0) {
5754 		PR_XDC("%s:%d:%d: WARNING: idnxdc(cpu %d) FAILED\n",
5755 		    f, domid, xx, cpuid);
5756 	}
5757 
5758 	return (rv);
5759 }
5760 
5761 caddr_t
5762 _idn_getstruct(char *structname, int size)
5763 {
5764 	caddr_t		ptr;
5765 	procname_t	proc = "GETSTRUCT";
5766 
5767 	ptr = kmem_zalloc(size, KM_SLEEP);
5768 
5769 	PR_ALLOC("%s: ptr 0x%p, struct(%s), size = %d\n",
5770 	    proc, ptr, structname, size);
5771 
5772 	return (ptr);
5773 }
5774 
5775 void
5776 _idn_freestruct(caddr_t ptr, char *structname, int size)
5777 {
5778 	procname_t	proc = "FREESTRUCT";
5779 
5780 	PR_ALLOC("%s: ptr 0x%p, struct(%s), size = %d\n",
5781 	    proc, ptr, structname, size);
5782 
5783 	ASSERT(ptr != NULL);
5784 	kmem_free(ptr, size);
5785 }
5786 #endif /* DEBUG */
5787