xref: /titanic_41/usr/src/cmd/fm/modules/sun4v/etm/etm.c (revision c5fb5d329e745a7b8cb9ff07eebb42948af7bc4e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * etm.c	FMA Event Transport Module implementation, a plugin of FMD
29  *		for sun4v/Ontario
30  *
31  * plugin for sending/receiving FMA events to/from service processor
32  */
33 
34 /*
35  * --------------------------------- includes --------------------------------
36  */
37 
38 #include <sys/fm/protocol.h>
39 #include <sys/fm/util.h>
40 #include <sys/fm/ldom.h>
41 #include <sys/strlog.h>
42 #include <sys/syslog.h>
43 #include <sys/libds.h>
44 #include <netinet/in.h>
45 #include <fm/fmd_api.h>
46 
47 #include "etm_xport_api.h"
48 #include "etm_etm_proto.h"
49 #include "etm_impl.h"
50 #include "etm_iosvc.h"
51 #include "etm_filter.h"
52 #include "etm_ckpt.h"
53 
54 #include <pthread.h>
55 #include <signal.h>
56 #include <stropts.h>
57 #include <locale.h>
58 #include <strings.h>
59 #include <stdlib.h>
60 #include <unistd.h>
61 #include <limits.h>
62 #include <values.h>
63 #include <alloca.h>
64 #include <errno.h>
65 #include <dlfcn.h>
66 #include <link.h>
67 #include <fcntl.h>
68 #include <time.h>
69 
70 /*
71  * ----------------------------- forward decls -------------------------------
72  */
73 
74 static void
75 etm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class);
76 
77 static int
78 etm_send(fmd_hdl_t *hdl, fmd_xprt_t *xp, fmd_event_t *event, nvlist_t *nvl);
79 
80 static void
81 etm_send_to_remote_root(void *arg);
82 
83 static void
84 etm_recv_from_remote_root(void *arg);
85 
86 static void
87 etm_ckpt_remove(fmd_hdl_t *hdl, etm_iosvc_q_ele_t *ele);
88 
89 /*
90  * ------------------------- data structs for FMD ----------------------------
91  */
92 
93 static const fmd_hdl_ops_t fmd_ops = {
94 	etm_recv,	/* fmdo_recv */
95 	NULL,		/* fmdo_timeout */
96 	NULL,		/* fmdo_close */
97 	NULL,		/* fmdo_stats */
98 	NULL,		/* fmdo_gc */
99 	etm_send,	/* fmdo_send */
100 };
101 
102 static const fmd_prop_t fmd_props[] = {
103 	{ ETM_PROP_NM_XPORT_ADDRS,		FMD_TYPE_STRING, "" },
104 	{ ETM_PROP_NM_DEBUG_LVL,		FMD_TYPE_INT32, "0" },
105 	{ ETM_PROP_NM_DEBUG_MAX_EV_CNT,		FMD_TYPE_INT32, "-1" },
106 	{ ETM_PROP_NM_CONSOLE,			FMD_TYPE_BOOL, "false" },
107 	{ ETM_PROP_NM_SYSLOGD,			FMD_TYPE_BOOL, "true" },
108 	{ ETM_PROP_NM_FACILITY,			FMD_TYPE_STRING, "LOG_DAEMON" },
109 	{ ETM_PROP_NM_MAX_RESP_Q_LEN,		FMD_TYPE_UINT32, "32" },
110 	{ ETM_PROP_NM_BAD_ACC_TO_SEC,		FMD_TYPE_UINT32, "1" },
111 	{ ETM_PROP_NM_FMA_RESP_WAIT_TIME,	FMD_TYPE_INT32, "240" },
112 	{ NULL, 0, NULL }
113 };
114 
115 
116 static const fmd_hdl_info_t fmd_info = {
117 	"FMA Event Transport Module", "1.2", &fmd_ops, fmd_props
118 };
119 
120 /*
121  * ----------------------- private consts and defns --------------------------
122  */
123 
124 /* misc buffer for variable sized protocol header fields */
125 
126 #define	ETM_MISC_BUF_SZ	(4 * 1024)
127 
128 static uint32_t
129 etm_ldom_type = LDOM_TYPE_LEGACY;
130 
131 /* try limit for IO operations w/ capped exp backoff sleep on retry */
132 
133 /*
134  * Design_Note:	ETM will potentially retry forever IO operations that the
135  *		transport fails with EAGAIN (aka EWOULDBLOCK) rather than
136  *		giving up after some number of seconds. This avoids
137  *		dropping FMA events while the service processor is down,
138  *		but at the risk of pending fmdo_recv() forever and
139  *		overflowing FMD's event queue for ETM.
140  *		A future TBD enhancement would be to always recv
141  *		and send each ETM msg in a single read/write() to reduce
142  *		the risk of failure between ETM msg hdr and body,
143  *		assuming the MTU_SZ is large enough.
144  */
145 
146 #define	ETM_TRY_MAX_CNT		(MAXINT - 1)
147 #define	ETM_TRY_BACKOFF_RATE	(4)
148 #define	ETM_TRY_BACKOFF_CAP	(60)
149 
150 /* amount to increment protocol transaction id on each new send */
151 
152 #define	ETM_XID_INC		(2)
153 
154 typedef struct etm_resp_q_ele {
155 
156 	etm_xport_conn_t	rqe_conn;	/* open connection to send on */
157 	etm_proto_v1_pp_t	*rqe_hdrp;	/* ptr to ETM msg hdr */
158 	size_t			rqe_hdr_sz;	/* sizeof ETM msg hdr */
159 	int32_t			rqe_resp_code;	/* response code to send */
160 
161 	struct etm_resp_q_ele	*rqe_nextp;	/* PRIVATE - next ele ptr */
162 
163 } etm_resp_q_ele_t;	/* responder queue element */
164 
165 /*
166  * ---------------------------- global data ----------------------------------
167  */
168 
169 static fmd_hdl_t
170 *init_hdl = NULL;	/* used in mem allocator and several other places */
171 
172 static int
173 etm_debug_lvl = 0;	/* debug level: 0 is off, 1 is on, 2 is more, etc */
174 
175 static int
176 etm_debug_max_ev_cnt = -1; /* max allowed event count for debugging */
177 
178 static fmd_xprt_t
179 *etm_fmd_xprt = NULL;	/* FMD transport layer handle */
180 
181 static pthread_t
182 etm_svr_tid = NULL;	/* thread id of connection acceptance server */
183 
184 static pthread_t
185 etm_resp_tid = NULL;	/* thread id of msg responder */
186 
187 static etm_resp_q_ele_t
188 *etm_resp_q_head = NULL; /* ptr to cur head of responder queue */
189 
190 static etm_resp_q_ele_t
191 *etm_resp_q_tail = NULL; /* ptr to cur tail of responder queue */
192 
193 static uint32_t
194 etm_resp_q_cur_len = 0;	/* cur length (ele cnt) of responder queue */
195 
196 static uint32_t
197 etm_resp_q_max_len = 0;	/* max length (ele cnt) of responder queue */
198 
199 static uint32_t
200 etm_bad_acc_to_sec = 0;	/* sleep timeout (in sec) after bad conn accept */
201 
202 static pthread_mutex_t
203 etm_resp_q_lock = PTHREAD_MUTEX_INITIALIZER;	/* protects responder queue */
204 
205 static pthread_cond_t
206 etm_resp_q_cv = PTHREAD_COND_INITIALIZER;	/* nudges msg responder */
207 
208 static volatile int
209 etm_is_dying = 0;	/* bool for dying (killing self) */
210 
211 static uint32_t
212 etm_xid_cur = 0;	/* current transaction id for sends */
213 
214 static uint32_t
215 etm_xid_ping = 0;	/* xid of last CONTROL msg sent requesting ping */
216 
217 static uint32_t
218 etm_xid_ver_negot = 0;	/* xid of last CONTROL msg sent requesting ver negot */
219 
220 static uint32_t
221 etm_xid_posted_logged_ev = 0;
222 			/* xid of last FMA_EVENT msg/event posted OK to FMD */
223 
224 static uint32_t
225 etm_xid_posted_sa = 0;	/* xid of last ALERT msg/event posted OK to syslog */
226 
227 static uint8_t
228 etm_resp_ver = ETM_PROTO_V1; /* proto ver [negotiated] for msg sends */
229 
230 static uint32_t
231 etm_fma_resp_wait_time = 30;	/*  time (sec) wait for fma event resp */
232 
233 static pthread_mutex_t
234 etm_write_lock = PTHREAD_MUTEX_INITIALIZER;	/* for write operations */
235 
236 static log_ctl_t syslog_ctl;	/* log(7D) meta-data for each msg */
237 static int syslog_facility;	/* log(7D) facility (part of priority) */
238 static int syslog_logfd = -1;	/* log(7D) file descriptor */
239 static int syslog_msgfd = -1;	/* sysmsg(7D) file descriptor */
240 static int syslog_file = 0;	/* log to syslog_logfd */
241 static int syslog_cons = 0;	/* log to syslog_msgfd */
242 
243 static const struct facility {
244 	const char *fac_name;
245 	int fac_value;
246 } syslog_facs[] = {
247 	{ "LOG_DAEMON", LOG_DAEMON },
248 	{ "LOG_LOCAL0", LOG_LOCAL0 },
249 	{ "LOG_LOCAL1", LOG_LOCAL1 },
250 	{ "LOG_LOCAL2", LOG_LOCAL2 },
251 	{ "LOG_LOCAL3", LOG_LOCAL3 },
252 	{ "LOG_LOCAL4", LOG_LOCAL4 },
253 	{ "LOG_LOCAL5", LOG_LOCAL5 },
254 	{ "LOG_LOCAL6", LOG_LOCAL6 },
255 	{ "LOG_LOCAL7", LOG_LOCAL7 },
256 	{ NULL, 0 }
257 };
258 
259 static struct stats {
260 
261 	/* ETM msg counters */
262 
263 	fmd_stat_t etm_rd_hdr_fmaevent;
264 	fmd_stat_t etm_rd_hdr_control;
265 	fmd_stat_t etm_rd_hdr_alert;
266 	fmd_stat_t etm_rd_hdr_response;
267 	fmd_stat_t etm_rd_body_fmaevent;
268 	fmd_stat_t etm_rd_body_control;
269 	fmd_stat_t etm_rd_body_alert;
270 	fmd_stat_t etm_rd_body_response;
271 	fmd_stat_t etm_wr_hdr_fmaevent;
272 	fmd_stat_t etm_wr_hdr_control;
273 	fmd_stat_t etm_wr_hdr_response;
274 	fmd_stat_t etm_wr_body_fmaevent;
275 	fmd_stat_t etm_wr_body_control;
276 	fmd_stat_t etm_wr_body_response;
277 
278 	fmd_stat_t etm_rd_max_ev_per_msg;
279 	fmd_stat_t etm_wr_max_ev_per_msg;
280 
281 	fmd_stat_t etm_resp_q_cur_len;
282 	fmd_stat_t etm_resp_q_max_len;
283 
284 	/* ETM byte counters */
285 
286 	fmd_stat_t etm_wr_fmd_bytes;
287 	fmd_stat_t etm_rd_fmd_bytes;
288 	fmd_stat_t etm_wr_xport_bytes;
289 	fmd_stat_t etm_rd_xport_bytes;
290 
291 	fmd_stat_t etm_magic_drop_bytes;
292 
293 	/* ETM [dropped] FMA event counters */
294 
295 	fmd_stat_t etm_rd_fmd_fmaevent;
296 	fmd_stat_t etm_wr_fmd_fmaevent;
297 
298 	fmd_stat_t etm_rd_drop_fmaevent;
299 	fmd_stat_t etm_wr_drop_fmaevent;
300 
301 	fmd_stat_t etm_rd_dup_fmaevent;
302 	fmd_stat_t etm_wr_dup_fmaevent;
303 
304 	fmd_stat_t etm_rd_dup_alert;
305 	fmd_stat_t etm_wr_dup_alert;
306 
307 	fmd_stat_t etm_enq_drop_resp_q;
308 	fmd_stat_t etm_deq_drop_resp_q;
309 
310 	/* ETM protocol failures */
311 
312 	fmd_stat_t etm_magic_bad;
313 	fmd_stat_t etm_ver_bad;
314 	fmd_stat_t etm_msgtype_bad;
315 	fmd_stat_t etm_subtype_bad;
316 	fmd_stat_t etm_xid_bad;
317 	fmd_stat_t etm_fmaeventlen_bad;
318 	fmd_stat_t etm_respcode_bad;
319 	fmd_stat_t etm_timeout_bad;
320 	fmd_stat_t etm_evlens_bad;
321 
322 	/* IO operation failures */
323 
324 	fmd_stat_t etm_xport_wr_fail;
325 	fmd_stat_t etm_xport_rd_fail;
326 	fmd_stat_t etm_xport_pk_fail;
327 
328 	/* IO operation retries */
329 
330 	fmd_stat_t etm_xport_wr_retry;
331 	fmd_stat_t etm_xport_rd_retry;
332 	fmd_stat_t etm_xport_pk_retry;
333 
334 	/* system and library failures */
335 
336 	fmd_stat_t etm_os_nvlist_pack_fail;
337 	fmd_stat_t etm_os_nvlist_unpack_fail;
338 	fmd_stat_t etm_os_nvlist_size_fail;
339 	fmd_stat_t etm_os_pthread_create_fail;
340 
341 	/* xport API failures */
342 
343 	fmd_stat_t etm_xport_get_ev_addrv_fail;
344 	fmd_stat_t etm_xport_open_fail;
345 	fmd_stat_t etm_xport_close_fail;
346 	fmd_stat_t etm_xport_accept_fail;
347 	fmd_stat_t etm_xport_open_retry;
348 
349 	/* FMD entry point bad arguments */
350 
351 	fmd_stat_t etm_fmd_init_badargs;
352 	fmd_stat_t etm_fmd_fini_badargs;
353 
354 	/* Alert logging errors */
355 
356 	fmd_stat_t etm_log_err;
357 	fmd_stat_t etm_msg_err;
358 
359 	/* miscellaneous stats */
360 
361 	fmd_stat_t etm_reset_xport;
362 
363 } etm_stats = {
364 
365 	/* ETM msg counters */
366 
367 	{ "etm_rd_hdr_fmaevent", FMD_TYPE_UINT64,
368 		"ETM fmaevent msg headers rcvd from xport" },
369 	{ "etm_rd_hdr_control", FMD_TYPE_UINT64,
370 		"ETM control msg headers rcvd from xport" },
371 	{ "etm_rd_hdr_alert", FMD_TYPE_UINT64,
372 		"ETM alert msg headers rcvd from xport" },
373 	{ "etm_rd_hdr_response", FMD_TYPE_UINT64,
374 		"ETM response msg headers rcvd from xport" },
375 	{ "etm_rd_body_fmaevent", FMD_TYPE_UINT64,
376 		"ETM fmaevent msg bodies rcvd from xport" },
377 	{ "etm_rd_body_control", FMD_TYPE_UINT64,
378 		"ETM control msg bodies rcvd from xport" },
379 	{ "etm_rd_body_alert", FMD_TYPE_UINT64,
380 		"ETM alert msg bodies rcvd from xport" },
381 	{ "etm_rd_body_response", FMD_TYPE_UINT64,
382 		"ETM response msg bodies rcvd from xport" },
383 	{ "etm_wr_hdr_fmaevent", FMD_TYPE_UINT64,
384 		"ETM fmaevent msg headers sent to xport" },
385 	{ "etm_wr_hdr_control", FMD_TYPE_UINT64,
386 		"ETM control msg headers sent to xport" },
387 	{ "etm_wr_hdr_response", FMD_TYPE_UINT64,
388 		"ETM response msg headers sent to xport" },
389 	{ "etm_wr_body_fmaevent", FMD_TYPE_UINT64,
390 		"ETM fmaevent msg bodies sent to xport" },
391 	{ "etm_wr_body_control", FMD_TYPE_UINT64,
392 		"ETM control msg bodies sent to xport" },
393 	{ "etm_wr_body_response", FMD_TYPE_UINT64,
394 		"ETM response msg bodies sent to xport" },
395 
396 	{ "etm_rd_max_ev_per_msg", FMD_TYPE_UINT64,
397 		"max FMA events per ETM msg from xport" },
398 	{ "etm_wr_max_ev_per_msg", FMD_TYPE_UINT64,
399 		"max FMA events per ETM msg to xport" },
400 
401 	{ "etm_resp_q_cur_len", FMD_TYPE_UINT64,
402 		"cur enqueued response msgs to xport" },
403 	{ "etm_resp_q_max_len", FMD_TYPE_UINT64,
404 		"max enqueable response msgs to xport" },
405 
406 	/* ETM byte counters */
407 
408 	{ "etm_wr_fmd_bytes", FMD_TYPE_UINT64,
409 		"bytes of FMA events sent to FMD" },
410 	{ "etm_rd_fmd_bytes", FMD_TYPE_UINT64,
411 		"bytes of FMA events rcvd from FMD" },
412 	{ "etm_wr_xport_bytes", FMD_TYPE_UINT64,
413 		"bytes of FMA events sent to xport" },
414 	{ "etm_rd_xport_bytes", FMD_TYPE_UINT64,
415 		"bytes of FMA events rcvd from xport" },
416 
417 	{ "etm_magic_drop_bytes", FMD_TYPE_UINT64,
418 		"bytes dropped from xport pre magic num" },
419 
420 	/* ETM [dropped] FMA event counters */
421 
422 	{ "etm_rd_fmd_fmaevent", FMD_TYPE_UINT64,
423 		"FMA events rcvd from FMD" },
424 	{ "etm_wr_fmd_fmaevent", FMD_TYPE_UINT64,
425 		"FMA events sent to FMD" },
426 
427 	{ "etm_rd_drop_fmaevent", FMD_TYPE_UINT64,
428 		"dropped FMA events from xport" },
429 	{ "etm_wr_drop_fmaevent", FMD_TYPE_UINT64,
430 		"dropped FMA events to xport" },
431 
432 	{ "etm_rd_dup_fmaevent", FMD_TYPE_UINT64,
433 	    "duplicate FMA events rcvd from xport" },
434 	{ "etm_wr_dup_fmaevent", FMD_TYPE_UINT64,
435 	    "duplicate FMA events sent to xport" },
436 
437 	{ "etm_rd_dup_alert", FMD_TYPE_UINT64,
438 	    "duplicate ALERTs rcvd from xport" },
439 	{ "etm_wr_dup_alert", FMD_TYPE_UINT64,
440 	    "duplicate ALERTs sent to xport" },
441 
442 	{ "etm_enq_drop_resp_q", FMD_TYPE_UINT64,
443 	    "dropped response msgs on enq" },
444 	{ "etm_deq_drop_resp_q", FMD_TYPE_UINT64,
445 	    "dropped response msgs on deq" },
446 
447 	/* ETM protocol failures */
448 
449 	{ "etm_magic_bad", FMD_TYPE_UINT64,
450 		"ETM msgs w/ invalid magic num" },
451 	{ "etm_ver_bad", FMD_TYPE_UINT64,
452 		"ETM msgs w/ invalid protocol version" },
453 	{ "etm_msgtype_bad", FMD_TYPE_UINT64,
454 		"ETM msgs w/ invalid message type" },
455 	{ "etm_subtype_bad", FMD_TYPE_UINT64,
456 		"ETM msgs w/ invalid sub type" },
457 	{ "etm_xid_bad", FMD_TYPE_UINT64,
458 		"ETM msgs w/ unmatched xid" },
459 	{ "etm_fmaeventlen_bad", FMD_TYPE_UINT64,
460 		"ETM msgs w/ invalid FMA event length" },
461 	{ "etm_respcode_bad", FMD_TYPE_UINT64,
462 		"ETM msgs w/ invalid response code" },
463 	{ "etm_timeout_bad", FMD_TYPE_UINT64,
464 		"ETM msgs w/ invalid timeout value" },
465 	{ "etm_evlens_bad", FMD_TYPE_UINT64,
466 		"ETM msgs w/ too many event lengths" },
467 
468 	/* IO operation failures */
469 
470 	{ "etm_xport_wr_fail", FMD_TYPE_UINT64,
471 		"xport write failures" },
472 	{ "etm_xport_rd_fail", FMD_TYPE_UINT64,
473 		"xport read failures" },
474 	{ "etm_xport_pk_fail", FMD_TYPE_UINT64,
475 		"xport peek failures" },
476 
477 	/* IO operation retries */
478 
479 	{ "etm_xport_wr_retry", FMD_TYPE_UINT64,
480 		"xport write retries" },
481 	{ "etm_xport_rd_retry", FMD_TYPE_UINT64,
482 		"xport read retries" },
483 	{ "etm_xport_pk_retry", FMD_TYPE_UINT64,
484 		"xport peek retries" },
485 
486 	/* system and library failures */
487 
488 	{ "etm_os_nvlist_pack_fail", FMD_TYPE_UINT64,
489 		"nvlist_pack failures" },
490 	{ "etm_os_nvlist_unpack_fail", FMD_TYPE_UINT64,
491 		"nvlist_unpack failures" },
492 	{ "etm_os_nvlist_size_fail", FMD_TYPE_UINT64,
493 		"nvlist_size failures" },
494 	{ "etm_os_pthread_create_fail", FMD_TYPE_UINT64,
495 		"pthread_create failures" },
496 
497 	/* transport API failures */
498 
499 	{ "etm_xport_get_ev_addrv_fail", FMD_TYPE_UINT64,
500 		"xport get event addrv API failures" },
501 	{ "etm_xport_open_fail", FMD_TYPE_UINT64,
502 		"xport open API failures" },
503 	{ "etm_xport_close_fail", FMD_TYPE_UINT64,
504 		"xport close API failures" },
505 	{ "etm_xport_accept_fail", FMD_TYPE_UINT64,
506 		"xport accept API failures" },
507 	{ "etm_xport_open_retry", FMD_TYPE_UINT64,
508 		"xport open API retries" },
509 
510 	/* FMD entry point bad arguments */
511 
512 	{ "etm_fmd_init_badargs", FMD_TYPE_UINT64,
513 	    "bad arguments from fmd_init entry point" },
514 	{ "etm_fmd_fini_badargs", FMD_TYPE_UINT64,
515 	    "bad arguments from fmd_fini entry point" },
516 
517 	/* Alert logging errors */
518 
519 	{ "etm_log_err", FMD_TYPE_UINT64,
520 		"failed to log message to log(7D)" },
521 	{ "etm_msg_err", FMD_TYPE_UINT64,
522 		"failed to log message to sysmsg(7D)" },
523 
524 	/* miscellaneous stats */
525 
526 	{ "etm_reset_xport", FMD_TYPE_UINT64,
527 		"xport resets after xport API failure" }
528 };
529 
530 
531 /*
532  * -------------------- global data for Root ldom-------------------------
533  */
534 
535 ldom_hdl_t
536 *etm_lhp = NULL;		/* ldom pointer */
537 
538 static void *etm_dl_hdl = (void *)NULL;
539 static const char *etm_dl_path = "libds.so.1";
540 static int etm_dl_mode = (RTLD_NOW | RTLD_LOCAL);
541 
542 static int(*etm_ds_svc_reg)(ds_capability_t *cap, ds_ops_t *ops) =
543 	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
544 static int(*etm_ds_clnt_reg)(ds_capability_t *cap, ds_ops_t *ops) =
545 	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
546 static int(*etm_ds_send_msg)(ds_hdl_t hdl, void *buf, size_t buflen) =
547 	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen))NULL;
548 static int(*etm_ds_recv_msg)(ds_hdl_t hdl, void *buf, size_t buflen,
549     size_t *msglen) =
550 	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen, size_t *msglen))NULL;
551 static int (*etm_ds_fini)(void) = (int (*)(void))NULL;
552 
553 static pthread_mutex_t
554 iosvc_list_lock =  PTHREAD_MUTEX_INITIALIZER;
555 
556 static pthread_t
557 etm_async_e_tid = NULL;	/* thread id of io svc async event handler */
558 
559 static etm_proto_v1_ev_hdr_t iosvc_hdr = {
560 	ETM_PROTO_MAGIC_NUM,	/* magic number */
561 	ETM_PROTO_V1,		/* default to V1, not checked */
562 	ETM_MSG_TYPE_FMA_EVENT,	/* Root Domain inteoduces only FMA events */
563 	0,			/* sub-type */
564 	0,			/* pad */
565 	0,			/* add the xid at the Q send time */
566 	ETM_PROTO_V1_TIMEOUT_NONE,
567 	0			/* ev_lens, 0-termed, after 1 FMA event */
568 };
569 
570 /*
571  * static iosvc_list
572  */
573 static etm_iosvc_t iosvc_list[NUM_OF_ROOT_DOMAINS] = {
574 	{"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0},
575 	{"", 0}, {"", 0}
576 };
577 
578 static etm_iosvc_t io_svc = {
579 	"\0",				/* ldom_name */
580 	PTHREAD_COND_INITIALIZER,	/* nudges */
581 	PTHREAD_MUTEX_INITIALIZER,	/* protects the iosvc msg Q */
582 	NULL,				/* iosvc msg Q head */
583 	NULL,				/* iosvc msg Q tail */
584 	0,				/* msg Q current length */
585 	100,				/* msg Q max length */
586 	0,				/* current transaction id */
587 	0,				/* xid of last event posted to FMD */
588 	DS_INVALID_HDL,			/* DS handle */
589 	NULL,				/* fmd xprt handle */
590 	NULL,				/* tid 4 send to remote RootDomain */
591 	NULL,				/* tid 4 recv from remote RootDomain */
592 	PTHREAD_COND_INITIALIZER,	/* nudges etm_send_to_remote_root */
593 	PTHREAD_MUTEX_INITIALIZER,	/* protects msg_ack_cv */
594 	0,				/* send/recv threads are not dying */
595 	0,				/* flag for start sending msg Q */
596 	0				/* indicate if the ACK has come  */
597 };
598 etm_iosvc_t *io_svc_p = &io_svc;
599 
600 
601 static uint32_t
602 flags;					/* flags for fmd_xprt_open */
603 
604 static etm_async_event_ele_t
605 async_event_q[ASYNC_EVENT_Q_SIZE];	/* holds the async events */
606 
607 static uint32_t
608 etm_async_q_head = 0;		/* ptr to cur head of async event queue */
609 
610 static uint32_t
611 etm_async_q_tail = 0;		/* ptr to cur tail of async event queue */
612 
613 static uint32_t
614 etm_async_q_cur_len = 0;	/* cur length (ele cnt) of async event queue */
615 
616 static uint32_t
617 etm_async_q_max_len = ASYNC_EVENT_Q_SIZE;
618 				/* max length (ele cnt) of async event queue */
619 
620 static pthread_cond_t
621 etm_async_event_q_cv = PTHREAD_COND_INITIALIZER;
622 				/* nudges  async event handler */
623 
624 static pthread_mutex_t
625 etm_async_event_q_lock = PTHREAD_MUTEX_INITIALIZER;
626 				/* protects async event q */
627 
628 static ds_ver_t
629 etm_iosvc_vers[] = { { 1, 0} };
630 
631 #define	ETM_NVERS	(sizeof (etm_iosvc_vers) / sizeof (ds_ver_t))
632 
633 static ds_capability_t
634 iosvc_caps = {
635 	"ETM",				/* svc_id */
636 	etm_iosvc_vers,			/* vers */
637 	ETM_NVERS			/* number of vers */
638 };
639 
640 static void
641 etm_iosvc_reg_handler(ds_hdl_t hdl, ds_cb_arg_t arg, ds_ver_t *ver,
642     ds_domain_hdl_t did);
643 
644 static void
645 etm_iosvc_unreg_handler(ds_hdl_t hdl, ds_cb_arg_t arg);
646 
647 static ds_ops_t
648 iosvc_ops = {
649 	etm_iosvc_reg_handler,		/* ds_reg_cb */
650 	etm_iosvc_unreg_handler,	/* ds_unreg_cb */
651 	NULL,				/* ds_data_cb */
652 	NULL				/* cb_arg */
653 };
654 
655 
656 /*
657  * -------------------------- support functions ------------------------------
658  */
659 
660 /*
661  * Design_Note:	Each failure worth reporting to FMD should be done using
662  *		a single call to fmd_hdl_error() as it logs an FMA event
663  *		for each call. Also be aware that all the fmd_hdl_*()
664  *		format strings currently use platform specific *printf()
665  *		routines; so "%p" under Solaris does not prepend "0x" to
666  *		the outputted hex digits, while Linux and VxWorks do.
667  */
668 
669 
670 /*
671  * etm_show_time - display the current time of day (for debugging) using
672  *		the given FMD module handle and annotation string
673  */
674 
675 static void
etm_show_time(fmd_hdl_t * hdl,char * note_str)676 etm_show_time(fmd_hdl_t *hdl, char *note_str)
677 {
678 	struct timeval		tmv;		/* timeval */
679 
680 	(void) gettimeofday(&tmv, NULL);
681 	fmd_hdl_debug(hdl, "info: %s: cur Unix Epoch time %d.%06d\n",
682 	    note_str, tmv.tv_sec, tmv.tv_usec);
683 
684 } /* etm_show_time() */
685 
686 /*
687  * etm_hexdump - hexdump the given buffer (for debugging) using
688  *		the given FMD module handle
689  */
690 
691 static void
etm_hexdump(fmd_hdl_t * hdl,void * buf,size_t byte_cnt)692 etm_hexdump(fmd_hdl_t *hdl, void *buf, size_t byte_cnt)
693 {
694 	uint8_t		*bp;		/* byte ptr */
695 	int		i, j;		/* index */
696 	char		cb[80];		/* char buf */
697 	unsigned int	n;		/* a byte of data for sprintf() */
698 
699 	bp = buf;
700 	j = 0;
701 
702 	/*
703 	 * Design_Note:	fmd_hdl_debug() auto adds a newline if missing;
704 	 *		hence cb exists to accumulate a longer string.
705 	 */
706 
707 	for (i = 1; i <= byte_cnt; i++) {
708 		n = *bp++;
709 		(void) sprintf(&cb[j], "%2.2x ", n);
710 		j += 3;
711 		/* add a newline every 16 bytes or at the buffer's end */
712 		if (((i % 16) == 0) || (i >= byte_cnt)) {
713 			cb[j-1] = '\0';
714 			fmd_hdl_debug(hdl, "%s\n", cb);
715 			j = 0;
716 		}
717 	} /* for each byte in the buffer */
718 
719 } /* etm_hexdump() */
720 
721 /*
722  * etm_sleep - sleep the caller for the given number of seconds,
723  *		return 0 or -errno value
724  *
725  * Design_Note:	To avoid interfering with FMD's signal mask (SIGALRM)
726  *		do not use [Solaris] sleep(3C) and instead use
727  *		pthread_cond_wait() or nanosleep(), both of which
728  *		are POSIX spec-ed to leave signal masks alone.
729  *		This is needed for Solaris and Linux (domain and SP).
730  */
731 
732 static int
etm_sleep(unsigned sleep_sec)733 etm_sleep(unsigned sleep_sec)
734 {
735 	struct timespec	tms;	/* for nanosleep() */
736 
737 	tms.tv_sec = sleep_sec;
738 	tms.tv_nsec = 0;
739 
740 	if (nanosleep(&tms, NULL) < 0) {
741 		/* errno assumed set by above call */
742 		return (-errno);
743 	}
744 	return (0);
745 
746 } /* etm_sleep() */
747 
748 /*
749  * etm_conn_open - open a connection to the given transport address,
750  *		return 0 and the opened connection handle
751  *		or -errno value
752  *
753  * caveats:	the err_substr is used in failure cases for calling
754  *		fmd_hdl_error()
755  */
756 
757 static int
etm_conn_open(fmd_hdl_t * hdl,char * err_substr,etm_xport_addr_t addr,etm_xport_conn_t * connp)758 etm_conn_open(fmd_hdl_t *hdl, char *err_substr,
759 		etm_xport_addr_t addr, etm_xport_conn_t *connp)
760 {
761 	etm_xport_conn_t	conn;	/* connection to return */
762 	int			nev;	/* -errno value */
763 
764 	if ((conn = etm_xport_open(hdl, addr)) == NULL) {
765 		nev = (-errno);
766 		fmd_hdl_error(hdl, "error: %s: errno %d\n",
767 		    err_substr, errno);
768 		etm_stats.etm_xport_open_fail.fmds_value.ui64++;
769 		return (nev);
770 	} else {
771 		*connp = conn;
772 		return (0);
773 	}
774 } /* etm_conn_open() */
775 
776 /*
777  * etm_conn_close - close the given connection,
778  *		return 0 or -errno value
779  *
780  * caveats:	the err_substr is used in failure cases for calling
781  *		fmd_hdl_error()
782  */
783 
784 static int
etm_conn_close(fmd_hdl_t * hdl,char * err_substr,etm_xport_conn_t conn)785 etm_conn_close(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn)
786 {
787 	int	nev;	/* -errno value */
788 
789 	if (etm_xport_close(hdl, conn) == NULL) {
790 		nev = (-errno);
791 		fmd_hdl_error(hdl, "warning: %s: errno %d\n",
792 		    err_substr, errno);
793 		etm_stats.etm_xport_close_fail.fmds_value.ui64++;
794 		return (nev);
795 	} else {
796 		return (0);
797 	}
798 } /* etm_conn_close() */
799 
800 /*
801  * etm_io_op - perform an IO operation on the given connection
802  *		with the given buffer,
803  *		accommodating MTU size and retrying op if needed,
804  *		return how many bytes actually done by the op
805  *		or -errno value
806  *
807  * caveats:	the err_substr is used in failure cases for calling
808  *		fmd_hdl_error()
809  */
810 
811 static ssize_t
etm_io_op(fmd_hdl_t * hdl,char * err_substr,etm_xport_conn_t conn,void * buf,size_t byte_cnt,int io_op)812 etm_io_op(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn,
813 				void *buf, size_t byte_cnt, int io_op)
814 {
815 	ssize_t		rv;		/* ret val / byte count */
816 	ssize_t		n;		/* gen use */
817 	uint8_t		*datap;		/* ptr to data */
818 	size_t		mtu_sz;		/* MTU size in bytes */
819 	int		(*io_func_ptr)(fmd_hdl_t *, etm_xport_conn_t,
820 	    void *, size_t);
821 	size_t		io_sz;		/* byte count for io_func_ptr */
822 	int		try_cnt;	/* number of tries done */
823 	int		sleep_sec;	/* exp backoff sleep period in sec */
824 	int		sleep_rv;	/* ret val from sleeping */
825 	fmd_stat_t	io_retry_stat;	/* IO retry stat to update */
826 	fmd_stat_t	io_fail_stat;	/* IO failure stat to update */
827 
828 	if ((conn == NULL) || (buf == NULL)) {
829 		return (-EINVAL);
830 	}
831 	switch (io_op) {
832 	case ETM_IO_OP_RD:
833 		io_func_ptr = etm_xport_read;
834 		io_retry_stat = etm_stats.etm_xport_rd_retry;
835 		io_fail_stat = etm_stats.etm_xport_rd_fail;
836 		break;
837 	case ETM_IO_OP_WR:
838 		io_func_ptr = etm_xport_write;
839 		io_retry_stat = etm_stats.etm_xport_wr_retry;
840 		io_fail_stat = etm_stats.etm_xport_wr_fail;
841 		break;
842 	default:
843 		return (-EINVAL);
844 	}
845 	if (byte_cnt == 0) {
846 		return (byte_cnt);	/* nop */
847 	}
848 
849 	/* obtain [current] MTU size */
850 
851 	if ((n = etm_xport_get_opt(hdl, conn, ETM_XPORT_OPT_MTU_SZ)) < 0) {
852 		mtu_sz = ETM_XPORT_MTU_SZ_DEF;
853 	} else {
854 		mtu_sz = n;
855 	}
856 
857 	/* loop until all IO done, try limit exceeded, or real failure */
858 
859 	rv = 0;
860 	datap = buf;
861 	while (rv < byte_cnt) {
862 		io_sz = MIN((byte_cnt - rv), mtu_sz);
863 		try_cnt = 0;
864 		sleep_sec = 0;
865 
866 		/* when give up, return -errno value even if partly done */
867 
868 		while ((n = (*io_func_ptr)(hdl, conn, datap, io_sz)) ==
869 		    (-EAGAIN)) {
870 			try_cnt++;
871 			if (try_cnt > ETM_TRY_MAX_CNT) {
872 				rv = n;
873 				goto func_ret;
874 			}
875 			if (etm_is_dying) {
876 				rv = (-EINTR);
877 				goto func_ret;
878 			}
879 			if ((sleep_rv = etm_sleep(sleep_sec)) < 0) {
880 				rv = sleep_rv;
881 				goto func_ret;
882 			}
883 			sleep_sec = ((sleep_sec == 0) ? 1 :
884 			    (sleep_sec * ETM_TRY_BACKOFF_RATE));
885 			sleep_sec = MIN(sleep_sec, ETM_TRY_BACKOFF_CAP);
886 			io_retry_stat.fmds_value.ui64++;
887 			if (etm_debug_lvl >= 1) {
888 				fmd_hdl_debug(hdl, "info: retrying io op %d "
889 				    "due to EAGAIN\n", io_op);
890 			}
891 		} /* while trying the io operation */
892 
893 		if (etm_is_dying) {
894 			rv = (-EINTR);
895 			goto func_ret;
896 		}
897 		if (n < 0) {
898 			rv = n;
899 			goto func_ret;
900 		}
901 		/* avoid spinning CPU when given 0 bytes but no error */
902 		if (n == 0) {
903 			if ((sleep_rv = etm_sleep(ETM_SLEEP_QUIK)) < 0) {
904 				rv = sleep_rv;
905 				goto func_ret;
906 			}
907 		}
908 		rv += n;
909 		datap += n;
910 	} /* while still have more data */
911 
912 func_ret:
913 
914 	if (rv < 0) {
915 		io_fail_stat.fmds_value.ui64++;
916 		fmd_hdl_debug(hdl, "error: %s: errno %d\n",
917 		    err_substr, (int)(-rv));
918 	}
919 	if (etm_debug_lvl >= 3) {
920 		fmd_hdl_debug(hdl, "info: io op %d ret %d of %d\n",
921 		    io_op, (int)rv, (int)byte_cnt);
922 	}
923 	return (rv);
924 
925 } /* etm_io_op() */
926 
927 /*
928  * etm_magic_read - read the magic number of an ETM message header
929  *		from the given connection into the given buffer,
930  *		return 0 or -errno value
931  *
932  * Design_Note:	This routine is intended to help protect ETM from protocol
933  *		framing errors as might be caused by an SP reset / crash in
934  *		the middle of an ETM message send; the connection will be
935  *		read from for as many bytes as needed until the magic number
936  *		is found using a sliding buffer for comparisons.
937  */
938 
939 static int
etm_magic_read(fmd_hdl_t * hdl,etm_xport_conn_t conn,uint32_t * magic_ptr)940 etm_magic_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, uint32_t *magic_ptr)
941 {
942 	int		rv;		/* ret val */
943 	uint32_t	magic_num;	/* magic number */
944 	int		byte_cnt;	/* count of bytes read */
945 	uint8_t		buf5[4+1];	/* sliding input buffer */
946 	int		i, j;		/* indices into buf5 */
947 	ssize_t		n;		/* gen use */
948 	uint8_t		drop_buf[1024];	/* dropped bytes buffer */
949 
950 	rv = 0;		/* assume success */
951 	magic_num = 0;
952 	byte_cnt = 0;
953 	j = 0;
954 
955 	/* magic number bytes are sent in network (big endian) order */
956 
957 	while (magic_num != ETM_PROTO_MAGIC_NUM) {
958 		if ((n = etm_io_op(hdl, "bad io read on magic",
959 		    conn, &buf5[j], 1, ETM_IO_OP_RD)) < 0) {
960 			rv = n;
961 			goto func_ret;
962 		}
963 		byte_cnt++;
964 		j = MIN((j + 1), sizeof (magic_num));
965 		if (byte_cnt < sizeof (magic_num)) {
966 			continue;
967 		}
968 
969 		if (byte_cnt > sizeof (magic_num)) {
970 			etm_stats.etm_magic_drop_bytes.fmds_value.ui64++;
971 			i = MIN(byte_cnt - j - 1, sizeof (drop_buf) - 1);
972 			drop_buf[i] = buf5[0];
973 			for (i = 0; i < j; i++) {
974 				buf5[i] = buf5[i+1];
975 			} /* for sliding the buffer contents */
976 		}
977 		(void) memcpy(&magic_num, &buf5[0], sizeof (magic_num));
978 		magic_num = ntohl(magic_num);
979 	} /* for reading bytes until find magic number */
980 
981 func_ret:
982 
983 	if (byte_cnt != sizeof (magic_num)) {
984 		fmd_hdl_debug(hdl, "warning: bad proto frame "
985 		    "implies corrupt/lost msg(s)\n");
986 	}
987 	if ((byte_cnt > sizeof (magic_num)) && (etm_debug_lvl >= 2)) {
988 		i = MIN(byte_cnt - sizeof (magic_num), sizeof (drop_buf));
989 		fmd_hdl_debug(hdl, "info: magic drop hexdump "
990 		    "first %d of %d bytes:\n", i,
991 		    byte_cnt - sizeof (magic_num));
992 		etm_hexdump(hdl, drop_buf, i);
993 	}
994 
995 	if (rv == 0) {
996 		*magic_ptr = magic_num;
997 	}
998 	return (rv);
999 
1000 } /* etm_magic_read() */
1001 
1002 /*
1003  * etm_hdr_read - allocate, read, and validate a [variable sized]
1004  *		ETM message header from the given connection,
1005  *		return the allocated ETM message header
1006  *		(which is guaranteed to be large enough to reuse as a
1007  *		RESPONSE msg hdr) and its size
1008  *		or NULL and set errno on failure
1009  */
1010 
1011 static void *
etm_hdr_read(fmd_hdl_t * hdl,etm_xport_conn_t conn,size_t * szp)1012 etm_hdr_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, size_t *szp)
1013 {
1014 	uint8_t			*hdrp;		/* ptr to header to return */
1015 	size_t			hdr_sz;		/* sizeof *hdrp */
1016 	etm_proto_v1_pp_t	pp; 		/* protocol preamble */
1017 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
1018 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
1019 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
1020 	etm_proto_v3_sa_hdr_t	*sa_hdrp;	/* for ALERT msg */
1021 	uint32_t		*lenp;		/* ptr to FMA event length */
1022 	ssize_t			i, n;		/* gen use */
1023 	uint8_t	misc_buf[ETM_MISC_BUF_SZ];	/* for var sized hdrs */
1024 	int			dummy_int;	/* dummy var to appease lint */
1025 
1026 	hdrp = NULL; hdr_sz = 0;
1027 
1028 	/* read the magic number which starts the protocol preamble */
1029 
1030 	if ((n = etm_magic_read(hdl, conn, &pp.pp_magic_num)) < 0) {
1031 		errno = (-n);
1032 		etm_stats.etm_magic_bad.fmds_value.ui64++;
1033 		return (NULL);
1034 	}
1035 
1036 	/* read the rest of the protocol preamble all at once */
1037 
1038 	if ((n = etm_io_op(hdl, "bad io read on preamble",
1039 	    conn, &pp.pp_proto_ver, sizeof (pp) - sizeof (pp.pp_magic_num),
1040 	    ETM_IO_OP_RD)) < 0) {
1041 		errno = (-n);
1042 		return (NULL);
1043 	}
1044 
1045 	/*
1046 	 * Design_Note:	The magic number was already network decoded; but
1047 	 *		some other preamble fields also need to be decoded,
1048 	 *		specifically pp_xid and pp_timeout. The rest of the
1049 	 *		preamble fields are byte sized and hence need no
1050 	 *		decoding.
1051 	 */
1052 
1053 	pp.pp_xid = ntohl(pp.pp_xid);
1054 	pp.pp_timeout = ntohl(pp.pp_timeout);
1055 
1056 	/* sanity check the header as best we can */
1057 
1058 	if ((pp.pp_proto_ver < ETM_PROTO_V1) ||
1059 	    (pp.pp_proto_ver > ETM_PROTO_V3)) {
1060 		fmd_hdl_error(hdl, "error: bad proto ver %d\n",
1061 		    (int)pp.pp_proto_ver);
1062 		errno = EPROTO;
1063 		etm_stats.etm_ver_bad.fmds_value.ui64++;
1064 		return (NULL);
1065 	}
1066 
1067 	dummy_int = pp.pp_msg_type;
1068 	if ((dummy_int <= ETM_MSG_TYPE_TOO_LOW) ||
1069 	    (dummy_int >= ETM_MSG_TYPE_TOO_BIG)) {
1070 		fmd_hdl_error(hdl, "error: bad msg type %d", dummy_int);
1071 		errno = EBADMSG;
1072 		etm_stats.etm_msgtype_bad.fmds_value.ui64++;
1073 		return (NULL);
1074 	}
1075 
1076 	/* handle [var sized] hdrs for FMA_EVENT, CONTROL, RESPONSE msgs */
1077 
1078 	if (pp.pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
1079 
1080 		ev_hdrp = (void*)&misc_buf[0];
1081 		hdr_sz = sizeof (*ev_hdrp);
1082 		(void) memcpy(&ev_hdrp->ev_pp, &pp, sizeof (pp));
1083 
1084 		/* sanity check the header's timeout */
1085 
1086 		if ((ev_hdrp->ev_pp.pp_proto_ver == ETM_PROTO_V1) &&
1087 		    (ev_hdrp->ev_pp.pp_timeout != ETM_PROTO_V1_TIMEOUT_NONE)) {
1088 			errno = ETIME;
1089 			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1090 			return (NULL);
1091 		}
1092 
1093 		/* get all FMA event lengths from the header */
1094 
1095 		lenp = (uint32_t *)&ev_hdrp->ev_lens[0]; lenp--;
1096 		i = -1;	/* cnt of length entries preceding 0 */
1097 		do {
1098 			i++; lenp++;
1099 			if ((sizeof (*ev_hdrp) + (i * sizeof (*lenp))) >=
1100 			    ETM_MISC_BUF_SZ) {
1101 				errno = E2BIG;	/* ridiculous size */
1102 				etm_stats.etm_evlens_bad.fmds_value.ui64++;
1103 				return (NULL);
1104 			}
1105 			if ((n = etm_io_op(hdl, "bad io read on event len",
1106 			    conn, lenp, sizeof (*lenp), ETM_IO_OP_RD)) < 0) {
1107 				errno = (-n);
1108 				return (NULL);
1109 			}
1110 			*lenp = ntohl(*lenp);
1111 
1112 		} while (*lenp != 0);
1113 		i += 0; /* first len already counted by sizeof(ev_hdr) */
1114 		hdr_sz += (i * sizeof (*lenp));
1115 
1116 		etm_stats.etm_rd_hdr_fmaevent.fmds_value.ui64++;
1117 
1118 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_CONTROL) {
1119 
1120 		ctl_hdrp = (void*)&misc_buf[0];
1121 		hdr_sz = sizeof (*ctl_hdrp);
1122 		(void) memcpy(&ctl_hdrp->ctl_pp, &pp, sizeof (pp));
1123 
1124 		/* sanity check the header's sub type (control selector) */
1125 
1126 		if ((ctl_hdrp->ctl_pp.pp_sub_type <= ETM_CTL_SEL_TOO_LOW) ||
1127 		    (ctl_hdrp->ctl_pp.pp_sub_type >= ETM_CTL_SEL_TOO_BIG)) {
1128 			fmd_hdl_error(hdl, "error: bad ctl sub type %d\n",
1129 			    (int)ctl_hdrp->ctl_pp.pp_sub_type);
1130 			errno = EBADMSG;
1131 			etm_stats.etm_subtype_bad.fmds_value.ui64++;
1132 			return (NULL);
1133 		}
1134 
1135 		/* get the control length */
1136 
1137 		if ((n = etm_io_op(hdl, "bad io read on ctl len",
1138 		    conn, &ctl_hdrp->ctl_len, sizeof (ctl_hdrp->ctl_len),
1139 		    ETM_IO_OP_RD)) < 0) {
1140 			errno = (-n);
1141 			return (NULL);
1142 		}
1143 
1144 		ctl_hdrp->ctl_len = ntohl(ctl_hdrp->ctl_len);
1145 
1146 		etm_stats.etm_rd_hdr_control.fmds_value.ui64++;
1147 
1148 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
1149 
1150 		resp_hdrp = (void*)&misc_buf[0];
1151 		hdr_sz = sizeof (*resp_hdrp);
1152 		(void) memcpy(&resp_hdrp->resp_pp, &pp, sizeof (pp));
1153 
1154 		/* sanity check the header's timeout */
1155 
1156 		if (resp_hdrp->resp_pp.pp_timeout !=
1157 		    ETM_PROTO_V1_TIMEOUT_NONE) {
1158 			errno = ETIME;
1159 			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1160 			return (NULL);
1161 		}
1162 
1163 		/* get the response code and length */
1164 
1165 		if ((n = etm_io_op(hdl, "bad io read on resp code+len",
1166 		    conn, &resp_hdrp->resp_code,
1167 		    sizeof (resp_hdrp->resp_code)
1168 		    + sizeof (resp_hdrp->resp_len),
1169 		    ETM_IO_OP_RD)) < 0) {
1170 			errno = (-n);
1171 			return (NULL);
1172 		}
1173 
1174 		resp_hdrp->resp_code = ntohl(resp_hdrp->resp_code);
1175 		resp_hdrp->resp_len = ntohl(resp_hdrp->resp_len);
1176 
1177 		etm_stats.etm_rd_hdr_response.fmds_value.ui64++;
1178 
1179 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_ALERT) {
1180 
1181 		sa_hdrp = (void*)&misc_buf[0];
1182 		hdr_sz = sizeof (*sa_hdrp);
1183 		(void) memcpy(&sa_hdrp->sa_pp, &pp, sizeof (pp));
1184 
1185 		/* sanity check the header's protocol version */
1186 
1187 		if (sa_hdrp->sa_pp.pp_proto_ver != ETM_PROTO_V3) {
1188 			errno = EPROTO;
1189 			etm_stats.etm_ver_bad.fmds_value.ui64++;
1190 			return (NULL);
1191 		}
1192 
1193 		/* get the priority and length */
1194 
1195 		if ((n = etm_io_op(hdl, "bad io read on sa priority+len",
1196 		    conn, &sa_hdrp->sa_priority,
1197 		    sizeof (sa_hdrp->sa_priority)
1198 		    + sizeof (sa_hdrp->sa_len),
1199 		    ETM_IO_OP_RD)) < 0) {
1200 			errno = (-n);
1201 			return (NULL);
1202 		}
1203 
1204 		sa_hdrp->sa_priority = ntohl(sa_hdrp->sa_priority);
1205 		sa_hdrp->sa_len = ntohl(sa_hdrp->sa_len);
1206 
1207 		etm_stats.etm_rd_hdr_alert.fmds_value.ui64++;
1208 
1209 	} /* whether we have FMA_EVENT, ALERT, CONTROL, or RESPONSE msg */
1210 
1211 	/*
1212 	 * choose a header size that allows hdr reuse for RESPONSE msgs,
1213 	 * allocate and populate the message header, and
1214 	 * return alloc size to caller for later free of hdrp
1215 	 */
1216 
1217 	hdr_sz = MAX(hdr_sz, sizeof (*resp_hdrp));
1218 	hdrp = fmd_hdl_zalloc(hdl, hdr_sz, FMD_SLEEP);
1219 	(void) memcpy(hdrp, misc_buf, hdr_sz);
1220 
1221 	if (etm_debug_lvl >= 3) {
1222 		fmd_hdl_debug(hdl, "info: msg hdr hexdump %d bytes:\n", hdr_sz);
1223 		etm_hexdump(hdl, hdrp, hdr_sz);
1224 	}
1225 	*szp = hdr_sz;
1226 	return (hdrp);
1227 
1228 } /* etm_hdr_read() */
1229 
1230 /*
1231  * etm_hdr_write - create and write a [variable sized] ETM message header
1232  *		to the given connection appropriate for the given FMA event
1233  *		and type of nvlist encoding,
1234  *		return the allocated ETM message header and its size
1235  *		or NULL and set errno on failure
1236  */
1237 
1238 static void*
etm_hdr_write(fmd_hdl_t * hdl,etm_xport_conn_t conn,nvlist_t * evp,int encoding,size_t * szp)1239 etm_hdr_write(fmd_hdl_t *hdl, etm_xport_conn_t conn, nvlist_t *evp,
1240 						int encoding, size_t *szp)
1241 {
1242 	etm_proto_v1_ev_hdr_t	*hdrp;		/* for FMA_EVENT msg */
1243 	size_t			hdr_sz;		/* sizeof *hdrp */
1244 	uint32_t		*lenp;		/* ptr to FMA event length */
1245 	size_t			evsz;		/* packed FMA event size */
1246 	ssize_t			n;		/* gen use */
1247 
1248 	/* allocate and populate the message header for 1 FMA event */
1249 
1250 	hdr_sz = sizeof (*hdrp) + (1 * sizeof (hdrp->ev_lens[0]));
1251 
1252 	hdrp = fmd_hdl_zalloc(hdl, hdr_sz, FMD_SLEEP);
1253 
1254 	/*
1255 	 * Design_Note: Although the ETM protocol supports it, we do not (yet)
1256 	 *		want responses/ACKs on FMA events that we send. All
1257 	 *		such messages are sent with ETM_PROTO_V1_TIMEOUT_NONE.
1258 	 */
1259 
1260 	hdrp->ev_pp.pp_magic_num = ETM_PROTO_MAGIC_NUM;
1261 	hdrp->ev_pp.pp_magic_num = htonl(hdrp->ev_pp.pp_magic_num);
1262 	hdrp->ev_pp.pp_proto_ver = ETM_PROTO_V1;
1263 	hdrp->ev_pp.pp_msg_type = ETM_MSG_TYPE_FMA_EVENT;
1264 	hdrp->ev_pp.pp_sub_type = 0;
1265 	hdrp->ev_pp.pp_rsvd_pad = 0;
1266 	hdrp->ev_pp.pp_xid = etm_xid_cur;
1267 	hdrp->ev_pp.pp_xid = htonl(hdrp->ev_pp.pp_xid);
1268 	etm_xid_cur += ETM_XID_INC;
1269 	hdrp->ev_pp.pp_timeout = ETM_PROTO_V1_TIMEOUT_NONE;
1270 	hdrp->ev_pp.pp_timeout = htonl(hdrp->ev_pp.pp_timeout);
1271 
1272 	lenp = &hdrp->ev_lens[0];
1273 
1274 	if ((n = nvlist_size(evp, &evsz, encoding)) != 0) {
1275 		errno = n;
1276 		fmd_hdl_free(hdl, hdrp, hdr_sz);
1277 		etm_stats.etm_os_nvlist_size_fail.fmds_value.ui64++;
1278 		return (NULL);
1279 	}
1280 
1281 	/* indicate 1 FMA event, network encode its length, and 0-terminate */
1282 
1283 	etm_stats.etm_wr_max_ev_per_msg.fmds_value.ui64 = 1;
1284 
1285 	*lenp = evsz; *lenp = htonl(*lenp); lenp++;
1286 	*lenp = 0; *lenp = htonl(*lenp); lenp++;
1287 
1288 	/*
1289 	 * write the network encoded header to the transport, and
1290 	 * return alloc size to caller for later free
1291 	 */
1292 
1293 	if ((n = etm_io_op(hdl, "bad io write on event hdr",
1294 	    conn, hdrp, hdr_sz, ETM_IO_OP_WR)) < 0) {
1295 		errno = (-n);
1296 		fmd_hdl_free(hdl, hdrp, hdr_sz);
1297 		return (NULL);
1298 	}
1299 
1300 	*szp = hdr_sz;
1301 	return (hdrp);
1302 
1303 } /* etm_hdr_write() */
1304 
1305 /*
1306  * etm_post_to_fmd - post the given FMA event to FMD
1307  *			via a FMD transport API call,
1308  *			return 0 or -errno value
1309  *
1310  * caveats:	the FMA event (evp) is freed by FMD,
1311  *		thus callers of this function should
1312  *		immediately discard any ptr they have to the
1313  *		nvlist without freeing or dereferencing it
1314  */
1315 
1316 static int
etm_post_to_fmd(fmd_hdl_t * hdl,fmd_xprt_t * fmd_xprt,nvlist_t * evp)1317 etm_post_to_fmd(fmd_hdl_t *hdl, fmd_xprt_t *fmd_xprt, nvlist_t *evp)
1318 {
1319 	ssize_t			ev_sz;		/* sizeof *evp */
1320 
1321 	(void) nvlist_size(evp, (size_t *)&ev_sz, NV_ENCODE_XDR);
1322 
1323 	if (etm_debug_lvl >= 2) {
1324 		etm_show_time(hdl, "ante ev post");
1325 	}
1326 	fmd_xprt_post(hdl, fmd_xprt, evp, 0);
1327 	etm_stats.etm_wr_fmd_fmaevent.fmds_value.ui64++;
1328 	etm_stats.etm_wr_fmd_bytes.fmds_value.ui64 += ev_sz;
1329 	if (etm_debug_lvl >= 1) {
1330 		fmd_hdl_debug(hdl, "info: event %p post ok to FMD\n", evp);
1331 	}
1332 	if (etm_debug_lvl >= 2) {
1333 		etm_show_time(hdl, "post ev post");
1334 	}
1335 	return (0);
1336 
1337 } /* etm_post_to_fmd() */
1338 
1339 /*
1340  * Ideally we would just use syslog(3C) for outputting our messages.
1341  * Unfortunately, as this module is running within the FMA daemon context,
1342  * that would create the situation where this module's openlog() would
1343  * have the monopoly on syslog(3C) for the daemon and all its modules.
1344  * To avoid that situation, this module uses the same logic as the
1345  * syslog-msgs FM module to directly call into the log(7D) and sysmsg(7D)
1346  * devices for syslog and console.
1347  */
1348 
1349 static int
etm_post_to_syslog(fmd_hdl_t * hdl,uint32_t priority,uint32_t body_sz,uint8_t * body_buf)1350 etm_post_to_syslog(fmd_hdl_t *hdl, uint32_t priority, uint32_t body_sz,
1351 							uint8_t *body_buf)
1352 {
1353 	char		*sysmessage;	/* Formatted message */
1354 	size_t		formatlen;	/* maximum length of sysmessage */
1355 	struct strbuf	ctl, dat;	/* structs pushed to the logfd */
1356 	uint32_t	msgid;		/* syslog message ID number */
1357 
1358 	if ((syslog_file == 0) && (syslog_cons == 0)) {
1359 		return (0);
1360 	}
1361 
1362 	if (etm_debug_lvl >= 2) {
1363 		etm_show_time(hdl, "ante syslog post");
1364 	}
1365 
1366 	formatlen = body_sz + 64; /* +64 for prefix strings added below */
1367 	sysmessage = fmd_hdl_zalloc(hdl, formatlen, FMD_SLEEP);
1368 
1369 	if (syslog_file) {
1370 		STRLOG_MAKE_MSGID(body_buf, msgid);
1371 		(void) snprintf(sysmessage, formatlen,
1372 		    "SC Alert: [ID %u FACILITY_AND_PRIORITY] %s", msgid,
1373 		    body_buf);
1374 
1375 		syslog_ctl.pri = syslog_facility | priority;
1376 
1377 		ctl.buf = (void *)&syslog_ctl;
1378 		ctl.len = sizeof (syslog_ctl);
1379 
1380 		dat.buf = sysmessage;
1381 		dat.len = strlen(sysmessage) + 1;
1382 
1383 		if (putmsg(syslog_logfd, &ctl, &dat, 0) != 0) {
1384 			fmd_hdl_debug(hdl, "putmsg failed: %s\n",
1385 			    strerror(errno));
1386 			etm_stats.etm_log_err.fmds_value.ui64++;
1387 		}
1388 	}
1389 
1390 	if (syslog_cons) {
1391 		(void) snprintf(sysmessage, formatlen,
1392 		    "SC Alert: %s\r\n", body_buf);
1393 
1394 		dat.buf = sysmessage;
1395 		dat.len = strlen(sysmessage) + 1;
1396 
1397 		if (write(syslog_msgfd, dat.buf, dat.len) != dat.len) {
1398 			fmd_hdl_debug(hdl, "write failed: %s\n",
1399 			    strerror(errno));
1400 			etm_stats.etm_msg_err.fmds_value.ui64++;
1401 		}
1402 	}
1403 
1404 	fmd_hdl_free(hdl, sysmessage, formatlen);
1405 
1406 	if (etm_debug_lvl >= 2) {
1407 		etm_show_time(hdl, "post syslog post");
1408 	}
1409 
1410 	return (0);
1411 }
1412 
1413 
1414 /*
1415  * etm_req_ver_negot - send an ETM control message to the other end requesting
1416  *			that the ETM protocol version be negotiated/set
1417  */
1418 
1419 static void
etm_req_ver_negot(fmd_hdl_t * hdl)1420 etm_req_ver_negot(fmd_hdl_t *hdl)
1421 {
1422 	etm_xport_addr_t	*addrv;		/* default dst addr(s) */
1423 	etm_xport_conn_t	conn;		/* connection to other end */
1424 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
1425 	size_t			hdr_sz;		/* sizeof header */
1426 	uint8_t			*body_buf;	/* msg body buffer */
1427 	uint32_t		body_sz;	/* sizeof *body_buf */
1428 	ssize_t			i;		/* gen use */
1429 
1430 	/* populate an ETM control msg to send */
1431 
1432 	hdr_sz = sizeof (*ctl_hdrp);
1433 	body_sz = (3 + 1);		/* version bytes plus null byte */
1434 
1435 	ctl_hdrp = fmd_hdl_zalloc(hdl, hdr_sz + body_sz, FMD_SLEEP);
1436 
1437 	ctl_hdrp->ctl_pp.pp_magic_num = htonl(ETM_PROTO_MAGIC_NUM);
1438 	ctl_hdrp->ctl_pp.pp_proto_ver = ETM_PROTO_V1;
1439 	ctl_hdrp->ctl_pp.pp_msg_type = ETM_MSG_TYPE_CONTROL;
1440 	ctl_hdrp->ctl_pp.pp_sub_type = ETM_CTL_SEL_VER_NEGOT_REQ;
1441 	ctl_hdrp->ctl_pp.pp_rsvd_pad = 0;
1442 	etm_xid_ver_negot = etm_xid_cur;
1443 	etm_xid_cur += ETM_XID_INC;
1444 	ctl_hdrp->ctl_pp.pp_xid = htonl(etm_xid_ver_negot);
1445 	ctl_hdrp->ctl_pp.pp_timeout = htonl(ETM_PROTO_V1_TIMEOUT_FOREVER);
1446 	ctl_hdrp->ctl_len = htonl(body_sz);
1447 
1448 	body_buf = (void*)&ctl_hdrp->ctl_len;
1449 	body_buf += sizeof (ctl_hdrp->ctl_len);
1450 	*body_buf++ = ETM_PROTO_V3;
1451 	*body_buf++ = ETM_PROTO_V2;
1452 	*body_buf++ = ETM_PROTO_V1;
1453 	*body_buf++ = '\0';
1454 
1455 	/*
1456 	 * open and close a connection to send the ETM control msg
1457 	 * to any/all of the default dst addrs
1458 	 */
1459 
1460 	if ((addrv = etm_xport_get_ev_addrv(hdl, NULL)) == NULL) {
1461 		fmd_hdl_error(hdl,
1462 		    "error: bad ctl dst addrs errno %d\n", errno);
1463 		etm_stats.etm_xport_get_ev_addrv_fail.fmds_value.ui64++;
1464 		goto func_ret;
1465 	}
1466 
1467 	for (i = 0; addrv[i] != NULL; i++) {
1468 
1469 		if (etm_conn_open(hdl, "bad conn open during ver negot",
1470 		    addrv[i], &conn) < 0) {
1471 			continue;
1472 		}
1473 		if (etm_io_op(hdl, "bad io write on ctl hdr+body",
1474 		    conn, ctl_hdrp, hdr_sz + body_sz, ETM_IO_OP_WR) >= 0) {
1475 			etm_stats.etm_wr_hdr_control.fmds_value.ui64++;
1476 			etm_stats.etm_wr_body_control.fmds_value.ui64++;
1477 		}
1478 		(void) etm_conn_close(hdl, "bad conn close during ver negot",
1479 		    conn);
1480 
1481 	} /* foreach dst addr */
1482 
1483 func_ret:
1484 
1485 	if (addrv != NULL) {
1486 		etm_xport_free_addrv(hdl, addrv);
1487 	}
1488 	fmd_hdl_free(hdl, ctl_hdrp, hdr_sz + body_sz);
1489 
1490 } /* etm_req_ver_negot() */
1491 
1492 
1493 
1494 /*
1495  * etm_iosvc_msg_enq - add element to tail of ETM iosvc msg queue
1496  * etm_iosvc_msg_deq - del element from head of ETM iosvc msg  queue
1497  * need to grab the mutex lock before calling this routine
1498  * return >0 for success, or -errno value
1499  */
1500 static int
etm_iosvc_msg_enq(fmd_hdl_t * hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msgp)1501 etm_iosvc_msg_enq(fmd_hdl_t *hdl, etm_iosvc_t *iosvc, etm_iosvc_q_ele_t *msgp)
1502 {
1503 	etm_iosvc_q_ele_t		*newp;	/* ptr to new msg q ele */
1504 
1505 	if (iosvc->msg_q_cur_len >= iosvc->msg_q_max_len) {
1506 		fmd_hdl_debug(hdl, "warning: enq to full msg queue\n");
1507 		return (-E2BIG);
1508 	}
1509 
1510 	newp = fmd_hdl_zalloc(hdl, sizeof (*newp), FMD_SLEEP);
1511 	(void) memcpy(newp, msgp, sizeof (*newp));
1512 	newp->msg_nextp = NULL;
1513 
1514 	if (iosvc->msg_q_cur_len == 0) {
1515 		iosvc->msg_q_head = newp;
1516 	} else {
1517 		iosvc->msg_q_tail->msg_nextp = newp;
1518 	}
1519 
1520 	iosvc->msg_q_tail = newp;
1521 	iosvc->msg_q_cur_len++;
1522 	fmd_hdl_debug(hdl, "info: current msg queue length %d\n",
1523 	    iosvc->msg_q_cur_len);
1524 
1525 	return (1);
1526 
1527 } /* etm_iosvc_msg_enq() */
1528 
1529 static int
etm_iosvc_msg_deq(fmd_hdl_t * hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msgp)1530 etm_iosvc_msg_deq(fmd_hdl_t *hdl, etm_iosvc_t *iosvc, etm_iosvc_q_ele_t *msgp)
1531 {
1532 	etm_iosvc_q_ele_t	*oldp;	/* ptr to old msg q ele */
1533 
1534 	if (iosvc->msg_q_cur_len == 0) {
1535 		fmd_hdl_debug(hdl, "warning: deq from empty responder queue\n");
1536 		return (-ENOENT);
1537 	}
1538 
1539 	(void) memcpy(msgp, iosvc->msg_q_head, sizeof (*msgp));
1540 	msgp->msg_nextp = NULL;
1541 
1542 	oldp = iosvc->msg_q_head;
1543 	iosvc->msg_q_head = iosvc->msg_q_head->msg_nextp;
1544 
1545 	/*
1546 	 * free the mem alloc-ed in etm_iosvc_msg_enq()
1547 	 */
1548 	fmd_hdl_free(hdl, oldp, sizeof (*oldp));
1549 
1550 	iosvc->msg_q_cur_len--;
1551 	if (iosvc->msg_q_cur_len == 0) {
1552 		iosvc->msg_q_tail = NULL;
1553 	}
1554 
1555 	return (1);
1556 
1557 } /* etm_iosvc_msg_deq() */
1558 
1559 
1560 /*
1561  * etm_msg_enq_head():
1562  * enq the msg to the head of the Q.
1563  * If the Q is full, drop the msg at the tail then enq the msg at head.
1564  * need to grab mutex lock iosvc->msg_q_lock before calling this routine.
1565  */
1566 static void
etm_msg_enq_head(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msg_ele)1567 etm_msg_enq_head(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
1568     etm_iosvc_q_ele_t *msg_ele)
1569 {
1570 
1571 	etm_iosvc_q_ele_t	*newp;	/* iosvc msg ele ptr */
1572 
1573 	if (iosvc->msg_q_cur_len >= iosvc->msg_q_max_len) {
1574 		fmd_hdl_debug(fmd_hdl,
1575 		    "warning: add to head of a full msg queue."
1576 		    " Drop the msg at the tail\n");
1577 		/*
1578 		 * drop the msg at the tail
1579 		 */
1580 		newp = iosvc->msg_q_head;
1581 		while (newp->msg_nextp != iosvc->msg_q_tail) {
1582 			newp = newp->msg_nextp;
1583 		}
1584 
1585 		/*
1586 		 * free the msg in iosvc->msg_q_tail->msg
1587 		 * free the mem pointed to by iosvc->msg_q_tail
1588 		 */
1589 		fmd_hdl_free(fmd_hdl, iosvc->msg_q_tail->msg,
1590 		    iosvc->msg_q_tail->msg_size);
1591 		fmd_hdl_free(fmd_hdl, iosvc->msg_q_tail, sizeof (*newp));
1592 		iosvc->msg_q_tail = newp;
1593 		iosvc->msg_q_tail->msg_nextp = NULL;
1594 		iosvc->msg_q_cur_len--;
1595 	}
1596 
1597 	/*
1598 	 * enq the msg to the head
1599 	 */
1600 	newp = fmd_hdl_zalloc(fmd_hdl, sizeof (*newp), FMD_SLEEP);
1601 	(void) memcpy(newp, msg_ele, sizeof (*newp));
1602 	if (iosvc->msg_q_cur_len == 0) {
1603 		newp->msg_nextp = NULL;
1604 		iosvc->msg_q_tail = newp;
1605 	} else {
1606 		newp->msg_nextp = iosvc->msg_q_head;
1607 	}
1608 	iosvc->msg_q_head = newp;
1609 	iosvc->msg_q_cur_len++;
1610 } /* etm_msg_enq_head() */
1611 
1612 /*
1613  * etm_iosvc_cleanup():
1614  * Clean up an iosvc structure
1615  * 1) close the fmd_xprt if it has not been closed
1616  * 2) Terminate the send/revc threads
1617  * 3) If the clean_msg_q flag is set, free all fma events in the queue. In
1618  *    addition, if the chpt_remove flag is set, delete the checkpoint so that
1619  *    the events are not persisted.
1620  */
1621 static void
etm_iosvc_cleanup(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,boolean_t clean_msg_q,boolean_t ckpt_remove)1622 etm_iosvc_cleanup(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc, boolean_t clean_msg_q,
1623     boolean_t ckpt_remove)
1624 {
1625 
1626 	etm_iosvc_q_ele_t	msg_ele;	/* io svc msg Q ele */
1627 
1628 	iosvc->thr_is_dying = 1;
1629 
1630 	iosvc->ds_hdl = DS_INVALID_HDL;
1631 	if (iosvc->fmd_xprt != NULL) {
1632 		fmd_xprt_close(fmd_hdl, iosvc->fmd_xprt);
1633 		iosvc->fmd_xprt = NULL;
1634 	} /* if fmd-xprt has been opened */
1635 
1636 	if (iosvc->send_tid != NULL) {
1637 		fmd_thr_signal(fmd_hdl, iosvc->send_tid);
1638 		fmd_thr_destroy(fmd_hdl, iosvc->send_tid);
1639 		iosvc->send_tid = NULL;
1640 	} /* if io svc send thread was created ok */
1641 
1642 	if (iosvc->recv_tid != NULL) {
1643 		fmd_thr_signal(fmd_hdl, iosvc->recv_tid);
1644 		fmd_thr_destroy(fmd_hdl, iosvc->recv_tid);
1645 		iosvc->recv_tid = NULL;
1646 	} /* if root domain recv thread was created */
1647 
1648 
1649 	if (clean_msg_q) {
1650 		iosvc->ldom_name[0] = '\0';
1651 
1652 		(void) pthread_mutex_lock(&iosvc->msg_q_lock);
1653 		while (iosvc->msg_q_cur_len > 0) {
1654 			(void) etm_iosvc_msg_deq(fmd_hdl, iosvc, &msg_ele);
1655 			if (ckpt_remove == B_TRUE &&
1656 			    msg_ele.ckpt_flag != ETM_CKPT_NOOP) {
1657 				etm_ckpt_remove(fmd_hdl, &msg_ele);
1658 			}
1659 			fmd_hdl_free(fmd_hdl, msg_ele.msg, msg_ele.msg_size);
1660 		}
1661 		(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
1662 	}
1663 
1664 	return;
1665 
1666 } /* etm_iosvc_cleanup() */
1667 
1668 /*
1669  * etm_iosvc_lookup(using ldom_name or ds_hdl when ldom_name is empty)
1670  * not found, create one, add to iosvc_list
1671  */
1672 etm_iosvc_t *
etm_iosvc_lookup(fmd_hdl_t * fmd_hdl,char * ldom_name,ds_hdl_t ds_hdl,boolean_t iosvc_create)1673 etm_iosvc_lookup(fmd_hdl_t *fmd_hdl, char *ldom_name, ds_hdl_t ds_hdl,
1674     boolean_t iosvc_create)
1675 {
1676 	uint32_t		i;			/* for loop var */
1677 	int32_t			first_empty_slot = -1;	/* remember that */
1678 
1679 	for (i = 0; i < NUM_OF_ROOT_DOMAINS; i++) {
1680 		if (ldom_name[0] == '\0') {
1681 			/*
1682 			 * search by hdl passed in
1683 			 * the only time this is used is at ds_unreg_cb time.
1684 			 * there is no ldom name, only the valid ds_hdl.
1685 			 * find an iosvc with the matching ds_hdl.
1686 			 * ignore the iosvc_create flag, should never need to
1687 			 * create an iosvc for ds_unreg_cb
1688 			 */
1689 			if (ds_hdl == iosvc_list[i].ds_hdl) {
1690 				if (etm_debug_lvl >= 2) {
1691 				fmd_hdl_debug(fmd_hdl,
1692 			    "info: found an iosvc at slot %d w/ ds_hdl %d \n",
1693 				    i, iosvc_list[i].ds_hdl);
1694 				}
1695 				if (iosvc_list[i].ldom_name[0] != '\0')
1696 					if (etm_debug_lvl >= 2) {
1697 						fmd_hdl_debug(fmd_hdl,
1698 				    "info: found an iosvc w/ ldom_name %s \n",
1699 						    iosvc_list[i].ldom_name);
1700 				}
1701 				return (&iosvc_list[i]);
1702 			} else {
1703 				continue;
1704 			}
1705 		} else if (iosvc_list[i].ldom_name[0] != '\0') {
1706 			/*
1707 			 * this is  an non-empty iosvc structure slot
1708 			 */
1709 			if (strcmp(ldom_name, iosvc_list[i].ldom_name) == 0) {
1710 				/*
1711 				 * found an iosvc structure that matches the
1712 				 * passed in ldom_name, return the ptr
1713 				 */
1714 				if (etm_debug_lvl >= 2) {
1715 					fmd_hdl_debug(fmd_hdl, "info: found an "
1716 					    "iosvc at slot %d w/ ds_hdl %d \n",
1717 					    i, iosvc_list[i].ds_hdl);
1718 					fmd_hdl_debug(fmd_hdl, "info: found an "
1719 					    "iosvc w/ ldom_name %s \n",
1720 					    iosvc_list[i].ldom_name);
1721 				}
1722 				return (&iosvc_list[i]);
1723 			} else {
1724 				/*
1725 				 * non-empty slot with no-matching name,
1726 				 * move on to next slot.
1727 				 */
1728 				continue;
1729 			}
1730 		} else {
1731 			/*
1732 			 * found the 1st slot with ldom name being empty
1733 			 * remember the slot #, will be used for creating one
1734 			 */
1735 			if (first_empty_slot == -1) {
1736 				first_empty_slot = i;
1737 			}
1738 		}
1739 	}
1740 	if (iosvc_create == B_TRUE && first_empty_slot >= 0) {
1741 		/*
1742 		 * this is the case we need to add an iosvc at first_empty_slot
1743 		 * for the ldom_name at iosvc_list[first_empty_slot]
1744 		 */
1745 		fmd_hdl_debug(fmd_hdl,
1746 		    "info: create an iosvc with ldom name %s\n",
1747 		    ldom_name);
1748 		i = first_empty_slot;
1749 		(void) memcpy(&iosvc_list[i], &io_svc, sizeof (etm_iosvc_t));
1750 		(void) strcpy(iosvc_list[i].ldom_name, ldom_name);
1751 		fmd_hdl_debug(fmd_hdl, "info: iosvc #%d has ldom name %s\n",
1752 		    i, iosvc_list[i].ldom_name);
1753 		return (&iosvc_list[i]);
1754 	} else {
1755 		return (NULL);
1756 	}
1757 
1758 } /* etm_iosvc_lookup() */
1759 
1760 
1761 /*
1762  * etm_ckpt_remove:
1763  * remove the ckpt for the iosvc element
1764  */
1765 static void
etm_ckpt_remove(fmd_hdl_t * hdl,etm_iosvc_q_ele_t * ele)1766 etm_ckpt_remove(fmd_hdl_t *hdl, etm_iosvc_q_ele_t *ele) {
1767 	int		err;			/* temp error */
1768 	nvlist_t	*evp = NULL;		/* event pointer */
1769 	etm_proto_v1_ev_hdr_t	*hdrp;		/* hdr for FMA_EVENT */
1770 	char		*buf;			/* packed event pointer */
1771 
1772 	if ((ele->ckpt_flag == ETM_CKPT_NOOP) ||
1773 	    (etm_ldom_type != LDOM_TYPE_CONTROL)) {
1774 		return;
1775 	}
1776 
1777 	/* the pointer to the packed event in the etm message */
1778 	hdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)ele->msg);
1779 	buf = (char *)((ptrdiff_t)hdrp + sizeof (*hdrp)
1780 	    + (1 * sizeof (hdrp->ev_lens[0])));
1781 
1782 	/* unpack it, then uncheckpoited it */
1783 	if ((err = nvlist_unpack(buf, hdrp->ev_lens[0], &evp, 0)) != 0) {
1784 		fmd_hdl_debug(hdl, "failed to unpack event(rc=%d)\n", err);
1785 		return;
1786 	}
1787 	(void) etm_ckpt_delete(hdl, evp);
1788 	nvlist_free(evp);
1789 }
1790 
1791 /*
1792  * etm_send_ds_msg()
1793  * call ds_send_msg() to send the msg passed in.
1794  * timedcond_wait for the ACK to come back.
1795  * if the ACK doesn't come in the specified time, retrun -EAGAIN.
1796  * other wise, return 1.
1797  */
1798 int
etm_send_ds_msg(fmd_hdl_t * fmd_hdl,boolean_t ckpt_remove,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msg_ele,etm_proto_v1_ev_hdr_t * evhdrp)1799 etm_send_ds_msg(fmd_hdl_t *fmd_hdl, boolean_t ckpt_remove, etm_iosvc_t *iosvc,
1800     etm_iosvc_q_ele_t *msg_ele, etm_proto_v1_ev_hdr_t *evhdrp)
1801 {
1802 	uint32_t		rc;		/* for return code  */
1803 
1804 	struct timeval		tv;
1805 	struct timespec		timeout;
1806 
1807 
1808 	/*
1809 	 * call ds_send_msg(). Return (-EAGAIN) if not successful
1810 	 */
1811 	if ((rc = (*etm_ds_send_msg)(iosvc->ds_hdl, msg_ele->msg,
1812 	    msg_ele->msg_size)) != 0) {
1813 		fmd_hdl_debug(fmd_hdl, "info: ds_send_msg rc %d xid %d\n",
1814 		    rc, evhdrp->ev_pp.pp_xid);
1815 			return (-EAGAIN);
1816 	}
1817 
1818 	/*
1819 	 * wait on the cv for resp msg for cur_send_xid
1820 	 */
1821 	(void *) pthread_mutex_lock(&iosvc->msg_ack_lock);
1822 
1823 	(void) gettimeofday(&tv, 0);
1824 	timeout.tv_sec = tv.tv_sec + etm_fma_resp_wait_time;
1825 	timeout.tv_nsec = 0;
1826 
1827 	fmd_hdl_debug(fmd_hdl, "info: waiting on msg_ack_cv for ldom %s\n",
1828 	    iosvc->ldom_name);
1829 	rc = pthread_cond_timedwait(&iosvc->msg_ack_cv, &iosvc->msg_ack_lock,
1830 	    &timeout);
1831 	(void *) pthread_mutex_unlock(&iosvc->msg_ack_lock);
1832 	fmd_hdl_debug(fmd_hdl,  "info: msg_ack_cv returns with rc %d\n", rc);
1833 
1834 	/*
1835 	 * check to see if ack_ok is non-zero
1836 	 * if non-zero, resp msg has been received
1837 	 */
1838 	if (iosvc->ack_ok != 0) {
1839 		/*
1840 		 * ACK came ok,  this send is successful,
1841 		 * tell the caller ready to send next.
1842 		 * free mem alloc-ed in
1843 		 * etm_pack_ds_msg
1844 		 */
1845 		if (ckpt_remove == B_TRUE &&
1846 		    etm_ldom_type == LDOM_TYPE_CONTROL) {
1847 			etm_ckpt_remove(fmd_hdl, msg_ele);
1848 		}
1849 		fmd_hdl_free(fmd_hdl, msg_ele->msg, msg_ele->msg_size);
1850 		iosvc->cur_send_xid++;
1851 		return (1);
1852 	} else {
1853 		/*
1854 		 * the ACK did not come on time
1855 		 * tell the caller to resend cur_send_xid
1856 		 */
1857 		return (-EAGAIN);
1858 	} /* iosvc->ack_ok != 0 */
1859 } /* etm_send_ds_msg() */
1860 
1861 /*
1862  * both events from fmdo_send entry point and from SP are using the
1863  * etm_proto_v1_ev_hdr_t as its header and it will be the same header for all
1864  * ds send/recv msgs.
1865  * Idealy, we should use the hdr coming with the SP FMA event. Since fmdo_send
1866  * entry point can be called before FMA events from SP, we can't rely on
1867  * the SP FMA event hdr. Use the static hdr for packing ds msgs for fmdo_send
1868  * events.
1869  * return >0 for success, or -errno value
1870  * Design assumption: there is one FMA event per ds msg
1871  */
1872 int
etm_pack_ds_msg(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_proto_v1_ev_hdr_t * ev_hdrp,size_t hdr_sz,nvlist_t * evp,etm_pack_msg_type_t msg_type,uint_t ckpt_opt)1873 etm_pack_ds_msg(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
1874 	etm_proto_v1_ev_hdr_t *ev_hdrp, size_t hdr_sz, nvlist_t *evp,
1875 	etm_pack_msg_type_t msg_type, uint_t ckpt_opt)
1876 {
1877 	etm_proto_v1_ev_hdr_t	*hdrp;		/* for FMA_EVENT msg */
1878 	uint32_t		*lenp;		/* ptr to FMA event length */
1879 	size_t			evsz;		/* packed FMA event size */
1880 	char 			*buf;
1881 	uint32_t		rc;		/* for return code  */
1882 	char 			*msg;		/* body of msg to be Qed */
1883 
1884 	etm_iosvc_q_ele_t	msg_ele;	/* io svc msg Q ele */
1885 	etm_proto_v1_ev_hdr_t	*evhdrp;
1886 
1887 
1888 	if (ev_hdrp == NULL) {
1889 		hdrp = &iosvc_hdr;
1890 	} else {
1891 		hdrp = ev_hdrp;
1892 	}
1893 
1894 	/*
1895 	 * determine hdr_sz if 0, otherwise use the one passed in hdr_sz
1896 	 */
1897 
1898 	if (hdr_sz == 0) {
1899 		hdr_sz = sizeof (*hdrp) + (1 * sizeof (hdrp->ev_lens[0]));
1900 	}
1901 
1902 	/*
1903 	 * determine evp size
1904 	 */
1905 	(void) nvlist_size(evp, &evsz, NV_ENCODE_XDR);
1906 
1907 	/* indicate 1 FMA event, no network encoding, and 0-terminate */
1908 	lenp = &hdrp->ev_lens[0];
1909 	*lenp = evsz;
1910 
1911 	/*
1912 	 * now the total of mem needs to be alloc-ed/ds msg size is
1913 	 * hdr_sz + evsz
1914 	 * msg will be freed in etm_send_to_remote_root() after ds_send_msg()
1915 	 */
1916 	msg = fmd_hdl_zalloc(fmd_hdl, hdr_sz + evsz, FMD_SLEEP);
1917 
1918 
1919 	/*
1920 	 * copy hdr, 0 terminate the length vector,  and then evp
1921 	 */
1922 	(void) memcpy(msg, hdrp, sizeof (*hdrp));
1923 	hdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)msg);
1924 	lenp = &hdrp->ev_lens[0];
1925 	lenp++;
1926 	*lenp = 0;
1927 
1928 	buf = fmd_hdl_zalloc(fmd_hdl, evsz, FMD_SLEEP);
1929 	(void) nvlist_pack(evp, (char **)&buf, &evsz, NV_ENCODE_XDR, 0);
1930 	(void) memcpy(msg + hdr_sz, buf, evsz);
1931 	fmd_hdl_free(fmd_hdl, buf, evsz);
1932 
1933 	fmd_hdl_debug(fmd_hdl, "info: hdr_sz= %d evsz= %d in etm_pack_ds_msg"
1934 	    "for ldom %s\n", hdr_sz, evsz, iosvc->ldom_name);
1935 	msg_ele.msg = msg;
1936 	msg_ele.msg_size = hdr_sz + evsz;
1937 	msg_ele.ckpt_flag = ckpt_opt;
1938 
1939 	/*
1940 	 * decide what to do with the msg:
1941 	 * if SP ereports (msg_type == SP_MSG), always enq the msg
1942 	 * if not SP ereports, ie, fmd xprt control msgs, enq it _only_ after
1943 	 * resource.fm.xprt.run has been sent (which sets start_sending_Q to 1)
1944 	 */
1945 	if ((msg_type == SP_MSG) ||
1946 	    (msg_type != SP_MSG) && (iosvc->start_sending_Q == 1)) {
1947 		/*
1948 		 * this is the case when the msg needs to be enq-ed
1949 		 */
1950 		(void) pthread_mutex_lock(&iosvc->msg_q_lock);
1951 		rc = etm_iosvc_msg_enq(fmd_hdl, iosvc, &msg_ele);
1952 		if ((rc > 0) && (ckpt_opt & ETM_CKPT_SAVE) &&
1953 		    (etm_ldom_type == LDOM_TYPE_CONTROL)) {
1954 			(void) etm_ckpt_add(fmd_hdl, evp);
1955 		}
1956 		if (iosvc->msg_q_cur_len == 1)
1957 			(void) pthread_cond_signal(&iosvc->msg_q_cv);
1958 		(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
1959 	} else {
1960 		/*
1961 		 * fmd RDWR xprt procotol startup msgs, send it now!
1962 		 */
1963 		iosvc->ack_ok = 0;
1964 		evhdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)msg_ele.msg);
1965 		evhdrp->ev_pp.pp_xid = iosvc->cur_send_xid + 1;
1966 		while (!iosvc->ack_ok && iosvc->ds_hdl != DS_INVALID_HDL &&
1967 		    !etm_is_dying) {
1968 			if (etm_send_ds_msg(fmd_hdl, B_FALSE, iosvc, &msg_ele,
1969 			    evhdrp) < 0) {
1970 				continue;
1971 			}
1972 		}
1973 		if (msg_type == FMD_XPRT_RUN_MSG)
1974 			iosvc->start_sending_Q = 1;
1975 	}
1976 
1977 	return (rc);
1978 
1979 } /* etm_pack_ds_msg() */
1980 
1981 /*
1982  * Design_Note:	For all etm_resp_q_*() functions and etm_resp_q_* globals,
1983  *		the mutex etm_resp_q_lock must be held by the caller.
1984  */
1985 
1986 /*
1987  * etm_resp_q_enq - add element to tail of ETM responder queue
1988  * etm_resp_q_deq - del element from head of ETM responder queue
1989  *
1990  * return >0 for success, or -errno value
1991  */
1992 
1993 static int
etm_resp_q_enq(fmd_hdl_t * hdl,etm_resp_q_ele_t * rqep)1994 etm_resp_q_enq(fmd_hdl_t *hdl, etm_resp_q_ele_t *rqep)
1995 {
1996 	etm_resp_q_ele_t	*newp;	/* ptr to new resp q ele */
1997 
1998 	if (etm_resp_q_cur_len >= etm_resp_q_max_len) {
1999 		fmd_hdl_debug(hdl, "warning: enq to full responder queue\n");
2000 		etm_stats.etm_enq_drop_resp_q.fmds_value.ui64++;
2001 		return (-E2BIG);
2002 	}
2003 
2004 	newp = fmd_hdl_zalloc(hdl, sizeof (*newp), FMD_SLEEP);
2005 	(void) memcpy(newp, rqep, sizeof (*newp));
2006 	newp->rqe_nextp = NULL;
2007 
2008 	if (etm_resp_q_cur_len == 0) {
2009 		etm_resp_q_head = newp;
2010 	} else {
2011 		etm_resp_q_tail->rqe_nextp = newp;
2012 	}
2013 	etm_resp_q_tail = newp;
2014 	etm_resp_q_cur_len++;
2015 	etm_stats.etm_resp_q_cur_len.fmds_value.ui64 = etm_resp_q_cur_len;
2016 
2017 	return (1);
2018 
2019 } /* etm_resp_q_enq() */
2020 
2021 static int
etm_resp_q_deq(fmd_hdl_t * hdl,etm_resp_q_ele_t * rqep)2022 etm_resp_q_deq(fmd_hdl_t *hdl, etm_resp_q_ele_t *rqep)
2023 {
2024 	etm_resp_q_ele_t	*oldp;	/* ptr to old resp q ele */
2025 
2026 	if (etm_resp_q_cur_len == 0) {
2027 		fmd_hdl_debug(hdl, "warning: deq from empty responder queue\n");
2028 		etm_stats.etm_deq_drop_resp_q.fmds_value.ui64++;
2029 		return (-ENOENT);
2030 	}
2031 
2032 	(void) memcpy(rqep, etm_resp_q_head, sizeof (*rqep));
2033 	rqep->rqe_nextp = NULL;
2034 
2035 	oldp = etm_resp_q_head;
2036 	etm_resp_q_head = etm_resp_q_head->rqe_nextp;
2037 	fmd_hdl_free(hdl, oldp, sizeof (*oldp));
2038 
2039 	etm_resp_q_cur_len--;
2040 	etm_stats.etm_resp_q_cur_len.fmds_value.ui64 = etm_resp_q_cur_len;
2041 	if (etm_resp_q_cur_len == 0) {
2042 		etm_resp_q_tail = NULL;
2043 	}
2044 
2045 	return (1);
2046 
2047 } /* etm_resp_q_deq() */
2048 
2049 /*
2050  * etm_maybe_enq_response - check the given message header to see
2051  *				whether a response has been requested,
2052  *				if so then enqueue the given connection
2053  *				and header for later transport by the
2054  *				responder thread as an ETM response msg,
2055  *				return 0 for nop, >0 success, or -errno value
2056  */
2057 
2058 static ssize_t
etm_maybe_enq_response(fmd_hdl_t * hdl,etm_xport_conn_t conn,void * hdrp,uint32_t hdr_sz,int32_t resp_code)2059 etm_maybe_enq_response(fmd_hdl_t *hdl, etm_xport_conn_t conn,
2060     void *hdrp, uint32_t hdr_sz, int32_t resp_code)
2061 {
2062 	ssize_t			rv;		/* ret val */
2063 	etm_proto_v1_pp_t	*ppp;		/* protocol preamble ptr */
2064 	uint8_t			orig_msg_type;	/* orig hdr's message type */
2065 	uint32_t		orig_timeout;	/* orig hdr's timeout */
2066 	etm_resp_q_ele_t	rqe;		/* responder queue ele */
2067 
2068 	ppp = hdrp;
2069 	orig_msg_type = ppp->pp_msg_type;
2070 	orig_timeout = ppp->pp_timeout;
2071 
2072 	/* bail out now if no response is to be sent */
2073 
2074 	if (orig_timeout == ETM_PROTO_V1_TIMEOUT_NONE) {
2075 		return (0);
2076 	} /* if a nop */
2077 
2078 	if ((orig_msg_type != ETM_MSG_TYPE_FMA_EVENT) &&
2079 	    (orig_msg_type != ETM_MSG_TYPE_ALERT) &&
2080 	    (orig_msg_type != ETM_MSG_TYPE_CONTROL)) {
2081 		fmd_hdl_debug(hdl, "warning: bad msg type 0x%x\n",
2082 		    orig_msg_type);
2083 		return (-EINVAL);
2084 	} /* if inappropriate hdr for a response msg */
2085 
2086 	/*
2087 	 * enqueue the msg hdr and nudge the responder thread
2088 	 * if the responder queue was previously empty
2089 	 */
2090 
2091 	rqe.rqe_conn = conn;
2092 	rqe.rqe_hdrp = hdrp;
2093 	rqe.rqe_hdr_sz = hdr_sz;
2094 	rqe.rqe_resp_code = resp_code;
2095 
2096 	(void) pthread_mutex_lock(&etm_resp_q_lock);
2097 
2098 	if (etm_resp_q_cur_len == etm_resp_q_max_len)
2099 		(void) pthread_cond_wait(&etm_resp_q_cv, &etm_resp_q_lock);
2100 
2101 	rv = etm_resp_q_enq(hdl, &rqe);
2102 	if (etm_resp_q_cur_len == 1)
2103 		(void) pthread_cond_signal(&etm_resp_q_cv);
2104 	(void) pthread_mutex_unlock(&etm_resp_q_lock);
2105 
2106 	return (rv);
2107 
2108 } /* etm_maybe_enq_response() */
2109 
2110 /*
2111  * Design_Note:	We rely on the fact that all message types have
2112  *		a common protocol preamble; if this fact should
2113  *		ever change it may break the code below. We also
2114  *		rely on the fact that FMA_EVENT and CONTROL headers
2115  *		returned by etm_hdr_read() will be sized large enough
2116  *		to reuse them as RESPONSE headers if the remote endpt
2117  *		asked for a response via the pp_timeout field.
2118  */
2119 
2120 /*
2121  * etm_send_response - use the given message header and response code
2122  *			to construct an appropriate response message,
2123  *			and send it back on the given connection,
2124  *			return >0 for success, or -errno value
2125  */
2126 
2127 static ssize_t
etm_send_response(fmd_hdl_t * hdl,etm_xport_conn_t conn,void * hdrp,int32_t resp_code)2128 etm_send_response(fmd_hdl_t *hdl, etm_xport_conn_t conn,
2129     void *hdrp, int32_t resp_code)
2130 {
2131 	ssize_t			rv;		/* ret val */
2132 	etm_proto_v1_pp_t	*ppp;		/* protocol preamble ptr */
2133 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
2134 	uint8_t			resp_body[4];	/* response body if needed */
2135 	uint8_t			*resp_msg;	/* response hdr+body */
2136 	size_t			hdr_sz;		/* sizeof response hdr */
2137 	uint8_t			orig_msg_type;	/* orig hdr's message type */
2138 
2139 	ppp = hdrp;
2140 	orig_msg_type = ppp->pp_msg_type;
2141 
2142 	if (etm_debug_lvl >= 2) {
2143 		etm_show_time(hdl, "ante resp send");
2144 	}
2145 
2146 	/* reuse the given header as a response header */
2147 
2148 	resp_hdrp = hdrp;
2149 	resp_hdrp->resp_code = resp_code;
2150 	resp_hdrp->resp_len = 0;		/* default is empty body */
2151 
2152 	if ((orig_msg_type == ETM_MSG_TYPE_CONTROL) &&
2153 	    (ppp->pp_sub_type == ETM_CTL_SEL_VER_NEGOT_REQ)) {
2154 		resp_body[0] = ETM_PROTO_V2;
2155 		resp_body[1] = ETM_PROTO_V3;
2156 		resp_body[2] = 0;
2157 		resp_hdrp->resp_len = 3;
2158 	} /* if should send our/negotiated proto ver in resp body */
2159 
2160 	/* respond with the proto ver that was negotiated */
2161 
2162 	resp_hdrp->resp_pp.pp_proto_ver = etm_resp_ver;
2163 	resp_hdrp->resp_pp.pp_msg_type = ETM_MSG_TYPE_RESPONSE;
2164 	resp_hdrp->resp_pp.pp_timeout = ETM_PROTO_V1_TIMEOUT_NONE;
2165 
2166 	/*
2167 	 * send the whole response msg in one write, header and body;
2168 	 * avoid the alloc-and-copy if we can reuse the hdr as the msg,
2169 	 * ie, if the body is empty. update the response stats.
2170 	 */
2171 
2172 	hdr_sz = sizeof (etm_proto_v1_resp_hdr_t);
2173 
2174 	resp_msg = hdrp;
2175 	if (resp_hdrp->resp_len > 0) {
2176 		resp_msg = fmd_hdl_zalloc(hdl, hdr_sz + resp_hdrp->resp_len,
2177 		    FMD_SLEEP);
2178 		(void) memcpy(resp_msg, resp_hdrp, hdr_sz);
2179 		(void) memcpy(resp_msg + hdr_sz, resp_body,
2180 		    resp_hdrp->resp_len);
2181 	}
2182 
2183 	(void) pthread_mutex_lock(&etm_write_lock);
2184 	rv = etm_io_op(hdl, "bad io write on resp msg", conn,
2185 	    resp_msg, hdr_sz + resp_hdrp->resp_len, ETM_IO_OP_WR);
2186 	(void) pthread_mutex_unlock(&etm_write_lock);
2187 	if (rv < 0) {
2188 		goto func_ret;
2189 	}
2190 
2191 	etm_stats.etm_wr_hdr_response.fmds_value.ui64++;
2192 	etm_stats.etm_wr_body_response.fmds_value.ui64++;
2193 
2194 	fmd_hdl_debug(hdl, "info: sent V%u RESPONSE msg to xport "
2195 	    "xid 0x%x code %d len %u\n",
2196 	    (unsigned int)resp_hdrp->resp_pp.pp_proto_ver,
2197 	    resp_hdrp->resp_pp.pp_xid, resp_hdrp->resp_code,
2198 	    resp_hdrp->resp_len);
2199 func_ret:
2200 
2201 	if (resp_hdrp->resp_len > 0) {
2202 		fmd_hdl_free(hdl, resp_msg, hdr_sz + resp_hdrp->resp_len);
2203 	}
2204 	if (etm_debug_lvl >= 2) {
2205 		etm_show_time(hdl, "post resp send");
2206 	}
2207 	return (rv);
2208 
2209 } /* etm_send_response() */
2210 
2211 /*
2212  * etm_reset_xport - reset the transport layer (via fini;init)
2213  *			presumably for an error condition we cannot
2214  *			otherwise recover from (ex: hung LDC channel)
2215  *
2216  * caveats - no checking/locking is done to ensure an existing connection
2217  *		is idle during an xport reset; we don't want to deadlock
2218  *		and presumably the transport is stuck/unusable anyway
2219  */
2220 
2221 static void
etm_reset_xport(fmd_hdl_t * hdl)2222 etm_reset_xport(fmd_hdl_t *hdl)
2223 {
2224 	(void) etm_xport_fini(hdl);
2225 	(void) etm_xport_init(hdl);
2226 	etm_stats.etm_reset_xport.fmds_value.ui64++;
2227 
2228 } /* etm_reset_xport() */
2229 
2230 /*
2231  * etm_handle_new_conn - receive an ETM message sent from the other end via
2232  *			the given open connection, pull out any FMA events
2233  *			and post them to the local FMD (or handle any ETM
2234  *			control or response msg); when done, close the
2235  *			connection
2236  */
2237 
2238 static void
etm_handle_new_conn(fmd_hdl_t * hdl,etm_xport_conn_t conn)2239 etm_handle_new_conn(fmd_hdl_t *hdl, etm_xport_conn_t conn)
2240 {
2241 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
2242 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
2243 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
2244 	etm_proto_v3_sa_hdr_t	*sa_hdrp;	/* for ALERT msg */
2245 	etm_iosvc_t		*iosvc;		/* iosvc data structure */
2246 	int32_t			resp_code;	/* response code */
2247 	ssize_t			enq_rv;		/* resp_q enqueue status */
2248 	size_t			hdr_sz;		/* sizeof header */
2249 	size_t			evsz;		/* FMA event size */
2250 	uint8_t			*body_buf;	/* msg body buffer */
2251 	uint32_t		body_sz;	/* sizeof body_buf */
2252 	uint32_t		ev_cnt;		/* count of FMA events */
2253 	uint8_t			*bp;		/* byte ptr within body_buf */
2254 	nvlist_t		*evp;		/* ptr to unpacked FMA event */
2255 	char			*class;		/* FMA event class */
2256 	ssize_t			i, n;		/* gen use */
2257 	int			should_reset_xport; /* bool to reset xport */
2258 	char			ldom_name[MAX_LDOM_NAME]; /* ldom name */
2259 	int			rc;		/* return code */
2260 	uint64_t		did;		/* domain id */
2261 
2262 
2263 	if (etm_debug_lvl >= 2) {
2264 		etm_show_time(hdl, "ante conn handle");
2265 	}
2266 	fmd_hdl_debug(hdl, "info: handling new conn %p\n", conn);
2267 
2268 	should_reset_xport = 0;
2269 	ev_hdrp = NULL;
2270 	ctl_hdrp = NULL;
2271 	resp_hdrp = NULL;
2272 	sa_hdrp = NULL;
2273 	body_buf = NULL;
2274 	class = NULL;
2275 	evp = NULL;
2276 	resp_code = 0;	/* default is success */
2277 	enq_rv = 0;	/* default is nop, ie, did not enqueue */
2278 
2279 	/* read a network decoded message header from the connection */
2280 
2281 	if ((ev_hdrp = etm_hdr_read(hdl, conn, &hdr_sz)) == NULL) {
2282 		/* errno assumed set by above call */
2283 		should_reset_xport = (errno == ENOTACTIVE);
2284 		fmd_hdl_debug(hdl, "error: FMA event dropped: "
2285 		    "bad hdr read errno %d\n", errno);
2286 		etm_stats.etm_rd_drop_fmaevent.fmds_value.ui64++;
2287 		goto func_ret;
2288 	}
2289 
2290 	/*
2291 	 * handle the message based on its preamble pp_msg_type
2292 	 * which is known to be valid from etm_hdr_read() checks
2293 	 */
2294 
2295 	if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
2296 
2297 		fmd_hdl_debug(hdl, "info: rcvd FMA_EVENT msg from xport\n");
2298 
2299 		/* allocate buf large enough for whole body / all FMA events */
2300 
2301 		body_sz = 0;
2302 		for (i = 0; ev_hdrp->ev_lens[i] != 0; i++) {
2303 			body_sz += ev_hdrp->ev_lens[i];
2304 		} /* for summing sizes of all FMA events */
2305 		if (i > etm_stats.etm_rd_max_ev_per_msg.fmds_value.ui64)
2306 			etm_stats.etm_rd_max_ev_per_msg.fmds_value.ui64 = i;
2307 		ev_cnt = i;
2308 
2309 		if (etm_debug_lvl >= 1) {
2310 			fmd_hdl_debug(hdl, "info: event lengths %u sum %u\n",
2311 			    ev_cnt, body_sz);
2312 		}
2313 
2314 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2315 
2316 		/* read all the FMA events at once */
2317 
2318 		if ((n = etm_io_op(hdl, "FMA event dropped: "
2319 		    "bad io read on event bodies", conn, body_buf, body_sz,
2320 		    ETM_IO_OP_RD)) < 0) {
2321 			should_reset_xport = (n == -ENOTACTIVE);
2322 			etm_stats.etm_rd_drop_fmaevent.fmds_value.ui64++;
2323 			goto func_ret;
2324 		}
2325 
2326 		etm_stats.etm_rd_xport_bytes.fmds_value.ui64 += body_sz;
2327 		etm_stats.etm_rd_body_fmaevent.fmds_value.ui64 += ev_cnt;
2328 
2329 		/*
2330 		 * now that we've read the entire ETM msg from the conn,
2331 		 * which avoids later ETM protocol framing errors if we didn't,
2332 		 * check for dup msg/xid against last good FMD posting,
2333 		 * if a dup then resend response but skip repost to FMD
2334 		 */
2335 
2336 		if (ev_hdrp->ev_pp.pp_xid == etm_xid_posted_logged_ev) {
2337 			enq_rv = etm_maybe_enq_response(hdl, conn,
2338 			    ev_hdrp, hdr_sz, 0);
2339 			fmd_hdl_debug(hdl, "info: skipping dup FMA event post "
2340 			    "xid 0x%x\n", etm_xid_posted_logged_ev);
2341 			etm_stats.etm_rd_dup_fmaevent.fmds_value.ui64++;
2342 			goto func_ret;
2343 		}
2344 
2345 		/* unpack each FMA event and post it to FMD */
2346 
2347 		bp = body_buf;
2348 		for (i = 0; i < ev_cnt; i++) {
2349 			if ((n = nvlist_unpack((char *)bp,
2350 			    ev_hdrp->ev_lens[i], &evp, 0)) != 0) {
2351 				resp_code = (-n);
2352 				enq_rv = etm_maybe_enq_response(hdl, conn,
2353 				    ev_hdrp, hdr_sz, resp_code);
2354 				fmd_hdl_error(hdl, "error: FMA event dropped: "
2355 				    "bad event body unpack errno %d\n", n);
2356 				if (etm_debug_lvl >= 2) {
2357 					fmd_hdl_debug(hdl, "info: FMA event "
2358 					    "hexdump %d bytes:\n",
2359 					    ev_hdrp->ev_lens[i]);
2360 					etm_hexdump(hdl, bp,
2361 					    ev_hdrp->ev_lens[i]);
2362 				}
2363 				etm_stats.etm_os_nvlist_unpack_fail.fmds_value.
2364 				    ui64++;
2365 				etm_stats.etm_rd_drop_fmaevent.fmds_value.
2366 				    ui64++;
2367 				bp += ev_hdrp->ev_lens[i];
2368 				continue;
2369 			}
2370 
2371 			if (etm_debug_lvl >= 1) {
2372 				(void) nvlist_lookup_string(evp, FM_CLASS,
2373 				    &class);
2374 				if (class == NULL) {
2375 					class = "NULL";
2376 				}
2377 				fmd_hdl_debug(hdl, "info: FMA event %p "
2378 				    "class %s\n", evp, class);
2379 			}
2380 
2381 			rc = nvlist_size(evp, &evsz, NV_ENCODE_XDR);
2382 			fmd_hdl_debug(hdl,
2383 			    "info: evp size before pack ds msg %d\n", evsz);
2384 			ldom_name[0] = '\0';
2385 			rc = etm_filter_find_ldom_id(hdl, evp, ldom_name,
2386 			    MAX_LDOM_NAME, &did);
2387 
2388 			/*
2389 			 * if rc is zero and the ldom_name is not "primary",
2390 			 * the evp belongs to a root domain, put the evp in an
2391 			 * outgoing etm queue,
2392 			 * in all other cases, whether ldom_name is primary or
2393 			 * can't find a ldom name, call etm_post_to_fmd
2394 			 */
2395 			if ((rc == 0) && strcmp(ldom_name, "primary") &&
2396 			    strcmp(ldom_name, "")) {
2397 				/*
2398 				 * use the ldom_name, guaranteered at this point
2399 				 * to be a valid ldom name/non-NULL, to find the
2400 				 * iosvc data.
2401 				 * add an iosvc struct if can not find one
2402 				 */
2403 				(void) pthread_mutex_unlock(&iosvc_list_lock);
2404 				iosvc = etm_iosvc_lookup(hdl, ldom_name,
2405 				    DS_INVALID_HDL, B_TRUE);
2406 				(void) pthread_mutex_unlock(&iosvc_list_lock);
2407 				if (iosvc == NULL) {
2408 					fmd_hdl_debug(hdl,
2409 					    "error: can't find iosvc for ldom "
2410 					    "name %s\n", ldom_name);
2411 				} else {
2412 					resp_code = 0;
2413 					(void) etm_pack_ds_msg(hdl, iosvc,
2414 					    ev_hdrp, hdr_sz, evp,
2415 					    SP_MSG, ETM_CKPT_SAVE);
2416 					/*
2417 					 * call the new fmd_xprt_log()
2418 					 */
2419 					fmd_xprt_log(hdl, etm_fmd_xprt, evp, 0);
2420 					etm_xid_posted_logged_ev =
2421 					    ev_hdrp->ev_pp.pp_xid;
2422 				}
2423 			} else {
2424 				/*
2425 				 * post the fma event to the control fmd
2426 				 */
2427 				resp_code = etm_post_to_fmd(hdl, etm_fmd_xprt,
2428 				    evp);
2429 				if (resp_code >= 0) {
2430 					etm_xid_posted_logged_ev =
2431 					    ev_hdrp->ev_pp.pp_xid;
2432 				}
2433 			}
2434 
2435 			evp = NULL;
2436 			enq_rv = etm_maybe_enq_response(hdl, conn,
2437 			    ev_hdrp, hdr_sz, resp_code);
2438 			bp += ev_hdrp->ev_lens[i];
2439 		} /* foreach FMA event in the body buffer */
2440 
2441 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_CONTROL) {
2442 
2443 		ctl_hdrp = (void*)ev_hdrp;
2444 
2445 		fmd_hdl_debug(hdl, "info: rcvd CONTROL msg from xport\n");
2446 		if (etm_debug_lvl >= 1) {
2447 			fmd_hdl_debug(hdl, "info: ctl sel %d xid 0x%x\n",
2448 			    (int)ctl_hdrp->ctl_pp.pp_sub_type,
2449 			    ctl_hdrp->ctl_pp.pp_xid);
2450 		}
2451 
2452 		/*
2453 		 * if we have a VER_NEGOT_REQ read the body and validate
2454 		 * the protocol version set contained therein,
2455 		 * otherwise we have a PING_REQ (which has no body)
2456 		 * and we [also] fall thru to the code which sends a
2457 		 * response msg if the pp_timeout field requested one
2458 		 */
2459 
2460 		if (ctl_hdrp->ctl_pp.pp_sub_type == ETM_CTL_SEL_VER_NEGOT_REQ) {
2461 
2462 			body_sz = ctl_hdrp->ctl_len;
2463 			body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2464 
2465 			if ((n = etm_io_op(hdl, "bad io read on ctl body",
2466 			    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2467 				should_reset_xport = (n == -ENOTACTIVE);
2468 				goto func_ret;
2469 			}
2470 
2471 			/* complain if version set completely incompatible */
2472 
2473 			for (i = 0; i < body_sz; i++) {
2474 				if ((body_buf[i] == ETM_PROTO_V1) ||
2475 				    (body_buf[i] == ETM_PROTO_V2) ||
2476 				    (body_buf[i] == ETM_PROTO_V3)) {
2477 					break;
2478 				}
2479 			}
2480 			if (i >= body_sz) {
2481 				etm_stats.etm_ver_bad.fmds_value.ui64++;
2482 				resp_code = (-EPROTO);
2483 			}
2484 
2485 		} /* if got version set request */
2486 
2487 		etm_stats.etm_rd_body_control.fmds_value.ui64++;
2488 
2489 		enq_rv = etm_maybe_enq_response(hdl, conn,
2490 		    ctl_hdrp, hdr_sz, resp_code);
2491 
2492 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
2493 
2494 		resp_hdrp = (void*)ev_hdrp;
2495 
2496 		fmd_hdl_debug(hdl, "info: rcvd RESPONSE msg from xport\n");
2497 		if (etm_debug_lvl >= 1) {
2498 			fmd_hdl_debug(hdl, "info: resp xid 0x%x\n",
2499 			    (int)resp_hdrp->resp_pp.pp_xid);
2500 		}
2501 
2502 		body_sz = resp_hdrp->resp_len;
2503 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2504 
2505 		if ((n = etm_io_op(hdl, "bad io read on resp len",
2506 		    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2507 			should_reset_xport = (n == -ENOTACTIVE);
2508 			goto func_ret;
2509 		}
2510 
2511 		etm_stats.etm_rd_body_response.fmds_value.ui64++;
2512 
2513 		/*
2514 		 * look up the xid to interpret the response body
2515 		 *
2516 		 * ping is a nop; for ver negot confirm that a supported
2517 		 * protocol version was negotiated and remember which one
2518 		 */
2519 
2520 		if ((resp_hdrp->resp_pp.pp_xid != etm_xid_ping) &&
2521 		    (resp_hdrp->resp_pp.pp_xid != etm_xid_ver_negot)) {
2522 			etm_stats.etm_xid_bad.fmds_value.ui64++;
2523 			goto func_ret;
2524 		}
2525 
2526 		if (resp_hdrp->resp_pp.pp_xid == etm_xid_ver_negot) {
2527 			if ((body_buf[0] < ETM_PROTO_V1) ||
2528 			    (body_buf[0] > ETM_PROTO_V3)) {
2529 				etm_stats.etm_ver_bad.fmds_value.ui64++;
2530 				goto func_ret;
2531 			}
2532 			etm_resp_ver = body_buf[0];
2533 		} /* if have resp to last req to negotiate proto ver */
2534 
2535 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_ALERT) {
2536 
2537 		sa_hdrp = (void*)ev_hdrp;
2538 
2539 		fmd_hdl_debug(hdl, "info: rcvd ALERT msg from xport\n");
2540 		if (etm_debug_lvl >= 1) {
2541 			fmd_hdl_debug(hdl, "info: sa sel %d xid 0x%x\n",
2542 			    (int)sa_hdrp->sa_pp.pp_sub_type,
2543 			    sa_hdrp->sa_pp.pp_xid);
2544 		}
2545 
2546 		body_sz = sa_hdrp->sa_len;
2547 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2548 
2549 		if ((n = etm_io_op(hdl, "bad io read on sa body",
2550 		    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2551 			should_reset_xport = (n == -ENOTACTIVE);
2552 			goto func_ret;
2553 		}
2554 
2555 		etm_stats.etm_rd_body_alert.fmds_value.ui64++;
2556 
2557 		/*
2558 		 * now that we've read the entire ETM msg from the conn,
2559 		 * which avoids later ETM protocol framing errors if we didn't,
2560 		 * check for dup msg/xid against last good syslog posting,
2561 		 * if a dup then resend response but skip repost to syslog
2562 		 */
2563 
2564 		if (sa_hdrp->sa_pp.pp_xid == etm_xid_posted_sa) {
2565 			enq_rv = etm_maybe_enq_response(hdl, conn,
2566 			    sa_hdrp, hdr_sz, 0);
2567 			fmd_hdl_debug(hdl, "info: skipping dup ALERT post "
2568 			    "xid 0x%x\n", etm_xid_posted_sa);
2569 			etm_stats.etm_rd_dup_alert.fmds_value.ui64++;
2570 			goto func_ret;
2571 		}
2572 
2573 		resp_code = etm_post_to_syslog(hdl, sa_hdrp->sa_priority,
2574 		    body_sz, body_buf);
2575 		if (resp_code >= 0) {
2576 			etm_xid_posted_sa = sa_hdrp->sa_pp.pp_xid;
2577 		}
2578 		enq_rv = etm_maybe_enq_response(hdl, conn,
2579 		    sa_hdrp, hdr_sz, resp_code);
2580 	} /* whether we have a FMA_EVENT, CONTROL, RESPONSE or ALERT msg */
2581 
2582 func_ret:
2583 
2584 	if (etm_debug_lvl >= 2) {
2585 		etm_show_time(hdl, "post conn handle");
2586 	}
2587 
2588 	/*
2589 	 * if no responder ele was enqueued, close the conn now
2590 	 * and free the ETM msg hdr; the ETM msg body is not needed
2591 	 * by the responder thread and should always be freed here
2592 	 */
2593 
2594 	if (enq_rv <= 0) {
2595 		(void) etm_conn_close(hdl, "bad conn close after msg recv",
2596 		    conn);
2597 		if (ev_hdrp != NULL) {
2598 			fmd_hdl_free(hdl, ev_hdrp, hdr_sz);
2599 		}
2600 	}
2601 	if (body_buf != NULL) {
2602 		fmd_hdl_free(hdl, body_buf, body_sz);
2603 	}
2604 	if (should_reset_xport) {
2605 		etm_reset_xport(hdl);
2606 	}
2607 } /* etm_handle_new_conn() */
2608 
2609 /*
2610  * etm_handle_bad_accept - recover from a failed connection acceptance
2611  */
2612 
2613 static void
etm_handle_bad_accept(fmd_hdl_t * hdl,int nev)2614 etm_handle_bad_accept(fmd_hdl_t *hdl, int nev)
2615 {
2616 	int	should_reset_xport; /* bool to reset xport */
2617 
2618 	should_reset_xport = (nev == -ENOTACTIVE);
2619 	fmd_hdl_debug(hdl, "error: bad conn accept errno %d\n", (-nev));
2620 	etm_stats.etm_xport_accept_fail.fmds_value.ui64++;
2621 	(void) etm_sleep(etm_bad_acc_to_sec); /* avoid spinning CPU */
2622 	if (should_reset_xport) {
2623 		etm_reset_xport(hdl);
2624 	}
2625 } /* etm_handle_bad_accept() */
2626 
2627 /*
2628  * etm_server - loop forever accepting new connections
2629  *		using the given FMD handle,
2630  *		handling any ETM msgs sent from the other side
2631  *		via each such connection
2632  */
2633 
2634 static void
etm_server(void * arg)2635 etm_server(void *arg)
2636 {
2637 	etm_xport_conn_t	conn;		/* connection handle */
2638 	int			nev;		/* -errno val */
2639 	fmd_hdl_t		*hdl;		/* FMD handle */
2640 
2641 	hdl = arg;
2642 
2643 	fmd_hdl_debug(hdl, "info: connection server starting\n");
2644 
2645 	/*
2646 	 * Restore the checkpointed events and dispatch them before starting to
2647 	 * receive more events from the sp.
2648 	 */
2649 	etm_ckpt_recover(hdl);
2650 
2651 	while (!etm_is_dying) {
2652 
2653 		if ((conn = etm_xport_accept(hdl, NULL)) == NULL) {
2654 			/* errno assumed set by above call */
2655 			nev = (-errno);
2656 			if (etm_is_dying) {
2657 				break;
2658 			}
2659 			etm_handle_bad_accept(hdl, nev);
2660 			continue;
2661 		}
2662 
2663 		/* handle the new message/connection, closing it when done */
2664 
2665 		etm_handle_new_conn(hdl, conn);
2666 
2667 	} /* while accepting new connections until ETM dies */
2668 
2669 	/* ETM is dying (probably due to "fmadm unload etm") */
2670 
2671 	fmd_hdl_debug(hdl, "info: connection server is dying\n");
2672 
2673 } /* etm_server() */
2674 
2675 /*
2676  * etm_responder - loop forever waiting for new responder queue elements
2677  *		to be enqueued, for each one constructing and sending
2678  *		an ETM response msg to the other side, and closing its
2679  *		associated connection when appropriate
2680  *
2681  *	this thread exists to ensure that the etm_server() thread
2682  *	never pends indefinitely waiting on the xport write lock, and is
2683  *	hence always available to accept new connections and handle
2684  *	incoming messages
2685  *
2686  *	this design relies on the fact that each connection accepted and
2687  *	returned by the ETM xport layer is unique, and each can be closed
2688  *	independently of the others while multiple connections are
2689  *	outstanding
2690  */
2691 
2692 static void
etm_responder(void * arg)2693 etm_responder(void *arg)
2694 {
2695 	ssize_t			n;		/* gen use */
2696 	fmd_hdl_t		*hdl;		/* FMD handle */
2697 	etm_resp_q_ele_t	rqe;		/* responder queue ele */
2698 
2699 	hdl = arg;
2700 
2701 	fmd_hdl_debug(hdl, "info: responder server starting\n");
2702 
2703 	while (!etm_is_dying) {
2704 
2705 		(void) pthread_mutex_lock(&etm_resp_q_lock);
2706 
2707 		while (etm_resp_q_cur_len == 0) {
2708 			(void) pthread_cond_wait(&etm_resp_q_cv,
2709 			    &etm_resp_q_lock);
2710 			if (etm_is_dying) {
2711 				(void) pthread_mutex_unlock(&etm_resp_q_lock);
2712 				goto func_ret;
2713 			}
2714 		} /* while the responder queue is empty, wait to be nudged */
2715 
2716 		/*
2717 		 * for every responder ele that has been enqueued,
2718 		 * dequeue and send it as an ETM response msg,
2719 		 * closing its associated conn and freeing its hdr
2720 		 *
2721 		 * enter the queue draining loop holding the responder
2722 		 * queue lock, but do not hold the lock indefinitely
2723 		 * (the actual send may pend us indefinitely),
2724 		 * so that other threads will never pend for long
2725 		 * trying to enqueue a new element
2726 		 */
2727 
2728 		while (etm_resp_q_cur_len > 0) {
2729 
2730 			(void) etm_resp_q_deq(hdl, &rqe);
2731 
2732 			if ((etm_resp_q_cur_len + 1) == etm_resp_q_max_len)
2733 				(void) pthread_cond_signal(&etm_resp_q_cv);
2734 
2735 			(void) pthread_mutex_unlock(&etm_resp_q_lock);
2736 
2737 			if ((n = etm_send_response(hdl, rqe.rqe_conn,
2738 			    rqe.rqe_hdrp, rqe.rqe_resp_code)) < 0) {
2739 				fmd_hdl_error(hdl, "error: bad resp send "
2740 				    "errno %d\n", (-n));
2741 			}
2742 
2743 			(void) etm_conn_close(hdl, "bad conn close after resp",
2744 			    rqe.rqe_conn);
2745 			fmd_hdl_free(hdl, rqe.rqe_hdrp, rqe.rqe_hdr_sz);
2746 
2747 			if (etm_is_dying) {
2748 				goto func_ret;
2749 			}
2750 			(void) pthread_mutex_lock(&etm_resp_q_lock);
2751 
2752 		} /* while draining the responder queue */
2753 
2754 		(void) pthread_mutex_unlock(&etm_resp_q_lock);
2755 
2756 	} /* while awaiting and sending resp msgs until ETM dies */
2757 
2758 func_ret:
2759 
2760 	/* ETM is dying (probably due to "fmadm unload etm") */
2761 
2762 	fmd_hdl_debug(hdl, "info: responder server is dying\n");
2763 
2764 	(void) pthread_mutex_lock(&etm_resp_q_lock);
2765 	if (etm_resp_q_cur_len > 0) {
2766 		fmd_hdl_error(hdl, "warning: %d response msgs dropped\n",
2767 		    (int)etm_resp_q_cur_len);
2768 		while (etm_resp_q_cur_len > 0) {
2769 			(void) etm_resp_q_deq(hdl, &rqe);
2770 			(void) etm_conn_close(hdl, "bad conn close after deq",
2771 			    rqe.rqe_conn);
2772 			fmd_hdl_free(hdl, rqe.rqe_hdrp, rqe.rqe_hdr_sz);
2773 		}
2774 	}
2775 	(void) pthread_mutex_unlock(&etm_resp_q_lock);
2776 
2777 } /* etm_responder() */
2778 
2779 static void *
etm_init_alloc(size_t size)2780 etm_init_alloc(size_t size)
2781 {
2782 	return (fmd_hdl_alloc(init_hdl, size, FMD_SLEEP));
2783 }
2784 
2785 static void
etm_init_free(void * addr,size_t size)2786 etm_init_free(void *addr, size_t size)
2787 {
2788 	fmd_hdl_free(init_hdl, addr, size);
2789 }
2790 
2791 /*
2792  * ---------------------root ldom support functions -----------------------
2793  */
2794 
2795 /*
2796  * use a static array async_event_q instead of dynamicaly allocated mem  queue
2797  * for etm_async_q_enq and etm_async_q_deq.
2798  * This is not running in an fmd aux thread, can't use the fmd_hdl_* funcs.
2799  * caller needs to grab the mutex lock before calling this func.
2800  * return >0 for success, or -errno value
2801  */
2802 static int
etm_async_q_enq(etm_async_event_ele_t * async_e)2803 etm_async_q_enq(etm_async_event_ele_t *async_e)
2804 {
2805 
2806 	if (etm_async_q_cur_len >= etm_async_q_max_len) {
2807 		/* etm_stats.etm_enq_drop_async_q.fmds_value.ui64++; */
2808 		return (-E2BIG);
2809 	}
2810 
2811 	(void) memcpy(&async_event_q[etm_async_q_tail], async_e,
2812 	    sizeof (*async_e));
2813 
2814 	etm_async_q_tail++;
2815 	if (etm_async_q_tail == etm_async_q_max_len) {
2816 		etm_async_q_tail = 0;
2817 	}
2818 	etm_async_q_cur_len++;
2819 
2820 /* etm_stats.etm_async_q_cur_len.fmds_value.ui64 = etm_async_q_cur_len; */
2821 
2822 	return (1);
2823 
2824 } /* etm_async_q_enq() */
2825 
2826 
2827 static int
etm_async_q_deq(etm_async_event_ele_t * async_e)2828 etm_async_q_deq(etm_async_event_ele_t *async_e)
2829 {
2830 
2831 	if (etm_async_q_cur_len == 0) {
2832 		/* etm_stats.etm_deq_drop_async_q.fmds_value.ui64++; */
2833 		return (-ENOENT);
2834 	}
2835 
2836 	(void) memcpy(async_e, &async_event_q[etm_async_q_head],
2837 	    sizeof (*async_e));
2838 
2839 	etm_async_q_head++;
2840 	if (etm_async_q_head == etm_async_q_max_len) {
2841 		etm_async_q_head = 0;
2842 	}
2843 	etm_async_q_cur_len--;
2844 
2845 	return (1);
2846 } /* etm_async_q_deq */
2847 
2848 
2849 /*
2850  * setting up the fields in iosvc at DS_REG_CB time
2851  */
2852 void
etm_iosvc_setup(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_async_event_ele_t * async_e)2853 etm_iosvc_setup(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
2854 	etm_async_event_ele_t *async_e)
2855 {
2856 	iosvc->ds_hdl = async_e->ds_hdl;
2857 	iosvc->cur_send_xid = 0;
2858 	iosvc->xid_posted_ev = 0;
2859 	iosvc->start_sending_Q = 0;
2860 
2861 	/*
2862 	 * open the fmd xprt if it
2863 	 * hasn't been previously opened
2864 	 */
2865 	fmd_hdl_debug(fmd_hdl,  "info: before fmd_xprt_open ldom_name is %s\n",
2866 	    async_e->ldom_name);
2867 
2868 	if (iosvc->fmd_xprt == NULL) {
2869 		iosvc->fmd_xprt = fmd_xprt_open(fmd_hdl, flags, NULL, iosvc);
2870 	}
2871 
2872 	iosvc->thr_is_dying = 0;
2873 	if (iosvc->recv_tid == NULL) {
2874 		iosvc->recv_tid = fmd_thr_create(fmd_hdl,
2875 		    etm_recv_from_remote_root, iosvc);
2876 	}
2877 	if (iosvc->send_tid == NULL) {
2878 		iosvc->send_tid = fmd_thr_create(fmd_hdl,
2879 		    etm_send_to_remote_root, iosvc);
2880 	}
2881 } /* etm_iosvc_setup() */
2882 
2883 
2884 /*
2885  * ds userland interface ds_reg_cb  callback func
2886  */
2887 
2888 /* ARGSUSED */
2889 static void
etm_iosvc_reg_handler(ds_hdl_t ds_hdl,ds_cb_arg_t arg,ds_ver_t * ver,ds_domain_hdl_t dhdl)2890 etm_iosvc_reg_handler(ds_hdl_t ds_hdl, ds_cb_arg_t arg, ds_ver_t *ver,
2891 	ds_domain_hdl_t dhdl)
2892 {
2893 	etm_async_event_ele_t	async_ele;
2894 
2895 
2896 	/*
2897 	 * do version check here.
2898 	 * checked the ver received here against etm_iosvc_vers here
2899 	 */
2900 	if (etm_iosvc_vers[0].major != ver->major ||
2901 	    etm_iosvc_vers[0].minor != ver->minor) {
2902 		/*
2903 		 * can't log an fmd debug msg,
2904 		 * not running in an fmd aux thread
2905 		 */
2906 		return;
2907 	}
2908 
2909 	/*
2910 	 * the callback should have a valid ldom_name
2911 	 * can't log fmd debugging msg here since this is not in an fmd aux
2912 	 * thread. log fmd debug msg in etm_async_event_handle()
2913 	 */
2914 	async_ele.ds_hdl = ds_hdl;
2915 	async_ele.dhdl = dhdl;
2916 	async_ele.ldom_name[0] = '\0';
2917 	async_ele.event_type = ETM_ASYNC_EVENT_DS_REG_CB;
2918 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2919 	(void) etm_async_q_enq(&async_ele);
2920 	if (etm_async_q_cur_len == 1)
2921 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2922 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2923 
2924 } /* etm_iosvc_reg_handler */
2925 
2926 
2927 /*
2928  * ds userland interface ds_unreg_cb  callback func
2929  */
2930 
2931 /*ARGSUSED*/
2932 static void
etm_iosvc_unreg_handler(ds_hdl_t hdl,ds_cb_arg_t arg)2933 etm_iosvc_unreg_handler(ds_hdl_t hdl, ds_cb_arg_t arg)
2934 {
2935 	etm_async_event_ele_t	async_ele;
2936 
2937 	/*
2938 	 * fill in async_ele and enqueue async_ele
2939 	 */
2940 	async_ele.ldom_name[0] = '\0';
2941 	async_ele.ds_hdl = hdl;
2942 	async_ele.event_type = ETM_ASYNC_EVENT_DS_UNREG_CB;
2943 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2944 	(void) etm_async_q_enq(&async_ele);
2945 	if (etm_async_q_cur_len == 1)
2946 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2947 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2948 } /* etm_iosvc_unreg_handler */
2949 
2950 /*
2951  * ldom event registration callback func
2952  */
2953 
2954 /* ARGSUSED */
2955 static void
ldom_event_handler(char * ldom_name,ldom_event_t event,ldom_cb_arg_t data)2956 ldom_event_handler(char *ldom_name, ldom_event_t event, ldom_cb_arg_t data)
2957 {
2958 	etm_async_event_ele_t	async_ele;
2959 
2960 	/*
2961 	 * the callback will have a valid ldom_name
2962 	 */
2963 	async_ele.ldom_name[0] = '\0';
2964 	if (ldom_name)
2965 		(void) strcpy(async_ele.ldom_name, ldom_name);
2966 	async_ele.ds_hdl = DS_INVALID_HDL;
2967 
2968 	/*
2969 	 * fill in async_ele and enq async_ele
2970 	 */
2971 	switch (event) {
2972 	case LDOM_EVENT_BIND:
2973 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_BIND;
2974 		break;
2975 	case LDOM_EVENT_UNBIND:
2976 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_UNBIND;
2977 		break;
2978 	case LDOM_EVENT_ADD:
2979 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_ADD;
2980 		break;
2981 	case LDOM_EVENT_REMOVE:
2982 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_REMOVE;
2983 		break;
2984 	default:
2985 		/*
2986 		 * for all other ldom events, do nothing
2987 		 */
2988 		return;
2989 	} /* switch (event) */
2990 
2991 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2992 	(void) etm_async_q_enq(&async_ele);
2993 	if (etm_async_q_cur_len == 1)
2994 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2995 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2996 
2997 } /* ldom_event_handler */
2998 
2999 
3000 /*
3001  * This is running as an fmd aux thread.
3002  * This is the func that actually handle the events, which include:
3003  * 1. ldom events. ldom events are  on Control Domain only
3004  * 2. any DS userland callback funcs
3005  * these events are already Q-ed in the async_event_ele_q
3006  * deQ and process the events accordingly
3007  */
3008 static void
etm_async_event_handler(void * arg)3009 etm_async_event_handler(void *arg)
3010 {
3011 
3012 	fmd_hdl_t		*fmd_hdl = (fmd_hdl_t *)arg;
3013 	etm_iosvc_t		*iosvc;		/* ptr 2 iosvc struct */
3014 	etm_async_event_ele_t	async_e;
3015 
3016 	fmd_hdl_debug(fmd_hdl, "info: etm_async_event_handler starting\n");
3017 	/*
3018 	 *  handle etm is not dying and Q len > 0
3019 	 */
3020 	while (!etm_is_dying) {
3021 		/*
3022 		 * grab the lock to check the Q len
3023 		 */
3024 		(void) pthread_mutex_lock(&etm_async_event_q_lock);
3025 		fmd_hdl_debug(fmd_hdl, "info: etm_async_q_cur_len %d\n",
3026 		    etm_async_q_cur_len);
3027 
3028 		while (etm_async_q_cur_len > 0) {
3029 			(void) etm_async_q_deq(&async_e);
3030 			(void) pthread_mutex_unlock(&etm_async_event_q_lock);
3031 			fmd_hdl_debug(fmd_hdl,
3032 			    "info: processing an async event type %d ds_hdl"
3033 			    " %d\n", async_e.event_type, async_e.ds_hdl);
3034 			if (async_e.ldom_name[0] != '\0') {
3035 				fmd_hdl_debug(fmd_hdl,
3036 				    "info: procssing async evt ldom_name %s\n",
3037 				    async_e.ldom_name);
3038 			}
3039 
3040 			/*
3041 			 * at this point, if async_e.ldom_name is not NULL,
3042 			 * we have a valid iosvc strcut ptr.
3043 			 * the only time async_e.ldom_name is NULL is  at
3044 			 * ds_unreg_cb()
3045 			 */
3046 			switch (async_e.event_type)  {
3047 			case ETM_ASYNC_EVENT_LDOM_UNBIND:
3048 			case ETM_ASYNC_EVENT_LDOM_REMOVE:
3049 				/*
3050 				 * we have a valid ldom_name,
3051 				 * etm_lookup_struct(ldom_name)
3052 				 * do nothing if can't find an iosvc
3053 				 * no iosvc clean up to do
3054 				 */
3055 				(void) pthread_mutex_lock(
3056 				    &iosvc_list_lock);
3057 				iosvc = etm_iosvc_lookup(fmd_hdl,
3058 				    async_e.ldom_name,
3059 				    async_e.ds_hdl, B_FALSE);
3060 				if (iosvc == NULL) {
3061 					fmd_hdl_debug(fmd_hdl,
3062 					    "error: can't find iosvc for ldom "
3063 					    "name %s\n",
3064 					    async_e.ldom_name);
3065 					(void) pthread_mutex_unlock(
3066 					    &iosvc_list_lock);
3067 					break;
3068 				}
3069 				/*
3070 				 * Clean up the queue, delete all messages and
3071 				 * do not persist checkpointed fma events.
3072 				 */
3073 				etm_iosvc_cleanup(fmd_hdl, iosvc, B_TRUE,
3074 				    B_TRUE);
3075 				(void) pthread_mutex_unlock(
3076 				    &iosvc_list_lock);
3077 				break;
3078 
3079 			case ETM_ASYNC_EVENT_LDOM_BIND:
3080 
3081 				/*
3082 				 * create iosvc if it has not been
3083 				 * created
3084 				 * async_e.ds_hdl is invalid
3085 				 * async_e.ldom_name is valid ldom_name
3086 				 */
3087 				(void) pthread_mutex_lock(
3088 				    &iosvc_list_lock);
3089 				iosvc = etm_iosvc_lookup(fmd_hdl,
3090 				    async_e.ldom_name,
3091 				    async_e.ds_hdl, B_TRUE);
3092 				if (iosvc == NULL) {
3093 					fmd_hdl_debug(fmd_hdl,
3094 					    "error: can't create iosvc for "
3095 					    "async evnt %d\n",
3096 					    async_e.event_type);
3097 					(void) pthread_mutex_unlock(
3098 					    &iosvc_list_lock);
3099 					break;
3100 				}
3101 				(void) strcpy(iosvc->ldom_name,
3102 				    async_e.ldom_name);
3103 				iosvc->ds_hdl = async_e.ds_hdl;
3104 				(void) pthread_mutex_unlock(
3105 				    &iosvc_list_lock);
3106 				break;
3107 
3108 			case ETM_ASYNC_EVENT_DS_REG_CB:
3109 				if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3110 					/*
3111 					 * find the root ldom name from
3112 					 * ldom domain hdl/id
3113 					 */
3114 					if (etm_filter_find_ldom_name(
3115 					    fmd_hdl, async_e.dhdl,
3116 					    async_e.ldom_name,
3117 					    MAX_LDOM_NAME) != 0) {
3118 						fmd_hdl_debug(fmd_hdl,
3119 						    "error: can't find root "
3120 						    "domain name from did %d\n",
3121 						    async_e.dhdl);
3122 						break;
3123 					} else {
3124 						fmd_hdl_debug(fmd_hdl,
3125 						    "info: etm_filter_find_"
3126 						    "ldom_name returned %s\n",
3127 						    async_e.ldom_name);
3128 					}
3129 					/*
3130 					 * now we should have a valid
3131 					 * root domain name.
3132 					 * lookup the iosvc struct
3133 					 * associated with the ldom_name
3134 					 * and init the iosvc struct
3135 					 */
3136 					(void) pthread_mutex_lock(
3137 					    &iosvc_list_lock);
3138 					iosvc = etm_iosvc_lookup(
3139 					    fmd_hdl, async_e.ldom_name,
3140 					    async_e.ds_hdl, B_TRUE);
3141 					if (iosvc == NULL) {
3142 						fmd_hdl_debug(fmd_hdl,
3143 						    "error: can't create iosvc "
3144 						    "for async evnt %d\n",
3145 						    async_e.event_type);
3146 						(void) pthread_mutex_unlock(
3147 						    &iosvc_list_lock);
3148 						break;
3149 					}
3150 
3151 					etm_iosvc_setup(fmd_hdl, iosvc,
3152 					    &async_e);
3153 					(void) pthread_mutex_unlock(
3154 					    &iosvc_list_lock);
3155 				} else {
3156 					iosvc = &io_svc;
3157 					(void) strcpy(iosvc->ldom_name,
3158 					    async_e.ldom_name);
3159 
3160 					etm_iosvc_setup(fmd_hdl, iosvc,
3161 					    &async_e);
3162 				}
3163 				break;
3164 
3165 			case ETM_ASYNC_EVENT_DS_UNREG_CB:
3166 				/*
3167 				 * decide which iosvc struct to perform
3168 				 * this UNREG callback on.
3169 				 */
3170 				if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3171 					(void) pthread_mutex_lock(
3172 					    &iosvc_list_lock);
3173 					/*
3174 					 * lookup the iosvc struct w/
3175 					 * ds_hdl
3176 					 */
3177 					iosvc = etm_iosvc_lookup(
3178 					    fmd_hdl, async_e.ldom_name,
3179 					    async_e.ds_hdl, B_FALSE);
3180 					if (iosvc == NULL) {
3181 						fmd_hdl_debug(fmd_hdl,
3182 						    "error: can't find iosvc "
3183 						    "for async evnt %d\n",
3184 						    async_e.event_type);
3185 					(void) pthread_mutex_unlock(
3186 					    &iosvc_list_lock);
3187 						break;
3188 					}
3189 
3190 					/*
3191 					 * ds_hdl and fmd_xprt_open
3192 					 * go hand to hand together
3193 					 * after unreg_cb,
3194 					 * ds_hdl is INVALID and
3195 					 * fmd_xprt is closed.
3196 					 * the ldom name and the msg Q
3197 					 * remains in iosvc_list
3198 					 */
3199 					if (iosvc->ldom_name != '\0')
3200 						fmd_hdl_debug(fmd_hdl,
3201 						    "info: iosvc  w/ ldom_name "
3202 						    "%s \n", iosvc->ldom_name);
3203 
3204 					/*
3205 					 * destroy send/recv threads and
3206 					 * other clean up on Control side.
3207 					 */
3208 					etm_iosvc_cleanup(fmd_hdl, iosvc,
3209 					    B_FALSE, B_FALSE);
3210 					(void) pthread_mutex_unlock(
3211 					    &iosvc_list_lock);
3212 				} else {
3213 					iosvc = &io_svc;
3214 					/*
3215 					 * destroy send/recv threads and
3216 					 * then clean up on Root side.
3217 					 */
3218 					etm_iosvc_cleanup(fmd_hdl, iosvc,
3219 					    B_FALSE, B_FALSE);
3220 				}
3221 				break;
3222 
3223 			default:
3224 				/*
3225 				 * for all other events, etm doesn't care.
3226 				 * already logged an fmd info msg w/
3227 				 * the event type. Do nothing here.
3228 				 */
3229 				break;
3230 			} /* switch (async_e.event_type) */
3231 
3232 			if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3233 				etm_filter_handle_ldom_event(fmd_hdl,
3234 				    async_e.event_type, async_e.ldom_name);
3235 			}
3236 
3237 			/*
3238 			 * grab the lock to check the q length again
3239 			 */
3240 			(void) pthread_mutex_lock(&etm_async_event_q_lock);
3241 
3242 			if (etm_is_dying) {
3243 				break;
3244 			}
3245 		}	/* etm_async_q_cur_len */
3246 
3247 		/*
3248 		 * we have the mutex lock at this point, whether
3249 		 * . etm_is_dying  and/or
3250 		 * . q_len == 0
3251 		 */
3252 		if (!etm_is_dying && etm_async_q_cur_len == 0) {
3253 			fmd_hdl_debug(fmd_hdl,
3254 			    "info: cond wait on async_event_q_cv\n");
3255 			(void) pthread_cond_wait(&etm_async_event_q_cv,
3256 			    &etm_async_event_q_lock);
3257 			fmd_hdl_debug(fmd_hdl,
3258 			    "info: cond wait on async_event_q_cv rtns\n");
3259 		}
3260 		(void) pthread_mutex_unlock(&etm_async_event_q_lock);
3261 	} /* etm_is_dying */
3262 
3263 	fmd_hdl_debug(fmd_hdl,
3264 	    "info: etm async event handler thread exiting\n");
3265 
3266 } /* etm_async_event_handler */
3267 
3268 /*
3269  * deQ what's in iosvc msg Q
3270  * send iosvc_msgp to the remote io svc ldom by calling ds_send_msg()
3271  * the iosvc_msgp already has the packed msg, which is hdr + 1 fma event
3272  */
3273 static void
etm_send_to_remote_root(void * arg)3274 etm_send_to_remote_root(void *arg)
3275 {
3276 
3277 	etm_iosvc_t		*iosvc = (etm_iosvc_t *)arg;	/* iosvc ptr */
3278 	etm_iosvc_q_ele_t	msg_ele;	/* iosvc msg ele */
3279 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* hdr for FMA_EVENT */
3280 	fmd_hdl_t		*fmd_hdl = init_hdl;	/* fmd handle */
3281 
3282 
3283 	fmd_hdl_debug(fmd_hdl,
3284 	    "info: send to remote iosvc starting w/ ldom_name %s\n",
3285 	    iosvc->ldom_name);
3286 
3287 	/*
3288 	 *  loop forever until etm_is_dying or thr_is_dying
3289 	 */
3290 	while (!etm_is_dying && !iosvc->thr_is_dying) {
3291 		if (iosvc->ds_hdl != DS_INVALID_HDL &&
3292 		    iosvc->start_sending_Q > 0) {
3293 			(void) pthread_mutex_lock(&iosvc->msg_q_lock);
3294 			while (iosvc->msg_q_cur_len > 0 &&
3295 			    iosvc->ds_hdl != DS_INVALID_HDL)  {
3296 				(void) etm_iosvc_msg_deq(fmd_hdl, iosvc,
3297 				    &msg_ele);
3298 				if (etm_debug_lvl >= 3) {
3299 					fmd_hdl_debug(fmd_hdl, "info: valid "
3300 					    "ds_hdl before ds_send_msg \n");
3301 				}
3302 				(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
3303 
3304 				iosvc->ack_ok = 0;
3305 				ev_hdrp = (etm_proto_v1_ev_hdr_t *)
3306 				    ((ptrdiff_t)msg_ele.msg);
3307 				ev_hdrp->ev_pp.pp_xid = iosvc->cur_send_xid + 1;
3308 				while (!iosvc->ack_ok &&
3309 				    iosvc->ds_hdl != DS_INVALID_HDL &&
3310 				    !etm_is_dying) {
3311 					/*
3312 					 * call ds_send_msg() to send the msg,
3313 					 * wait for the recv end to send the
3314 					 * resp msg back.
3315 					 * If resp msg is recv-ed, ack_ok
3316 					 * will be set to 1.
3317 					 * otherwise, retry.
3318 					 */
3319 					if (etm_send_ds_msg(fmd_hdl, B_TRUE,
3320 					    iosvc, &msg_ele, ev_hdrp) < 0) {
3321 						continue;
3322 					}
3323 
3324 					if (etm_is_dying || iosvc->thr_is_dying)
3325 						break;
3326 				}
3327 
3328 				/*
3329 				 * if out of the while loop but !ack_ok, ie,
3330 				 * ds_hdl becomes invalid at some point
3331 				 * while waiting the resp msg, we need to put
3332 				 * the msg back to the head of the Q.
3333 				 */
3334 				if (!iosvc->ack_ok) {
3335 					(void) pthread_mutex_lock(
3336 					    &iosvc->msg_q_lock);
3337 					/*
3338 					 * put the msg back to the head of Q.
3339 					 * If the Q is full at this point,
3340 					 * drop the msg at the tail, enq this
3341 					 * msg to the head.
3342 					 */
3343 					etm_msg_enq_head(fmd_hdl, iosvc,
3344 					    &msg_ele);
3345 					(void) pthread_mutex_unlock(
3346 					    &iosvc->msg_q_lock);
3347 				}
3348 
3349 				/*
3350 				 *
3351 				 * grab the lock to check the Q len again
3352 				 */
3353 				(void) pthread_mutex_lock(&iosvc->msg_q_lock);
3354 				if (etm_is_dying || iosvc->thr_is_dying) {
3355 					break;
3356 				}
3357 			} /* while dequeing iosvc msgs to send */
3358 
3359 			/*
3360 			 * we have the mutex lock for msg_q_lock at this point
3361 			 * we are here because
3362 			 * 1) q_len == 0: then wait on the cv for Q to be filled
3363 			 * 2) etm_is_dying
3364 			 */
3365 			if (!etm_is_dying && !iosvc->thr_is_dying &&
3366 			    iosvc->msg_q_cur_len == 0) {
3367 				fmd_hdl_debug(fmd_hdl,
3368 				    "info: waiting on msg_q_cv\n");
3369 				(void) pthread_cond_wait(&iosvc->msg_q_cv,
3370 				    &iosvc->msg_q_lock);
3371 			}
3372 			(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
3373 			if (etm_is_dying || iosvc->thr_is_dying)  {
3374 				break;
3375 			}
3376 		} else {
3377 			(void) etm_sleep(1);
3378 		} /* wait for the start_sendingQ > 0 */
3379 	} /* etm_is_dying or thr_is_dying */
3380 	fmd_hdl_debug(fmd_hdl, "info; etm send thread exiting \n");
3381 } /* etm_send_to_remote_root */
3382 
3383 
3384 /*
3385  * receive etm msgs from the remote root ldom by calling ds_recv_msg()
3386  * if FMA events/ereports, call fmd_xprt_post() to post to fmd
3387  * send ACK back by calling ds_send_msg()
3388  */
3389 static void
etm_recv_from_remote_root(void * arg)3390 etm_recv_from_remote_root(void *arg)
3391 {
3392 	etm_iosvc_t		*iosvc = (etm_iosvc_t *)arg;	/* iosvc ptr */
3393 	etm_proto_v1_pp_t	*pp;		/* protocol preamble */
3394 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
3395 	etm_proto_v1_resp_hdr_t	*resp_hdrp;	/* for RESPONSE msg */
3396 	int32_t			resp_code = 0;	/* default is success */
3397 	int32_t			rc;		/* return value */
3398 	size_t			maxlen = MAXLEN;
3399 						/* max msg len */
3400 	char 			msgbuf[MAXLEN];	/* recv msg buf */
3401 	size_t			msg_size;	/* recv msg size */
3402 	size_t			hdr_sz;		/* sizeof *hdrp */
3403 	size_t			evsz;		/* sizeof *evp */
3404 	size_t			fma_event_size;	/* sizeof FMA event  */
3405 	nvlist_t 		*evp;		/* ptr to the nvlist */
3406 	char			*buf;		/* ptr to the nvlist */
3407 	static uint32_t		mem_alloc = 0;	/* indicate if alloc mem */
3408 	char 			*msg;		/* ptr to alloc mem */
3409 	fmd_hdl_t		*fmd_hdl = init_hdl;
3410 
3411 
3412 
3413 	fmd_hdl_debug(fmd_hdl,
3414 	    "info: recv from remote iosvc starting with ldom name %s \n",
3415 	    iosvc->ldom_name);
3416 
3417 	/*
3418 	 * loop forever until etm_is_dying or the thread is dying
3419 	 */
3420 
3421 	msg = msgbuf;
3422 	while (!etm_is_dying && !iosvc->thr_is_dying) {
3423 		if (iosvc->ds_hdl == DS_INVALID_HDL) {
3424 			fmd_hdl_debug(fmd_hdl,
3425 			    "info: ds_hdl is invalid in recv thr\n");
3426 			(void) etm_sleep(1);
3427 			continue;
3428 		}
3429 
3430 		/*
3431 		 * for now, there are FMA_EVENT and ACK msg type.
3432 		 * use FMA_EVENT buf as the maxlen, hdr+1 fma event.
3433 		 * FMA_EVENT is big enough to hold an ACK msg.
3434 		 * the actual msg size received is in msg_size.
3435 		 */
3436 		rc = (*etm_ds_recv_msg)(iosvc->ds_hdl, msg, maxlen, &msg_size);
3437 		if (rc == EFBIG) {
3438 			fmd_hdl_debug(fmd_hdl,
3439 			    "info: ds_recv_msg needs mem the size of %d\n",
3440 			    msg_size);
3441 			msg = fmd_hdl_zalloc(fmd_hdl, msg_size, FMD_SLEEP);
3442 			mem_alloc = 1;
3443 		} else if (rc == 0) {
3444 			fmd_hdl_debug(fmd_hdl,
3445 			    "info: ds_recv_msg received a msg ok\n");
3446 			/*
3447 			 * check the magic # in  msg.hdr
3448 			 */
3449 			pp = (etm_proto_v1_pp_t *)((ptrdiff_t)msg);
3450 			if (pp->pp_magic_num != ETM_PROTO_MAGIC_NUM) {
3451 				fmd_hdl_debug(fmd_hdl,
3452 				    "info: bad ds recv on magic\n");
3453 				continue;
3454 			}
3455 
3456 			/*
3457 			 * check the msg type against msg_size to be sure
3458 			 * that received msg is not a truncated msg
3459 			 */
3460 			if (pp->pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
3461 
3462 				ev_hdrp = (etm_proto_v1_ev_hdr_t *)
3463 				    ((ptrdiff_t)msg);
3464 				fmd_hdl_debug(fmd_hdl, "info: ds received "
3465 				    "FMA EVENT xid=%d msg_size=%d\n",
3466 				    ev_hdrp->ev_pp.pp_xid, msg_size);
3467 				hdr_sz = sizeof (*ev_hdrp) +
3468 				    1*(sizeof (ev_hdrp->ev_lens[0]));
3469 				fma_event_size = hdr_sz + ev_hdrp->ev_lens[0];
3470 				if (fma_event_size != msg_size) {
3471 					fmd_hdl_debug(fmd_hdl, "info: wrong "
3472 					    "ev msg size received\n");
3473 					continue;
3474 					/*
3475 					 * Simply  do nothing. The send side
3476 					 * will timedcond_wait waiting on the
3477 					 * resp msg will timeout and
3478 					 * re-send the same msg.
3479 					 */
3480 				}
3481 				if (etm_debug_lvl >= 3) {
3482 					fmd_hdl_debug(fmd_hdl,  "info: recv msg"
3483 					    " size %d hdrsz %d evp size %d\n",
3484 					    msg_size, hdr_sz,
3485 					    ev_hdrp->ev_lens[0]);
3486 				}
3487 
3488 				if (ev_hdrp->ev_pp.pp_xid !=
3489 				    iosvc->xid_posted_ev) {
3490 					/*
3491 					 * different from last xid posted to
3492 					 * fmd, post to fmd now.
3493 					 */
3494 					buf = msg + hdr_sz;
3495 					rc = nvlist_unpack(buf,
3496 					    ev_hdrp->ev_lens[0], &evp, 0);
3497 					rc = nvlist_size(evp, &evsz,
3498 					    NV_ENCODE_XDR);
3499 					fmd_hdl_debug(fmd_hdl,
3500 					    "info: evp size %d before fmd"
3501 					    "post\n", evsz);
3502 
3503 					if ((rc = etm_post_to_fmd(fmd_hdl,
3504 					    iosvc->fmd_xprt, evp)) >= 0) {
3505 						fmd_hdl_debug(fmd_hdl,
3506 						    "info: xid posted to fmd %d"
3507 						    "\n",
3508 						    ev_hdrp->ev_pp.pp_xid);
3509 						iosvc->xid_posted_ev =
3510 						    ev_hdrp->ev_pp.pp_xid;
3511 					}
3512 				}
3513 
3514 				/*
3515 				 * ready to  send the RESPONSE msg back
3516 				 * reuse the msg buffer as the response buffer
3517 				 */
3518 				resp_hdrp = (etm_proto_v1_resp_hdr_t *)
3519 				    ((ptrdiff_t)msg);
3520 				resp_hdrp->resp_pp.pp_msg_type =
3521 				    ETM_MSG_TYPE_RESPONSE;
3522 
3523 				resp_hdrp->resp_code = resp_code;
3524 				resp_hdrp->resp_len = sizeof (*resp_hdrp);
3525 
3526 				/*
3527 				 * send the whole response msg in one send
3528 				 */
3529 				if ((*etm_ds_send_msg)(iosvc->ds_hdl, msg,
3530 				    sizeof (*resp_hdrp)) != 0) {
3531 					fmd_hdl_debug(fmd_hdl,
3532 					    "info: send response msg failed\n");
3533 				} else {
3534 					fmd_hdl_debug(fmd_hdl,
3535 					    "info: ds send resp msg ok"
3536 					    "size %d\n", sizeof (*resp_hdrp));
3537 				}
3538 			} else if (pp->pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
3539 				fmd_hdl_debug(fmd_hdl,
3540 				    "info: ds received respond msg xid=%d"
3541 				    "msg_size=%d for ldom %s\n", pp->pp_xid,
3542 				    msg_size, iosvc->ldom_name);
3543 				if (sizeof (*resp_hdrp) != msg_size) {
3544 					fmd_hdl_debug(fmd_hdl,
3545 					    "info: wrong resp msg size"
3546 					    "received\n");
3547 					fmd_hdl_debug(fmd_hdl,
3548 					    "info: resp msg size %d recv resp"
3549 					    "msg size %d\n",
3550 					    sizeof (*resp_hdrp), msg_size);
3551 					continue;
3552 				}
3553 				/*
3554 				 * is the pp.pp_xid == iosvc->cur_send_xid+1,
3555 				 * if so, nudge the send routine to send next
3556 				 */
3557 				if (pp->pp_xid != iosvc->cur_send_xid+1) {
3558 					fmd_hdl_debug(fmd_hdl,
3559 					    "info: ds received resp msg xid=%d "
3560 					    "doesn't match cur_send_id=%d\n",
3561 					    pp->pp_xid, iosvc->cur_send_xid+1);
3562 					continue;
3563 				}
3564 				(void) pthread_mutex_lock(&iosvc->msg_ack_lock);
3565 				iosvc->ack_ok = 1;
3566 				(void) pthread_cond_signal(&iosvc->msg_ack_cv);
3567 				(void) pthread_mutex_unlock(
3568 				    &iosvc->msg_ack_lock);
3569 				fmd_hdl_debug(fmd_hdl,
3570 				    "info: signaling msg_ack_cv\n");
3571 			} else {
3572 				/*
3573 				 * place holder for future msg types
3574 				 */
3575 				fmd_hdl_debug(fmd_hdl,
3576 				    "info: ds received unrecognized msg\n");
3577 			}
3578 			if (mem_alloc) {
3579 				fmd_hdl_free(fmd_hdl, msg, msg_size);
3580 				mem_alloc = 0;
3581 				msg = msgbuf;
3582 			}
3583 		} else {
3584 			if (etm_debug_lvl >= 3) {
3585 				fmd_hdl_debug(fmd_hdl,
3586 				    "info: ds_recv_msg() failed\n");
3587 			}
3588 		} /* ds_recv_msg() returns */
3589 	} /* etm_is_dying */
3590 
3591 	/*
3592 	 * need to free the mem allocated in msg upon exiting the thread
3593 	 */
3594 	if (mem_alloc) {
3595 		fmd_hdl_free(fmd_hdl, msg, msg_size);
3596 		mem_alloc = 0;
3597 		msg = msgbuf;
3598 	}
3599 	fmd_hdl_debug(fmd_hdl, "info; etm recv thread exiting \n");
3600 } /* etm_recv_from_remote_root */
3601 
3602 
3603 
3604 /*
3605  * etm_ds_init
3606  *		initialize DS services function pointers by calling
3607  *		dlopen() followed by  dlsym() for each ds func.
3608  *		if any dlopen() or dlsym() call fails, return -ENOENT
3609  *		return >0 for successs, -ENOENT for failure
3610  */
3611 static int
etm_ds_init(fmd_hdl_t * hdl)3612 etm_ds_init(fmd_hdl_t *hdl)
3613 {
3614 	int rc = 0;
3615 
3616 	if ((etm_dl_hdl = dlopen(etm_dl_path, etm_dl_mode)) == NULL) {
3617 		fmd_hdl_debug(hdl, "error: failed to dlopen %s\n", etm_dl_path);
3618 		return (-ENOENT);
3619 	}
3620 
3621 	etm_ds_svc_reg = (int (*)(ds_capability_t *cap, ds_ops_t *ops))
3622 	    dlsym(etm_dl_hdl, "ds_svc_reg");
3623 	if (etm_ds_svc_reg == NULL) {
3624 		fmd_hdl_debug(hdl,
3625 		    "error: failed to dlsym ds_svc_reg() w/ error %s\n",
3626 		    dlerror());
3627 		rc = -ENOENT;
3628 	}
3629 
3630 
3631 	etm_ds_clnt_reg = (int (*)(ds_capability_t *cap, ds_ops_t *ops))
3632 	    dlsym(etm_dl_hdl, "ds_clnt_reg");
3633 	if (etm_ds_clnt_reg == NULL) {
3634 		fmd_hdl_debug(hdl,
3635 		    "error: dlsym(ds_clnt_reg) failed w/ errno %d\n", errno);
3636 		rc = -ENOENT;
3637 	}
3638 
3639 	etm_ds_send_msg = (int (*)(ds_hdl_t hdl, void *buf, size_t buflen))
3640 	    dlsym(etm_dl_hdl, "ds_send_msg");
3641 	if (etm_ds_send_msg == NULL) {
3642 		fmd_hdl_debug(hdl, "error: dlsym(ds_send_msg) failed\n");
3643 		rc = -ENOENT;
3644 	}
3645 
3646 	etm_ds_recv_msg = (int (*)(ds_hdl_t hdl, void *buf, size_t buflen,
3647 	    size_t *msglen))dlsym(etm_dl_hdl, "ds_recv_msg");
3648 	if (etm_ds_recv_msg == NULL) {
3649 		fmd_hdl_debug(hdl, "error: dlsym(ds_recv_msg) failed\n");
3650 		rc = -ENOENT;
3651 	}
3652 
3653 	etm_ds_fini = (int (*)(void))dlsym(etm_dl_hdl, "ds_fini");
3654 	if (etm_ds_fini == NULL) {
3655 		fmd_hdl_debug(hdl, "error: dlsym(ds_fini) failed\n");
3656 		rc = -ENOENT;
3657 	}
3658 
3659 	if (rc == -ENOENT) {
3660 		(void) dlclose(etm_dl_hdl);
3661 	}
3662 	return (rc);
3663 
3664 } /* etm_ds_init() */
3665 
3666 
3667 /*
3668  * -------------------------- FMD entry points -------------------------------
3669  */
3670 
3671 /*
3672  * _fmd_init - initialize the transport for use by ETM and start the
3673  *		server daemon to accept new connections to us
3674  *
3675  *		FMD will read our *.conf and subscribe us to FMA events
3676  */
3677 
3678 void
_fmd_init(fmd_hdl_t * hdl)3679 _fmd_init(fmd_hdl_t *hdl)
3680 {
3681 	struct timeval		tmv;		/* timeval */
3682 	ssize_t			n;		/* gen use */
3683 	const struct facility	*fp;		/* syslog facility matching */
3684 	char			*facname;	/* syslog facility property */
3685 	uint32_t		type_mask;	/* type of the local host */
3686 	int			rc;		/* funcs return code */
3687 
3688 
3689 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
3690 		return; /* invalid data in configuration file */
3691 	}
3692 
3693 	fmd_hdl_debug(hdl, "info: module initializing\n");
3694 
3695 	init_hdl = hdl;
3696 	etm_lhp = ldom_init(etm_init_alloc, etm_init_free);
3697 
3698 	/*
3699 	 * decide the ldom type, do initialization accordingly
3700 	 */
3701 	if ((rc = ldom_get_type(etm_lhp, &type_mask)) != 0) {
3702 		fmd_hdl_debug(hdl, "error: can't decide ldom type\n");
3703 		fmd_hdl_debug(hdl, "info: module unregistering\n");
3704 		ldom_fini(etm_lhp);
3705 		fmd_hdl_unregister(hdl);
3706 		return;
3707 	}
3708 
3709 	if ((type_mask & LDOM_TYPE_LEGACY) || (type_mask & LDOM_TYPE_CONTROL)) {
3710 		if (type_mask & LDOM_TYPE_LEGACY) {
3711 			/*
3712 			 * running on a legacy sun4v domain,
3713 			 * act as the the old sun4v
3714 			 */
3715 			etm_ldom_type = LDOM_TYPE_LEGACY;
3716 			fmd_hdl_debug(hdl, "info: running as the old sun4v\n");
3717 			ldom_fini(etm_lhp);
3718 		} else if (type_mask & LDOM_TYPE_CONTROL) {
3719 			etm_ldom_type = LDOM_TYPE_CONTROL;
3720 			fmd_hdl_debug(hdl, "info: running as control domain\n");
3721 
3722 			/*
3723 			 * looking for libds.so.1.
3724 			 * If not found, don't do DS registration. As a result,
3725 			 * there will be no DS callbacks or other DS services.
3726 			 */
3727 			if (etm_ds_init(hdl) >= 0) {
3728 				etm_filter_init(hdl);
3729 				etm_ckpt_init(hdl);
3730 
3731 				flags = FMD_XPRT_RDWR | FMD_XPRT_ACCEPT;
3732 
3733 				/*
3734 				 * ds client registration
3735 				 */
3736 				if ((rc = (*etm_ds_clnt_reg)(&iosvc_caps,
3737 				    &iosvc_ops))) {
3738 					fmd_hdl_debug(hdl,
3739 					"error: ds_clnt_reg(): errno %d\n", rc);
3740 				}
3741 			} else {
3742 				fmd_hdl_debug(hdl, "error: dlopen() libds "
3743 				    "failed, continue without the DS services");
3744 			}
3745 
3746 			/*
3747 			 * register for ldom status events
3748 			 */
3749 			if ((rc = ldom_register_event(etm_lhp,
3750 			    ldom_event_handler, hdl))) {
3751 				fmd_hdl_debug(hdl,
3752 				    "error: ldom_register_event():"
3753 				    " errno %d\n", rc);
3754 			}
3755 
3756 			/*
3757 			 * create the thread for handling both the ldom status
3758 			 * change and service events
3759 			 */
3760 			etm_async_e_tid = fmd_thr_create(hdl,
3761 			    etm_async_event_handler, hdl);
3762 		}
3763 
3764 		/* setup statistics and properties from FMD */
3765 
3766 		(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC,
3767 		    sizeof (etm_stats) / sizeof (fmd_stat_t),
3768 		    (fmd_stat_t *)&etm_stats);
3769 
3770 		etm_fma_resp_wait_time = fmd_prop_get_int32(hdl,
3771 		    ETM_PROP_NM_FMA_RESP_WAIT_TIME);
3772 		etm_debug_lvl = fmd_prop_get_int32(hdl, ETM_PROP_NM_DEBUG_LVL);
3773 		etm_debug_max_ev_cnt = fmd_prop_get_int32(hdl,
3774 		    ETM_PROP_NM_DEBUG_MAX_EV_CNT);
3775 		fmd_hdl_debug(hdl, "info: etm_debug_lvl %d "
3776 		    "etm_debug_max_ev_cnt %d\n", etm_debug_lvl,
3777 		    etm_debug_max_ev_cnt);
3778 
3779 		etm_resp_q_max_len = fmd_prop_get_int32(hdl,
3780 		    ETM_PROP_NM_MAX_RESP_Q_LEN);
3781 		etm_stats.etm_resp_q_max_len.fmds_value.ui64 =
3782 		    etm_resp_q_max_len;
3783 		etm_bad_acc_to_sec = fmd_prop_get_int32(hdl,
3784 		    ETM_PROP_NM_BAD_ACC_TO_SEC);
3785 
3786 		/*
3787 		 * obtain an FMD transport handle so we can post
3788 		 * FMA events later
3789 		 */
3790 
3791 		etm_fmd_xprt = fmd_xprt_open(hdl, FMD_XPRT_RDONLY, NULL, NULL);
3792 
3793 		/*
3794 		 * encourage protocol transaction id to be unique per module
3795 		 * load
3796 		 */
3797 
3798 		(void) gettimeofday(&tmv, NULL);
3799 		etm_xid_cur = (uint32_t)((tmv.tv_sec << 10) |
3800 		    ((unsigned long)tmv.tv_usec >> 10));
3801 
3802 		/* init the ETM transport */
3803 
3804 		if ((n = etm_xport_init(hdl)) != 0) {
3805 			fmd_hdl_error(hdl, "error: bad xport init errno %d\n",
3806 			    (-n));
3807 			fmd_hdl_unregister(hdl);
3808 			return;
3809 		}
3810 
3811 		/*
3812 		 * Cache any properties we use every time we receive an alert.
3813 		 */
3814 		syslog_file = fmd_prop_get_int32(hdl, ETM_PROP_NM_SYSLOGD);
3815 		syslog_cons = fmd_prop_get_int32(hdl, ETM_PROP_NM_CONSOLE);
3816 
3817 		if (syslog_file && (syslog_logfd = open("/dev/conslog",
3818 		    O_WRONLY | O_NOCTTY)) == -1) {
3819 			fmd_hdl_error(hdl,
3820 			    "error: failed to open /dev/conslog");
3821 			syslog_file = 0;
3822 		}
3823 
3824 		if (syslog_cons && (syslog_msgfd = open("/dev/sysmsg",
3825 		    O_WRONLY | O_NOCTTY)) == -1) {
3826 			fmd_hdl_error(hdl, "error: failed to open /dev/sysmsg");
3827 			syslog_cons = 0;
3828 		}
3829 
3830 		if (syslog_file) {
3831 			/*
3832 			 * Look up the value of the "facility" property and
3833 			 * use it to determine * what syslog LOG_* facility
3834 			 * value we use to fill in our log_ctl_t.
3835 			 */
3836 			facname = fmd_prop_get_string(hdl,
3837 			    ETM_PROP_NM_FACILITY);
3838 
3839 			for (fp = syslog_facs; fp->fac_name != NULL; fp++) {
3840 				if (strcmp(fp->fac_name, facname) == 0)
3841 					break;
3842 			}
3843 
3844 			if (fp->fac_name == NULL) {
3845 				fmd_hdl_error(hdl, "error: invalid 'facility'"
3846 				    " setting: %s\n", facname);
3847 				syslog_file = 0;
3848 			} else {
3849 				syslog_facility = fp->fac_value;
3850 				syslog_ctl.flags = SL_CONSOLE | SL_LOGONLY;
3851 			}
3852 
3853 			fmd_prop_free_string(hdl, facname);
3854 		}
3855 
3856 		/*
3857 		 * start the message responder and the connection acceptance
3858 		 * server; request protocol version be negotiated after waiting
3859 		 * a second for the receiver to be ready to start handshaking
3860 		 */
3861 
3862 		etm_resp_tid = fmd_thr_create(hdl, etm_responder, hdl);
3863 		etm_svr_tid = fmd_thr_create(hdl, etm_server, hdl);
3864 
3865 		(void) etm_sleep(ETM_SLEEP_QUIK);
3866 		etm_req_ver_negot(hdl);
3867 
3868 	} else if (type_mask & LDOM_TYPE_ROOT) {
3869 		etm_ldom_type = LDOM_TYPE_ROOT;
3870 		fmd_hdl_debug(hdl, "info: running as root domain\n");
3871 
3872 		/*
3873 		 * looking for libds.so.1.
3874 		 * If not found, don't do DS registration. As a result,
3875 		 * there will be no DS callbacks or other DS services.
3876 		 */
3877 		if (etm_ds_init(hdl) < 0) {
3878 			fmd_hdl_debug(hdl,
3879 			    "error: dlopen() libds failed, "
3880 			    "module unregistering\n");
3881 			ldom_fini(etm_lhp);
3882 			fmd_hdl_unregister(hdl);
3883 			return;
3884 		}
3885 
3886 		/*
3887 		 * DS service registration
3888 		 */
3889 		if ((rc = (*etm_ds_svc_reg)(&iosvc_caps, &iosvc_ops))) {
3890 			fmd_hdl_debug(hdl, "error: ds_svc_reg(): errno %d\n",
3891 			    rc);
3892 		}
3893 
3894 		/*
3895 		 * this thread is created for ds_reg_cb/ds_unreg_cb
3896 		 */
3897 		etm_async_e_tid = fmd_thr_create(hdl,
3898 		    etm_async_event_handler, hdl);
3899 
3900 		flags = FMD_XPRT_RDWR;
3901 	} else if ((type_mask & LDOM_TYPE_IO) || (type_mask == 0)) {
3902 		/*
3903 		 * Do not load this module if it is
3904 		 * . runing on a non-root ldom
3905 		 * . the domain owns no io devices
3906 		 */
3907 		fmd_hdl_debug(hdl,
3908 		    "info: non-root ldom, module unregistering\n");
3909 		ldom_fini(etm_lhp);
3910 		fmd_hdl_unregister(hdl);
3911 		return;
3912 	} else {
3913 		/*
3914 		 * place holder, all other cases. unload etm for now
3915 		 */
3916 		fmd_hdl_debug(hdl,
3917 		    "info: other ldom type, module unregistering\n");
3918 		ldom_fini(etm_lhp);
3919 		fmd_hdl_unregister(hdl);
3920 		return;
3921 	}
3922 
3923 	fmd_hdl_debug(hdl, "info: module initialized ok\n");
3924 
3925 } /* _fmd_init() */
3926 
3927 /*
3928  * etm_recv - receive an FMA event from FMD and transport it
3929  *		to the remote endpoint
3930  */
3931 
3932 /*ARGSUSED*/
3933 void
etm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * evp,const char * class)3934 etm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *evp, const char *class)
3935 {
3936 	etm_xport_addr_t	*addrv;	/* vector of transport addresses */
3937 	etm_xport_conn_t	conn;	/* connection handle */
3938 	etm_proto_v1_ev_hdr_t	*hdrp;	/* for FMA_EVENT msg */
3939 	ssize_t			i, n;	/* gen use */
3940 	size_t			sz;	/* header size */
3941 	size_t			buflen;	/* size of packed FMA event */
3942 	uint8_t			*buf;	/* tmp buffer for packed FMA event */
3943 
3944 	/*
3945 	 * if this is running on a Root Domain, ignore the events,
3946 	 * return right away
3947 	 */
3948 	if (etm_ldom_type == LDOM_TYPE_ROOT)
3949 		return;
3950 
3951 	buflen = 0;
3952 	if ((n = nvlist_size(evp, &buflen, NV_ENCODE_XDR)) != 0) {
3953 		fmd_hdl_error(hdl, "error: FMA event dropped: "
3954 		    "event size errno %d class %s\n", n, class);
3955 		etm_stats.etm_os_nvlist_size_fail.fmds_value.ui64++;
3956 		etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
3957 		return;
3958 	}
3959 
3960 	fmd_hdl_debug(hdl, "info: rcvd event %p from FMD\n", evp);
3961 	fmd_hdl_debug(hdl, "info: cnt %llu class %s\n",
3962 	    etm_stats.etm_rd_fmd_fmaevent.fmds_value.ui64, class);
3963 
3964 	etm_stats.etm_rd_fmd_bytes.fmds_value.ui64 += buflen;
3965 	etm_stats.etm_rd_fmd_fmaevent.fmds_value.ui64++;
3966 
3967 	/*
3968 	 * if the debug limit has been set, avoid excessive traffic,
3969 	 * for example, an infinite cycle using loopback nodes
3970 	 */
3971 
3972 	if ((etm_debug_max_ev_cnt >= 0) &&
3973 	    (etm_stats.etm_rd_fmd_fmaevent.fmds_value.ui64 >
3974 	    etm_debug_max_ev_cnt)) {
3975 		fmd_hdl_debug(hdl, "warning: FMA event dropped: "
3976 		    "event %p cnt %llu > debug max %d\n", evp,
3977 		    etm_stats.etm_rd_fmd_fmaevent.fmds_value.ui64,
3978 		    etm_debug_max_ev_cnt);
3979 		etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
3980 		return;
3981 	}
3982 
3983 	/* allocate a buffer for the FMA event and nvlist pack it */
3984 
3985 	buf = fmd_hdl_zalloc(hdl, buflen, FMD_SLEEP);
3986 
3987 	/*
3988 	 * increment the ttl value if the event is from remote (a root domain)
3989 	 * uncomment this when enabling fault forwarding from Root domains
3990 	 * to Control domain.
3991 	 *
3992 	 * uint8_t			ttl;
3993 	 * if (fmd_event_local(hdl, evp) != FMD_EVF_LOCAL) {
3994 	 *	if (nvlist_lookup_uint8(evp, FMD_EVN_TTL, &ttl) == 0) {
3995 	 *		(void) nvlist_remove(evp, FMD_EVN_TTL, DATA_TYPE_UINT8);
3996 	 *		(void) nvlist_add_uint8(evp, FMD_EVN_TTL, ttl + 1);
3997 	 *	}
3998 	 * }
3999 	 */
4000 
4001 	if ((n = nvlist_pack(evp, (char **)&buf, &buflen,
4002 	    NV_ENCODE_XDR, 0)) != 0) {
4003 		fmd_hdl_error(hdl, "error: FMA event dropped: "
4004 		    "event pack errno %d class %s\n", n, class);
4005 		etm_stats.etm_os_nvlist_pack_fail.fmds_value.ui64++;
4006 		etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
4007 		fmd_hdl_free(hdl, buf, buflen);
4008 		return;
4009 	}
4010 
4011 	/* get vector of dst addrs and send the FMA event to each one */
4012 
4013 	if ((addrv = etm_xport_get_ev_addrv(hdl, evp)) == NULL) {
4014 		fmd_hdl_error(hdl, "error: FMA event dropped: "
4015 		    "bad event dst addrs errno %d\n", errno);
4016 		etm_stats.etm_xport_get_ev_addrv_fail.fmds_value.ui64++;
4017 		etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
4018 		fmd_hdl_free(hdl, buf, buflen);
4019 		return;
4020 	}
4021 
4022 	for (i = 0; addrv[i] != NULL; i++) {
4023 
4024 		/* open a new connection to this dst addr */
4025 
4026 		if ((n = etm_conn_open(hdl, "FMA event dropped: "
4027 		    "bad conn open on new ev", addrv[i], &conn)) < 0) {
4028 			etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
4029 			continue;
4030 		}
4031 
4032 		(void) pthread_mutex_lock(&etm_write_lock);
4033 
4034 		/* write the ETM message header */
4035 
4036 		if ((hdrp = etm_hdr_write(hdl, conn, evp, NV_ENCODE_XDR,
4037 		    &sz)) == NULL) {
4038 			(void) pthread_mutex_unlock(&etm_write_lock);
4039 			fmd_hdl_error(hdl, "error: FMA event dropped: "
4040 			    "bad hdr write errno %d\n", errno);
4041 			(void) etm_conn_close(hdl,
4042 			    "bad conn close per bad hdr wr", conn);
4043 			etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
4044 			continue;
4045 		}
4046 
4047 		fmd_hdl_free(hdl, hdrp, sz);	/* header not needed */
4048 		etm_stats.etm_wr_hdr_fmaevent.fmds_value.ui64++;
4049 		fmd_hdl_debug(hdl, "info: hdr xport write ok for event %p\n",
4050 		    evp);
4051 
4052 		/* write the ETM message body, ie, the packed nvlist */
4053 
4054 		if ((n = etm_io_op(hdl, "FMA event dropped: "
4055 		    "bad io write on event", conn,
4056 		    buf, buflen, ETM_IO_OP_WR)) < 0) {
4057 			(void) pthread_mutex_unlock(&etm_write_lock);
4058 			(void) etm_conn_close(hdl,
4059 			    "bad conn close per bad body wr", conn);
4060 			etm_stats.etm_wr_drop_fmaevent.fmds_value.ui64++;
4061 			continue;
4062 		}
4063 
4064 		(void) pthread_mutex_unlock(&etm_write_lock);
4065 
4066 		etm_stats.etm_wr_body_fmaevent.fmds_value.ui64++;
4067 		etm_stats.etm_wr_xport_bytes.fmds_value.ui64 += buflen;
4068 		fmd_hdl_debug(hdl, "info: body xport write ok for event %p\n",
4069 		    evp);
4070 
4071 		/* close the connection */
4072 
4073 		(void) etm_conn_close(hdl, "bad conn close after event send",
4074 		    conn);
4075 	} /* foreach dst addr in the vector */
4076 
4077 	etm_xport_free_addrv(hdl, addrv);
4078 	fmd_hdl_free(hdl, buf, buflen);
4079 
4080 } /* etm_recv() */
4081 
4082 
4083 /*
4084  * etm_send -	receive an FMA event from FMD and enQ it in the iosvc.Q.
4085  *		etm_send_to_remote_root() deQ and xprt the FMA events to a
4086  *		remote root domain
4087  *		return FMD_SEND_SUCCESS for success,
4088  *		       FMD_SEND_FAILED for error
4089  */
4090 
4091 /*ARGSUSED*/
4092 int
etm_send(fmd_hdl_t * fmd_hdl,fmd_xprt_t * xp,fmd_event_t * ep,nvlist_t * nvl)4093 etm_send(fmd_hdl_t *fmd_hdl, fmd_xprt_t *xp, fmd_event_t *ep, nvlist_t *nvl)
4094 {
4095 	uint32_t	pack_it;	/* whether to pack/enq the event */
4096 	etm_pack_msg_type_t	msg_type;
4097 					/* tell etm_pack_ds_msg() what to do */
4098 	etm_iosvc_t	*iosvc;		/* ptr to cur iosvc struct */
4099 	char 		*class;		/* nvlist class name */
4100 
4101 	pack_it = 1;
4102 	msg_type = FMD_XPRT_OTHER_MSG;
4103 
4104 	(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
4105 	if (class == NULL) {
4106 		pack_it = 0;
4107 	} else  {
4108 		if (etm_debug_lvl >= 1) {
4109 			fmd_hdl_debug(fmd_hdl,
4110 			    "info: evp class= %s in etm_send\n", class);
4111 		}
4112 
4113 		if (etm_ldom_type ==  LDOM_TYPE_CONTROL) {
4114 			iosvc =
4115 			    (etm_iosvc_t *)fmd_xprt_getspecific(fmd_hdl, xp);
4116 
4117 			/*
4118 			 * check the flag FORWARDING_FAULTS_TO_CONTROL to
4119 			 * decide if or not to drop fault subscription
4120 			 * control msgs
4121 			 */
4122 			if (strcmp(class, "resource.fm.xprt.subscribe") == 0) {
4123 				pack_it = 0;
4124 				/*
4125 				 * if (FORWARDING_FAULTS_TO_CONTROL == 1) {
4126 				 * (void) nvlist_lookup_string(nvl,
4127 				 *    FM_RSRC_XPRT_SUBCLASS, &subclass);
4128 				 * if (strcmp(subclass, "list.suspect")
4129 				 *    == 0) {
4130 				 *	pack_it = 1;
4131 				 *	msg_action = FMD_XPRT_OTHER_MSG;
4132 				 * }
4133 				 * if (strcmp(subclass, "list.repaired")
4134 				 *    == 0) {
4135 				 *	pack_it = 1;
4136 				 *	msg_action = FMD_XPRT_OTHER_MSG;
4137 				 * }
4138 				 * }
4139 				 */
4140 			}
4141 			if (strcmp(class, "resource.fm.xprt.run") == 0) {
4142 				pack_it = 1;
4143 				msg_type = FMD_XPRT_RUN_MSG;
4144 			}
4145 		} else { /* has to be the root domain ldom */
4146 			iosvc = &io_svc;
4147 			/*
4148 			 * drop all ereport and fault subscriptions
4149 			 * are we dropping too much here, more than just ereport
4150 			 * and fault subscriptions? need to check
4151 			 */
4152 			if (strcmp(class, "resource.fm.xprt.subscribe") == 0)
4153 				pack_it = 0;
4154 			if (strcmp(class, "resource.fm.xprt.run") == 0) {
4155 				pack_it = 1;
4156 				msg_type = FMD_XPRT_RUN_MSG;
4157 			}
4158 		}
4159 	}
4160 
4161 	if (pack_it)  {
4162 		if (etm_debug_lvl >= 1) {
4163 			fmd_hdl_debug(fmd_hdl,
4164 			    "info: ldom name returned from xprt get specific="
4165 			    "%s xprt=%lld\n", iosvc->ldom_name, xp);
4166 		}
4167 		/*
4168 		 * pack the etm msg for the DS library and  enq in io_svc->Q
4169 		 * when the hdrp is NULL, the packing func will use the static
4170 		 * iosvc_hdr
4171 		 */
4172 		(void) etm_pack_ds_msg(fmd_hdl, iosvc, NULL, 0, nvl, msg_type,
4173 		    ETM_CKPT_NOOP);
4174 	}
4175 
4176 	return (FMD_SEND_SUCCESS);
4177 
4178 } /* etm_send() */
4179 
4180 
4181 
4182 /*
4183  * _fmd_fini - stop the server daemon and teardown the transport
4184  */
4185 
4186 void
_fmd_fini(fmd_hdl_t * hdl)4187 _fmd_fini(fmd_hdl_t *hdl)
4188 {
4189 	ssize_t			n;		/* gen use */
4190 	etm_iosvc_t		*iosvc;		/* ptr to insvc struct */
4191 	etm_iosvc_q_ele_t	msg_ele;	/* iosvc msg ele */
4192 	uint32_t		i;		/* for loop var */
4193 
4194 	fmd_hdl_debug(hdl, "info: module finalizing\n");
4195 
4196 	/* kill the connection server and responder ; wait for them to die */
4197 
4198 	etm_is_dying = 1;
4199 
4200 	if (etm_svr_tid != NULL) {
4201 		fmd_thr_signal(hdl, etm_svr_tid);
4202 		fmd_thr_destroy(hdl, etm_svr_tid);
4203 		etm_svr_tid = NULL;
4204 	} /* if server thread was successfully created */
4205 
4206 	if (etm_resp_tid != NULL) {
4207 		fmd_thr_signal(hdl, etm_resp_tid);
4208 		fmd_thr_destroy(hdl, etm_resp_tid);
4209 		etm_resp_tid = NULL;
4210 	} /* if responder thread was successfully created */
4211 
4212 	if (etm_async_e_tid != NULL) {
4213 		fmd_thr_signal(hdl, etm_async_e_tid);
4214 		fmd_thr_destroy(hdl, etm_async_e_tid);
4215 		etm_async_e_tid = NULL;
4216 	} /* if async event handler thread was successfully created */
4217 
4218 
4219 	if ((etm_ldom_type == LDOM_TYPE_LEGACY) ||
4220 	    (etm_ldom_type == LDOM_TYPE_CONTROL)) {
4221 
4222 		/* teardown the transport and cleanup syslogging */
4223 		if ((n = etm_xport_fini(hdl)) != 0) {
4224 			fmd_hdl_error(hdl, "warning: xport fini errno %d\n",
4225 			    (-n));
4226 		}
4227 		if (etm_fmd_xprt != NULL) {
4228 			fmd_xprt_close(hdl, etm_fmd_xprt);
4229 		}
4230 
4231 		if (syslog_logfd != -1) {
4232 			(void) close(syslog_logfd);
4233 		}
4234 		if (syslog_msgfd != -1) {
4235 			(void) close(syslog_msgfd);
4236 		}
4237 	}
4238 
4239 	if (etm_ldom_type == LDOM_TYPE_CONTROL)  {
4240 		if (ldom_unregister_event(etm_lhp))
4241 			fmd_hdl_debug(hdl, "ldom_unregister_event() failed\n");
4242 
4243 		/*
4244 		 * On control domain side, there may be multiple iosvc struct
4245 		 * in use, one for each bound/active domain. Each struct
4246 		 * manages a queue of fma events destined to the root domain.
4247 		 * Need to go thru every iosvc struct to clean up its resources.
4248 		 */
4249 		for (i = 0; i < NUM_OF_ROOT_DOMAINS; i++) {
4250 			if (iosvc_list[i].ldom_name[0] != '\0') {
4251 				/*
4252 				 * found an iosvc struct for a root domain
4253 				 */
4254 				iosvc = &iosvc_list[i];
4255 				(void) pthread_mutex_lock(&iosvc_list_lock);
4256 				etm_iosvc_cleanup(hdl, iosvc, B_TRUE, B_FALSE);
4257 				(void) pthread_mutex_unlock(&iosvc_list_lock);
4258 
4259 			} else {
4260 				/*
4261 				 * reach the end of existing iosvc structures
4262 				 */
4263 				continue;
4264 			}
4265 		} /* for i<NUM_OF_ROOT_DOMAINS */
4266 		etm_ckpt_fini(hdl);
4267 		etm_filter_fini(hdl);
4268 
4269 		ldom_fini(etm_lhp);
4270 
4271 	} else if (etm_ldom_type == LDOM_TYPE_ROOT) {
4272 		/*
4273 		 * On root domain side, there is only one iosvc struct in use.
4274 		 */
4275 		iosvc = &io_svc;
4276 		if (iosvc->send_tid != NULL) {
4277 			fmd_thr_signal(hdl, iosvc->send_tid);
4278 			fmd_thr_destroy(hdl, iosvc->send_tid);
4279 			iosvc->send_tid = NULL;
4280 		} /* if io svc send thread was successfully created */
4281 
4282 		if (iosvc->recv_tid != NULL) {
4283 			fmd_thr_signal(hdl, iosvc->recv_tid);
4284 			fmd_thr_destroy(hdl, iosvc->recv_tid);
4285 			iosvc->recv_tid = NULL;
4286 		} /* if io svc receive thread was successfully created */
4287 
4288 		(void) pthread_mutex_lock(&iosvc->msg_q_lock);
4289 		while (iosvc->msg_q_cur_len > 0) {
4290 			(void) etm_iosvc_msg_deq(hdl, iosvc, &msg_ele);
4291 			fmd_hdl_free(hdl, msg_ele.msg, msg_ele.msg_size);
4292 		}
4293 		(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
4294 
4295 		if (iosvc->fmd_xprt != NULL)
4296 			fmd_xprt_close(hdl, iosvc->fmd_xprt);
4297 		ldom_fini(etm_lhp);
4298 	}
4299 	if (etm_ds_fini) {
4300 		(*etm_ds_fini)();
4301 		(void) dlclose(etm_dl_hdl);
4302 	}
4303 
4304 	fmd_hdl_debug(hdl, "info: module finalized ok\n");
4305 
4306 } /* _fmd_fini() */
4307