xref: /titanic_44/usr/src/lib/fm/libfmd_snmp/common/problem.c (revision 734b6a94890be549309b21156f8ed6d4561cac51)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/fm/protocol.h>
30 #include <fm/fmd_adm.h>
31 #include <fm/fmd_snmp.h>
32 #include <net-snmp/net-snmp-config.h>
33 #include <net-snmp/net-snmp-includes.h>
34 #include <net-snmp/agent/net-snmp-agent-includes.h>
35 #include <pthread.h>
36 #include <stddef.h>
37 #include <errno.h>
38 #include <alloca.h>
39 #include <locale.h>
40 #include <libuutil.h>
41 #include <libnvpair.h>
42 #include "sunFM_impl.h"
43 #include "problem.h"
44 
45 /*
46  * We assume that the number of suspect fault events associated with a
47  * particular case will generally be sufficiently small that the overhead
48  * associated with indexing them in a tree would exceed the gain from
49  * not traversing the fault list for each request.
50  */
51 static uu_avl_pool_t	*problem_uuid_avl_pool;
52 static uu_avl_t		*problem_uuid_avl;
53 
54 #define	VALID_AVL_STATE	(problem_uuid_avl_pool != NULL &&	\
55 	problem_uuid_avl != NULL)
56 
57 #define	UPDATE_WAIT_MILLIS	10	/* poll interval in milliseconds */
58 
59 /*
60  * Update types.  Single-index and all are mutually exclusive.
61  */
62 #define	UCT_INDEX	0x1
63 #define	UCT_ALL		0x2
64 #define	UCT_FLAGS	0x3
65 
66 #define	MODULE_DATA_VALID(d)	((d)->d_valid == valid_stamp)
67 
68 /*
69  * Locking strategy is described in module.c.
70  */
71 static int		valid_stamp;
72 static pthread_mutex_t	update_lock;
73 static pthread_cond_t	update_cv;
74 static volatile enum { US_QUIET, US_NEEDED, US_INPROGRESS } update_status;
75 
76 static const char SNMP_URL[] = SNMP_URL_MSG;
77 #define	URL_BASE	"http://sun.com/msg/"
78 
79 static Netsnmp_Node_Handler	sunFmProblemTable_handler;
80 static Netsnmp_Node_Handler	sunFmFaultEventTable_handler;
81 
82 static sunFmProblem_data_t *
83 problem_key_build(const char *uuid)
84 {
85 	static sunFmProblem_data_t	key;
86 
87 	key.d_aci_uuid = uuid;
88 
89 	return (&key);
90 }
91 
92 static sunFmProblem_data_t *
93 problem_lookup_uuid_exact(const char *uuid)
94 {
95 	sunFmProblem_data_t	*key, *data;
96 
97 	key = problem_key_build(uuid);
98 
99 	DEBUGMSGTL((MODNAME_STR, "lookup_exact for uuid %s\n", uuid));
100 	data = uu_avl_find(problem_uuid_avl, key, NULL, NULL);
101 
102 	return (data);
103 }
104 
105 static sunFmProblem_data_t *
106 problem_lookup_uuid_next(const char *uuid)
107 {
108 	sunFmProblem_data_t	*key, *data;
109 	uu_avl_index_t		idx;
110 
111 	key = problem_key_build(uuid);
112 
113 	DEBUGMSGTL((MODNAME_STR, "lookup_next for uuid %s\n", uuid));
114 	(void) uu_avl_find(problem_uuid_avl, key, NULL, &idx);
115 
116 	data = uu_avl_nearest_next(problem_uuid_avl, idx);
117 
118 	DEBUGMSGTL((MODNAME_STR, "lookup_next: entry is %p\n", data));
119 
120 	return (data);
121 }
122 
123 static sunFmFaultEvent_data_t *
124 faultevent_lookup_index_exact(sunFmProblem_data_t *data, ulong_t index)
125 {
126 	if (index > data->d_nsuspects)
127 		return (NULL);
128 
129 	if (data->d_suspects == NULL)
130 		return (NULL);
131 
132 	return (data->d_suspects[index - 1]);
133 }
134 
135 static int
136 problem_update_one(const fmd_adm_caseinfo_t *acp, void *arg)
137 {
138 	const sunFmProblem_update_ctx_t	*update_ctx =
139 	    (sunFmProblem_update_ctx_t *)arg;
140 	sunFmProblem_data_t		*data;
141 	nvlist_t			*nvl;
142 	int64_t				*diag_time;
143 	uint_t				nelem;
144 	uint32_t			nsusp;
145 	int				err;
146 
147 	DEBUGMSGTL((MODNAME_STR, "update_one\n"));
148 
149 	ASSERT(acp->aci_uuid != NULL);
150 
151 	if ((data = problem_lookup_uuid_exact(acp->aci_uuid)) == NULL) {
152 		uu_avl_index_t idx;
153 
154 		DEBUGMSGTL((MODNAME_STR, "found new problem %s\n",
155 		    acp->aci_uuid));
156 		if ((data = SNMP_MALLOC_TYPEDEF(sunFmProblem_data_t)) == NULL) {
157 			snmp_log(LOG_ERR, MODNAME_STR ": Out of memory for "
158 			    "new problem data at %s:%d\n", __FILE__, __LINE__);
159 			return (0);
160 		}
161 		if ((err = nvlist_dup(acp->aci_event, &data->d_aci_event, 0))
162 		    != 0) {
163 			snmp_log(LOG_ERR, MODNAME_STR ": Problem data setup "
164 			    "failed: %s\n", strerror(err));
165 			SNMP_FREE(data);
166 			return (0);
167 		}
168 
169 		data->d_aci_uuid = data->d_aci_code = data->d_aci_url = "-";
170 		(void) nvlist_lookup_string(data->d_aci_event, FM_SUSPECT_UUID,
171 		    (char **)&data->d_aci_uuid);
172 		(void) nvlist_lookup_string(data->d_aci_event,
173 		    FM_SUSPECT_DIAG_CODE, (char **)&data->d_aci_code);
174 		data->d_aci_url = strdup(acp->aci_url);
175 
176 		if (nvlist_lookup_nvlist(data->d_aci_event, FM_SUSPECT_DE,
177 		    &nvl) == 0)
178 			if ((data->d_diag_engine = sunFm_nvl2str(nvl)) == NULL)
179 				data->d_diag_engine = "-";
180 
181 		if (nvlist_lookup_int64_array(data->d_aci_event,
182 		    FM_SUSPECT_DIAG_TIME, &diag_time, &nelem) == 0 &&
183 		    nelem >= 2) {
184 			data->d_diag_time.tv_sec = (long)diag_time[0];
185 			data->d_diag_time.tv_usec = (long)diag_time[1];
186 		}
187 
188 		(void) nvlist_lookup_uint32(data->d_aci_event,
189 		    FM_SUSPECT_FAULT_SZ, &nsusp);
190 		data->d_nsuspects = (ulong_t)nsusp;
191 
192 		(void) nvlist_lookup_nvlist_array(data->d_aci_event,
193 		    FM_SUSPECT_FAULT_LIST, &data->d_suspects, &nelem);
194 
195 		ASSERT(nelem == data->d_nsuspects);
196 
197 		uu_avl_node_init(data, &data->d_uuid_avl,
198 		    problem_uuid_avl_pool);
199 		(void) uu_avl_find(problem_uuid_avl, data, NULL, &idx);
200 		uu_avl_insert(problem_uuid_avl, data, idx);
201 
202 		data->d_valid = valid_stamp;
203 
204 		DEBUGMSGTL((MODNAME_STR, "completed new problem %s@%p\n",
205 		    data->d_aci_uuid, data));
206 	}
207 
208 	/*
209 	 * We don't touch problems we've seen before; they shouldn't change
210 	 * in any way we care about, since they've already been solved.  The
211 	 * state, however, could change, and if we later expose that to the
212 	 * client we need to update it here.
213 	 */
214 
215 	return (0);
216 }
217 
218 static int
219 problem_update(sunFmProblem_update_ctx_t *update_ctx)
220 {
221 	fmd_adm_t *adm;
222 
223 	ASSERT(update_ctx != NULL);
224 	ASSERT((update_ctx->uc_type & (UCT_INDEX|UCT_ALL)) !=
225 	    (UCT_INDEX|UCT_ALL));
226 	ASSERT((update_ctx->uc_type & ~UCT_FLAGS) == 0);
227 	ASSERT(VALID_AVL_STATE);
228 
229 	if ((adm = fmd_adm_open(update_ctx->uc_host, update_ctx->uc_prog,
230 	    update_ctx->uc_version)) == NULL) {
231 		snmp_log(LOG_ERR, MODNAME_STR ": Communication with fmd "
232 		    "failed: %s\n", strerror(errno));
233 		return (SNMP_ERR_RESOURCEUNAVAILABLE);
234 	}
235 
236 	++valid_stamp;
237 	if (fmd_adm_case_iter(adm, SNMP_URL_MSG, problem_update_one,
238 	    update_ctx) != 0) {
239 		snmp_log(LOG_ERR, MODNAME_STR ": fmd case information update "
240 		    "failed: %s\n", fmd_adm_errmsg(adm));
241 		fmd_adm_close(adm);
242 		return (SNMP_ERR_RESOURCEUNAVAILABLE);
243 	}
244 
245 	DEBUGMSGTL((MODNAME_STR, "case iteration completed\n"));
246 
247 	fmd_adm_close(adm);
248 	return (SNMP_ERR_NOERROR);
249 }
250 
251 /*ARGSUSED*/
252 static void
253 update_thread(void *arg)
254 {
255 	/*
256 	 * The current problem_update implementation offers minimal savings
257 	 * for the use of index-only updates; therefore we always do a full
258 	 * update.  If it becomes advantageous to limit updates to a single
259 	 * index, the contexts can be queued by the handler instead.
260 	 */
261 	sunFmProblem_update_ctx_t	uc;
262 
263 	uc.uc_host = NULL;
264 	uc.uc_prog = FMD_ADM_PROGRAM;
265 	uc.uc_version = FMD_ADM_VERSION;
266 
267 	uc.uc_index = NULL;
268 	uc.uc_type = UCT_ALL;
269 
270 	for (;;) {
271 		(void) pthread_mutex_lock(&update_lock);
272 		update_status = US_QUIET;
273 		while (update_status == US_QUIET)
274 			(void) pthread_cond_wait(&update_cv, &update_lock);
275 		update_status = US_INPROGRESS;
276 		(void) pthread_mutex_unlock(&update_lock);
277 		(void) problem_update(&uc);
278 	}
279 }
280 
281 static void
282 request_update(void)
283 {
284 	(void) pthread_mutex_lock(&update_lock);
285 	if (update_status != US_QUIET) {
286 		(void) pthread_mutex_unlock(&update_lock);
287 		return;
288 	}
289 	update_status = US_NEEDED;
290 	(void) pthread_cond_signal(&update_cv);
291 	(void) pthread_mutex_unlock(&update_lock);
292 }
293 
294 /*ARGSUSED*/
295 static int
296 problem_compare_uuid(const void *l, const void *r, void *private)
297 {
298 	sunFmProblem_data_t	*l_data = (sunFmProblem_data_t *)l;
299 	sunFmProblem_data_t	*r_data = (sunFmProblem_data_t *)r;
300 
301 	ASSERT(l_data != NULL && r_data != NULL);
302 
303 	return (strcmp(l_data->d_aci_uuid, r_data->d_aci_uuid));
304 }
305 
306 int
307 sunFmProblemTable_init(void)
308 {
309 	static oid sunFmProblemTable_oid[] = { SUNFMPROBLEMTABLE_OID };
310 	netsnmp_table_registration_info *table_info;
311 	netsnmp_handler_registration *handler;
312 	int err;
313 
314 	if ((err = pthread_mutex_init(&update_lock, NULL)) != 0) {
315 		snmp_log(LOG_ERR, MODNAME_STR ": mutex_init failure: %s\n",
316 		    strerror(err));
317 		return (MIB_REGISTRATION_FAILED);
318 	}
319 	if ((err = pthread_cond_init(&update_cv, NULL)) != 0) {
320 		snmp_log(LOG_ERR, MODNAME_STR ": cond_init failure: %s\n",
321 		    strerror(err));
322 		return (MIB_REGISTRATION_FAILED);
323 	}
324 
325 	if ((err = pthread_create(NULL, NULL, (void *(*)(void *))update_thread,
326 	    NULL)) != 0) {
327 		snmp_log(LOG_ERR, MODNAME_STR ": error creating update "
328 		    "thread: %s\n", strerror(err));
329 		return (MIB_REGISTRATION_FAILED);
330 	}
331 
332 	if ((table_info =
333 	    SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL)
334 		return (MIB_REGISTRATION_FAILED);
335 
336 	if ((handler = netsnmp_create_handler_registration("sunFmProblemTable",
337 	    sunFmProblemTable_handler, sunFmProblemTable_oid,
338 	    OID_LENGTH(sunFmProblemTable_oid), HANDLER_CAN_RONLY)) == NULL) {
339 		SNMP_FREE(table_info);
340 		return (MIB_REGISTRATION_FAILED);
341 	}
342 
343 	/*
344 	 * The Net-SNMP template uses add_indexes here, but that
345 	 * function is unsafe because it does not check for failure.
346 	 */
347 	if (netsnmp_table_helper_add_index(table_info, ASN_OCTET_STR) == NULL) {
348 		SNMP_FREE(table_info);
349 		SNMP_FREE(handler);
350 		return (MIB_REGISTRATION_FAILED);
351 	}
352 
353 	table_info->min_column = SUNFMPROBLEM_COLMIN;
354 	table_info->max_column = SUNFMPROBLEM_COLMAX;
355 
356 	if ((problem_uuid_avl_pool = uu_avl_pool_create("problem_uuid",
357 	    sizeof (sunFmProblem_data_t),
358 	    offsetof(sunFmProblem_data_t, d_uuid_avl), problem_compare_uuid,
359 	    UU_AVL_DEBUG)) == NULL) {
360 		snmp_log(LOG_ERR, MODNAME_STR ": problem_uuid avl pool "
361 		    "creation failed: %s\n", uu_strerror(uu_error()));
362 		snmp_free_varbind(table_info->indexes);
363 		SNMP_FREE(table_info);
364 		SNMP_FREE(handler);
365 		return (MIB_REGISTRATION_FAILED);
366 	}
367 
368 	if ((problem_uuid_avl = uu_avl_create(problem_uuid_avl_pool, NULL,
369 	    UU_AVL_DEBUG)) == NULL) {
370 		snmp_log(LOG_ERR, MODNAME_STR ": problem_uuid avl creation "
371 		    "failed: %s\n", uu_strerror(uu_error()));
372 		snmp_free_varbind(table_info->indexes);
373 		SNMP_FREE(table_info);
374 		SNMP_FREE(handler);
375 		uu_avl_pool_destroy(problem_uuid_avl_pool);
376 		return (MIB_REGISTRATION_FAILED);
377 	}
378 
379 	if ((err = netsnmp_register_table(handler, table_info)) !=
380 	    MIB_REGISTERED_OK) {
381 		snmp_free_varbind(table_info->indexes);
382 		SNMP_FREE(table_info);
383 		SNMP_FREE(handler);
384 		uu_avl_destroy(problem_uuid_avl);
385 		uu_avl_pool_destroy(problem_uuid_avl_pool);
386 		return (err);
387 	}
388 
389 	return (MIB_REGISTERED_OK);
390 }
391 
392 int
393 sunFmFaultEventTable_init(void)
394 {
395 	static oid sunFmFaultEventTable_oid[] = { SUNFMFAULTEVENTTABLE_OID };
396 	netsnmp_table_registration_info *table_info;
397 	netsnmp_handler_registration *handler;
398 	int err;
399 
400 	if ((table_info =
401 	    SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info)) == NULL)
402 		return (MIB_REGISTRATION_FAILED);
403 
404 	if ((handler =
405 	    netsnmp_create_handler_registration("sunFmFaultEventTable",
406 	    sunFmFaultEventTable_handler, sunFmFaultEventTable_oid,
407 	    OID_LENGTH(sunFmFaultEventTable_oid), HANDLER_CAN_RONLY)) == NULL) {
408 		SNMP_FREE(table_info);
409 		return (MIB_REGISTRATION_FAILED);
410 	}
411 
412 	/*
413 	 * The Net-SNMP template uses add_indexes here, but that
414 	 * function is unsafe because it does not check for failure.
415 	 */
416 	if (netsnmp_table_helper_add_index(table_info, ASN_OCTET_STR) == NULL) {
417 		SNMP_FREE(table_info);
418 		SNMP_FREE(handler);
419 		return (MIB_REGISTRATION_FAILED);
420 	}
421 	if (netsnmp_table_helper_add_index(table_info, ASN_UNSIGNED) == NULL) {
422 		snmp_free_varbind(table_info->indexes);
423 		SNMP_FREE(table_info);
424 		SNMP_FREE(handler);
425 		return (MIB_REGISTRATION_FAILED);
426 	}
427 
428 	table_info->min_column = SUNFMFAULTEVENT_COLMIN;
429 	table_info->max_column = SUNFMFAULTEVENT_COLMAX;
430 
431 	if ((err = netsnmp_register_table(handler, table_info)) !=
432 	    MIB_REGISTERED_OK) {
433 		snmp_free_varbind(table_info->indexes);
434 		SNMP_FREE(table_info);
435 		SNMP_FREE(handler);
436 		return (err);
437 	}
438 
439 	return (MIB_REGISTERED_OK);
440 }
441 
442 /*
443  * Returns the problem data for the problem whose uuid is next according
444  * to ASN.1 lexical ordering after the request in table_info.  Indexes are
445  * updated to reflect the OID of the value being returned.  This allows
446  * us to implement GETNEXT.
447  */
448 static sunFmProblem_data_t *
449 sunFmProblemTable_nextpr(netsnmp_handler_registration *reginfo,
450     netsnmp_table_request_info *table_info)
451 {
452 	sunFmProblem_data_t	*data;
453 	netsnmp_variable_list	*var;
454 	char			*uuid = "";
455 
456 	if (table_info->number_indexes < 1) {
457 		oid tmpoid[MAX_OID_LEN];
458 
459 		DEBUGMSGTL((MODNAME_STR, "nextpr: no indexes given\n"));
460 
461 		table_info->indexes =
462 		    SNMP_MALLOC_TYPEDEF(netsnmp_variable_list);
463 		snmp_set_var_typed_value(table_info->indexes, ASN_OCTET_STR,
464 		    (const uchar_t *)uuid, 0);
465 		memcpy(tmpoid, reginfo->rootoid,
466 		    reginfo->rootoid_len * sizeof (oid));
467 		tmpoid[reginfo->rootoid_len] = 1;
468 		tmpoid[reginfo->rootoid_len + 1] = table_info->colnum;
469 		if (build_oid_segment(table_info->indexes) != SNMPERR_SUCCESS) {
470 			snmp_free_varbind(table_info->indexes);
471 			return (NULL);
472 		}
473 		table_info->number_indexes = 1;
474 		table_info->index_oid_len = table_info->indexes->name_length;
475 		memcpy(table_info->index_oid, table_info->indexes->name,
476 		    table_info->indexes->name_length);
477 
478 		DEBUGMSGTL((MODNAME_STR, "nextpr: built fake index:\n"));
479 		DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
480 		DEBUGMSG((MODNAME_STR, "\n"));
481 	} else {
482 		/*
483 		 * Construct the next possible UUID to look for.  We can
484 		 * simply increment the least significant byte of the last
485 		 * UUID because (a) that preserves SNMP lex order and (b)
486 		 * the characters that may appear in a UUID do not include
487 		 * 127 nor 255.
488 		 */
489 		uuid = alloca(table_info->indexes->val_len + 1);
490 		(void) strlcpy(uuid,
491 		    (const char *)table_info->indexes->val.string,
492 		    table_info->indexes->val_len + 1);
493 		++uuid[table_info->indexes->val_len - 1];
494 
495 		DEBUGMSGTL((MODNAME_STR, "nextpr: received index:\n"));
496 		DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
497 		DEBUGMSG((MODNAME_STR, "\n"));
498 	}
499 
500 	if ((data = problem_lookup_uuid_next(uuid)) == NULL) {
501 		DEBUGMSGTL((MODNAME_STR, "nextpr: next match not found for "
502 		    "%s; trying next column\n", uuid));
503 		if (table_info->colnum >= SUNFMPROBLEM_COLMAX) {
504 			snmp_free_varbind(table_info->indexes);
505 			table_info->indexes = NULL;
506 			table_info->number_indexes = 0;
507 			DEBUGMSGTL((MODNAME_STR, "nextpr: out of columns\n"));
508 			return (NULL);
509 		}
510 		table_info->colnum++;
511 		DEBUGMSGTL((MODNAME_STR, "nextpr: search for col %u empty "
512 		    "uuid\n", table_info->colnum, uuid));
513 
514 		if ((data = problem_lookup_uuid_next("")) == NULL) {
515 			DEBUGMSGTL((MODNAME_STR, "nextpr: next match not found "
516 			    "for empty uuid; stopping\n"));
517 			snmp_free_varbind(table_info->indexes);
518 			table_info->indexes = NULL;
519 			table_info->number_indexes = 0;
520 			return (NULL);
521 		}
522 	}
523 
524 	snmp_set_var_typed_value(table_info->indexes, ASN_OCTET_STR,
525 	    (uchar_t *)data->d_aci_uuid, strlen(data->d_aci_uuid));
526 	table_info->number_indexes = 1;
527 
528 	DEBUGMSGTL((MODNAME_STR, "matching data is %s@%p\n", data->d_aci_uuid,
529 	    data));
530 
531 	return (data);
532 }
533 
534 /*
535  * Returns the problem data corresponding to the request in table_info.
536  * All request parameters are unmodified.
537  */
538 static sunFmProblem_data_t *
539 sunFmProblemTable_pr(netsnmp_handler_registration *reginfo,
540     netsnmp_table_request_info *table_info)
541 {
542 	sunFmProblem_data_t	*data;
543 	char			*uuid;
544 
545 	ASSERT(table_info->number_indexes >= 1);
546 
547 	uuid = alloca(table_info->indexes->val_len + 1);
548 	(void) strlcpy(uuid, (const char *)table_info->indexes->val.string,
549 	    table_info->indexes->val_len + 1);
550 
551 	return (problem_lookup_uuid_exact(uuid));
552 }
553 
554 /*
555  * Returns the ASN.1 lexicographically first fault event after the one
556  * identified by table_info.  Indexes are updated to reflect the OID
557  * of the data returned.  This allows us to implement GETNEXT.
558  */
559 static sunFmFaultEvent_data_t *
560 sunFmFaultEventTable_nextfe(netsnmp_handler_registration *reginfo,
561     netsnmp_table_request_info *table_info)
562 {
563 	sunFmProblem_data_t	*data;
564 	sunFmFaultEvent_data_t	*rv;
565 	netsnmp_variable_list	*var;
566 	char			*uuid;
567 	ulong_t			index;
568 
569 	for (;;) {
570 		switch (table_info->number_indexes) {
571 		case 2:
572 		default:
573 			DEBUGMSGTL((MODNAME_STR, "nextfe: 2 indices:\n"));
574 			DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
575 			DEBUGMSG((MODNAME_STR, "\n"));
576 			DEBUGMSGVAR((MODNAME_STR,
577 			    table_info->indexes->next_variable));
578 			DEBUGMSG((MODNAME_STR, "\n"));
579 			index = *(ulong_t *)
580 			    table_info->indexes->next_variable->val.integer + 1;
581 
582 			if ((data = sunFmProblemTable_pr(reginfo,
583 			    table_info)) != NULL &&
584 			    (rv = faultevent_lookup_index_exact(data, index)) !=
585 			    NULL) {
586 				snmp_set_var_typed_value(
587 				    table_info->indexes->next_variable,
588 				    ASN_UNSIGNED, (uchar_t *)&index,
589 				    sizeof (index));
590 				return (rv);
591 			}
592 
593 			if (sunFmProblemTable_nextpr(reginfo, table_info) ==
594 			    NULL)
595 				return (NULL);
596 			break;
597 		case 1:
598 			if ((data = sunFmProblemTable_pr(reginfo,
599 			    table_info)) != NULL) {
600 				oid tmpoid[MAX_OID_LEN];
601 				index = 0;
602 
603 				DEBUGMSGTL((MODNAME_STR, "nextfe: 1 index:\n"));
604 				DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
605 				DEBUGMSG((MODNAME_STR, "\n"));
606 				var =
607 				    SNMP_MALLOC_TYPEDEF(netsnmp_variable_list);
608 				snmp_set_var_typed_value(var, ASN_UNSIGNED,
609 				    (uchar_t *)&index, sizeof (index));
610 				memcpy(tmpoid, reginfo->rootoid,
611 				    reginfo->rootoid_len * sizeof (oid));
612 				tmpoid[reginfo->rootoid_len] = 1;
613 				tmpoid[reginfo->rootoid_len + 1] =
614 				    table_info->colnum;
615 				if (build_oid_segment(var) != SNMPERR_SUCCESS) {
616 					snmp_free_varbind(var);
617 					return (NULL);
618 				}
619 				table_info->indexes->next_variable = var;
620 				table_info->number_indexes = 2;
621 				DEBUGMSGTL((MODNAME_STR, "nextfe: built fake "
622 				    "index:\n"));
623 				DEBUGMSGVAR((MODNAME_STR, table_info->indexes));
624 				DEBUGMSG((MODNAME_STR, "\n"));
625 				DEBUGMSGVAR((MODNAME_STR,
626 				    table_info->indexes->next_variable));
627 				DEBUGMSG((MODNAME_STR, "\n"));
628 			} else {
629 				if (sunFmProblemTable_nextpr(reginfo,
630 				    table_info) == NULL)
631 					return (NULL);
632 			}
633 			break;
634 		case 0:
635 			if (sunFmProblemTable_nextpr(reginfo, table_info) ==
636 			    NULL)
637 				return (NULL);
638 			break;
639 		}
640 	}
641 }
642 
643 static sunFmFaultEvent_data_t *
644 sunFmFaultEventTable_fe(netsnmp_handler_registration *reginfo,
645     netsnmp_table_request_info *table_info)
646 {
647 	sunFmProblem_data_t	*data;
648 
649 	ASSERT(table_info->number_indexes == 2);
650 
651 	if ((data = sunFmProblemTable_pr(reginfo, table_info)) == NULL)
652 		return (NULL);
653 
654 	return (faultevent_lookup_index_exact(data,
655 	    *(ulong_t *)table_info->indexes->next_variable->val.integer));
656 }
657 
658 static void
659 sunFmProblemTable_return(unsigned int reg, void *arg)
660 {
661 	netsnmp_delegated_cache		*cache = (netsnmp_delegated_cache *)arg;
662 	netsnmp_request_info		*request;
663 	netsnmp_agent_request_info	*reqinfo;
664 	netsnmp_handler_registration	*reginfo;
665 	netsnmp_mib_handler		*handler;
666 	netsnmp_table_request_info	*table_info;
667 	netsnmp_variable_list		*var;
668 	sunFmProblem_data_t		*data;
669 
670 	ASSERT(netsnmp_handler_check_cache(cache) != NULL);
671 
672 	(void) pthread_mutex_lock(&update_lock);
673 	if (update_status != US_QUIET) {
674 		struct timeval			tv;
675 
676 		tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
677 		tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
678 
679 		(void) snmp_alarm_register_hr(tv, 0, sunFmProblemTable_return,
680 		    cache);
681 		(void) pthread_mutex_unlock(&update_lock);
682 		return;
683 	}
684 
685 	request = cache->requests;
686 	reqinfo = cache->reqinfo;
687 	reginfo = cache->reginfo;
688 	handler = cache->handler;
689 
690 	var = request->requestvb;
691 	table_info = netsnmp_extract_table_info(request);
692 	request->delegated = 0;
693 
694 	ASSERT(table_info->colnum >= SUNFMPROBLEM_COLMIN);
695 	ASSERT(table_info->colnum <= SUNFMPROBLEM_COLMAX);
696 
697 	/*
698 	 * table_info->colnum contains the column number requested.
699 	 * table_info->indexes contains a linked list of snmp variable
700 	 * bindings for the indexes of the table.  Values in the list
701 	 * have been set corresponding to the indexes of the
702 	 * request.  We have other guarantees as well:
703 	 *
704 	 * - The column number is always within range.
705 	 * - If we have no index data, table_info->index_oid_len is 0.
706 	 * - We will never receive requests outside our table nor
707 	 *   those with the first subid anything other than 1 (Entry)
708 	 *   nor those without a column number.  This is true even
709 	 *   for GETNEXT requests.
710 	 */
711 
712 	switch (reqinfo->mode) {
713 	case MODE_GET:
714 		if ((data = sunFmProblemTable_pr(reginfo, table_info)) ==
715 		    NULL) {
716 			netsnmp_free_delegated_cache(cache);
717 			(void) pthread_mutex_unlock(&update_lock);
718 			return;
719 		}
720 		break;
721 	case MODE_GETNEXT:
722 	case MODE_GETBULK:
723 		if ((data = sunFmProblemTable_nextpr(reginfo, table_info)) ==
724 		    NULL) {
725 			netsnmp_free_delegated_cache(cache);
726 			(void) pthread_mutex_unlock(&update_lock);
727 			return;
728 		}
729 		break;
730 	default:
731 		snmp_log(LOG_ERR, MODNAME_STR ": Unsupported request mode %d\n",
732 		    reqinfo->mode);
733 		netsnmp_free_delegated_cache(cache);
734 		(void) pthread_mutex_unlock(&update_lock);
735 		return;
736 	}
737 
738 	switch (table_info->colnum) {
739 	case SUNFMPROBLEM_COL_UUID:
740 	{
741 		netsnmp_table_build_result(reginfo, request, table_info,
742 		    ASN_OCTET_STR, (uchar_t *)data->d_aci_uuid,
743 		    strlen(data->d_aci_uuid));
744 		break;
745 	}
746 	case SUNFMPROBLEM_COL_CODE:
747 	{
748 		netsnmp_table_build_result(reginfo, request, table_info,
749 		    ASN_OCTET_STR, (uchar_t *)data->d_aci_code,
750 		    strlen(data->d_aci_code));
751 		break;
752 	}
753 	case SUNFMPROBLEM_COL_URL:
754 	{
755 		netsnmp_table_build_result(reginfo, request, table_info,
756 		    ASN_OCTET_STR, (uchar_t *)data->d_aci_url,
757 		    strlen(data->d_aci_url));
758 		break;
759 	}
760 	case SUNFMPROBLEM_COL_DIAGENGINE:
761 	{
762 		netsnmp_table_build_result(reginfo, request, table_info,
763 		    ASN_OCTET_STR, (uchar_t *)data->d_diag_engine,
764 		    strlen(data->d_diag_engine));
765 		break;
766 	}
767 	case SUNFMPROBLEM_COL_DIAGTIME:
768 	{
769 		/*
770 		 * The date_n_time function is not Y2038-safe; this may
771 		 * need to be updated when a suitable Y2038-safe Net-SNMP
772 		 * API is available.
773 		 */
774 		size_t	dt_size;
775 		time_t	dt_time = (time_t)data->d_diag_time.tv_sec;
776 		uchar_t	*dt = date_n_time(&dt_time, &dt_size);
777 
778 		netsnmp_table_build_result(reginfo, request, table_info,
779 		    ASN_OCTET_STR, dt, dt_size);
780 		break;
781 	}
782 	case SUNFMPROBLEM_COL_SUSPECTCOUNT:
783 	{
784 		netsnmp_table_build_result(reginfo, request, table_info,
785 		    ASN_UNSIGNED, (uchar_t *)&data->d_nsuspects,
786 		    sizeof (data->d_nsuspects));
787 		break;
788 	}
789 	default:
790 		break;
791 	}
792 
793 	netsnmp_free_delegated_cache(cache);
794 	(void) pthread_mutex_unlock(&update_lock);
795 }
796 
797 static int
798 sunFmProblemTable_handler(netsnmp_mib_handler *handler,
799     netsnmp_handler_registration *reginfo, netsnmp_agent_request_info *reqinfo,
800     netsnmp_request_info *requests)
801 {
802 	netsnmp_request_info		*request;
803 	struct timeval			tv;
804 
805 	tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
806 	tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
807 
808 	request_update();
809 
810 	for (request = requests; request; request = request->next) {
811 		if (request->processed != 0)
812 			continue;
813 
814 		if (netsnmp_extract_table_info(request) == NULL)
815 			continue;
816 
817 		request->delegated = 1;
818 		(void) snmp_alarm_register_hr(tv, 0,
819 		    sunFmProblemTable_return,
820 		    (void *) netsnmp_create_delegated_cache(handler, reginfo,
821 		    reqinfo, request, NULL));
822 	}
823 
824 	return (SNMP_ERR_NOERROR);
825 }
826 
827 static void
828 sunFmFaultEventTable_return(unsigned int reg, void *arg)
829 {
830 	netsnmp_delegated_cache		*cache = (netsnmp_delegated_cache *)arg;
831 	netsnmp_request_info		*request;
832 	netsnmp_agent_request_info	*reqinfo;
833 	netsnmp_handler_registration	*reginfo;
834 	netsnmp_mib_handler		*handler;
835 	netsnmp_table_request_info	*table_info;
836 	netsnmp_variable_list		*var;
837 	sunFmProblem_data_t		*pdata;
838 	sunFmFaultEvent_data_t		*data;
839 
840 	ASSERT(netsnmp_handler_check_cache(cache) != NULL);
841 
842 	(void) pthread_mutex_lock(&update_lock);
843 	if (update_status != US_QUIET) {
844 		struct timeval			tv;
845 
846 		tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
847 		tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
848 
849 		(void) snmp_alarm_register_hr(tv, 0,
850 		    sunFmFaultEventTable_return, cache);
851 		(void) pthread_mutex_unlock(&update_lock);
852 		return;
853 	}
854 
855 	request = cache->requests;
856 	reqinfo = cache->reqinfo;
857 	reginfo = cache->reginfo;
858 	handler = cache->handler;
859 
860 	var = request->requestvb;
861 	table_info = netsnmp_extract_table_info(request);
862 	request->delegated = 0;
863 
864 	ASSERT(table_info->colnum >= SUNFMFAULTEVENT_COLMIN);
865 	ASSERT(table_info->colnum <= SUNFMFAULTEVENT_COLMAX);
866 
867 	/*
868 	 * table_info->colnum contains the column number requested.
869 	 * table_info->indexes contains a linked list of snmp variable
870 	 * bindings for the indexes of the table.  Values in the list
871 	 * have been set corresponding to the indexes of the
872 	 * request.  We have other guarantees as well:
873 	 *
874 	 * - The column number is always within range.
875 	 * - If we have no index data, table_info->index_oid_len is 0.
876 	 * - We will never receive requests outside our table nor
877 	 *   those with the first subid anything other than 1 (Entry)
878 	 *   nor those without a column number.  This is true even
879 	 *   for GETNEXT requests.
880 	 */
881 
882 	switch (reqinfo->mode) {
883 	case MODE_GET:
884 		if ((data = sunFmFaultEventTable_fe(reginfo, table_info)) ==
885 		    NULL) {
886 			netsnmp_free_delegated_cache(cache);
887 			(void) pthread_mutex_unlock(&update_lock);
888 			return;
889 		}
890 		break;
891 	case MODE_GETNEXT:
892 	case MODE_GETBULK:
893 		if ((data = sunFmFaultEventTable_nextfe(reginfo, table_info)) ==
894 		    NULL) {
895 			netsnmp_free_delegated_cache(cache);
896 			(void) pthread_mutex_unlock(&update_lock);
897 			return;
898 		}
899 		break;
900 	default:
901 		snmp_log(LOG_ERR, MODNAME_STR ": Unsupported request mode %d\n",
902 		    reqinfo->mode);
903 		netsnmp_free_delegated_cache(cache);
904 		(void) pthread_mutex_unlock(&update_lock);
905 		return;
906 	}
907 
908 	switch (table_info->colnum) {
909 	case SUNFMFAULTEVENT_COL_PROBLEMUUID:
910 	{
911 		if ((pdata = sunFmProblemTable_pr(reginfo, table_info))
912 		    == NULL) {
913 			netsnmp_table_build_result(reginfo, request, table_info,
914 			    ASN_OCTET_STR, NULL, 0);
915 			break;
916 		}
917 		netsnmp_table_build_result(reginfo, request, table_info,
918 		    ASN_OCTET_STR, (uchar_t *)pdata->d_aci_uuid,
919 		    strlen(pdata->d_aci_uuid));
920 		break;
921 	}
922 	case SUNFMFAULTEVENT_COL_CLASS:
923 	{
924 		char	*class = "-";
925 
926 		(void) nvlist_lookup_string(data, FM_CLASS, &class);
927 		netsnmp_table_build_result(reginfo, request, table_info,
928 		    ASN_OCTET_STR, (uchar_t *)class, strlen(class));
929 		break;
930 	}
931 	case SUNFMFAULTEVENT_COL_CERTAINTY:
932 	{
933 		uint8_t	pct = 0;
934 		ulong_t	pl;
935 
936 		(void) nvlist_lookup_uint8(data, FM_FAULT_CERTAINTY,
937 		    &pct);
938 		pl = (ulong_t)pct;
939 		netsnmp_table_build_result(reginfo, request, table_info,
940 		    ASN_UNSIGNED, (uchar_t *)&pl, sizeof (pl));
941 		break;
942 	}
943 	case SUNFMFAULTEVENT_COL_ASRU:
944 	{
945 		nvlist_t	*asru = NULL;
946 		char		*fmri;
947 
948 		(void) nvlist_lookup_nvlist(data, FM_FAULT_ASRU, &asru);
949 		if ((fmri = sunFm_nvl2str(asru)) == NULL)
950 			fmri = "-";
951 		netsnmp_table_build_result(reginfo, request, table_info,
952 		    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
953 		break;
954 	}
955 	case SUNFMFAULTEVENT_COL_FRU:
956 	{
957 		nvlist_t	*fru = NULL;
958 		char		*fmri;
959 
960 		(void) nvlist_lookup_nvlist(data, FM_FAULT_FRU, &fru);
961 		if ((fmri = sunFm_nvl2str(fru)) == NULL)
962 			fmri = "-";
963 		netsnmp_table_build_result(reginfo, request, table_info,
964 		    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
965 		break;
966 	}
967 	case SUNFMFAULTEVENT_COL_RESOURCE:
968 	{
969 		nvlist_t	*rsrc = NULL;
970 		char		*fmri;
971 
972 		(void) nvlist_lookup_nvlist(data, FM_FAULT_ASRU, &rsrc);
973 		if ((fmri = sunFm_nvl2str(rsrc)) == NULL)
974 			fmri = "-";
975 		netsnmp_table_build_result(reginfo, request, table_info,
976 		    ASN_OCTET_STR, (uchar_t *)fmri, strlen(fmri));
977 		break;
978 	}
979 	default:
980 		break;
981 	}
982 
983 	netsnmp_free_delegated_cache(cache);
984 	(void) pthread_mutex_unlock(&update_lock);
985 }
986 
987 static int
988 sunFmFaultEventTable_handler(netsnmp_mib_handler *handler,
989     netsnmp_handler_registration *reginfo, netsnmp_agent_request_info *reqinfo,
990     netsnmp_request_info *requests)
991 {
992 	netsnmp_request_info		*request;
993 	struct timeval			tv;
994 
995 	tv.tv_sec = UPDATE_WAIT_MILLIS / 1000;
996 	tv.tv_usec = (UPDATE_WAIT_MILLIS % 1000) * 1000;
997 
998 	request_update();
999 
1000 	for (request = requests; request; request = request->next) {
1001 		if (request->processed != 0)
1002 			continue;
1003 
1004 		if (netsnmp_extract_table_info(request) == NULL)
1005 			continue;
1006 
1007 		request->delegated = 1;
1008 		(void) snmp_alarm_register_hr(tv, 0,
1009 		    sunFmFaultEventTable_return,
1010 		    (void *) netsnmp_create_delegated_cache(handler, reginfo,
1011 		    reqinfo, request, NULL));
1012 	}
1013 
1014 	return (SNMP_ERR_NOERROR);
1015 }
1016