xref: /titanic_51/usr/src/cmd/fm/fmd/common/fmd_mdb.c (revision 4372c2457c83ba786dcbd57c9896aa05174f2673)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/mdb_modapi.h>
28 #include <limits.h>
29 
30 #include <fmd_trace.h>
31 #include <fmd_module.h>
32 #include <fmd_thread.h>
33 #include <fmd_ustat.h>
34 #include <fmd_event.h>
35 #include <fmd_case.h>
36 #include <fmd_buf.h>
37 #include <fmd_asru.h>
38 #include <fmd_ckpt.h>
39 #include <fmd_timerq.h>
40 #include <fmd_xprt.h>
41 
42 #include <fmd.h>
43 
44 typedef struct trwalk_state {
45 	struct trwalk_state *trw_next;
46 	fmd_tracebuf_t trw_data;
47 	pthread_t trw_tid;
48 	uintptr_t trw_base;
49 	const fmd_tracerec_t *trw_stop;
50 	fmd_tracerec_t *trw_xrec;
51 } trwalk_state_t;
52 
53 typedef struct hashwalk_data {
54 	uintptr_t *hw_hash;
55 	uint_t hw_hashlen;
56 	uint_t hw_hashidx;
57 	const char *hw_name;
58 	void *hw_data;
59 	size_t hw_size;
60 	size_t hw_next;
61 } hashwalk_data_t;
62 
63 static int fmd_stat(uintptr_t, uint_t, int, const mdb_arg_t *);
64 static int fmd_ustat(uintptr_t, uint_t, int, const mdb_arg_t *);
65 
66 static int
67 trwalk_init(mdb_walk_state_t *wsp)
68 {
69 	uintptr_t addr;
70 	fmd_thread_t thr;
71 	fmd_t F;
72 
73 	if (wsp->walk_addr != NULL) {
74 		mdb_warn("fmd_trace only supports global walks\n");
75 		return (WALK_ERR);
76 	}
77 
78 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
79 		mdb_warn("failed to read fmd meta-data");
80 		return (WALK_ERR);
81 	}
82 
83 	for (addr = (uintptr_t)F.d_thr_list.l_next; addr != NULL;
84 	    addr = (uintptr_t)thr.thr_list.l_next) {
85 
86 		size_t len, ptr_off, end_off;
87 		fmd_tracerec_t *buf;
88 		trwalk_state_t *t;
89 
90 		if (mdb_vread(&thr, sizeof (thr), addr) != sizeof (thr)) {
91 			mdb_warn("failed to read thread at %p "
92 			    "(some trace data will be unavailable)", addr);
93 			break;
94 		}
95 
96 		t = mdb_zalloc(sizeof (trwalk_state_t), UM_SLEEP);
97 		t->trw_next = wsp->walk_data;
98 		wsp->walk_data = t;
99 
100 		(void) mdb_vread(&t->trw_data,
101 		    sizeof (t->trw_data), (uintptr_t)thr.thr_trdata);
102 
103 		if (t->trw_data.tb_recs == 0)
104 			continue; /* no trace buffer allocated for thread */
105 
106 		len = t->trw_data.tb_recs * t->trw_data.tb_size;
107 		buf = mdb_alloc(len, UM_SLEEP);
108 
109 		t->trw_tid = thr.thr_tid;
110 		t->trw_base = (uintptr_t)t->trw_data.tb_buf;
111 
112 		if (mdb_vread(buf, len, t->trw_base) == -1) {
113 			mdb_warn("failed to read buffer for t%u", t->trw_tid);
114 			bzero(buf, len);
115 		}
116 
117 		end_off = (uintptr_t)t->trw_data.tb_end - t->trw_base;
118 		ptr_off = (uintptr_t)t->trw_data.tb_ptr - t->trw_base;
119 
120 		t->trw_data.tb_buf = buf;
121 		t->trw_data.tb_end = (void *)((uintptr_t)buf + end_off);
122 		t->trw_data.tb_ptr = (void *)((uintptr_t)buf + ptr_off);
123 
124 		if (t->trw_data.tb_ptr < t->trw_data.tb_buf ||
125 		    t->trw_data.tb_ptr > t->trw_data.tb_end) {
126 			mdb_warn("trace record ptr for t%u is corrupt "
127 			    "(some data may be unavailable)\n", t->trw_tid);
128 			t->trw_data.tb_ptr = t->trw_data.tb_buf;
129 		}
130 
131 		t->trw_stop = t->trw_data.tb_ptr;
132 		t->trw_xrec = mdb_alloc(
133 		    t->trw_data.tb_size + sizeof (uintptr_t), UM_SLEEP);
134 	}
135 
136 	return (WALK_NEXT);
137 }
138 
139 static fmd_tracerec_t *
140 trwalk_nextrec(trwalk_state_t *t)
141 {
142 	if (t->trw_stop == NULL)
143 		return (t->trw_data.tb_ptr);
144 
145 	if (t->trw_data.tb_ptr == t->trw_data.tb_buf)
146 		t->trw_data.tb_ptr = t->trw_data.tb_end;
147 	else
148 		t->trw_data.tb_ptr = (fmd_tracerec_t *)
149 		    ((uintptr_t)t->trw_data.tb_ptr - t->trw_data.tb_size);
150 
151 	if (t->trw_data.tb_ptr == t->trw_stop)
152 		t->trw_stop = NULL; /* mark buffer as empty */
153 
154 	return (t->trw_data.tb_ptr);
155 }
156 
157 static int
158 trwalk_step(mdb_walk_state_t *wsp)
159 {
160 	trwalk_state_t *t, *oldest_t;
161 	hrtime_t oldest_time = 0;
162 	fmd_tracerec_t *trp;
163 	int status;
164 
165 	for (t = wsp->walk_data; t != NULL; t = t->trw_next) {
166 		for (trp = t->trw_data.tb_ptr; t->trw_stop != NULL &&
167 		    trp->tr_time == 0; trp = trwalk_nextrec(t))
168 			continue;
169 
170 		if (t->trw_stop == NULL)
171 			continue; /* buffer has been emptied */
172 
173 		if (trp->tr_time > oldest_time) {
174 			oldest_time = trp->tr_time;
175 			oldest_t = t;
176 		}
177 	}
178 
179 	if (oldest_time == 0)
180 		return (WALK_DONE);
181 
182 	t = oldest_t;
183 	trp = t->trw_data.tb_ptr;
184 
185 	bcopy(trp, t->trw_xrec, t->trw_data.tb_size);
186 	t->trw_xrec->tr_depth = MIN(trp->tr_depth, t->trw_data.tb_frames);
187 	t->trw_xrec->tr_stack[t->trw_xrec->tr_depth] = t->trw_tid;
188 
189 	status = wsp->walk_callback((uintptr_t)trp - (uintptr_t)
190 	    t->trw_data.tb_buf + t->trw_base, t->trw_xrec, wsp->walk_cbdata);
191 
192 	(void) trwalk_nextrec(t);
193 	return (status);
194 }
195 
196 static void
197 trwalk_fini(mdb_walk_state_t *wsp)
198 {
199 	trwalk_state_t *t, *u;
200 
201 	for (t = wsp->walk_data; t != NULL; t = u) {
202 		u = t->trw_next;
203 		mdb_free(t->trw_data.tb_buf,
204 		    t->trw_data.tb_recs * t->trw_data.tb_size);
205 		mdb_free(t->trw_xrec, t->trw_data.tb_size + sizeof (uintptr_t));
206 		mdb_free(t, sizeof (trwalk_state_t));
207 	}
208 }
209 
210 /*ARGSUSED*/
211 static int
212 trprint_msg(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
213 {
214 	if (tid == 0)
215 		mdb_printf("%3lu ", trp->tr_stack[trp->tr_depth]);
216 	else if (trp->tr_stack[trp->tr_depth] != tid)
217 		return (WALK_NEXT);
218 
219 	mdb_printf("%016llx %04x %-5u %s\n",
220 	    trp->tr_time, 1 << trp->tr_tag, trp->tr_errno, trp->tr_msg);
221 
222 	return (WALK_NEXT);
223 }
224 
225 /*ARGSUSED*/
226 static int
227 trprint_cpp(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
228 {
229 	char file[64];
230 
231 	if (tid == 0)
232 		mdb_printf("%3lu ", trp->tr_stack[trp->tr_depth]);
233 	else if (trp->tr_stack[trp->tr_depth] != tid)
234 		return (WALK_NEXT);
235 
236 	if (mdb_readstr(file, sizeof (file), (uintptr_t)trp->tr_file) <= 0)
237 		(void) strcpy(file, "???");
238 
239 	mdb_printf("%016llx %04x %s: %u\n",
240 	    trp->tr_time, 1 << trp->tr_tag, file, trp->tr_line);
241 
242 	return (WALK_NEXT);
243 }
244 
245 static void
246 trprint_stack(const fmd_tracerec_t *trp)
247 {
248 	uint8_t i;
249 
250 	for (i = 0; i < trp->tr_depth; i++)
251 		mdb_printf("\t%a\n", trp->tr_stack[i]);
252 
253 	if (trp->tr_depth != 0)
254 		mdb_printf("\n");
255 }
256 
257 static int
258 trprint_msg_stack(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
259 {
260 	int status = trprint_msg(addr, trp, tid);
261 	trprint_stack(trp);
262 	return (status);
263 }
264 
265 static int
266 trprint_cpp_stack(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
267 {
268 	int status = trprint_cpp(addr, trp, tid);
269 	trprint_stack(trp);
270 	return (status);
271 }
272 
273 static int
274 fmd_trace(uintptr_t tid, uint_t flags, int argc, const mdb_arg_t *argv)
275 {
276 	int (*func)(uintptr_t, const fmd_tracerec_t *, uintptr_t);
277 	uint_t opt_c = FALSE, opt_s = FALSE;
278 
279 	if (mdb_getopts(argc, argv,
280 	    'c', MDB_OPT_SETBITS, TRUE, &opt_c,
281 	    's', MDB_OPT_SETBITS, TRUE, &opt_s, NULL) != argc)
282 		return (DCMD_USAGE);
283 
284 	if (!(flags & DCMD_ADDRSPEC)) {
285 		mdb_printf("TID ");
286 		tid = 0;
287 	}
288 
289 	if (opt_c) {
290 		mdb_printf("%-16s %-4s FILE:LINE\n", "TIME", "TAG");
291 		func = opt_s ? trprint_cpp_stack : trprint_cpp;
292 	} else {
293 		mdb_printf("%-16s %-4s %-5s MSG\n", "TIME", "TAG", "ERRNO");
294 		func = opt_s ? trprint_msg_stack : trprint_msg;
295 	}
296 
297 	if (mdb_walk("fmd_trace", (mdb_walk_cb_t)func, (void *)tid) == -1) {
298 		mdb_warn("failed to walk fmd_trace");
299 		return (DCMD_ERR);
300 	}
301 
302 	return (DCMD_OK);
303 }
304 
305 static int
306 hash_walk_init(mdb_walk_state_t *wsp, uintptr_t addr, uint_t hashlen,
307     const char *name, size_t size, size_t next)
308 {
309 	hashwalk_data_t *hwp;
310 	size_t len = sizeof (uintptr_t) * hashlen;
311 
312 	if (len == 0) {
313 		mdb_warn("failed to walk hash: invalid hash length\n");
314 		return (WALK_ERR);
315 	}
316 
317 	hwp = mdb_alloc(sizeof (hashwalk_data_t), UM_SLEEP);
318 	hwp->hw_hash = mdb_zalloc(len, UM_SLEEP);
319 	(void) mdb_vread(hwp->hw_hash, len, addr);
320 	hwp->hw_hashlen = hashlen;
321 	hwp->hw_hashidx = 0;
322 	hwp->hw_name = name;
323 	hwp->hw_data = mdb_zalloc(size, UM_SLEEP);
324 	hwp->hw_size = size;
325 	hwp->hw_next = next;
326 
327 	wsp->walk_addr = hwp->hw_hash[0];
328 	wsp->walk_data = hwp;
329 
330 	return (WALK_NEXT);
331 }
332 
333 static int
334 hash_walk_step(mdb_walk_state_t *wsp)
335 {
336 	hashwalk_data_t *hwp = wsp->walk_data;
337 	int rv;
338 
339 	while (wsp->walk_addr == NULL) {
340 		if (++hwp->hw_hashidx < hwp->hw_hashlen)
341 			wsp->walk_addr = hwp->hw_hash[hwp->hw_hashidx];
342 		else
343 			return (WALK_DONE);
344 	}
345 
346 	if (mdb_vread(hwp->hw_data, hwp->hw_size, wsp->walk_addr) == -1) {
347 		mdb_warn("failed to read %s at %p",
348 		    hwp->hw_name, wsp->walk_addr);
349 		return (WALK_ERR);
350 	}
351 
352 	rv = wsp->walk_callback(wsp->walk_addr, hwp->hw_data, wsp->walk_cbdata);
353 	wsp->walk_addr = *(uintptr_t *)((uintptr_t)hwp->hw_data + hwp->hw_next);
354 	return (rv);
355 }
356 
357 static void
358 hash_walk_fini(mdb_walk_state_t *wsp)
359 {
360 	hashwalk_data_t *hwp = wsp->walk_data;
361 
362 	mdb_free(hwp->hw_hash, sizeof (uintptr_t) * hwp->hw_hashlen);
363 	mdb_free(hwp->hw_data, hwp->hw_size);
364 	mdb_free(hwp, sizeof (hashwalk_data_t));
365 }
366 
367 static int
368 ustat_walk_init(mdb_walk_state_t *wsp)
369 {
370 	fmd_ustat_t us;
371 
372 	if (mdb_vread(&us, sizeof (us), wsp->walk_addr) != sizeof (us)) {
373 		mdb_warn("failed to read fmd_ustat_t at %p", wsp->walk_addr);
374 		return (WALK_ERR);
375 	}
376 
377 	return (hash_walk_init(wsp,
378 	    (uintptr_t)us.us_hash, us.us_hashlen, NULL, 0, 0));
379 }
380 
381 static int
382 ustat_walk_step(mdb_walk_state_t *wsp)
383 {
384 	hashwalk_data_t *hwp = wsp->walk_data;
385 	fmd_ustat_elem_t ue;
386 	fmd_stat_t s;
387 
388 	while (wsp->walk_addr == NULL) {
389 		if (++hwp->hw_hashidx < hwp->hw_hashlen)
390 			wsp->walk_addr = hwp->hw_hash[hwp->hw_hashidx];
391 		else
392 			return (WALK_DONE);
393 	}
394 
395 	if (mdb_vread(&ue, sizeof (ue), wsp->walk_addr) != sizeof (ue) ||
396 	    mdb_vread(&s, sizeof (s), (uintptr_t)ue.use_stat) != sizeof (s)) {
397 		mdb_warn("failed to read stat element at %p", wsp->walk_addr);
398 		return (WALK_ERR);
399 	}
400 
401 	wsp->walk_addr = (uintptr_t)ue.use_next;
402 
403 	return (wsp->walk_callback(
404 	    (uintptr_t)ue.use_stat, &s, wsp->walk_cbdata));
405 }
406 
407 struct fmd_cmd_data {
408 	int argc;
409 	const mdb_arg_t *argv;
410 };
411 
412 /* ARGSUSED */
413 static int
414 module_ustat(uintptr_t addr, const void *data, void *wsp)
415 {
416 	fmd_module_t *modp = (fmd_module_t *)data;
417 	char name[PATH_MAX];
418 	const struct fmd_cmd_data *udp = wsp;
419 
420 	if (mdb_readstr(name, sizeof (name), (uintptr_t)modp->mod_name) <= 0)
421 		(void) mdb_snprintf(name, sizeof (name), "<%p>",
422 		    modp->mod_name);
423 	mdb_printf("%s\n", name);
424 	(void) fmd_ustat((uintptr_t)modp->mod_ustat,
425 	    DCMD_ADDRSPEC | DCMD_LOOPFIRST, udp->argc, udp->argv);
426 	return (WALK_NEXT);
427 }
428 
429 static int
430 fmd_ustat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
431 {
432 	if (!(flags & DCMD_ADDRSPEC)) {
433 		struct fmd_cmd_data ud;
434 
435 		ud.argc = argc;
436 		ud.argv = argv;
437 		if (mdb_walk("fmd_module", module_ustat, &ud) == -1) {
438 			mdb_warn("failed to walk 'fmd_module'");
439 			return (DCMD_ERR);
440 		}
441 		return (DCMD_OK);
442 	}
443 
444 	if (mdb_pwalk_dcmd("fmd_ustat", "fmd_stat", argc, argv, addr) != 0) {
445 		mdb_warn("failed to walk fmd_ustat at %p", addr);
446 		return (DCMD_ERR);
447 	}
448 
449 	return (DCMD_OK);
450 }
451 
452 /* ARGSUSED */
453 static int
454 module_stat(uintptr_t addr, const void *data, void *wsp)
455 {
456 	fmd_module_t *modp = (fmd_module_t *)data;
457 	char name[PATH_MAX];
458 	const struct fmd_cmd_data *udp = wsp;
459 	fmd_modstat_t *mod_stats;
460 
461 	if (mdb_readstr(name, sizeof (name), (uintptr_t)modp->mod_name) <= 0) {
462 		(void) mdb_snprintf(name, sizeof (name), "<%p>",
463 		    modp->mod_name);
464 	}
465 	mdb_printf("%s\n", name);
466 	mod_stats = modp->mod_stats;
467 	(void) fmd_stat((uintptr_t)&mod_stats->ms_loadtime,
468 	    DCMD_ADDRSPEC | DCMD_LOOPFIRST, udp->argc, udp->argv);
469 	(void) fmd_stat((uintptr_t)&mod_stats->ms_snaptime,
470 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
471 	(void) fmd_stat((uintptr_t)&mod_stats->ms_accepted,
472 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
473 	(void) fmd_stat((uintptr_t)&mod_stats->ms_debugdrop,
474 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
475 	(void) fmd_stat((uintptr_t)&mod_stats->ms_memtotal,
476 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
477 	(void) fmd_stat((uintptr_t)&mod_stats->ms_memlimit,
478 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
479 	(void) fmd_stat((uintptr_t)&mod_stats->ms_buftotal,
480 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
481 	(void) fmd_stat((uintptr_t)&mod_stats->ms_buflimit,
482 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
483 	(void) fmd_stat((uintptr_t)&mod_stats->ms_thrtotal,
484 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
485 	(void) fmd_stat((uintptr_t)&mod_stats->ms_thrlimit,
486 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
487 	(void) fmd_stat((uintptr_t)&mod_stats->ms_caseopen,
488 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
489 	(void) fmd_stat((uintptr_t)&mod_stats->ms_casesolved,
490 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
491 	(void) fmd_stat((uintptr_t)&mod_stats->ms_caseclosed,
492 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
493 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_save,
494 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
495 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_restore,
496 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
497 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_zeroed,
498 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
499 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_cnt,
500 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
501 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_time,
502 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
503 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtopen,
504 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
505 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtlimit,
506 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
507 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtqlimit,
508 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
509 	return (WALK_NEXT);
510 }
511 
512 /*ARGSUSED*/
513 static int
514 fmd_stat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
515 {
516 	char buf[512];
517 	fmd_stat_t s;
518 
519 	if (argc != 0)
520 		return (DCMD_USAGE);
521 
522 	if (DCMD_HDRSPEC(flags))
523 		mdb_printf("%<u>%-11s %-4s %-32s %s%</u>\n",
524 		    "ADDR", "TYPE", "NAME", "VALUE");
525 
526 	if (!(flags & DCMD_ADDRSPEC)) {
527 		struct fmd_cmd_data ud;
528 
529 		ud.argc = argc;
530 		ud.argv = argv;
531 
532 		if (mdb_walk("fmd_module", module_stat, &ud) == -1) {
533 			mdb_warn("failed to walk 'fmd_module'");
534 			return (DCMD_ERR);
535 		}
536 		return (DCMD_OK);
537 	}
538 
539 	if (mdb_vread(&s, sizeof (s), addr) != sizeof (s)) {
540 		mdb_warn("failed to read statistic at %p", addr);
541 		return (DCMD_ERR);
542 	}
543 
544 	switch (s.fmds_type) {
545 	case FMD_TYPE_BOOL:
546 		mdb_printf("%-11p %-4s %-32s %s\n", addr, "bool",
547 		    s.fmds_name, s.fmds_value.bool ? "true" : "false");
548 		break;
549 	case FMD_TYPE_INT32:
550 		mdb_printf("%-11p %-4s %-32s %d\n", addr, "i32",
551 		    s.fmds_name, s.fmds_value.i32);
552 		break;
553 	case FMD_TYPE_UINT32:
554 		mdb_printf("%-11p %-4s %-32s %u\n", addr, "ui32",
555 		    s.fmds_name, s.fmds_value.i32);
556 		break;
557 	case FMD_TYPE_INT64:
558 		mdb_printf("%-11p %-4s %-32s %lld\n", addr, "i64",
559 		    s.fmds_name, s.fmds_value.i64);
560 		break;
561 	case FMD_TYPE_UINT64:
562 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "ui64",
563 		    s.fmds_name, s.fmds_value.ui64);
564 		break;
565 	case FMD_TYPE_STRING:
566 		if (mdb_readstr(buf, sizeof (buf),
567 		    (uintptr_t)s.fmds_value.str) < 0) {
568 			(void) mdb_snprintf(buf, sizeof (buf), "<%p>",
569 			    s.fmds_value.str);
570 		}
571 		mdb_printf("%-11p %-4s %-32s %s\n", addr, "str",
572 		    s.fmds_name, buf);
573 		break;
574 	case FMD_TYPE_TIME:
575 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "time",
576 		    s.fmds_name, s.fmds_value.ui64);
577 		break;
578 	case FMD_TYPE_SIZE:
579 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "size",
580 		    s.fmds_name, s.fmds_value.ui64);
581 		break;
582 	default:
583 		mdb_printf("%-11p %-4u %-32s ???\n", addr,
584 		    s.fmds_type, s.fmds_name);
585 		break;
586 	}
587 
588 	return (DCMD_OK);
589 }
590 
591 /*ARGSUSED*/
592 static int
593 fmd_event(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
594 {
595 	char type[16], name[16];
596 	fmd_event_impl_t ev;
597 
598 	if (argc != 0)
599 		return (DCMD_USAGE);
600 
601 	if (mdb_vread(&ev, sizeof (ev), addr) != sizeof (ev)) {
602 		mdb_warn("failed to read fmd_event at %p", addr);
603 		return (DCMD_ERR);
604 	}
605 
606 	if (DCMD_HDRSPEC(flags)) {
607 		mdb_printf("%<u>%-11s %-4s %-5s %-3s %-?s%</u>\n",
608 		    "ADDR", "TYPE", "STATE", "REF", "NVPAIR");
609 	}
610 
611 	switch (ev.ev_type) {
612 	case FMD_EVT_PROTOCOL:
613 		(void) strcpy(type, "PROT");
614 		break;
615 	case FMD_EVT_GC:
616 		(void) strcpy(type, "GC");
617 		break;
618 	case FMD_EVT_CLOSE:
619 		(void) strcpy(type, "CLSE");
620 		break;
621 	case FMD_EVT_TIMEOUT:
622 		(void) strcpy(type, "TIME");
623 		break;
624 	case FMD_EVT_STATS:
625 		(void) strcpy(type, "STAT");
626 		break;
627 	case FMD_EVT_PUBLISH:
628 		(void) strcpy(type, "PUBL");
629 		break;
630 	case FMD_EVT_TOPO:
631 		(void) strcpy(type, "TOPO");
632 		break;
633 	default:
634 		(void) mdb_snprintf(type, sizeof (type), "%u", ev.ev_type);
635 	}
636 
637 	switch (ev.ev_state) {
638 	case FMD_EVS_RECEIVED:
639 		(void) strcpy(name, "RECVD");
640 		break;
641 	case FMD_EVS_ACCEPTED:
642 		(void) strcpy(name, "ACCPT");
643 		break;
644 	case FMD_EVS_DISCARDED:
645 		(void) strcpy(name, "DSCRD");
646 		break;
647 	case FMD_EVS_DIAGNOSED:
648 		(void) strcpy(name, "DIAGN");
649 		break;
650 	default:
651 		(void) mdb_snprintf(name, sizeof (name), "%u", ev.ev_state);
652 	}
653 
654 	mdb_printf("%-11p %-4s %-5s %-3u %p\n",
655 	    addr, type, name, ev.ev_refs, ev.ev_nvl);
656 
657 	return (DCMD_OK);
658 }
659 
660 static int
661 thread_walk_init(mdb_walk_state_t *wsp)
662 {
663 	fmd_t F;
664 
665 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
666 		mdb_warn("failed to read fmd meta-data");
667 		return (WALK_ERR);
668 	}
669 
670 	wsp->walk_addr = (uintptr_t)F.d_thr_list.l_next;
671 	return (WALK_NEXT);
672 }
673 
674 static int
675 thread_walk_step(mdb_walk_state_t *wsp)
676 {
677 	uintptr_t addr = wsp->walk_addr;
678 	fmd_thread_t t;
679 
680 	if (addr == NULL)
681 		return (WALK_DONE);
682 
683 	if (mdb_vread(&t, sizeof (t), addr) != sizeof (t)) {
684 		mdb_warn("failed to read fmd_thread at %p", addr);
685 		return (WALK_ERR);
686 	}
687 
688 	wsp->walk_addr = (uintptr_t)t.thr_list.l_next;
689 	return (wsp->walk_callback(addr, &t, wsp->walk_cbdata));
690 }
691 
692 static int
693 fmd_thread(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
694 {
695 	fmd_thread_t thr;
696 
697 	if (!(flags & DCMD_ADDRSPEC))
698 		return (mdb_walk_dcmd("fmd_thread", "fmd_thread", argc, argv));
699 
700 	if (argc != 0)
701 		return (DCMD_USAGE);
702 
703 	if (mdb_vread(&thr, sizeof (thr), addr) != sizeof (thr)) {
704 		mdb_warn("failed to read fmd_thread at %p", addr);
705 		return (DCMD_ERR);
706 	}
707 
708 	if (DCMD_HDRSPEC(flags)) {
709 		mdb_printf("%<u>%-11s %-11s %-8s %-16s%</u>\n",
710 		    "ADDR", "MOD", "TID", "FUNC");
711 	}
712 
713 	mdb_printf("%-11p %-11p %-8u %a\n",
714 	    addr, thr.thr_mod, thr.thr_tid, thr.thr_func);
715 
716 	return (DCMD_OK);
717 }
718 
719 static int
720 mod_walk_init(mdb_walk_state_t *wsp)
721 {
722 	fmd_t F;
723 
724 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
725 		mdb_warn("failed to read fmd meta-data");
726 		return (WALK_ERR);
727 	}
728 
729 	wsp->walk_addr = (uintptr_t)F.d_mod_list.l_next;
730 	return (WALK_NEXT);
731 }
732 
733 static int
734 mod_walk_step(mdb_walk_state_t *wsp)
735 {
736 	uintptr_t addr = wsp->walk_addr;
737 	fmd_module_t m;
738 
739 	if (addr == NULL)
740 		return (WALK_DONE);
741 
742 	if (mdb_vread(&m, sizeof (m), addr) != sizeof (m)) {
743 		mdb_warn("failed to read fmd_module at %p", addr);
744 		return (WALK_ERR);
745 	}
746 
747 	wsp->walk_addr = (uintptr_t)m.mod_list.l_next;
748 	return (wsp->walk_callback(addr, &m, wsp->walk_cbdata));
749 }
750 
751 static int
752 fmd_module(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
753 {
754 	fmd_module_t mod;
755 	char name[PATH_MAX];
756 
757 	if (!(flags & DCMD_ADDRSPEC))
758 		return (mdb_walk_dcmd("fmd_module", "fmd_module", argc, argv));
759 
760 	if (argc != 0)
761 		return (DCMD_USAGE);
762 
763 	if (mdb_vread(&mod, sizeof (mod), addr) != sizeof (mod)) {
764 		mdb_warn("failed to read fmd_module at %p", addr);
765 		return (DCMD_ERR);
766 	}
767 
768 	if (DCMD_HDRSPEC(flags)) {
769 		mdb_printf("%<u>%-11s %-16s %-11s %-4s %-?s %-16s%</u>\n",
770 		    "ADDR", "OPS", "DATA", "FLAG", "USTAT", "NAME");
771 	}
772 
773 	if (mdb_readstr(name, sizeof (name), (uintptr_t)mod.mod_name) <= 0)
774 		(void) mdb_snprintf(name, sizeof (name), "<%p>", mod.mod_name);
775 
776 	mdb_printf("%-11p %-16a %-11p 0x%02x %-?p %s\n", addr,
777 	    mod.mod_ops, mod.mod_data, mod.mod_flags, mod.mod_ustat, name);
778 
779 	return (DCMD_OK);
780 }
781 
782 static int
783 case_walk_init(mdb_walk_state_t *wsp)
784 {
785 	fmd_module_t mod;
786 	fmd_case_hash_t ch;
787 	fmd_t F;
788 
789 	if (wsp->walk_addr != NULL) {
790 		if (mdb_vread(&mod, sizeof (mod), wsp->walk_addr) == -1) {
791 			mdb_warn("failed to read module at %p", wsp->walk_addr);
792 			return (WALK_ERR);
793 		}
794 
795 		wsp->walk_addr = (uintptr_t)mod.mod_cases.l_next;
796 		return (WALK_NEXT);
797 	}
798 
799 	if (mdb_readvar(&F, "fmd") != sizeof (F) ||
800 	    mdb_vread(&ch, sizeof (ch), (uintptr_t)F.d_cases) != sizeof (ch)) {
801 		mdb_warn("failed to read fmd meta-data");
802 		return (WALK_ERR);
803 	}
804 
805 	return (hash_walk_init(wsp, (uintptr_t)ch.ch_hash, ch.ch_hashlen,
806 	    "fmd_case", sizeof (fmd_case_impl_t),
807 	    OFFSETOF(fmd_case_impl_t, ci_next)));
808 }
809 
810 static int
811 case_walk_step(mdb_walk_state_t *wsp)
812 {
813 	uintptr_t addr = wsp->walk_addr;
814 	fmd_case_impl_t ci;
815 
816 	if (wsp->walk_data != NULL)
817 		return (hash_walk_step(wsp));
818 
819 	if (addr == NULL)
820 		return (WALK_DONE);
821 
822 	if (mdb_vread(&ci, sizeof (ci), addr) != sizeof (ci)) {
823 		mdb_warn("failed to read fmd_case at %p", addr);
824 		return (WALK_ERR);
825 	}
826 
827 	wsp->walk_addr = (uintptr_t)ci.ci_list.l_next;
828 	return (wsp->walk_callback(addr, &ci, wsp->walk_cbdata));
829 }
830 
831 static void
832 case_walk_fini(mdb_walk_state_t *wsp)
833 {
834 	if (wsp->walk_data != NULL)
835 		hash_walk_fini(wsp);
836 }
837 
838 static int
839 fmd_case(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
840 {
841 	char uuid[48], name[16];
842 	fmd_case_impl_t ci;
843 
844 	if (!(flags & DCMD_ADDRSPEC)) {
845 		if (mdb_walk_dcmd("fmd_case", "fmd_case", argc, argv) != 0) {
846 			mdb_warn("failed to walk fmd_case hash");
847 			return (DCMD_ERR);
848 		}
849 		return (DCMD_OK);
850 	}
851 
852 	if (mdb_vread(&ci, sizeof (ci), addr) != sizeof (ci)) {
853 		mdb_warn("failed to read fmd_case at %p", addr);
854 		return (DCMD_ERR);
855 	}
856 
857 	if (DCMD_HDRSPEC(flags)) {
858 		mdb_printf("%<u>%-11s %-5s %-3s %-?s %-36s%</u>\n",
859 		    "ADDR", "STATE", "REF", "DATA", "UUID");
860 	}
861 
862 	if (mdb_readstr(uuid, sizeof (uuid), (uintptr_t)ci.ci_uuid) <= 0)
863 		(void) mdb_snprintf(uuid, sizeof (uuid), "<%p>", ci.ci_uuid);
864 
865 	switch (ci.ci_state) {
866 	case FMD_CASE_UNSOLVED:
867 		(void) strcpy(name, "UNSLV");
868 		break;
869 	case FMD_CASE_SOLVED:
870 		(void) strcpy(name, "SOLVE");
871 		break;
872 	case FMD_CASE_CLOSE_WAIT:
873 		(void) strcpy(name, "CWAIT");
874 		break;
875 	case FMD_CASE_CLOSED:
876 		(void) strcpy(name, "CLOSE");
877 		break;
878 	case FMD_CASE_REPAIRED:
879 		(void) strcpy(name, "RPAIR");
880 		break;
881 	default:
882 		(void) mdb_snprintf(name, sizeof (name), "%u", ci.ci_state);
883 	}
884 
885 	mdb_printf("%-11p %-5s %-3u %-?p %s\n",
886 	    addr, name, ci.ci_refs, ci.ci_data, uuid);
887 
888 	return (DCMD_OK);
889 }
890 
891 static int
892 buf_walk_init(mdb_walk_state_t *wsp)
893 {
894 	fmd_buf_hash_t bh;
895 
896 	if (mdb_vread(&bh, sizeof (bh), wsp->walk_addr) != sizeof (bh)) {
897 		mdb_warn("failed to read fmd_buf_hash_t at %p", wsp->walk_addr);
898 		return (WALK_ERR);
899 	}
900 
901 	return (hash_walk_init(wsp, (uintptr_t)bh.bh_hash, bh.bh_hashlen,
902 	    "fmd_buf", sizeof (fmd_buf_t), OFFSETOF(fmd_buf_t, buf_next)));
903 }
904 
905 /*ARGSUSED*/
906 static int
907 fmd_buf(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
908 {
909 	char name[PATH_MAX];
910 	fmd_buf_t b;
911 
912 	if (argc != 0 || !(flags & DCMD_ADDRSPEC))
913 		return (DCMD_USAGE);
914 
915 	if (mdb_vread(&b, sizeof (b), addr) != sizeof (b)) {
916 		mdb_warn("failed to read fmd_buf at %p", addr);
917 		return (DCMD_ERR);
918 	}
919 
920 	if (DCMD_HDRSPEC(flags)) {
921 		mdb_printf("%<u>%-11s %-32s %-5s %-?s %s%</u>\n",
922 		    "ADDR", "NAME", "FLAGS", "DATA", "SIZE");
923 	}
924 
925 	if (mdb_readstr(name, sizeof (name), (uintptr_t)b.buf_name) <= 0)
926 		(void) mdb_snprintf(name, sizeof (name), "<%p>", b.buf_name);
927 
928 	mdb_printf("%-11p %-32s %-#5x %-?p %lu\n",
929 	    addr, name, b.buf_flags, b.buf_data, b.buf_size);
930 
931 	return (DCMD_OK);
932 }
933 
934 static int
935 serd_walk_init(mdb_walk_state_t *wsp)
936 {
937 	fmd_serd_hash_t sh;
938 
939 	if (mdb_vread(&sh, sizeof (sh), wsp->walk_addr) != sizeof (sh)) {
940 		mdb_warn("failed to read fmd_serd_hash at %p", wsp->walk_addr);
941 		return (WALK_ERR);
942 	}
943 
944 	return (hash_walk_init(wsp, (uintptr_t)sh.sh_hash, sh.sh_hashlen,
945 	    "fmd_serd_eng", sizeof (fmd_serd_eng_t),
946 	    OFFSETOF(fmd_serd_eng_t, sg_next)));
947 }
948 
949 /* ARGSUSED */
950 static int
951 module_serd(uintptr_t addr, const void *data, void *wsp)
952 {
953 	fmd_module_t *modp = (fmd_module_t *)data;
954 
955 	if (modp->mod_serds.sh_count != 0) {
956 		modp = (fmd_module_t *)addr;
957 		(void) mdb_pwalk_dcmd("fmd_serd", "fmd_serd", 0, 0,
958 		    (uintptr_t)&modp->mod_serds);
959 	}
960 	return (WALK_NEXT);
961 }
962 
963 /*ARGSUSED*/
964 static int
965 fmd_serd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
966 {
967 	char name[PATH_MAX];
968 	fmd_serd_eng_t sg;
969 
970 	if (argc != 0)
971 		return (DCMD_USAGE);
972 	if (!(flags & DCMD_ADDRSPEC)) {
973 		if (mdb_walk("fmd_module", module_serd, 0) == -1) {
974 			mdb_warn("failed to walk 'fmd_module'");
975 			return (DCMD_ERR);
976 		}
977 		return (DCMD_OK);
978 	}
979 
980 	if (mdb_vread(&sg, sizeof (sg), addr) != sizeof (sg)) {
981 		mdb_warn("failed to read fmd_serd_eng at %p", addr);
982 		return (DCMD_ERR);
983 	}
984 
985 	if (DCMD_HDRSPEC(flags)) {
986 		mdb_printf("%<u>%-11s %-32s %-3s F >%-2s %-16s%</u>\n",
987 		    "ADDR", "NAME", "CNT", "N", "T");
988 	}
989 
990 	if (mdb_readstr(name, sizeof (name), (uintptr_t)sg.sg_name) <= 0)
991 		(void) mdb_snprintf(name, sizeof (name), "<%p>", sg.sg_name);
992 
993 	mdb_printf("%-11p %-32s %-3u %c >%-2u %lluns\n",
994 	    addr, name, sg.sg_count, (sg.sg_flags & FMD_SERD_FIRED) ? 'F' : ' ',
995 	    sg.sg_n, (u_longlong_t)sg.sg_t);
996 
997 	return (DCMD_OK);
998 }
999 
1000 static int
1001 asru_walk_init(mdb_walk_state_t *wsp)
1002 {
1003 	fmd_asru_hash_t ah;
1004 	fmd_t F;
1005 
1006 	if (wsp->walk_addr == NULL && mdb_readvar(&F, "fmd") != sizeof (F)) {
1007 		mdb_warn("failed to read fmd meta-data");
1008 		return (WALK_ERR);
1009 	}
1010 
1011 	if (wsp->walk_addr == NULL)
1012 		wsp->walk_addr = (uintptr_t)F.d_asrus;
1013 
1014 	if (mdb_vread(&ah, sizeof (ah), wsp->walk_addr) != sizeof (ah)) {
1015 		mdb_warn("failed to read asru_hash at %p", wsp->walk_addr);
1016 		return (WALK_ERR);
1017 	}
1018 
1019 	return (hash_walk_init(wsp, (uintptr_t)ah.ah_hash, ah.ah_hashlen,
1020 	    "fmd_asru", sizeof (fmd_asru_t), OFFSETOF(fmd_asru_t, asru_next)));
1021 }
1022 
1023 static int
1024 fmd_asru(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1025 {
1026 	char uuid[48], name[PATH_MAX];
1027 	fmd_asru_t a;
1028 
1029 	if (!(flags & DCMD_ADDRSPEC)) {
1030 		if (mdb_walk_dcmd("fmd_asru", "fmd_asru", argc, argv) != 0) {
1031 			mdb_warn("failed to walk fmd_asru hash");
1032 			return (DCMD_ERR);
1033 		}
1034 		return (DCMD_OK);
1035 	}
1036 
1037 	if (mdb_vread(&a, sizeof (a), addr) != sizeof (a)) {
1038 		mdb_warn("failed to read fmd_asru at %p", addr);
1039 		return (DCMD_ERR);
1040 	}
1041 
1042 	if (DCMD_HDRSPEC(flags))
1043 		mdb_printf("%<u>%-8s %-36s %s%</u>\n", "ADDR", "UUID", "NAME");
1044 
1045 	if (mdb_readstr(uuid, sizeof (uuid), (uintptr_t)a.asru_uuid) <= 0)
1046 		(void) mdb_snprintf(uuid, sizeof (uuid), "<%p>", a.asru_uuid);
1047 	if (mdb_readstr(name, sizeof (name), (uintptr_t)a.asru_name) <= 0)
1048 		(void) mdb_snprintf(name, sizeof (name), "<%p>", a.asru_name);
1049 
1050 	mdb_printf("%-8p %-36s %s\n", addr, uuid, name);
1051 	return (DCMD_OK);
1052 }
1053 
1054 static int
1055 al_walk_init(mdb_walk_state_t *wsp)
1056 {
1057 	fmd_asru_hash_t ah;
1058 	fmd_t F;
1059 
1060 	if (wsp->walk_addr == NULL && mdb_readvar(&F, "fmd") != sizeof (F)) {
1061 		mdb_warn("failed to read fmd meta-data");
1062 		return (WALK_ERR);
1063 	}
1064 
1065 	if (wsp->walk_addr == NULL)
1066 		wsp->walk_addr = (uintptr_t)F.d_asrus;
1067 
1068 	if (mdb_vread(&ah, sizeof (ah), wsp->walk_addr) != sizeof (ah)) {
1069 		mdb_warn("failed to read asru_hash at %p", wsp->walk_addr);
1070 		return (WALK_ERR);
1071 	}
1072 
1073 	return (hash_walk_init(wsp, (uintptr_t)ah.ah_rsrc_hash, ah.ah_hashlen,
1074 	    "fmd_asru_link", sizeof (fmd_asru_link_t), OFFSETOF(fmd_asru_link_t,
1075 	    al_rsrc_next)));
1076 }
1077 
1078 static int
1079 fmd_asru_link(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1080 {
1081 	char uuid[48], name[PATH_MAX];
1082 	fmd_asru_link_t a;
1083 
1084 	if (!(flags & DCMD_ADDRSPEC)) {
1085 		if (mdb_walk_dcmd("fmd_asru_link", "fmd_asru_link", argc,
1086 		    argv) != 0) {
1087 			mdb_warn("failed to walk fmd_asru_link hash");
1088 			return (DCMD_ERR);
1089 		}
1090 		return (DCMD_OK);
1091 	}
1092 
1093 	if (mdb_vread(&a, sizeof (a), addr) != sizeof (a)) {
1094 		mdb_warn("failed to read fmd_asru_link at %p", addr);
1095 		return (DCMD_ERR);
1096 	}
1097 
1098 	if (DCMD_HDRSPEC(flags))
1099 		mdb_printf("%<u>%-8s %-36s %s%</u>\n", "ADDR", "UUID", "NAME");
1100 
1101 	if (mdb_readstr(uuid, sizeof (uuid), (uintptr_t)a.al_uuid) <= 0)
1102 		(void) mdb_snprintf(uuid, sizeof (uuid), "<%p>", a.al_uuid);
1103 	if (mdb_readstr(name, sizeof (name), (uintptr_t)a.al_rsrc_name) <= 0)
1104 		(void) mdb_snprintf(name, sizeof (name), "<%p>",
1105 		    a.al_rsrc_name);
1106 
1107 	mdb_printf("%-8p %-36s %s\n", addr, uuid, name);
1108 	return (DCMD_OK);
1109 }
1110 
1111 /*ARGSUSED*/
1112 static int
1113 fcf_hdr(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1114 {
1115 	fcf_hdr_t h;
1116 
1117 	if (argc != 0)
1118 		return (DCMD_USAGE);
1119 
1120 	if (!(flags & DCMD_ADDRSPEC))
1121 		addr = 0; /* assume base of file in file target */
1122 
1123 	if (mdb_vread(&h, sizeof (h), addr) != sizeof (h)) {
1124 		mdb_warn("failed to read header at %p", addr);
1125 		return (DCMD_ERR);
1126 	}
1127 
1128 	mdb_printf("fcfh_ident.id_magic = 0x%x, %c, %c, %c\n",
1129 	    h.fcfh_ident[FCF_ID_MAG0], h.fcfh_ident[FCF_ID_MAG1],
1130 	    h.fcfh_ident[FCF_ID_MAG2], h.fcfh_ident[FCF_ID_MAG3]);
1131 
1132 	switch (h.fcfh_ident[FCF_ID_MODEL]) {
1133 	case FCF_MODEL_ILP32:
1134 		mdb_printf("fcfh_ident.id_model = ILP32\n");
1135 		break;
1136 	case FCF_MODEL_LP64:
1137 		mdb_printf("fcfh_ident.id_model = LP64\n");
1138 		break;
1139 	default:
1140 		mdb_printf("fcfh_ident.id_model = 0x%x\n",
1141 		    h.fcfh_ident[FCF_ID_MODEL]);
1142 	}
1143 
1144 	switch (h.fcfh_ident[FCF_ID_ENCODING]) {
1145 	case FCF_ENCODE_LSB:
1146 		mdb_printf("fcfh_ident.id_encoding = LSB\n");
1147 		break;
1148 	case FCF_ENCODE_MSB:
1149 		mdb_printf("fcfh_ident.id_encoding = MSB\n");
1150 		break;
1151 	default:
1152 		mdb_printf("fcfh_ident.id_encoding = 0x%x\n",
1153 		    h.fcfh_ident[FCF_ID_ENCODING]);
1154 	}
1155 
1156 	mdb_printf("fcfh_ident.id_version = %u\n",
1157 	    h.fcfh_ident[FCF_ID_VERSION]);
1158 
1159 	mdb_printf("fcfh_flags = 0x%x\n", h.fcfh_flags);
1160 	mdb_printf("fcfh_hdrsize = %u\n", h.fcfh_hdrsize);
1161 	mdb_printf("fcfh_secsize = %u\n", h.fcfh_secsize);
1162 	mdb_printf("fcfh_secnum = %u\n", h.fcfh_secnum);
1163 	mdb_printf("fcfh_secoff = %llu\n", h.fcfh_secoff);
1164 	mdb_printf("fcfh_filesz = %llu\n", h.fcfh_filesz);
1165 	mdb_printf("fcfh_cgen = %llu\n", h.fcfh_cgen);
1166 
1167 	return (DCMD_OK);
1168 }
1169 
1170 /*ARGSUSED*/
1171 static int
1172 fcf_sec_one(uintptr_t addr, void *ignored, uint_t *secp)
1173 {
1174 	static int fcf_sec(uintptr_t, uint_t, int, const mdb_arg_t *);
1175 
1176 	mdb_printf("%3d ", (*secp)++);
1177 	(void) fcf_sec(addr, DCMD_ADDRSPEC | DCMD_LOOP, 0, NULL);
1178 	return (WALK_NEXT);
1179 }
1180 
1181 /*ARGSUSED*/
1182 static int
1183 fcf_sec(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1184 {
1185 	static const char *const types[] = {
1186 		"none",		/* FCF_SECT_NONE */
1187 		"strtab",	/* FCF_SECT_STRTAB */
1188 		"module",	/* FCF_SECT_MODULE */
1189 		"case",		/* FCF_SECT_CASE */
1190 		"bufs",		/* FCF_SECT_BUFS */
1191 		"buffer",	/* FCF_SECT_BUFFER */
1192 		"serd",		/* FCF_SECT_SERD */
1193 		"events",	/* FCF_SECT_EVENTS */
1194 		"nvlists",	/* FCF_SECT_NVLISTS */
1195 	};
1196 
1197 	uint_t sec = 0;
1198 	fcf_sec_t s;
1199 
1200 	if (!(flags & DCMD_ADDRSPEC))
1201 		mdb_printf("%<u>%-3s ", "NDX");
1202 
1203 	if (!(flags & DCMD_ADDRSPEC) || DCMD_HDRSPEC(flags)) {
1204 		mdb_printf("%<u>%?s %-10s %-5s %-5s %-5s %-6s %-5s%</u>\n",
1205 		    "ADDR", "TYPE", "ALIGN", "FLAGS", "ENTSZ", "OFF", "SIZE");
1206 	}
1207 
1208 	if (!(flags & DCMD_ADDRSPEC)) {
1209 		if (mdb_walk("fcf_sec", (mdb_walk_cb_t)fcf_sec_one, &sec) < 0) {
1210 			mdb_warn("failed to walk fcf_sec");
1211 			return (DCMD_ERR);
1212 		}
1213 		return (DCMD_OK);
1214 	}
1215 
1216 	if (argc != 0)
1217 		return (DCMD_USAGE);
1218 
1219 	if (mdb_vread(&s, sizeof (s), addr) != sizeof (s)) {
1220 		mdb_warn("failed to read section header at %p", addr);
1221 		return (DCMD_ERR);
1222 	}
1223 
1224 	mdb_printf("%?p ", addr);
1225 
1226 	if (s.fcfs_type < sizeof (types) / sizeof (types[0]))
1227 		mdb_printf("%-10s ", types[s.fcfs_type]);
1228 	else
1229 		mdb_printf("%-10u ", s.fcfs_type);
1230 
1231 	mdb_printf("%-5u %-#5x %-#5x %-6llx %-#5llx\n", s.fcfs_align,
1232 	    s.fcfs_flags, s.fcfs_entsize, s.fcfs_offset, s.fcfs_size);
1233 
1234 	return (DCMD_OK);
1235 }
1236 
1237 static int
1238 fcf_sec_walk_init(mdb_walk_state_t *wsp)
1239 {
1240 	fcf_hdr_t h, *hp;
1241 	size_t size;
1242 
1243 	if (mdb_vread(&h, sizeof (h), wsp->walk_addr) != sizeof (h)) {
1244 		mdb_warn("failed to read FCF header at %p", wsp->walk_addr);
1245 		return (WALK_ERR);
1246 	}
1247 
1248 	size = sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * h.fcfh_secnum;
1249 	hp = mdb_alloc(size, UM_SLEEP);
1250 
1251 	if (mdb_vread(hp, size, wsp->walk_addr) != size) {
1252 		mdb_warn("failed to read FCF sections at %p", wsp->walk_addr);
1253 		mdb_free(hp, size);
1254 		return (WALK_ERR);
1255 	}
1256 
1257 	wsp->walk_data = hp;
1258 	wsp->walk_arg = 0;
1259 
1260 	return (WALK_NEXT);
1261 }
1262 
1263 static int
1264 fcf_sec_walk_step(mdb_walk_state_t *wsp)
1265 {
1266 	uint_t i = (uint_t)wsp->walk_arg;
1267 	size_t off = sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * i;
1268 	fcf_hdr_t *hp = wsp->walk_data;
1269 	fcf_sec_t *sp = (fcf_sec_t *)((uintptr_t)hp + off);
1270 
1271 	if (i >= hp->fcfh_secnum)
1272 		return (WALK_DONE);
1273 
1274 	wsp->walk_arg = (void *)(i + 1);
1275 	return (wsp->walk_callback(wsp->walk_addr + off, sp, wsp->walk_cbdata));
1276 }
1277 
1278 static void
1279 fcf_sec_walk_fini(mdb_walk_state_t *wsp)
1280 {
1281 	fcf_hdr_t *hp = wsp->walk_data;
1282 	mdb_free(hp, sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * hp->fcfh_secnum);
1283 }
1284 
1285 /*ARGSUSED*/
1286 static int
1287 fcf_case(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1288 {
1289 	fcf_case_t fcfc;
1290 
1291 	if (argc != 0)
1292 		return (DCMD_USAGE);
1293 
1294 	if (mdb_vread(&fcfc, sizeof (fcfc), addr) != sizeof (fcfc)) {
1295 		mdb_warn("failed to read case at %p", addr);
1296 		return (DCMD_ERR);
1297 	}
1298 
1299 	mdb_printf("fcfc_uuid = 0x%x\n", fcfc.fcfc_uuid);
1300 	mdb_printf("fcfc_state = %u\n", fcfc.fcfc_state);
1301 	mdb_printf("fcfc_bufs = %u\n", fcfc.fcfc_bufs);
1302 	mdb_printf("fcfc_events = %u\n", fcfc.fcfc_events);
1303 	mdb_printf("fcfc_suspects = %u\n", fcfc.fcfc_suspects);
1304 
1305 	return (DCMD_OK);
1306 }
1307 
1308 /*ARGSUSED*/
1309 static int
1310 fcf_event(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1311 {
1312 	fcf_event_t fcfe;
1313 
1314 	if (argc != 0)
1315 		return (DCMD_USAGE);
1316 
1317 	if (mdb_vread(&fcfe, sizeof (fcfe), addr) != sizeof (fcfe)) {
1318 		mdb_warn("failed to read event at %p", addr);
1319 		return (DCMD_ERR);
1320 	}
1321 
1322 	mdb_printf("fcfe_todsec = %llu (%Y)\n",
1323 	    fcfe.fcfe_todsec, (time_t)fcfe.fcfe_todsec);
1324 	mdb_printf("fcfe_todnsec = %llu\n", fcfe.fcfe_todnsec);
1325 	mdb_printf("fcfe_major = %u\n", fcfe.fcfe_major);
1326 	mdb_printf("fcfe_minor = %u\n", fcfe.fcfe_minor);
1327 	mdb_printf("fcfe_inode = %llu\n", fcfe.fcfe_inode);
1328 	mdb_printf("fcfe_offset = %llu\n", fcfe.fcfe_offset);
1329 
1330 	return (DCMD_OK);
1331 }
1332 
1333 /*ARGSUSED*/
1334 static int
1335 fcf_serd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1336 {
1337 	fcf_serd_t fcfd;
1338 
1339 	if (argc != 0)
1340 		return (DCMD_USAGE);
1341 
1342 	if (mdb_vread(&fcfd, sizeof (fcfd), addr) != sizeof (fcfd)) {
1343 		mdb_warn("failed to read serd at %p", addr);
1344 		return (DCMD_ERR);
1345 	}
1346 
1347 	mdb_printf("fcfd_name = 0x%x\n", fcfd.fcfd_name);
1348 	mdb_printf("fcfd_events = %u\n", fcfd.fcfd_events);
1349 	mdb_printf("fcfd_n = >%u\n", fcfd.fcfd_n);
1350 	mdb_printf("fcfd_t = %lluns\n", fcfd.fcfd_t);
1351 
1352 	return (DCMD_OK);
1353 }
1354 
1355 static int
1356 tmq_walk_init(mdb_walk_state_t *wsp)
1357 {
1358 	fmd_timerq_t tmq;
1359 	fmd_t F;
1360 
1361 	if (wsp->walk_addr == NULL && mdb_readvar(&F, "fmd") != sizeof (F)) {
1362 		mdb_warn("failed to read fmd meta-data");
1363 		return (WALK_ERR);
1364 	}
1365 
1366 	if (wsp->walk_addr == NULL)
1367 		wsp->walk_addr = (uintptr_t)F.d_timers;
1368 
1369 	if (mdb_vread(&tmq, sizeof (tmq), wsp->walk_addr) != sizeof (tmq)) {
1370 		mdb_warn("failed to read timerq at %p", wsp->walk_addr);
1371 		return (WALK_ERR);
1372 	}
1373 
1374 	wsp->walk_addr = (uintptr_t)tmq.tmq_list.l_next;
1375 	return (WALK_NEXT);
1376 }
1377 
1378 static int
1379 tmq_walk_step(mdb_walk_state_t *wsp)
1380 {
1381 	uintptr_t addr = wsp->walk_addr;
1382 	fmd_timer_t tmr;
1383 
1384 	if (addr == NULL)
1385 		return (WALK_DONE);
1386 
1387 	if (mdb_vread(&tmr, sizeof (tmr), addr) != sizeof (tmr)) {
1388 		mdb_warn("failed to read fmd_timer at %p", addr);
1389 		return (WALK_ERR);
1390 	}
1391 
1392 	wsp->walk_addr = (uintptr_t)tmr.tmr_list.l_next;
1393 	return (wsp->walk_callback(addr, &tmr, wsp->walk_cbdata));
1394 }
1395 
1396 static int
1397 fmd_timer(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1398 {
1399 	char name[32], func[MDB_SYM_NAMLEN];
1400 	fmd_timer_t t;
1401 
1402 	if (!(flags & DCMD_ADDRSPEC)) {
1403 		if (mdb_walk_dcmd("fmd_timerq", "fmd_timer", argc, argv) != 0) {
1404 			mdb_warn("failed to walk fmd_timerq");
1405 			return (DCMD_ERR);
1406 		}
1407 		return (DCMD_OK);
1408 	}
1409 
1410 	if (mdb_vread(&t, sizeof (t), addr) != sizeof (t)) {
1411 		mdb_warn("failed to read fmd_timer at %p", addr);
1412 		return (DCMD_ERR);
1413 	}
1414 
1415 	if (DCMD_HDRSPEC(flags)) {
1416 		mdb_printf("%<u>%-8s %-20s %-4s %-18s %-8s %s%</u>\n",
1417 		    "ADDR", "MODULE", "ID", "HRTIME", "ARG", "FUNC");
1418 	}
1419 
1420 	if (mdb_readstr(name, sizeof (name), (uintptr_t)
1421 	    t.tmr_ids + OFFSETOF(fmd_idspace_t, ids_name)) <= 0)
1422 		(void) mdb_snprintf(name, sizeof (name), "<%p>", t.tmr_ids);
1423 
1424 	if (mdb_lookup_by_addr((uintptr_t)t.tmr_func, MDB_SYM_FUZZY,
1425 	    func, sizeof (func), NULL) != 0)
1426 		(void) mdb_snprintf(func, sizeof (func), "<%p>", t.tmr_func);
1427 
1428 	mdb_printf("%-8p %-20s %4d 0x%-16llx %-8p %s\n",
1429 	    addr, name, t.tmr_id, t.tmr_hrt, t.tmr_arg, func);
1430 	return (DCMD_OK);
1431 }
1432 
1433 static int
1434 xprt_walk_init(mdb_walk_state_t *wsp)
1435 {
1436 	fmd_module_t m;
1437 
1438 	if (wsp->walk_addr == NULL) {
1439 		mdb_warn("transport walker requires fmd_module_t address\n");
1440 		return (WALK_ERR);
1441 	}
1442 
1443 	if (mdb_vread(&m, sizeof (m), wsp->walk_addr) != sizeof (m)) {
1444 		mdb_warn("failed to read module at %p", wsp->walk_addr);
1445 		return (WALK_ERR);
1446 	}
1447 
1448 	wsp->walk_addr = (uintptr_t)m.mod_transports.l_next;
1449 	return (WALK_NEXT);
1450 }
1451 
1452 static int
1453 xprt_walk_step(mdb_walk_state_t *wsp)
1454 {
1455 	uintptr_t addr = wsp->walk_addr;
1456 	fmd_xprt_impl_t xi;
1457 
1458 	if (addr == NULL)
1459 		return (WALK_DONE);
1460 
1461 	if (mdb_vread(&xi, sizeof (xi), addr) != sizeof (xi)) {
1462 		mdb_warn("failed to read fmd_xprt at %p", addr);
1463 		return (WALK_ERR);
1464 	}
1465 
1466 	wsp->walk_addr = (uintptr_t)xi.xi_list.l_next;
1467 	return (wsp->walk_callback(addr, &xi, wsp->walk_cbdata));
1468 }
1469 
1470 static int
1471 xpc_walk_init(mdb_walk_state_t *wsp)
1472 {
1473 	fmd_xprt_class_hash_t xch;
1474 
1475 	if (mdb_vread(&xch, sizeof (xch), wsp->walk_addr) != sizeof (xch)) {
1476 		mdb_warn("failed to read fmd_xprt_class_hash at %p",
1477 		    wsp->walk_addr);
1478 		return (WALK_ERR);
1479 	}
1480 
1481 	return (hash_walk_init(wsp, (uintptr_t)xch.xch_hash, xch.xch_hashlen,
1482 	    "fmd_xprt_class", sizeof (fmd_xprt_class_t),
1483 	    OFFSETOF(fmd_xprt_class_t, xc_next)));
1484 }
1485 
1486 /*ARGSUSED*/
1487 static int
1488 fmd_xprt_class(uintptr_t addr, const void *data, void *arg)
1489 {
1490 	const fmd_xprt_class_t *xcp = data;
1491 	char name[1024];
1492 
1493 	if (mdb_readstr(name, sizeof (name), (uintptr_t)xcp->xc_class) <= 0)
1494 		(void) mdb_snprintf(name, sizeof (name), "<%p>", xcp->xc_class);
1495 
1496 	mdb_printf("%-8p %-4u %s\n", addr, xcp->xc_refs, name);
1497 	return (WALK_NEXT);
1498 }
1499 
1500 static int
1501 fmd_xprt(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1502 {
1503 	uint_t opt_s = FALSE, opt_l = FALSE, opt_r = FALSE, opt_u = FALSE;
1504 	fmd_xprt_impl_t xi;
1505 
1506 	if (mdb_getopts(argc, argv,
1507 	    'l', MDB_OPT_SETBITS, TRUE, &opt_l,
1508 	    'r', MDB_OPT_SETBITS, TRUE, &opt_r,
1509 	    's', MDB_OPT_SETBITS, TRUE, &opt_s,
1510 	    'u', MDB_OPT_SETBITS, TRUE, &opt_u, NULL) != argc)
1511 		return (DCMD_USAGE);
1512 
1513 	if (!(flags & DCMD_ADDRSPEC)) {
1514 		if (mdb_walk_dcmd("fmd_xprt", "fmd_xprt", argc, argv) != 0) {
1515 			mdb_warn("failed to walk fmd_xprt");
1516 			return (DCMD_ERR);
1517 		}
1518 		return (DCMD_OK);
1519 	}
1520 
1521 	if (mdb_vread(&xi, sizeof (xi), addr) != sizeof (xi)) {
1522 		mdb_warn("failed to read fmd_xprt at %p", addr);
1523 		return (DCMD_ERR);
1524 	}
1525 
1526 	if (DCMD_HDRSPEC(flags)) {
1527 		mdb_printf("%<u>%-8s %-4s %-4s %-5s %s%</u>\n",
1528 		    "ADDR", "ID", "VERS", "FLAGS", "STATE");
1529 	}
1530 
1531 	mdb_printf("%-8p %-4d %-4u %-5x %a\n",
1532 	    addr, xi.xi_id, xi.xi_version, xi.xi_flags, xi.xi_state);
1533 
1534 	if (opt_l | opt_s) {
1535 		(void) mdb_inc_indent(4);
1536 		mdb_printf("Local subscriptions requested by peer:\n");
1537 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1538 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1539 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_lsub));
1540 		(void) mdb_dec_indent(4);
1541 	}
1542 
1543 	if (opt_r | opt_s) {
1544 		(void) mdb_inc_indent(4);
1545 		mdb_printf("Remote subscriptions requested of peer:\n");
1546 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1547 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1548 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_rsub));
1549 		(void) mdb_dec_indent(4);
1550 	}
1551 
1552 	if (opt_u | opt_s) {
1553 		(void) mdb_inc_indent(4);
1554 		mdb_printf("Pending unsubscription acknowledgements:\n");
1555 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1556 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1557 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_usub));
1558 		(void) mdb_dec_indent(4);
1559 	}
1560 
1561 	return (DCMD_OK);
1562 }
1563 
1564 static const mdb_dcmd_t dcmds[] = {
1565 	{ "fcf_case", "?", "print a FCF case", fcf_case },
1566 	{ "fcf_event", "?", "print a FCF event", fcf_event },
1567 	{ "fcf_hdr", "?", "print a FCF header", fcf_hdr },
1568 	{ "fcf_sec", ":", "print a FCF section header", fcf_sec },
1569 	{ "fcf_serd", "?", "print a FCF serd engine", fcf_serd },
1570 	{ "fmd_trace", "?[-cs]", "display thread trace buffer(s)", fmd_trace },
1571 	{ "fmd_ustat", "[:]", "display statistics collection", fmd_ustat },
1572 	{ "fmd_stat", "[:]", "display statistic structure", fmd_stat },
1573 	{ "fmd_event", NULL, "display event structure", fmd_event },
1574 	{ "fmd_thread", "?", "display thread or list of threads", fmd_thread },
1575 	{ "fmd_module", "?", "display module or list of modules", fmd_module },
1576 	{ "fmd_case", ":", "display case file structure", fmd_case },
1577 	{ "fmd_buf", ":", "display buffer structure", fmd_buf },
1578 	{ "fmd_serd", "[:]", "display serd engine structure", fmd_serd },
1579 	{ "fmd_asru", "?", "display asru resource structure", fmd_asru },
1580 	{ "fmd_asru_link", "?", "display resource structure", fmd_asru_link },
1581 	{ "fmd_timer", "?", "display pending timer(s)", fmd_timer },
1582 	{ "fmd_xprt", "?[-lrsu]", "display event transport(s)", fmd_xprt },
1583 	{ NULL }
1584 };
1585 
1586 static const mdb_walker_t walkers[] = {
1587 	{ "fcf_sec", "walk FCF section header table given header address",
1588 		fcf_sec_walk_init, fcf_sec_walk_step, fcf_sec_walk_fini },
1589 	{ "fmd_trace", "walk per-thread trace buffers",
1590 		trwalk_init, trwalk_step, trwalk_fini },
1591 	{ "fmd_ustat", "walk per-collection statistics",
1592 		ustat_walk_init, ustat_walk_step, hash_walk_fini },
1593 	{ "fmd_thread", "walk list of all fmd_thread_t's",
1594 		thread_walk_init, thread_walk_step, NULL },
1595 	{ "fmd_module", "walk list of all fmd_module_t's",
1596 		mod_walk_init, mod_walk_step, NULL },
1597 	{ "fmd_case", "walk per-module case objects",
1598 		case_walk_init, case_walk_step, case_walk_fini },
1599 	{ "fmd_buf", "walk per-buf_hash buffers",
1600 		buf_walk_init, hash_walk_step, hash_walk_fini },
1601 	{ "fmd_serd", "walk per-serd_hash engines",
1602 		serd_walk_init, hash_walk_step, hash_walk_fini },
1603 	{ "fmd_asru", "walk asru resource hash",
1604 		asru_walk_init, hash_walk_step, hash_walk_fini },
1605 	{ "fmd_asru_link", "walk resource hash",
1606 		al_walk_init, hash_walk_step, hash_walk_fini },
1607 	{ "fmd_timerq", "walk timer queue",
1608 		tmq_walk_init, tmq_walk_step, NULL },
1609 	{ "fmd_xprt", "walk per-module list of transports",
1610 		xprt_walk_init, xprt_walk_step, NULL },
1611 	{ "fmd_xprt_class", "walk hash table of subscription classes",
1612 		xpc_walk_init, hash_walk_step, hash_walk_fini },
1613 	{ NULL, NULL, NULL, NULL, NULL }
1614 };
1615 
1616 static const mdb_modinfo_t modinfo = { MDB_API_VERSION, dcmds, walkers };
1617 
1618 const mdb_modinfo_t *
1619 _mdb_init(void)
1620 {
1621 	return (&modinfo);
1622 }
1623