xref: /illumos-gate/usr/src/cmd/fm/fmd/common/fmd_mdb.c (revision a73c0fe4e90b82a478f821ef3adb5cf34f6a9346)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/mdb_modapi.h>
30 #include <limits.h>
31 
32 #include <fmd_trace.h>
33 #include <fmd_module.h>
34 #include <fmd_thread.h>
35 #include <fmd_ustat.h>
36 #include <fmd_event.h>
37 #include <fmd_case.h>
38 #include <fmd_buf.h>
39 #include <fmd_asru.h>
40 #include <fmd_ckpt.h>
41 #include <fmd_timerq.h>
42 #include <fmd_xprt.h>
43 
44 #include <fmd.h>
45 
46 typedef struct trwalk_state {
47 	struct trwalk_state *trw_next;
48 	fmd_tracebuf_t trw_data;
49 	pthread_t trw_tid;
50 	uintptr_t trw_base;
51 	const fmd_tracerec_t *trw_stop;
52 	fmd_tracerec_t *trw_xrec;
53 } trwalk_state_t;
54 
55 typedef struct hashwalk_data {
56 	uintptr_t *hw_hash;
57 	uint_t hw_hashlen;
58 	uint_t hw_hashidx;
59 	const char *hw_name;
60 	void *hw_data;
61 	size_t hw_size;
62 	size_t hw_next;
63 } hashwalk_data_t;
64 
65 static int fmd_stat(uintptr_t, uint_t, int, const mdb_arg_t *);
66 static int fmd_ustat(uintptr_t, uint_t, int, const mdb_arg_t *);
67 
68 static int
69 trwalk_init(mdb_walk_state_t *wsp)
70 {
71 	uintptr_t addr;
72 	fmd_thread_t thr;
73 	fmd_t F;
74 
75 	if (wsp->walk_addr != NULL) {
76 		mdb_warn("fmd_trace only supports global walks\n");
77 		return (WALK_ERR);
78 	}
79 
80 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
81 		mdb_warn("failed to read fmd meta-data");
82 		return (WALK_ERR);
83 	}
84 
85 	for (addr = (uintptr_t)F.d_thr_list.l_next; addr != NULL;
86 	    addr = (uintptr_t)thr.thr_list.l_next) {
87 
88 		size_t len, ptr_off, end_off;
89 		fmd_tracerec_t *buf;
90 		trwalk_state_t *t;
91 
92 		if (mdb_vread(&thr, sizeof (thr), addr) != sizeof (thr)) {
93 			mdb_warn("failed to read thread at %p "
94 			    "(some trace data will be unavailable)", addr);
95 			break;
96 		}
97 
98 		t = mdb_zalloc(sizeof (trwalk_state_t), UM_SLEEP);
99 		t->trw_next = wsp->walk_data;
100 		wsp->walk_data = t;
101 
102 		(void) mdb_vread(&t->trw_data,
103 		    sizeof (t->trw_data), (uintptr_t)thr.thr_trdata);
104 
105 		if (t->trw_data.tb_recs == 0)
106 			continue; /* no trace buffer allocated for thread */
107 
108 		len = t->trw_data.tb_recs * t->trw_data.tb_size;
109 		buf = mdb_alloc(len, UM_SLEEP);
110 
111 		t->trw_tid = thr.thr_tid;
112 		t->trw_base = (uintptr_t)t->trw_data.tb_buf;
113 
114 		if (mdb_vread(buf, len, t->trw_base) == -1) {
115 			mdb_warn("failed to read buffer for t%u", t->trw_tid);
116 			bzero(buf, len);
117 		}
118 
119 		end_off = (uintptr_t)t->trw_data.tb_end - t->trw_base;
120 		ptr_off = (uintptr_t)t->trw_data.tb_ptr - t->trw_base;
121 
122 		t->trw_data.tb_buf = buf;
123 		t->trw_data.tb_end = (void *)((uintptr_t)buf + end_off);
124 		t->trw_data.tb_ptr = (void *)((uintptr_t)buf + ptr_off);
125 
126 		if (t->trw_data.tb_ptr < t->trw_data.tb_buf ||
127 		    t->trw_data.tb_ptr > t->trw_data.tb_end) {
128 			mdb_warn("trace record ptr for t%u is corrupt "
129 			    "(some data may be unavailable)\n", t->trw_tid);
130 			t->trw_data.tb_ptr = t->trw_data.tb_buf;
131 		}
132 
133 		t->trw_stop = t->trw_data.tb_ptr;
134 		t->trw_xrec = mdb_alloc(
135 		    t->trw_data.tb_size + sizeof (uintptr_t), UM_SLEEP);
136 	}
137 
138 	return (WALK_NEXT);
139 }
140 
141 static fmd_tracerec_t *
142 trwalk_nextrec(trwalk_state_t *t)
143 {
144 	if (t->trw_stop == NULL)
145 		return (t->trw_data.tb_ptr);
146 
147 	if (t->trw_data.tb_ptr == t->trw_data.tb_buf)
148 		t->trw_data.tb_ptr = t->trw_data.tb_end;
149 	else
150 		t->trw_data.tb_ptr = (fmd_tracerec_t *)
151 		    ((uintptr_t)t->trw_data.tb_ptr - t->trw_data.tb_size);
152 
153 	if (t->trw_data.tb_ptr == t->trw_stop)
154 		t->trw_stop = NULL; /* mark buffer as empty */
155 
156 	return (t->trw_data.tb_ptr);
157 }
158 
159 static int
160 trwalk_step(mdb_walk_state_t *wsp)
161 {
162 	trwalk_state_t *t, *oldest_t;
163 	hrtime_t oldest_time = 0;
164 	fmd_tracerec_t *trp;
165 	int status;
166 
167 	for (t = wsp->walk_data; t != NULL; t = t->trw_next) {
168 		for (trp = t->trw_data.tb_ptr; t->trw_stop != NULL &&
169 		    trp->tr_time == 0; trp = trwalk_nextrec(t))
170 			continue;
171 
172 		if (t->trw_stop == NULL)
173 			continue; /* buffer has been emptied */
174 
175 		if (trp->tr_time > oldest_time) {
176 			oldest_time = trp->tr_time;
177 			oldest_t = t;
178 		}
179 	}
180 
181 	if (oldest_time == 0)
182 		return (WALK_DONE);
183 
184 	t = oldest_t;
185 	trp = t->trw_data.tb_ptr;
186 
187 	bcopy(trp, t->trw_xrec, t->trw_data.tb_size);
188 	t->trw_xrec->tr_depth = MIN(trp->tr_depth, t->trw_data.tb_frames);
189 	t->trw_xrec->tr_stack[t->trw_xrec->tr_depth] = t->trw_tid;
190 
191 	status = wsp->walk_callback((uintptr_t)trp - (uintptr_t)
192 	    t->trw_data.tb_buf + t->trw_base, t->trw_xrec, wsp->walk_cbdata);
193 
194 	(void) trwalk_nextrec(t);
195 	return (status);
196 }
197 
198 static void
199 trwalk_fini(mdb_walk_state_t *wsp)
200 {
201 	trwalk_state_t *t, *u;
202 
203 	for (t = wsp->walk_data; t != NULL; t = u) {
204 		u = t->trw_next;
205 		mdb_free(t->trw_data.tb_buf,
206 		    t->trw_data.tb_recs * t->trw_data.tb_size);
207 		mdb_free(t->trw_xrec, t->trw_data.tb_size + sizeof (uintptr_t));
208 		mdb_free(t, sizeof (trwalk_state_t));
209 	}
210 }
211 
212 /*ARGSUSED*/
213 static int
214 trprint_msg(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
215 {
216 	if (tid == 0)
217 		mdb_printf("%3lu ", trp->tr_stack[trp->tr_depth]);
218 	else if (trp->tr_stack[trp->tr_depth] != tid)
219 		return (WALK_NEXT);
220 
221 	mdb_printf("%016llx %04x %-5u %s\n",
222 	    trp->tr_time, 1 << trp->tr_tag, trp->tr_errno, trp->tr_msg);
223 
224 	return (WALK_NEXT);
225 }
226 
227 /*ARGSUSED*/
228 static int
229 trprint_cpp(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
230 {
231 	char file[64];
232 
233 	if (tid == 0)
234 		mdb_printf("%3lu ", trp->tr_stack[trp->tr_depth]);
235 	else if (trp->tr_stack[trp->tr_depth] != tid)
236 		return (WALK_NEXT);
237 
238 	if (mdb_readstr(file, sizeof (file), (uintptr_t)trp->tr_file) <= 0)
239 		(void) strcpy(file, "???");
240 
241 	mdb_printf("%016llx %04x %s: %u\n",
242 	    trp->tr_time, 1 << trp->tr_tag, file, trp->tr_line);
243 
244 	return (WALK_NEXT);
245 }
246 
247 static void
248 trprint_stack(const fmd_tracerec_t *trp)
249 {
250 	uint8_t i;
251 
252 	for (i = 0; i < trp->tr_depth; i++)
253 		mdb_printf("\t%a\n", trp->tr_stack[i]);
254 
255 	if (trp->tr_depth != 0)
256 		mdb_printf("\n");
257 }
258 
259 static int
260 trprint_msg_stack(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
261 {
262 	int status = trprint_msg(addr, trp, tid);
263 	trprint_stack(trp);
264 	return (status);
265 }
266 
267 static int
268 trprint_cpp_stack(uintptr_t addr, const fmd_tracerec_t *trp, uintptr_t tid)
269 {
270 	int status = trprint_cpp(addr, trp, tid);
271 	trprint_stack(trp);
272 	return (status);
273 }
274 
275 static int
276 fmd_trace(uintptr_t tid, uint_t flags, int argc, const mdb_arg_t *argv)
277 {
278 	int (*func)(uintptr_t, const fmd_tracerec_t *, uintptr_t);
279 	uint_t opt_c = FALSE, opt_s = FALSE;
280 
281 	if (mdb_getopts(argc, argv,
282 	    'c', MDB_OPT_SETBITS, TRUE, &opt_c,
283 	    's', MDB_OPT_SETBITS, TRUE, &opt_s, NULL) != argc)
284 		return (DCMD_USAGE);
285 
286 	if (!(flags & DCMD_ADDRSPEC)) {
287 		mdb_printf("TID ");
288 		tid = 0;
289 	}
290 
291 	if (opt_c) {
292 		mdb_printf("%-16s %-4s FILE:LINE\n", "TIME", "TAG");
293 		func = opt_s ? trprint_cpp_stack : trprint_cpp;
294 	} else {
295 		mdb_printf("%-16s %-4s %-5s MSG\n", "TIME", "TAG", "ERRNO");
296 		func = opt_s ? trprint_msg_stack : trprint_msg;
297 	}
298 
299 	if (mdb_walk("fmd_trace", (mdb_walk_cb_t)func, (void *)tid) == -1) {
300 		mdb_warn("failed to walk fmd_trace");
301 		return (DCMD_ERR);
302 	}
303 
304 	return (DCMD_OK);
305 }
306 
307 static int
308 hash_walk_init(mdb_walk_state_t *wsp, uintptr_t addr, uint_t hashlen,
309     const char *name, size_t size, size_t next)
310 {
311 	hashwalk_data_t *hwp;
312 	size_t len = sizeof (uintptr_t) * hashlen;
313 
314 	if (len == 0) {
315 		mdb_warn("failed to walk hash: invalid hash length\n");
316 		return (WALK_ERR);
317 	}
318 
319 	hwp = mdb_alloc(sizeof (hashwalk_data_t), UM_SLEEP);
320 	hwp->hw_hash = mdb_zalloc(len, UM_SLEEP);
321 	(void) mdb_vread(hwp->hw_hash, len, addr);
322 	hwp->hw_hashlen = hashlen;
323 	hwp->hw_hashidx = 0;
324 	hwp->hw_name = name;
325 	hwp->hw_data = mdb_zalloc(size, UM_SLEEP);
326 	hwp->hw_size = size;
327 	hwp->hw_next = next;
328 
329 	wsp->walk_addr = hwp->hw_hash[0];
330 	wsp->walk_data = hwp;
331 
332 	return (WALK_NEXT);
333 }
334 
335 static int
336 hash_walk_step(mdb_walk_state_t *wsp)
337 {
338 	hashwalk_data_t *hwp = wsp->walk_data;
339 	int rv;
340 
341 	while (wsp->walk_addr == NULL) {
342 		if (++hwp->hw_hashidx < hwp->hw_hashlen)
343 			wsp->walk_addr = hwp->hw_hash[hwp->hw_hashidx];
344 		else
345 			return (WALK_DONE);
346 	}
347 
348 	if (mdb_vread(hwp->hw_data, hwp->hw_size, wsp->walk_addr) == -1) {
349 		mdb_warn("failed to read %s at %p",
350 		    hwp->hw_name, wsp->walk_addr);
351 		return (WALK_ERR);
352 	}
353 
354 	rv = wsp->walk_callback(wsp->walk_addr, hwp->hw_data, wsp->walk_cbdata);
355 	wsp->walk_addr = *(uintptr_t *)((uintptr_t)hwp->hw_data + hwp->hw_next);
356 	return (rv);
357 }
358 
359 static void
360 hash_walk_fini(mdb_walk_state_t *wsp)
361 {
362 	hashwalk_data_t *hwp = wsp->walk_data;
363 
364 	mdb_free(hwp->hw_hash, sizeof (uintptr_t) * hwp->hw_hashlen);
365 	mdb_free(hwp->hw_data, hwp->hw_size);
366 	mdb_free(hwp, sizeof (hashwalk_data_t));
367 }
368 
369 static int
370 ustat_walk_init(mdb_walk_state_t *wsp)
371 {
372 	fmd_ustat_t us;
373 
374 	if (mdb_vread(&us, sizeof (us), wsp->walk_addr) != sizeof (us)) {
375 		mdb_warn("failed to read fmd_ustat_t at %p", wsp->walk_addr);
376 		return (WALK_ERR);
377 	}
378 
379 	return (hash_walk_init(wsp,
380 	    (uintptr_t)us.us_hash, us.us_hashlen, NULL, 0, 0));
381 }
382 
383 static int
384 ustat_walk_step(mdb_walk_state_t *wsp)
385 {
386 	hashwalk_data_t *hwp = wsp->walk_data;
387 	fmd_ustat_elem_t ue;
388 	fmd_stat_t s;
389 
390 	while (wsp->walk_addr == NULL) {
391 		if (++hwp->hw_hashidx < hwp->hw_hashlen)
392 			wsp->walk_addr = hwp->hw_hash[hwp->hw_hashidx];
393 		else
394 			return (WALK_DONE);
395 	}
396 
397 	if (mdb_vread(&ue, sizeof (ue), wsp->walk_addr) != sizeof (ue) ||
398 	    mdb_vread(&s, sizeof (s), (uintptr_t)ue.use_stat) != sizeof (s)) {
399 		mdb_warn("failed to read stat element at %p", wsp->walk_addr);
400 		return (WALK_ERR);
401 	}
402 
403 	wsp->walk_addr = (uintptr_t)ue.use_next;
404 
405 	return (wsp->walk_callback(
406 	    (uintptr_t)ue.use_stat, &s, wsp->walk_cbdata));
407 }
408 
409 struct fmd_cmd_data {
410 	int argc;
411 	const mdb_arg_t *argv;
412 };
413 
414 /* ARGSUSED */
415 static int
416 module_ustat(uintptr_t addr, const void *data, void *wsp)
417 {
418 	fmd_module_t *modp = (fmd_module_t *)data;
419 	char name[PATH_MAX];
420 	const struct fmd_cmd_data *udp = wsp;
421 
422 	if (mdb_readstr(name, sizeof (name), (uintptr_t)modp->mod_name) <= 0)
423 		(void) mdb_snprintf(name, sizeof (name), "<%p>",
424 		    modp->mod_name);
425 	mdb_printf("%s\n", name);
426 	(void) fmd_ustat((uintptr_t)modp->mod_ustat,
427 	    DCMD_ADDRSPEC | DCMD_LOOPFIRST, udp->argc, udp->argv);
428 	return (WALK_NEXT);
429 }
430 
431 static int
432 fmd_ustat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
433 {
434 	if (!(flags & DCMD_ADDRSPEC)) {
435 		struct fmd_cmd_data ud;
436 
437 		ud.argc = argc;
438 		ud.argv = argv;
439 		if (mdb_walk("fmd_module", module_ustat, &ud) == -1) {
440 			mdb_warn("failed to walk 'fmd_module'");
441 			return (DCMD_ERR);
442 		}
443 		return (DCMD_OK);
444 	}
445 
446 	if (mdb_pwalk_dcmd("fmd_ustat", "fmd_stat", argc, argv, addr) != 0) {
447 		mdb_warn("failed to walk fmd_ustat at %p", addr);
448 		return (DCMD_ERR);
449 	}
450 
451 	return (DCMD_OK);
452 }
453 
454 /* ARGSUSED */
455 static int
456 module_stat(uintptr_t addr, const void *data, void *wsp)
457 {
458 	fmd_module_t *modp = (fmd_module_t *)data;
459 	char name[PATH_MAX];
460 	const struct fmd_cmd_data *udp = wsp;
461 	fmd_modstat_t *mod_stats;
462 
463 	if (mdb_readstr(name, sizeof (name), (uintptr_t)modp->mod_name) <= 0) {
464 		(void) mdb_snprintf(name, sizeof (name), "<%p>",
465 		    modp->mod_name);
466 	}
467 	mdb_printf("%s\n", name);
468 	mod_stats = modp->mod_stats;
469 	(void) fmd_stat((uintptr_t)&mod_stats->ms_loadtime,
470 	    DCMD_ADDRSPEC | DCMD_LOOPFIRST, udp->argc, udp->argv);
471 	(void) fmd_stat((uintptr_t)&mod_stats->ms_snaptime,
472 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
473 	(void) fmd_stat((uintptr_t)&mod_stats->ms_accepted,
474 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
475 	(void) fmd_stat((uintptr_t)&mod_stats->ms_debugdrop,
476 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
477 	(void) fmd_stat((uintptr_t)&mod_stats->ms_memtotal,
478 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
479 	(void) fmd_stat((uintptr_t)&mod_stats->ms_memlimit,
480 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
481 	(void) fmd_stat((uintptr_t)&mod_stats->ms_buftotal,
482 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
483 	(void) fmd_stat((uintptr_t)&mod_stats->ms_buflimit,
484 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
485 	(void) fmd_stat((uintptr_t)&mod_stats->ms_thrtotal,
486 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
487 	(void) fmd_stat((uintptr_t)&mod_stats->ms_thrlimit,
488 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
489 	(void) fmd_stat((uintptr_t)&mod_stats->ms_caseopen,
490 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
491 	(void) fmd_stat((uintptr_t)&mod_stats->ms_casesolved,
492 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
493 	(void) fmd_stat((uintptr_t)&mod_stats->ms_caseclosed,
494 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
495 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_save,
496 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
497 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_restore,
498 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
499 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_zeroed,
500 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
501 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_cnt,
502 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
503 	(void) fmd_stat((uintptr_t)&mod_stats->ms_ckpt_time,
504 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
505 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtopen,
506 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
507 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtlimit,
508 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
509 	(void) fmd_stat((uintptr_t)&mod_stats->ms_xprtqlimit,
510 	    DCMD_ADDRSPEC | DCMD_LOOP, udp->argc, udp->argv);
511 	return (WALK_NEXT);
512 }
513 
514 /*ARGSUSED*/
515 static int
516 fmd_stat(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
517 {
518 	char buf[512];
519 	fmd_stat_t s;
520 
521 	if (argc != 0)
522 		return (DCMD_USAGE);
523 
524 	if (DCMD_HDRSPEC(flags))
525 		mdb_printf("%<u>%-11s %-4s %-32s %s%</u>\n",
526 		    "ADDR", "TYPE", "NAME", "VALUE");
527 
528 	if (!(flags & DCMD_ADDRSPEC)) {
529 		struct fmd_cmd_data ud;
530 
531 		ud.argc = argc;
532 		ud.argv = argv;
533 
534 		if (mdb_walk("fmd_module", module_stat, &ud) == -1) {
535 			mdb_warn("failed to walk 'fmd_module'");
536 			return (DCMD_ERR);
537 		}
538 		return (DCMD_OK);
539 	}
540 
541 	if (mdb_vread(&s, sizeof (s), addr) != sizeof (s)) {
542 		mdb_warn("failed to read statistic at %p", addr);
543 		return (DCMD_ERR);
544 	}
545 
546 	switch (s.fmds_type) {
547 	case FMD_TYPE_BOOL:
548 		mdb_printf("%-11p %-4s %-32s %s\n", addr, "bool",
549 		    s.fmds_name, s.fmds_value.bool ? "true" : "false");
550 		break;
551 	case FMD_TYPE_INT32:
552 		mdb_printf("%-11p %-4s %-32s %d\n", addr, "i32",
553 		    s.fmds_name, s.fmds_value.i32);
554 		break;
555 	case FMD_TYPE_UINT32:
556 		mdb_printf("%-11p %-4s %-32s %u\n", addr, "ui32",
557 		    s.fmds_name, s.fmds_value.i32);
558 		break;
559 	case FMD_TYPE_INT64:
560 		mdb_printf("%-11p %-4s %-32s %lld\n", addr, "i64",
561 		    s.fmds_name, s.fmds_value.i64);
562 		break;
563 	case FMD_TYPE_UINT64:
564 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "ui64",
565 		    s.fmds_name, s.fmds_value.ui64);
566 		break;
567 	case FMD_TYPE_STRING:
568 		if (mdb_readstr(buf, sizeof (buf),
569 		    (uintptr_t)s.fmds_value.str) < 0) {
570 			(void) mdb_snprintf(buf, sizeof (buf), "<%p>",
571 			    s.fmds_value.str);
572 		}
573 		mdb_printf("%-11p %-4s %-32s %s\n", addr, "str",
574 		    s.fmds_name, buf);
575 		break;
576 	case FMD_TYPE_TIME:
577 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "time",
578 		    s.fmds_name, s.fmds_value.ui64);
579 		break;
580 	case FMD_TYPE_SIZE:
581 		mdb_printf("%-11p %-4s %-32s %llu\n", addr, "size",
582 		    s.fmds_name, s.fmds_value.ui64);
583 		break;
584 	default:
585 		mdb_printf("%-11p %-4u %-32s ???\n", addr,
586 		    s.fmds_type, s.fmds_name);
587 		break;
588 	}
589 
590 	return (DCMD_OK);
591 }
592 
593 /*ARGSUSED*/
594 static int
595 fmd_event(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
596 {
597 	char type[16], name[16];
598 	fmd_event_impl_t ev;
599 
600 	if (argc != 0)
601 		return (DCMD_USAGE);
602 
603 	if (mdb_vread(&ev, sizeof (ev), addr) != sizeof (ev)) {
604 		mdb_warn("failed to read fmd_event at %p", addr);
605 		return (DCMD_ERR);
606 	}
607 
608 	if (DCMD_HDRSPEC(flags)) {
609 		mdb_printf("%<u>%-11s %-4s %-5s %-3s %-?s%</u>\n",
610 		    "ADDR", "TYPE", "STATE", "REF", "NVPAIR");
611 	}
612 
613 	switch (ev.ev_type) {
614 	case FMD_EVT_PROTOCOL:
615 		(void) strcpy(type, "PROT");
616 		break;
617 	case FMD_EVT_GC:
618 		(void) strcpy(type, "GC");
619 		break;
620 	case FMD_EVT_CLOSE:
621 		(void) strcpy(type, "CLSE");
622 		break;
623 	case FMD_EVT_TIMEOUT:
624 		(void) strcpy(type, "TIME");
625 		break;
626 	case FMD_EVT_STATS:
627 		(void) strcpy(type, "STAT");
628 		break;
629 	case FMD_EVT_PUBLISH:
630 		(void) strcpy(type, "PUBL");
631 		break;
632 	case FMD_EVT_TOPO:
633 		(void) strcpy(type, "TOPO");
634 		break;
635 	default:
636 		(void) mdb_snprintf(type, sizeof (type), "%u", ev.ev_type);
637 	}
638 
639 	switch (ev.ev_state) {
640 	case FMD_EVS_RECEIVED:
641 		(void) strcpy(name, "RECVD");
642 		break;
643 	case FMD_EVS_ACCEPTED:
644 		(void) strcpy(name, "ACCPT");
645 		break;
646 	case FMD_EVS_DISCARDED:
647 		(void) strcpy(name, "DSCRD");
648 		break;
649 	case FMD_EVS_DIAGNOSED:
650 		(void) strcpy(name, "DIAGN");
651 		break;
652 	default:
653 		(void) mdb_snprintf(name, sizeof (name), "%u", ev.ev_state);
654 	}
655 
656 	mdb_printf("%-11p %-4s %-5s %-3u %p\n",
657 	    addr, type, name, ev.ev_refs, ev.ev_nvl);
658 
659 	return (DCMD_OK);
660 }
661 
662 static int
663 thread_walk_init(mdb_walk_state_t *wsp)
664 {
665 	fmd_t F;
666 
667 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
668 		mdb_warn("failed to read fmd meta-data");
669 		return (WALK_ERR);
670 	}
671 
672 	wsp->walk_addr = (uintptr_t)F.d_thr_list.l_next;
673 	return (WALK_NEXT);
674 }
675 
676 static int
677 thread_walk_step(mdb_walk_state_t *wsp)
678 {
679 	uintptr_t addr = wsp->walk_addr;
680 	fmd_thread_t t;
681 
682 	if (addr == NULL)
683 		return (WALK_DONE);
684 
685 	if (mdb_vread(&t, sizeof (t), addr) != sizeof (t)) {
686 		mdb_warn("failed to read fmd_thread at %p", addr);
687 		return (WALK_ERR);
688 	}
689 
690 	wsp->walk_addr = (uintptr_t)t.thr_list.l_next;
691 	return (wsp->walk_callback(addr, &t, wsp->walk_cbdata));
692 }
693 
694 static int
695 fmd_thread(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
696 {
697 	fmd_thread_t thr;
698 
699 	if (!(flags & DCMD_ADDRSPEC))
700 		return (mdb_walk_dcmd("fmd_thread", "fmd_thread", argc, argv));
701 
702 	if (argc != 0)
703 		return (DCMD_USAGE);
704 
705 	if (mdb_vread(&thr, sizeof (thr), addr) != sizeof (thr)) {
706 		mdb_warn("failed to read fmd_thread at %p", addr);
707 		return (DCMD_ERR);
708 	}
709 
710 	if (DCMD_HDRSPEC(flags)) {
711 		mdb_printf("%<u>%-11s %-11s %-8s %-16s%</u>\n",
712 		    "ADDR", "MOD", "TID", "FUNC");
713 	}
714 
715 	mdb_printf("%-11p %-11p %-8u %a\n",
716 	    addr, thr.thr_mod, thr.thr_tid, thr.thr_func);
717 
718 	return (DCMD_OK);
719 }
720 
721 static int
722 mod_walk_init(mdb_walk_state_t *wsp)
723 {
724 	fmd_t F;
725 
726 	if (mdb_readvar(&F, "fmd") != sizeof (F)) {
727 		mdb_warn("failed to read fmd meta-data");
728 		return (WALK_ERR);
729 	}
730 
731 	wsp->walk_addr = (uintptr_t)F.d_mod_list.l_next;
732 	return (WALK_NEXT);
733 }
734 
735 static int
736 mod_walk_step(mdb_walk_state_t *wsp)
737 {
738 	uintptr_t addr = wsp->walk_addr;
739 	fmd_module_t m;
740 
741 	if (addr == NULL)
742 		return (WALK_DONE);
743 
744 	if (mdb_vread(&m, sizeof (m), addr) != sizeof (m)) {
745 		mdb_warn("failed to read fmd_module at %p", addr);
746 		return (WALK_ERR);
747 	}
748 
749 	wsp->walk_addr = (uintptr_t)m.mod_list.l_next;
750 	return (wsp->walk_callback(addr, &m, wsp->walk_cbdata));
751 }
752 
753 static int
754 fmd_module(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
755 {
756 	fmd_module_t mod;
757 	char name[PATH_MAX];
758 
759 	if (!(flags & DCMD_ADDRSPEC))
760 		return (mdb_walk_dcmd("fmd_module", "fmd_module", argc, argv));
761 
762 	if (argc != 0)
763 		return (DCMD_USAGE);
764 
765 	if (mdb_vread(&mod, sizeof (mod), addr) != sizeof (mod)) {
766 		mdb_warn("failed to read fmd_module at %p", addr);
767 		return (DCMD_ERR);
768 	}
769 
770 	if (DCMD_HDRSPEC(flags)) {
771 		mdb_printf("%<u>%-11s %-16s %-11s %-4s %-?s %-16s%</u>\n",
772 		    "ADDR", "OPS", "DATA", "FLAG", "USTAT", "NAME");
773 	}
774 
775 	if (mdb_readstr(name, sizeof (name), (uintptr_t)mod.mod_name) <= 0)
776 		(void) mdb_snprintf(name, sizeof (name), "<%p>", mod.mod_name);
777 
778 	mdb_printf("%-11p %-16a %-11p 0x%02x %-?p %s\n", addr,
779 	    mod.mod_ops, mod.mod_data, mod.mod_flags, mod.mod_ustat, name);
780 
781 	return (DCMD_OK);
782 }
783 
784 static int
785 case_walk_init(mdb_walk_state_t *wsp)
786 {
787 	fmd_module_t mod;
788 	fmd_case_hash_t ch;
789 	fmd_t F;
790 
791 	if (wsp->walk_addr != NULL) {
792 		if (mdb_vread(&mod, sizeof (mod), wsp->walk_addr) == -1) {
793 			mdb_warn("failed to read module at %p", wsp->walk_addr);
794 			return (WALK_ERR);
795 		}
796 
797 		wsp->walk_addr = (uintptr_t)mod.mod_cases.l_next;
798 		return (WALK_NEXT);
799 	}
800 
801 	if (mdb_readvar(&F, "fmd") != sizeof (F) ||
802 	    mdb_vread(&ch, sizeof (ch), (uintptr_t)F.d_cases) != sizeof (ch)) {
803 		mdb_warn("failed to read fmd meta-data");
804 		return (WALK_ERR);
805 	}
806 
807 	return (hash_walk_init(wsp, (uintptr_t)ch.ch_hash, ch.ch_hashlen,
808 	    "fmd_case", sizeof (fmd_case_impl_t),
809 	    OFFSETOF(fmd_case_impl_t, ci_next)));
810 }
811 
812 static int
813 case_walk_step(mdb_walk_state_t *wsp)
814 {
815 	uintptr_t addr = wsp->walk_addr;
816 	fmd_case_impl_t ci;
817 
818 	if (wsp->walk_data != NULL)
819 		return (hash_walk_step(wsp));
820 
821 	if (addr == NULL)
822 		return (WALK_DONE);
823 
824 	if (mdb_vread(&ci, sizeof (ci), addr) != sizeof (ci)) {
825 		mdb_warn("failed to read fmd_case at %p", addr);
826 		return (WALK_ERR);
827 	}
828 
829 	wsp->walk_addr = (uintptr_t)ci.ci_list.l_next;
830 	return (wsp->walk_callback(addr, &ci, wsp->walk_cbdata));
831 }
832 
833 static void
834 case_walk_fini(mdb_walk_state_t *wsp)
835 {
836 	if (wsp->walk_data != NULL)
837 		hash_walk_fini(wsp);
838 }
839 
840 static int
841 fmd_case(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
842 {
843 	char uuid[48], name[16];
844 	fmd_case_impl_t ci;
845 
846 	if (!(flags & DCMD_ADDRSPEC)) {
847 		if (mdb_walk_dcmd("fmd_case", "fmd_case", argc, argv) != 0) {
848 			mdb_warn("failed to walk fmd_case hash");
849 			return (DCMD_ERR);
850 		}
851 		return (DCMD_OK);
852 	}
853 
854 	if (mdb_vread(&ci, sizeof (ci), addr) != sizeof (ci)) {
855 		mdb_warn("failed to read fmd_case at %p", addr);
856 		return (DCMD_ERR);
857 	}
858 
859 	if (DCMD_HDRSPEC(flags)) {
860 		mdb_printf("%<u>%-11s %-5s %-3s %-?s %-36s%</u>\n",
861 		    "ADDR", "STATE", "REF", "DATA", "UUID");
862 	}
863 
864 	if (mdb_readstr(uuid, sizeof (uuid), (uintptr_t)ci.ci_uuid) <= 0)
865 		(void) mdb_snprintf(uuid, sizeof (uuid), "<%p>", ci.ci_uuid);
866 
867 	switch (ci.ci_state) {
868 	case FMD_CASE_UNSOLVED:
869 		(void) strcpy(name, "UNSLV");
870 		break;
871 	case FMD_CASE_SOLVED:
872 		(void) strcpy(name, "SOLVE");
873 		break;
874 	case FMD_CASE_CLOSE_WAIT:
875 		(void) strcpy(name, "CWAIT");
876 		break;
877 	case FMD_CASE_CLOSED:
878 		(void) strcpy(name, "CLOSE");
879 		break;
880 	case FMD_CASE_REPAIRED:
881 		(void) strcpy(name, "RPAIR");
882 		break;
883 	default:
884 		(void) mdb_snprintf(name, sizeof (name), "%u", ci.ci_state);
885 	}
886 
887 	mdb_printf("%-11p %-5s %-3u %-?p %s\n",
888 	    addr, name, ci.ci_refs, ci.ci_data, uuid);
889 
890 	return (DCMD_OK);
891 }
892 
893 static int
894 buf_walk_init(mdb_walk_state_t *wsp)
895 {
896 	fmd_buf_hash_t bh;
897 
898 	if (mdb_vread(&bh, sizeof (bh), wsp->walk_addr) != sizeof (bh)) {
899 		mdb_warn("failed to read fmd_buf_hash_t at %p", wsp->walk_addr);
900 		return (WALK_ERR);
901 	}
902 
903 	return (hash_walk_init(wsp, (uintptr_t)bh.bh_hash, bh.bh_hashlen,
904 	    "fmd_buf", sizeof (fmd_buf_t), OFFSETOF(fmd_buf_t, buf_next)));
905 }
906 
907 /*ARGSUSED*/
908 static int
909 fmd_buf(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
910 {
911 	char name[PATH_MAX];
912 	fmd_buf_t b;
913 
914 	if (argc != 0 || !(flags & DCMD_ADDRSPEC))
915 		return (DCMD_USAGE);
916 
917 	if (mdb_vread(&b, sizeof (b), addr) != sizeof (b)) {
918 		mdb_warn("failed to read fmd_buf at %p", addr);
919 		return (DCMD_ERR);
920 	}
921 
922 	if (DCMD_HDRSPEC(flags)) {
923 		mdb_printf("%<u>%-11s %-32s %-5s %-?s %s%</u>\n",
924 		    "ADDR", "NAME", "FLAGS", "DATA", "SIZE");
925 	}
926 
927 	if (mdb_readstr(name, sizeof (name), (uintptr_t)b.buf_name) <= 0)
928 		(void) mdb_snprintf(name, sizeof (name), "<%p>", b.buf_name);
929 
930 	mdb_printf("%-11p %-32s %-#5x %-?p %lu\n",
931 	    addr, name, b.buf_flags, b.buf_data, b.buf_size);
932 
933 	return (DCMD_OK);
934 }
935 
936 static int
937 serd_walk_init(mdb_walk_state_t *wsp)
938 {
939 	fmd_serd_hash_t sh;
940 
941 	if (mdb_vread(&sh, sizeof (sh), wsp->walk_addr) != sizeof (sh)) {
942 		mdb_warn("failed to read fmd_serd_hash at %p", wsp->walk_addr);
943 		return (WALK_ERR);
944 	}
945 
946 	return (hash_walk_init(wsp, (uintptr_t)sh.sh_hash, sh.sh_hashlen,
947 	    "fmd_serd_eng", sizeof (fmd_serd_eng_t),
948 	    OFFSETOF(fmd_serd_eng_t, sg_next)));
949 }
950 
951 /* ARGSUSED */
952 static int
953 module_serd(uintptr_t addr, const void *data, void *wsp)
954 {
955 	fmd_module_t *modp = (fmd_module_t *)data;
956 
957 	if (modp->mod_serds.sh_count != 0) {
958 		modp = (fmd_module_t *)addr;
959 		(void) mdb_pwalk_dcmd("fmd_serd", "fmd_serd", 0, 0,
960 		    (uintptr_t)&modp->mod_serds);
961 	}
962 	return (WALK_NEXT);
963 }
964 
965 /*ARGSUSED*/
966 static int
967 fmd_serd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
968 {
969 	char name[PATH_MAX];
970 	fmd_serd_eng_t sg;
971 
972 	if (argc != 0)
973 		return (DCMD_USAGE);
974 	if (!(flags & DCMD_ADDRSPEC)) {
975 		if (mdb_walk("fmd_module", module_serd, 0) == -1) {
976 			mdb_warn("failed to walk 'fmd_module'");
977 			return (DCMD_ERR);
978 		}
979 		return (DCMD_OK);
980 	}
981 
982 	if (mdb_vread(&sg, sizeof (sg), addr) != sizeof (sg)) {
983 		mdb_warn("failed to read fmd_serd_eng at %p", addr);
984 		return (DCMD_ERR);
985 	}
986 
987 	if (DCMD_HDRSPEC(flags)) {
988 		mdb_printf("%<u>%-11s %-32s %-3s F >%-2s %-16s%</u>\n",
989 		    "ADDR", "NAME", "CNT", "N", "T");
990 	}
991 
992 	if (mdb_readstr(name, sizeof (name), (uintptr_t)sg.sg_name) <= 0)
993 		(void) mdb_snprintf(name, sizeof (name), "<%p>", sg.sg_name);
994 
995 	mdb_printf("%-11p %-32s %-3u %c >%-2u %lluns\n",
996 	    addr, name, sg.sg_count, (sg.sg_flags & FMD_SERD_FIRED) ? 'F' : ' ',
997 	    sg.sg_n, (u_longlong_t)sg.sg_t);
998 
999 	return (DCMD_OK);
1000 }
1001 
1002 static int
1003 asru_walk_init(mdb_walk_state_t *wsp)
1004 {
1005 	fmd_asru_hash_t ah;
1006 	fmd_t F;
1007 
1008 	if (wsp->walk_addr == NULL && mdb_readvar(&F, "fmd") != sizeof (F)) {
1009 		mdb_warn("failed to read fmd meta-data");
1010 		return (WALK_ERR);
1011 	}
1012 
1013 	if (wsp->walk_addr == NULL)
1014 		wsp->walk_addr = (uintptr_t)F.d_asrus;
1015 
1016 	if (mdb_vread(&ah, sizeof (ah), wsp->walk_addr) != sizeof (ah)) {
1017 		mdb_warn("failed to read asru_hash at %p", wsp->walk_addr);
1018 		return (WALK_ERR);
1019 	}
1020 
1021 	return (hash_walk_init(wsp, (uintptr_t)ah.ah_hash, ah.ah_hashlen,
1022 	    "fmd_asru", sizeof (fmd_asru_t), OFFSETOF(fmd_asru_t, asru_next)));
1023 }
1024 
1025 static int
1026 fmd_asru(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1027 {
1028 	char uuid[48], name[PATH_MAX];
1029 	fmd_asru_t a;
1030 
1031 	if (!(flags & DCMD_ADDRSPEC)) {
1032 		if (mdb_walk_dcmd("fmd_asru", "fmd_asru", argc, argv) != 0) {
1033 			mdb_warn("failed to walk fmd_asru hash");
1034 			return (DCMD_ERR);
1035 		}
1036 		return (DCMD_OK);
1037 	}
1038 
1039 	if (mdb_vread(&a, sizeof (a), addr) != sizeof (a)) {
1040 		mdb_warn("failed to read fmd_asru at %p", addr);
1041 		return (DCMD_ERR);
1042 	}
1043 
1044 	if (DCMD_HDRSPEC(flags))
1045 		mdb_printf("%<u>%-8s %-36s %s%</u>\n", "ADDR", "UUID", "NAME");
1046 
1047 	if (mdb_readstr(uuid, sizeof (uuid), (uintptr_t)a.asru_uuid) <= 0)
1048 		(void) mdb_snprintf(uuid, sizeof (uuid), "<%p>", a.asru_uuid);
1049 	if (mdb_readstr(name, sizeof (name), (uintptr_t)a.asru_name) <= 0)
1050 		(void) mdb_snprintf(name, sizeof (name), "<%p>", a.asru_name);
1051 
1052 	mdb_printf("%-8p %-36s %s\n", addr, uuid, name);
1053 	return (DCMD_OK);
1054 }
1055 
1056 /*ARGSUSED*/
1057 static int
1058 fcf_hdr(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1059 {
1060 	fcf_hdr_t h;
1061 
1062 	if (argc != 0)
1063 		return (DCMD_USAGE);
1064 
1065 	if (!(flags & DCMD_ADDRSPEC))
1066 		addr = 0; /* assume base of file in file target */
1067 
1068 	if (mdb_vread(&h, sizeof (h), addr) != sizeof (h)) {
1069 		mdb_warn("failed to read header at %p", addr);
1070 		return (DCMD_ERR);
1071 	}
1072 
1073 	mdb_printf("fcfh_ident.id_magic = 0x%x, %c, %c, %c\n",
1074 	    h.fcfh_ident[FCF_ID_MAG0], h.fcfh_ident[FCF_ID_MAG1],
1075 	    h.fcfh_ident[FCF_ID_MAG2], h.fcfh_ident[FCF_ID_MAG3]);
1076 
1077 	switch (h.fcfh_ident[FCF_ID_MODEL]) {
1078 	case FCF_MODEL_ILP32:
1079 		mdb_printf("fcfh_ident.id_model = ILP32\n");
1080 		break;
1081 	case FCF_MODEL_LP64:
1082 		mdb_printf("fcfh_ident.id_model = LP64\n");
1083 		break;
1084 	default:
1085 		mdb_printf("fcfh_ident.id_model = 0x%x\n",
1086 		    h.fcfh_ident[FCF_ID_MODEL]);
1087 	}
1088 
1089 	switch (h.fcfh_ident[FCF_ID_ENCODING]) {
1090 	case FCF_ENCODE_LSB:
1091 		mdb_printf("fcfh_ident.id_encoding = LSB\n");
1092 		break;
1093 	case FCF_ENCODE_MSB:
1094 		mdb_printf("fcfh_ident.id_encoding = MSB\n");
1095 		break;
1096 	default:
1097 		mdb_printf("fcfh_ident.id_encoding = 0x%x\n",
1098 		    h.fcfh_ident[FCF_ID_ENCODING]);
1099 	}
1100 
1101 	mdb_printf("fcfh_ident.id_version = %u\n",
1102 	    h.fcfh_ident[FCF_ID_VERSION]);
1103 
1104 	mdb_printf("fcfh_flags = 0x%x\n", h.fcfh_flags);
1105 	mdb_printf("fcfh_hdrsize = %u\n", h.fcfh_hdrsize);
1106 	mdb_printf("fcfh_secsize = %u\n", h.fcfh_secsize);
1107 	mdb_printf("fcfh_secnum = %u\n", h.fcfh_secnum);
1108 	mdb_printf("fcfh_secoff = %llu\n", h.fcfh_secoff);
1109 	mdb_printf("fcfh_filesz = %llu\n", h.fcfh_filesz);
1110 	mdb_printf("fcfh_cgen = %llu\n", h.fcfh_cgen);
1111 
1112 	return (DCMD_OK);
1113 }
1114 
1115 /*ARGSUSED*/
1116 static int
1117 fcf_sec_one(uintptr_t addr, void *ignored, uint_t *secp)
1118 {
1119 	static int fcf_sec(uintptr_t, uint_t, int, const mdb_arg_t *);
1120 
1121 	mdb_printf("%3d ", (*secp)++);
1122 	(void) fcf_sec(addr, DCMD_ADDRSPEC | DCMD_LOOP, 0, NULL);
1123 	return (WALK_NEXT);
1124 }
1125 
1126 /*ARGSUSED*/
1127 static int
1128 fcf_sec(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1129 {
1130 	static const char *const types[] = {
1131 		"none",		/* FCF_SECT_NONE */
1132 		"strtab",	/* FCF_SECT_STRTAB */
1133 		"module",	/* FCF_SECT_MODULE */
1134 		"case",		/* FCF_SECT_CASE */
1135 		"bufs",		/* FCF_SECT_BUFS */
1136 		"buffer",	/* FCF_SECT_BUFFER */
1137 		"serd",		/* FCF_SECT_SERD */
1138 		"events",	/* FCF_SECT_EVENTS */
1139 		"nvlists",	/* FCF_SECT_NVLISTS */
1140 	};
1141 
1142 	uint_t sec = 0;
1143 	fcf_sec_t s;
1144 
1145 	if (!(flags & DCMD_ADDRSPEC))
1146 		mdb_printf("%<u>%-3s ", "NDX");
1147 
1148 	if (!(flags & DCMD_ADDRSPEC) || DCMD_HDRSPEC(flags)) {
1149 		mdb_printf("%<u>%?s %-10s %-5s %-5s %-5s %-6s %-5s%</u>\n",
1150 		    "ADDR", "TYPE", "ALIGN", "FLAGS", "ENTSZ", "OFF", "SIZE");
1151 	}
1152 
1153 	if (!(flags & DCMD_ADDRSPEC)) {
1154 		if (mdb_walk("fcf_sec", (mdb_walk_cb_t)fcf_sec_one, &sec) < 0) {
1155 			mdb_warn("failed to walk fcf_sec");
1156 			return (DCMD_ERR);
1157 		}
1158 		return (DCMD_OK);
1159 	}
1160 
1161 	if (argc != 0)
1162 		return (DCMD_USAGE);
1163 
1164 	if (mdb_vread(&s, sizeof (s), addr) != sizeof (s)) {
1165 		mdb_warn("failed to read section header at %p", addr);
1166 		return (DCMD_ERR);
1167 	}
1168 
1169 	mdb_printf("%?p ", addr);
1170 
1171 	if (s.fcfs_type < sizeof (types) / sizeof (types[0]))
1172 		mdb_printf("%-10s ", types[s.fcfs_type]);
1173 	else
1174 		mdb_printf("%-10u ", s.fcfs_type);
1175 
1176 	mdb_printf("%-5u %-#5x %-#5x %-6llx %-#5llx\n", s.fcfs_align,
1177 	    s.fcfs_flags, s.fcfs_entsize, s.fcfs_offset, s.fcfs_size);
1178 
1179 	return (DCMD_OK);
1180 }
1181 
1182 static int
1183 fcf_sec_walk_init(mdb_walk_state_t *wsp)
1184 {
1185 	fcf_hdr_t h, *hp;
1186 	size_t size;
1187 
1188 	if (mdb_vread(&h, sizeof (h), wsp->walk_addr) != sizeof (h)) {
1189 		mdb_warn("failed to read FCF header at %p", wsp->walk_addr);
1190 		return (WALK_ERR);
1191 	}
1192 
1193 	size = sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * h.fcfh_secnum;
1194 	hp = mdb_alloc(size, UM_SLEEP);
1195 
1196 	if (mdb_vread(hp, size, wsp->walk_addr) != size) {
1197 		mdb_warn("failed to read FCF sections at %p", wsp->walk_addr);
1198 		mdb_free(hp, size);
1199 		return (WALK_ERR);
1200 	}
1201 
1202 	wsp->walk_data = hp;
1203 	wsp->walk_arg = 0;
1204 
1205 	return (WALK_NEXT);
1206 }
1207 
1208 static int
1209 fcf_sec_walk_step(mdb_walk_state_t *wsp)
1210 {
1211 	uint_t i = (uint_t)wsp->walk_arg;
1212 	size_t off = sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * i;
1213 	fcf_hdr_t *hp = wsp->walk_data;
1214 	fcf_sec_t *sp = (fcf_sec_t *)((uintptr_t)hp + off);
1215 
1216 	if (i >= hp->fcfh_secnum)
1217 		return (WALK_DONE);
1218 
1219 	wsp->walk_arg = (void *)(i + 1);
1220 	return (wsp->walk_callback(wsp->walk_addr + off, sp, wsp->walk_cbdata));
1221 }
1222 
1223 static void
1224 fcf_sec_walk_fini(mdb_walk_state_t *wsp)
1225 {
1226 	fcf_hdr_t *hp = wsp->walk_data;
1227 	mdb_free(hp, sizeof (fcf_hdr_t) + sizeof (fcf_sec_t) * hp->fcfh_secnum);
1228 }
1229 
1230 /*ARGSUSED*/
1231 static int
1232 fcf_case(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1233 {
1234 	fcf_case_t fcfc;
1235 
1236 	if (argc != 0)
1237 		return (DCMD_USAGE);
1238 
1239 	if (mdb_vread(&fcfc, sizeof (fcfc), addr) != sizeof (fcfc)) {
1240 		mdb_warn("failed to read case at %p", addr);
1241 		return (DCMD_ERR);
1242 	}
1243 
1244 	mdb_printf("fcfc_uuid = 0x%x\n", fcfc.fcfc_uuid);
1245 	mdb_printf("fcfc_state = %u\n", fcfc.fcfc_state);
1246 	mdb_printf("fcfc_bufs = %u\n", fcfc.fcfc_bufs);
1247 	mdb_printf("fcfc_events = %u\n", fcfc.fcfc_events);
1248 	mdb_printf("fcfc_suspects = %u\n", fcfc.fcfc_suspects);
1249 
1250 	return (DCMD_OK);
1251 }
1252 
1253 /*ARGSUSED*/
1254 static int
1255 fcf_event(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1256 {
1257 	fcf_event_t fcfe;
1258 
1259 	if (argc != 0)
1260 		return (DCMD_USAGE);
1261 
1262 	if (mdb_vread(&fcfe, sizeof (fcfe), addr) != sizeof (fcfe)) {
1263 		mdb_warn("failed to read event at %p", addr);
1264 		return (DCMD_ERR);
1265 	}
1266 
1267 	mdb_printf("fcfe_todsec = %llu (%Y)\n",
1268 	    fcfe.fcfe_todsec, (time_t)fcfe.fcfe_todsec);
1269 	mdb_printf("fcfe_todnsec = %llu\n", fcfe.fcfe_todnsec);
1270 	mdb_printf("fcfe_major = %u\n", fcfe.fcfe_major);
1271 	mdb_printf("fcfe_minor = %u\n", fcfe.fcfe_minor);
1272 	mdb_printf("fcfe_inode = %llu\n", fcfe.fcfe_inode);
1273 	mdb_printf("fcfe_offset = %llu\n", fcfe.fcfe_offset);
1274 
1275 	return (DCMD_OK);
1276 }
1277 
1278 /*ARGSUSED*/
1279 static int
1280 fcf_serd(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1281 {
1282 	fcf_serd_t fcfd;
1283 
1284 	if (argc != 0)
1285 		return (DCMD_USAGE);
1286 
1287 	if (mdb_vread(&fcfd, sizeof (fcfd), addr) != sizeof (fcfd)) {
1288 		mdb_warn("failed to read serd at %p", addr);
1289 		return (DCMD_ERR);
1290 	}
1291 
1292 	mdb_printf("fcfd_name = 0x%x\n", fcfd.fcfd_name);
1293 	mdb_printf("fcfd_events = %u\n", fcfd.fcfd_events);
1294 	mdb_printf("fcfd_n = >%u\n", fcfd.fcfd_n);
1295 	mdb_printf("fcfd_t = %lluns\n", fcfd.fcfd_t);
1296 
1297 	return (DCMD_OK);
1298 }
1299 
1300 static int
1301 tmq_walk_init(mdb_walk_state_t *wsp)
1302 {
1303 	fmd_timerq_t tmq;
1304 	fmd_t F;
1305 
1306 	if (wsp->walk_addr == NULL && mdb_readvar(&F, "fmd") != sizeof (F)) {
1307 		mdb_warn("failed to read fmd meta-data");
1308 		return (WALK_ERR);
1309 	}
1310 
1311 	if (wsp->walk_addr == NULL)
1312 		wsp->walk_addr = (uintptr_t)F.d_timers;
1313 
1314 	if (mdb_vread(&tmq, sizeof (tmq), wsp->walk_addr) != sizeof (tmq)) {
1315 		mdb_warn("failed to read timerq at %p", wsp->walk_addr);
1316 		return (WALK_ERR);
1317 	}
1318 
1319 	wsp->walk_addr = (uintptr_t)tmq.tmq_list.l_next;
1320 	return (WALK_NEXT);
1321 }
1322 
1323 static int
1324 tmq_walk_step(mdb_walk_state_t *wsp)
1325 {
1326 	uintptr_t addr = wsp->walk_addr;
1327 	fmd_timer_t tmr;
1328 
1329 	if (addr == NULL)
1330 		return (WALK_DONE);
1331 
1332 	if (mdb_vread(&tmr, sizeof (tmr), addr) != sizeof (tmr)) {
1333 		mdb_warn("failed to read fmd_timer at %p", addr);
1334 		return (WALK_ERR);
1335 	}
1336 
1337 	wsp->walk_addr = (uintptr_t)tmr.tmr_list.l_next;
1338 	return (wsp->walk_callback(addr, &tmr, wsp->walk_cbdata));
1339 }
1340 
1341 static int
1342 fmd_timer(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1343 {
1344 	char name[32], func[MDB_SYM_NAMLEN];
1345 	fmd_timer_t t;
1346 
1347 	if (!(flags & DCMD_ADDRSPEC)) {
1348 		if (mdb_walk_dcmd("fmd_timerq", "fmd_timer", argc, argv) != 0) {
1349 			mdb_warn("failed to walk fmd_timerq");
1350 			return (DCMD_ERR);
1351 		}
1352 		return (DCMD_OK);
1353 	}
1354 
1355 	if (mdb_vread(&t, sizeof (t), addr) != sizeof (t)) {
1356 		mdb_warn("failed to read fmd_timer at %p", addr);
1357 		return (DCMD_ERR);
1358 	}
1359 
1360 	if (DCMD_HDRSPEC(flags)) {
1361 		mdb_printf("%<u>%-8s %-20s %-4s %-18s %-8s %s%</u>\n",
1362 		    "ADDR", "MODULE", "ID", "HRTIME", "ARG", "FUNC");
1363 	}
1364 
1365 	if (mdb_readstr(name, sizeof (name), (uintptr_t)
1366 	    t.tmr_ids + OFFSETOF(fmd_idspace_t, ids_name)) <= 0)
1367 		(void) mdb_snprintf(name, sizeof (name), "<%p>", t.tmr_ids);
1368 
1369 	if (mdb_lookup_by_addr((uintptr_t)t.tmr_func, MDB_SYM_FUZZY,
1370 	    func, sizeof (func), NULL) != 0)
1371 		(void) mdb_snprintf(func, sizeof (func), "<%p>", t.tmr_func);
1372 
1373 	mdb_printf("%-8p %-20s %4d 0x%-16llx %-8p %s\n",
1374 	    addr, name, t.tmr_id, t.tmr_hrt, t.tmr_arg, func);
1375 	return (DCMD_OK);
1376 }
1377 
1378 static int
1379 xprt_walk_init(mdb_walk_state_t *wsp)
1380 {
1381 	fmd_module_t m;
1382 
1383 	if (wsp->walk_addr == NULL) {
1384 		mdb_warn("transport walker requires fmd_module_t address\n");
1385 		return (WALK_ERR);
1386 	}
1387 
1388 	if (mdb_vread(&m, sizeof (m), wsp->walk_addr) != sizeof (m)) {
1389 		mdb_warn("failed to read module at %p", wsp->walk_addr);
1390 		return (WALK_ERR);
1391 	}
1392 
1393 	wsp->walk_addr = (uintptr_t)m.mod_transports.l_next;
1394 	return (WALK_NEXT);
1395 }
1396 
1397 static int
1398 xprt_walk_step(mdb_walk_state_t *wsp)
1399 {
1400 	uintptr_t addr = wsp->walk_addr;
1401 	fmd_xprt_impl_t xi;
1402 
1403 	if (addr == NULL)
1404 		return (WALK_DONE);
1405 
1406 	if (mdb_vread(&xi, sizeof (xi), addr) != sizeof (xi)) {
1407 		mdb_warn("failed to read fmd_xprt at %p", addr);
1408 		return (WALK_ERR);
1409 	}
1410 
1411 	wsp->walk_addr = (uintptr_t)xi.xi_list.l_next;
1412 	return (wsp->walk_callback(addr, &xi, wsp->walk_cbdata));
1413 }
1414 
1415 static int
1416 xpc_walk_init(mdb_walk_state_t *wsp)
1417 {
1418 	fmd_xprt_class_hash_t xch;
1419 
1420 	if (mdb_vread(&xch, sizeof (xch), wsp->walk_addr) != sizeof (xch)) {
1421 		mdb_warn("failed to read fmd_xprt_class_hash at %p",
1422 		    wsp->walk_addr);
1423 		return (WALK_ERR);
1424 	}
1425 
1426 	return (hash_walk_init(wsp, (uintptr_t)xch.xch_hash, xch.xch_hashlen,
1427 	    "fmd_xprt_class", sizeof (fmd_xprt_class_t),
1428 	    OFFSETOF(fmd_xprt_class_t, xc_next)));
1429 }
1430 
1431 /*ARGSUSED*/
1432 static int
1433 fmd_xprt_class(uintptr_t addr, const void *data, void *arg)
1434 {
1435 	const fmd_xprt_class_t *xcp = data;
1436 	char name[1024];
1437 
1438 	if (mdb_readstr(name, sizeof (name), (uintptr_t)xcp->xc_class) <= 0)
1439 		(void) mdb_snprintf(name, sizeof (name), "<%p>", xcp->xc_class);
1440 
1441 	mdb_printf("%-8p %-4u %s\n", addr, xcp->xc_refs, name);
1442 	return (WALK_NEXT);
1443 }
1444 
1445 static int
1446 fmd_xprt(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
1447 {
1448 	uint_t opt_s = FALSE, opt_l = FALSE, opt_r = FALSE, opt_u = FALSE;
1449 	fmd_xprt_impl_t xi;
1450 
1451 	if (mdb_getopts(argc, argv,
1452 	    'l', MDB_OPT_SETBITS, TRUE, &opt_l,
1453 	    'r', MDB_OPT_SETBITS, TRUE, &opt_r,
1454 	    's', MDB_OPT_SETBITS, TRUE, &opt_s,
1455 	    'u', MDB_OPT_SETBITS, TRUE, &opt_u, NULL) != argc)
1456 		return (DCMD_USAGE);
1457 
1458 	if (!(flags & DCMD_ADDRSPEC)) {
1459 		if (mdb_walk_dcmd("fmd_xprt", "fmd_xprt", argc, argv) != 0) {
1460 			mdb_warn("failed to walk fmd_xprt");
1461 			return (DCMD_ERR);
1462 		}
1463 		return (DCMD_OK);
1464 	}
1465 
1466 	if (mdb_vread(&xi, sizeof (xi), addr) != sizeof (xi)) {
1467 		mdb_warn("failed to read fmd_xprt at %p", addr);
1468 		return (DCMD_ERR);
1469 	}
1470 
1471 	if (DCMD_HDRSPEC(flags)) {
1472 		mdb_printf("%<u>%-8s %-4s %-4s %-5s %s%</u>\n",
1473 		    "ADDR", "ID", "VERS", "FLAGS", "STATE");
1474 	}
1475 
1476 	mdb_printf("%-8p %-4d %-4u %-5x %a\n",
1477 	    addr, xi.xi_id, xi.xi_version, xi.xi_flags, xi.xi_state);
1478 
1479 	if (opt_l | opt_s) {
1480 		(void) mdb_inc_indent(4);
1481 		mdb_printf("Local subscriptions requested by peer:\n");
1482 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1483 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1484 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_lsub));
1485 		(void) mdb_dec_indent(4);
1486 	}
1487 
1488 	if (opt_r | opt_s) {
1489 		(void) mdb_inc_indent(4);
1490 		mdb_printf("Remote subscriptions requested of peer:\n");
1491 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1492 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1493 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_rsub));
1494 		(void) mdb_dec_indent(4);
1495 	}
1496 
1497 	if (opt_u | opt_s) {
1498 		(void) mdb_inc_indent(4);
1499 		mdb_printf("Pending unsubscription acknowledgements:\n");
1500 		mdb_printf("%<u>%-8s %-4s %s%</u>\n", "ADDR", "REFS", "CLASS");
1501 		(void) mdb_pwalk("fmd_xprt_class", fmd_xprt_class, &xi,
1502 		    addr + OFFSETOF(fmd_xprt_impl_t, xi_usub));
1503 		(void) mdb_dec_indent(4);
1504 	}
1505 
1506 	return (DCMD_OK);
1507 }
1508 
1509 static const mdb_dcmd_t dcmds[] = {
1510 	{ "fcf_case", "?", "print a FCF case", fcf_case },
1511 	{ "fcf_event", "?", "print a FCF event", fcf_event },
1512 	{ "fcf_hdr", "?", "print a FCF header", fcf_hdr },
1513 	{ "fcf_sec", ":", "print a FCF section header", fcf_sec },
1514 	{ "fcf_serd", "?", "print a FCF serd engine", fcf_serd },
1515 	{ "fmd_trace", "?[-cs]", "display thread trace buffer(s)", fmd_trace },
1516 	{ "fmd_ustat", "[:]", "display statistics collection", fmd_ustat },
1517 	{ "fmd_stat", "[:]", "display statistic structure", fmd_stat },
1518 	{ "fmd_event", NULL, "display event structure", fmd_event },
1519 	{ "fmd_thread", "?", "display thread or list of threads", fmd_thread },
1520 	{ "fmd_module", "?", "display module or list of modules", fmd_module },
1521 	{ "fmd_case", ":", "display case file structure", fmd_case },
1522 	{ "fmd_buf", ":", "display buffer structure", fmd_buf },
1523 	{ "fmd_serd", "[:]", "display serd engine structure", fmd_serd },
1524 	{ "fmd_asru", "?", "display asru resource structure", fmd_asru },
1525 	{ "fmd_timer", "?", "display pending timer(s)", fmd_timer },
1526 	{ "fmd_xprt", "?[-lrsu]", "display event transport(s)", fmd_xprt },
1527 	{ NULL }
1528 };
1529 
1530 static const mdb_walker_t walkers[] = {
1531 	{ "fcf_sec", "walk FCF section header table given header address",
1532 		fcf_sec_walk_init, fcf_sec_walk_step, fcf_sec_walk_fini },
1533 	{ "fmd_trace", "walk per-thread trace buffers",
1534 		trwalk_init, trwalk_step, trwalk_fini },
1535 	{ "fmd_ustat", "walk per-collection statistics",
1536 		ustat_walk_init, ustat_walk_step, hash_walk_fini },
1537 	{ "fmd_thread", "walk list of all fmd_thread_t's",
1538 		thread_walk_init, thread_walk_step, NULL },
1539 	{ "fmd_module", "walk list of all fmd_module_t's",
1540 		mod_walk_init, mod_walk_step, NULL },
1541 	{ "fmd_case", "walk per-module case objects",
1542 		case_walk_init, case_walk_step, case_walk_fini },
1543 	{ "fmd_buf", "walk per-buf_hash buffers",
1544 		buf_walk_init, hash_walk_step, hash_walk_fini },
1545 	{ "fmd_serd", "walk per-serd_hash engines",
1546 		serd_walk_init, hash_walk_step, hash_walk_fini },
1547 	{ "fmd_asru", "walk asru resource hash",
1548 		asru_walk_init, hash_walk_step, hash_walk_fini },
1549 	{ "fmd_timerq", "walk timer queue",
1550 		tmq_walk_init, tmq_walk_step, NULL },
1551 	{ "fmd_xprt", "walk per-module list of transports",
1552 		xprt_walk_init, xprt_walk_step, NULL },
1553 	{ "fmd_xprt_class", "walk hash table of subscription classes",
1554 		xpc_walk_init, hash_walk_step, hash_walk_fini },
1555 	{ NULL, NULL, NULL, NULL, NULL }
1556 };
1557 
1558 static const mdb_modinfo_t modinfo = { MDB_API_VERSION, dcmds, walkers };
1559 
1560 const mdb_modinfo_t *
1561 _mdb_init(void)
1562 {
1563 	return (&modinfo);
1564 }
1565