xref: /titanic_50/usr/src/cmd/mdb/common/modules/genunix/findstack.c (revision 98144673ce45bddc6d5dbe7e2afab720c660b5d7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2013, Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
25  */
26 
27 #include <mdb/mdb_modapi.h>
28 #include <mdb/mdb_ctf.h>
29 
30 #include <sys/types.h>
31 #include <sys/regset.h>
32 #include <sys/stack.h>
33 #include <sys/thread.h>
34 #include <sys/modctl.h>
35 #include <assert.h>
36 
37 #include "findstack.h"
38 #include "thread.h"
39 #include "sobj.h"
40 
41 int findstack_debug_on = 0;
42 
43 /*
44  * "sp" is a kernel VA.
45  */
46 static int
print_stack(uintptr_t sp,uintptr_t pc,uintptr_t addr,int argc,const mdb_arg_t * argv,int free_state)47 print_stack(uintptr_t sp, uintptr_t pc, uintptr_t addr,
48     int argc, const mdb_arg_t *argv, int free_state)
49 {
50 	int showargs = 0, count, err;
51 
52 	count = mdb_getopts(argc, argv,
53 	    'v', MDB_OPT_SETBITS, TRUE, &showargs, NULL);
54 	argc -= count;
55 	argv += count;
56 
57 	if (argc > 1 || (argc == 1 && argv->a_type != MDB_TYPE_STRING))
58 		return (DCMD_USAGE);
59 
60 	mdb_printf("stack pointer for thread %p%s: %p\n",
61 	    addr, (free_state ? " (TS_FREE)" : ""), sp);
62 	if (pc != 0)
63 		mdb_printf("[ %0?lr %a() ]\n", sp, pc);
64 
65 	mdb_inc_indent(2);
66 	mdb_set_dot(sp);
67 
68 	if (argc == 1)
69 		err = mdb_eval(argv->a_un.a_str);
70 	else if (showargs)
71 		err = mdb_eval("<.$C");
72 	else
73 		err = mdb_eval("<.$C0");
74 
75 	mdb_dec_indent(2);
76 
77 	return ((err == -1) ? DCMD_ABORT : DCMD_OK);
78 }
79 
80 int
findstack(uintptr_t addr,uint_t flags,int argc,const mdb_arg_t * argv)81 findstack(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
82 {
83 	findstack_info_t fsi;
84 	int retval;
85 
86 	if (!(flags & DCMD_ADDRSPEC))
87 		return (DCMD_USAGE);
88 
89 	bzero(&fsi, sizeof (fsi));
90 
91 	if ((retval = stacks_findstack(addr, &fsi, 1)) != DCMD_OK ||
92 	    fsi.fsi_failed)
93 		return (retval);
94 
95 	return (print_stack(fsi.fsi_sp, fsi.fsi_pc, addr,
96 	    argc, argv, fsi.fsi_tstate == TS_FREE));
97 }
98 
99 /*ARGSUSED*/
100 int
findstack_debug(uintptr_t addr,uint_t flags,int argc,const mdb_arg_t * av)101 findstack_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *av)
102 {
103 	findstack_debug_on ^= 1;
104 
105 	mdb_printf("findstack: debugging is now %s\n",
106 	    findstack_debug_on ? "on" : "off");
107 
108 	return (DCMD_OK);
109 }
110 
111 static void
uppercase(char * p)112 uppercase(char *p)
113 {
114 	for (; *p != '\0'; p++) {
115 		if (*p >= 'a' && *p <= 'z')
116 			*p += 'A' - 'a';
117 	}
118 }
119 
120 static void
sobj_to_text(uintptr_t addr,char * out,size_t out_sz)121 sobj_to_text(uintptr_t addr, char *out, size_t out_sz)
122 {
123 	sobj_ops_to_text(addr, out, out_sz);
124 	uppercase(out);
125 }
126 
127 #define	SOBJ_ALL	1
128 
129 static int
text_to_sobj(const char * text,uintptr_t * out)130 text_to_sobj(const char *text, uintptr_t *out)
131 {
132 	if (strcasecmp(text, "ALL") == 0) {
133 		*out = SOBJ_ALL;
134 		return (0);
135 	}
136 
137 	return (sobj_text_to_ops(text, out));
138 }
139 
140 #define	TSTATE_PANIC	-2U
141 static int
text_to_tstate(const char * text,uint_t * out)142 text_to_tstate(const char *text, uint_t *out)
143 {
144 	if (strcasecmp(text, "panic") == 0)
145 		*out = TSTATE_PANIC;
146 	else if (thread_text_to_state(text, out) != 0) {
147 		mdb_warn("tstate \"%s\" not recognized\n", text);
148 		return (-1);
149 	}
150 	return (0);
151 }
152 
153 static void
tstate_to_text(uint_t tstate,uint_t paniced,char * out,size_t out_sz)154 tstate_to_text(uint_t tstate, uint_t paniced, char *out, size_t out_sz)
155 {
156 	if (paniced)
157 		mdb_snprintf(out, out_sz, "panic");
158 	else
159 		thread_state_to_text(tstate, out, out_sz);
160 	uppercase(out);
161 }
162 
163 typedef struct stacks_entry {
164 	struct stacks_entry	*se_next;
165 	struct stacks_entry	*se_dup;	/* dups of this stack */
166 	uintptr_t		se_thread;
167 	uintptr_t		se_sp;
168 	uintptr_t		se_sobj_ops;
169 	uint32_t		se_tstate;
170 	uint32_t		se_count;	/* # threads w/ this stack */
171 	uint8_t			se_overflow;
172 	uint8_t			se_depth;
173 	uint8_t			se_failed;	/* failure reason; FSI_FAIL_* */
174 	uint8_t			se_panic;
175 	uintptr_t		se_stack[1];
176 } stacks_entry_t;
177 #define	STACKS_ENTRY_SIZE(x) OFFSETOF(stacks_entry_t, se_stack[(x)])
178 
179 #define	STACKS_HSIZE 127
180 
181 /* Maximum stack depth reported in stacks */
182 #define	STACKS_MAX_DEPTH	254
183 
184 typedef struct stacks_info {
185 	size_t		si_count;	/* total stacks_entry_ts (incl dups) */
186 	size_t		si_entries;	/* # entries in hash table */
187 	stacks_entry_t	**si_hash;	/* hash table */
188 	findstack_info_t si_fsi;	/* transient callback state */
189 } stacks_info_t;
190 
191 /* global state cached between invocations */
192 #define	STACKS_STATE_CLEAN	0
193 #define	STACKS_STATE_DIRTY	1
194 #define	STACKS_STATE_DONE	2
195 static uint_t stacks_state = STACKS_STATE_CLEAN;
196 static stacks_entry_t **stacks_hash;
197 static stacks_entry_t **stacks_array;
198 static size_t stacks_array_size;
199 
200 size_t
stacks_hash_entry(stacks_entry_t * sep)201 stacks_hash_entry(stacks_entry_t *sep)
202 {
203 	size_t depth = sep->se_depth;
204 	uintptr_t *stack = sep->se_stack;
205 
206 	uint64_t total = depth;
207 
208 	while (depth > 0) {
209 		total += *stack;
210 		stack++; depth--;
211 	}
212 
213 	return (total % STACKS_HSIZE);
214 }
215 
216 /*
217  * This is used to both compare stacks for equality and to sort the final
218  * list of unique stacks.  forsort specifies the latter behavior, which
219  * additionally:
220  *	compares se_count, and
221  *	sorts the stacks by text function name.
222  *
223  * The equality test is independent of se_count, and doesn't care about
224  * relative ordering, so we don't do the extra work of looking up symbols
225  * for the stack addresses.
226  */
227 int
stacks_entry_comp_impl(stacks_entry_t * l,stacks_entry_t * r,uint_t forsort)228 stacks_entry_comp_impl(stacks_entry_t *l, stacks_entry_t *r,
229     uint_t forsort)
230 {
231 	int idx;
232 
233 	int depth = MIN(l->se_depth, r->se_depth);
234 
235 	/* no matter what, panic stacks come last. */
236 	if (l->se_panic > r->se_panic)
237 		return (1);
238 	if (l->se_panic < r->se_panic)
239 		return (-1);
240 
241 	if (forsort) {
242 		/* put large counts earlier */
243 		if (l->se_count > r->se_count)
244 			return (-1);
245 		if (l->se_count < r->se_count)
246 			return (1);
247 	}
248 
249 	if (l->se_tstate > r->se_tstate)
250 		return (1);
251 	if (l->se_tstate < r->se_tstate)
252 		return (-1);
253 
254 	if (l->se_failed > r->se_failed)
255 		return (1);
256 	if (l->se_failed < r->se_failed)
257 		return (-1);
258 
259 	for (idx = 0; idx < depth; idx++) {
260 		char lbuf[MDB_SYM_NAMLEN];
261 		char rbuf[MDB_SYM_NAMLEN];
262 
263 		int rval;
264 		uintptr_t laddr = l->se_stack[idx];
265 		uintptr_t raddr = r->se_stack[idx];
266 
267 		if (laddr == raddr)
268 			continue;
269 
270 		if (forsort &&
271 		    mdb_lookup_by_addr(laddr, MDB_SYM_FUZZY,
272 		    lbuf, sizeof (lbuf), NULL) != -1 &&
273 		    mdb_lookup_by_addr(raddr, MDB_SYM_FUZZY,
274 		    rbuf, sizeof (rbuf), NULL) != -1 &&
275 		    (rval = strcmp(lbuf, rbuf)) != 0)
276 			return (rval);
277 
278 		if (laddr > raddr)
279 			return (1);
280 		return (-1);
281 	}
282 
283 	if (l->se_overflow > r->se_overflow)
284 		return (-1);
285 	if (l->se_overflow < r->se_overflow)
286 		return (1);
287 
288 	if (l->se_depth > r->se_depth)
289 		return (1);
290 	if (l->se_depth < r->se_depth)
291 		return (-1);
292 
293 	if (l->se_sobj_ops > r->se_sobj_ops)
294 		return (1);
295 	if (l->se_sobj_ops < r->se_sobj_ops)
296 		return (-1);
297 
298 	return (0);
299 }
300 
301 int
stacks_entry_comp(const void * l_arg,const void * r_arg)302 stacks_entry_comp(const void *l_arg, const void *r_arg)
303 {
304 	stacks_entry_t * const *lp = l_arg;
305 	stacks_entry_t * const *rp = r_arg;
306 
307 	return (stacks_entry_comp_impl(*lp, *rp, 1));
308 }
309 
310 void
stacks_cleanup(int force)311 stacks_cleanup(int force)
312 {
313 	int idx = 0;
314 	stacks_entry_t *cur, *next;
315 
316 	if (stacks_state == STACKS_STATE_CLEAN)
317 		return;
318 
319 	if (!force && stacks_state == STACKS_STATE_DONE)
320 		return;
321 
322 	/*
323 	 * Until the array is sorted and stable, stacks_hash will be non-NULL.
324 	 * This way, we can get at all of the data, even if qsort() was
325 	 * interrupted while mucking with the array.
326 	 */
327 	if (stacks_hash != NULL) {
328 		for (idx = 0; idx < STACKS_HSIZE; idx++) {
329 			while ((cur = stacks_hash[idx]) != NULL) {
330 				while ((next = cur->se_dup) != NULL) {
331 					cur->se_dup = next->se_dup;
332 					mdb_free(next,
333 					    STACKS_ENTRY_SIZE(next->se_depth));
334 				}
335 				next = cur->se_next;
336 				stacks_hash[idx] = next;
337 				mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
338 			}
339 		}
340 		if (stacks_array != NULL)
341 			mdb_free(stacks_array,
342 			    stacks_array_size * sizeof (*stacks_array));
343 
344 		mdb_free(stacks_hash, STACKS_HSIZE * sizeof (*stacks_hash));
345 
346 	} else if (stacks_array != NULL) {
347 		for (idx = 0; idx < stacks_array_size; idx++) {
348 			if ((cur = stacks_array[idx]) != NULL) {
349 				while ((next = cur->se_dup) != NULL) {
350 					cur->se_dup = next->se_dup;
351 					mdb_free(next,
352 					    STACKS_ENTRY_SIZE(next->se_depth));
353 				}
354 				stacks_array[idx] = NULL;
355 				mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
356 			}
357 		}
358 		mdb_free(stacks_array,
359 		    stacks_array_size * sizeof (*stacks_array));
360 	}
361 
362 	stacks_findstack_cleanup();
363 
364 	stacks_array_size = 0;
365 	stacks_state = STACKS_STATE_CLEAN;
366 	stacks_hash = NULL;
367 	stacks_array = NULL;
368 }
369 
370 /*ARGSUSED*/
371 int
stacks_thread_cb(uintptr_t addr,const void * ignored,void * cbarg)372 stacks_thread_cb(uintptr_t addr, const void *ignored, void *cbarg)
373 {
374 	stacks_info_t *sip = cbarg;
375 	findstack_info_t *fsip = &sip->si_fsi;
376 
377 	stacks_entry_t **sepp, *nsep, *sep;
378 	int idx;
379 	size_t depth;
380 
381 	if (stacks_findstack(addr, fsip, 0) != DCMD_OK &&
382 	    fsip->fsi_failed == FSI_FAIL_BADTHREAD) {
383 		mdb_warn("couldn't read thread at %p\n", addr);
384 		return (WALK_NEXT);
385 	}
386 
387 	sip->si_count++;
388 
389 	depth = fsip->fsi_depth;
390 	nsep = mdb_zalloc(STACKS_ENTRY_SIZE(depth), UM_SLEEP);
391 	nsep->se_thread = addr;
392 	nsep->se_sp = fsip->fsi_sp;
393 	nsep->se_sobj_ops = fsip->fsi_sobj_ops;
394 	nsep->se_tstate = fsip->fsi_tstate;
395 	nsep->se_count = 1;
396 	nsep->se_overflow = fsip->fsi_overflow;
397 	nsep->se_depth = depth;
398 	nsep->se_failed = fsip->fsi_failed;
399 	nsep->se_panic = fsip->fsi_panic;
400 
401 	for (idx = 0; idx < depth; idx++)
402 		nsep->se_stack[idx] = fsip->fsi_stack[idx];
403 
404 	for (sepp = &sip->si_hash[stacks_hash_entry(nsep)];
405 	    (sep = *sepp) != NULL;
406 	    sepp = &sep->se_next) {
407 
408 		if (stacks_entry_comp_impl(sep, nsep, 0) != 0)
409 			continue;
410 
411 		nsep->se_dup = sep->se_dup;
412 		sep->se_dup = nsep;
413 		sep->se_count++;
414 		return (WALK_NEXT);
415 	}
416 
417 	nsep->se_next = NULL;
418 	*sepp = nsep;
419 	sip->si_entries++;
420 
421 	return (WALK_NEXT);
422 }
423 
424 int
stacks_run_tlist(mdb_pipe_t * tlist,stacks_info_t * si)425 stacks_run_tlist(mdb_pipe_t *tlist, stacks_info_t *si)
426 {
427 	size_t idx;
428 	size_t found = 0;
429 	int ret;
430 
431 	for (idx = 0; idx < tlist->pipe_len; idx++) {
432 		uintptr_t addr = tlist->pipe_data[idx];
433 
434 		found++;
435 
436 		ret = stacks_thread_cb(addr, NULL, si);
437 		if (ret == WALK_DONE)
438 			break;
439 		if (ret != WALK_NEXT)
440 			return (-1);
441 	}
442 
443 	if (found)
444 		return (0);
445 	return (-1);
446 }
447 
448 int
stacks_run(int verbose,mdb_pipe_t * tlist)449 stacks_run(int verbose, mdb_pipe_t *tlist)
450 {
451 	stacks_info_t si;
452 	findstack_info_t *fsip = &si.si_fsi;
453 	size_t idx;
454 	stacks_entry_t **cur;
455 
456 	bzero(&si, sizeof (si));
457 
458 	stacks_state = STACKS_STATE_DIRTY;
459 
460 	stacks_hash = si.si_hash =
461 	    mdb_zalloc(STACKS_HSIZE * sizeof (*si.si_hash), UM_SLEEP);
462 	si.si_entries = 0;
463 	si.si_count = 0;
464 
465 	fsip->fsi_max_depth = STACKS_MAX_DEPTH;
466 	fsip->fsi_stack =
467 	    mdb_alloc(fsip->fsi_max_depth * sizeof (*fsip->fsi_stack),
468 	    UM_SLEEP | UM_GC);
469 
470 	if (verbose)
471 		mdb_warn("stacks: processing kernel threads\n");
472 
473 	if (tlist != NULL) {
474 		if (stacks_run_tlist(tlist, &si))
475 			return (DCMD_ERR);
476 	} else {
477 		if (mdb_walk("thread", stacks_thread_cb, &si) != 0) {
478 			mdb_warn("cannot walk \"thread\"");
479 			return (DCMD_ERR);
480 		}
481 	}
482 
483 	if (verbose)
484 		mdb_warn("stacks: %d unique stacks / %d threads\n",
485 		    si.si_entries, si.si_count);
486 
487 	stacks_array_size = si.si_entries;
488 	stacks_array =
489 	    mdb_zalloc(si.si_entries * sizeof (*stacks_array), UM_SLEEP);
490 	cur = stacks_array;
491 	for (idx = 0; idx < STACKS_HSIZE; idx++) {
492 		stacks_entry_t *sep;
493 		for (sep = si.si_hash[idx]; sep != NULL; sep = sep->se_next)
494 			*(cur++) = sep;
495 	}
496 
497 	if (cur != stacks_array + si.si_entries) {
498 		mdb_warn("stacks: miscounted array size (%d != size: %d)\n",
499 		    (cur - stacks_array), stacks_array_size);
500 		return (DCMD_ERR);
501 	}
502 	qsort(stacks_array, si.si_entries, sizeof (*stacks_array),
503 	    stacks_entry_comp);
504 
505 	/* Now that we're done, free the hash table */
506 	stacks_hash = NULL;
507 	mdb_free(si.si_hash, STACKS_HSIZE * sizeof (*si.si_hash));
508 
509 	if (tlist == NULL)
510 		stacks_state = STACKS_STATE_DONE;
511 
512 	if (verbose)
513 		mdb_warn("stacks: done\n");
514 
515 	return (DCMD_OK);
516 }
517 
518 static int
stacks_has_caller(stacks_entry_t * sep,uintptr_t addr)519 stacks_has_caller(stacks_entry_t *sep, uintptr_t addr)
520 {
521 	uintptr_t laddr = addr;
522 	uintptr_t haddr = addr + 1;
523 	int idx;
524 	char c[MDB_SYM_NAMLEN];
525 	GElf_Sym sym;
526 
527 	if (mdb_lookup_by_addr(addr, MDB_SYM_FUZZY,
528 	    c, sizeof (c), &sym) != -1 &&
529 	    addr == (uintptr_t)sym.st_value) {
530 		laddr = (uintptr_t)sym.st_value;
531 		haddr = (uintptr_t)sym.st_value + sym.st_size;
532 	}
533 
534 	for (idx = 0; idx < sep->se_depth; idx++)
535 		if (sep->se_stack[idx] >= laddr && sep->se_stack[idx] < haddr)
536 			return (1);
537 
538 	return (0);
539 }
540 
541 static int
stacks_has_module(stacks_entry_t * sep,stacks_module_t * mp)542 stacks_has_module(stacks_entry_t *sep, stacks_module_t *mp)
543 {
544 	int idx;
545 
546 	for (idx = 0; idx < sep->se_depth; idx++) {
547 		if (sep->se_stack[idx] >= mp->sm_text &&
548 		    sep->se_stack[idx] < mp->sm_text + mp->sm_size)
549 			return (1);
550 	}
551 
552 	return (0);
553 }
554 
555 static int
stacks_module_find(const char * name,stacks_module_t * mp)556 stacks_module_find(const char *name, stacks_module_t *mp)
557 {
558 	(void) strncpy(mp->sm_name, name, sizeof (mp->sm_name));
559 
560 	if (stacks_module(mp) != 0)
561 		return (-1);
562 
563 	if (mp->sm_size == 0) {
564 		mdb_warn("stacks: module \"%s\" is unknown\n", name);
565 		return (-1);
566 	}
567 
568 	return (0);
569 }
570 
571 static int
uintptrcomp(const void * lp,const void * rp)572 uintptrcomp(const void *lp, const void *rp)
573 {
574 	uintptr_t lhs = *(const uintptr_t *)lp;
575 	uintptr_t rhs = *(const uintptr_t *)rp;
576 	if (lhs > rhs)
577 		return (1);
578 	if (lhs < rhs)
579 		return (-1);
580 	return (0);
581 }
582 
583 /*ARGSUSED*/
584 int
stacks(uintptr_t addr,uint_t flags,int argc,const mdb_arg_t * argv)585 stacks(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
586 {
587 	size_t idx;
588 
589 	char *seen = NULL;
590 
591 	const char *caller_str = NULL;
592 	const char *excl_caller_str = NULL;
593 	uintptr_t caller = 0, excl_caller = 0;
594 	const char *module_str = NULL;
595 	const char *excl_module_str = NULL;
596 	stacks_module_t module, excl_module;
597 	const char *sobj = NULL;
598 	const char *excl_sobj = NULL;
599 	uintptr_t sobj_ops = 0, excl_sobj_ops = 0;
600 	const char *tstate_str = NULL;
601 	const char *excl_tstate_str = NULL;
602 	uint_t tstate = -1U;
603 	uint_t excl_tstate = -1U;
604 	uint_t printed = 0;
605 
606 	uint_t all = 0;
607 	uint_t force = 0;
608 	uint_t interesting = 0;
609 	uint_t verbose = 0;
610 
611 	/*
612 	 * We have a slight behavior difference between having piped
613 	 * input and 'addr::stacks'.  Without a pipe, we assume the
614 	 * thread pointer given is a representative thread, and so
615 	 * we include all similar threads in the system in our output.
616 	 *
617 	 * With a pipe, we filter down to just the threads in our
618 	 * input.
619 	 */
620 	uint_t addrspec = (flags & DCMD_ADDRSPEC);
621 	uint_t only_matching = addrspec && (flags & DCMD_PIPE);
622 
623 	mdb_pipe_t p;
624 
625 	bzero(&module, sizeof (module));
626 	bzero(&excl_module, sizeof (excl_module));
627 
628 	if (mdb_getopts(argc, argv,
629 	    'a', MDB_OPT_SETBITS, TRUE, &all,
630 	    'f', MDB_OPT_SETBITS, TRUE, &force,
631 	    'i', MDB_OPT_SETBITS, TRUE, &interesting,
632 	    'v', MDB_OPT_SETBITS, TRUE, &verbose,
633 	    'c', MDB_OPT_STR, &caller_str,
634 	    'C', MDB_OPT_STR, &excl_caller_str,
635 	    'm', MDB_OPT_STR, &module_str,
636 	    'M', MDB_OPT_STR, &excl_module_str,
637 	    's', MDB_OPT_STR, &sobj,
638 	    'S', MDB_OPT_STR, &excl_sobj,
639 	    't', MDB_OPT_STR, &tstate_str,
640 	    'T', MDB_OPT_STR, &excl_tstate_str,
641 	    NULL) != argc)
642 		return (DCMD_USAGE);
643 
644 	if (interesting) {
645 		if (sobj != NULL || excl_sobj != NULL ||
646 		    tstate_str != NULL || excl_tstate_str != NULL) {
647 			mdb_warn(
648 			    "stacks: -i is incompatible with -[sStT]\n");
649 			return (DCMD_USAGE);
650 		}
651 		excl_sobj = "CV";
652 		excl_tstate_str = "FREE";
653 	}
654 
655 	if (caller_str != NULL) {
656 		mdb_set_dot(0);
657 		if (mdb_eval(caller_str) != 0) {
658 			mdb_warn("stacks: evaluation of \"%s\" failed",
659 			    caller_str);
660 			return (DCMD_ABORT);
661 		}
662 		caller = mdb_get_dot();
663 	}
664 
665 	if (excl_caller_str != NULL) {
666 		mdb_set_dot(0);
667 		if (mdb_eval(excl_caller_str) != 0) {
668 			mdb_warn("stacks: evaluation of \"%s\" failed",
669 			    excl_caller_str);
670 			return (DCMD_ABORT);
671 		}
672 		excl_caller = mdb_get_dot();
673 	}
674 	mdb_set_dot(addr);
675 
676 	if (module_str != NULL && stacks_module_find(module_str, &module) != 0)
677 		return (DCMD_ABORT);
678 
679 	if (excl_module_str != NULL &&
680 	    stacks_module_find(excl_module_str, &excl_module) != 0)
681 		return (DCMD_ABORT);
682 
683 	if (sobj != NULL && text_to_sobj(sobj, &sobj_ops) != 0)
684 		return (DCMD_USAGE);
685 
686 	if (excl_sobj != NULL && text_to_sobj(excl_sobj, &excl_sobj_ops) != 0)
687 		return (DCMD_USAGE);
688 
689 	if (sobj_ops != 0 && excl_sobj_ops != 0) {
690 		mdb_warn("stacks: only one of -s and -S can be specified\n");
691 		return (DCMD_USAGE);
692 	}
693 
694 	if (tstate_str != NULL && text_to_tstate(tstate_str, &tstate) != 0)
695 		return (DCMD_USAGE);
696 
697 	if (excl_tstate_str != NULL &&
698 	    text_to_tstate(excl_tstate_str, &excl_tstate) != 0)
699 		return (DCMD_USAGE);
700 
701 	if (tstate != -1U && excl_tstate != -1U) {
702 		mdb_warn("stacks: only one of -t and -T can be specified\n");
703 		return (DCMD_USAGE);
704 	}
705 
706 	/*
707 	 * If there's an address specified, we're going to further filter
708 	 * to only entries which have an address in the input.  To reduce
709 	 * overhead (and make the sorted output come out right), we
710 	 * use mdb_get_pipe() to grab the entire pipeline of input, then
711 	 * use qsort() and bsearch() to speed up the search.
712 	 */
713 	if (addrspec) {
714 		mdb_get_pipe(&p);
715 		if (p.pipe_data == NULL || p.pipe_len == 0) {
716 			p.pipe_data = &addr;
717 			p.pipe_len = 1;
718 		}
719 		qsort(p.pipe_data, p.pipe_len, sizeof (uintptr_t),
720 		    uintptrcomp);
721 
722 		/* remove any duplicates in the data */
723 		idx = 0;
724 		while (idx < p.pipe_len - 1) {
725 			uintptr_t *data = &p.pipe_data[idx];
726 			size_t len = p.pipe_len - idx;
727 
728 			if (data[0] == data[1]) {
729 				memmove(data, data + 1,
730 				    (len - 1) * sizeof (*data));
731 				p.pipe_len--;
732 				continue; /* repeat without incrementing idx */
733 			}
734 			idx++;
735 		}
736 
737 		seen = mdb_zalloc(p.pipe_len, UM_SLEEP | UM_GC);
738 	}
739 
740 	/*
741 	 * Force a cleanup if we're connected to a live system. Never
742 	 * do a cleanup after the first invocation around the loop.
743 	 */
744 	force |= (mdb_get_state() == MDB_STATE_RUNNING);
745 	if (force && (flags & (DCMD_LOOPFIRST|DCMD_LOOP)) == DCMD_LOOP)
746 		force = 0;
747 
748 	stacks_cleanup(force);
749 
750 	if (stacks_state == STACKS_STATE_CLEAN) {
751 		int res = stacks_run(verbose, addrspec ? &p : NULL);
752 		if (res != DCMD_OK)
753 			return (res);
754 	}
755 
756 	for (idx = 0; idx < stacks_array_size; idx++) {
757 		stacks_entry_t *sep = stacks_array[idx];
758 		stacks_entry_t *cur = sep;
759 		int frame;
760 		size_t count = sep->se_count;
761 
762 		if (addrspec) {
763 			stacks_entry_t *head = NULL, *tail = NULL, *sp;
764 			size_t foundcount = 0;
765 			/*
766 			 * We use the now-unused hash chain field se_next to
767 			 * link together the dups which match our list.
768 			 */
769 			for (sp = sep; sp != NULL; sp = sp->se_dup) {
770 				uintptr_t *entry = bsearch(&sp->se_thread,
771 				    p.pipe_data, p.pipe_len, sizeof (uintptr_t),
772 				    uintptrcomp);
773 				if (entry != NULL) {
774 					foundcount++;
775 					seen[entry - p.pipe_data]++;
776 					if (head == NULL)
777 						head = sp;
778 					else
779 						tail->se_next = sp;
780 					tail = sp;
781 					sp->se_next = NULL;
782 				}
783 			}
784 			if (head == NULL)
785 				continue;	/* no match, skip entry */
786 
787 			if (only_matching) {
788 				cur = sep = head;
789 				count = foundcount;
790 			}
791 		}
792 
793 		if (caller != 0 && !stacks_has_caller(sep, caller))
794 			continue;
795 
796 		if (excl_caller != 0 && stacks_has_caller(sep, excl_caller))
797 			continue;
798 
799 		if (module.sm_size != 0 && !stacks_has_module(sep, &module))
800 			continue;
801 
802 		if (excl_module.sm_size != 0 &&
803 		    stacks_has_module(sep, &excl_module))
804 			continue;
805 
806 		if (tstate != -1U) {
807 			if (tstate == TSTATE_PANIC) {
808 				if (!sep->se_panic)
809 					continue;
810 			} else if (sep->se_panic || sep->se_tstate != tstate)
811 				continue;
812 		}
813 		if (excl_tstate != -1U) {
814 			if (excl_tstate == TSTATE_PANIC) {
815 				if (sep->se_panic)
816 					continue;
817 			} else if (!sep->se_panic &&
818 			    sep->se_tstate == excl_tstate)
819 				continue;
820 		}
821 
822 		if (sobj_ops == SOBJ_ALL) {
823 			if (sep->se_sobj_ops == 0)
824 				continue;
825 		} else if (sobj_ops != 0) {
826 			if (sobj_ops != sep->se_sobj_ops)
827 				continue;
828 		}
829 
830 		if (!(interesting && sep->se_panic)) {
831 			if (excl_sobj_ops == SOBJ_ALL) {
832 				if (sep->se_sobj_ops != 0)
833 					continue;
834 			} else if (excl_sobj_ops != 0) {
835 				if (excl_sobj_ops == sep->se_sobj_ops)
836 					continue;
837 			}
838 		}
839 
840 		if (flags & DCMD_PIPE_OUT) {
841 			while (sep != NULL) {
842 				mdb_printf("%lr\n", sep->se_thread);
843 				sep = only_matching ?
844 				    sep->se_next : sep->se_dup;
845 			}
846 			continue;
847 		}
848 
849 		if (all || !printed) {
850 			mdb_printf("%<u>%-?s %-8s %-?s %8s%</u>\n",
851 			    "THREAD", "STATE", "SOBJ", "COUNT");
852 			printed = 1;
853 		}
854 
855 		do {
856 			char state[20];
857 			char sobj[100];
858 
859 			tstate_to_text(cur->se_tstate, cur->se_panic,
860 			    state, sizeof (state));
861 			sobj_to_text(cur->se_sobj_ops,
862 			    sobj, sizeof (sobj));
863 
864 			if (cur == sep)
865 				mdb_printf("%-?p %-8s %-?s %8d\n",
866 				    cur->se_thread, state, sobj, count);
867 			else
868 				mdb_printf("%-?p %-8s %-?s %8s\n",
869 				    cur->se_thread, state, sobj, "-");
870 
871 			cur = only_matching ? cur->se_next : cur->se_dup;
872 		} while (all && cur != NULL);
873 
874 		if (sep->se_failed != 0) {
875 			char *reason;
876 			switch (sep->se_failed) {
877 			case FSI_FAIL_NOTINMEMORY:
878 				reason = "thread not in memory";
879 				break;
880 			case FSI_FAIL_THREADCORRUPT:
881 				reason = "thread structure stack info corrupt";
882 				break;
883 			case FSI_FAIL_STACKNOTFOUND:
884 				reason = "no consistent stack found";
885 				break;
886 			default:
887 				reason = "unknown failure";
888 				break;
889 			}
890 			mdb_printf("%?s <%s>\n", "", reason);
891 		}
892 
893 		for (frame = 0; frame < sep->se_depth; frame++)
894 			mdb_printf("%?s %a\n", "", sep->se_stack[frame]);
895 		if (sep->se_overflow)
896 			mdb_printf("%?s ... truncated ...\n", "");
897 		mdb_printf("\n");
898 	}
899 
900 	if (flags & DCMD_ADDRSPEC) {
901 		for (idx = 0; idx < p.pipe_len; idx++)
902 			if (seen[idx] == 0)
903 				mdb_warn("stacks: %p not in thread list\n",
904 				    p.pipe_data[idx]);
905 	}
906 	return (DCMD_OK);
907 }
908