xref: /titanic_44/usr/src/cmd/mdb/common/modules/genunix/cyclic.c (revision 672986541be54a7a471bb088e60780c37e371d7e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include "cyclic.h"
29 
30 #define	CYCLIC_TRACE
31 
32 #include <mdb/mdb_modapi.h>
33 #include <sys/timer.h>
34 #include <sys/cyclic_impl.h>
35 #include <sys/sysmacros.h>
36 #include <stdio.h>
37 
38 int
39 cyccpu_vread(cyc_cpu_t *cpu, uintptr_t addr)
40 {
41 	static int inited = 0;
42 	static int cyc_trace_enabled = 0;
43 	static size_t cyccpu_size;
44 
45 	if (!inited) {
46 		inited = 1;
47 		(void) mdb_readvar(&cyc_trace_enabled, "cyc_trace_enabled");
48 		cyccpu_size = (cyc_trace_enabled) ? sizeof (*cpu) :
49 		    OFFSETOF(cyc_cpu_t, cyp_trace);
50 	}
51 
52 	if (mdb_vread(cpu, cyccpu_size, addr) == -1)
53 		return (-1);
54 
55 	if (!cyc_trace_enabled)
56 		bzero(cpu->cyp_trace, sizeof (cpu->cyp_trace));
57 
58 	return (0);
59 }
60 
61 int
62 cyccpu_walk_init(mdb_walk_state_t *wsp)
63 {
64 	if (mdb_layered_walk("cpu", wsp) == -1) {
65 		mdb_warn("couldn't walk 'cpu'");
66 		return (WALK_ERR);
67 	}
68 
69 	return (WALK_NEXT);
70 }
71 
72 int
73 cyccpu_walk_step(mdb_walk_state_t *wsp)
74 {
75 	uintptr_t addr = (uintptr_t)((cpu_t *)wsp->walk_layer)->cpu_cyclic;
76 	cyc_cpu_t cpu;
77 
78 	if (cyccpu_vread(&cpu, addr) == -1) {
79 		mdb_warn("couldn't read cyc_cpu at %p", addr);
80 		return (WALK_ERR);
81 	}
82 
83 	return (wsp->walk_callback(addr, &cpu, wsp->walk_cbdata));
84 }
85 
86 int
87 cycomni_walk_init(mdb_walk_state_t *wsp)
88 {
89 	cyc_id_t id;
90 
91 	if (wsp->walk_addr == NULL) {
92 		mdb_warn("must provide a cyclic id\n");
93 		return (WALK_ERR);
94 	}
95 
96 	if (mdb_vread(&id, sizeof (id), wsp->walk_addr) == -1) {
97 		mdb_warn("couldn't read cyc_id_t at %p", wsp->walk_addr);
98 		return (WALK_ERR);
99 	}
100 
101 	if (id.cyi_cpu != NULL || id.cyi_omni_list == NULL ||
102 	    id.cyi_omni_hdlr.cyo_online == NULL) {
103 		mdb_warn("%p is not an omnipresent cyclic.\n", wsp->walk_addr);
104 		return (WALK_ERR);
105 	}
106 
107 	wsp->walk_addr = (uintptr_t)id.cyi_omni_list;
108 
109 	return (WALK_NEXT);
110 }
111 
112 int
113 cycomni_walk_step(mdb_walk_state_t *wsp)
114 {
115 	uintptr_t addr = wsp->walk_addr;
116 	cyc_omni_cpu_t omni;
117 
118 	if (addr == NULL)
119 		return (WALK_DONE);
120 
121 	if (mdb_vread(&omni, sizeof (omni), addr) == -1) {
122 		mdb_warn("couldn't read cyc_omni_cpu at %p", addr);
123 		return (WALK_ERR);
124 	}
125 
126 	wsp->walk_addr = (uintptr_t)omni.cyo_next;
127 
128 	return (wsp->walk_callback(addr, &omni, wsp->walk_cbdata));
129 }
130 
131 void
132 cyclic_dump_node(cyc_cpu_t *cpu, cyc_index_t *heap, char **c, size_t w,
133     int ndx, int l, int r, int depth)
134 {
135 	int heap_left, heap_right;
136 	int me;
137 	int i, x = l + (r - l) / 2;
138 	size_t n = w - (x - 1); /* n bytes left for snprintf after c[][x - 1] */
139 
140 	heap_left = CYC_HEAP_LEFT(ndx);
141 	heap_right = CYC_HEAP_RIGHT(ndx);
142 	me = heap[ndx];
143 
144 	if (ndx >= cpu->cyp_nelems)
145 		return;
146 
147 	if (me < 10) {
148 		(void) mdb_snprintf(&c[depth][x - 1], n, " %d", me);
149 	} else if (me >= 100) {
150 		(void) mdb_snprintf(&c[depth][x - 1], n, "%3d", me);
151 	} else {
152 		(void) mdb_snprintf(&c[depth][x - 1], n, "%s%2d%s",
153 		    CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? " " : "", me,
154 		    CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? "" : " ");
155 	}
156 
157 	if (r - l > 5) {
158 		c[++depth][x] = '|';
159 		depth++;
160 
161 		for (i = l + (r - l) / 4; i < r - (r - l) / 4; i++)
162 			c[depth][i] = '-';
163 		c[depth][l + (r - l) / 4] = '+';
164 		c[depth][r - (r - l) / 4 - 1] = '+';
165 		c[depth][x] = '+';
166 	} else {
167 
168 		if (heap_left >= cpu->cyp_nelems)
169 			return;
170 
171 		(void) mdb_snprintf(&c[++depth][x - 1], n, "L%d",
172 		    heap[heap_left]);
173 
174 		if (heap_right >= cpu->cyp_nelems)
175 			return;
176 
177 		(void) mdb_snprintf(&c[++depth][x - 1], n, "R%d",
178 		    heap[heap_right]);
179 		return;
180 	}
181 
182 	if (heap_left < cpu->cyp_nelems)
183 		cyclic_dump_node(cpu, heap, c, w, heap_left, l, x, depth + 1);
184 
185 	if (heap_right < cpu->cyp_nelems)
186 		cyclic_dump_node(cpu, heap, c, w, heap_right, x, r, depth + 1);
187 }
188 
189 #define	LINES_PER_LEVEL 3
190 
191 void
192 cyclic_pretty_dump(cyc_cpu_t *cpu)
193 {
194 	char **c;
195 	int i, j;
196 	int width = 80;
197 	int depth;
198 	cyc_index_t *heap;
199 	size_t hsize = sizeof (cyc_index_t) * cpu->cyp_size;
200 
201 	heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);
202 
203 	if (mdb_vread(heap, hsize, (uintptr_t)cpu->cyp_heap) == -1) {
204 		mdb_warn("couldn't read heap at %p", (uintptr_t)cpu->cyp_heap);
205 		return;
206 	}
207 
208 	for (depth = 0; (1 << depth) < cpu->cyp_nelems; depth++)
209 		continue;
210 	depth++;
211 	depth = (depth + 1) * LINES_PER_LEVEL;
212 
213 	c = mdb_zalloc(sizeof (char *) * depth, UM_SLEEP|UM_GC);
214 
215 	for (i = 0; i < depth; i++)
216 		c[i] = mdb_zalloc(width, UM_SLEEP|UM_GC);
217 
218 	cyclic_dump_node(cpu, heap, c, width, 0, 1, width - 2, 0);
219 
220 	for (i = 0; i < depth; i++) {
221 		int dump = 0;
222 		for (j = 0; j < width - 1; j++) {
223 			if (c[i][j] == '\0')
224 				c[i][j] = ' ';
225 			else
226 				dump = 1;
227 		}
228 		c[i][width - 2] = '\n';
229 
230 		if (dump)
231 			mdb_printf(c[i]);
232 	}
233 }
234 
235 /*
236  * Yes, this is very weak.  Full 16-column-wide 64-bit addresses screw up
237  * ::cycinfo's very carefully planned 80-column layout.  We set the column
238  * width for addresses to be 11 (instead of 16), knowing that the kernel
239  * heap (from which these data structures are allocated) starts at
240  * 0x0000030000000000, and isn't likely to extend to 0x0000100000000000.
241  */
242 #ifdef _LP64
243 #define	CYC_ADDR_WIDTH	11
244 #else
245 #define	CYC_ADDR_WIDTH	8
246 #endif
247 
248 int
249 cycinfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
250 {
251 	cyc_cpu_t cpu;
252 	cpu_t c;
253 	cyc_index_t root, i, *heap;
254 	size_t hsize;
255 	cyclic_t *cyc;
256 	uintptr_t caddr;
257 	uint_t verbose = FALSE, Verbose = FALSE;
258 	int header = 0;
259 	cyc_level_t lev;
260 
261 	if (!(flags & DCMD_ADDRSPEC)) {
262 		if (mdb_walk_dcmd("cyccpu", "cycinfo", argc, argv) == -1) {
263 			mdb_warn("can't walk 'cyccpu'");
264 			return (DCMD_ERR);
265 		}
266 		return (DCMD_OK);
267 	}
268 
269 	if (mdb_getopts(argc, argv,
270 	    'v', MDB_OPT_SETBITS, TRUE, &verbose,
271 	    'V', MDB_OPT_SETBITS, TRUE, &Verbose, NULL) != argc)
272 		return (DCMD_USAGE);
273 
274 	if (!DCMD_HDRSPEC(flags) && (verbose || Verbose))
275 		mdb_printf("\n\n");
276 
277 	if (DCMD_HDRSPEC(flags) || verbose || Verbose)
278 		mdb_printf("%3s %*s %7s %6s %*s %15s %s\n", "CPU",
279 		    CYC_ADDR_WIDTH, "CYC_CPU", "STATE", "NELEMS",
280 		    CYC_ADDR_WIDTH, "ROOT", "FIRE", "HANDLER");
281 
282 	if (cyccpu_vread(&cpu, addr) == -1) {
283 		mdb_warn("couldn't read cyc_cpu at %p", addr);
284 		return (DCMD_ERR);
285 	}
286 
287 	if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
288 		mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
289 		return (DCMD_ERR);
290 	}
291 
292 	cyc = mdb_alloc(sizeof (cyclic_t) * cpu.cyp_size, UM_SLEEP | UM_GC);
293 	caddr = (uintptr_t)cpu.cyp_cyclics;
294 
295 	if (mdb_vread(cyc, sizeof (cyclic_t) * cpu.cyp_size, caddr) == -1) {
296 		mdb_warn("couldn't read cyclic at %p", caddr);
297 		return (DCMD_ERR);
298 	}
299 
300 	hsize = sizeof (cyc_index_t) * cpu.cyp_size;
301 	heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);
302 
303 	if (mdb_vread(heap, hsize, (uintptr_t)cpu.cyp_heap) == -1) {
304 		mdb_warn("couldn't read heap at %p", cpu.cyp_heap);
305 		return (DCMD_ERR);
306 	}
307 
308 	root = heap[0];
309 
310 	mdb_printf("%3d %0*p %7s %6d ", c.cpu_id, CYC_ADDR_WIDTH, addr,
311 	    cpu.cyp_state == CYS_ONLINE ? "online" :
312 	    cpu.cyp_state == CYS_OFFLINE ? "offline" :
313 	    cpu.cyp_state == CYS_EXPANDING ? "expand" :
314 	    cpu.cyp_state == CYS_REMOVING ? "remove" :
315 	    cpu.cyp_state == CYS_SUSPENDED ? "suspend" : "????",
316 	    cpu.cyp_nelems);
317 
318 	if (cpu.cyp_nelems > 0)
319 		mdb_printf("%0*p %15llx %a\n", CYC_ADDR_WIDTH,
320 		    caddr, cyc[root].cy_expire, cyc[root].cy_handler);
321 	else
322 		mdb_printf("%*s %15s %s\n", CYC_ADDR_WIDTH, "-", "-", "-");
323 
324 	if (!verbose && !Verbose)
325 		return (DCMD_OK);
326 
327 	mdb_printf("\n");
328 
329 	cyclic_pretty_dump(&cpu);
330 
331 	mdb_inc_indent(2);
332 
333 	for (i = 0; i < cpu.cyp_size; i++) {
334 		int j;
335 
336 		for (j = 0; j < cpu.cyp_size; j++) {
337 			if (heap[j] == i)
338 				break;
339 		}
340 
341 		if (!Verbose && j >= cpu.cyp_nelems)
342 			continue;
343 
344 		if (!header) {
345 			header = 1;
346 			mdb_printf("\n%*s %3s %4s %4s %5s %15s %7s %s\n",
347 			    CYC_ADDR_WIDTH, "ADDR", "NDX", "HEAP", "LEVL",
348 			    "PEND", "FIRE", "USECINT", "HANDLER");
349 		}
350 
351 		mdb_printf("%0*p %3d ", CYC_ADDR_WIDTH,
352 		    caddr + i * sizeof (cyclic_t), i);
353 
354 		mdb_printf("%4d ", j);
355 
356 		if (j >= cpu.cyp_nelems) {
357 			mdb_printf("%4s %5s %15s %7s %s\n", "-", "-",
358 			    "-", "-", "-");
359 			continue;
360 		}
361 
362 		mdb_printf("%4s %5d %15llx ",
363 		    cyc[i].cy_level == CY_HIGH_LEVEL ? "high" :
364 		    cyc[i].cy_level == CY_LOCK_LEVEL ? "lock" :
365 		    cyc[i].cy_level == CY_LOW_LEVEL ? "low" : "????",
366 		    cyc[i].cy_pend, cyc[i].cy_expire);
367 
368 		if (cyc[i].cy_interval + cyc[i].cy_expire != INT64_MAX)
369 			mdb_printf("%7lld ", cyc[i].cy_interval /
370 			    (uint64_t)(NANOSEC / MICROSEC));
371 		else
372 			mdb_printf("%7s ", "-");
373 
374 		mdb_printf("%a\n", cyc[i].cy_handler);
375 	}
376 
377 
378 	if (!Verbose)
379 		goto out;
380 
381 	for (lev = CY_LOW_LEVEL; lev < CY_LOW_LEVEL + CY_SOFT_LEVELS; lev++) {
382 		cyc_softbuf_t *softbuf = &cpu.cyp_softbuf[lev];
383 		char which = softbuf->cys_hard, shared = 1;
384 		cyc_pcbuffer_t *pc;
385 		size_t bufsiz;
386 		cyc_index_t *buf;
387 
388 		if (softbuf->cys_hard != softbuf->cys_soft)
389 			shared = 0;
390 
391 again:
392 		pc = &softbuf->cys_buf[which];
393 		bufsiz = (pc->cypc_sizemask + 1) * sizeof (cyc_index_t);
394 		buf = mdb_alloc(bufsiz, UM_SLEEP | UM_GC);
395 
396 		if (mdb_vread(buf, bufsiz, (uintptr_t)pc->cypc_buf) == -1) {
397 			mdb_warn("couldn't read cypc_buf at %p", pc->cypc_buf);
398 			continue;
399 		}
400 
401 		mdb_printf("\n%3s %4s %4s %4s %*s %4s %*s\n", "CPU",
402 		    "LEVL", "USER", "NDX", CYC_ADDR_WIDTH, "ADDR", "CYC",
403 		    CYC_ADDR_WIDTH, "CYC_ADDR", "PEND");
404 
405 		for (i = 0; i <= pc->cypc_sizemask &&
406 		    i <= pc->cypc_prodndx; i++) {
407 			uintptr_t cyc_addr = caddr + buf[i] * sizeof (cyclic_t);
408 
409 			mdb_printf("%3d %4s %4s ", c.cpu_id,
410 			    lev == CY_HIGH_LEVEL ? "high" :
411 			    lev == CY_LOCK_LEVEL ? "lock" :
412 			    lev == CY_LOW_LEVEL ? "low" : "????",
413 			    shared ? "shrd" : which == softbuf->cys_hard ?
414 			    "hard" : "soft");
415 
416 			mdb_printf("%4d %0*p ", i, CYC_ADDR_WIDTH,
417 			    (uintptr_t)&buf[i] - (uintptr_t)&buf[0] +
418 			    (uintptr_t)pc->cypc_buf, buf[i],
419 			    caddr + buf[i] * sizeof (cyclic_t));
420 
421 			if (i >= pc->cypc_prodndx)
422 				mdb_printf("%4s %*s %5s  ",
423 				    "-", CYC_ADDR_WIDTH, "-", "-");
424 			else {
425 				cyclic_t c;
426 
427 				if (mdb_vread(&c, sizeof (c), cyc_addr) == -1) {
428 					mdb_warn("\ncouldn't read cyclic at "
429 					    "%p", cyc_addr);
430 					continue;
431 				}
432 
433 				mdb_printf("%4d %0*p %5d  ", buf[i],
434 				    CYC_ADDR_WIDTH, cyc_addr, c.cy_pend);
435 			}
436 
437 			if (i == (pc->cypc_consndx & pc->cypc_sizemask)) {
438 				mdb_printf("<-- consndx");
439 				if (i == (pc->cypc_prodndx & pc->cypc_sizemask))
440 					mdb_printf(",prodndx");
441 				mdb_printf("\n");
442 				continue;
443 			}
444 
445 			if (i == (pc->cypc_prodndx & pc->cypc_sizemask)) {
446 				mdb_printf("<-- prodndx\n");
447 				continue;
448 			}
449 			mdb_printf("\n");
450 
451 			if (i >= pc->cypc_prodndx)
452 				break;
453 		}
454 
455 		if (!shared && which == softbuf->cys_hard) {
456 			which = softbuf->cys_soft;
457 			goto again;
458 		}
459 	}
460 
461 out:
462 	mdb_dec_indent(2);
463 	return (DCMD_OK);
464 }
465 
466 int
467 cyctrace_walk_init(mdb_walk_state_t *wsp)
468 {
469 	cyc_cpu_t *cpu;
470 	int i;
471 
472 	cpu = mdb_zalloc(sizeof (cyc_cpu_t), UM_SLEEP);
473 
474 	if (wsp->walk_addr == NULL) {
475 		/*
476 		 * If an address isn't provided, we'll use the passive buffer.
477 		 */
478 		GElf_Sym sym;
479 		cyc_tracebuf_t *tr = &cpu->cyp_trace[0];
480 		uintptr_t addr;
481 
482 		if (mdb_lookup_by_name("cyc_ptrace", &sym) == -1) {
483 			mdb_warn("couldn't find passive buffer");
484 			return (-1);
485 		}
486 
487 		addr = (uintptr_t)sym.st_value;
488 
489 		if (mdb_vread(tr, sizeof (cyc_tracebuf_t), addr) == -1) {
490 			mdb_warn("couldn't read passive buffer");
491 			return (-1);
492 		}
493 
494 		wsp->walk_addr = addr - offsetof(cyc_cpu_t, cyp_trace[0]);
495 	} else {
496 		if (cyccpu_vread(cpu, wsp->walk_addr) == -1) {
497 			mdb_warn("couldn't read cyc_cpu at %p", wsp->walk_addr);
498 			mdb_free(cpu, sizeof (cyc_cpu_t));
499 			return (-1);
500 		}
501 	}
502 
503 	for (i = 0; i < CY_LEVELS; i++) {
504 		if (cpu->cyp_trace[i].cyt_ndx-- == 0)
505 			cpu->cyp_trace[i].cyt_ndx = CY_NTRACEREC - 1;
506 	}
507 
508 	wsp->walk_data = cpu;
509 
510 	return (0);
511 }
512 
513 int
514 cyctrace_walk_step(mdb_walk_state_t *wsp)
515 {
516 	cyc_cpu_t *cpu = wsp->walk_data;
517 	cyc_tracebuf_t *buf = cpu->cyp_trace;
518 	hrtime_t latest = 0;
519 	int i, ndx, new_ndx, lev, rval;
520 	uintptr_t addr;
521 
522 	for (i = 0; i < CY_LEVELS; i++) {
523 		if ((ndx = buf[i].cyt_ndx) == -1)
524 			continue;
525 
526 		/*
527 		 * Account for NPT.
528 		 */
529 		buf[i].cyt_buf[ndx].cyt_tstamp <<= 1;
530 		buf[i].cyt_buf[ndx].cyt_tstamp >>= 1;
531 
532 		if (buf[i].cyt_buf[ndx].cyt_tstamp > latest) {
533 			latest = buf[i].cyt_buf[ndx].cyt_tstamp;
534 			lev = i;
535 		}
536 	}
537 
538 	/*
539 	 * If we didn't find one, we're done.
540 	 */
541 	if (latest == 0)
542 		return (-1);
543 
544 	buf = &buf[lev];
545 	ndx = buf->cyt_ndx;
546 	addr = wsp->walk_addr +
547 	    (uintptr_t)&(buf->cyt_buf[ndx]) - (uintptr_t)cpu;
548 
549 	rval = wsp->walk_callback(addr, &buf->cyt_buf[ndx], wsp->walk_cbdata);
550 
551 	new_ndx = ndx == 0 ? CY_NTRACEREC - 1 : ndx - 1;
552 
553 	if (buf->cyt_buf[new_ndx].cyt_tstamp != 0 &&
554 	    buf->cyt_buf[new_ndx].cyt_tstamp > buf->cyt_buf[ndx].cyt_tstamp)
555 		new_ndx = -1;
556 
557 	buf->cyt_ndx = new_ndx;
558 
559 	return (rval);
560 }
561 
562 void
563 cyctrace_walk_fini(mdb_walk_state_t *wsp)
564 {
565 	cyc_cpu_t *cpu = wsp->walk_data;
566 
567 	mdb_free(cpu, sizeof (cyc_cpu_t));
568 }
569 
570 #define	WHYLEN	17
571 
572 int
573 cyctrace_walk(uintptr_t addr, const cyc_tracerec_t *rec, cyc_cpu_t *cpu)
574 {
575 	int i;
576 	char c[WHYLEN];
577 
578 	for (i = 0; cpu != NULL && i < CY_LEVELS; i++)
579 		if (addr < (uintptr_t)&cpu->cyp_trace[i + 1].cyt_buf[0])
580 			break;
581 
582 	(void) mdb_readstr(c, WHYLEN, (uintptr_t)rec->cyt_why);
583 
584 	mdb_printf("%08p %4s %15llx %-*s %15llx %15llx\n",
585 	    addr & UINT_MAX, cpu == NULL ? "pasv" :
586 	    i == CY_HIGH_LEVEL ? "high" : i == CY_LOCK_LEVEL ? "lock" :
587 	    i == CY_LOW_LEVEL ? "low" : "????", rec->cyt_tstamp, WHYLEN, c,
588 	    rec->cyt_arg0, rec->cyt_arg1);
589 
590 	return (0);
591 }
592 
593 /*ARGSUSED*/
594 int
595 cyctrace(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
596 {
597 	if (!(flags & DCMD_ADDRSPEC) || argc != 0)
598 		addr = NULL;
599 
600 	if (mdb_pwalk("cyctrace", (mdb_walk_cb_t)cyctrace_walk,
601 	    (void *)addr, addr) == -1) {
602 		mdb_warn("couldn't walk cyctrace");
603 		return (DCMD_ERR);
604 	}
605 
606 	return (DCMD_OK);
607 }
608 
609 int
610 cyccover_comp(const void *l, const void *r)
611 {
612 	cyc_coverage_t *lhs = (cyc_coverage_t *)l;
613 	cyc_coverage_t *rhs = (cyc_coverage_t *)r;
614 
615 	char ly[WHYLEN], ry[WHYLEN];
616 
617 	if (rhs->cyv_why == lhs->cyv_why)
618 		return (0);
619 
620 	if (rhs->cyv_why == NULL)
621 		return (-1);
622 
623 	if (lhs->cyv_why == NULL)
624 		return (1);
625 
626 	(void) mdb_readstr(ly, WHYLEN, (uintptr_t)lhs->cyv_why);
627 	(void) mdb_readstr(ry, WHYLEN, (uintptr_t)rhs->cyv_why);
628 
629 	return (strcmp(ly, ry));
630 }
631 
632 /*ARGSUSED*/
633 int
634 cyccover(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
635 {
636 	cyc_coverage_t cv[CY_NCOVERAGE];
637 	char c[WHYLEN];
638 	GElf_Sym sym;
639 	int i;
640 
641 	if ((flags & DCMD_ADDRSPEC) || argc != 0)
642 		return (DCMD_USAGE);
643 
644 	if (mdb_lookup_by_name("cyc_coverage", &sym) == -1) {
645 		mdb_warn("couldn't find coverage information");
646 		return (DCMD_ABORT);
647 	}
648 
649 	addr = (uintptr_t)sym.st_value;
650 
651 	if (mdb_vread(cv, sizeof (cyc_coverage_t) * CY_NCOVERAGE, addr) == -1) {
652 		mdb_warn("couldn't read coverage array at %p", addr);
653 		return (DCMD_ABORT);
654 	}
655 
656 	mdb_printf("%-*s %8s %8s %8s %15s %15s\n",
657 	    WHYLEN, "POINT", "HIGH", "LOCK", "LOW/PASV", "ARG0", "ARG1");
658 
659 	qsort(cv, CY_NCOVERAGE, sizeof (cyc_coverage_t), cyccover_comp);
660 
661 	for (i = 0; i < CY_NCOVERAGE; i++) {
662 		if (cv[i].cyv_why != NULL) {
663 			(void) mdb_readstr(c, WHYLEN, (uintptr_t)cv[i].cyv_why);
664 			mdb_printf("%-*s %8d %8d %8d %15llx %15llx\n",
665 			    WHYLEN, c,
666 			    cv[i].cyv_count[CY_HIGH_LEVEL],
667 			    cv[i].cyv_count[CY_LOCK_LEVEL],
668 			    cv[i].cyv_passive_count != 0 ?
669 			    cv[i].cyv_passive_count :
670 			    cv[i].cyv_count[CY_LOW_LEVEL],
671 			    cv[i].cyv_arg0, cv[i].cyv_arg1);
672 		}
673 	}
674 
675 	return (DCMD_OK);
676 }
677 
678 /*ARGSUSED*/
679 int
680 cyclic(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
681 {
682 	cyclic_t cyc;
683 
684 	if (!(flags & DCMD_ADDRSPEC) || argc != 0)
685 		return (DCMD_USAGE);
686 
687 	if (DCMD_HDRSPEC(flags))
688 		mdb_printf("%?s %4s %5s %5s %15s %7s %s\n", "ADDR", "LEVL",
689 		    "PEND", "FLAGS", "FIRE", "USECINT", "HANDLER");
690 
691 	if (mdb_vread(&cyc, sizeof (cyclic_t), addr) == -1) {
692 		mdb_warn("couldn't read cyclic at %p", addr);
693 		return (DCMD_ERR);
694 	}
695 
696 	mdb_printf("%0?p %4s %5d  %04x %15llx %7lld %a\n", addr,
697 	    cyc.cy_level == CY_HIGH_LEVEL ? "high" :
698 	    cyc.cy_level == CY_LOCK_LEVEL ? "lock" :
699 	    cyc.cy_level == CY_LOW_LEVEL ? "low" : "????",
700 	    cyc.cy_pend, cyc.cy_flags, cyc.cy_expire,
701 	    cyc.cy_interval / (uint64_t)(NANOSEC / MICROSEC),
702 	    cyc.cy_handler);
703 
704 	return (DCMD_OK);
705 }
706 
707 static int
708 cycid_cpu(cyc_cpu_t *addr, int ndx)
709 {
710 	cyc_cpu_t cpu;
711 	cpu_t c;
712 	uintptr_t caddr;
713 	cyclic_t cyc;
714 
715 	if (cyccpu_vread(&cpu, (uintptr_t)addr) == -1) {
716 		mdb_warn("couldn't read cyc_cpu at %p", addr);
717 		return (DCMD_ERR);
718 	}
719 
720 	if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
721 		mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
722 		return (DCMD_ERR);
723 	}
724 
725 	caddr = (uintptr_t)cpu.cyp_cyclics + ndx * sizeof (cyclic_t);
726 
727 	if (mdb_vread(&cyc, sizeof (cyc), caddr) == -1) {
728 		mdb_warn("couldn't read cyclic at %p", caddr);
729 		return (DCMD_ERR);
730 	}
731 
732 	mdb_printf("%4d %3d %?p %a\n", c.cpu_id, ndx, caddr, cyc.cy_handler);
733 
734 	return (DCMD_OK);
735 }
736 
737 /*ARGSUSED*/
738 static int
739 cycid_walk_omni(uintptr_t addr, const cyc_omni_cpu_t *omni, int *ignored)
740 {
741 	mdb_printf("%?s        ");
742 	cycid_cpu(omni->cyo_cpu, omni->cyo_ndx);
743 
744 	return (WALK_NEXT);
745 }
746 
747 /*ARGSUSED*/
748 int
749 cycid(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *av)
750 {
751 	cyc_id_t id;
752 
753 	if (!(flags & DCMD_ADDRSPEC)) {
754 		if (mdb_walk_dcmd("cyclic_id_cache", "cycid", ac, av) == -1) {
755 			mdb_warn("can't walk cyclic_id_cache");
756 			return (DCMD_ERR);
757 		}
758 
759 		return (DCMD_OK);
760 	}
761 
762 	if (DCMD_HDRSPEC(flags)) {
763 		mdb_printf("%?s %4s %3s %?s %s\n", "ADDR", "CPU", "NDX",
764 		    "CYCLIC", "HANDLER");
765 	}
766 
767 	if (mdb_vread(&id, sizeof (id), addr) == -1) {
768 		mdb_warn("couldn't read cyc_id_t at %p", addr);
769 		return (DCMD_ERR);
770 	}
771 
772 	if (id.cyi_cpu == NULL) {
773 		/*
774 		 * This is an omnipresent cyclic.
775 		 */
776 		mdb_printf("%?p %4s %3s %?s %a\n", addr, "omni", "-", "-",
777 		    id.cyi_omni_hdlr.cyo_online);
778 		mdb_printf("%?s    |\n", "");
779 		mdb_printf("%?s    +-->%4s %3s %?s %s\n", "",
780 		    "CPU", "NDX", "CYCLIC", "HANDLER");
781 
782 		if (mdb_pwalk("cycomni",
783 		    (mdb_walk_cb_t)cycid_walk_omni, NULL, addr) == -1) {
784 			mdb_warn("couldn't walk cycomni for %p", addr);
785 			return (DCMD_ERR);
786 		}
787 
788 		mdb_printf("\n");
789 
790 		return (DCMD_OK);
791 	}
792 
793 	mdb_printf("%?p ", addr);
794 
795 	return (cycid_cpu(id.cyi_cpu, id.cyi_ndx));
796 }
797