xref: /illumos-gate/usr/src/cmd/mdb/common/modules/genunix/cyclic.c (revision 66582b606a8194f7f3ba5b3a3a6dca5b0d346361)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include "cyclic.h"
27 
28 #define	CYCLIC_TRACE
29 
30 #include <mdb/mdb_modapi.h>
31 #include <sys/timer.h>
32 #include <sys/cyclic_impl.h>
33 #include <sys/sysmacros.h>
34 #include <stdio.h>
35 
36 int
37 cyccpu_vread(cyc_cpu_t *cpu, uintptr_t addr)
38 {
39 	static int inited = 0;
40 	static int cyc_trace_enabled = 0;
41 	static size_t cyccpu_size;
42 
43 	if (!inited) {
44 		inited = 1;
45 		(void) mdb_readvar(&cyc_trace_enabled, "cyc_trace_enabled");
46 		cyccpu_size = (cyc_trace_enabled) ? sizeof (*cpu) :
47 		    OFFSETOF(cyc_cpu_t, cyp_trace);
48 	}
49 
50 	if (mdb_vread(cpu, cyccpu_size, addr) == -1)
51 		return (-1);
52 
53 	if (!cyc_trace_enabled)
54 		bzero(cpu->cyp_trace, sizeof (cpu->cyp_trace));
55 
56 	return (0);
57 }
58 
59 int
60 cyccpu_walk_init(mdb_walk_state_t *wsp)
61 {
62 	if (mdb_layered_walk("cpu", wsp) == -1) {
63 		mdb_warn("couldn't walk 'cpu'");
64 		return (WALK_ERR);
65 	}
66 
67 	return (WALK_NEXT);
68 }
69 
70 int
71 cyccpu_walk_step(mdb_walk_state_t *wsp)
72 {
73 	uintptr_t addr = (uintptr_t)((cpu_t *)wsp->walk_layer)->cpu_cyclic;
74 	cyc_cpu_t cpu;
75 
76 	if (cyccpu_vread(&cpu, addr) == -1) {
77 		mdb_warn("couldn't read cyc_cpu at %p", addr);
78 		return (WALK_ERR);
79 	}
80 
81 	return (wsp->walk_callback(addr, &cpu, wsp->walk_cbdata));
82 }
83 
84 int
85 cycomni_walk_init(mdb_walk_state_t *wsp)
86 {
87 	cyc_id_t id;
88 
89 	if (wsp->walk_addr == 0) {
90 		mdb_warn("must provide a cyclic id\n");
91 		return (WALK_ERR);
92 	}
93 
94 	if (mdb_vread(&id, sizeof (id), wsp->walk_addr) == -1) {
95 		mdb_warn("couldn't read cyc_id_t at %p", wsp->walk_addr);
96 		return (WALK_ERR);
97 	}
98 
99 	if (id.cyi_cpu != NULL || id.cyi_omni_list == NULL ||
100 	    id.cyi_omni_hdlr.cyo_online == NULL) {
101 		mdb_warn("%p is not an omnipresent cyclic.\n", wsp->walk_addr);
102 		return (WALK_ERR);
103 	}
104 
105 	wsp->walk_addr = (uintptr_t)id.cyi_omni_list;
106 
107 	return (WALK_NEXT);
108 }
109 
110 int
111 cycomni_walk_step(mdb_walk_state_t *wsp)
112 {
113 	uintptr_t addr = wsp->walk_addr;
114 	cyc_omni_cpu_t omni;
115 
116 	if (addr == 0)
117 		return (WALK_DONE);
118 
119 	if (mdb_vread(&omni, sizeof (omni), addr) == -1) {
120 		mdb_warn("couldn't read cyc_omni_cpu at %p", addr);
121 		return (WALK_ERR);
122 	}
123 
124 	wsp->walk_addr = (uintptr_t)omni.cyo_next;
125 
126 	return (wsp->walk_callback(addr, &omni, wsp->walk_cbdata));
127 }
128 
129 void
130 cyclic_dump_node(cyc_cpu_t *cpu, cyc_index_t *heap, char **c, size_t w,
131     int ndx, int l, int r, int depth)
132 {
133 	int heap_left, heap_right;
134 	int me;
135 	int i, x = l + (r - l) / 2;
136 	size_t n = w - (x - 1); /* n bytes left for snprintf after c[][x - 1] */
137 
138 	heap_left = CYC_HEAP_LEFT(ndx);
139 	heap_right = CYC_HEAP_RIGHT(ndx);
140 	me = heap[ndx];
141 
142 	if (ndx >= cpu->cyp_nelems)
143 		return;
144 
145 	if (me < 10) {
146 		(void) mdb_snprintf(&c[depth][x - 1], n, " %d", me);
147 	} else if (me >= 100) {
148 		(void) mdb_snprintf(&c[depth][x - 1], n, "%3d", me);
149 	} else {
150 		(void) mdb_snprintf(&c[depth][x - 1], n, "%s%2d%s",
151 		    CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? " " : "", me,
152 		    CYC_HEAP_LEFT(CYC_HEAP_PARENT(ndx)) == ndx ? "" : " ");
153 	}
154 
155 	if (r - l > 5) {
156 		c[++depth][x] = '|';
157 		depth++;
158 
159 		for (i = l + (r - l) / 4; i < r - (r - l) / 4; i++)
160 			c[depth][i] = '-';
161 		c[depth][l + (r - l) / 4] = '+';
162 		c[depth][r - (r - l) / 4 - 1] = '+';
163 		c[depth][x] = '+';
164 	} else {
165 
166 		if (heap_left >= cpu->cyp_nelems)
167 			return;
168 
169 		(void) mdb_snprintf(&c[++depth][x - 1], n, "L%d",
170 		    heap[heap_left]);
171 
172 		if (heap_right >= cpu->cyp_nelems)
173 			return;
174 
175 		(void) mdb_snprintf(&c[++depth][x - 1], n, "R%d",
176 		    heap[heap_right]);
177 		return;
178 	}
179 
180 	if (heap_left < cpu->cyp_nelems)
181 		cyclic_dump_node(cpu, heap, c, w, heap_left, l, x, depth + 1);
182 
183 	if (heap_right < cpu->cyp_nelems)
184 		cyclic_dump_node(cpu, heap, c, w, heap_right, x, r, depth + 1);
185 }
186 
187 #define	LINES_PER_LEVEL 3
188 
189 void
190 cyclic_pretty_dump(cyc_cpu_t *cpu)
191 {
192 	char **c;
193 	int i, j;
194 	int width = 80;
195 	int depth;
196 	cyc_index_t *heap;
197 	size_t hsize = sizeof (cyc_index_t) * cpu->cyp_size;
198 
199 	heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);
200 
201 	if (mdb_vread(heap, hsize, (uintptr_t)cpu->cyp_heap) == -1) {
202 		mdb_warn("couldn't read heap at %p", (uintptr_t)cpu->cyp_heap);
203 		return;
204 	}
205 
206 	for (depth = 0; (1 << depth) < cpu->cyp_nelems; depth++)
207 		continue;
208 	depth++;
209 	depth = (depth + 1) * LINES_PER_LEVEL;
210 
211 	c = mdb_zalloc(sizeof (char *) * depth, UM_SLEEP|UM_GC);
212 
213 	for (i = 0; i < depth; i++)
214 		c[i] = mdb_zalloc(width, UM_SLEEP|UM_GC);
215 
216 	cyclic_dump_node(cpu, heap, c, width, 0, 1, width - 2, 0);
217 
218 	for (i = 0; i < depth; i++) {
219 		int dump = 0;
220 		for (j = 0; j < width - 1; j++) {
221 			if (c[i][j] == '\0')
222 				c[i][j] = ' ';
223 			else
224 				dump = 1;
225 		}
226 		c[i][width - 2] = '\n';
227 
228 		if (dump)
229 			mdb_printf(c[i]);
230 	}
231 }
232 
233 /*
234  * Yes, this is very weak.  Full 16-column-wide 64-bit addresses screw up
235  * ::cycinfo's very carefully planned 80-column layout.  We set the column
236  * width for addresses to be 11 (instead of 16), knowing that the kernel
237  * heap (from which these data structures are allocated) starts at
238  * 0x0000030000000000, and isn't likely to extend to 0x0000100000000000.
239  */
240 #ifdef _LP64
241 #define	CYC_ADDR_WIDTH	11
242 #else
243 #define	CYC_ADDR_WIDTH	8
244 #endif
245 
246 int
247 cycinfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
248 {
249 	cyc_cpu_t cpu;
250 	cpu_t c;
251 	cyc_index_t root, i, *heap;
252 	size_t hsize;
253 	cyclic_t *cyc;
254 	uintptr_t caddr;
255 	uint_t verbose = FALSE, Verbose = FALSE;
256 	int header = 0;
257 	cyc_level_t lev;
258 
259 	if (!(flags & DCMD_ADDRSPEC)) {
260 		if (mdb_walk_dcmd("cyccpu", "cycinfo", argc, argv) == -1) {
261 			mdb_warn("can't walk 'cyccpu'");
262 			return (DCMD_ERR);
263 		}
264 		return (DCMD_OK);
265 	}
266 
267 	if (mdb_getopts(argc, argv,
268 	    'v', MDB_OPT_SETBITS, TRUE, &verbose,
269 	    'V', MDB_OPT_SETBITS, TRUE, &Verbose, NULL) != argc)
270 		return (DCMD_USAGE);
271 
272 	if (!DCMD_HDRSPEC(flags) && (verbose || Verbose))
273 		mdb_printf("\n\n");
274 
275 	if (DCMD_HDRSPEC(flags) || verbose || Verbose)
276 		mdb_printf("%3s %*s %7s %6s %*s %15s %s\n", "CPU",
277 		    CYC_ADDR_WIDTH, "CYC_CPU", "STATE", "NELEMS",
278 		    CYC_ADDR_WIDTH, "ROOT", "FIRE", "HANDLER");
279 
280 	if (cyccpu_vread(&cpu, addr) == -1) {
281 		mdb_warn("couldn't read cyc_cpu at %p", addr);
282 		return (DCMD_ERR);
283 	}
284 
285 	if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
286 		mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
287 		return (DCMD_ERR);
288 	}
289 
290 	cyc = mdb_alloc(sizeof (cyclic_t) * cpu.cyp_size, UM_SLEEP | UM_GC);
291 	caddr = (uintptr_t)cpu.cyp_cyclics;
292 
293 	if (mdb_vread(cyc, sizeof (cyclic_t) * cpu.cyp_size, caddr) == -1) {
294 		mdb_warn("couldn't read cyclic at %p", caddr);
295 		return (DCMD_ERR);
296 	}
297 
298 	hsize = sizeof (cyc_index_t) * cpu.cyp_size;
299 	heap = mdb_alloc(hsize, UM_SLEEP | UM_GC);
300 
301 	if (mdb_vread(heap, hsize, (uintptr_t)cpu.cyp_heap) == -1) {
302 		mdb_warn("couldn't read heap at %p", cpu.cyp_heap);
303 		return (DCMD_ERR);
304 	}
305 
306 	root = heap[0];
307 
308 	mdb_printf("%3d %0*p %7s %6d ", c.cpu_id, CYC_ADDR_WIDTH, addr,
309 	    cpu.cyp_state == CYS_ONLINE ? "online" :
310 	    cpu.cyp_state == CYS_OFFLINE ? "offline" :
311 	    cpu.cyp_state == CYS_EXPANDING ? "expand" :
312 	    cpu.cyp_state == CYS_REMOVING ? "remove" :
313 	    cpu.cyp_state == CYS_SUSPENDED ? "suspend" : "????",
314 	    cpu.cyp_nelems);
315 
316 	if (cpu.cyp_nelems > 0)
317 		mdb_printf("%0*p %15llx %a\n", CYC_ADDR_WIDTH,
318 		    caddr, cyc[root].cy_expire, cyc[root].cy_handler);
319 	else
320 		mdb_printf("%*s %15s %s\n", CYC_ADDR_WIDTH, "-", "-", "-");
321 
322 	if (!verbose && !Verbose)
323 		return (DCMD_OK);
324 
325 	mdb_printf("\n");
326 
327 	cyclic_pretty_dump(&cpu);
328 
329 	mdb_inc_indent(2);
330 
331 	for (i = 0; i < cpu.cyp_size; i++) {
332 		int j;
333 
334 		for (j = 0; j < cpu.cyp_size; j++) {
335 			if (heap[j] == i)
336 				break;
337 		}
338 
339 		if (!Verbose && j >= cpu.cyp_nelems)
340 			continue;
341 
342 		if (!header) {
343 			header = 1;
344 			mdb_printf("\n%*s %3s %4s %4s %5s %15s %7s %s\n",
345 			    CYC_ADDR_WIDTH, "ADDR", "NDX", "HEAP", "LEVL",
346 			    "PEND", "FIRE", "USECINT", "HANDLER");
347 		}
348 
349 		mdb_printf("%0*p %3d ", CYC_ADDR_WIDTH,
350 		    caddr + i * sizeof (cyclic_t), i);
351 
352 		mdb_printf("%4d ", j);
353 
354 		if (j >= cpu.cyp_nelems) {
355 			mdb_printf("%4s %5s %15s %7s %s\n", "-", "-",
356 			    "-", "-", "-");
357 			continue;
358 		}
359 
360 		mdb_printf("%4s %5d %15llx ",
361 		    cyc[i].cy_level == CY_HIGH_LEVEL ? "high" :
362 		    cyc[i].cy_level == CY_LOCK_LEVEL ? "lock" :
363 		    cyc[i].cy_level == CY_LOW_LEVEL ? "low" : "????",
364 		    cyc[i].cy_pend, cyc[i].cy_expire);
365 
366 		if (cyc[i].cy_interval + cyc[i].cy_expire != INT64_MAX)
367 			mdb_printf("%7lld ", cyc[i].cy_interval /
368 			    (uint64_t)(NANOSEC / MICROSEC));
369 		else
370 			mdb_printf("%7s ", "-");
371 
372 		mdb_printf("%a\n", cyc[i].cy_handler);
373 	}
374 
375 
376 	if (!Verbose)
377 		goto out;
378 
379 	for (lev = CY_LOW_LEVEL; lev < CY_LOW_LEVEL + CY_SOFT_LEVELS; lev++) {
380 		cyc_softbuf_t *softbuf = &cpu.cyp_softbuf[lev];
381 		char which = softbuf->cys_hard, shared = 1;
382 		cyc_pcbuffer_t *pc;
383 		size_t bufsiz;
384 		cyc_index_t *buf;
385 
386 		if (softbuf->cys_hard != softbuf->cys_soft)
387 			shared = 0;
388 
389 again:
390 		pc = &softbuf->cys_buf[which];
391 		bufsiz = (pc->cypc_sizemask + 1) * sizeof (cyc_index_t);
392 		buf = mdb_alloc(bufsiz, UM_SLEEP | UM_GC);
393 
394 		if (mdb_vread(buf, bufsiz, (uintptr_t)pc->cypc_buf) == -1) {
395 			mdb_warn("couldn't read cypc_buf at %p", pc->cypc_buf);
396 			continue;
397 		}
398 
399 		mdb_printf("\n%3s %4s %4s %4s %*s %4s %*s\n", "CPU",
400 		    "LEVL", "USER", "NDX", CYC_ADDR_WIDTH, "ADDR", "CYC",
401 		    CYC_ADDR_WIDTH, "CYC_ADDR", "PEND");
402 
403 		for (i = 0; i <= pc->cypc_sizemask &&
404 		    i <= pc->cypc_prodndx; i++) {
405 			uintptr_t cyc_addr = caddr + buf[i] * sizeof (cyclic_t);
406 
407 			mdb_printf("%3d %4s %4s ", c.cpu_id,
408 			    lev == CY_HIGH_LEVEL ? "high" :
409 			    lev == CY_LOCK_LEVEL ? "lock" :
410 			    lev == CY_LOW_LEVEL ? "low" : "????",
411 			    shared ? "shrd" : which == softbuf->cys_hard ?
412 			    "hard" : "soft");
413 
414 			mdb_printf("%4d %0*p ", i, CYC_ADDR_WIDTH,
415 			    (uintptr_t)&buf[i] - (uintptr_t)&buf[0] +
416 			    (uintptr_t)pc->cypc_buf, buf[i],
417 			    caddr + buf[i] * sizeof (cyclic_t));
418 
419 			if (i >= pc->cypc_prodndx)
420 				mdb_printf("%4s %*s %5s  ",
421 				    "-", CYC_ADDR_WIDTH, "-", "-");
422 			else {
423 				cyclic_t c;
424 
425 				if (mdb_vread(&c, sizeof (c), cyc_addr) == -1) {
426 					mdb_warn("\ncouldn't read cyclic at "
427 					    "%p", cyc_addr);
428 					continue;
429 				}
430 
431 				mdb_printf("%4d %0*p %5d  ", buf[i],
432 				    CYC_ADDR_WIDTH, cyc_addr, c.cy_pend);
433 			}
434 
435 			if (i == (pc->cypc_consndx & pc->cypc_sizemask)) {
436 				mdb_printf("<-- consndx");
437 				if (i == (pc->cypc_prodndx & pc->cypc_sizemask))
438 					mdb_printf(",prodndx");
439 				mdb_printf("\n");
440 				continue;
441 			}
442 
443 			if (i == (pc->cypc_prodndx & pc->cypc_sizemask)) {
444 				mdb_printf("<-- prodndx\n");
445 				continue;
446 			}
447 			mdb_printf("\n");
448 
449 			if (i >= pc->cypc_prodndx)
450 				break;
451 		}
452 
453 		if (!shared && which == softbuf->cys_hard) {
454 			which = softbuf->cys_soft;
455 			goto again;
456 		}
457 	}
458 
459 out:
460 	mdb_dec_indent(2);
461 	return (DCMD_OK);
462 }
463 
464 int
465 cyctrace_walk_init(mdb_walk_state_t *wsp)
466 {
467 	cyc_cpu_t *cpu;
468 	int i;
469 
470 	cpu = mdb_zalloc(sizeof (cyc_cpu_t), UM_SLEEP);
471 
472 	if (wsp->walk_addr == 0) {
473 		/*
474 		 * If an address isn't provided, we'll use the passive buffer.
475 		 */
476 		GElf_Sym sym;
477 		cyc_tracebuf_t *tr = &cpu->cyp_trace[0];
478 		uintptr_t addr;
479 
480 		if (mdb_lookup_by_name("cyc_ptrace", &sym) == -1) {
481 			mdb_warn("couldn't find passive buffer");
482 			return (-1);
483 		}
484 
485 		addr = (uintptr_t)sym.st_value;
486 
487 		if (mdb_vread(tr, sizeof (cyc_tracebuf_t), addr) == -1) {
488 			mdb_warn("couldn't read passive buffer");
489 			return (-1);
490 		}
491 
492 		wsp->walk_addr = addr - offsetof(cyc_cpu_t, cyp_trace[0]);
493 	} else {
494 		if (cyccpu_vread(cpu, wsp->walk_addr) == -1) {
495 			mdb_warn("couldn't read cyc_cpu at %p", wsp->walk_addr);
496 			mdb_free(cpu, sizeof (cyc_cpu_t));
497 			return (-1);
498 		}
499 	}
500 
501 	for (i = 0; i < CY_LEVELS; i++) {
502 		if (cpu->cyp_trace[i].cyt_ndx-- == 0)
503 			cpu->cyp_trace[i].cyt_ndx = CY_NTRACEREC - 1;
504 	}
505 
506 	wsp->walk_data = cpu;
507 
508 	return (0);
509 }
510 
511 int
512 cyctrace_walk_step(mdb_walk_state_t *wsp)
513 {
514 	cyc_cpu_t *cpu = wsp->walk_data;
515 	cyc_tracebuf_t *buf = cpu->cyp_trace;
516 	hrtime_t latest = 0;
517 	int i, ndx, new_ndx, lev, rval;
518 	uintptr_t addr;
519 
520 	for (i = 0; i < CY_LEVELS; i++) {
521 		if ((ndx = buf[i].cyt_ndx) == -1)
522 			continue;
523 
524 		/*
525 		 * Account for NPT.
526 		 */
527 		buf[i].cyt_buf[ndx].cyt_tstamp <<= 1;
528 		buf[i].cyt_buf[ndx].cyt_tstamp >>= 1;
529 
530 		if (buf[i].cyt_buf[ndx].cyt_tstamp > latest) {
531 			latest = buf[i].cyt_buf[ndx].cyt_tstamp;
532 			lev = i;
533 		}
534 	}
535 
536 	/*
537 	 * If we didn't find one, we're done.
538 	 */
539 	if (latest == 0)
540 		return (-1);
541 
542 	buf = &buf[lev];
543 	ndx = buf->cyt_ndx;
544 	addr = wsp->walk_addr +
545 	    (uintptr_t)&(buf->cyt_buf[ndx]) - (uintptr_t)cpu;
546 
547 	rval = wsp->walk_callback(addr, &buf->cyt_buf[ndx], wsp->walk_cbdata);
548 
549 	new_ndx = ndx == 0 ? CY_NTRACEREC - 1 : ndx - 1;
550 
551 	if (buf->cyt_buf[new_ndx].cyt_tstamp != 0 &&
552 	    buf->cyt_buf[new_ndx].cyt_tstamp > buf->cyt_buf[ndx].cyt_tstamp)
553 		new_ndx = -1;
554 
555 	buf->cyt_ndx = new_ndx;
556 
557 	return (rval);
558 }
559 
560 void
561 cyctrace_walk_fini(mdb_walk_state_t *wsp)
562 {
563 	cyc_cpu_t *cpu = wsp->walk_data;
564 
565 	mdb_free(cpu, sizeof (cyc_cpu_t));
566 }
567 
568 #define	WHYLEN	17
569 
570 int
571 cyctrace_walk(uintptr_t addr, const cyc_tracerec_t *rec, cyc_cpu_t *cpu)
572 {
573 	int i;
574 	char c[WHYLEN];
575 
576 	for (i = 0; cpu != NULL && i < CY_LEVELS; i++)
577 		if (addr < (uintptr_t)&cpu->cyp_trace[i + 1].cyt_buf[0])
578 			break;
579 
580 	(void) mdb_readstr(c, WHYLEN, (uintptr_t)rec->cyt_why);
581 
582 	mdb_printf("%08p %4s %15llx %-*s %15llx %15llx\n",
583 	    addr & UINT_MAX, cpu == NULL ? "pasv" :
584 	    i == CY_HIGH_LEVEL ? "high" : i == CY_LOCK_LEVEL ? "lock" :
585 	    i == CY_LOW_LEVEL ? "low" : "????", rec->cyt_tstamp, WHYLEN, c,
586 	    rec->cyt_arg0, rec->cyt_arg1);
587 
588 	return (0);
589 }
590 
591 /*ARGSUSED*/
592 int
593 cyctrace(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
594 {
595 	if (!(flags & DCMD_ADDRSPEC) || argc != 0)
596 		addr = 0;
597 
598 	if (mdb_pwalk("cyctrace", (mdb_walk_cb_t)cyctrace_walk,
599 	    (void *)addr, addr) == -1) {
600 		mdb_warn("couldn't walk cyctrace");
601 		return (DCMD_ERR);
602 	}
603 
604 	return (DCMD_OK);
605 }
606 
607 int
608 cyccover_comp(const void *l, const void *r)
609 {
610 	cyc_coverage_t *lhs = (cyc_coverage_t *)l;
611 	cyc_coverage_t *rhs = (cyc_coverage_t *)r;
612 
613 	char ly[WHYLEN], ry[WHYLEN];
614 
615 	if (rhs->cyv_why == lhs->cyv_why)
616 		return (0);
617 
618 	if (rhs->cyv_why == NULL)
619 		return (-1);
620 
621 	if (lhs->cyv_why == NULL)
622 		return (1);
623 
624 	(void) mdb_readstr(ly, WHYLEN, (uintptr_t)lhs->cyv_why);
625 	(void) mdb_readstr(ry, WHYLEN, (uintptr_t)rhs->cyv_why);
626 
627 	return (strcmp(ly, ry));
628 }
629 
630 /*ARGSUSED*/
631 int
632 cyccover(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
633 {
634 	cyc_coverage_t cv[CY_NCOVERAGE];
635 	char c[WHYLEN];
636 	GElf_Sym sym;
637 	int i;
638 
639 	if ((flags & DCMD_ADDRSPEC) || argc != 0)
640 		return (DCMD_USAGE);
641 
642 	if (mdb_lookup_by_name("cyc_coverage", &sym) == -1) {
643 		mdb_warn("couldn't find coverage information");
644 		return (DCMD_ABORT);
645 	}
646 
647 	addr = (uintptr_t)sym.st_value;
648 
649 	if (mdb_vread(cv, sizeof (cyc_coverage_t) * CY_NCOVERAGE, addr) == -1) {
650 		mdb_warn("couldn't read coverage array at %p", addr);
651 		return (DCMD_ABORT);
652 	}
653 
654 	mdb_printf("%-*s %8s %8s %8s %15s %15s\n",
655 	    WHYLEN, "POINT", "HIGH", "LOCK", "LOW/PASV", "ARG0", "ARG1");
656 
657 	qsort(cv, CY_NCOVERAGE, sizeof (cyc_coverage_t), cyccover_comp);
658 
659 	for (i = 0; i < CY_NCOVERAGE; i++) {
660 		if (cv[i].cyv_why != NULL) {
661 			(void) mdb_readstr(c, WHYLEN, (uintptr_t)cv[i].cyv_why);
662 			mdb_printf("%-*s %8d %8d %8d %15llx %15llx\n",
663 			    WHYLEN, c,
664 			    cv[i].cyv_count[CY_HIGH_LEVEL],
665 			    cv[i].cyv_count[CY_LOCK_LEVEL],
666 			    cv[i].cyv_passive_count != 0 ?
667 			    cv[i].cyv_passive_count :
668 			    cv[i].cyv_count[CY_LOW_LEVEL],
669 			    cv[i].cyv_arg0, cv[i].cyv_arg1);
670 		}
671 	}
672 
673 	return (DCMD_OK);
674 }
675 
676 /*ARGSUSED*/
677 int
678 cyclic(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
679 {
680 	cyclic_t cyc;
681 
682 	if (!(flags & DCMD_ADDRSPEC) || argc != 0)
683 		return (DCMD_USAGE);
684 
685 	if (DCMD_HDRSPEC(flags))
686 		mdb_printf("%?s %4s %5s %5s %15s %7s %s\n", "ADDR", "LEVL",
687 		    "PEND", "FLAGS", "FIRE", "USECINT", "HANDLER");
688 
689 	if (mdb_vread(&cyc, sizeof (cyclic_t), addr) == -1) {
690 		mdb_warn("couldn't read cyclic at %p", addr);
691 		return (DCMD_ERR);
692 	}
693 
694 	mdb_printf("%0?p %4s %5d  %04x %15llx %7lld %a\n", addr,
695 	    cyc.cy_level == CY_HIGH_LEVEL ? "high" :
696 	    cyc.cy_level == CY_LOCK_LEVEL ? "lock" :
697 	    cyc.cy_level == CY_LOW_LEVEL ? "low" : "????",
698 	    cyc.cy_pend, cyc.cy_flags, cyc.cy_expire,
699 	    cyc.cy_interval / (uint64_t)(NANOSEC / MICROSEC),
700 	    cyc.cy_handler);
701 
702 	return (DCMD_OK);
703 }
704 
705 static int
706 cycid_cpu(cyc_cpu_t *addr, int ndx)
707 {
708 	cyc_cpu_t cpu;
709 	cpu_t c;
710 	uintptr_t caddr;
711 	cyclic_t cyc;
712 
713 	if (cyccpu_vread(&cpu, (uintptr_t)addr) == -1) {
714 		mdb_warn("couldn't read cyc_cpu at %p", addr);
715 		return (DCMD_ERR);
716 	}
717 
718 	if (mdb_vread(&c, sizeof (c), (uintptr_t)cpu.cyp_cpu) == -1) {
719 		mdb_warn("couldn't read cpu at %p", cpu.cyp_cpu);
720 		return (DCMD_ERR);
721 	}
722 
723 	caddr = (uintptr_t)cpu.cyp_cyclics + ndx * sizeof (cyclic_t);
724 
725 	if (mdb_vread(&cyc, sizeof (cyc), caddr) == -1) {
726 		mdb_warn("couldn't read cyclic at %p", caddr);
727 		return (DCMD_ERR);
728 	}
729 
730 	mdb_printf("%4d %3d %?p %a\n", c.cpu_id, ndx, caddr, cyc.cy_handler);
731 
732 	return (DCMD_OK);
733 }
734 
735 /*ARGSUSED*/
736 static int
737 cycid_walk_omni(uintptr_t addr, const cyc_omni_cpu_t *omni, int *ignored)
738 {
739 	mdb_printf("%?s        ");
740 	cycid_cpu(omni->cyo_cpu, omni->cyo_ndx);
741 
742 	return (WALK_NEXT);
743 }
744 
745 /*ARGSUSED*/
746 int
747 cycid(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *av)
748 {
749 	cyc_id_t id;
750 
751 	if (!(flags & DCMD_ADDRSPEC)) {
752 		if (mdb_walk_dcmd("cyclic_id_cache", "cycid", ac, av) == -1) {
753 			mdb_warn("can't walk cyclic_id_cache");
754 			return (DCMD_ERR);
755 		}
756 
757 		return (DCMD_OK);
758 	}
759 
760 	if (DCMD_HDRSPEC(flags)) {
761 		mdb_printf("%?s %4s %3s %?s %s\n", "ADDR", "CPU", "NDX",
762 		    "CYCLIC", "HANDLER");
763 	}
764 
765 	if (mdb_vread(&id, sizeof (id), addr) == -1) {
766 		mdb_warn("couldn't read cyc_id_t at %p", addr);
767 		return (DCMD_ERR);
768 	}
769 
770 	if (id.cyi_cpu == NULL) {
771 		/*
772 		 * This is an omnipresent cyclic.
773 		 */
774 		mdb_printf("%?p %4s %3s %?s %a\n", addr, "omni", "-", "-",
775 		    id.cyi_omni_hdlr.cyo_online);
776 		mdb_printf("%?s    |\n", "");
777 		mdb_printf("%?s    +-->%4s %3s %?s %s\n", "",
778 		    "CPU", "NDX", "CYCLIC", "HANDLER");
779 
780 		if (mdb_pwalk("cycomni",
781 		    (mdb_walk_cb_t)cycid_walk_omni, NULL, addr) == -1) {
782 			mdb_warn("couldn't walk cycomni for %p", addr);
783 			return (DCMD_ERR);
784 		}
785 
786 		mdb_printf("\n");
787 
788 		return (DCMD_OK);
789 	}
790 
791 	mdb_printf("%?p ", addr);
792 
793 	return (cycid_cpu(id.cyi_cpu, id.cyi_ndx));
794 }
795