xref: /freebsd/usr.bin/netstat/netisr.c (revision 4232f826683298e85d469c0ef17259cecd13b2c7)
1 /*-
2  * Copyright (c) 2010-2011 Juniper Networks, Inc.
3  * All rights reserved.
4  *
5  * This software was developed by Robert N. M. Watson under contract
6  * to Juniper Networks, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/sysctl.h>
36 
37 #include <sys/_lock.h>
38 #include <sys/_mutex.h>
39 
40 #define	_WANT_NETISR_INTERNAL
41 #include <net/netisr.h>
42 #include <net/netisr_internal.h>
43 
44 #include <err.h>
45 #include <kvm.h>
46 #include <stdint.h>
47 #include <stdio.h>
48 #include <stdlib.h>
49 #include <stdbool.h>
50 #include <string.h>
51 #include <libxo/xo.h>
52 #include "netstat.h"
53 
54 /*
55  * Print statistics for the kernel netisr subsystem.
56  */
57 static u_int				 bindthreads;
58 static u_int				 maxthreads;
59 static u_int				 numthreads;
60 
61 static u_int				 defaultqlimit;
62 static u_int				 maxqlimit;
63 
64 static char				 dispatch_policy[20];
65 
66 static struct sysctl_netisr_proto	*proto_array;
67 static u_int				 proto_array_len;
68 
69 static struct sysctl_netisr_workstream	*workstream_array;
70 static u_int				 workstream_array_len;
71 
72 static struct sysctl_netisr_work	*work_array;
73 static u_int				 work_array_len;
74 
75 static u_int				*nws_array;
76 
77 static u_int				 maxprot;
78 
79 static void
80 netisr_dispatch_policy_to_string(u_int policy, char *buf,
81     size_t buflen)
82 {
83 	const char *str;
84 
85 	switch (policy) {
86 	case NETISR_DISPATCH_DEFAULT:
87 		str = "default";
88 		break;
89 	case NETISR_DISPATCH_DEFERRED:
90 		str = "deferred";
91 		break;
92 	case NETISR_DISPATCH_HYBRID:
93 		str = "hybrid";
94 		break;
95 	case NETISR_DISPATCH_DIRECT:
96 		str = "direct";
97 		break;
98 	default:
99 		str = "unknown";
100 		break;
101 	}
102 	snprintf(buf, buflen, "%s", str);
103 }
104 
105 static void
106 netisr_load_kvm_uint(kvm_t *kd, const char *name, u_int *p)
107 {
108 	struct nlist nl[] = {
109 		{ .n_name = name },
110 		{ .n_name = NULL },
111 	};
112 	int ret;
113 
114 	ret = kvm_nlist(kd, nl);
115 	if (ret < 0)
116 		xo_errx(-1, "%s: kvm_nlist(%s): %s", __func__, name,
117 		    kvm_geterr(kd));
118 	if (ret != 0)
119 		xo_errx(-1, "%s: kvm_nlist(%s): unresolved symbol", __func__,
120 		    name);
121 	if (kvm_read(kd, nl[0].n_value, p, sizeof(*p)) != sizeof(*p))
122 		xo_errx(-1, "%s: kvm_read(%s): %s", __func__, name,
123 		    kvm_geterr(kd));
124 }
125 
126 /*
127  * Load a nul-terminated string from KVM up to 'limit', guarantee that the
128  * string in local memory is nul-terminated.
129  */
130 static void
131 netisr_load_kvm_string(kvm_t *kd, uintptr_t addr, char *dest, u_int limit)
132 {
133 	u_int i;
134 
135 	for (i = 0; i < limit; i++) {
136 		if (kvm_read(kd, addr + i, &dest[i], sizeof(dest[i])) !=
137 		    sizeof(dest[i]))
138 			xo_err(-1, "%s: kvm_read: %s", __func__,
139 			    kvm_geterr(kd));
140 		if (dest[i] == '\0')
141 			break;
142 	}
143 	dest[limit - 1] = '\0';
144 }
145 
146 static const char *
147 netisr_proto2name(u_int proto)
148 {
149 	u_int i;
150 
151 	for (i = 0; i < proto_array_len; i++) {
152 		if (proto_array[i].snp_proto == proto)
153 			return (proto_array[i].snp_name);
154 	}
155 	return ("unknown");
156 }
157 
158 static int
159 netisr_protoispresent(u_int proto)
160 {
161 	u_int i;
162 
163 	for (i = 0; i < proto_array_len; i++) {
164 		if (proto_array[i].snp_proto == proto)
165 			return (1);
166 	}
167 	return (0);
168 }
169 
170 static void
171 netisr_load_kvm_config(kvm_t *kd)
172 {
173 	u_int tmp;
174 
175 	netisr_load_kvm_uint(kd, "_netisr_bindthreads", &bindthreads);
176 	netisr_load_kvm_uint(kd, "_netisr_maxthreads", &maxthreads);
177 	netisr_load_kvm_uint(kd, "_nws_count", &numthreads);
178 
179 	netisr_load_kvm_uint(kd, "_netisr_defaultqlimit", &defaultqlimit);
180 	netisr_load_kvm_uint(kd, "_netisr_maxqlimit", &maxqlimit);
181 
182 	netisr_load_kvm_uint(kd, "_netisr_dispatch_policy", &tmp);
183 	netisr_dispatch_policy_to_string(tmp, dispatch_policy,
184 	    sizeof(dispatch_policy));
185 }
186 
187 static void
188 netisr_load_sysctl_uint(const char *name, u_int *p)
189 {
190 	size_t retlen;
191 
192 	retlen = sizeof(u_int);
193 	if (sysctlbyname(name, p, &retlen, NULL, 0) < 0)
194 		xo_err(-1, "%s", name);
195 	if (retlen != sizeof(u_int))
196 		xo_errx(-1, "%s: invalid len %ju", name, (uintmax_t)retlen);
197 }
198 
199 static void
200 netisr_load_sysctl_string(const char *name, char *p, size_t len)
201 {
202 	size_t retlen;
203 
204 	retlen = len;
205 	if (sysctlbyname(name, p, &retlen, NULL, 0) < 0)
206 		xo_err(-1, "%s", name);
207 	p[len - 1] = '\0';
208 }
209 
210 static void
211 netisr_load_sysctl_config(void)
212 {
213 
214 	netisr_load_sysctl_uint("net.isr.bindthreads", &bindthreads);
215 	netisr_load_sysctl_uint("net.isr.maxthreads", &maxthreads);
216 	netisr_load_sysctl_uint("net.isr.numthreads", &numthreads);
217 
218 	netisr_load_sysctl_uint("net.isr.defaultqlimit", &defaultqlimit);
219 	netisr_load_sysctl_uint("net.isr.maxqlimit", &maxqlimit);
220 
221 	netisr_load_sysctl_string("net.isr.dispatch", dispatch_policy,
222 	    sizeof(dispatch_policy));
223 }
224 
225 static void
226 netisr_load_kvm_proto(kvm_t *kd)
227 {
228 	struct nlist nl[] = {
229 #define	NLIST_NETISR_PROTO	0
230 		{ .n_name = "_netisr_proto" },
231 		{ .n_name = NULL },
232 	};
233 	struct netisr_proto *np_array, *npp;
234 	u_int i, protocount;
235 	struct sysctl_netisr_proto *snpp;
236 	size_t len;
237 	int ret;
238 
239 	/*
240 	 * Kernel compile-time and user compile-time definitions of
241 	 * NETISR_MAXPROT must match, as we use that to size work arrays.
242 	 */
243 	netisr_load_kvm_uint(kd, "_netisr_maxprot", &maxprot);
244 	if (maxprot != NETISR_MAXPROT)
245 		xo_errx(-1, "%s: NETISR_MAXPROT mismatch", __func__);
246 	len = maxprot * sizeof(*np_array);
247 	np_array = malloc(len);
248 	if (np_array == NULL)
249 		xo_err(-1, "%s: malloc", __func__);
250 	ret = kvm_nlist(kd, nl);
251 	if (ret < 0)
252 		xo_errx(-1, "%s: kvm_nlist(_netisr_proto): %s", __func__,
253 		    kvm_geterr(kd));
254 	if (ret != 0)
255 		xo_errx(-1, "%s: kvm_nlist(_netisr_proto): unresolved symbol",
256 		    __func__);
257 	if (kvm_read(kd, nl[NLIST_NETISR_PROTO].n_value, np_array, len) !=
258 	    (ssize_t)len)
259 		xo_errx(-1, "%s: kvm_read(_netisr_proto): %s", __func__,
260 		    kvm_geterr(kd));
261 
262 	/*
263 	 * Size and allocate memory to hold only live protocols.
264 	 */
265 	protocount = 0;
266 	for (i = 0; i < maxprot; i++) {
267 		if (np_array[i].np_name == NULL)
268 			continue;
269 		protocount++;
270 	}
271 	proto_array = calloc(protocount, sizeof(*proto_array));
272 	if (proto_array == NULL)
273 		err(-1, "malloc");
274 	protocount = 0;
275 	for (i = 0; i < maxprot; i++) {
276 		npp = &np_array[i];
277 		if (npp->np_name == NULL)
278 			continue;
279 		snpp = &proto_array[protocount];
280 		snpp->snp_version = sizeof(*snpp);
281 		netisr_load_kvm_string(kd, (uintptr_t)npp->np_name,
282 		    snpp->snp_name, sizeof(snpp->snp_name));
283 		snpp->snp_proto = i;
284 		snpp->snp_qlimit = npp->np_qlimit;
285 		snpp->snp_policy = npp->np_policy;
286 		snpp->snp_dispatch = npp->np_dispatch;
287 		if (npp->np_m2flow != NULL)
288 			snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW;
289 		if (npp->np_m2cpuid != NULL)
290 			snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID;
291 		if (npp->np_drainedcpu != NULL)
292 			snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU;
293 		protocount++;
294 	}
295 	proto_array_len = protocount;
296 	free(np_array);
297 }
298 
299 static void
300 netisr_load_sysctl_proto(void)
301 {
302 	size_t len;
303 
304 	if (sysctlbyname("net.isr.proto", NULL, &len, NULL, 0) < 0)
305 		xo_err(-1, "net.isr.proto: query len");
306 	if (len % sizeof(*proto_array) != 0)
307 		xo_errx(-1, "net.isr.proto: invalid len");
308 	proto_array = malloc(len);
309 	if (proto_array == NULL)
310 		xo_err(-1, "malloc");
311 	if (sysctlbyname("net.isr.proto", proto_array, &len, NULL, 0) < 0)
312 		xo_err(-1, "net.isr.proto: query data");
313 	if (len % sizeof(*proto_array) != 0)
314 		xo_errx(-1, "net.isr.proto: invalid len");
315 	proto_array_len = len / sizeof(*proto_array);
316 	if (proto_array_len < 1)
317 		xo_errx(-1, "net.isr.proto: no data");
318 	if (proto_array[0].snp_version != sizeof(proto_array[0]))
319 		xo_errx(-1, "net.isr.proto: invalid version");
320 }
321 
322 static void
323 netisr_load_kvm_workstream(kvm_t *kd)
324 {
325 	struct nlist nl[] = {
326 #define	NLIST_NWS_ARRAY		0
327 		{ .n_name = "_nws_array" },
328 		{ .n_name = NULL },
329 	};
330 	struct netisr_workstream nws;
331 	struct sysctl_netisr_workstream *snwsp;
332 	struct sysctl_netisr_work *snwp;
333 	struct netisr_work *nwp;
334 	struct nlist nl_nws[2];
335 	u_int counter, cpuid, proto, wsid;
336 	size_t len;
337 	int ret;
338 
339 	len = numthreads * sizeof(*nws_array);
340 	nws_array = malloc(len);
341 	if (nws_array == NULL)
342 		xo_err(-1, "malloc");
343 	ret = kvm_nlist(kd, nl);
344 	if (ret < 0)
345 		xo_errx(-1, "%s: kvm_nlist: %s", __func__, kvm_geterr(kd));
346 	if (ret != 0)
347 		xo_errx(-1, "%s: kvm_nlist: unresolved symbol", __func__);
348 	if (kvm_read(kd, nl[NLIST_NWS_ARRAY].n_value, nws_array, len) !=
349 	    (ssize_t)len)
350 		xo_errx(-1, "%s: kvm_read(_nws_array): %s", __func__,
351 		    kvm_geterr(kd));
352 	workstream_array = calloc(numthreads, sizeof(*workstream_array));
353 	if (workstream_array == NULL)
354 		xo_err(-1, "calloc");
355 	workstream_array_len = numthreads;
356 	work_array = calloc(numthreads * proto_array_len, sizeof(*work_array));
357 	if (work_array == NULL)
358 		xo_err(-1, "calloc");
359 	counter = 0;
360 	for (wsid = 0; wsid < numthreads; wsid++) {
361 		cpuid = nws_array[wsid];
362 		if (kvm_dpcpu_setcpu(kd, cpuid) < 0)
363 			xo_errx(-1, "%s: kvm_dpcpu_setcpu(%u): %s", __func__,
364 			    cpuid, kvm_geterr(kd));
365 		bzero(nl_nws, sizeof(nl_nws));
366 		nl_nws[0].n_name = "_nws";
367 		ret = kvm_nlist(kd, nl_nws);
368 		if (ret < 0)
369 			xo_errx(-1, "%s: kvm_nlist looking up nws on CPU "
370 			    "%u: %s", __func__, cpuid, kvm_geterr(kd));
371 		if (ret != 0)
372 			xo_errx(-1, "%s: kvm_nlist(nws): unresolved symbol on "
373 			    "CPU %u", __func__, cpuid);
374 		if (kvm_read(kd, nl_nws[0].n_value, &nws, sizeof(nws)) !=
375 		    sizeof(nws))
376 			xo_errx(-1, "%s: kvm_read(nw): %s", __func__,
377 			    kvm_geterr(kd));
378 		snwsp = &workstream_array[wsid];
379 		snwsp->snws_version = sizeof(*snwsp);
380 		snwsp->snws_wsid = cpuid;
381 		snwsp->snws_cpu = cpuid;
382 		if (nws.nws_intr_event != NULL)
383 			snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR;
384 
385 		/*
386 		 * Extract the CPU's per-protocol work information.
387 		 */
388 		xo_emit("counting to maxprot: {:maxprot/%u}\n", maxprot);
389 		for (proto = 0; proto < maxprot; proto++) {
390 			if (!netisr_protoispresent(proto))
391 				continue;
392 			nwp = &nws.nws_work[proto];
393 			snwp = &work_array[counter];
394 			snwp->snw_version = sizeof(*snwp);
395 			snwp->snw_wsid = cpuid;
396 			snwp->snw_proto = proto;
397 			snwp->snw_len = nwp->nw_len;
398 			snwp->snw_watermark = nwp->nw_watermark;
399 			snwp->snw_dispatched = nwp->nw_dispatched;
400 			snwp->snw_hybrid_dispatched =
401 			    nwp->nw_hybrid_dispatched;
402 			snwp->snw_qdrops = nwp->nw_qdrops;
403 			snwp->snw_queued = nwp->nw_queued;
404 			snwp->snw_handled = nwp->nw_handled;
405 			counter++;
406 		}
407 	}
408 	work_array_len = counter;
409 }
410 
411 static void
412 netisr_load_sysctl_workstream(void)
413 {
414 	size_t len;
415 
416 	if (sysctlbyname("net.isr.workstream", NULL, &len, NULL, 0) < 0)
417 		xo_err(-1, "net.isr.workstream: query len");
418 	if (len % sizeof(*workstream_array) != 0)
419 		xo_errx(-1, "net.isr.workstream: invalid len");
420 	workstream_array = malloc(len);
421 	if (workstream_array == NULL)
422 		xo_err(-1, "malloc");
423 	if (sysctlbyname("net.isr.workstream", workstream_array, &len, NULL,
424 	    0) < 0)
425 		xo_err(-1, "net.isr.workstream: query data");
426 	if (len % sizeof(*workstream_array) != 0)
427 		xo_errx(-1, "net.isr.workstream: invalid len");
428 	workstream_array_len = len / sizeof(*workstream_array);
429 	if (workstream_array_len < 1)
430 		xo_errx(-1, "net.isr.workstream: no data");
431 	if (workstream_array[0].snws_version != sizeof(workstream_array[0]))
432 		xo_errx(-1, "net.isr.workstream: invalid version");
433 }
434 
435 static void
436 netisr_load_sysctl_work(void)
437 {
438 	size_t len;
439 
440 	if (sysctlbyname("net.isr.work", NULL, &len, NULL, 0) < 0)
441 		xo_err(-1, "net.isr.work: query len");
442 	if (len % sizeof(*work_array) != 0)
443 		xo_errx(-1, "net.isr.work: invalid len");
444 	work_array = malloc(len);
445 	if (work_array == NULL)
446 		xo_err(-1, "malloc");
447 	if (sysctlbyname("net.isr.work", work_array, &len, NULL, 0) < 0)
448 		xo_err(-1, "net.isr.work: query data");
449 	if (len % sizeof(*work_array) != 0)
450 		xo_errx(-1, "net.isr.work: invalid len");
451 	work_array_len = len / sizeof(*work_array);
452 	if (work_array_len < 1)
453 		xo_errx(-1, "net.isr.work: no data");
454 	if (work_array[0].snw_version != sizeof(work_array[0]))
455 		xo_errx(-1, "net.isr.work: invalid version");
456 }
457 
458 static void
459 netisr_print_proto(struct sysctl_netisr_proto *snpp)
460 {
461 	char tmp[20];
462 
463 	xo_emit("{[:-6}{k:name/%s}{]:}", snpp->snp_name);
464 	xo_emit(" {:protocol/%5u}", snpp->snp_proto);
465 	xo_emit(" {:queue-limit/%6u}", snpp->snp_qlimit);
466 	xo_emit(" {:policy-type/%6s}",
467 	    (snpp->snp_policy == NETISR_POLICY_SOURCE) ?  "source" :
468 	    (snpp->snp_policy == NETISR_POLICY_FLOW) ? "flow" :
469 	    (snpp->snp_policy == NETISR_POLICY_CPU) ? "cpu" : "-");
470 	netisr_dispatch_policy_to_string(snpp->snp_dispatch, tmp,
471 	    sizeof(tmp));
472 	xo_emit(" {:policy/%8s}", tmp);
473 	xo_emit("   {:flags/%s%s%s}\n",
474 	    (snpp->snp_flags & NETISR_SNP_FLAGS_M2CPUID) ?  "C" : "-",
475 	    (snpp->snp_flags & NETISR_SNP_FLAGS_DRAINEDCPU) ?  "D" : "-",
476 	    (snpp->snp_flags & NETISR_SNP_FLAGS_M2FLOW) ? "F" : "-");
477 }
478 
479 static void
480 netisr_print_workstream(struct sysctl_netisr_workstream *snwsp)
481 {
482 	struct sysctl_netisr_work *snwp;
483 	u_int i;
484 
485 	xo_open_list("work");
486 	for (i = 0; i < work_array_len; i++) {
487 		snwp = &work_array[i];
488 		if (snwp->snw_wsid != snwsp->snws_wsid)
489 			continue;
490 		xo_open_instance("work");
491 		xo_emit("{t:workstream/%4u} ", snwsp->snws_wsid);
492 		xo_emit("{t:cpu/%3u} ", snwsp->snws_cpu);
493 		xo_emit("{P:  }");
494 		xo_emit("{t:name/%-6s}", netisr_proto2name(snwp->snw_proto));
495 		xo_emit(" {t:length/%5u}", snwp->snw_len);
496 		xo_emit(" {t:watermark/%5u}", snwp->snw_watermark);
497 		xo_emit(" {t:dispatched/%8ju}", snwp->snw_dispatched);
498 		xo_emit(" {t:hybrid-dispatched/%8ju}",
499 		    snwp->snw_hybrid_dispatched);
500 		xo_emit(" {t:queue-drops/%8ju}", snwp->snw_qdrops);
501 		xo_emit(" {t:queued/%8ju}", snwp->snw_queued);
502 		xo_emit(" {t:handled/%8ju}", snwp->snw_handled);
503 		xo_emit("\n");
504 		xo_close_instance("work");
505 	}
506 	xo_close_list("work");
507 }
508 
509 void
510 netisr_stats(void *kvmd)
511 {
512 	struct sysctl_netisr_workstream *snwsp;
513 	struct sysctl_netisr_proto *snpp;
514 	kvm_t *kd = kvmd;
515 	u_int i;
516 
517 	if (live) {
518 		netisr_load_sysctl_config();
519 		netisr_load_sysctl_proto();
520 		netisr_load_sysctl_workstream();
521 		netisr_load_sysctl_work();
522 	} else {
523 		if (kd == NULL)
524 			xo_errx(-1, "netisr_stats: !live but !kd");
525 		netisr_load_kvm_config(kd);
526 		netisr_load_kvm_proto(kd);
527 		netisr_load_kvm_workstream(kd);		/* Also does work. */
528 	}
529 
530 	xo_open_container("netisr");
531 
532 	xo_emit("{T:Configuration}:\n");
533 	xo_emit("{T:/%-25s} {T:/%12s} {T:/%12s}\n",
534 	    "Setting", "Current", "Limit");
535 	xo_emit("{T:/%-25s} {T:/%12u} {T:/%12u}\n",
536 	    "Thread count", numthreads, maxthreads);
537 	xo_emit("{T:/%-25s} {T:/%12u} {T:/%12u}\n",
538 	    "Default queue limit", defaultqlimit, maxqlimit);
539 	xo_emit("{T:/%-25s} {T:/%12s} {T:/%12s}\n",
540 	    "Dispatch policy", dispatch_policy, "n/a");
541 	xo_emit("{T:/%-25s} {T:/%12s} {T:/%12s}\n",
542 	    "Threads bound to CPUs", bindthreads ? "enabled" : "disabled",
543 	    "n/a");
544 	xo_emit("\n");
545 
546 	xo_emit("{T:Protocols}:\n");
547 	xo_emit("{T:/%-6s} {T:/%5s} {T:/%6s} {T:/%-6s} {T:/%-8s} {T:/%-5s}\n",
548 	    "Name", "Proto", "QLimit", "Policy", "Dispatch", "Flags");
549 	xo_open_list("protocol");
550 	for (i = 0; i < proto_array_len; i++) {
551 		xo_open_instance("protocol");
552 		snpp = &proto_array[i];
553 		netisr_print_proto(snpp);
554 		xo_close_instance("protocol");
555 	}
556 	xo_close_list("protocol");
557 	xo_emit("\n");
558 
559 	xo_emit("{T:Workstreams}:\n");
560 	xo_emit("{T:/%4s} {T:/%3s} ", "WSID", "CPU");
561 	xo_emit("{P:/%2s}", "");
562 	xo_emit("{T:/%-6s} {T:/%5s} {T:/%5s} {T:/%8s} {T:/%8s} {T:/%8s} "
563 	    "{T:/%8s} {T:/%8s}\n",
564 	    "Name", "Len", "WMark", "Disp'd", "HDisp'd", "QDrops", "Queued",
565 	    "Handled");
566 	xo_open_list("workstream");
567 	for (i = 0; i < workstream_array_len; i++) {
568 		xo_open_instance("workstream");
569 		snwsp = &workstream_array[i];
570 		netisr_print_workstream(snwsp);
571 		xo_close_instance("workstream");
572 	}
573 	xo_close_list("workstream");
574 	xo_close_container("netisr");
575 }
576