1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
29 * Copyright (c) 2012 by Delphix. All rights reserved.
30 */
31
32 #include <stdlib.h>
33 #include <strings.h>
34 #include <errno.h>
35 #include <unistd.h>
36 #include <dt_impl.h>
37 #include <assert.h>
38 #include <dt_oformat.h>
39 #ifdef illumos
40 #include <alloca.h>
41 #else
42 #include <sys/sysctl.h>
43 #include <libproc_compat.h>
44 #endif
45 #include <limits.h>
46
47 #define DTRACE_AHASHSIZE 32779 /* big 'ol prime */
48
49 /*
50 * Because qsort(3C) does not allow an argument to be passed to a comparison
51 * function, the variables that affect comparison must regrettably be global;
52 * they are protected by a global static lock, dt_qsort_lock.
53 */
54 static pthread_mutex_t dt_qsort_lock = PTHREAD_MUTEX_INITIALIZER;
55
56 static int dt_revsort;
57 static int dt_keysort;
58 static int dt_keypos;
59
60 #define DT_LESSTHAN (dt_revsort == 0 ? -1 : 1)
61 #define DT_GREATERTHAN (dt_revsort == 0 ? 1 : -1)
62
63 static void
dt_aggregate_count(int64_t * existing,int64_t * new,size_t size)64 dt_aggregate_count(int64_t *existing, int64_t *new, size_t size)
65 {
66 uint_t i;
67
68 for (i = 0; i < size / sizeof (int64_t); i++)
69 existing[i] = existing[i] + new[i];
70 }
71
72 static int
dt_aggregate_countcmp(int64_t * lhs,int64_t * rhs)73 dt_aggregate_countcmp(int64_t *lhs, int64_t *rhs)
74 {
75 int64_t lvar = *lhs;
76 int64_t rvar = *rhs;
77
78 if (lvar < rvar)
79 return (DT_LESSTHAN);
80
81 if (lvar > rvar)
82 return (DT_GREATERTHAN);
83
84 return (0);
85 }
86
87 /*ARGSUSED*/
88 static void
dt_aggregate_min(int64_t * existing,int64_t * new,size_t size)89 dt_aggregate_min(int64_t *existing, int64_t *new, size_t size)
90 {
91 if (*new < *existing)
92 *existing = *new;
93 }
94
95 /*ARGSUSED*/
96 static void
dt_aggregate_max(int64_t * existing,int64_t * new,size_t size)97 dt_aggregate_max(int64_t *existing, int64_t *new, size_t size)
98 {
99 if (*new > *existing)
100 *existing = *new;
101 }
102
103 static int
dt_aggregate_averagecmp(int64_t * lhs,int64_t * rhs)104 dt_aggregate_averagecmp(int64_t *lhs, int64_t *rhs)
105 {
106 int64_t lavg = lhs[0] ? (lhs[1] / lhs[0]) : 0;
107 int64_t ravg = rhs[0] ? (rhs[1] / rhs[0]) : 0;
108
109 if (lavg < ravg)
110 return (DT_LESSTHAN);
111
112 if (lavg > ravg)
113 return (DT_GREATERTHAN);
114
115 return (0);
116 }
117
118 static int
dt_aggregate_stddevcmp(int64_t * lhs,int64_t * rhs)119 dt_aggregate_stddevcmp(int64_t *lhs, int64_t *rhs)
120 {
121 uint64_t lsd = dt_stddev((uint64_t *)lhs, 1);
122 uint64_t rsd = dt_stddev((uint64_t *)rhs, 1);
123
124 if (lsd < rsd)
125 return (DT_LESSTHAN);
126
127 if (lsd > rsd)
128 return (DT_GREATERTHAN);
129
130 return (0);
131 }
132
133 /*ARGSUSED*/
134 static void
dt_aggregate_lquantize(int64_t * existing,int64_t * new,size_t size)135 dt_aggregate_lquantize(int64_t *existing, int64_t *new, size_t size)
136 {
137 int64_t arg = *existing++;
138 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
139 int i;
140
141 for (i = 0; i <= levels + 1; i++)
142 existing[i] = existing[i] + new[i + 1];
143 }
144
145 static long double
dt_aggregate_lquantizedsum(int64_t * lquanta)146 dt_aggregate_lquantizedsum(int64_t *lquanta)
147 {
148 int64_t arg = *lquanta++;
149 int32_t base = DTRACE_LQUANTIZE_BASE(arg);
150 uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
151 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
152 long double total = (long double)lquanta[0] * (long double)(base - 1);
153
154 for (i = 0; i < levels; base += step, i++)
155 total += (long double)lquanta[i + 1] * (long double)base;
156
157 return (total + (long double)lquanta[levels + 1] *
158 (long double)(base + 1));
159 }
160
161 static int64_t
dt_aggregate_lquantizedzero(int64_t * lquanta)162 dt_aggregate_lquantizedzero(int64_t *lquanta)
163 {
164 int64_t arg = *lquanta++;
165 int32_t base = DTRACE_LQUANTIZE_BASE(arg);
166 uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
167 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
168
169 if (base - 1 == 0)
170 return (lquanta[0]);
171
172 for (i = 0; i < levels; base += step, i++) {
173 if (base != 0)
174 continue;
175
176 return (lquanta[i + 1]);
177 }
178
179 if (base + 1 == 0)
180 return (lquanta[levels + 1]);
181
182 return (0);
183 }
184
185 static int
dt_aggregate_lquantizedcmp(int64_t * lhs,int64_t * rhs)186 dt_aggregate_lquantizedcmp(int64_t *lhs, int64_t *rhs)
187 {
188 long double lsum = dt_aggregate_lquantizedsum(lhs);
189 long double rsum = dt_aggregate_lquantizedsum(rhs);
190 int64_t lzero, rzero;
191
192 if (lsum < rsum)
193 return (DT_LESSTHAN);
194
195 if (lsum > rsum)
196 return (DT_GREATERTHAN);
197
198 /*
199 * If they're both equal, then we will compare based on the weights at
200 * zero. If the weights at zero are equal (or if zero is not within
201 * the range of the linear quantization), then this will be judged a
202 * tie and will be resolved based on the key comparison.
203 */
204 lzero = dt_aggregate_lquantizedzero(lhs);
205 rzero = dt_aggregate_lquantizedzero(rhs);
206
207 if (lzero < rzero)
208 return (DT_LESSTHAN);
209
210 if (lzero > rzero)
211 return (DT_GREATERTHAN);
212
213 return (0);
214 }
215
216 static void
dt_aggregate_llquantize(int64_t * existing,int64_t * new,size_t size)217 dt_aggregate_llquantize(int64_t *existing, int64_t *new, size_t size)
218 {
219 int i;
220
221 for (i = 1; i < size / sizeof (int64_t); i++)
222 existing[i] = existing[i] + new[i];
223 }
224
225 static long double
dt_aggregate_llquantizedsum(int64_t * llquanta)226 dt_aggregate_llquantizedsum(int64_t *llquanta)
227 {
228 int64_t arg = *llquanta++;
229 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
230 uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
231 uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
232 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
233 int bin = 0, order;
234 int64_t value = 1, next, step;
235 long double total;
236
237 assert(nsteps >= factor);
238 assert(nsteps % factor == 0);
239
240 for (order = 0; order < low; order++)
241 value *= factor;
242
243 total = (long double)llquanta[bin++] * (long double)(value - 1);
244
245 next = value * factor;
246 step = next > nsteps ? next / nsteps : 1;
247
248 while (order <= high) {
249 assert(value < next);
250 total += (long double)llquanta[bin++] * (long double)(value);
251
252 if ((value += step) != next)
253 continue;
254
255 next = value * factor;
256 step = next > nsteps ? next / nsteps : 1;
257 order++;
258 }
259
260 return (total + (long double)llquanta[bin] * (long double)value);
261 }
262
263 static int
dt_aggregate_llquantizedcmp(int64_t * lhs,int64_t * rhs)264 dt_aggregate_llquantizedcmp(int64_t *lhs, int64_t *rhs)
265 {
266 long double lsum = dt_aggregate_llquantizedsum(lhs);
267 long double rsum = dt_aggregate_llquantizedsum(rhs);
268 int64_t lzero, rzero;
269
270 if (lsum < rsum)
271 return (DT_LESSTHAN);
272
273 if (lsum > rsum)
274 return (DT_GREATERTHAN);
275
276 /*
277 * If they're both equal, then we will compare based on the weights at
278 * zero. If the weights at zero are equal, then this will be judged a
279 * tie and will be resolved based on the key comparison.
280 */
281 lzero = lhs[1];
282 rzero = rhs[1];
283
284 if (lzero < rzero)
285 return (DT_LESSTHAN);
286
287 if (lzero > rzero)
288 return (DT_GREATERTHAN);
289
290 return (0);
291 }
292
293 static int
dt_aggregate_quantizedcmp(int64_t * lhs,int64_t * rhs)294 dt_aggregate_quantizedcmp(int64_t *lhs, int64_t *rhs)
295 {
296 int nbuckets = DTRACE_QUANTIZE_NBUCKETS;
297 long double ltotal = 0, rtotal = 0;
298 int64_t lzero, rzero;
299 uint_t i;
300
301 for (i = 0; i < nbuckets; i++) {
302 int64_t bucketval = DTRACE_QUANTIZE_BUCKETVAL(i);
303
304 if (bucketval == 0) {
305 lzero = lhs[i];
306 rzero = rhs[i];
307 }
308
309 ltotal += (long double)bucketval * (long double)lhs[i];
310 rtotal += (long double)bucketval * (long double)rhs[i];
311 }
312
313 if (ltotal < rtotal)
314 return (DT_LESSTHAN);
315
316 if (ltotal > rtotal)
317 return (DT_GREATERTHAN);
318
319 /*
320 * If they're both equal, then we will compare based on the weights at
321 * zero. If the weights at zero are equal, then this will be judged a
322 * tie and will be resolved based on the key comparison.
323 */
324 if (lzero < rzero)
325 return (DT_LESSTHAN);
326
327 if (lzero > rzero)
328 return (DT_GREATERTHAN);
329
330 return (0);
331 }
332
333 static void
dt_aggregate_usym(dtrace_hdl_t * dtp,uint64_t * data)334 dt_aggregate_usym(dtrace_hdl_t *dtp, uint64_t *data)
335 {
336 uint64_t pid = data[0];
337 uint64_t *pc = &data[1];
338 struct ps_prochandle *P;
339 GElf_Sym sym;
340
341 if (dtp->dt_vector != NULL)
342 return;
343
344 if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
345 return;
346
347 dt_proc_lock(dtp, P);
348
349 if (Plookup_by_addr(P, *pc, NULL, 0, &sym) == 0)
350 *pc = sym.st_value;
351
352 dt_proc_unlock(dtp, P);
353 dt_proc_release(dtp, P);
354 }
355
356 static void
dt_aggregate_umod(dtrace_hdl_t * dtp,uint64_t * data)357 dt_aggregate_umod(dtrace_hdl_t *dtp, uint64_t *data)
358 {
359 uint64_t pid = data[0];
360 uint64_t *pc = &data[1];
361 struct ps_prochandle *P;
362 const prmap_t *map;
363
364 if (dtp->dt_vector != NULL)
365 return;
366
367 if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
368 return;
369
370 dt_proc_lock(dtp, P);
371
372 if ((map = Paddr_to_map(P, *pc)) != NULL)
373 *pc = map->pr_vaddr;
374
375 dt_proc_unlock(dtp, P);
376 dt_proc_release(dtp, P);
377 }
378
379 static void
dt_aggregate_sym(dtrace_hdl_t * dtp,uint64_t * data)380 dt_aggregate_sym(dtrace_hdl_t *dtp, uint64_t *data)
381 {
382 GElf_Sym sym;
383 uint64_t *pc = data;
384
385 if (dtrace_lookup_by_addr(dtp, *pc, &sym, NULL) == 0)
386 *pc = sym.st_value;
387 }
388
389 static void
dt_aggregate_mod(dtrace_hdl_t * dtp,uint64_t * data)390 dt_aggregate_mod(dtrace_hdl_t *dtp, uint64_t *data)
391 {
392 uint64_t *pc = data;
393 dt_module_t *dmp;
394
395 if (dtp->dt_vector != NULL) {
396 /*
397 * We don't have a way of just getting the module for a
398 * vectored open, and it doesn't seem to be worth defining
399 * one. This means that use of mod() won't get true
400 * aggregation in the postmortem case (some modules may
401 * appear more than once in aggregation output). It seems
402 * unlikely that anyone will ever notice or care...
403 */
404 return;
405 }
406
407 for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
408 dmp = dt_list_next(dmp)) {
409 if (*pc - dmp->dm_text_va < dmp->dm_text_size) {
410 *pc = dmp->dm_text_va;
411 return;
412 }
413 }
414 }
415
416 static dtrace_aggvarid_t
dt_aggregate_aggvarid(dt_ahashent_t * ent)417 dt_aggregate_aggvarid(dt_ahashent_t *ent)
418 {
419 dtrace_aggdesc_t *agg = ent->dtahe_data.dtada_desc;
420 caddr_t data = ent->dtahe_data.dtada_data;
421 dtrace_recdesc_t *rec = agg->dtagd_rec;
422
423 /*
424 * First, we'll check the variable ID in the aggdesc. If it's valid,
425 * we'll return it. If not, we'll use the compiler-generated ID
426 * present as the first record.
427 */
428 if (agg->dtagd_varid != DTRACE_AGGVARIDNONE)
429 return (agg->dtagd_varid);
430
431 agg->dtagd_varid = *((dtrace_aggvarid_t *)(uintptr_t)(data +
432 rec->dtrd_offset));
433
434 return (agg->dtagd_varid);
435 }
436
437
438 static int
dt_aggregate_snap_cpu(dtrace_hdl_t * dtp,processorid_t cpu)439 dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu)
440 {
441 dtrace_epid_t id;
442 uint64_t hashval;
443 size_t offs, roffs, size, ndx;
444 int i, j, rval;
445 caddr_t addr, data;
446 dtrace_recdesc_t *rec;
447 dt_aggregate_t *agp = &dtp->dt_aggregate;
448 dtrace_aggdesc_t *agg;
449 dt_ahash_t *hash = &agp->dtat_hash;
450 dt_ahashent_t *h;
451 dtrace_bufdesc_t b = agp->dtat_buf, *buf = &b;
452 dtrace_aggdata_t *aggdata;
453 int flags = agp->dtat_flags;
454
455 buf->dtbd_cpu = cpu;
456
457 #ifdef illumos
458 if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, buf) == -1) {
459 #else
460 if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, &buf) == -1) {
461 #endif
462 if (errno == ENOENT) {
463 /*
464 * If that failed with ENOENT, it may be because the
465 * CPU was unconfigured. This is okay; we'll just
466 * do nothing but return success.
467 */
468 return (0);
469 }
470
471 return (dt_set_errno(dtp, errno));
472 }
473
474 if (buf->dtbd_drops != 0) {
475 xo_open_instance("probes");
476 dt_oformat_drop(dtp, cpu);
477 if (dt_handle_cpudrop(dtp, cpu,
478 DTRACEDROP_AGGREGATION, buf->dtbd_drops) == -1) {
479 xo_close_instance("probes");
480 return (-1);
481 }
482 xo_close_instance("probes");
483 }
484
485 if (buf->dtbd_size == 0)
486 return (0);
487
488 if (hash->dtah_hash == NULL) {
489 size_t size;
490
491 hash->dtah_size = DTRACE_AHASHSIZE;
492 size = hash->dtah_size * sizeof (dt_ahashent_t *);
493
494 if ((hash->dtah_hash = malloc(size)) == NULL)
495 return (dt_set_errno(dtp, EDT_NOMEM));
496
497 bzero(hash->dtah_hash, size);
498 }
499
500 for (offs = 0; offs < buf->dtbd_size; ) {
501 /*
502 * We're guaranteed to have an ID.
503 */
504 id = *((dtrace_epid_t *)((uintptr_t)buf->dtbd_data +
505 (uintptr_t)offs));
506
507 if (id == DTRACE_AGGIDNONE) {
508 /*
509 * This is filler to assure proper alignment of the
510 * next record; we simply ignore it.
511 */
512 offs += sizeof (id);
513 continue;
514 }
515
516 if ((rval = dt_aggid_lookup(dtp, id, &agg)) != 0)
517 return (rval);
518
519 addr = buf->dtbd_data + offs;
520 size = agg->dtagd_size;
521 hashval = 0;
522
523 for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
524 rec = &agg->dtagd_rec[j];
525 roffs = rec->dtrd_offset;
526
527 switch (rec->dtrd_action) {
528 case DTRACEACT_USYM:
529 dt_aggregate_usym(dtp,
530 /* LINTED - alignment */
531 (uint64_t *)&addr[roffs]);
532 break;
533
534 case DTRACEACT_UMOD:
535 dt_aggregate_umod(dtp,
536 /* LINTED - alignment */
537 (uint64_t *)&addr[roffs]);
538 break;
539
540 case DTRACEACT_SYM:
541 /* LINTED - alignment */
542 dt_aggregate_sym(dtp, (uint64_t *)&addr[roffs]);
543 break;
544
545 case DTRACEACT_MOD:
546 /* LINTED - alignment */
547 dt_aggregate_mod(dtp, (uint64_t *)&addr[roffs]);
548 break;
549
550 default:
551 break;
552 }
553
554 for (i = 0; i < rec->dtrd_size; i++)
555 hashval += addr[roffs + i];
556 }
557
558 ndx = hashval % hash->dtah_size;
559
560 for (h = hash->dtah_hash[ndx]; h != NULL; h = h->dtahe_next) {
561 if (h->dtahe_hashval != hashval)
562 continue;
563
564 if (h->dtahe_size != size)
565 continue;
566
567 aggdata = &h->dtahe_data;
568 data = aggdata->dtada_data;
569
570 for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
571 rec = &agg->dtagd_rec[j];
572 roffs = rec->dtrd_offset;
573
574 for (i = 0; i < rec->dtrd_size; i++)
575 if (addr[roffs + i] != data[roffs + i])
576 goto hashnext;
577 }
578
579 /*
580 * We found it. Now we need to apply the aggregating
581 * action on the data here.
582 */
583 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
584 roffs = rec->dtrd_offset;
585 /* LINTED - alignment */
586 h->dtahe_aggregate((int64_t *)&data[roffs],
587 /* LINTED - alignment */
588 (int64_t *)&addr[roffs], rec->dtrd_size);
589
590 /*
591 * If we're keeping per CPU data, apply the aggregating
592 * action there as well.
593 */
594 if (aggdata->dtada_percpu != NULL) {
595 data = aggdata->dtada_percpu[cpu];
596
597 /* LINTED - alignment */
598 h->dtahe_aggregate((int64_t *)data,
599 /* LINTED - alignment */
600 (int64_t *)&addr[roffs], rec->dtrd_size);
601 }
602
603 goto bufnext;
604 hashnext:
605 continue;
606 }
607
608 /*
609 * If we're here, we couldn't find an entry for this record.
610 */
611 if ((h = malloc(sizeof (dt_ahashent_t))) == NULL)
612 return (dt_set_errno(dtp, EDT_NOMEM));
613 bzero(h, sizeof (dt_ahashent_t));
614 aggdata = &h->dtahe_data;
615
616 if ((aggdata->dtada_data = malloc(size)) == NULL) {
617 free(h);
618 return (dt_set_errno(dtp, EDT_NOMEM));
619 }
620
621 bcopy(addr, aggdata->dtada_data, size);
622 aggdata->dtada_size = size;
623 aggdata->dtada_desc = agg;
624 aggdata->dtada_handle = dtp;
625 (void) dt_epid_lookup(dtp, agg->dtagd_epid,
626 &aggdata->dtada_edesc, &aggdata->dtada_pdesc);
627 aggdata->dtada_normal = 1;
628
629 h->dtahe_hashval = hashval;
630 h->dtahe_size = size;
631 (void) dt_aggregate_aggvarid(h);
632
633 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
634
635 if (flags & DTRACE_A_PERCPU) {
636 int max_cpus = agp->dtat_maxcpu;
637 caddr_t *percpu = malloc(max_cpus * sizeof (caddr_t));
638
639 if (percpu == NULL) {
640 free(aggdata->dtada_data);
641 free(h);
642 return (dt_set_errno(dtp, EDT_NOMEM));
643 }
644
645 for (j = 0; j < max_cpus; j++) {
646 percpu[j] = malloc(rec->dtrd_size);
647
648 if (percpu[j] == NULL) {
649 while (--j >= 0)
650 free(percpu[j]);
651
652 free(aggdata->dtada_data);
653 free(h);
654 return (dt_set_errno(dtp, EDT_NOMEM));
655 }
656
657 if (j == cpu) {
658 bcopy(&addr[rec->dtrd_offset],
659 percpu[j], rec->dtrd_size);
660 } else {
661 bzero(percpu[j], rec->dtrd_size);
662 }
663 }
664
665 aggdata->dtada_percpu = percpu;
666 }
667
668 switch (rec->dtrd_action) {
669 case DTRACEAGG_MIN:
670 h->dtahe_aggregate = dt_aggregate_min;
671 break;
672
673 case DTRACEAGG_MAX:
674 h->dtahe_aggregate = dt_aggregate_max;
675 break;
676
677 case DTRACEAGG_LQUANTIZE:
678 h->dtahe_aggregate = dt_aggregate_lquantize;
679 break;
680
681 case DTRACEAGG_LLQUANTIZE:
682 h->dtahe_aggregate = dt_aggregate_llquantize;
683 break;
684
685 case DTRACEAGG_COUNT:
686 case DTRACEAGG_SUM:
687 case DTRACEAGG_AVG:
688 case DTRACEAGG_STDDEV:
689 case DTRACEAGG_QUANTIZE:
690 h->dtahe_aggregate = dt_aggregate_count;
691 break;
692
693 default:
694 return (dt_set_errno(dtp, EDT_BADAGG));
695 }
696
697 if (hash->dtah_hash[ndx] != NULL)
698 hash->dtah_hash[ndx]->dtahe_prev = h;
699
700 h->dtahe_next = hash->dtah_hash[ndx];
701 hash->dtah_hash[ndx] = h;
702
703 if (hash->dtah_all != NULL)
704 hash->dtah_all->dtahe_prevall = h;
705
706 h->dtahe_nextall = hash->dtah_all;
707 hash->dtah_all = h;
708 bufnext:
709 offs += agg->dtagd_size;
710 }
711
712 return (0);
713 }
714
715 int
716 dtrace_aggregate_snap(dtrace_hdl_t *dtp)
717 {
718 int i, rval;
719 dt_aggregate_t *agp = &dtp->dt_aggregate;
720 hrtime_t now = gethrtime();
721 dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_AGGRATE];
722
723 if (dtp->dt_lastagg != 0) {
724 if (now - dtp->dt_lastagg < interval)
725 return (0);
726
727 dtp->dt_lastagg += interval;
728 } else {
729 dtp->dt_lastagg = now;
730 }
731
732 if (!dtp->dt_active)
733 return (dt_set_errno(dtp, EINVAL));
734
735 if (agp->dtat_buf.dtbd_size == 0)
736 return (0);
737
738 for (i = 0; i < agp->dtat_ncpus; i++) {
739 if ((rval = dt_aggregate_snap_cpu(dtp, agp->dtat_cpus[i])))
740 return (rval);
741 }
742
743 return (0);
744 }
745
746 static int
747 dt_aggregate_hashcmp(const void *lhs, const void *rhs)
748 {
749 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
750 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
751 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
752 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
753
754 if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
755 return (DT_LESSTHAN);
756
757 if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
758 return (DT_GREATERTHAN);
759
760 return (0);
761 }
762
763 static int
764 dt_aggregate_varcmp(const void *lhs, const void *rhs)
765 {
766 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
767 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
768 dtrace_aggvarid_t lid, rid;
769
770 lid = dt_aggregate_aggvarid(lh);
771 rid = dt_aggregate_aggvarid(rh);
772
773 if (lid < rid)
774 return (DT_LESSTHAN);
775
776 if (lid > rid)
777 return (DT_GREATERTHAN);
778
779 return (0);
780 }
781
782 static int
783 dt_aggregate_keycmp(const void *lhs, const void *rhs)
784 {
785 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
786 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
787 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
788 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
789 dtrace_recdesc_t *lrec, *rrec;
790 char *ldata, *rdata;
791 int rval, i, j, keypos, nrecs;
792
793 if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
794 return (rval);
795
796 nrecs = lagg->dtagd_nrecs - 1;
797 assert(nrecs == ragg->dtagd_nrecs - 1);
798
799 keypos = dt_keypos + 1 >= nrecs ? 0 : dt_keypos;
800
801 for (i = 1; i < nrecs; i++) {
802 uint64_t lval, rval;
803 int ndx = i + keypos;
804
805 if (ndx >= nrecs)
806 ndx = ndx - nrecs + 1;
807
808 lrec = &lagg->dtagd_rec[ndx];
809 rrec = &ragg->dtagd_rec[ndx];
810
811 ldata = lh->dtahe_data.dtada_data + lrec->dtrd_offset;
812 rdata = rh->dtahe_data.dtada_data + rrec->dtrd_offset;
813
814 if (lrec->dtrd_size < rrec->dtrd_size)
815 return (DT_LESSTHAN);
816
817 if (lrec->dtrd_size > rrec->dtrd_size)
818 return (DT_GREATERTHAN);
819
820 switch (lrec->dtrd_size) {
821 case sizeof (uint64_t):
822 /* LINTED - alignment */
823 lval = *((uint64_t *)ldata);
824 /* LINTED - alignment */
825 rval = *((uint64_t *)rdata);
826 break;
827
828 case sizeof (uint32_t):
829 /* LINTED - alignment */
830 lval = *((uint32_t *)ldata);
831 /* LINTED - alignment */
832 rval = *((uint32_t *)rdata);
833 break;
834
835 case sizeof (uint16_t):
836 /* LINTED - alignment */
837 lval = *((uint16_t *)ldata);
838 /* LINTED - alignment */
839 rval = *((uint16_t *)rdata);
840 break;
841
842 case sizeof (uint8_t):
843 lval = *((uint8_t *)ldata);
844 rval = *((uint8_t *)rdata);
845 break;
846
847 default:
848 switch (lrec->dtrd_action) {
849 case DTRACEACT_UMOD:
850 case DTRACEACT_UADDR:
851 case DTRACEACT_USYM:
852 for (j = 0; j < 2; j++) {
853 /* LINTED - alignment */
854 lval = ((uint64_t *)ldata)[j];
855 /* LINTED - alignment */
856 rval = ((uint64_t *)rdata)[j];
857
858 if (lval < rval)
859 return (DT_LESSTHAN);
860
861 if (lval > rval)
862 return (DT_GREATERTHAN);
863 }
864
865 break;
866
867 default:
868 for (j = 0; j < lrec->dtrd_size; j++) {
869 lval = ((uint8_t *)ldata)[j];
870 rval = ((uint8_t *)rdata)[j];
871
872 if (lval < rval)
873 return (DT_LESSTHAN);
874
875 if (lval > rval)
876 return (DT_GREATERTHAN);
877 }
878 }
879
880 continue;
881 }
882
883 if (lval < rval)
884 return (DT_LESSTHAN);
885
886 if (lval > rval)
887 return (DT_GREATERTHAN);
888 }
889
890 return (0);
891 }
892
893 static int
894 dt_aggregate_valcmp(const void *lhs, const void *rhs)
895 {
896 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
897 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
898 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
899 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
900 caddr_t ldata = lh->dtahe_data.dtada_data;
901 caddr_t rdata = rh->dtahe_data.dtada_data;
902 dtrace_recdesc_t *lrec, *rrec;
903 int64_t *laddr, *raddr;
904 int rval;
905
906 assert(lagg->dtagd_nrecs == ragg->dtagd_nrecs);
907
908 lrec = &lagg->dtagd_rec[lagg->dtagd_nrecs - 1];
909 rrec = &ragg->dtagd_rec[ragg->dtagd_nrecs - 1];
910
911 assert(lrec->dtrd_action == rrec->dtrd_action);
912
913 laddr = (int64_t *)(uintptr_t)(ldata + lrec->dtrd_offset);
914 raddr = (int64_t *)(uintptr_t)(rdata + rrec->dtrd_offset);
915
916 switch (lrec->dtrd_action) {
917 case DTRACEAGG_AVG:
918 rval = dt_aggregate_averagecmp(laddr, raddr);
919 break;
920
921 case DTRACEAGG_STDDEV:
922 rval = dt_aggregate_stddevcmp(laddr, raddr);
923 break;
924
925 case DTRACEAGG_QUANTIZE:
926 rval = dt_aggregate_quantizedcmp(laddr, raddr);
927 break;
928
929 case DTRACEAGG_LQUANTIZE:
930 rval = dt_aggregate_lquantizedcmp(laddr, raddr);
931 break;
932
933 case DTRACEAGG_LLQUANTIZE:
934 rval = dt_aggregate_llquantizedcmp(laddr, raddr);
935 break;
936
937 case DTRACEAGG_COUNT:
938 case DTRACEAGG_SUM:
939 case DTRACEAGG_MIN:
940 case DTRACEAGG_MAX:
941 rval = dt_aggregate_countcmp(laddr, raddr);
942 break;
943
944 default:
945 assert(0);
946 }
947
948 return (rval);
949 }
950
951 static int
952 dt_aggregate_valkeycmp(const void *lhs, const void *rhs)
953 {
954 int rval;
955
956 if ((rval = dt_aggregate_valcmp(lhs, rhs)) != 0)
957 return (rval);
958
959 /*
960 * If we're here, the values for the two aggregation elements are
961 * equal. We already know that the key layout is the same for the two
962 * elements; we must now compare the keys themselves as a tie-breaker.
963 */
964 return (dt_aggregate_keycmp(lhs, rhs));
965 }
966
967 static int
968 dt_aggregate_keyvarcmp(const void *lhs, const void *rhs)
969 {
970 int rval;
971
972 if ((rval = dt_aggregate_keycmp(lhs, rhs)) != 0)
973 return (rval);
974
975 return (dt_aggregate_varcmp(lhs, rhs));
976 }
977
978 static int
979 dt_aggregate_varkeycmp(const void *lhs, const void *rhs)
980 {
981 int rval;
982
983 if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
984 return (rval);
985
986 return (dt_aggregate_keycmp(lhs, rhs));
987 }
988
989 static int
990 dt_aggregate_valvarcmp(const void *lhs, const void *rhs)
991 {
992 int rval;
993
994 if ((rval = dt_aggregate_valkeycmp(lhs, rhs)) != 0)
995 return (rval);
996
997 return (dt_aggregate_varcmp(lhs, rhs));
998 }
999
1000 static int
1001 dt_aggregate_varvalcmp(const void *lhs, const void *rhs)
1002 {
1003 int rval;
1004
1005 if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
1006 return (rval);
1007
1008 return (dt_aggregate_valkeycmp(lhs, rhs));
1009 }
1010
1011 static int
1012 dt_aggregate_keyvarrevcmp(const void *lhs, const void *rhs)
1013 {
1014 return (dt_aggregate_keyvarcmp(rhs, lhs));
1015 }
1016
1017 static int
1018 dt_aggregate_varkeyrevcmp(const void *lhs, const void *rhs)
1019 {
1020 return (dt_aggregate_varkeycmp(rhs, lhs));
1021 }
1022
1023 static int
1024 dt_aggregate_valvarrevcmp(const void *lhs, const void *rhs)
1025 {
1026 return (dt_aggregate_valvarcmp(rhs, lhs));
1027 }
1028
1029 static int
1030 dt_aggregate_varvalrevcmp(const void *lhs, const void *rhs)
1031 {
1032 return (dt_aggregate_varvalcmp(rhs, lhs));
1033 }
1034
1035 static int
1036 dt_aggregate_bundlecmp(const void *lhs, const void *rhs)
1037 {
1038 dt_ahashent_t **lh = *((dt_ahashent_t ***)lhs);
1039 dt_ahashent_t **rh = *((dt_ahashent_t ***)rhs);
1040 int i, rval;
1041
1042 if (dt_keysort) {
1043 /*
1044 * If we're sorting on keys, we need to scan until we find the
1045 * last entry -- that's the representative key. (The order of
1046 * the bundle is values followed by key to accommodate the
1047 * default behavior of sorting by value.) If the keys are
1048 * equal, we'll fall into the value comparison loop, below.
1049 */
1050 for (i = 0; lh[i + 1] != NULL; i++)
1051 continue;
1052
1053 assert(i != 0);
1054 assert(rh[i + 1] == NULL);
1055
1056 if ((rval = dt_aggregate_keycmp(&lh[i], &rh[i])) != 0)
1057 return (rval);
1058 }
1059
1060 for (i = 0; ; i++) {
1061 if (lh[i + 1] == NULL) {
1062 /*
1063 * All of the values are equal; if we're sorting on
1064 * keys, then we're only here because the keys were
1065 * found to be equal and these records are therefore
1066 * equal. If we're not sorting on keys, we'll use the
1067 * key comparison from the representative key as the
1068 * tie-breaker.
1069 */
1070 if (dt_keysort)
1071 return (0);
1072
1073 assert(i != 0);
1074 assert(rh[i + 1] == NULL);
1075 return (dt_aggregate_keycmp(&lh[i], &rh[i]));
1076 } else {
1077 if ((rval = dt_aggregate_valcmp(&lh[i], &rh[i])) != 0)
1078 return (rval);
1079 }
1080 }
1081 }
1082
1083 int
1084 dt_aggregate_go(dtrace_hdl_t *dtp)
1085 {
1086 dt_aggregate_t *agp = &dtp->dt_aggregate;
1087 dtrace_optval_t size, cpu;
1088 dtrace_bufdesc_t *buf = &agp->dtat_buf;
1089 int rval, i;
1090
1091 assert(agp->dtat_maxcpu == 0);
1092 assert(agp->dtat_ncpu == 0);
1093 assert(agp->dtat_cpus == NULL);
1094
1095 agp->dtat_maxcpu = dt_cpu_maxid(dtp) + 1;
1096 if (agp->dtat_maxcpu <= 0)
1097 return (-1);
1098 agp->dtat_ncpu = dt_sysconf(dtp, _SC_NPROCESSORS_CONF);
1099 agp->dtat_cpus = malloc(agp->dtat_ncpu * sizeof (processorid_t));
1100
1101 if (agp->dtat_cpus == NULL)
1102 return (dt_set_errno(dtp, EDT_NOMEM));
1103
1104 /*
1105 * Use the aggregation buffer size as reloaded from the kernel.
1106 */
1107 size = dtp->dt_options[DTRACEOPT_AGGSIZE];
1108
1109 rval = dtrace_getopt(dtp, "aggsize", &size);
1110 assert(rval == 0);
1111
1112 if (size == 0 || size == DTRACEOPT_UNSET)
1113 return (0);
1114
1115 buf = &agp->dtat_buf;
1116 buf->dtbd_size = size;
1117
1118 if ((buf->dtbd_data = malloc(buf->dtbd_size)) == NULL)
1119 return (dt_set_errno(dtp, EDT_NOMEM));
1120
1121 /*
1122 * Now query for the CPUs enabled.
1123 */
1124 rval = dtrace_getopt(dtp, "cpu", &cpu);
1125 assert(rval == 0 && cpu != DTRACEOPT_UNSET);
1126
1127 if (cpu != DTRACE_CPUALL) {
1128 assert(cpu < agp->dtat_ncpu);
1129 agp->dtat_cpus[agp->dtat_ncpus++] = (processorid_t)cpu;
1130
1131 return (0);
1132 }
1133
1134 agp->dtat_ncpus = 0;
1135 for (i = 0; i < agp->dtat_maxcpu; i++) {
1136 if (dt_status(dtp, i) == -1)
1137 continue;
1138
1139 agp->dtat_cpus[agp->dtat_ncpus++] = i;
1140 }
1141
1142 return (0);
1143 }
1144
1145 static int
1146 dt_aggwalk_rval(dtrace_hdl_t *dtp, dt_ahashent_t *h, int rval)
1147 {
1148 dt_aggregate_t *agp = &dtp->dt_aggregate;
1149 dtrace_aggdata_t *data;
1150 dtrace_aggdesc_t *aggdesc;
1151 dtrace_recdesc_t *rec;
1152 int i;
1153
1154 switch (rval) {
1155 case DTRACE_AGGWALK_NEXT:
1156 break;
1157
1158 case DTRACE_AGGWALK_CLEAR: {
1159 uint32_t size, offs = 0;
1160
1161 aggdesc = h->dtahe_data.dtada_desc;
1162 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
1163 size = rec->dtrd_size;
1164 data = &h->dtahe_data;
1165
1166 if (rec->dtrd_action == DTRACEAGG_LQUANTIZE) {
1167 offs = sizeof (uint64_t);
1168 size -= sizeof (uint64_t);
1169 }
1170
1171 bzero(&data->dtada_data[rec->dtrd_offset] + offs, size);
1172
1173 if (data->dtada_percpu == NULL)
1174 break;
1175
1176 for (i = 0; i < dtp->dt_aggregate.dtat_maxcpu; i++)
1177 bzero(data->dtada_percpu[i] + offs, size);
1178 break;
1179 }
1180
1181 case DTRACE_AGGWALK_ERROR:
1182 /*
1183 * We assume that errno is already set in this case.
1184 */
1185 return (dt_set_errno(dtp, errno));
1186
1187 case DTRACE_AGGWALK_ABORT:
1188 return (dt_set_errno(dtp, EDT_DIRABORT));
1189
1190 case DTRACE_AGGWALK_DENORMALIZE:
1191 h->dtahe_data.dtada_normal = 1;
1192 return (0);
1193
1194 case DTRACE_AGGWALK_NORMALIZE:
1195 if (h->dtahe_data.dtada_normal == 0) {
1196 h->dtahe_data.dtada_normal = 1;
1197 return (dt_set_errno(dtp, EDT_BADRVAL));
1198 }
1199
1200 return (0);
1201
1202 case DTRACE_AGGWALK_REMOVE: {
1203 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1204 int max_cpus = agp->dtat_maxcpu;
1205
1206 /*
1207 * First, remove this hash entry from its hash chain.
1208 */
1209 if (h->dtahe_prev != NULL) {
1210 h->dtahe_prev->dtahe_next = h->dtahe_next;
1211 } else {
1212 dt_ahash_t *hash = &agp->dtat_hash;
1213 size_t ndx = h->dtahe_hashval % hash->dtah_size;
1214
1215 assert(hash->dtah_hash[ndx] == h);
1216 hash->dtah_hash[ndx] = h->dtahe_next;
1217 }
1218
1219 if (h->dtahe_next != NULL)
1220 h->dtahe_next->dtahe_prev = h->dtahe_prev;
1221
1222 /*
1223 * Now remove it from the list of all hash entries.
1224 */
1225 if (h->dtahe_prevall != NULL) {
1226 h->dtahe_prevall->dtahe_nextall = h->dtahe_nextall;
1227 } else {
1228 dt_ahash_t *hash = &agp->dtat_hash;
1229
1230 assert(hash->dtah_all == h);
1231 hash->dtah_all = h->dtahe_nextall;
1232 }
1233
1234 if (h->dtahe_nextall != NULL)
1235 h->dtahe_nextall->dtahe_prevall = h->dtahe_prevall;
1236
1237 /*
1238 * We're unlinked. We can safely destroy the data.
1239 */
1240 if (aggdata->dtada_percpu != NULL) {
1241 for (i = 0; i < max_cpus; i++)
1242 free(aggdata->dtada_percpu[i]);
1243 free(aggdata->dtada_percpu);
1244 }
1245
1246 free(aggdata->dtada_data);
1247 free(h);
1248
1249 return (0);
1250 }
1251
1252 default:
1253 return (dt_set_errno(dtp, EDT_BADRVAL));
1254 }
1255
1256 return (0);
1257 }
1258
1259 void
1260 dt_aggregate_qsort(dtrace_hdl_t *dtp, void *base, size_t nel, size_t width,
1261 int (*compar)(const void *, const void *))
1262 {
1263 int rev = dt_revsort, key = dt_keysort, keypos = dt_keypos;
1264 dtrace_optval_t keyposopt = dtp->dt_options[DTRACEOPT_AGGSORTKEYPOS];
1265
1266 dt_revsort = (dtp->dt_options[DTRACEOPT_AGGSORTREV] != DTRACEOPT_UNSET);
1267 dt_keysort = (dtp->dt_options[DTRACEOPT_AGGSORTKEY] != DTRACEOPT_UNSET);
1268
1269 if (keyposopt != DTRACEOPT_UNSET && keyposopt <= INT_MAX) {
1270 dt_keypos = (int)keyposopt;
1271 } else {
1272 dt_keypos = 0;
1273 }
1274
1275 if (compar == NULL) {
1276 if (!dt_keysort) {
1277 compar = dt_aggregate_varvalcmp;
1278 } else {
1279 compar = dt_aggregate_varkeycmp;
1280 }
1281 }
1282
1283 qsort(base, nel, width, compar);
1284
1285 dt_revsort = rev;
1286 dt_keysort = key;
1287 dt_keypos = keypos;
1288 }
1289
1290 int
1291 dtrace_aggregate_walk(dtrace_hdl_t *dtp, dtrace_aggregate_f *func, void *arg)
1292 {
1293 dt_ahashent_t *h, *next;
1294 dt_ahash_t *hash = &dtp->dt_aggregate.dtat_hash;
1295
1296 for (h = hash->dtah_all; h != NULL; h = next) {
1297 /*
1298 * dt_aggwalk_rval() can potentially remove the current hash
1299 * entry; we need to load the next hash entry before calling
1300 * into it.
1301 */
1302 next = h->dtahe_nextall;
1303
1304 if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
1305 return (-1);
1306 }
1307
1308 return (0);
1309 }
1310
1311 static int
1312 dt_aggregate_total(dtrace_hdl_t *dtp, boolean_t clear)
1313 {
1314 dt_ahashent_t *h;
1315 dtrace_aggdata_t **total;
1316 dtrace_aggid_t max = DTRACE_AGGVARIDNONE, id;
1317 dt_aggregate_t *agp = &dtp->dt_aggregate;
1318 dt_ahash_t *hash = &agp->dtat_hash;
1319 uint32_t tflags;
1320
1321 tflags = DTRACE_A_TOTAL | DTRACE_A_HASNEGATIVES | DTRACE_A_HASPOSITIVES;
1322
1323 /*
1324 * If we need to deliver per-aggregation totals, we're going to take
1325 * three passes over the aggregate: one to clear everything out and
1326 * determine our maximum aggregation ID, one to actually total
1327 * everything up, and a final pass to assign the totals to the
1328 * individual elements.
1329 */
1330 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1331 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1332
1333 if ((id = dt_aggregate_aggvarid(h)) > max)
1334 max = id;
1335
1336 aggdata->dtada_total = 0;
1337 aggdata->dtada_flags &= ~tflags;
1338 }
1339
1340 if (clear || max == DTRACE_AGGVARIDNONE)
1341 return (0);
1342
1343 total = dt_zalloc(dtp, (max + 1) * sizeof (dtrace_aggdata_t *));
1344
1345 if (total == NULL)
1346 return (-1);
1347
1348 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1349 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1350 dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1351 dtrace_recdesc_t *rec;
1352 caddr_t data;
1353 int64_t val, *addr;
1354
1355 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
1356 data = aggdata->dtada_data;
1357 addr = (int64_t *)(uintptr_t)(data + rec->dtrd_offset);
1358
1359 switch (rec->dtrd_action) {
1360 case DTRACEAGG_STDDEV:
1361 val = dt_stddev((uint64_t *)addr, 1);
1362 break;
1363
1364 case DTRACEAGG_SUM:
1365 case DTRACEAGG_COUNT:
1366 val = *addr;
1367 break;
1368
1369 case DTRACEAGG_AVG:
1370 val = addr[0] ? (addr[1] / addr[0]) : 0;
1371 break;
1372
1373 default:
1374 continue;
1375 }
1376
1377 if (total[agg->dtagd_varid] == NULL) {
1378 total[agg->dtagd_varid] = aggdata;
1379 aggdata->dtada_flags |= DTRACE_A_TOTAL;
1380 } else {
1381 aggdata = total[agg->dtagd_varid];
1382 }
1383
1384 if (val > 0)
1385 aggdata->dtada_flags |= DTRACE_A_HASPOSITIVES;
1386
1387 if (val < 0) {
1388 aggdata->dtada_flags |= DTRACE_A_HASNEGATIVES;
1389 val = -val;
1390 }
1391
1392 if (dtp->dt_options[DTRACEOPT_AGGZOOM] != DTRACEOPT_UNSET) {
1393 val = (int64_t)((long double)val *
1394 (1 / DTRACE_AGGZOOM_MAX));
1395
1396 if (val > aggdata->dtada_total)
1397 aggdata->dtada_total = val;
1398 } else {
1399 aggdata->dtada_total += val;
1400 }
1401 }
1402
1403 /*
1404 * And now one final pass to set everyone's total.
1405 */
1406 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1407 dtrace_aggdata_t *aggdata = &h->dtahe_data, *t;
1408 dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1409
1410 if ((t = total[agg->dtagd_varid]) == NULL || aggdata == t)
1411 continue;
1412
1413 aggdata->dtada_total = t->dtada_total;
1414 aggdata->dtada_flags |= (t->dtada_flags & tflags);
1415 }
1416
1417 dt_free(dtp, total);
1418
1419 return (0);
1420 }
1421
1422 static int
1423 dt_aggregate_minmaxbin(dtrace_hdl_t *dtp, boolean_t clear)
1424 {
1425 dt_ahashent_t *h;
1426 dtrace_aggdata_t **minmax;
1427 dtrace_aggid_t max = DTRACE_AGGVARIDNONE, id;
1428 dt_aggregate_t *agp = &dtp->dt_aggregate;
1429 dt_ahash_t *hash = &agp->dtat_hash;
1430
1431 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1432 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1433
1434 if ((id = dt_aggregate_aggvarid(h)) > max)
1435 max = id;
1436
1437 aggdata->dtada_minbin = 0;
1438 aggdata->dtada_maxbin = 0;
1439 aggdata->dtada_flags &= ~DTRACE_A_MINMAXBIN;
1440 }
1441
1442 if (clear || max == DTRACE_AGGVARIDNONE)
1443 return (0);
1444
1445 minmax = dt_zalloc(dtp, (max + 1) * sizeof (dtrace_aggdata_t *));
1446
1447 if (minmax == NULL)
1448 return (-1);
1449
1450 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1451 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1452 dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1453 dtrace_recdesc_t *rec;
1454 caddr_t data;
1455 int64_t *addr;
1456 int minbin = -1, maxbin = -1, i;
1457 int start = 0, size;
1458
1459 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
1460 size = rec->dtrd_size / sizeof (int64_t);
1461 data = aggdata->dtada_data;
1462 addr = (int64_t *)(uintptr_t)(data + rec->dtrd_offset);
1463
1464 switch (rec->dtrd_action) {
1465 case DTRACEAGG_LQUANTIZE:
1466 /*
1467 * For lquantize(), we always display the entire range
1468 * of the aggregation when aggpack is set.
1469 */
1470 start = 1;
1471 minbin = start;
1472 maxbin = size - 1 - start;
1473 break;
1474
1475 case DTRACEAGG_QUANTIZE:
1476 for (i = start; i < size; i++) {
1477 if (!addr[i])
1478 continue;
1479
1480 if (minbin == -1)
1481 minbin = i - start;
1482
1483 maxbin = i - start;
1484 }
1485
1486 if (minbin == -1) {
1487 /*
1488 * If we have no data (e.g., due to a clear()
1489 * or negative increments), we'll use the
1490 * zero bucket as both our min and max.
1491 */
1492 minbin = maxbin = DTRACE_QUANTIZE_ZEROBUCKET;
1493 }
1494
1495 break;
1496
1497 default:
1498 continue;
1499 }
1500
1501 if (minmax[agg->dtagd_varid] == NULL) {
1502 minmax[agg->dtagd_varid] = aggdata;
1503 aggdata->dtada_flags |= DTRACE_A_MINMAXBIN;
1504 aggdata->dtada_minbin = minbin;
1505 aggdata->dtada_maxbin = maxbin;
1506 continue;
1507 }
1508
1509 if (minbin < minmax[agg->dtagd_varid]->dtada_minbin)
1510 minmax[agg->dtagd_varid]->dtada_minbin = minbin;
1511
1512 if (maxbin > minmax[agg->dtagd_varid]->dtada_maxbin)
1513 minmax[agg->dtagd_varid]->dtada_maxbin = maxbin;
1514 }
1515
1516 /*
1517 * And now one final pass to set everyone's minbin and maxbin.
1518 */
1519 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1520 dtrace_aggdata_t *aggdata = &h->dtahe_data, *mm;
1521 dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1522
1523 if ((mm = minmax[agg->dtagd_varid]) == NULL || aggdata == mm)
1524 continue;
1525
1526 aggdata->dtada_minbin = mm->dtada_minbin;
1527 aggdata->dtada_maxbin = mm->dtada_maxbin;
1528 aggdata->dtada_flags |= DTRACE_A_MINMAXBIN;
1529 }
1530
1531 dt_free(dtp, minmax);
1532
1533 return (0);
1534 }
1535
1536 static int
1537 dt_aggregate_walk_sorted(dtrace_hdl_t *dtp,
1538 dtrace_aggregate_f *func, void *arg,
1539 int (*sfunc)(const void *, const void *))
1540 {
1541 dt_aggregate_t *agp = &dtp->dt_aggregate;
1542 dt_ahashent_t *h, **sorted;
1543 dt_ahash_t *hash = &agp->dtat_hash;
1544 size_t i, nentries = 0;
1545 int rval = -1;
1546
1547 agp->dtat_flags &= ~(DTRACE_A_TOTAL | DTRACE_A_MINMAXBIN);
1548
1549 if (dtp->dt_options[DTRACEOPT_AGGHIST] != DTRACEOPT_UNSET) {
1550 agp->dtat_flags |= DTRACE_A_TOTAL;
1551
1552 if (dt_aggregate_total(dtp, B_FALSE) != 0)
1553 return (-1);
1554 }
1555
1556 if (dtp->dt_options[DTRACEOPT_AGGPACK] != DTRACEOPT_UNSET) {
1557 agp->dtat_flags |= DTRACE_A_MINMAXBIN;
1558
1559 if (dt_aggregate_minmaxbin(dtp, B_FALSE) != 0)
1560 return (-1);
1561 }
1562
1563 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall)
1564 nentries++;
1565
1566 sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
1567
1568 if (sorted == NULL)
1569 goto out;
1570
1571 for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall)
1572 sorted[i++] = h;
1573
1574 (void) pthread_mutex_lock(&dt_qsort_lock);
1575
1576 if (sfunc == NULL) {
1577 dt_aggregate_qsort(dtp, sorted, nentries,
1578 sizeof (dt_ahashent_t *), NULL);
1579 } else {
1580 /*
1581 * If we've been explicitly passed a sorting function,
1582 * we'll use that -- ignoring the values of the "aggsortrev",
1583 * "aggsortkey" and "aggsortkeypos" options.
1584 */
1585 qsort(sorted, nentries, sizeof (dt_ahashent_t *), sfunc);
1586 }
1587
1588 (void) pthread_mutex_unlock(&dt_qsort_lock);
1589
1590 for (i = 0; i < nentries; i++) {
1591 h = sorted[i];
1592
1593 if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
1594 goto out;
1595 }
1596
1597 rval = 0;
1598 out:
1599 if (agp->dtat_flags & DTRACE_A_TOTAL)
1600 (void) dt_aggregate_total(dtp, B_TRUE);
1601
1602 if (agp->dtat_flags & DTRACE_A_MINMAXBIN)
1603 (void) dt_aggregate_minmaxbin(dtp, B_TRUE);
1604
1605 dt_free(dtp, sorted);
1606 return (rval);
1607 }
1608
1609 int
1610 dtrace_aggregate_walk_sorted(dtrace_hdl_t *dtp,
1611 dtrace_aggregate_f *func, void *arg)
1612 {
1613 return (dt_aggregate_walk_sorted(dtp, func, arg, NULL));
1614 }
1615
1616 int
1617 dtrace_aggregate_walk_keysorted(dtrace_hdl_t *dtp,
1618 dtrace_aggregate_f *func, void *arg)
1619 {
1620 return (dt_aggregate_walk_sorted(dtp, func,
1621 arg, dt_aggregate_varkeycmp));
1622 }
1623
1624 int
1625 dtrace_aggregate_walk_valsorted(dtrace_hdl_t *dtp,
1626 dtrace_aggregate_f *func, void *arg)
1627 {
1628 return (dt_aggregate_walk_sorted(dtp, func,
1629 arg, dt_aggregate_varvalcmp));
1630 }
1631
1632 int
1633 dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t *dtp,
1634 dtrace_aggregate_f *func, void *arg)
1635 {
1636 return (dt_aggregate_walk_sorted(dtp, func,
1637 arg, dt_aggregate_keyvarcmp));
1638 }
1639
1640 int
1641 dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t *dtp,
1642 dtrace_aggregate_f *func, void *arg)
1643 {
1644 return (dt_aggregate_walk_sorted(dtp, func,
1645 arg, dt_aggregate_valvarcmp));
1646 }
1647
1648 int
1649 dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t *dtp,
1650 dtrace_aggregate_f *func, void *arg)
1651 {
1652 return (dt_aggregate_walk_sorted(dtp, func,
1653 arg, dt_aggregate_varkeyrevcmp));
1654 }
1655
1656 int
1657 dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t *dtp,
1658 dtrace_aggregate_f *func, void *arg)
1659 {
1660 return (dt_aggregate_walk_sorted(dtp, func,
1661 arg, dt_aggregate_varvalrevcmp));
1662 }
1663
1664 int
1665 dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t *dtp,
1666 dtrace_aggregate_f *func, void *arg)
1667 {
1668 return (dt_aggregate_walk_sorted(dtp, func,
1669 arg, dt_aggregate_keyvarrevcmp));
1670 }
1671
1672 int
1673 dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t *dtp,
1674 dtrace_aggregate_f *func, void *arg)
1675 {
1676 return (dt_aggregate_walk_sorted(dtp, func,
1677 arg, dt_aggregate_valvarrevcmp));
1678 }
1679
1680 int
1681 dtrace_aggregate_walk_joined(dtrace_hdl_t *dtp, dtrace_aggvarid_t *aggvars,
1682 int naggvars, dtrace_aggregate_walk_joined_f *func, void *arg)
1683 {
1684 dt_aggregate_t *agp = &dtp->dt_aggregate;
1685 dt_ahashent_t *h, **sorted = NULL, ***bundle, **nbundle;
1686 const dtrace_aggdata_t **data;
1687 dt_ahashent_t *zaggdata = NULL;
1688 dt_ahash_t *hash = &agp->dtat_hash;
1689 size_t nentries = 0, nbundles = 0, start, zsize = 0, bundlesize;
1690 dtrace_aggvarid_t max = 0, aggvar;
1691 int rval = -1, *map, *remap = NULL;
1692 int i, j;
1693 dtrace_optval_t sortpos = dtp->dt_options[DTRACEOPT_AGGSORTPOS];
1694
1695 /*
1696 * If the sorting position is greater than the number of aggregation
1697 * variable IDs, we silently set it to 0.
1698 */
1699 if (sortpos == DTRACEOPT_UNSET || sortpos >= naggvars)
1700 sortpos = 0;
1701
1702 /*
1703 * First we need to translate the specified aggregation variable IDs
1704 * into a linear map that will allow us to translate an aggregation
1705 * variable ID into its position in the specified aggvars.
1706 */
1707 for (i = 0; i < naggvars; i++) {
1708 if (aggvars[i] == DTRACE_AGGVARIDNONE || aggvars[i] < 0)
1709 return (dt_set_errno(dtp, EDT_BADAGGVAR));
1710
1711 if (aggvars[i] > max)
1712 max = aggvars[i];
1713 }
1714
1715 if ((map = dt_zalloc(dtp, (max + 1) * sizeof (int))) == NULL)
1716 return (-1);
1717
1718 zaggdata = dt_zalloc(dtp, naggvars * sizeof (dt_ahashent_t));
1719
1720 if (zaggdata == NULL)
1721 goto out;
1722
1723 for (i = 0; i < naggvars; i++) {
1724 int ndx = i + sortpos;
1725
1726 if (ndx >= naggvars)
1727 ndx -= naggvars;
1728
1729 aggvar = aggvars[ndx];
1730 assert(aggvar <= max);
1731
1732 if (map[aggvar]) {
1733 /*
1734 * We have an aggregation variable that is present
1735 * more than once in the array of aggregation
1736 * variables. While it's unclear why one might want
1737 * to do this, it's legal. To support this construct,
1738 * we will allocate a remap that will indicate the
1739 * position from which this aggregation variable
1740 * should be pulled. (That is, where the remap will
1741 * map from one position to another.)
1742 */
1743 if (remap == NULL) {
1744 remap = dt_zalloc(dtp, naggvars * sizeof (int));
1745
1746 if (remap == NULL)
1747 goto out;
1748 }
1749
1750 /*
1751 * Given that the variable is already present, assert
1752 * that following through the mapping and adjusting
1753 * for the sort position yields the same aggregation
1754 * variable ID.
1755 */
1756 assert(aggvars[(map[aggvar] - 1 + sortpos) %
1757 naggvars] == aggvars[ndx]);
1758
1759 remap[i] = map[aggvar];
1760 continue;
1761 }
1762
1763 map[aggvar] = i + 1;
1764 }
1765
1766 /*
1767 * We need to take two passes over the data to size our allocation, so
1768 * we'll use the first pass to also fill in the zero-filled data to be
1769 * used to properly format a zero-valued aggregation.
1770 */
1771 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1772 dtrace_aggvarid_t id;
1773 int ndx;
1774
1775 if ((id = dt_aggregate_aggvarid(h)) > max || !(ndx = map[id]))
1776 continue;
1777
1778 if (zaggdata[ndx - 1].dtahe_size == 0) {
1779 zaggdata[ndx - 1].dtahe_size = h->dtahe_size;
1780 zaggdata[ndx - 1].dtahe_data = h->dtahe_data;
1781 }
1782
1783 nentries++;
1784 }
1785
1786 if (nentries == 0) {
1787 /*
1788 * We couldn't find any entries; there is nothing else to do.
1789 */
1790 rval = 0;
1791 goto out;
1792 }
1793
1794 /*
1795 * Before we sort the data, we're going to look for any holes in our
1796 * zero-filled data. This will occur if an aggregation variable that
1797 * we are being asked to print has not yet been assigned the result of
1798 * any aggregating action for _any_ tuple. The issue becomes that we
1799 * would like a zero value to be printed for all columns for this
1800 * aggregation, but without any record description, we don't know the
1801 * aggregating action that corresponds to the aggregation variable. To
1802 * try to find a match, we're simply going to lookup aggregation IDs
1803 * (which are guaranteed to be contiguous and to start from 1), looking
1804 * for the specified aggregation variable ID. If we find a match,
1805 * we'll use that. If we iterate over all aggregation IDs and don't
1806 * find a match, then we must be an anonymous enabling. (Anonymous
1807 * enablings can't currently derive either aggregation variable IDs or
1808 * aggregation variable names given only an aggregation ID.) In this
1809 * obscure case (anonymous enabling, multiple aggregation printa() with
1810 * some aggregations not represented for any tuple), our defined
1811 * behavior is that the zero will be printed in the format of the first
1812 * aggregation variable that contains any non-zero value.
1813 */
1814 for (i = 0; i < naggvars; i++) {
1815 if (zaggdata[i].dtahe_size == 0) {
1816 dtrace_aggvarid_t aggvar;
1817
1818 aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
1819 assert(zaggdata[i].dtahe_data.dtada_data == NULL);
1820
1821 for (j = DTRACE_AGGIDNONE + 1; ; j++) {
1822 dtrace_aggdesc_t *agg;
1823 dtrace_aggdata_t *aggdata;
1824
1825 if (dt_aggid_lookup(dtp, j, &agg) != 0)
1826 break;
1827
1828 if (agg->dtagd_varid != aggvar)
1829 continue;
1830
1831 /*
1832 * We have our description -- now we need to
1833 * cons up the zaggdata entry for it.
1834 */
1835 aggdata = &zaggdata[i].dtahe_data;
1836 aggdata->dtada_size = agg->dtagd_size;
1837 aggdata->dtada_desc = agg;
1838 aggdata->dtada_handle = dtp;
1839 (void) dt_epid_lookup(dtp, agg->dtagd_epid,
1840 &aggdata->dtada_edesc,
1841 &aggdata->dtada_pdesc);
1842 aggdata->dtada_normal = 1;
1843 zaggdata[i].dtahe_hashval = 0;
1844 zaggdata[i].dtahe_size = agg->dtagd_size;
1845 break;
1846 }
1847
1848 if (zaggdata[i].dtahe_size == 0) {
1849 caddr_t data;
1850
1851 /*
1852 * We couldn't find this aggregation, meaning
1853 * that we have never seen it before for any
1854 * tuple _and_ this is an anonymous enabling.
1855 * That is, we're in the obscure case outlined
1856 * above. In this case, our defined behavior
1857 * is to format the data in the format of the
1858 * first non-zero aggregation -- of which, of
1859 * course, we know there to be at least one
1860 * (or nentries would have been zero).
1861 */
1862 for (j = 0; j < naggvars; j++) {
1863 if (zaggdata[j].dtahe_size != 0)
1864 break;
1865 }
1866
1867 assert(j < naggvars);
1868 zaggdata[i] = zaggdata[j];
1869
1870 data = zaggdata[i].dtahe_data.dtada_data;
1871 assert(data != NULL);
1872 }
1873 }
1874 }
1875
1876 /*
1877 * Now we need to allocate our zero-filled data for use for
1878 * aggregations that don't have a value corresponding to a given key.
1879 */
1880 for (i = 0; i < naggvars; i++) {
1881 dtrace_aggdata_t *aggdata = &zaggdata[i].dtahe_data;
1882 dtrace_aggdesc_t *aggdesc = aggdata->dtada_desc;
1883 dtrace_recdesc_t *rec;
1884 uint64_t larg;
1885 caddr_t zdata;
1886
1887 zsize = zaggdata[i].dtahe_size;
1888 assert(zsize != 0);
1889
1890 if ((zdata = dt_zalloc(dtp, zsize)) == NULL) {
1891 /*
1892 * If we failed to allocated some zero-filled data, we
1893 * need to zero out the remaining dtada_data pointers
1894 * to prevent the wrong data from being freed below.
1895 */
1896 for (j = i; j < naggvars; j++)
1897 zaggdata[j].dtahe_data.dtada_data = NULL;
1898 goto out;
1899 }
1900
1901 aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
1902
1903 /*
1904 * First, the easy bit. To maintain compatibility with
1905 * consumers that pull the compiler-generated ID out of the
1906 * data, we put that ID at the top of the zero-filled data.
1907 */
1908 rec = &aggdesc->dtagd_rec[0];
1909 /* LINTED - alignment */
1910 *((dtrace_aggvarid_t *)(zdata + rec->dtrd_offset)) = aggvar;
1911
1912 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
1913
1914 /*
1915 * Now for the more complicated part. If (and only if) this
1916 * is an lquantize() aggregating action, zero-filled data is
1917 * not equivalent to an empty record: we must also get the
1918 * parameters for the lquantize().
1919 */
1920 if (rec->dtrd_action == DTRACEAGG_LQUANTIZE) {
1921 if (aggdata->dtada_data != NULL) {
1922 /*
1923 * The easier case here is if we actually have
1924 * some prototype data -- in which case we
1925 * manually dig it out of the aggregation
1926 * record.
1927 */
1928 /* LINTED - alignment */
1929 larg = *((uint64_t *)(aggdata->dtada_data +
1930 rec->dtrd_offset));
1931 } else {
1932 /*
1933 * We don't have any prototype data. As a
1934 * result, we know that we _do_ have the
1935 * compiler-generated information. (If this
1936 * were an anonymous enabling, all of our
1937 * zero-filled data would have prototype data
1938 * -- either directly or indirectly.) So as
1939 * gross as it is, we'll grovel around in the
1940 * compiler-generated information to find the
1941 * lquantize() parameters.
1942 */
1943 dtrace_stmtdesc_t *sdp;
1944 dt_ident_t *aid;
1945 dt_idsig_t *isp;
1946
1947 sdp = (dtrace_stmtdesc_t *)(uintptr_t)
1948 aggdesc->dtagd_rec[0].dtrd_uarg;
1949 aid = sdp->dtsd_aggdata;
1950 isp = (dt_idsig_t *)aid->di_data;
1951 assert(isp->dis_auxinfo != 0);
1952 larg = isp->dis_auxinfo;
1953 }
1954
1955 /* LINTED - alignment */
1956 *((uint64_t *)(zdata + rec->dtrd_offset)) = larg;
1957 }
1958
1959 aggdata->dtada_data = zdata;
1960 }
1961
1962 /*
1963 * Now that we've dealt with setting up our zero-filled data, we can
1964 * allocate our sorted array, and take another pass over the data to
1965 * fill it.
1966 */
1967 sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
1968
1969 if (sorted == NULL)
1970 goto out;
1971
1972 for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall) {
1973 dtrace_aggvarid_t id;
1974
1975 if ((id = dt_aggregate_aggvarid(h)) > max || !map[id])
1976 continue;
1977
1978 sorted[i++] = h;
1979 }
1980
1981 assert(i == nentries);
1982
1983 /*
1984 * We've loaded our array; now we need to sort by value to allow us
1985 * to create bundles of like value. We're going to acquire the
1986 * dt_qsort_lock here, and hold it across all of our subsequent
1987 * comparison and sorting.
1988 */
1989 (void) pthread_mutex_lock(&dt_qsort_lock);
1990
1991 qsort(sorted, nentries, sizeof (dt_ahashent_t *),
1992 dt_aggregate_keyvarcmp);
1993
1994 /*
1995 * Now we need to go through and create bundles. Because the number
1996 * of bundles is bounded by the size of the sorted array, we're going
1997 * to reuse the underlying storage. And note that "bundle" is an
1998 * array of pointers to arrays of pointers to dt_ahashent_t -- making
1999 * its type (regrettably) "dt_ahashent_t ***". (Regrettable because
2000 * '*' -- like '_' and 'X' -- should never appear in triplicate in
2001 * an ideal world.)
2002 */
2003 bundle = (dt_ahashent_t ***)sorted;
2004
2005 for (i = 1, start = 0; i <= nentries; i++) {
2006 if (i < nentries &&
2007 dt_aggregate_keycmp(&sorted[i], &sorted[i - 1]) == 0)
2008 continue;
2009
2010 /*
2011 * We have a bundle boundary. Everything from start to
2012 * (i - 1) belongs in one bundle.
2013 */
2014 assert(i - start <= naggvars);
2015 bundlesize = (naggvars + 2) * sizeof (dt_ahashent_t *);
2016
2017 if ((nbundle = dt_zalloc(dtp, bundlesize)) == NULL) {
2018 (void) pthread_mutex_unlock(&dt_qsort_lock);
2019 goto out;
2020 }
2021
2022 for (j = start; j < i; j++) {
2023 dtrace_aggvarid_t id = dt_aggregate_aggvarid(sorted[j]);
2024
2025 assert(id <= max);
2026 assert(map[id] != 0);
2027 assert(map[id] - 1 < naggvars);
2028 assert(nbundle[map[id] - 1] == NULL);
2029 nbundle[map[id] - 1] = sorted[j];
2030
2031 if (nbundle[naggvars] == NULL)
2032 nbundle[naggvars] = sorted[j];
2033 }
2034
2035 for (j = 0; j < naggvars; j++) {
2036 if (nbundle[j] != NULL)
2037 continue;
2038
2039 /*
2040 * Before we assume that this aggregation variable
2041 * isn't present (and fall back to using the
2042 * zero-filled data allocated earlier), check the
2043 * remap. If we have a remapping, we'll drop it in
2044 * here. Note that we might be remapping an
2045 * aggregation variable that isn't present for this
2046 * key; in this case, the aggregation data that we
2047 * copy will point to the zeroed data.
2048 */
2049 if (remap != NULL && remap[j]) {
2050 assert(remap[j] - 1 < j);
2051 assert(nbundle[remap[j] - 1] != NULL);
2052 nbundle[j] = nbundle[remap[j] - 1];
2053 } else {
2054 nbundle[j] = &zaggdata[j];
2055 }
2056 }
2057
2058 bundle[nbundles++] = nbundle;
2059 start = i;
2060 }
2061
2062 /*
2063 * Now we need to re-sort based on the first value.
2064 */
2065 dt_aggregate_qsort(dtp, bundle, nbundles, sizeof (dt_ahashent_t **),
2066 dt_aggregate_bundlecmp);
2067
2068 (void) pthread_mutex_unlock(&dt_qsort_lock);
2069
2070 /*
2071 * We're done! Now we just need to go back over the sorted bundles,
2072 * calling the function.
2073 */
2074 data = alloca((naggvars + 1) * sizeof (dtrace_aggdata_t *));
2075
2076 for (i = 0; i < nbundles; i++) {
2077 for (j = 0; j < naggvars; j++)
2078 data[j + 1] = NULL;
2079
2080 for (j = 0; j < naggvars; j++) {
2081 int ndx = j - sortpos;
2082
2083 if (ndx < 0)
2084 ndx += naggvars;
2085
2086 assert(bundle[i][ndx] != NULL);
2087 data[j + 1] = &bundle[i][ndx]->dtahe_data;
2088 }
2089
2090 for (j = 0; j < naggvars; j++)
2091 assert(data[j + 1] != NULL);
2092
2093 /*
2094 * The representative key is the last element in the bundle.
2095 * Assert that we have one, and then set it to be the first
2096 * element of data.
2097 */
2098 assert(bundle[i][j] != NULL);
2099 data[0] = &bundle[i][j]->dtahe_data;
2100
2101 if ((rval = func(data, naggvars + 1, arg)) == -1)
2102 goto out;
2103 }
2104
2105 rval = 0;
2106 out:
2107 for (i = 0; i < nbundles; i++)
2108 dt_free(dtp, bundle[i]);
2109
2110 if (zaggdata != NULL) {
2111 for (i = 0; i < naggvars; i++)
2112 dt_free(dtp, zaggdata[i].dtahe_data.dtada_data);
2113 }
2114
2115 dt_free(dtp, zaggdata);
2116 dt_free(dtp, sorted);
2117 dt_free(dtp, remap);
2118 dt_free(dtp, map);
2119
2120 return (rval);
2121 }
2122
2123 int
2124 dtrace_aggregate_print(dtrace_hdl_t *dtp, FILE *fp,
2125 dtrace_aggregate_walk_f *func)
2126 {
2127 dt_print_aggdata_t pd;
2128
2129 bzero(&pd, sizeof (pd));
2130
2131 pd.dtpa_dtp = dtp;
2132 pd.dtpa_fp = fp;
2133 pd.dtpa_allunprint = 1;
2134
2135 if (func == NULL)
2136 func = dtrace_aggregate_walk_sorted;
2137
2138 if (dtp->dt_oformat) {
2139 if ((*func)(dtp, dt_format_agg, &pd) == -1)
2140 return (dt_set_errno(dtp, dtp->dt_errno));
2141 } else {
2142 if ((*func)(dtp, dt_print_agg, &pd) == -1)
2143 return (dt_set_errno(dtp, dtp->dt_errno));
2144 }
2145
2146 return (0);
2147 }
2148
2149 void
2150 dtrace_aggregate_clear(dtrace_hdl_t *dtp)
2151 {
2152 dt_aggregate_t *agp = &dtp->dt_aggregate;
2153 dt_ahash_t *hash = &agp->dtat_hash;
2154 dt_ahashent_t *h;
2155 dtrace_aggdata_t *data;
2156 dtrace_aggdesc_t *aggdesc;
2157 dtrace_recdesc_t *rec;
2158 int i, max_cpus = agp->dtat_maxcpu;
2159
2160 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
2161 aggdesc = h->dtahe_data.dtada_desc;
2162 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
2163 data = &h->dtahe_data;
2164
2165 bzero(&data->dtada_data[rec->dtrd_offset], rec->dtrd_size);
2166
2167 if (data->dtada_percpu == NULL)
2168 continue;
2169
2170 for (i = 0; i < max_cpus; i++)
2171 bzero(data->dtada_percpu[i], rec->dtrd_size);
2172 }
2173 }
2174
2175 void
2176 dt_aggregate_destroy(dtrace_hdl_t *dtp)
2177 {
2178 dt_aggregate_t *agp = &dtp->dt_aggregate;
2179 dt_ahash_t *hash = &agp->dtat_hash;
2180 dt_ahashent_t *h, *next;
2181 dtrace_aggdata_t *aggdata;
2182 int i, max_cpus = agp->dtat_maxcpu;
2183
2184 if (hash->dtah_hash == NULL) {
2185 assert(hash->dtah_all == NULL);
2186 } else {
2187 free(hash->dtah_hash);
2188
2189 for (h = hash->dtah_all; h != NULL; h = next) {
2190 next = h->dtahe_nextall;
2191
2192 aggdata = &h->dtahe_data;
2193
2194 if (aggdata->dtada_percpu != NULL) {
2195 for (i = 0; i < max_cpus; i++)
2196 free(aggdata->dtada_percpu[i]);
2197 free(aggdata->dtada_percpu);
2198 }
2199
2200 free(aggdata->dtada_data);
2201 free(h);
2202 }
2203
2204 hash->dtah_hash = NULL;
2205 hash->dtah_all = NULL;
2206 hash->dtah_size = 0;
2207 }
2208
2209 free(agp->dtat_buf.dtbd_data);
2210 free(agp->dtat_cpus);
2211 }
2212