4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
28 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
29 * Copyright (c) 2012 by Delphix. All rights reserved.
41 #define DTRACE_AHASHSIZE 32779 /* big 'ol prime */
44 * Because qsort(3C) does not allow an argument to be passed to a comparison
45 * function, the variables that affect comparison must regrettably be global;
46 * they are protected by a global static lock, dt_qsort_lock.
48 static pthread_mutex_t dt_qsort_lock
= PTHREAD_MUTEX_INITIALIZER
;
50 static int dt_revsort
;
51 static int dt_keysort
;
54 #define DT_LESSTHAN (dt_revsort == 0 ? -1 : 1)
55 #define DT_GREATERTHAN (dt_revsort == 0 ? 1 : -1)
58 dt_aggregate_count(int64_t *existing
, int64_t *new, size_t size
)
62 for (i
= 0; i
< size
/ sizeof (int64_t); i
++)
63 existing
[i
] = existing
[i
] + new[i
];
67 dt_aggregate_countcmp(int64_t *lhs
, int64_t *rhs
)
76 return (DT_GREATERTHAN
);
83 dt_aggregate_min(int64_t *existing
, int64_t *new, size_t size
)
91 dt_aggregate_max(int64_t *existing
, int64_t *new, size_t size
)
98 dt_aggregate_averagecmp(int64_t *lhs
, int64_t *rhs
)
100 int64_t lavg
= lhs
[0] ? (lhs
[1] / lhs
[0]) : 0;
101 int64_t ravg
= rhs
[0] ? (rhs
[1] / rhs
[0]) : 0;
104 return (DT_LESSTHAN
);
107 return (DT_GREATERTHAN
);
113 dt_aggregate_stddevcmp(int64_t *lhs
, int64_t *rhs
)
115 uint64_t lsd
= dt_stddev((uint64_t *)lhs
, 1);
116 uint64_t rsd
= dt_stddev((uint64_t *)rhs
, 1);
119 return (DT_LESSTHAN
);
122 return (DT_GREATERTHAN
);
129 dt_aggregate_lquantize(int64_t *existing
, int64_t *new, size_t size
)
131 int64_t arg
= *existing
++;
132 uint16_t levels
= DTRACE_LQUANTIZE_LEVELS(arg
);
135 for (i
= 0; i
<= levels
+ 1; i
++)
136 existing
[i
] = existing
[i
] + new[i
+ 1];
140 dt_aggregate_lquantizedsum(int64_t *lquanta
)
142 int64_t arg
= *lquanta
++;
143 int32_t base
= DTRACE_LQUANTIZE_BASE(arg
);
144 uint16_t step
= DTRACE_LQUANTIZE_STEP(arg
);
145 uint16_t levels
= DTRACE_LQUANTIZE_LEVELS(arg
), i
;
146 long double total
= (long double)lquanta
[0] * (long double)(base
- 1);
148 for (i
= 0; i
< levels
; base
+= step
, i
++)
149 total
+= (long double)lquanta
[i
+ 1] * (long double)base
;
151 return (total
+ (long double)lquanta
[levels
+ 1] *
152 (long double)(base
+ 1));
156 dt_aggregate_lquantizedzero(int64_t *lquanta
)
158 int64_t arg
= *lquanta
++;
159 int32_t base
= DTRACE_LQUANTIZE_BASE(arg
);
160 uint16_t step
= DTRACE_LQUANTIZE_STEP(arg
);
161 uint16_t levels
= DTRACE_LQUANTIZE_LEVELS(arg
), i
;
166 for (i
= 0; i
< levels
; base
+= step
, i
++) {
170 return (lquanta
[i
+ 1]);
174 return (lquanta
[levels
+ 1]);
180 dt_aggregate_lquantizedcmp(int64_t *lhs
, int64_t *rhs
)
182 long double lsum
= dt_aggregate_lquantizedsum(lhs
);
183 long double rsum
= dt_aggregate_lquantizedsum(rhs
);
184 int64_t lzero
, rzero
;
187 return (DT_LESSTHAN
);
190 return (DT_GREATERTHAN
);
193 * If they're both equal, then we will compare based on the weights at
194 * zero. If the weights at zero are equal (or if zero is not within
195 * the range of the linear quantization), then this will be judged a
196 * tie and will be resolved based on the key comparison.
198 lzero
= dt_aggregate_lquantizedzero(lhs
);
199 rzero
= dt_aggregate_lquantizedzero(rhs
);
202 return (DT_LESSTHAN
);
205 return (DT_GREATERTHAN
);
211 dt_aggregate_llquantize(int64_t *existing
, int64_t *new, size_t size
)
215 for (i
= 1; i
< size
/ sizeof (int64_t); i
++)
216 existing
[i
] = existing
[i
] + new[i
];
220 dt_aggregate_llquantizedsum(int64_t *llquanta
)
222 int64_t arg
= *llquanta
++;
223 uint16_t factor
= DTRACE_LLQUANTIZE_FACTOR(arg
);
224 uint16_t low
= DTRACE_LLQUANTIZE_LOW(arg
);
225 uint16_t high
= DTRACE_LLQUANTIZE_HIGH(arg
);
226 uint16_t nsteps
= DTRACE_LLQUANTIZE_NSTEP(arg
);
228 int64_t value
= 1, next
, step
;
231 assert(nsteps
>= factor
);
232 assert(nsteps
% factor
== 0);
234 for (order
= 0; order
< low
; order
++)
237 total
= (long double)llquanta
[bin
++] * (long double)(value
- 1);
239 next
= value
* factor
;
240 step
= next
> nsteps
? next
/ nsteps
: 1;
242 while (order
<= high
) {
243 assert(value
< next
);
244 total
+= (long double)llquanta
[bin
++] * (long double)(value
);
246 if ((value
+= step
) != next
)
249 next
= value
* factor
;
250 step
= next
> nsteps
? next
/ nsteps
: 1;
254 return (total
+ (long double)llquanta
[bin
] * (long double)value
);
258 dt_aggregate_llquantizedcmp(int64_t *lhs
, int64_t *rhs
)
260 long double lsum
= dt_aggregate_llquantizedsum(lhs
);
261 long double rsum
= dt_aggregate_llquantizedsum(rhs
);
262 int64_t lzero
, rzero
;
265 return (DT_LESSTHAN
);
268 return (DT_GREATERTHAN
);
271 * If they're both equal, then we will compare based on the weights at
272 * zero. If the weights at zero are equal, then this will be judged a
273 * tie and will be resolved based on the key comparison.
279 return (DT_LESSTHAN
);
282 return (DT_GREATERTHAN
);
288 dt_aggregate_quantizedcmp(int64_t *lhs
, int64_t *rhs
)
290 int nbuckets
= DTRACE_QUANTIZE_NBUCKETS
, i
;
291 long double ltotal
= 0, rtotal
= 0;
292 int64_t lzero
, rzero
;
294 for (i
= 0; i
< nbuckets
; i
++) {
295 int64_t bucketval
= DTRACE_QUANTIZE_BUCKETVAL(i
);
297 if (bucketval
== 0) {
302 ltotal
+= (long double)bucketval
* (long double)lhs
[i
];
303 rtotal
+= (long double)bucketval
* (long double)rhs
[i
];
307 return (DT_LESSTHAN
);
310 return (DT_GREATERTHAN
);
313 * If they're both equal, then we will compare based on the weights at
314 * zero. If the weights at zero are equal, then this will be judged a
315 * tie and will be resolved based on the key comparison.
318 return (DT_LESSTHAN
);
321 return (DT_GREATERTHAN
);
327 dt_aggregate_usym(dtrace_hdl_t
*dtp
, uint64_t *data
)
329 uint64_t pid
= data
[0];
330 uint64_t *pc
= &data
[1];
331 struct ps_prochandle
*P
;
334 if (dtp
->dt_vector
!= NULL
)
337 if ((P
= dt_proc_grab(dtp
, pid
, PGRAB_RDONLY
| PGRAB_FORCE
, 0)) == NULL
)
340 dt_proc_lock(dtp
, P
);
342 if (Plookup_by_addr(P
, *pc
, NULL
, 0, &sym
) == 0)
345 dt_proc_unlock(dtp
, P
);
346 dt_proc_release(dtp
, P
);
350 dt_aggregate_umod(dtrace_hdl_t
*dtp
, uint64_t *data
)
352 uint64_t pid
= data
[0];
353 uint64_t *pc
= &data
[1];
354 struct ps_prochandle
*P
;
357 if (dtp
->dt_vector
!= NULL
)
360 if ((P
= dt_proc_grab(dtp
, pid
, PGRAB_RDONLY
| PGRAB_FORCE
, 0)) == NULL
)
363 dt_proc_lock(dtp
, P
);
365 if ((map
= Paddr_to_map(P
, *pc
)) != NULL
)
368 dt_proc_unlock(dtp
, P
);
369 dt_proc_release(dtp
, P
);
373 dt_aggregate_sym(dtrace_hdl_t
*dtp
, uint64_t *data
)
378 if (dtrace_lookup_by_addr(dtp
, *pc
, &sym
, NULL
) == 0)
383 dt_aggregate_mod(dtrace_hdl_t
*dtp
, uint64_t *data
)
388 if (dtp
->dt_vector
!= NULL
) {
390 * We don't have a way of just getting the module for a
391 * vectored open, and it doesn't seem to be worth defining
392 * one. This means that use of mod() won't get true
393 * aggregation in the postmortem case (some modules may
394 * appear more than once in aggregation output). It seems
395 * unlikely that anyone will ever notice or care...
400 for (dmp
= dt_list_next(&dtp
->dt_modlist
); dmp
!= NULL
;
401 dmp
= dt_list_next(dmp
)) {
402 if (*pc
- dmp
->dm_text_va
< dmp
->dm_text_size
) {
403 *pc
= dmp
->dm_text_va
;
409 static dtrace_aggvarid_t
410 dt_aggregate_aggvarid(dt_ahashent_t
*ent
)
412 dtrace_aggdesc_t
*agg
= ent
->dtahe_data
.dtada_desc
;
413 caddr_t data
= ent
->dtahe_data
.dtada_data
;
414 dtrace_recdesc_t
*rec
= agg
->dtagd_rec
;
417 * First, we'll check the variable ID in the aggdesc. If it's valid,
418 * we'll return it. If not, we'll use the compiler-generated ID
419 * present as the first record.
421 if (agg
->dtagd_varid
!= DTRACE_AGGVARIDNONE
)
422 return (agg
->dtagd_varid
);
424 agg
->dtagd_varid
= *((dtrace_aggvarid_t
*)(uintptr_t)(data
+
427 return (agg
->dtagd_varid
);
432 dt_aggregate_snap_cpu(dtrace_hdl_t
*dtp
, processorid_t cpu
)
436 size_t offs
, roffs
, size
, ndx
;
439 dtrace_recdesc_t
*rec
;
440 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
441 dtrace_aggdesc_t
*agg
;
442 dt_ahash_t
*hash
= &agp
->dtat_hash
;
444 dtrace_bufdesc_t b
= agp
->dtat_buf
, *buf
= &b
;
445 dtrace_aggdata_t
*aggdata
;
446 int flags
= agp
->dtat_flags
;
450 if (dt_ioctl(dtp
, DTRACEIOC_AGGSNAP
, buf
) == -1) {
451 if (errno
== ENOENT
) {
453 * If that failed with ENOENT, it may be because the
454 * CPU was unconfigured. This is okay; we'll just
455 * do nothing but return success.
460 return (dt_set_errno(dtp
, errno
));
463 if (buf
->dtbd_drops
!= 0) {
464 if (dt_handle_cpudrop(dtp
, cpu
,
465 DTRACEDROP_AGGREGATION
, buf
->dtbd_drops
) == -1)
469 if (buf
->dtbd_size
== 0)
472 if (hash
->dtah_hash
== NULL
) {
475 hash
->dtah_size
= DTRACE_AHASHSIZE
;
476 size
= hash
->dtah_size
* sizeof (dt_ahashent_t
*);
478 if ((hash
->dtah_hash
= malloc(size
)) == NULL
)
479 return (dt_set_errno(dtp
, EDT_NOMEM
));
481 bzero(hash
->dtah_hash
, size
);
484 for (offs
= 0; offs
< buf
->dtbd_size
; ) {
486 * We're guaranteed to have an ID.
488 id
= *((dtrace_epid_t
*)((uintptr_t)buf
->dtbd_data
+
491 if (id
== DTRACE_AGGIDNONE
) {
493 * This is filler to assure proper alignment of the
494 * next record; we simply ignore it.
500 if ((rval
= dt_aggid_lookup(dtp
, id
, &agg
)) != 0)
503 addr
= buf
->dtbd_data
+ offs
;
504 size
= agg
->dtagd_size
;
507 for (j
= 0; j
< agg
->dtagd_nrecs
- 1; j
++) {
508 rec
= &agg
->dtagd_rec
[j
];
509 roffs
= rec
->dtrd_offset
;
511 switch (rec
->dtrd_action
) {
513 dt_aggregate_usym(dtp
,
514 /* LINTED - alignment */
515 (uint64_t *)&addr
[roffs
]);
519 dt_aggregate_umod(dtp
,
520 /* LINTED - alignment */
521 (uint64_t *)&addr
[roffs
]);
525 /* LINTED - alignment */
526 dt_aggregate_sym(dtp
, (uint64_t *)&addr
[roffs
]);
530 /* LINTED - alignment */
531 dt_aggregate_mod(dtp
, (uint64_t *)&addr
[roffs
]);
538 for (i
= 0; i
< rec
->dtrd_size
; i
++)
539 hashval
+= addr
[roffs
+ i
];
542 ndx
= hashval
% hash
->dtah_size
;
544 for (h
= hash
->dtah_hash
[ndx
]; h
!= NULL
; h
= h
->dtahe_next
) {
545 if (h
->dtahe_hashval
!= hashval
)
548 if (h
->dtahe_size
!= size
)
551 aggdata
= &h
->dtahe_data
;
552 data
= aggdata
->dtada_data
;
554 for (j
= 0; j
< agg
->dtagd_nrecs
- 1; j
++) {
555 rec
= &agg
->dtagd_rec
[j
];
556 roffs
= rec
->dtrd_offset
;
558 for (i
= 0; i
< rec
->dtrd_size
; i
++)
559 if (addr
[roffs
+ i
] != data
[roffs
+ i
])
564 * We found it. Now we need to apply the aggregating
565 * action on the data here.
567 rec
= &agg
->dtagd_rec
[agg
->dtagd_nrecs
- 1];
568 roffs
= rec
->dtrd_offset
;
569 /* LINTED - alignment */
570 h
->dtahe_aggregate((int64_t *)&data
[roffs
],
571 /* LINTED - alignment */
572 (int64_t *)&addr
[roffs
], rec
->dtrd_size
);
575 * If we're keeping per CPU data, apply the aggregating
576 * action there as well.
578 if (aggdata
->dtada_percpu
!= NULL
) {
579 data
= aggdata
->dtada_percpu
[cpu
];
581 /* LINTED - alignment */
582 h
->dtahe_aggregate((int64_t *)data
,
583 /* LINTED - alignment */
584 (int64_t *)&addr
[roffs
], rec
->dtrd_size
);
593 * If we're here, we couldn't find an entry for this record.
595 if ((h
= malloc(sizeof (dt_ahashent_t
))) == NULL
)
596 return (dt_set_errno(dtp
, EDT_NOMEM
));
597 bzero(h
, sizeof (dt_ahashent_t
));
598 aggdata
= &h
->dtahe_data
;
600 if ((aggdata
->dtada_data
= malloc(size
)) == NULL
) {
602 return (dt_set_errno(dtp
, EDT_NOMEM
));
605 bcopy(addr
, aggdata
->dtada_data
, size
);
606 aggdata
->dtada_size
= size
;
607 aggdata
->dtada_desc
= agg
;
608 aggdata
->dtada_handle
= dtp
;
609 (void) dt_epid_lookup(dtp
, agg
->dtagd_epid
,
610 &aggdata
->dtada_edesc
, &aggdata
->dtada_pdesc
);
611 aggdata
->dtada_normal
= 1;
613 h
->dtahe_hashval
= hashval
;
614 h
->dtahe_size
= size
;
615 (void) dt_aggregate_aggvarid(h
);
617 rec
= &agg
->dtagd_rec
[agg
->dtagd_nrecs
- 1];
619 if (flags
& DTRACE_A_PERCPU
) {
620 int max_cpus
= agp
->dtat_maxcpu
;
621 caddr_t
*percpu
= malloc(max_cpus
* sizeof (caddr_t
));
623 if (percpu
== NULL
) {
624 free(aggdata
->dtada_data
);
626 return (dt_set_errno(dtp
, EDT_NOMEM
));
629 for (j
= 0; j
< max_cpus
; j
++) {
630 percpu
[j
] = malloc(rec
->dtrd_size
);
632 if (percpu
[j
] == NULL
) {
636 free(aggdata
->dtada_data
);
638 return (dt_set_errno(dtp
, EDT_NOMEM
));
642 bcopy(&addr
[rec
->dtrd_offset
],
643 percpu
[j
], rec
->dtrd_size
);
645 bzero(percpu
[j
], rec
->dtrd_size
);
649 aggdata
->dtada_percpu
= percpu
;
652 switch (rec
->dtrd_action
) {
654 h
->dtahe_aggregate
= dt_aggregate_min
;
658 h
->dtahe_aggregate
= dt_aggregate_max
;
661 case DTRACEAGG_LQUANTIZE
:
662 h
->dtahe_aggregate
= dt_aggregate_lquantize
;
665 case DTRACEAGG_LLQUANTIZE
:
666 h
->dtahe_aggregate
= dt_aggregate_llquantize
;
669 case DTRACEAGG_COUNT
:
672 case DTRACEAGG_STDDEV
:
673 case DTRACEAGG_QUANTIZE
:
674 h
->dtahe_aggregate
= dt_aggregate_count
;
678 return (dt_set_errno(dtp
, EDT_BADAGG
));
681 if (hash
->dtah_hash
[ndx
] != NULL
)
682 hash
->dtah_hash
[ndx
]->dtahe_prev
= h
;
684 h
->dtahe_next
= hash
->dtah_hash
[ndx
];
685 hash
->dtah_hash
[ndx
] = h
;
687 if (hash
->dtah_all
!= NULL
)
688 hash
->dtah_all
->dtahe_prevall
= h
;
690 h
->dtahe_nextall
= hash
->dtah_all
;
693 offs
+= agg
->dtagd_size
;
700 dtrace_aggregate_snap(dtrace_hdl_t
*dtp
)
703 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
704 hrtime_t now
= gethrtime();
705 dtrace_optval_t interval
= dtp
->dt_options
[DTRACEOPT_AGGRATE
];
707 if (dtp
->dt_lastagg
!= 0) {
708 if (now
- dtp
->dt_lastagg
< interval
)
711 dtp
->dt_lastagg
+= interval
;
713 dtp
->dt_lastagg
= now
;
717 return (dt_set_errno(dtp
, EINVAL
));
719 if (agp
->dtat_buf
.dtbd_size
== 0)
722 for (i
= 0; i
< agp
->dtat_ncpus
; i
++) {
723 if (rval
= dt_aggregate_snap_cpu(dtp
, agp
->dtat_cpus
[i
]))
731 dt_aggregate_hashcmp(const void *lhs
, const void *rhs
)
733 dt_ahashent_t
*lh
= *((dt_ahashent_t
**)lhs
);
734 dt_ahashent_t
*rh
= *((dt_ahashent_t
**)rhs
);
735 dtrace_aggdesc_t
*lagg
= lh
->dtahe_data
.dtada_desc
;
736 dtrace_aggdesc_t
*ragg
= rh
->dtahe_data
.dtada_desc
;
738 if (lagg
->dtagd_nrecs
< ragg
->dtagd_nrecs
)
739 return (DT_LESSTHAN
);
741 if (lagg
->dtagd_nrecs
> ragg
->dtagd_nrecs
)
742 return (DT_GREATERTHAN
);
748 dt_aggregate_varcmp(const void *lhs
, const void *rhs
)
750 dt_ahashent_t
*lh
= *((dt_ahashent_t
**)lhs
);
751 dt_ahashent_t
*rh
= *((dt_ahashent_t
**)rhs
);
752 dtrace_aggvarid_t lid
, rid
;
754 lid
= dt_aggregate_aggvarid(lh
);
755 rid
= dt_aggregate_aggvarid(rh
);
758 return (DT_LESSTHAN
);
761 return (DT_GREATERTHAN
);
767 dt_aggregate_keycmp(const void *lhs
, const void *rhs
)
769 dt_ahashent_t
*lh
= *((dt_ahashent_t
**)lhs
);
770 dt_ahashent_t
*rh
= *((dt_ahashent_t
**)rhs
);
771 dtrace_aggdesc_t
*lagg
= lh
->dtahe_data
.dtada_desc
;
772 dtrace_aggdesc_t
*ragg
= rh
->dtahe_data
.dtada_desc
;
773 dtrace_recdesc_t
*lrec
, *rrec
;
775 int rval
, i
, j
, keypos
, nrecs
;
777 if ((rval
= dt_aggregate_hashcmp(lhs
, rhs
)) != 0)
780 nrecs
= lagg
->dtagd_nrecs
- 1;
781 assert(nrecs
== ragg
->dtagd_nrecs
- 1);
783 keypos
= dt_keypos
+ 1 >= nrecs
? 0 : dt_keypos
;
785 for (i
= 1; i
< nrecs
; i
++) {
787 int ndx
= i
+ keypos
;
790 ndx
= ndx
- nrecs
+ 1;
792 lrec
= &lagg
->dtagd_rec
[ndx
];
793 rrec
= &ragg
->dtagd_rec
[ndx
];
795 ldata
= lh
->dtahe_data
.dtada_data
+ lrec
->dtrd_offset
;
796 rdata
= rh
->dtahe_data
.dtada_data
+ rrec
->dtrd_offset
;
798 if (lrec
->dtrd_size
< rrec
->dtrd_size
)
799 return (DT_LESSTHAN
);
801 if (lrec
->dtrd_size
> rrec
->dtrd_size
)
802 return (DT_GREATERTHAN
);
804 switch (lrec
->dtrd_size
) {
805 case sizeof (uint64_t):
806 /* LINTED - alignment */
807 lval
= *((uint64_t *)ldata
);
808 /* LINTED - alignment */
809 rval
= *((uint64_t *)rdata
);
812 case sizeof (uint32_t):
813 /* LINTED - alignment */
814 lval
= *((uint32_t *)ldata
);
815 /* LINTED - alignment */
816 rval
= *((uint32_t *)rdata
);
819 case sizeof (uint16_t):
820 /* LINTED - alignment */
821 lval
= *((uint16_t *)ldata
);
822 /* LINTED - alignment */
823 rval
= *((uint16_t *)rdata
);
826 case sizeof (uint8_t):
827 lval
= *((uint8_t *)ldata
);
828 rval
= *((uint8_t *)rdata
);
832 switch (lrec
->dtrd_action
) {
834 case DTRACEACT_UADDR
:
836 for (j
= 0; j
< 2; j
++) {
837 /* LINTED - alignment */
838 lval
= ((uint64_t *)ldata
)[j
];
839 /* LINTED - alignment */
840 rval
= ((uint64_t *)rdata
)[j
];
843 return (DT_LESSTHAN
);
846 return (DT_GREATERTHAN
);
852 for (j
= 0; j
< lrec
->dtrd_size
; j
++) {
853 lval
= ((uint8_t *)ldata
)[j
];
854 rval
= ((uint8_t *)rdata
)[j
];
857 return (DT_LESSTHAN
);
860 return (DT_GREATERTHAN
);
868 return (DT_LESSTHAN
);
871 return (DT_GREATERTHAN
);
878 dt_aggregate_valcmp(const void *lhs
, const void *rhs
)
880 dt_ahashent_t
*lh
= *((dt_ahashent_t
**)lhs
);
881 dt_ahashent_t
*rh
= *((dt_ahashent_t
**)rhs
);
882 dtrace_aggdesc_t
*lagg
= lh
->dtahe_data
.dtada_desc
;
883 dtrace_aggdesc_t
*ragg
= rh
->dtahe_data
.dtada_desc
;
884 caddr_t ldata
= lh
->dtahe_data
.dtada_data
;
885 caddr_t rdata
= rh
->dtahe_data
.dtada_data
;
886 dtrace_recdesc_t
*lrec
, *rrec
;
887 int64_t *laddr
, *raddr
;
890 assert(lagg
->dtagd_nrecs
== ragg
->dtagd_nrecs
);
892 lrec
= &lagg
->dtagd_rec
[lagg
->dtagd_nrecs
- 1];
893 rrec
= &ragg
->dtagd_rec
[ragg
->dtagd_nrecs
- 1];
895 assert(lrec
->dtrd_action
== rrec
->dtrd_action
);
897 laddr
= (int64_t *)(uintptr_t)(ldata
+ lrec
->dtrd_offset
);
898 raddr
= (int64_t *)(uintptr_t)(rdata
+ rrec
->dtrd_offset
);
900 switch (lrec
->dtrd_action
) {
902 rval
= dt_aggregate_averagecmp(laddr
, raddr
);
905 case DTRACEAGG_STDDEV
:
906 rval
= dt_aggregate_stddevcmp(laddr
, raddr
);
909 case DTRACEAGG_QUANTIZE
:
910 rval
= dt_aggregate_quantizedcmp(laddr
, raddr
);
913 case DTRACEAGG_LQUANTIZE
:
914 rval
= dt_aggregate_lquantizedcmp(laddr
, raddr
);
917 case DTRACEAGG_LLQUANTIZE
:
918 rval
= dt_aggregate_llquantizedcmp(laddr
, raddr
);
921 case DTRACEAGG_COUNT
:
925 rval
= dt_aggregate_countcmp(laddr
, raddr
);
936 dt_aggregate_valkeycmp(const void *lhs
, const void *rhs
)
940 if ((rval
= dt_aggregate_valcmp(lhs
, rhs
)) != 0)
944 * If we're here, the values for the two aggregation elements are
945 * equal. We already know that the key layout is the same for the two
946 * elements; we must now compare the keys themselves as a tie-breaker.
948 return (dt_aggregate_keycmp(lhs
, rhs
));
952 dt_aggregate_keyvarcmp(const void *lhs
, const void *rhs
)
956 if ((rval
= dt_aggregate_keycmp(lhs
, rhs
)) != 0)
959 return (dt_aggregate_varcmp(lhs
, rhs
));
963 dt_aggregate_varkeycmp(const void *lhs
, const void *rhs
)
967 if ((rval
= dt_aggregate_varcmp(lhs
, rhs
)) != 0)
970 return (dt_aggregate_keycmp(lhs
, rhs
));
974 dt_aggregate_valvarcmp(const void *lhs
, const void *rhs
)
978 if ((rval
= dt_aggregate_valkeycmp(lhs
, rhs
)) != 0)
981 return (dt_aggregate_varcmp(lhs
, rhs
));
985 dt_aggregate_varvalcmp(const void *lhs
, const void *rhs
)
989 if ((rval
= dt_aggregate_varcmp(lhs
, rhs
)) != 0)
992 return (dt_aggregate_valkeycmp(lhs
, rhs
));
996 dt_aggregate_keyvarrevcmp(const void *lhs
, const void *rhs
)
998 return (dt_aggregate_keyvarcmp(rhs
, lhs
));
1002 dt_aggregate_varkeyrevcmp(const void *lhs
, const void *rhs
)
1004 return (dt_aggregate_varkeycmp(rhs
, lhs
));
1008 dt_aggregate_valvarrevcmp(const void *lhs
, const void *rhs
)
1010 return (dt_aggregate_valvarcmp(rhs
, lhs
));
1014 dt_aggregate_varvalrevcmp(const void *lhs
, const void *rhs
)
1016 return (dt_aggregate_varvalcmp(rhs
, lhs
));
1020 dt_aggregate_bundlecmp(const void *lhs
, const void *rhs
)
1022 dt_ahashent_t
**lh
= *((dt_ahashent_t
***)lhs
);
1023 dt_ahashent_t
**rh
= *((dt_ahashent_t
***)rhs
);
1028 * If we're sorting on keys, we need to scan until we find the
1029 * last entry -- that's the representative key. (The order of
1030 * the bundle is values followed by key to accommodate the
1031 * default behavior of sorting by value.) If the keys are
1032 * equal, we'll fall into the value comparison loop, below.
1034 for (i
= 0; lh
[i
+ 1] != NULL
; i
++)
1038 assert(rh
[i
+ 1] == NULL
);
1040 if ((rval
= dt_aggregate_keycmp(&lh
[i
], &rh
[i
])) != 0)
1044 for (i
= 0; ; i
++) {
1045 if (lh
[i
+ 1] == NULL
) {
1047 * All of the values are equal; if we're sorting on
1048 * keys, then we're only here because the keys were
1049 * found to be equal and these records are therefore
1050 * equal. If we're not sorting on keys, we'll use the
1051 * key comparison from the representative key as the
1058 assert(rh
[i
+ 1] == NULL
);
1059 return (dt_aggregate_keycmp(&lh
[i
], &rh
[i
]));
1061 if ((rval
= dt_aggregate_valcmp(&lh
[i
], &rh
[i
])) != 0)
1068 dt_aggregate_go(dtrace_hdl_t
*dtp
)
1070 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1071 dtrace_optval_t size
, cpu
;
1072 dtrace_bufdesc_t
*buf
= &agp
->dtat_buf
;
1075 assert(agp
->dtat_maxcpu
== 0);
1076 assert(agp
->dtat_ncpu
== 0);
1077 assert(agp
->dtat_cpus
== NULL
);
1079 agp
->dtat_maxcpu
= dt_sysconf(dtp
, _SC_CPUID_MAX
) + 1;
1080 agp
->dtat_ncpu
= dt_sysconf(dtp
, _SC_NPROCESSORS_MAX
);
1081 agp
->dtat_cpus
= malloc(agp
->dtat_ncpu
* sizeof (processorid_t
));
1083 if (agp
->dtat_cpus
== NULL
)
1084 return (dt_set_errno(dtp
, EDT_NOMEM
));
1087 * Use the aggregation buffer size as reloaded from the kernel.
1089 size
= dtp
->dt_options
[DTRACEOPT_AGGSIZE
];
1091 rval
= dtrace_getopt(dtp
, "aggsize", &size
);
1094 if (size
== 0 || size
== DTRACEOPT_UNSET
)
1097 buf
= &agp
->dtat_buf
;
1098 buf
->dtbd_size
= size
;
1100 if ((buf
->dtbd_data
= malloc(buf
->dtbd_size
)) == NULL
)
1101 return (dt_set_errno(dtp
, EDT_NOMEM
));
1104 * Now query for the CPUs enabled.
1106 rval
= dtrace_getopt(dtp
, "cpu", &cpu
);
1107 assert(rval
== 0 && cpu
!= DTRACEOPT_UNSET
);
1109 if (cpu
!= DTRACE_CPUALL
) {
1110 assert(cpu
< agp
->dtat_ncpu
);
1111 agp
->dtat_cpus
[agp
->dtat_ncpus
++] = (processorid_t
)cpu
;
1116 agp
->dtat_ncpus
= 0;
1117 for (i
= 0; i
< agp
->dtat_maxcpu
; i
++) {
1118 if (dt_status(dtp
, i
) == -1)
1121 agp
->dtat_cpus
[agp
->dtat_ncpus
++] = i
;
1128 dt_aggwalk_rval(dtrace_hdl_t
*dtp
, dt_ahashent_t
*h
, int rval
)
1130 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1131 dtrace_aggdata_t
*data
;
1132 dtrace_aggdesc_t
*aggdesc
;
1133 dtrace_recdesc_t
*rec
;
1137 case DTRACE_AGGWALK_NEXT
:
1140 case DTRACE_AGGWALK_CLEAR
: {
1141 uint32_t size
, offs
= 0;
1143 aggdesc
= h
->dtahe_data
.dtada_desc
;
1144 rec
= &aggdesc
->dtagd_rec
[aggdesc
->dtagd_nrecs
- 1];
1145 size
= rec
->dtrd_size
;
1146 data
= &h
->dtahe_data
;
1148 if (rec
->dtrd_action
== DTRACEAGG_LQUANTIZE
) {
1149 offs
= sizeof (uint64_t);
1150 size
-= sizeof (uint64_t);
1153 bzero(&data
->dtada_data
[rec
->dtrd_offset
] + offs
, size
);
1155 if (data
->dtada_percpu
== NULL
)
1158 for (i
= 0; i
< dtp
->dt_aggregate
.dtat_maxcpu
; i
++)
1159 bzero(data
->dtada_percpu
[i
] + offs
, size
);
1163 case DTRACE_AGGWALK_ERROR
:
1165 * We assume that errno is already set in this case.
1167 return (dt_set_errno(dtp
, errno
));
1169 case DTRACE_AGGWALK_ABORT
:
1170 return (dt_set_errno(dtp
, EDT_DIRABORT
));
1172 case DTRACE_AGGWALK_DENORMALIZE
:
1173 h
->dtahe_data
.dtada_normal
= 1;
1176 case DTRACE_AGGWALK_NORMALIZE
:
1177 if (h
->dtahe_data
.dtada_normal
== 0) {
1178 h
->dtahe_data
.dtada_normal
= 1;
1179 return (dt_set_errno(dtp
, EDT_BADRVAL
));
1184 case DTRACE_AGGWALK_REMOVE
: {
1185 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
;
1186 int i
, max_cpus
= agp
->dtat_maxcpu
;
1189 * First, remove this hash entry from its hash chain.
1191 if (h
->dtahe_prev
!= NULL
) {
1192 h
->dtahe_prev
->dtahe_next
= h
->dtahe_next
;
1194 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1195 size_t ndx
= h
->dtahe_hashval
% hash
->dtah_size
;
1197 assert(hash
->dtah_hash
[ndx
] == h
);
1198 hash
->dtah_hash
[ndx
] = h
->dtahe_next
;
1201 if (h
->dtahe_next
!= NULL
)
1202 h
->dtahe_next
->dtahe_prev
= h
->dtahe_prev
;
1205 * Now remove it from the list of all hash entries.
1207 if (h
->dtahe_prevall
!= NULL
) {
1208 h
->dtahe_prevall
->dtahe_nextall
= h
->dtahe_nextall
;
1210 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1212 assert(hash
->dtah_all
== h
);
1213 hash
->dtah_all
= h
->dtahe_nextall
;
1216 if (h
->dtahe_nextall
!= NULL
)
1217 h
->dtahe_nextall
->dtahe_prevall
= h
->dtahe_prevall
;
1220 * We're unlinked. We can safely destroy the data.
1222 if (aggdata
->dtada_percpu
!= NULL
) {
1223 for (i
= 0; i
< max_cpus
; i
++)
1224 free(aggdata
->dtada_percpu
[i
]);
1225 free(aggdata
->dtada_percpu
);
1228 free(aggdata
->dtada_data
);
1235 return (dt_set_errno(dtp
, EDT_BADRVAL
));
1242 dt_aggregate_qsort(dtrace_hdl_t
*dtp
, void *base
, size_t nel
, size_t width
,
1243 int (*compar
)(const void *, const void *))
1245 int rev
= dt_revsort
, key
= dt_keysort
, keypos
= dt_keypos
;
1246 dtrace_optval_t keyposopt
= dtp
->dt_options
[DTRACEOPT_AGGSORTKEYPOS
];
1248 dt_revsort
= (dtp
->dt_options
[DTRACEOPT_AGGSORTREV
] != DTRACEOPT_UNSET
);
1249 dt_keysort
= (dtp
->dt_options
[DTRACEOPT_AGGSORTKEY
] != DTRACEOPT_UNSET
);
1251 if (keyposopt
!= DTRACEOPT_UNSET
&& keyposopt
<= INT_MAX
) {
1252 dt_keypos
= (int)keyposopt
;
1257 if (compar
== NULL
) {
1259 compar
= dt_aggregate_varvalcmp
;
1261 compar
= dt_aggregate_varkeycmp
;
1265 qsort(base
, nel
, width
, compar
);
1273 dtrace_aggregate_walk(dtrace_hdl_t
*dtp
, dtrace_aggregate_f
*func
, void *arg
)
1275 dt_ahashent_t
*h
, *next
;
1276 dt_ahash_t
*hash
= &dtp
->dt_aggregate
.dtat_hash
;
1278 for (h
= hash
->dtah_all
; h
!= NULL
; h
= next
) {
1280 * dt_aggwalk_rval() can potentially remove the current hash
1281 * entry; we need to load the next hash entry before calling
1284 next
= h
->dtahe_nextall
;
1286 if (dt_aggwalk_rval(dtp
, h
, func(&h
->dtahe_data
, arg
)) == -1)
1294 dt_aggregate_total(dtrace_hdl_t
*dtp
, boolean_t clear
)
1297 dtrace_aggdata_t
**total
;
1298 dtrace_aggid_t max
= DTRACE_AGGVARIDNONE
, id
;
1299 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1300 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1303 tflags
= DTRACE_A_TOTAL
| DTRACE_A_HASNEGATIVES
| DTRACE_A_HASPOSITIVES
;
1306 * If we need to deliver per-aggregation totals, we're going to take
1307 * three passes over the aggregate: one to clear everything out and
1308 * determine our maximum aggregation ID, one to actually total
1309 * everything up, and a final pass to assign the totals to the
1310 * individual elements.
1312 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1313 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
;
1315 if ((id
= dt_aggregate_aggvarid(h
)) > max
)
1318 aggdata
->dtada_total
= 0;
1319 aggdata
->dtada_flags
&= ~tflags
;
1322 if (clear
|| max
== DTRACE_AGGVARIDNONE
)
1325 total
= dt_zalloc(dtp
, (max
+ 1) * sizeof (dtrace_aggdata_t
*));
1330 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1331 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
;
1332 dtrace_aggdesc_t
*agg
= aggdata
->dtada_desc
;
1333 dtrace_recdesc_t
*rec
;
1337 rec
= &agg
->dtagd_rec
[agg
->dtagd_nrecs
- 1];
1338 data
= aggdata
->dtada_data
;
1339 addr
= (int64_t *)(uintptr_t)(data
+ rec
->dtrd_offset
);
1341 switch (rec
->dtrd_action
) {
1342 case DTRACEAGG_STDDEV
:
1343 val
= dt_stddev((uint64_t *)addr
, 1);
1347 case DTRACEAGG_COUNT
:
1352 val
= addr
[0] ? (addr
[1] / addr
[0]) : 0;
1359 if (total
[agg
->dtagd_varid
] == NULL
) {
1360 total
[agg
->dtagd_varid
] = aggdata
;
1361 aggdata
->dtada_flags
|= DTRACE_A_TOTAL
;
1363 aggdata
= total
[agg
->dtagd_varid
];
1367 aggdata
->dtada_flags
|= DTRACE_A_HASPOSITIVES
;
1370 aggdata
->dtada_flags
|= DTRACE_A_HASNEGATIVES
;
1374 if (dtp
->dt_options
[DTRACEOPT_AGGZOOM
] != DTRACEOPT_UNSET
) {
1375 val
= (int64_t)((long double)val
*
1376 (1 / DTRACE_AGGZOOM_MAX
));
1378 if (val
> aggdata
->dtada_total
)
1379 aggdata
->dtada_total
= val
;
1381 aggdata
->dtada_total
+= val
;
1386 * And now one final pass to set everyone's total.
1388 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1389 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
, *t
;
1390 dtrace_aggdesc_t
*agg
= aggdata
->dtada_desc
;
1392 if ((t
= total
[agg
->dtagd_varid
]) == NULL
|| aggdata
== t
)
1395 aggdata
->dtada_total
= t
->dtada_total
;
1396 aggdata
->dtada_flags
|= (t
->dtada_flags
& tflags
);
1399 dt_free(dtp
, total
);
1405 dt_aggregate_minmaxbin(dtrace_hdl_t
*dtp
, boolean_t clear
)
1408 dtrace_aggdata_t
**minmax
;
1409 dtrace_aggid_t max
= DTRACE_AGGVARIDNONE
, id
;
1410 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1411 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1413 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1414 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
;
1416 if ((id
= dt_aggregate_aggvarid(h
)) > max
)
1419 aggdata
->dtada_minbin
= 0;
1420 aggdata
->dtada_maxbin
= 0;
1421 aggdata
->dtada_flags
&= ~DTRACE_A_MINMAXBIN
;
1424 if (clear
|| max
== DTRACE_AGGVARIDNONE
)
1427 minmax
= dt_zalloc(dtp
, (max
+ 1) * sizeof (dtrace_aggdata_t
*));
1432 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1433 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
;
1434 dtrace_aggdesc_t
*agg
= aggdata
->dtada_desc
;
1435 dtrace_recdesc_t
*rec
;
1438 int minbin
= -1, maxbin
= -1, i
;
1439 int start
= 0, size
;
1441 rec
= &agg
->dtagd_rec
[agg
->dtagd_nrecs
- 1];
1442 size
= rec
->dtrd_size
/ sizeof (int64_t);
1443 data
= aggdata
->dtada_data
;
1444 addr
= (int64_t *)(uintptr_t)(data
+ rec
->dtrd_offset
);
1446 switch (rec
->dtrd_action
) {
1447 case DTRACEAGG_LQUANTIZE
:
1449 * For lquantize(), we always display the entire range
1450 * of the aggregation when aggpack is set.
1454 maxbin
= size
- 1 - start
;
1457 case DTRACEAGG_QUANTIZE
:
1458 for (i
= start
; i
< size
; i
++) {
1470 * If we have no data (e.g., due to a clear()
1471 * or negative increments), we'll use the
1472 * zero bucket as both our min and max.
1474 minbin
= maxbin
= DTRACE_QUANTIZE_ZEROBUCKET
;
1483 if (minmax
[agg
->dtagd_varid
] == NULL
) {
1484 minmax
[agg
->dtagd_varid
] = aggdata
;
1485 aggdata
->dtada_flags
|= DTRACE_A_MINMAXBIN
;
1486 aggdata
->dtada_minbin
= minbin
;
1487 aggdata
->dtada_maxbin
= maxbin
;
1491 if (minbin
< minmax
[agg
->dtagd_varid
]->dtada_minbin
)
1492 minmax
[agg
->dtagd_varid
]->dtada_minbin
= minbin
;
1494 if (maxbin
> minmax
[agg
->dtagd_varid
]->dtada_maxbin
)
1495 minmax
[agg
->dtagd_varid
]->dtada_maxbin
= maxbin
;
1499 * And now one final pass to set everyone's minbin and maxbin.
1501 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1502 dtrace_aggdata_t
*aggdata
= &h
->dtahe_data
, *mm
;
1503 dtrace_aggdesc_t
*agg
= aggdata
->dtada_desc
;
1505 if ((mm
= minmax
[agg
->dtagd_varid
]) == NULL
|| aggdata
== mm
)
1508 aggdata
->dtada_minbin
= mm
->dtada_minbin
;
1509 aggdata
->dtada_maxbin
= mm
->dtada_maxbin
;
1510 aggdata
->dtada_flags
|= DTRACE_A_MINMAXBIN
;
1513 dt_free(dtp
, minmax
);
1519 dt_aggregate_walk_sorted(dtrace_hdl_t
*dtp
,
1520 dtrace_aggregate_f
*func
, void *arg
,
1521 int (*sfunc
)(const void *, const void *))
1523 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1524 dt_ahashent_t
*h
, **sorted
;
1525 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1526 size_t i
, nentries
= 0;
1529 agp
->dtat_flags
&= ~(DTRACE_A_TOTAL
| DTRACE_A_MINMAXBIN
);
1531 if (dtp
->dt_options
[DTRACEOPT_AGGHIST
] != DTRACEOPT_UNSET
) {
1532 agp
->dtat_flags
|= DTRACE_A_TOTAL
;
1534 if (dt_aggregate_total(dtp
, B_FALSE
) != 0)
1538 if (dtp
->dt_options
[DTRACEOPT_AGGPACK
] != DTRACEOPT_UNSET
) {
1539 agp
->dtat_flags
|= DTRACE_A_MINMAXBIN
;
1541 if (dt_aggregate_minmaxbin(dtp
, B_FALSE
) != 0)
1545 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
)
1548 sorted
= dt_alloc(dtp
, nentries
* sizeof (dt_ahashent_t
*));
1553 for (h
= hash
->dtah_all
, i
= 0; h
!= NULL
; h
= h
->dtahe_nextall
)
1556 (void) pthread_mutex_lock(&dt_qsort_lock
);
1558 if (sfunc
== NULL
) {
1559 dt_aggregate_qsort(dtp
, sorted
, nentries
,
1560 sizeof (dt_ahashent_t
*), NULL
);
1563 * If we've been explicitly passed a sorting function,
1564 * we'll use that -- ignoring the values of the "aggsortrev",
1565 * "aggsortkey" and "aggsortkeypos" options.
1567 qsort(sorted
, nentries
, sizeof (dt_ahashent_t
*), sfunc
);
1570 (void) pthread_mutex_unlock(&dt_qsort_lock
);
1572 for (i
= 0; i
< nentries
; i
++) {
1575 if (dt_aggwalk_rval(dtp
, h
, func(&h
->dtahe_data
, arg
)) == -1)
1581 if (agp
->dtat_flags
& DTRACE_A_TOTAL
)
1582 (void) dt_aggregate_total(dtp
, B_TRUE
);
1584 if (agp
->dtat_flags
& DTRACE_A_MINMAXBIN
)
1585 (void) dt_aggregate_minmaxbin(dtp
, B_TRUE
);
1587 dt_free(dtp
, sorted
);
1592 dtrace_aggregate_walk_sorted(dtrace_hdl_t
*dtp
,
1593 dtrace_aggregate_f
*func
, void *arg
)
1595 return (dt_aggregate_walk_sorted(dtp
, func
, arg
, NULL
));
1599 dtrace_aggregate_walk_keysorted(dtrace_hdl_t
*dtp
,
1600 dtrace_aggregate_f
*func
, void *arg
)
1602 return (dt_aggregate_walk_sorted(dtp
, func
,
1603 arg
, dt_aggregate_varkeycmp
));
1607 dtrace_aggregate_walk_valsorted(dtrace_hdl_t
*dtp
,
1608 dtrace_aggregate_f
*func
, void *arg
)
1610 return (dt_aggregate_walk_sorted(dtp
, func
,
1611 arg
, dt_aggregate_varvalcmp
));
1615 dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t
*dtp
,
1616 dtrace_aggregate_f
*func
, void *arg
)
1618 return (dt_aggregate_walk_sorted(dtp
, func
,
1619 arg
, dt_aggregate_keyvarcmp
));
1623 dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t
*dtp
,
1624 dtrace_aggregate_f
*func
, void *arg
)
1626 return (dt_aggregate_walk_sorted(dtp
, func
,
1627 arg
, dt_aggregate_valvarcmp
));
1631 dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t
*dtp
,
1632 dtrace_aggregate_f
*func
, void *arg
)
1634 return (dt_aggregate_walk_sorted(dtp
, func
,
1635 arg
, dt_aggregate_varkeyrevcmp
));
1639 dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t
*dtp
,
1640 dtrace_aggregate_f
*func
, void *arg
)
1642 return (dt_aggregate_walk_sorted(dtp
, func
,
1643 arg
, dt_aggregate_varvalrevcmp
));
1647 dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t
*dtp
,
1648 dtrace_aggregate_f
*func
, void *arg
)
1650 return (dt_aggregate_walk_sorted(dtp
, func
,
1651 arg
, dt_aggregate_keyvarrevcmp
));
1655 dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t
*dtp
,
1656 dtrace_aggregate_f
*func
, void *arg
)
1658 return (dt_aggregate_walk_sorted(dtp
, func
,
1659 arg
, dt_aggregate_valvarrevcmp
));
1663 dtrace_aggregate_walk_joined(dtrace_hdl_t
*dtp
, dtrace_aggvarid_t
*aggvars
,
1664 int naggvars
, dtrace_aggregate_walk_joined_f
*func
, void *arg
)
1666 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
1667 dt_ahashent_t
*h
, **sorted
= NULL
, ***bundle
, **nbundle
;
1668 const dtrace_aggdata_t
**data
;
1669 dt_ahashent_t
*zaggdata
= NULL
;
1670 dt_ahash_t
*hash
= &agp
->dtat_hash
;
1671 size_t nentries
= 0, nbundles
= 0, start
, zsize
= 0, bundlesize
;
1672 dtrace_aggvarid_t max
= 0, aggvar
;
1673 int rval
= -1, *map
, *remap
= NULL
;
1675 dtrace_optval_t sortpos
= dtp
->dt_options
[DTRACEOPT_AGGSORTPOS
];
1678 * If the sorting position is greater than the number of aggregation
1679 * variable IDs, we silently set it to 0.
1681 if (sortpos
== DTRACEOPT_UNSET
|| sortpos
>= naggvars
)
1685 * First we need to translate the specified aggregation variable IDs
1686 * into a linear map that will allow us to translate an aggregation
1687 * variable ID into its position in the specified aggvars.
1689 for (i
= 0; i
< naggvars
; i
++) {
1690 if (aggvars
[i
] == DTRACE_AGGVARIDNONE
|| aggvars
[i
] < 0)
1691 return (dt_set_errno(dtp
, EDT_BADAGGVAR
));
1693 if (aggvars
[i
] > max
)
1697 if ((map
= dt_zalloc(dtp
, (max
+ 1) * sizeof (int))) == NULL
)
1700 zaggdata
= dt_zalloc(dtp
, naggvars
* sizeof (dt_ahashent_t
));
1702 if (zaggdata
== NULL
)
1705 for (i
= 0; i
< naggvars
; i
++) {
1706 int ndx
= i
+ sortpos
;
1708 if (ndx
>= naggvars
)
1711 aggvar
= aggvars
[ndx
];
1712 assert(aggvar
<= max
);
1716 * We have an aggregation variable that is present
1717 * more than once in the array of aggregation
1718 * variables. While it's unclear why one might want
1719 * to do this, it's legal. To support this construct,
1720 * we will allocate a remap that will indicate the
1721 * position from which this aggregation variable
1722 * should be pulled. (That is, where the remap will
1723 * map from one position to another.)
1725 if (remap
== NULL
) {
1726 remap
= dt_zalloc(dtp
, naggvars
* sizeof (int));
1733 * Given that the variable is already present, assert
1734 * that following through the mapping and adjusting
1735 * for the sort position yields the same aggregation
1738 assert(aggvars
[(map
[aggvar
] - 1 + sortpos
) %
1739 naggvars
] == aggvars
[ndx
]);
1741 remap
[i
] = map
[aggvar
];
1745 map
[aggvar
] = i
+ 1;
1749 * We need to take two passes over the data to size our allocation, so
1750 * we'll use the first pass to also fill in the zero-filled data to be
1751 * used to properly format a zero-valued aggregation.
1753 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
1754 dtrace_aggvarid_t id
;
1757 if ((id
= dt_aggregate_aggvarid(h
)) > max
|| !(ndx
= map
[id
]))
1760 if (zaggdata
[ndx
- 1].dtahe_size
== 0) {
1761 zaggdata
[ndx
- 1].dtahe_size
= h
->dtahe_size
;
1762 zaggdata
[ndx
- 1].dtahe_data
= h
->dtahe_data
;
1768 if (nentries
== 0) {
1770 * We couldn't find any entries; there is nothing else to do.
1777 * Before we sort the data, we're going to look for any holes in our
1778 * zero-filled data. This will occur if an aggregation variable that
1779 * we are being asked to print has not yet been assigned the result of
1780 * any aggregating action for _any_ tuple. The issue becomes that we
1781 * would like a zero value to be printed for all columns for this
1782 * aggregation, but without any record description, we don't know the
1783 * aggregating action that corresponds to the aggregation variable. To
1784 * try to find a match, we're simply going to lookup aggregation IDs
1785 * (which are guaranteed to be contiguous and to start from 1), looking
1786 * for the specified aggregation variable ID. If we find a match,
1787 * we'll use that. If we iterate over all aggregation IDs and don't
1788 * find a match, then we must be an anonymous enabling. (Anonymous
1789 * enablings can't currently derive either aggregation variable IDs or
1790 * aggregation variable names given only an aggregation ID.) In this
1791 * obscure case (anonymous enabling, multiple aggregation printa() with
1792 * some aggregations not represented for any tuple), our defined
1793 * behavior is that the zero will be printed in the format of the first
1794 * aggregation variable that contains any non-zero value.
1796 for (i
= 0; i
< naggvars
; i
++) {
1797 if (zaggdata
[i
].dtahe_size
== 0) {
1798 dtrace_aggvarid_t aggvar
;
1800 aggvar
= aggvars
[(i
- sortpos
+ naggvars
) % naggvars
];
1801 assert(zaggdata
[i
].dtahe_data
.dtada_data
== NULL
);
1803 for (j
= DTRACE_AGGIDNONE
+ 1; ; j
++) {
1804 dtrace_aggdesc_t
*agg
;
1805 dtrace_aggdata_t
*aggdata
;
1807 if (dt_aggid_lookup(dtp
, j
, &agg
) != 0)
1810 if (agg
->dtagd_varid
!= aggvar
)
1814 * We have our description -- now we need to
1815 * cons up the zaggdata entry for it.
1817 aggdata
= &zaggdata
[i
].dtahe_data
;
1818 aggdata
->dtada_size
= agg
->dtagd_size
;
1819 aggdata
->dtada_desc
= agg
;
1820 aggdata
->dtada_handle
= dtp
;
1821 (void) dt_epid_lookup(dtp
, agg
->dtagd_epid
,
1822 &aggdata
->dtada_edesc
,
1823 &aggdata
->dtada_pdesc
);
1824 aggdata
->dtada_normal
= 1;
1825 zaggdata
[i
].dtahe_hashval
= 0;
1826 zaggdata
[i
].dtahe_size
= agg
->dtagd_size
;
1830 if (zaggdata
[i
].dtahe_size
== 0) {
1834 * We couldn't find this aggregation, meaning
1835 * that we have never seen it before for any
1836 * tuple _and_ this is an anonymous enabling.
1837 * That is, we're in the obscure case outlined
1838 * above. In this case, our defined behavior
1839 * is to format the data in the format of the
1840 * first non-zero aggregation -- of which, of
1841 * course, we know there to be at least one
1842 * (or nentries would have been zero).
1844 for (j
= 0; j
< naggvars
; j
++) {
1845 if (zaggdata
[j
].dtahe_size
!= 0)
1849 assert(j
< naggvars
);
1850 zaggdata
[i
] = zaggdata
[j
];
1852 data
= zaggdata
[i
].dtahe_data
.dtada_data
;
1853 assert(data
!= NULL
);
1859 * Now we need to allocate our zero-filled data for use for
1860 * aggregations that don't have a value corresponding to a given key.
1862 for (i
= 0; i
< naggvars
; i
++) {
1863 dtrace_aggdata_t
*aggdata
= &zaggdata
[i
].dtahe_data
;
1864 dtrace_aggdesc_t
*aggdesc
= aggdata
->dtada_desc
;
1865 dtrace_recdesc_t
*rec
;
1869 zsize
= zaggdata
[i
].dtahe_size
;
1872 if ((zdata
= dt_zalloc(dtp
, zsize
)) == NULL
) {
1874 * If we failed to allocated some zero-filled data, we
1875 * need to zero out the remaining dtada_data pointers
1876 * to prevent the wrong data from being freed below.
1878 for (j
= i
; j
< naggvars
; j
++)
1879 zaggdata
[j
].dtahe_data
.dtada_data
= NULL
;
1883 aggvar
= aggvars
[(i
- sortpos
+ naggvars
) % naggvars
];
1886 * First, the easy bit. To maintain compatibility with
1887 * consumers that pull the compiler-generated ID out of the
1888 * data, we put that ID at the top of the zero-filled data.
1890 rec
= &aggdesc
->dtagd_rec
[0];
1891 /* LINTED - alignment */
1892 *((dtrace_aggvarid_t
*)(zdata
+ rec
->dtrd_offset
)) = aggvar
;
1894 rec
= &aggdesc
->dtagd_rec
[aggdesc
->dtagd_nrecs
- 1];
1897 * Now for the more complicated part. If (and only if) this
1898 * is an lquantize() aggregating action, zero-filled data is
1899 * not equivalent to an empty record: we must also get the
1900 * parameters for the lquantize().
1902 if (rec
->dtrd_action
== DTRACEAGG_LQUANTIZE
) {
1903 if (aggdata
->dtada_data
!= NULL
) {
1905 * The easier case here is if we actually have
1906 * some prototype data -- in which case we
1907 * manually dig it out of the aggregation
1910 /* LINTED - alignment */
1911 larg
= *((uint64_t *)(aggdata
->dtada_data
+
1915 * We don't have any prototype data. As a
1916 * result, we know that we _do_ have the
1917 * compiler-generated information. (If this
1918 * were an anonymous enabling, all of our
1919 * zero-filled data would have prototype data
1920 * -- either directly or indirectly.) So as
1921 * gross as it is, we'll grovel around in the
1922 * compiler-generated information to find the
1923 * lquantize() parameters.
1925 dtrace_stmtdesc_t
*sdp
;
1929 sdp
= (dtrace_stmtdesc_t
*)(uintptr_t)
1930 aggdesc
->dtagd_rec
[0].dtrd_uarg
;
1931 aid
= sdp
->dtsd_aggdata
;
1932 isp
= (dt_idsig_t
*)aid
->di_data
;
1933 assert(isp
->dis_auxinfo
!= 0);
1934 larg
= isp
->dis_auxinfo
;
1937 /* LINTED - alignment */
1938 *((uint64_t *)(zdata
+ rec
->dtrd_offset
)) = larg
;
1941 aggdata
->dtada_data
= zdata
;
1945 * Now that we've dealt with setting up our zero-filled data, we can
1946 * allocate our sorted array, and take another pass over the data to
1949 sorted
= dt_alloc(dtp
, nentries
* sizeof (dt_ahashent_t
*));
1954 for (h
= hash
->dtah_all
, i
= 0; h
!= NULL
; h
= h
->dtahe_nextall
) {
1955 dtrace_aggvarid_t id
;
1957 if ((id
= dt_aggregate_aggvarid(h
)) > max
|| !map
[id
])
1963 assert(i
== nentries
);
1966 * We've loaded our array; now we need to sort by value to allow us
1967 * to create bundles of like value. We're going to acquire the
1968 * dt_qsort_lock here, and hold it across all of our subsequent
1969 * comparison and sorting.
1971 (void) pthread_mutex_lock(&dt_qsort_lock
);
1973 qsort(sorted
, nentries
, sizeof (dt_ahashent_t
*),
1974 dt_aggregate_keyvarcmp
);
1977 * Now we need to go through and create bundles. Because the number
1978 * of bundles is bounded by the size of the sorted array, we're going
1979 * to reuse the underlying storage. And note that "bundle" is an
1980 * array of pointers to arrays of pointers to dt_ahashent_t -- making
1981 * its type (regrettably) "dt_ahashent_t ***". (Regrettable because
1982 * '*' -- like '_' and 'X' -- should never appear in triplicate in
1985 bundle
= (dt_ahashent_t
***)sorted
;
1987 for (i
= 1, start
= 0; i
<= nentries
; i
++) {
1989 dt_aggregate_keycmp(&sorted
[i
], &sorted
[i
- 1]) == 0)
1993 * We have a bundle boundary. Everything from start to
1994 * (i - 1) belongs in one bundle.
1996 assert(i
- start
<= naggvars
);
1997 bundlesize
= (naggvars
+ 2) * sizeof (dt_ahashent_t
*);
1999 if ((nbundle
= dt_zalloc(dtp
, bundlesize
)) == NULL
) {
2000 (void) pthread_mutex_unlock(&dt_qsort_lock
);
2004 for (j
= start
; j
< i
; j
++) {
2005 dtrace_aggvarid_t id
= dt_aggregate_aggvarid(sorted
[j
]);
2008 assert(map
[id
] != 0);
2009 assert(map
[id
] - 1 < naggvars
);
2010 assert(nbundle
[map
[id
] - 1] == NULL
);
2011 nbundle
[map
[id
] - 1] = sorted
[j
];
2013 if (nbundle
[naggvars
] == NULL
)
2014 nbundle
[naggvars
] = sorted
[j
];
2017 for (j
= 0; j
< naggvars
; j
++) {
2018 if (nbundle
[j
] != NULL
)
2022 * Before we assume that this aggregation variable
2023 * isn't present (and fall back to using the
2024 * zero-filled data allocated earlier), check the
2025 * remap. If we have a remapping, we'll drop it in
2026 * here. Note that we might be remapping an
2027 * aggregation variable that isn't present for this
2028 * key; in this case, the aggregation data that we
2029 * copy will point to the zeroed data.
2031 if (remap
!= NULL
&& remap
[j
]) {
2032 assert(remap
[j
] - 1 < j
);
2033 assert(nbundle
[remap
[j
] - 1] != NULL
);
2034 nbundle
[j
] = nbundle
[remap
[j
] - 1];
2036 nbundle
[j
] = &zaggdata
[j
];
2040 bundle
[nbundles
++] = nbundle
;
2045 * Now we need to re-sort based on the first value.
2047 dt_aggregate_qsort(dtp
, bundle
, nbundles
, sizeof (dt_ahashent_t
**),
2048 dt_aggregate_bundlecmp
);
2050 (void) pthread_mutex_unlock(&dt_qsort_lock
);
2053 * We're done! Now we just need to go back over the sorted bundles,
2054 * calling the function.
2056 data
= alloca((naggvars
+ 1) * sizeof (dtrace_aggdata_t
*));
2058 for (i
= 0; i
< nbundles
; i
++) {
2059 for (j
= 0; j
< naggvars
; j
++)
2062 for (j
= 0; j
< naggvars
; j
++) {
2063 int ndx
= j
- sortpos
;
2068 assert(bundle
[i
][ndx
] != NULL
);
2069 data
[j
+ 1] = &bundle
[i
][ndx
]->dtahe_data
;
2072 for (j
= 0; j
< naggvars
; j
++)
2073 assert(data
[j
+ 1] != NULL
);
2076 * The representative key is the last element in the bundle.
2077 * Assert that we have one, and then set it to be the first
2080 assert(bundle
[i
][j
] != NULL
);
2081 data
[0] = &bundle
[i
][j
]->dtahe_data
;
2083 if ((rval
= func(data
, naggvars
+ 1, arg
)) == -1)
2089 for (i
= 0; i
< nbundles
; i
++)
2090 dt_free(dtp
, bundle
[i
]);
2092 if (zaggdata
!= NULL
) {
2093 for (i
= 0; i
< naggvars
; i
++)
2094 dt_free(dtp
, zaggdata
[i
].dtahe_data
.dtada_data
);
2097 dt_free(dtp
, zaggdata
);
2098 dt_free(dtp
, sorted
);
2099 dt_free(dtp
, remap
);
2106 dtrace_aggregate_print(dtrace_hdl_t
*dtp
, FILE *fp
,
2107 dtrace_aggregate_walk_f
*func
)
2109 dt_print_aggdata_t pd
;
2111 bzero(&pd
, sizeof (pd
));
2115 pd
.dtpa_allunprint
= 1;
2118 func
= dtrace_aggregate_walk_sorted
;
2120 if ((*func
)(dtp
, dt_print_agg
, &pd
) == -1)
2121 return (dt_set_errno(dtp
, dtp
->dt_errno
));
2127 dtrace_aggregate_clear(dtrace_hdl_t
*dtp
)
2129 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
2130 dt_ahash_t
*hash
= &agp
->dtat_hash
;
2132 dtrace_aggdata_t
*data
;
2133 dtrace_aggdesc_t
*aggdesc
;
2134 dtrace_recdesc_t
*rec
;
2135 int i
, max_cpus
= agp
->dtat_maxcpu
;
2137 for (h
= hash
->dtah_all
; h
!= NULL
; h
= h
->dtahe_nextall
) {
2138 aggdesc
= h
->dtahe_data
.dtada_desc
;
2139 rec
= &aggdesc
->dtagd_rec
[aggdesc
->dtagd_nrecs
- 1];
2140 data
= &h
->dtahe_data
;
2142 bzero(&data
->dtada_data
[rec
->dtrd_offset
], rec
->dtrd_size
);
2144 if (data
->dtada_percpu
== NULL
)
2147 for (i
= 0; i
< max_cpus
; i
++)
2148 bzero(data
->dtada_percpu
[i
], rec
->dtrd_size
);
2153 dt_aggregate_destroy(dtrace_hdl_t
*dtp
)
2155 dt_aggregate_t
*agp
= &dtp
->dt_aggregate
;
2156 dt_ahash_t
*hash
= &agp
->dtat_hash
;
2157 dt_ahashent_t
*h
, *next
;
2158 dtrace_aggdata_t
*aggdata
;
2159 int i
, max_cpus
= agp
->dtat_maxcpu
;
2161 if (hash
->dtah_hash
== NULL
) {
2162 assert(hash
->dtah_all
== NULL
);
2164 free(hash
->dtah_hash
);
2166 for (h
= hash
->dtah_all
; h
!= NULL
; h
= next
) {
2167 next
= h
->dtahe_nextall
;
2169 aggdata
= &h
->dtahe_data
;
2171 if (aggdata
->dtada_percpu
!= NULL
) {
2172 for (i
= 0; i
< max_cpus
; i
++)
2173 free(aggdata
->dtada_percpu
[i
]);
2174 free(aggdata
->dtada_percpu
);
2177 free(aggdata
->dtada_data
);
2181 hash
->dtah_hash
= NULL
;
2182 hash
->dtah_all
= NULL
;
2183 hash
->dtah_size
= 0;
2186 free(agp
->dtat_buf
.dtbd_data
);
2187 free(agp
->dtat_cpus
);